mirror of
https://github.com/stablyai/orca.git
synced 2026-09-22 00:02:31 +00:00
feat(ssh): bound relay PTY output end to end (#11005)
* docs: design SSH relay PTY backpressure * fix(ssh): bound relay frame decoding * fix(relay): bound PTY output publication * fix(ssh): bound PTY model admission * fix(ssh): settle closed model admissions * feat(ssh): negotiate bounded PTY consumer sessions * fix(ssh): fence exit on renderer settlement * feat(ssh): track PTY source credit end to end * fix(ssh): recover bounded PTY output across reconnect * feat(ssh): complete relay PTY output backpressure * fix(ssh): close final PTY source credit races * docs(ssh): record final backpressure validation * feat(ssh): complete relay PTY source-credit lifecycle * test(ssh): complete provider notification fixture * fix(ssh): preserve terminal source credit across rotation * fix(ssh): fail closed on recovery cancellation * fix(ssh): prioritize mux control writes after drain * fix(ssh): retire canceled relay restore deliveries * fix(ssh): order exit cancellation cleanup * fix(ssh): gate provisional source activation * test(ssh): register mux drain-priority coverage * fix(ssh): type stale owner recovery mismatches * fix(ssh): close projection replacement races * fix(relay): contain streaming edge failures * fix(ssh): secure relay endpoint credentials * docs(ssh): reconcile final backpressure lifecycle * fix(ssh): bound main IPC output lifecycle * fix(ssh): close recovery ownership gaps * docs(ssh): record exact artifact validation * fix(ssh): reject reclaimed snapshot replacements * fix(ssh): fence model admission across reconnect * fix(ssh): contain migration failure per PTY * docs(ssh): record final exact-head validation * test(ssh): align deploy fixtures with credential publication * feat(ssh): add per-target bounded output setting * fix(ssh): close source recovery review gaps * fix(ssh): latch source credit environment override * feat(ssh): make PTY source credit the default * docs(ssh): record always-on relay validation * docs(ssh): bind validation to current main * test(ssh): grant source credit in IPC fixture * test(ssh): grant source credit in fake relay --------- Co-authored-by: OrcaWin <293788423+OrcaWin@users.noreply.github.com>
This commit is contained in:
+705
-856
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -16,14 +16,20 @@ type FakeChild = ChildProcessWithoutNullStreams & { kill: ReturnType<typeof vi.f
|
||||
|
||||
function fakeChild(): FakeChild {
|
||||
const child = new EventEmitter() as EventEmitter & {
|
||||
stdout: EventEmitter
|
||||
stdout: EventEmitter & {
|
||||
pause: ReturnType<typeof vi.fn>
|
||||
resume: ReturnType<typeof vi.fn>
|
||||
}
|
||||
stderr: EventEmitter
|
||||
stdin: { write: ReturnType<typeof vi.fn> }
|
||||
stdin: EventEmitter & { write: ReturnType<typeof vi.fn> }
|
||||
kill: ReturnType<typeof vi.fn>
|
||||
}
|
||||
child.stdout = new EventEmitter()
|
||||
child.stdout = Object.assign(new EventEmitter(), {
|
||||
pause: vi.fn(),
|
||||
resume: vi.fn()
|
||||
})
|
||||
child.stderr = new EventEmitter()
|
||||
child.stdin = { write: vi.fn(() => true) }
|
||||
child.stdin = Object.assign(new EventEmitter(), { write: vi.fn(() => true) })
|
||||
child.kill = vi.fn()
|
||||
return child as unknown as FakeChild
|
||||
}
|
||||
@@ -62,6 +68,10 @@ describe('waitForWslRelaySentinel', () => {
|
||||
const transport = await promise
|
||||
expect(typeof transport.write).toBe('function')
|
||||
expect(typeof transport.onData).toBe('function')
|
||||
transport.pauseReads?.()
|
||||
transport.resumeReads?.()
|
||||
expect(child.stdout.pause).toHaveBeenCalledOnce()
|
||||
expect(child.stdout.resume).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('resolves past leading garbage and hands trailing bytes to onData', async () => {
|
||||
@@ -149,4 +159,25 @@ describe('waitForWslRelaySentinel', () => {
|
||||
expect(err.message).toContain('E_FAIL')
|
||||
expect(err.message).not.toContain(String.fromCharCode(0))
|
||||
})
|
||||
|
||||
it('forwards write(false), callback settlement, and drain from WSL stdin', async () => {
|
||||
const child = fakeChild()
|
||||
const callback = vi.fn()
|
||||
const drain = vi.fn()
|
||||
const writeMock = child.stdin.write as ReturnType<typeof vi.fn>
|
||||
writeMock.mockImplementation((_data, onWritten) => {
|
||||
onWritten(null)
|
||||
return false
|
||||
})
|
||||
const promise = waitForWslRelaySentinel(child)
|
||||
emitStdout(child, RELAY_SENTINEL)
|
||||
const transport = await promise
|
||||
|
||||
transport.onDrain?.(drain)
|
||||
expect(transport.write(Buffer.from('frame'), callback)).toBe(false)
|
||||
expect(callback).toHaveBeenCalledWith({ ok: true })
|
||||
child.stdin.emit('drain')
|
||||
expect(drain).toHaveBeenCalledOnce()
|
||||
expect(transport.supportsWriteSettlement).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -123,12 +123,15 @@ export function waitForWslRelaySentinel(
|
||||
pendingChunks.push(trailing)
|
||||
}
|
||||
const transport: MultiplexerTransport = {
|
||||
write: (data) => {
|
||||
try {
|
||||
child.stdin.write(data)
|
||||
} catch {
|
||||
// Channel already closing — mux close handling takes over.
|
||||
}
|
||||
write: (data, onSettled) => {
|
||||
return child.stdin.write(data, (error?: Error | null) => {
|
||||
onSettled?.(error ? { ok: false, error } : { ok: true })
|
||||
})
|
||||
},
|
||||
supportsWriteSettlement: true,
|
||||
onDrain: (cb) => {
|
||||
child.stdin.on('drain', cb)
|
||||
return () => child.stdin.off('drain', cb)
|
||||
},
|
||||
onData: (cb) => {
|
||||
dataCallbacks.push(cb)
|
||||
@@ -143,6 +146,8 @@ export function waitForWslRelaySentinel(
|
||||
}
|
||||
},
|
||||
onClose: (cb) => closeCallbacks.push(cb),
|
||||
pauseReads: () => child.stdout.pause(),
|
||||
resumeReads: () => child.stdout.resume(),
|
||||
close: () => child.kill()
|
||||
}
|
||||
resolve(transport)
|
||||
|
||||
@@ -9,6 +9,8 @@ export type PendingPtyData = {
|
||||
droppedOutput?: true
|
||||
droppedMode2031Data?: string
|
||||
droppedMode2031ScanState?: Mode2031ReplyScanState
|
||||
projectionAdmissionIds?: readonly string[]
|
||||
projectionAdmissionsTransferred?: true
|
||||
}
|
||||
|
||||
export type PtyPendingDataDrainDisposition = 'active' | 'background' | 'blocked'
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
appendPendingProjectionAdmission,
|
||||
compactPendingProjectionAdmissions,
|
||||
PTY_PENDING_PROJECTION_ADMISSION_MAX_IDS,
|
||||
propagatePendingProjectionRemainder
|
||||
} from './pty-pending-projection-admissions'
|
||||
|
||||
describe('pending PTY projection admissions', () => {
|
||||
it('retains the exact cap then transfers the whole ordered run on overflow', () => {
|
||||
const transfer = vi.fn()
|
||||
let state = compactPendingProjectionAdmissions()
|
||||
|
||||
for (let index = 0; index < PTY_PENDING_PROJECTION_ADMISSION_MAX_IDS; index++) {
|
||||
state = appendPendingProjectionAdmission(state, `projection-${index}`, {
|
||||
isPending: () => true,
|
||||
transfer
|
||||
})
|
||||
}
|
||||
|
||||
expect(state.projectionAdmissionIds).toHaveLength(PTY_PENDING_PROJECTION_ADMISSION_MAX_IDS)
|
||||
expect(transfer).not.toHaveBeenCalled()
|
||||
|
||||
state = appendPendingProjectionAdmission(state, 'projection-overflow', {
|
||||
isPending: () => true,
|
||||
transfer
|
||||
})
|
||||
|
||||
expect(state).toEqual({ projectionAdmissionsTransferred: true })
|
||||
expect(transfer).toHaveBeenCalledOnce()
|
||||
expect(transfer.mock.calls[0]?.[0]).toHaveLength(PTY_PENDING_PROJECTION_ADMISSION_MAX_IDS + 1)
|
||||
})
|
||||
|
||||
it('compacts terminal prefixes and transfers later admissions until the remainder drains', () => {
|
||||
const transfer = vi.fn()
|
||||
const compacted = propagatePendingProjectionRemainder(
|
||||
{
|
||||
projectionAdmissionIds: ['projection-published', 'projection-partial', 'projection-tail']
|
||||
},
|
||||
{ sent: true, projectionsTransferred: false },
|
||||
{ isPending: (id) => id !== 'projection-published', transfer }
|
||||
)
|
||||
|
||||
expect(compacted).toEqual({
|
||||
projectionAdmissionIds: ['projection-partial', 'projection-tail']
|
||||
})
|
||||
|
||||
const transferred = appendPendingProjectionAdmission(
|
||||
{ projectionAdmissionsTransferred: true },
|
||||
'projection-after-transfer',
|
||||
{ isPending: () => true, transfer }
|
||||
)
|
||||
expect(transferred).toEqual({ projectionAdmissionsTransferred: true })
|
||||
expect(transfer).toHaveBeenCalledWith(['projection-after-transfer'], 'pending-projection-cap')
|
||||
|
||||
expect(
|
||||
propagatePendingProjectionRemainder(
|
||||
compacted,
|
||||
{ sent: true, projectionsTransferred: true },
|
||||
{ isPending: () => true, transfer }
|
||||
)
|
||||
).toEqual({ projectionAdmissionsTransferred: true })
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,57 @@
|
||||
export const PTY_PENDING_PROJECTION_ADMISSION_MAX_IDS = 1024
|
||||
|
||||
export type PendingProjectionAdmissions = Readonly<{
|
||||
projectionAdmissionIds?: readonly string[]
|
||||
projectionAdmissionsTransferred?: true
|
||||
}>
|
||||
|
||||
type PendingProjectionAdmissionOptions = Readonly<{
|
||||
isPending: (id: string) => boolean
|
||||
transfer: (ids: readonly string[], reason: string) => void
|
||||
}>
|
||||
|
||||
const DEFAULT_OPTIONS: PendingProjectionAdmissionOptions = {
|
||||
isPending: () => true,
|
||||
transfer: () => {}
|
||||
}
|
||||
|
||||
export function compactPendingProjectionAdmissions(
|
||||
state: PendingProjectionAdmissions = {},
|
||||
options: PendingProjectionAdmissionOptions = DEFAULT_OPTIONS
|
||||
): PendingProjectionAdmissions {
|
||||
if (state.projectionAdmissionsTransferred) {
|
||||
return { projectionAdmissionsTransferred: true }
|
||||
}
|
||||
const ids = Array.from(
|
||||
new Set((state.projectionAdmissionIds ?? []).filter((id) => options.isPending(id)))
|
||||
)
|
||||
return ids.length > 0 ? { projectionAdmissionIds: ids } : {}
|
||||
}
|
||||
|
||||
export function appendPendingProjectionAdmission(
|
||||
state: PendingProjectionAdmissions,
|
||||
id: string,
|
||||
options: PendingProjectionAdmissionOptions
|
||||
): PendingProjectionAdmissions {
|
||||
if (state.projectionAdmissionsTransferred) {
|
||||
options.transfer([id], 'pending-projection-cap')
|
||||
return { projectionAdmissionsTransferred: true }
|
||||
}
|
||||
const compacted = compactPendingProjectionAdmissions(state, options)
|
||||
const ids = [...(compacted.projectionAdmissionIds ?? []), id]
|
||||
if (ids.length <= PTY_PENDING_PROJECTION_ADMISSION_MAX_IDS) {
|
||||
return { projectionAdmissionIds: ids }
|
||||
}
|
||||
options.transfer(ids, 'pending-projection-cap')
|
||||
return { projectionAdmissionsTransferred: true }
|
||||
}
|
||||
|
||||
export function propagatePendingProjectionRemainder(
|
||||
state: PendingProjectionAdmissions,
|
||||
delivery: Readonly<{ sent: boolean; projectionsTransferred: boolean }>,
|
||||
options: PendingProjectionAdmissionOptions
|
||||
): PendingProjectionAdmissions {
|
||||
return delivery.sent && !delivery.projectionsTransferred
|
||||
? compactPendingProjectionAdmissions(state, options)
|
||||
: { projectionAdmissionsTransferred: true }
|
||||
}
|
||||
+198
-1
@@ -247,6 +247,11 @@ import { resolveWindowsShellLaunchArgs } from '../providers/windows-shell-args'
|
||||
import { _resetWslCachesForTests, _setWslCachesForTests } from '../wsl'
|
||||
import { wslHookRelayManager } from '../agent-hooks/wsl-hook-relay-manager'
|
||||
import { acquireWatcherRemovalGate } from './watcher-removal-gate'
|
||||
import {
|
||||
acceptSshPtyOutputData,
|
||||
acceptSshPtyOutputExit,
|
||||
closeSshPtyOutputGeneration
|
||||
} from './ssh-pty-output-intake-registry'
|
||||
|
||||
// Why: Windows resolves a bare PowerShell name to an absolute exe before ConPTY, else CreateProcessW fails with error 5 (PR #6537 / #5161).
|
||||
const RESOLVED_WINDOWS_POWERSHELL = 'C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe'
|
||||
@@ -449,7 +454,8 @@ describe('registerPtyHandlers', () => {
|
||||
'ssh-reattach-1',
|
||||
'ssh-reattach-fail',
|
||||
'ssh-reattach-ok',
|
||||
'ssh-runtime-env'
|
||||
'ssh-runtime-env',
|
||||
'ssh-generation-replacement'
|
||||
]) {
|
||||
unregisterSshPtyProvider(leakedConnectionId)
|
||||
}
|
||||
@@ -11556,6 +11562,197 @@ describe('registerPtyHandlers', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps negotiated source-credit overflow off the legacy PTY-global pause path', async () => {
|
||||
vi.useFakeTimers()
|
||||
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
|
||||
try {
|
||||
const provider = installObservableDaemonTestProvider()
|
||||
let modelSequence = 0
|
||||
const runtime = {
|
||||
setPtyController: vi.fn(),
|
||||
setRemoteTerminalSourceRangeConsumerHooks: vi.fn(),
|
||||
getPtyOutputSequence: vi.fn(() => modelSequence),
|
||||
onPtyData: vi.fn(
|
||||
(_id: string, data: string, _at: number, rawLength = data.length) =>
|
||||
(modelSequence += rawLength)
|
||||
),
|
||||
acceptPtyDataBounded: vi.fn(
|
||||
(_id: string, _data: string, _at: number, rawLength: number) => {
|
||||
modelSequence += rawLength
|
||||
return { sequence: modelSequence, completion: Promise.resolve() }
|
||||
}
|
||||
)
|
||||
}
|
||||
registerPtyHandlers(mainWindow as never, runtime as never)
|
||||
mainWindow.webContents.send.mockClear()
|
||||
|
||||
const sourceChunk = 's'.repeat(128 * 1024)
|
||||
for (let index = 0; index < 17; index++) {
|
||||
const sourceStartSu = index * sourceChunk.length
|
||||
await acceptSshPtyOutputData({
|
||||
id: 'source-credit-pty',
|
||||
data: sourceChunk,
|
||||
providerGeneration: 41,
|
||||
ptyIncarnation: 'source-incarnation',
|
||||
rawLength: sourceChunk.length,
|
||||
transformed: false,
|
||||
source: {
|
||||
relayPtyId: 'relay-source-pty',
|
||||
spanId: `source-token:${sourceStartSu}:${sourceStartSu + sourceChunk.length}`,
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'source-token',
|
||||
sourceStartSu,
|
||||
sourceEndSu: sourceStartSu + sourceChunk.length
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
expect(getPtyRendererDeliveryDebugSnapshot()).toMatchObject({
|
||||
pendingPtyCount: 1,
|
||||
pendingChars: 0
|
||||
})
|
||||
expect(provider.pauseProducer).not.toHaveBeenCalledWith('source-credit-pty')
|
||||
expect(provider.resumeProducer).not.toHaveBeenCalledWith('source-credit-pty')
|
||||
|
||||
provider.emitData('legacy-pty', 'l'.repeat(320 * 1024))
|
||||
expect(provider.pauseProducer).toHaveBeenCalledTimes(1)
|
||||
expect(provider.pauseProducer).toHaveBeenCalledWith('legacy-pty')
|
||||
expect(provider.pauseProducer).not.toHaveBeenCalledWith('unrelated-pty')
|
||||
|
||||
vi.runAllTimers()
|
||||
expect(provider.resumeProducer).toHaveBeenCalledTimes(1)
|
||||
expect(provider.resumeProducer).toHaveBeenCalledWith('legacy-pty')
|
||||
} finally {
|
||||
errorSpy.mockRestore()
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('pauses and resumes the exact SSH provider generation across reconnect replacement', async () => {
|
||||
vi.useFakeTimers()
|
||||
const completion = makeDeferred()
|
||||
let sequence = 0
|
||||
let captures = 0
|
||||
const runtime = {
|
||||
setPtyController: vi.fn(),
|
||||
setRemoteTerminalSourceRangeConsumerHooks: vi.fn(),
|
||||
getPtyOutputSequence: vi.fn(() => sequence),
|
||||
acceptPtyDataBounded: vi.fn((_id: string, _data: string, _at: number, rawLength: number) => {
|
||||
sequence += rawLength
|
||||
captures++
|
||||
return {
|
||||
sequence,
|
||||
completion: captures === 1 ? completion.promise : Promise.resolve()
|
||||
}
|
||||
})
|
||||
}
|
||||
const original = {
|
||||
providerGeneration: 41,
|
||||
hasPtyDeliveryPauseAdapter: () => true,
|
||||
pauseProducer: vi.fn(),
|
||||
resumeProducer: vi.fn()
|
||||
}
|
||||
const replacement = {
|
||||
providerGeneration: 42,
|
||||
hasPtyDeliveryPauseAdapter: () => true,
|
||||
pauseProducer: vi.fn(),
|
||||
resumeProducer: vi.fn()
|
||||
}
|
||||
const id = 'ssh:ssh-generation-replacement@@relay-pty'
|
||||
const receipts: Promise<unknown>[] = []
|
||||
|
||||
try {
|
||||
registerPtyHandlers(mainWindow as never, runtime as never)
|
||||
registerSshPtyProvider('ssh-generation-replacement', original as never)
|
||||
const running = acceptSshPtyOutputData({
|
||||
id,
|
||||
data: 'a'.repeat(256 * 1024),
|
||||
providerGeneration: 41,
|
||||
ptyIncarnation: 'incarnation-41',
|
||||
rawLength: 256 * 1024,
|
||||
transformed: false
|
||||
})
|
||||
receipts.push(running)
|
||||
registerSshPtyProvider('ssh-generation-replacement', replacement as never)
|
||||
const pressured = acceptSshPtyOutputData({
|
||||
id,
|
||||
data: 'b',
|
||||
providerGeneration: 41,
|
||||
ptyIncarnation: 'incarnation-41',
|
||||
rawLength: 1,
|
||||
transformed: false
|
||||
})
|
||||
receipts.push(pressured)
|
||||
|
||||
expect(original.pauseProducer).toHaveBeenCalledWith(id)
|
||||
expect(replacement.pauseProducer).not.toHaveBeenCalled()
|
||||
|
||||
completion.resolve()
|
||||
await Promise.all([running, pressured])
|
||||
expect(original.resumeProducer).toHaveBeenCalledWith(id)
|
||||
expect(replacement.resumeProducer).not.toHaveBeenCalled()
|
||||
} finally {
|
||||
completion.resolve()
|
||||
await Promise.allSettled(receipts)
|
||||
closeSshPtyOutputGeneration(41, 'test-cleanup')
|
||||
unregisterSshPtyProvider('ssh-generation-replacement')
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects local data while an SSH renderer exit waits for projection settlement', async () => {
|
||||
const provider = installObservableDaemonTestProvider()
|
||||
let sequence = 0
|
||||
const runtime = {
|
||||
setPtyController: vi.fn(),
|
||||
setRemoteTerminalSourceRangeConsumerHooks: vi.fn(),
|
||||
getPtyOutputSequence: vi.fn(() => sequence),
|
||||
acceptPtyDataBounded: vi.fn((_id: string, _data: string, _at: number, rawLength: number) => {
|
||||
sequence += rawLength
|
||||
return { sequence, completion: Promise.resolve() }
|
||||
}),
|
||||
onPtyData: vi.fn(),
|
||||
onPtyExit: vi.fn()
|
||||
}
|
||||
const id = 'ssh:exit-data-race@@relay-pty'
|
||||
|
||||
registerPtyHandlers(mainWindow as never, runtime as never)
|
||||
mainWindow.webContents.send.mockClear()
|
||||
await acceptSshPtyOutputData({
|
||||
id,
|
||||
data: 'before-exit',
|
||||
providerGeneration: 51,
|
||||
ptyIncarnation: 'incarnation-51',
|
||||
rawLength: 'before-exit'.length,
|
||||
transformed: false
|
||||
})
|
||||
const exit = acceptSshPtyOutputExit({
|
||||
id,
|
||||
code: 0,
|
||||
providerGeneration: 51,
|
||||
ptyIncarnation: 'incarnation-51'
|
||||
})
|
||||
await Promise.resolve()
|
||||
|
||||
provider.emitData(id, 'must-not-follow-exit')
|
||||
expect(mainWindow.webContents.send).not.toHaveBeenCalledWith('pty:data', {
|
||||
id,
|
||||
data: 'must-not-follow-exit'
|
||||
})
|
||||
|
||||
getPtyAckDataListener()(null, { id, processedChars: 'before-exit'.length })
|
||||
await exit
|
||||
expect(mainWindow.webContents.send.mock.calls.at(-1)).toEqual([
|
||||
'pty:exit',
|
||||
{
|
||||
id,
|
||||
code: 0,
|
||||
providerGeneration: 51,
|
||||
ptyIncarnation: 'incarnation-51'
|
||||
}
|
||||
])
|
||||
})
|
||||
|
||||
it('resumes a paused producer when the PTY exits before draining', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
|
||||
+471
-168
@@ -158,6 +158,19 @@ import {
|
||||
unmarkHiddenRendererPty
|
||||
} from './pty-hidden-delivery-gate'
|
||||
import { PtyPendingDataDrainQueue, type PendingPtyData } from './pty-pending-data-drain-queue'
|
||||
import {
|
||||
appendPendingProjectionAdmission,
|
||||
compactPendingProjectionAdmissions,
|
||||
propagatePendingProjectionRemainder,
|
||||
type PendingProjectionAdmissions
|
||||
} from './pty-pending-projection-admissions'
|
||||
import { SshPtyOutputIntake } from './ssh-pty-output-intake'
|
||||
import {
|
||||
cancelSshPtySourceDelivery,
|
||||
installSshPtyOutputIntake,
|
||||
publishSshPtySourceAck
|
||||
} from './ssh-pty-output-intake-registry'
|
||||
import type { LegacySshProjectionSemantics } from './ssh-pty-legacy-projection'
|
||||
import {
|
||||
clearNativeWindowsConptyPty,
|
||||
isNativeWindowsLocalPtySpawn,
|
||||
@@ -202,6 +215,7 @@ type FreshLocalFallbackProvider = IPtyProvider & {
|
||||
routesFreshSpawnsToLocalProvider?: true
|
||||
}
|
||||
const sshProviders = new Map<string, IPtyProvider>()
|
||||
const sshProvidersByGeneration = new Map<number, IPtyProvider>()
|
||||
|
||||
type RegisteredPtyProvider = {
|
||||
provider: IPtyProvider
|
||||
@@ -1296,10 +1310,19 @@ function beginPtySpawnForWorktree(
|
||||
/** Register an SSH PTY provider for a connection. */
|
||||
export function registerSshPtyProvider(connectionId: string, provider: IPtyProvider): void {
|
||||
sshProviders.set(connectionId, provider)
|
||||
const generation = (provider as { providerGeneration?: number }).providerGeneration
|
||||
if (Number.isSafeInteger(generation) && generation! > 0) {
|
||||
sshProvidersByGeneration.set(generation!, provider)
|
||||
}
|
||||
}
|
||||
|
||||
/** Remove an SSH PTY provider when a connection is closed. */
|
||||
export function unregisterSshPtyProvider(connectionId: string): void {
|
||||
const provider = sshProviders.get(connectionId)
|
||||
const generation = (provider as { providerGeneration?: number } | undefined)?.providerGeneration
|
||||
if (generation !== undefined && sshProvidersByGeneration.get(generation) === provider) {
|
||||
sshProvidersByGeneration.delete(generation)
|
||||
}
|
||||
sshProviders.delete(connectionId)
|
||||
}
|
||||
|
||||
@@ -1467,6 +1490,7 @@ let rendererDidStartLoadingHandler: (() => void) | null = null
|
||||
|
||||
// Why: Restart daemon must re-bind provider→renderer listeners after replaceDaemonProvider swaps localProvider, else subscribers stay bound to the disposed adapter and new PTY data silently drops.
|
||||
let rebindProviderListeners: (() => void) | null = null
|
||||
let sshOutputIntakeCleanup: (() => void) | null = null
|
||||
|
||||
export function rebindLocalProviderListeners(): void {
|
||||
rebindProviderListeners?.()
|
||||
@@ -1840,8 +1864,10 @@ export function registerPtyHandlers(
|
||||
},
|
||||
() => isHiddenPtyDeliveryGateEnabled(getSettings?.())
|
||||
)
|
||||
let sshOutputIntake: SshPtyOutputIntake | null = null
|
||||
// Why: resuming a paused producer during exit can synchronously emit; those bytes must not queue behind pty:exit.
|
||||
const rendererExitingPtyIds = new Set<string>()
|
||||
const rendererCreditBeforeExitByPty = new Map<string, boolean>()
|
||||
const rendererDeliveryRestoreNeededPtys = new Set<string>()
|
||||
|
||||
function transitionHiddenRendererPtyDeliveryState(id: string, hidden: boolean) {
|
||||
@@ -1927,11 +1953,18 @@ export function registerPtyHandlers(
|
||||
pauseProducer: (id) => tryGetProviderForPty(id)?.pauseProducer?.(id),
|
||||
resumeProducer: (id) => tryGetProviderForPty(id)?.resumeProducer?.(id)
|
||||
})
|
||||
const sourceCreditPendingPtys = new Set<string>()
|
||||
|
||||
function updateProducerFlowControl(id: string): void {
|
||||
if (!PRODUCER_FLOW_CONTROL_ENABLED) {
|
||||
return
|
||||
}
|
||||
if (sourceCreditPendingPtys.has(id)) {
|
||||
if (pendingData.get(id)) {
|
||||
return
|
||||
}
|
||||
sourceCreditPendingPtys.delete(id)
|
||||
}
|
||||
producerFlowControl.update(id, pendingData.get(id)?.data.length ?? 0)
|
||||
}
|
||||
|
||||
@@ -2000,7 +2033,16 @@ export function registerPtyHandlers(
|
||||
}
|
||||
|
||||
function clearPendingPtyData(): void {
|
||||
for (const pending of pendingData.values()) {
|
||||
if (pending.projectionAdmissionIds) {
|
||||
sshOutputIntake?.transferProjections(
|
||||
pending.projectionAdmissionIds,
|
||||
'renderer-lifecycle-reset'
|
||||
)
|
||||
}
|
||||
}
|
||||
pendingData.clear()
|
||||
sourceCreditPendingPtys.clear()
|
||||
}
|
||||
|
||||
function readCurrentPtyRendererDeliveryDebugSnapshot(): PtyRendererDeliveryDebugSnapshot {
|
||||
@@ -2169,6 +2211,9 @@ export function registerPtyHandlers(
|
||||
producerFlowControl.releaseAll()
|
||||
clearDeliveryResyncProbe()
|
||||
deliveryResyncUnansweredWarnLogged = false
|
||||
for (const id of rendererDeliveryAccountingByPty.keys()) {
|
||||
sshOutputIntake?.transferPtyProjections(id, 'renderer-lifecycle-reset')
|
||||
}
|
||||
rendererDeliveryAccountingByPty.clear()
|
||||
rendererInFlightTotalChars = 0
|
||||
clearPendingPtyData()
|
||||
@@ -2265,6 +2310,9 @@ export function registerPtyHandlers(
|
||||
accounting.lastAckAtMs = Date.now()
|
||||
}
|
||||
rendererInFlightTotalChars = Math.max(0, rendererInFlightTotalChars - acknowledged)
|
||||
if (acknowledged > 0) {
|
||||
sshOutputIntake?.settleProjectionPrefix(id, acknowledged)
|
||||
}
|
||||
return acknowledged
|
||||
}
|
||||
|
||||
@@ -2336,6 +2384,12 @@ export function registerPtyHandlers(
|
||||
// Why drop pending: everything at/before markerSeq comes from the snapshot, so flushing pre-marker bytes would double-paint the restore.
|
||||
const pending = pendingData.get(id)
|
||||
if (pending) {
|
||||
if (pending.projectionAdmissionIds) {
|
||||
sshOutputIntake?.transferProjections(
|
||||
pending.projectionAdmissionIds,
|
||||
'renderer-delivery-writeoff'
|
||||
)
|
||||
}
|
||||
pendingDroppedChars += pending.data.length
|
||||
deletePendingPtyData(id)
|
||||
pendingOverflowMarkedPtys.delete(id)
|
||||
@@ -2365,7 +2419,11 @@ export function registerPtyHandlers(
|
||||
return writtenOff
|
||||
}
|
||||
|
||||
function sendPtyDataToRenderer(id: string, payload: PtyDataPayload): boolean {
|
||||
function sendPtyDataToRenderer(
|
||||
id: string,
|
||||
payload: PtyDataPayload,
|
||||
projectionAdmissionIds?: readonly string[]
|
||||
): { sent: boolean; projectionsTransferred: boolean } {
|
||||
const charCount = getPtyPayloadCharCount(payload)
|
||||
const accounting = rendererDeliveryAccountingByPty.get(id)
|
||||
const hadAccounting = accounting !== undefined
|
||||
@@ -2400,12 +2458,28 @@ export function registerPtyHandlers(
|
||||
}
|
||||
}
|
||||
rendererDeliveryRestoreNeededPtys.add(id)
|
||||
if (projectionAdmissionIds) {
|
||||
sshOutputIntake?.transferProjections(projectionAdmissionIds, 'renderer-send-failed')
|
||||
}
|
||||
mainDeliveryBreadcrumbs.record('pty-data-send-failed', {
|
||||
id: redactPtyIdForDiagnostics(id),
|
||||
chars: charCount
|
||||
})
|
||||
console.error('[pty] renderer data send failed; payload will not be retried', error)
|
||||
return false
|
||||
return { sent: false, projectionsTransferred: projectionAdmissionIds !== undefined }
|
||||
}
|
||||
let projectionsTransferred = false
|
||||
if (projectionAdmissionIds) {
|
||||
try {
|
||||
sshOutputIntake?.publishProjectionPrefix(
|
||||
projectionAdmissionIds,
|
||||
payload.data.length,
|
||||
charCount
|
||||
)
|
||||
} catch {
|
||||
sshOutputIntake?.transferProjections(projectionAdmissionIds, 'projection-publish-failed')
|
||||
projectionsTransferred = true
|
||||
}
|
||||
}
|
||||
if (rendererDeliveryRestoreNeededPtys.has(id)) {
|
||||
try {
|
||||
@@ -2418,7 +2492,7 @@ export function registerPtyHandlers(
|
||||
)
|
||||
}
|
||||
}
|
||||
return true
|
||||
return { sent: true, projectionsTransferred }
|
||||
}
|
||||
|
||||
function rendererPtyIsKnownHidden(id: string): boolean {
|
||||
@@ -2521,6 +2595,9 @@ export function registerPtyHandlers(
|
||||
pendingOverflowMarkedPtys.add(id)
|
||||
}
|
||||
pendingDroppedChars += pending.data.length
|
||||
if (pending.projectionAdmissionIds) {
|
||||
sshOutputIntake?.transferProjections(pending.projectionAdmissionIds, 'pending-cap')
|
||||
}
|
||||
const mode2031 = scanDroppedMode2031Data(pending.data, INITIAL_MODE_2031_REPLY_SCAN_STATE)
|
||||
// Why no trimmed content tail: a mid-stream gap would corrupt the pane; the droppedOutput sentinel repaints from the snapshot and realigns by sequence (only query bytes ride along).
|
||||
return {
|
||||
@@ -2531,6 +2608,39 @@ export function registerPtyHandlers(
|
||||
}
|
||||
}
|
||||
|
||||
function updatePendingProjectionAdmissions(
|
||||
pending: PendingPtyData,
|
||||
state: PendingProjectionAdmissions
|
||||
): void {
|
||||
delete pending.projectionAdmissionIds
|
||||
delete pending.projectionAdmissionsTransferred
|
||||
if (state.projectionAdmissionIds) {
|
||||
pending.projectionAdmissionIds = state.projectionAdmissionIds
|
||||
}
|
||||
if (state.projectionAdmissionsTransferred) {
|
||||
pending.projectionAdmissionsTransferred = true
|
||||
}
|
||||
}
|
||||
|
||||
function compactPendingProjectionState(
|
||||
pending: PendingProjectionAdmissions,
|
||||
projectionSemanticsId?: string
|
||||
): PendingProjectionAdmissions {
|
||||
const options = pendingProjectionAdmissionOptions()
|
||||
const compacted = compactPendingProjectionAdmissions(pending, options)
|
||||
return projectionSemanticsId
|
||||
? appendPendingProjectionAdmission(compacted, projectionSemanticsId, options)
|
||||
: compacted
|
||||
}
|
||||
|
||||
function pendingProjectionAdmissionOptions() {
|
||||
return {
|
||||
isPending: (id: string) => sshOutputIntake?.hasUnpublishedProjection(id) ?? false,
|
||||
transfer: (ids: readonly string[], reason: string) =>
|
||||
sshOutputIntake?.transferProjections(ids, reason)
|
||||
}
|
||||
}
|
||||
|
||||
function appendPendingPtyData(
|
||||
id: string,
|
||||
existing: PendingPtyData | undefined,
|
||||
@@ -2539,10 +2649,14 @@ export function registerPtyHandlers(
|
||||
preservesSeq: boolean,
|
||||
containsBackgroundOutput: boolean,
|
||||
rawLength = data.length,
|
||||
transformed = false
|
||||
transformed = false,
|
||||
projectionSemanticsId?: string
|
||||
): PendingPtyData {
|
||||
// Why stay dropped at O(1): once over the cap the restore sentinel supersedes interim bytes; queries still get carved out (bounded) so replies survive the whole episode.
|
||||
if (existing?.droppedOutput === true) {
|
||||
if (projectionSemanticsId) {
|
||||
sshOutputIntake?.transferProjections([projectionSemanticsId], 'pending-cap')
|
||||
}
|
||||
const mode2031 = scanDroppedMode2031Data(
|
||||
data,
|
||||
existing.droppedMode2031ScanState ?? INITIAL_MODE_2031_REPLY_SCAN_STATE
|
||||
@@ -2559,16 +2673,19 @@ export function registerPtyHandlers(
|
||||
droppedMode2031ScanState: mode2031.state
|
||||
}
|
||||
}
|
||||
const projectionState = compactPendingProjectionState(existing ?? {}, projectionSemanticsId)
|
||||
const nextContainsBackgroundOutput =
|
||||
existing?.containsBackgroundOutput === true || containsBackgroundOutput
|
||||
if (!existing) {
|
||||
return dropOversizedPendingPtyData(id, {
|
||||
const pending: PendingPtyData = {
|
||||
data,
|
||||
...(typeof startSeq === 'number' ? { startSeq } : {}),
|
||||
...(rawLength !== data.length ? { rawLength } : {}),
|
||||
...(transformed ? { transformed: true } : {}),
|
||||
...(nextContainsBackgroundOutput ? { containsBackgroundOutput: true } : {})
|
||||
})
|
||||
}
|
||||
updatePendingProjectionAdmissions(pending, projectionState)
|
||||
return dropOversizedPendingPtyData(id, pending)
|
||||
}
|
||||
const existingRawLength = existing.rawLength ?? existing.data.length
|
||||
const next: PendingPtyData = {
|
||||
@@ -2578,6 +2695,7 @@ export function registerPtyHandlers(
|
||||
: {}),
|
||||
...(nextContainsBackgroundOutput ? { containsBackgroundOutput: true } : {})
|
||||
}
|
||||
updatePendingProjectionAdmissions(next, projectionState)
|
||||
if (typeof existing.startSeq === 'number') {
|
||||
next.startSeq = existing.startSeq
|
||||
}
|
||||
@@ -2661,6 +2779,9 @@ export function registerPtyHandlers(
|
||||
pendingOverflowMarkedPtys.delete(id)
|
||||
updateProducerFlowControl(id)
|
||||
const drop = recordHiddenRendererPtyDataDrop(id, pending.data.length)
|
||||
if (pending.projectionAdmissionIds) {
|
||||
sshOutputIntake?.transferProjections(pending.projectionAdmissionIds, 'hidden-drop')
|
||||
}
|
||||
warnIfDroppingHiddenBytesForVisiblePty(id, pending.data.length)
|
||||
if (drop.shouldEmitRestoreMarker) {
|
||||
sendModelRestoreNeededMarker(id, 'hidden-drop', runtime?.getPtyOutputSequence(id))
|
||||
@@ -2676,11 +2797,15 @@ export function registerPtyHandlers(
|
||||
updateProducerFlowControl(id)
|
||||
// Why droppedOutput sentinel: pending-cap drop means the pane must repaint from the snapshot, not continue a gapped stream (data = carved query bytes only).
|
||||
if (
|
||||
!sendPtyDataToRenderer(id, {
|
||||
!sendPtyDataToRenderer(
|
||||
id,
|
||||
data: pending.data + getDroppedMode2031RendererData(pending),
|
||||
droppedOutput: true
|
||||
})
|
||||
{
|
||||
id,
|
||||
data: pending.data + getDroppedMode2031RendererData(pending),
|
||||
droppedOutput: true
|
||||
},
|
||||
pending.projectionAdmissionIds
|
||||
).sent
|
||||
) {
|
||||
sendFailed = true
|
||||
break
|
||||
@@ -2692,33 +2817,50 @@ export function registerPtyHandlers(
|
||||
const indivisible = pending.transformed === true
|
||||
const chunk = indivisible ? data : data.slice(0, PTY_BATCH_FLUSH_CHUNK_CHARS)
|
||||
const remaining = indivisible ? '' : data.slice(PTY_BATCH_FLUSH_CHUNK_CHARS)
|
||||
let nextPending: PendingPtyData | undefined
|
||||
if (remaining) {
|
||||
const nextPending: PendingPtyData = { data: remaining }
|
||||
nextPending = { data: remaining }
|
||||
if (typeof pending.startSeq === 'number') {
|
||||
nextPending.startSeq = pending.startSeq + chunk.length
|
||||
}
|
||||
if (pending.containsBackgroundOutput === true) {
|
||||
nextPending.containsBackgroundOutput = true
|
||||
}
|
||||
if (pending.projectionAdmissionIds) {
|
||||
nextPending.projectionAdmissionIds = pending.projectionAdmissionIds
|
||||
}
|
||||
if (pending.projectionAdmissionsTransferred) {
|
||||
nextPending.projectionAdmissionsTransferred = true
|
||||
}
|
||||
pendingData.replaceWithRemainder(selection, nextPending)
|
||||
} else {
|
||||
pendingData.remove(selection)
|
||||
pendingOverflowMarkedPtys.delete(id)
|
||||
}
|
||||
updateProducerFlowControl(id)
|
||||
if (
|
||||
!sendPtyDataToRenderer(
|
||||
const delivery = sendPtyDataToRenderer(
|
||||
id,
|
||||
makePtyDataPayload(
|
||||
id,
|
||||
makePtyDataPayload(
|
||||
id,
|
||||
chunk,
|
||||
pending.startSeq,
|
||||
pending.containsBackgroundOutput,
|
||||
pending.rawLength,
|
||||
pending.transformed
|
||||
chunk,
|
||||
pending.startSeq,
|
||||
pending.containsBackgroundOutput,
|
||||
pending.rawLength,
|
||||
pending.transformed
|
||||
),
|
||||
pending.projectionAdmissionIds
|
||||
)
|
||||
if (nextPending) {
|
||||
updatePendingProjectionAdmissions(
|
||||
nextPending,
|
||||
propagatePendingProjectionRemainder(
|
||||
nextPending,
|
||||
delivery,
|
||||
pendingProjectionAdmissionOptions()
|
||||
)
|
||||
)
|
||||
) {
|
||||
}
|
||||
if (!delivery.sent) {
|
||||
sendFailed = true
|
||||
break
|
||||
}
|
||||
@@ -2781,27 +2923,45 @@ export function registerPtyHandlers(
|
||||
return true
|
||||
}
|
||||
|
||||
function sendPtyExitToRenderer(payload: { id: string; code: number }): void {
|
||||
function preparePtyExitForRenderer(payload: { id: string; code: number }): (() => void) | null {
|
||||
if (mainWindow.isDestroyed()) {
|
||||
return
|
||||
sshOutputIntake?.transferPtyProjections(payload.id, 'renderer-destroyed')
|
||||
return () => {}
|
||||
}
|
||||
if (rendererExitingPtyIds.has(payload.id)) {
|
||||
return
|
||||
return null
|
||||
}
|
||||
rendererExitingPtyIds.add(payload.id)
|
||||
let released = false
|
||||
const release = (): void => {
|
||||
if (released) {
|
||||
return
|
||||
}
|
||||
released = true
|
||||
rendererExitingPtyIds.delete(payload.id)
|
||||
}
|
||||
try {
|
||||
const hadReleasableRendererCredit = getRendererInFlightCharsForPty(payload.id) > 0
|
||||
if (!rendererCreditBeforeExitByPty.has(payload.id)) {
|
||||
rendererCreditBeforeExitByPty.set(
|
||||
payload.id,
|
||||
getRendererInFlightCharsForPty(payload.id) > 0
|
||||
)
|
||||
}
|
||||
// Why flush before exit: the renderer tears down the terminal on pty:exit, so any batched output not yet flushed would be silently lost.
|
||||
const remaining = pendingData.delete(payload.id)
|
||||
clearFlushTimerIfIdle()
|
||||
if (remaining) {
|
||||
if (remaining.droppedOutput === true) {
|
||||
// Sentinel entry: only salvaged query bytes remain; keep the flag so the renderer knows the span was dropped.
|
||||
sendPtyDataToRenderer(payload.id, {
|
||||
id: payload.id,
|
||||
data: remaining.data,
|
||||
droppedOutput: true
|
||||
})
|
||||
sendPtyDataToRenderer(
|
||||
payload.id,
|
||||
{
|
||||
id: payload.id,
|
||||
data: remaining.data,
|
||||
droppedOutput: true
|
||||
},
|
||||
remaining.projectionAdmissionIds
|
||||
)
|
||||
} else {
|
||||
sendPtyDataToRenderer(
|
||||
payload.id,
|
||||
@@ -2812,35 +2972,63 @@ export function registerPtyHandlers(
|
||||
remaining.containsBackgroundOutput,
|
||||
remaining.rawLength,
|
||||
remaining.transformed
|
||||
)
|
||||
),
|
||||
remaining.projectionAdmissionIds
|
||||
)
|
||||
}
|
||||
}
|
||||
// Why resume a dead PTY (no-op): avoid leaving a stale paused mark behind for a reused id.
|
||||
producerFlowControl.release(payload.id)
|
||||
pendingOverflowMarkedPtys.delete(payload.id)
|
||||
rendererDeliveryRestoreNeededPtys.delete(payload.id)
|
||||
lastInputAtByPty.delete(payload.id)
|
||||
interactiveOutputCharsByPty.delete(payload.id)
|
||||
const releasedRendererCredit = getRendererInFlightCharsForPty(payload.id)
|
||||
rendererInFlightTotalChars = Math.max(0, rendererInFlightTotalChars - releasedRendererCredit)
|
||||
// Why: the renderer also drops its cumulative total on pty:exit, so a reused id restarts aligned at zero on both sides.
|
||||
rendererDeliveryAccountingByPty.delete(payload.id)
|
||||
if (hadReleasableRendererCredit) {
|
||||
if (pendingDataFlushActive) {
|
||||
// Why: let the open round coalesce this wake into its one post-round continuation.
|
||||
const reactivatedBlocked = pendingData.reactivateBlocked()
|
||||
pendingDataCreditReleasedDuringFlush ||= reactivatedBlocked
|
||||
} else {
|
||||
schedulePendingDataAfterCreditReport(true)
|
||||
}
|
||||
return release
|
||||
} catch (error) {
|
||||
release()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
function finalizePtyExitForRenderer(payload: { id: string; code: number }): void {
|
||||
if (mainWindow.isDestroyed()) {
|
||||
rendererCreditBeforeExitByPty.delete(payload.id)
|
||||
return
|
||||
}
|
||||
const hadReleasableRendererCredit =
|
||||
rendererCreditBeforeExitByPty.get(payload.id) ??
|
||||
getRendererInFlightCharsForPty(payload.id) > 0
|
||||
rendererCreditBeforeExitByPty.delete(payload.id)
|
||||
// Why resume a dead PTY (no-op): avoid leaving a stale paused mark behind for a reused id.
|
||||
producerFlowControl.release(payload.id)
|
||||
sourceCreditPendingPtys.delete(payload.id)
|
||||
pendingOverflowMarkedPtys.delete(payload.id)
|
||||
rendererDeliveryRestoreNeededPtys.delete(payload.id)
|
||||
lastInputAtByPty.delete(payload.id)
|
||||
interactiveOutputCharsByPty.delete(payload.id)
|
||||
const releasedRendererCredit = getRendererInFlightCharsForPty(payload.id)
|
||||
rendererInFlightTotalChars = Math.max(0, rendererInFlightTotalChars - releasedRendererCredit)
|
||||
// Why: the renderer also drops its cumulative total on pty:exit, so a reused id restarts aligned at zero on both sides.
|
||||
rendererDeliveryAccountingByPty.delete(payload.id)
|
||||
if (hadReleasableRendererCredit) {
|
||||
if (pendingDataFlushActive) {
|
||||
// Why: let the open round coalesce this wake into its one post-round continuation.
|
||||
const reactivatedBlocked = pendingData.reactivateBlocked()
|
||||
pendingDataCreditReleasedDuringFlush ||= reactivatedBlocked
|
||||
} else {
|
||||
schedulePendingDataAfterCreditReport(true)
|
||||
}
|
||||
mainWindow.webContents.send('pty:exit', {
|
||||
...payload,
|
||||
...(reversibleStopOwnersByPtyId.has(payload.id) ? { preserveRendererBinding: true } : {})
|
||||
})
|
||||
}
|
||||
mainWindow.webContents.send('pty:exit', {
|
||||
...payload,
|
||||
...(reversibleStopOwnersByPtyId.has(payload.id) ? { preserveRendererBinding: true } : {})
|
||||
})
|
||||
}
|
||||
|
||||
function sendPtyExitToRenderer(payload: { id: string; code: number }): void {
|
||||
const release = preparePtyExitForRenderer(payload)
|
||||
if (!release) {
|
||||
return
|
||||
}
|
||||
try {
|
||||
sshOutputIntake?.transferPtyProjections(payload.id, 'legacy-pty-exit')
|
||||
finalizePtyExitForRenderer(payload)
|
||||
} finally {
|
||||
rendererExitingPtyIds.delete(payload.id)
|
||||
release()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2850,6 +3038,231 @@ export function registerPtyHandlers(
|
||||
}
|
||||
}
|
||||
|
||||
function acceptPtyDataForRenderer(
|
||||
payload: {
|
||||
id: string
|
||||
data: string
|
||||
sequenceChars?: number
|
||||
transformed?: boolean
|
||||
},
|
||||
outputSeq: number | undefined,
|
||||
projection?: LegacySshProjectionSemantics
|
||||
): void {
|
||||
const rawLength = payload.sequenceChars ?? payload.data.length
|
||||
const preservesSeq = !payload.transformed && rawLength === payload.data.length
|
||||
const startSeq = typeof outputSeq === 'number' ? Math.max(0, outputSeq - rawLength) : undefined
|
||||
const projectionId = projection?.identity.projectionSemanticsId
|
||||
if (mainWindow.isDestroyed()) {
|
||||
if (projectionId) {
|
||||
sshOutputIntake?.transferProjections([projectionId], 'renderer-destroyed')
|
||||
}
|
||||
if (flushTimer) {
|
||||
clearTimeout(flushTimer)
|
||||
flushTimer = null
|
||||
}
|
||||
producerFlowControl.releaseAll()
|
||||
clearDeliveryResyncProbe()
|
||||
clearPendingPtyData()
|
||||
pendingOverflowMarkedPtys.clear()
|
||||
rendererDeliveryAccountingByPty.clear()
|
||||
rendererInFlightTotalChars = 0
|
||||
clearDispatcherReadyWatchdog()
|
||||
return
|
||||
}
|
||||
if (rendererExitingPtyIds.has(payload.id)) {
|
||||
if (projectionId) {
|
||||
sshOutputIntake?.transferProjections([projectionId], 'pty-exiting')
|
||||
}
|
||||
return
|
||||
}
|
||||
if (shouldDropHiddenRendererPtyData(payload.id, getSettings?.())) {
|
||||
if (projectionId) {
|
||||
sshOutputIntake?.transferProjections([projectionId], 'hidden-drop')
|
||||
}
|
||||
const droppedChars = projection ? rawLength : payload.data.length
|
||||
const drop = recordHiddenRendererPtyDataDrop(payload.id, droppedChars)
|
||||
warnIfDroppingHiddenBytesForVisiblePty(payload.id, droppedChars)
|
||||
if (drop.shouldEmitRestoreMarker) {
|
||||
sendModelRestoreNeededMarker(payload.id, 'hidden-drop', outputSeq)
|
||||
}
|
||||
return
|
||||
}
|
||||
if (payload.data.length === 0 && !payload.transformed) {
|
||||
if (projectionId) {
|
||||
sshOutputIntake?.transferProjections([projectionId], 'empty-projection')
|
||||
}
|
||||
return
|
||||
}
|
||||
const containsBackgroundOutput =
|
||||
rendererPtyIsKnownHidden(payload.id) || ptyHasHiddenRendererResizeOutput(payload.id)
|
||||
if (containsBackgroundOutput) {
|
||||
markHiddenRendererResizeOutputDelivered(payload.id)
|
||||
}
|
||||
const overflowMarkedBeforeAppend = pendingOverflowMarkedPtys.has(payload.id)
|
||||
if (projection?.desktopSpan) {
|
||||
sourceCreditPendingPtys.add(payload.id)
|
||||
}
|
||||
const pending = appendPendingPtyData(
|
||||
payload.id,
|
||||
pendingData.get(payload.id),
|
||||
payload.data,
|
||||
startSeq,
|
||||
preservesSeq,
|
||||
containsBackgroundOutput,
|
||||
rawLength,
|
||||
payload.transformed === true,
|
||||
projectionId
|
||||
)
|
||||
const shouldEmitPendingCapRestoreMarker =
|
||||
pending.droppedOutput === true &&
|
||||
!overflowMarkedBeforeAppend &&
|
||||
pendingOverflowMarkedPtys.has(payload.id)
|
||||
const nextData = pending.data + getDroppedMode2031RendererData(pending)
|
||||
const isInteractiveOutput = shouldSendInteractiveOutputNow(
|
||||
payload.id,
|
||||
nextData,
|
||||
performance.now()
|
||||
)
|
||||
if (isInteractiveOutput && rendererPtyDispatcherReady) {
|
||||
if (!canSendPtyDataToRenderer(payload.id, { interactive: true })) {
|
||||
setPendingPtyData(payload.id, pending)
|
||||
if (shouldEmitPendingCapRestoreMarker) {
|
||||
sendModelRestoreNeededMarker(payload.id, 'pending-cap', outputSeq)
|
||||
}
|
||||
updateProducerFlowControl(payload.id)
|
||||
requestDeliveryResyncForGatedPty()
|
||||
return
|
||||
}
|
||||
deletePendingPtyData(payload.id)
|
||||
clearFlushTimerIfIdle()
|
||||
if (shouldEmitPendingCapRestoreMarker) {
|
||||
sendModelRestoreNeededMarker(payload.id, 'pending-cap', outputSeq)
|
||||
}
|
||||
pendingOverflowMarkedPtys.delete(payload.id)
|
||||
try {
|
||||
sendPtyDataToRenderer(
|
||||
payload.id,
|
||||
{
|
||||
id: payload.id,
|
||||
data: nextData,
|
||||
...(typeof pending.startSeq === 'number'
|
||||
? {
|
||||
seq: pending.startSeq + (pending.rawLength ?? nextData.length),
|
||||
rawLength: pending.rawLength ?? nextData.length
|
||||
}
|
||||
: {}),
|
||||
...(pending.transformed ? { transformed: true } : {}),
|
||||
...(pending.containsBackgroundOutput === true ? { background: true } : {}),
|
||||
...(pending.droppedOutput === true ? { droppedOutput: true } : {})
|
||||
},
|
||||
pending.projectionAdmissionIds
|
||||
)
|
||||
} finally {
|
||||
updateProducerFlowControl(payload.id)
|
||||
}
|
||||
return
|
||||
}
|
||||
setPendingPtyData(payload.id, pending)
|
||||
if (shouldEmitPendingCapRestoreMarker) {
|
||||
sendModelRestoreNeededMarker(payload.id, 'pending-cap', outputSeq)
|
||||
}
|
||||
updateProducerFlowControl(payload.id)
|
||||
if (
|
||||
!canSendPtyDataToRenderer(payload.id, { interactive: activeRendererPtys.has(payload.id) })
|
||||
) {
|
||||
requestDeliveryResyncForGatedPty()
|
||||
}
|
||||
if (!flushTimer) {
|
||||
schedulePendingDataFlush(PTY_BATCH_INTERVAL_MS)
|
||||
}
|
||||
}
|
||||
|
||||
sshOutputIntakeCleanup?.()
|
||||
sshOutputIntake = new SshPtyOutputIntake({
|
||||
getModelSequence: (id) => runtime?.getPtyOutputSequence(id) ?? 0,
|
||||
acceptModel: (event, projection) => {
|
||||
if (!runtime) {
|
||||
throw new Error('SSH PTY output requires the main terminal model')
|
||||
}
|
||||
return runtime.acceptPtyDataBounded(
|
||||
event.id,
|
||||
event.data,
|
||||
Date.now(),
|
||||
event.rawLength,
|
||||
event.transformed,
|
||||
projection.desktopSpan ? [projection.desktopSpan] : undefined
|
||||
)
|
||||
},
|
||||
project: (event, projection) =>
|
||||
acceptPtyDataForRenderer(
|
||||
{
|
||||
id: event.id,
|
||||
data: event.data,
|
||||
sequenceChars: event.rawLength,
|
||||
transformed: event.transformed
|
||||
},
|
||||
projection.identity.sequenceEnd,
|
||||
projection
|
||||
),
|
||||
prepareExit: (event) => {
|
||||
const release = preparePtyExitForRenderer(event)
|
||||
if (!release) {
|
||||
throw new Error('pty_renderer_exit_in_progress')
|
||||
}
|
||||
return release
|
||||
},
|
||||
finalizeExit: (event) => {
|
||||
runtime?.onPtyExit(event.id, event.code, event.ptyIncarnation)
|
||||
finalizePtyExitForRenderer(event)
|
||||
},
|
||||
pauseProvider: (generation, id) => {
|
||||
const provider = sshProvidersByGeneration.get(generation) as
|
||||
| (IPtyProvider & { hasPtyDeliveryPauseAdapter?: () => boolean })
|
||||
| undefined
|
||||
if (!provider?.hasPtyDeliveryPauseAdapter?.()) {
|
||||
return false
|
||||
}
|
||||
provider.pauseProducer?.(id)
|
||||
return true
|
||||
},
|
||||
resumeProvider: (generation, id) =>
|
||||
sshProvidersByGeneration.get(generation)?.resumeProducer?.(id),
|
||||
closeProvider: (generation, reason) => {
|
||||
const provider = sshProvidersByGeneration.get(generation)
|
||||
;(
|
||||
provider as (IPtyProvider & { closeOutputIntake?: (reason: string) => void }) | undefined
|
||||
)?.closeOutputIntake?.(reason)
|
||||
},
|
||||
resetModelForMigration: (_generation, id) => runtime?.resetPtyModelAfterMigrationFailure(id),
|
||||
onGenerationClosed: (providerGeneration) => {
|
||||
for (const id of pendingData.keys()) {
|
||||
const pending = pendingData.get(id)
|
||||
if (
|
||||
pending?.projectionAdmissionIds &&
|
||||
sshOutputIntake?.hasProjectionFromGeneration(
|
||||
pending.projectionAdmissionIds,
|
||||
providerGeneration
|
||||
)
|
||||
) {
|
||||
pendingData.delete(id)
|
||||
updateProducerFlowControl(id)
|
||||
pendingOverflowMarkedPtys.delete(id)
|
||||
}
|
||||
}
|
||||
sshProvidersByGeneration.delete(providerGeneration)
|
||||
},
|
||||
publishSourceAck: publishSshPtySourceAck,
|
||||
cancelSourceDelivery: cancelSshPtySourceDelivery
|
||||
})
|
||||
runtime?.setRemoteTerminalSourceRangeConsumerHooks?.(
|
||||
sshOutputIntake.getRemoteSourceRangeConsumerHooks()
|
||||
)
|
||||
const cleanupSshOutputIntakeRegistry = installSshPtyOutputIntake(sshOutputIntake)
|
||||
sshOutputIntakeCleanup = () => {
|
||||
runtime?.setRemoteTerminalSourceRangeConsumerHooks?.(null)
|
||||
cleanupSshOutputIntakeRegistry()
|
||||
}
|
||||
|
||||
async function shutdownProviderAndDetectExit(
|
||||
provider: IPtyProvider,
|
||||
id: string,
|
||||
@@ -2933,120 +3346,7 @@ export function registerPtyHandlers(
|
||||
const outputSeq = isLocalProvider
|
||||
? runtime?.getPtyOutputSequence(payload.id)
|
||||
: runtime?.onPtyData(payload.id, payload.data, Date.now(), rawLength, payload.transformed)
|
||||
const rendererData = payload.data
|
||||
const preservesSeq = !payload.transformed && rawLength === payload.data.length
|
||||
const startSeq =
|
||||
typeof outputSeq === 'number' ? Math.max(0, outputSeq - rawLength) : undefined
|
||||
if (mainWindow.isDestroyed()) {
|
||||
// Why clear the flush timer: macOS app re-activation otherwise leaks orphaned timers from the previous window's registration.
|
||||
if (flushTimer) {
|
||||
clearTimeout(flushTimer)
|
||||
flushTimer = null
|
||||
}
|
||||
producerFlowControl.releaseAll()
|
||||
clearDeliveryResyncProbe()
|
||||
clearPendingPtyData()
|
||||
pendingOverflowMarkedPtys.clear()
|
||||
rendererDeliveryAccountingByPty.clear()
|
||||
rendererInFlightTotalChars = 0
|
||||
clearDispatcherReadyWatchdog()
|
||||
return
|
||||
}
|
||||
if (rendererExitingPtyIds.size > 0 && rendererExitingPtyIds.has(payload.id)) {
|
||||
return
|
||||
}
|
||||
const settings = getSettings?.()
|
||||
// Why drop before the interactive bypass: runtime already ingested the chunk, so gated PTYs skip both renderer paths and reveal restores from the snapshot.
|
||||
if (shouldDropHiddenRendererPtyData(payload.id, settings)) {
|
||||
const drop = recordHiddenRendererPtyDataDrop(payload.id, payload.data.length)
|
||||
warnIfDroppingHiddenBytesForVisiblePty(payload.id, payload.data.length)
|
||||
if (drop.shouldEmitRestoreMarker) {
|
||||
sendModelRestoreNeededMarker(payload.id, 'hidden-drop', outputSeq)
|
||||
}
|
||||
return
|
||||
}
|
||||
if (rendererData.length === 0 && !payload.transformed) {
|
||||
return
|
||||
}
|
||||
const containsBackgroundOutput =
|
||||
rendererPtyIsKnownHidden(payload.id) || ptyHasHiddenRendererResizeOutput(payload.id)
|
||||
if (containsBackgroundOutput) {
|
||||
markHiddenRendererResizeOutputDelivered(payload.id)
|
||||
}
|
||||
const existing = pendingData.get(payload.id)
|
||||
const overflowMarkedBeforeAppend = pendingOverflowMarkedPtys.has(payload.id)
|
||||
const pending = appendPendingPtyData(
|
||||
payload.id,
|
||||
existing,
|
||||
rendererData,
|
||||
startSeq,
|
||||
preservesSeq,
|
||||
containsBackgroundOutput,
|
||||
rawLength,
|
||||
payload.transformed === true
|
||||
)
|
||||
const shouldEmitPendingCapRestoreMarker =
|
||||
pending.droppedOutput === true &&
|
||||
!overflowMarkedBeforeAppend &&
|
||||
pendingOverflowMarkedPtys.has(payload.id)
|
||||
const nextData = pending.data + getDroppedMode2031RendererData(pending)
|
||||
const isInteractiveOutput = shouldSendInteractiveOutputNow(
|
||||
payload.id,
|
||||
nextData,
|
||||
performance.now()
|
||||
)
|
||||
// Why gate the fast path on the handshake too: else boot-window keystroke echo is sent into a listener-less page and pins the gate.
|
||||
if (isInteractiveOutput && rendererPtyDispatcherReady) {
|
||||
// Why the reserve: keep input echo from being pinned behind unrelated bulk output; it's bounded and the per-PTY cap still prevents an active TUI runaway.
|
||||
if (!canSendPtyDataToRenderer(payload.id, { interactive: true })) {
|
||||
setPendingPtyData(payload.id, pending)
|
||||
if (shouldEmitPendingCapRestoreMarker) {
|
||||
sendModelRestoreNeededMarker(payload.id, 'pending-cap', outputSeq)
|
||||
}
|
||||
updateProducerFlowControl(payload.id)
|
||||
requestDeliveryResyncForGatedPty()
|
||||
return
|
||||
}
|
||||
deletePendingPtyData(payload.id)
|
||||
clearFlushTimerIfIdle()
|
||||
if (shouldEmitPendingCapRestoreMarker) {
|
||||
sendModelRestoreNeededMarker(payload.id, 'pending-cap', outputSeq)
|
||||
}
|
||||
pendingOverflowMarkedPtys.delete(payload.id)
|
||||
// Why immediate: agent TUIs redraw small prompt regions per keystroke; the throughput batch timer would add visible input latency.
|
||||
try {
|
||||
sendPtyDataToRenderer(payload.id, {
|
||||
id: payload.id,
|
||||
data: nextData,
|
||||
...(typeof pending.startSeq === 'number'
|
||||
? {
|
||||
seq: pending.startSeq + (pending.rawLength ?? nextData.length),
|
||||
rawLength: pending.rawLength ?? nextData.length
|
||||
}
|
||||
: {}),
|
||||
...(pending.transformed ? { transformed: true } : {}),
|
||||
...(pending.containsBackgroundOutput === true ? { background: true } : {}),
|
||||
...(pending.droppedOutput === true ? { droppedOutput: true } : {})
|
||||
})
|
||||
} finally {
|
||||
updateProducerFlowControl(payload.id)
|
||||
}
|
||||
return
|
||||
}
|
||||
setPendingPtyData(payload.id, pending)
|
||||
if (shouldEmitPendingCapRestoreMarker) {
|
||||
sendModelRestoreNeededMarker(payload.id, 'pending-cap', outputSeq)
|
||||
}
|
||||
updateProducerFlowControl(payload.id)
|
||||
// Why probe on data arrival (not flush skips): new output for a fully gated PTY is the moment stuck delivery becomes observable.
|
||||
if (
|
||||
!canSendPtyDataToRenderer(payload.id, { interactive: activeRendererPtys.has(payload.id) })
|
||||
) {
|
||||
requestDeliveryResyncForGatedPty()
|
||||
}
|
||||
if (!flushTimer) {
|
||||
schedulePendingDataFlush(PTY_BATCH_INTERVAL_MS)
|
||||
}
|
||||
acceptPtyDataForRenderer(payload, outputSeq)
|
||||
})
|
||||
localExitUnsub = localProvider.onExit((payload) => {
|
||||
if (!isCurrentPtyExit(payload)) {
|
||||
@@ -5646,6 +5946,9 @@ export function registerPtyHandlers(
|
||||
const pending = pendingData.get(args.id)
|
||||
if (pending && transition.droppable) {
|
||||
pendingData.delete(args.id)
|
||||
if (pending.projectionAdmissionIds) {
|
||||
sshOutputIntake?.transferProjections(pending.projectionAdmissionIds, 'hidden-drop')
|
||||
}
|
||||
updateProducerFlowControl(args.id)
|
||||
pendingOverflowMarkedPtys.delete(args.id)
|
||||
const drop = recordHiddenRendererPtyDataDrop(args.id, pending.data.length)
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
type ClosedGenerationRange = {
|
||||
start: number
|
||||
end: number
|
||||
}
|
||||
|
||||
export class SshPtyClosedGenerationRanges {
|
||||
private readonly ranges: ClosedGenerationRange[] = []
|
||||
|
||||
add(generation: number): void {
|
||||
let index = 0
|
||||
while (index < this.ranges.length && this.ranges[index]!.end + 1 < generation) {
|
||||
index++
|
||||
}
|
||||
const current = this.ranges[index]
|
||||
if (!current || generation + 1 < current.start) {
|
||||
this.ranges.splice(index, 0, { start: generation, end: generation })
|
||||
return
|
||||
}
|
||||
current.start = Math.min(current.start, generation)
|
||||
current.end = Math.max(current.end, generation)
|
||||
const next = this.ranges[index + 1]
|
||||
if (next && current.end + 1 >= next.start) {
|
||||
current.end = Math.max(current.end, next.end)
|
||||
this.ranges.splice(index + 1, 1)
|
||||
}
|
||||
}
|
||||
|
||||
has(generation: number): boolean {
|
||||
for (const range of this.ranges) {
|
||||
if (generation < range.start) {
|
||||
return false
|
||||
}
|
||||
if (generation <= range.end) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
get size(): number {
|
||||
return this.ranges.length
|
||||
}
|
||||
|
||||
get activeGaps(): number {
|
||||
const highWater = this.ranges.at(-1)?.end ?? 0
|
||||
let closedGenerations = 0
|
||||
for (const range of this.ranges) {
|
||||
closedGenerations += range.end - range.start + 1
|
||||
}
|
||||
// Why: provider generations allocate from 1, so unclosed IDs below high-water remain active.
|
||||
return highWater - closedGenerations
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
import {
|
||||
projectionError,
|
||||
reclaimProjectionRecord,
|
||||
type DesktopProjectionSpan,
|
||||
type ProjectionRecord
|
||||
} from './ssh-pty-legacy-projection-record'
|
||||
|
||||
export function publishLegacyProjectionPrefix(
|
||||
records: Map<string, ProjectionRecord>,
|
||||
ids: readonly string[],
|
||||
displayChars: number,
|
||||
accountingChars: number
|
||||
): void {
|
||||
let displayRemaining = Math.max(0, displayChars)
|
||||
let accountingRemaining = Math.max(0, accountingChars)
|
||||
for (const id of ids) {
|
||||
const record = records.get(id)
|
||||
if (!record) {
|
||||
continue
|
||||
}
|
||||
const displayLength =
|
||||
record.semantics.identity.displayEnd - record.semantics.identity.displayStart
|
||||
const unpublishedDisplay = displayLength - record.publishedDisplay
|
||||
const unpublishedAccounting = record.semantics.identity.rawLength - record.publishedAccounting
|
||||
if (displayLength === 0 && unpublishedAccounting > 0) {
|
||||
const publishAccounting = Math.min(accountingRemaining, unpublishedAccounting)
|
||||
if (publishAccounting !== unpublishedAccounting) {
|
||||
throw projectionError('ssh_projection_indivisible_split')
|
||||
}
|
||||
record.publishedAccounting += publishAccounting
|
||||
record.state = 'published'
|
||||
accountingRemaining -= publishAccounting
|
||||
continue
|
||||
}
|
||||
if (unpublishedDisplay <= 0) {
|
||||
continue
|
||||
}
|
||||
const publishDisplay = Math.min(displayRemaining, unpublishedDisplay)
|
||||
if (publishDisplay <= 0) {
|
||||
break
|
||||
}
|
||||
if (record.semantics.identity.transformed && publishDisplay !== unpublishedDisplay) {
|
||||
throw projectionError('ssh_projection_indivisible_split')
|
||||
}
|
||||
const publishAccounting =
|
||||
publishDisplay === unpublishedDisplay
|
||||
? Math.min(accountingRemaining, unpublishedAccounting)
|
||||
: publishDisplay
|
||||
record.publishedDisplay += publishDisplay
|
||||
record.publishedAccounting += publishAccounting
|
||||
record.state = 'published'
|
||||
displayRemaining -= publishDisplay
|
||||
accountingRemaining -= publishAccounting
|
||||
}
|
||||
if (displayRemaining !== 0 || accountingRemaining !== 0) {
|
||||
throw projectionError('ssh_projection_publish_range_mismatch')
|
||||
}
|
||||
}
|
||||
|
||||
export function hasUnpublishedLegacyProjection(
|
||||
records: ReadonlyMap<string, ProjectionRecord>,
|
||||
id: string
|
||||
): boolean {
|
||||
const record = records.get(id)
|
||||
if (!record || record.state === 'reserved') {
|
||||
return false
|
||||
}
|
||||
const displayLength =
|
||||
record.semantics.identity.displayEnd - record.semantics.identity.displayStart
|
||||
return (
|
||||
record.publishedDisplay < displayLength ||
|
||||
record.publishedAccounting < record.semantics.identity.rawLength
|
||||
)
|
||||
}
|
||||
|
||||
export function settlePublishedLegacyProjectionPrefix(
|
||||
records: Map<string, ProjectionRecord>,
|
||||
idsByPty: Map<string, string[]>,
|
||||
ptyId: string,
|
||||
accountingChars: number,
|
||||
onSettled: ((span: DesktopProjectionSpan, reason: string) => void) | undefined
|
||||
): { settled: number; completed: number } {
|
||||
let remaining = Math.max(0, accountingChars)
|
||||
let settled = 0
|
||||
let completed = 0
|
||||
for (const id of idsByPty.get(ptyId)?.slice() ?? []) {
|
||||
const record = records.get(id)
|
||||
if (!record) {
|
||||
continue
|
||||
}
|
||||
const available = record.publishedAccounting - record.settledAccounting
|
||||
if (available <= 0) {
|
||||
continue
|
||||
}
|
||||
const take = Math.min(remaining, available)
|
||||
const finishes =
|
||||
record.settledAccounting + take === record.semantics.identity.rawLength &&
|
||||
record.publishedDisplay ===
|
||||
record.semantics.identity.displayEnd - record.semantics.identity.displayStart
|
||||
if (finishes && record.semantics.desktopSpan) {
|
||||
onSettled?.(record.semantics.desktopSpan, 'renderer-parse')
|
||||
}
|
||||
record.settledAccounting += take
|
||||
settled += take
|
||||
remaining -= take
|
||||
if (finishes) {
|
||||
completed++
|
||||
reclaimProjectionRecord(records, idsByPty, id, ptyId)
|
||||
}
|
||||
if (remaining === 0) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return { settled, completed }
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
import type {
|
||||
Mode2031ReplyDecision,
|
||||
Mode2031ReplyScanState
|
||||
} from '../../shared/terminal-color-scheme-protocol'
|
||||
import { INITIAL_MODE_2031_REPLY_SCAN_STATE } from '../../shared/terminal-color-scheme-protocol'
|
||||
import type { TerminalOutputSourceRange } from '../../shared/terminal-output-source-range'
|
||||
|
||||
export type LegacySshProjectionIdentity = Readonly<{
|
||||
projectionSemanticsId: string
|
||||
ptyId: string
|
||||
providerGeneration: number
|
||||
ptyIncarnation: string
|
||||
displayStart: number
|
||||
displayEnd: number
|
||||
sequenceEnd: number
|
||||
rawLength: number
|
||||
transformed: boolean
|
||||
}>
|
||||
|
||||
export type LegacySshProjectionSemantics = Readonly<{
|
||||
identity: LegacySshProjectionIdentity
|
||||
desktopSpan?: DesktopProjectionSpan
|
||||
beforeScanner: Readonly<Mode2031ReplyScanState>
|
||||
afterScanner: Readonly<Mode2031ReplyScanState>
|
||||
decision: Mode2031ReplyDecision
|
||||
}>
|
||||
|
||||
export type DesktopProjectionSpan = Readonly<
|
||||
TerminalOutputSourceRange & {
|
||||
projectionSemanticsId: string
|
||||
transform: Readonly<{
|
||||
transformed: boolean
|
||||
rawLengthSu: number
|
||||
scalarSafe: boolean
|
||||
}>
|
||||
}
|
||||
>
|
||||
|
||||
export type ProjectionState = 'reserved' | 'committed' | 'published'
|
||||
|
||||
export type ProjectionRecord = {
|
||||
semantics: LegacySshProjectionSemantics
|
||||
state: ProjectionState
|
||||
publishedDisplay: number
|
||||
publishedAccounting: number
|
||||
settledAccounting: number
|
||||
}
|
||||
|
||||
export type PtyProjectionCursor = {
|
||||
providerGeneration: number
|
||||
ptyIncarnation: string
|
||||
displayEnd: number
|
||||
scanner: Mode2031ReplyScanState
|
||||
}
|
||||
|
||||
export type LegacySshProjectionReservation = Readonly<{
|
||||
semantics: LegacySshProjectionSemantics
|
||||
}>
|
||||
|
||||
export type LegacySshProjectionDebugSnapshot = {
|
||||
reserved: number
|
||||
committed: number
|
||||
published: number
|
||||
settled: number
|
||||
transferred: number
|
||||
rolledBack: number
|
||||
records: number
|
||||
cursors: number
|
||||
}
|
||||
|
||||
export function scannerSnapshot(state: Mode2031ReplyScanState): Readonly<Mode2031ReplyScanState> {
|
||||
return Object.freeze({ tail: state.tail, pendingSubscribe: state.pendingSubscribe })
|
||||
}
|
||||
|
||||
export function resetProjectionCursorForGap(
|
||||
cursors: ReadonlyMap<string, PtyProjectionCursor>,
|
||||
ptyId: string
|
||||
): void {
|
||||
const cursor = cursors.get(ptyId)
|
||||
if (cursor) {
|
||||
cursor.scanner = { ...INITIAL_MODE_2031_REPLY_SCAN_STATE }
|
||||
}
|
||||
}
|
||||
|
||||
export function projectionError(code: string): Error {
|
||||
return Object.assign(new Error(code), { code })
|
||||
}
|
||||
|
||||
export function projectionDebugSnapshot(
|
||||
records: ReadonlyMap<string, ProjectionRecord>,
|
||||
cursors: ReadonlyMap<string, PtyProjectionCursor>,
|
||||
terminalCounts: { settled: number; transferred: number; rolledBack: number }
|
||||
): LegacySshProjectionDebugSnapshot {
|
||||
const result = {
|
||||
reserved: 0,
|
||||
committed: 0,
|
||||
published: 0,
|
||||
...terminalCounts,
|
||||
records: records.size,
|
||||
cursors: cursors.size
|
||||
}
|
||||
for (const record of records.values()) {
|
||||
result[record.state]++
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
export function reclaimProjectionRecord(
|
||||
records: Map<string, ProjectionRecord>,
|
||||
idsByPty: Map<string, string[]>,
|
||||
id: string,
|
||||
ptyId: string
|
||||
): void {
|
||||
records.delete(id)
|
||||
const ids = idsByPty.get(ptyId)
|
||||
if (!ids) {
|
||||
return
|
||||
}
|
||||
const index = ids.indexOf(id)
|
||||
if (index >= 0) {
|
||||
ids.splice(index, 1)
|
||||
}
|
||||
if (ids.length === 0) {
|
||||
idsByPty.delete(ptyId)
|
||||
}
|
||||
}
|
||||
|
||||
export function requireProjectionRecord(
|
||||
records: ReadonlyMap<string, ProjectionRecord>,
|
||||
id: string
|
||||
): ProjectionRecord {
|
||||
const record = records.get(id)
|
||||
if (!record) {
|
||||
throw projectionError('ssh_projection_reservation_missing')
|
||||
}
|
||||
return record
|
||||
}
|
||||
|
||||
export function rollbackCommittedProjectionRecord(
|
||||
records: Map<string, ProjectionRecord>,
|
||||
idsByPty: Map<string, string[]>,
|
||||
cursors: Map<string, PtyProjectionCursor>,
|
||||
reservation: LegacySshProjectionReservation
|
||||
): string | null {
|
||||
const id = reservation.semantics.identity.projectionSemanticsId
|
||||
const record = records.get(id)
|
||||
if (!record || record.state !== 'committed') {
|
||||
return null
|
||||
}
|
||||
const { identity, beforeScanner } = record.semantics
|
||||
const cursor = cursors.get(identity.ptyId)
|
||||
if (
|
||||
idsByPty.get(identity.ptyId)?.at(-1) !== id ||
|
||||
!cursor ||
|
||||
cursor.providerGeneration !== identity.providerGeneration ||
|
||||
cursor.ptyIncarnation !== identity.ptyIncarnation ||
|
||||
cursor.displayEnd !== identity.displayEnd
|
||||
) {
|
||||
return null
|
||||
}
|
||||
cursor.displayEnd = identity.displayStart
|
||||
cursor.scanner = { ...beforeScanner }
|
||||
reclaimProjectionRecord(records, idsByPty, id, identity.ptyId)
|
||||
return identity.ptyId
|
||||
}
|
||||
|
||||
export function closeProjectionPty(
|
||||
cursors: Map<string, PtyProjectionCursor>,
|
||||
ptyId: string,
|
||||
providerGeneration: number,
|
||||
ptyIncarnation: string,
|
||||
beforeDelete: () => void
|
||||
): void {
|
||||
const cursor = cursors.get(ptyId)
|
||||
if (
|
||||
!cursor ||
|
||||
cursor.providerGeneration !== providerGeneration ||
|
||||
cursor.ptyIncarnation !== ptyIncarnation
|
||||
) {
|
||||
return
|
||||
}
|
||||
beforeDelete()
|
||||
cursors.delete(ptyId)
|
||||
}
|
||||
|
||||
export function getOrCreateProjectionCursor(
|
||||
cursors: Map<string, PtyProjectionCursor>,
|
||||
args: { ptyId: string; providerGeneration: number; ptyIncarnation: string },
|
||||
replaceGeneration: (providerGeneration: number) => void
|
||||
): PtyProjectionCursor {
|
||||
const existing = cursors.get(args.ptyId)
|
||||
if (!existing) {
|
||||
const cursor = {
|
||||
providerGeneration: args.providerGeneration,
|
||||
ptyIncarnation: args.ptyIncarnation,
|
||||
displayEnd: 0,
|
||||
scanner: { ...INITIAL_MODE_2031_REPLY_SCAN_STATE }
|
||||
}
|
||||
cursors.set(args.ptyId, cursor)
|
||||
return cursor
|
||||
}
|
||||
if (args.providerGeneration < existing.providerGeneration) {
|
||||
throw projectionError('ssh_projection_stale_generation')
|
||||
}
|
||||
if (
|
||||
args.providerGeneration === existing.providerGeneration &&
|
||||
args.ptyIncarnation !== existing.ptyIncarnation
|
||||
) {
|
||||
throw projectionError('ssh_projection_stale_incarnation')
|
||||
}
|
||||
if (args.providerGeneration > existing.providerGeneration) {
|
||||
replaceGeneration(existing.providerGeneration)
|
||||
existing.providerGeneration = args.providerGeneration
|
||||
existing.ptyIncarnation = args.ptyIncarnation
|
||||
existing.scanner = { ...INITIAL_MODE_2031_REPLY_SCAN_STATE }
|
||||
}
|
||||
return existing
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { SshPtyLegacyProjectionLedger } from './ssh-pty-legacy-projection'
|
||||
import { SshPtyProjectionTerminality } from './ssh-pty-projection-terminality'
|
||||
|
||||
function reserve(
|
||||
ledger: SshPtyLegacyProjectionLedger,
|
||||
overrides: Partial<Parameters<SshPtyLegacyProjectionLedger['reserve']>[0]> = {}
|
||||
) {
|
||||
return ledger.reserve({
|
||||
ptyId: 'pty-1',
|
||||
providerGeneration: 3,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
data: 'abc',
|
||||
sequenceEnd: 3,
|
||||
rawLength: 3,
|
||||
transformed: false,
|
||||
...overrides
|
||||
})
|
||||
}
|
||||
|
||||
function source(sourceStartSu: number, sourceEndSu: number, deliveryToken = 'token-1') {
|
||||
return {
|
||||
spanId: `span-${sourceStartSu}`,
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 4,
|
||||
deliveryToken,
|
||||
sourceStartSu,
|
||||
sourceEndSu
|
||||
}
|
||||
}
|
||||
|
||||
describe('SshPtyLegacyProjectionLedger', () => {
|
||||
it('drains exact terminality waiters when their provider generation closes', async () => {
|
||||
const terminality = new SshPtyProjectionTerminality()
|
||||
const terminal = terminality.whenTerminal('pty-1', 3, 'incarnation-1', () => true)
|
||||
let nextResolved = false
|
||||
const next = terminality
|
||||
.whenTerminal('pty-1', 4, 'incarnation-2', () => true)
|
||||
.then(() => {
|
||||
nextResolved = true
|
||||
})
|
||||
|
||||
terminality.closeGeneration(3)
|
||||
|
||||
await expect(terminal).resolves.toBeUndefined()
|
||||
expect(nextResolved).toBe(false)
|
||||
terminality.closeGeneration(4)
|
||||
await next
|
||||
})
|
||||
|
||||
it('rolls back scanner and display reservations before commit', () => {
|
||||
const ledger = new SshPtyLegacyProjectionLedger()
|
||||
const partial = reserve(ledger, { data: '\x1b[?20', rawLength: 5, sequenceEnd: 5 })
|
||||
expect(ledger.rollback(partial)).toBe(true)
|
||||
|
||||
const next = reserve(ledger, { data: '31h', sequenceEnd: 3 })
|
||||
expect(next.semantics.identity.displayStart).toBe(0)
|
||||
expect(next.semantics.beforeScanner).toEqual({ tail: '', pendingSubscribe: false })
|
||||
expect(next.semantics.decision).toBeNull()
|
||||
})
|
||||
|
||||
it('keeps immutable generation, incarnation, display, sequence, raw length, and scanner facts', () => {
|
||||
const ledger = new SshPtyLegacyProjectionLedger()
|
||||
const first = reserve(ledger, {
|
||||
data: '\x1b[?2031h',
|
||||
rawLength: 11,
|
||||
sequenceEnd: 11
|
||||
})
|
||||
const semantics = ledger.commit(first)
|
||||
|
||||
expect(semantics.identity).toMatchObject({
|
||||
providerGeneration: 3,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
displayStart: 0,
|
||||
displayEnd: 8,
|
||||
sequenceEnd: 11,
|
||||
rawLength: 11
|
||||
})
|
||||
expect(semantics.decision).toBe('subscribed')
|
||||
expect(Object.isFrozen(semantics)).toBe(true)
|
||||
expect(Object.isFrozen(semantics.identity)).toBe(true)
|
||||
})
|
||||
|
||||
it('rejects stale generations and resets cross-chunk scanner state on gaps', () => {
|
||||
const ledger = new SshPtyLegacyProjectionLedger()
|
||||
ledger.commit(reserve(ledger, { data: '\x1b[?20', rawLength: 5, sequenceEnd: 5 }))
|
||||
ledger.resetForGap('pty-1')
|
||||
const next = reserve(ledger, { data: '31h', sequenceEnd: 8 })
|
||||
expect(next.semantics.beforeScanner).toEqual({ tail: '', pendingSubscribe: false })
|
||||
|
||||
expect(() => reserve(ledger, { providerGeneration: 2 })).toThrow(
|
||||
'ssh_projection_stale_generation'
|
||||
)
|
||||
})
|
||||
|
||||
it('publishes, settles, and transfers explicit ranges', () => {
|
||||
const ledger = new SshPtyLegacyProjectionLedger()
|
||||
const first = ledger.commit(reserve(ledger))
|
||||
ledger.publishPrefix([first.identity.projectionSemanticsId], 3, 3)
|
||||
expect(ledger.settlePublishedPrefix('pty-1', 2)).toBe(2)
|
||||
expect(ledger.transfer([first.identity.projectionSemanticsId], 'renderer-reload')).toBe(1)
|
||||
expect(ledger.getDebugSnapshot()).toMatchObject({ transferred: 1, records: 0 })
|
||||
})
|
||||
|
||||
it('resolves exact PTY terminal waiters only after settlement or transfer', async () => {
|
||||
const ledger = new SshPtyLegacyProjectionLedger()
|
||||
const projection = ledger.commit(reserve(ledger))
|
||||
ledger.publishPrefix([projection.identity.projectionSemanticsId], 3, 3)
|
||||
|
||||
let settled = false
|
||||
const terminal = ledger.whenPtyTerminal('pty-1', 3, 'incarnation-1').then(() => {
|
||||
settled = true
|
||||
})
|
||||
await Promise.resolve()
|
||||
expect(settled).toBe(false)
|
||||
|
||||
ledger.settlePublishedPrefix('pty-1', 2)
|
||||
await Promise.resolve()
|
||||
expect(settled).toBe(false)
|
||||
ledger.transfer([projection.identity.projectionSemanticsId], 'renderer-reload')
|
||||
await terminal
|
||||
expect(settled).toBe(true)
|
||||
})
|
||||
|
||||
it('publishes and settles transformed source accounting with no display text', () => {
|
||||
const ledger = new SshPtyLegacyProjectionLedger()
|
||||
const projection = ledger.commit(
|
||||
reserve(ledger, {
|
||||
data: '',
|
||||
sequenceEnd: 9,
|
||||
rawLength: 9,
|
||||
transformed: true
|
||||
})
|
||||
)
|
||||
|
||||
ledger.publishPrefix([projection.identity.projectionSemanticsId], 0, 9)
|
||||
expect(ledger.settlePublishedPrefix('pty-1', 9)).toBe(9)
|
||||
expect(ledger.getDebugSnapshot()).toMatchObject({ settled: 1, records: 0 })
|
||||
})
|
||||
|
||||
it('reclaims a closed PTY cursor so its id can be reused by a new incarnation', () => {
|
||||
const ledger = new SshPtyLegacyProjectionLedger()
|
||||
ledger.commit(reserve(ledger))
|
||||
ledger.closePty('pty-1', 3, 'incarnation-1', 'pty-exit')
|
||||
|
||||
const next = reserve(ledger, {
|
||||
ptyIncarnation: 'incarnation-2',
|
||||
data: 'next',
|
||||
sequenceEnd: 4,
|
||||
rawLength: 4
|
||||
})
|
||||
expect(next.semantics.identity).toMatchObject({
|
||||
ptyIncarnation: 'incarnation-2',
|
||||
displayStart: 0
|
||||
})
|
||||
expect(ledger.getDebugSnapshot()).toMatchObject({ records: 1, cursors: 1 })
|
||||
})
|
||||
|
||||
it('keeps split publication attached to one immutable desktop span', () => {
|
||||
const settled: unknown[] = []
|
||||
const ledger = new SshPtyLegacyProjectionLedger({
|
||||
onSettled: (span) => settled.push(span)
|
||||
})
|
||||
const projection = ledger.commit(
|
||||
reserve(ledger, {
|
||||
data: 'abcd',
|
||||
rawLength: 4,
|
||||
sequenceEnd: 4,
|
||||
source: source(0, 4)
|
||||
})
|
||||
)
|
||||
const id = projection.identity.projectionSemanticsId
|
||||
expect(ledger.hasUnpublished(id)).toBe(true)
|
||||
|
||||
ledger.publishPrefix([id], 2, 2)
|
||||
expect(ledger.hasUnpublished(id)).toBe(true)
|
||||
ledger.settlePublishedPrefix('pty-1', 2)
|
||||
expect(settled).toEqual([])
|
||||
ledger.publishPrefix([id], 2, 2)
|
||||
expect(ledger.hasUnpublished(id)).toBe(false)
|
||||
ledger.settlePublishedPrefix('pty-1', 2)
|
||||
|
||||
expect(settled).toEqual([
|
||||
expect.objectContaining({
|
||||
spanId: 'span-0',
|
||||
projectionSemanticsId: id,
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4,
|
||||
displayStart: 0,
|
||||
displayEnd: 4
|
||||
})
|
||||
])
|
||||
})
|
||||
|
||||
it('preserves scanner and display facts after transfer and delivery-token replacement', () => {
|
||||
const transferred: unknown[] = []
|
||||
const ledger = new SshPtyLegacyProjectionLedger({
|
||||
onTransferred: (span) => transferred.push(span)
|
||||
})
|
||||
const partial = ledger.commit(
|
||||
reserve(ledger, {
|
||||
data: '\x1b[?20',
|
||||
rawLength: 5,
|
||||
sequenceEnd: 5,
|
||||
source: source(0, 5)
|
||||
})
|
||||
)
|
||||
ledger.transfer([partial.identity.projectionSemanticsId], 'renderer-reload')
|
||||
const continuation = reserve(ledger, {
|
||||
data: '31h',
|
||||
rawLength: 3,
|
||||
sequenceEnd: 8,
|
||||
source: source(5, 8, 'token-2')
|
||||
})
|
||||
|
||||
expect(continuation.semantics.identity.displayStart).toBe(5)
|
||||
expect(continuation.semantics.beforeScanner.tail).toBe('\x1b[?20')
|
||||
expect(continuation.semantics.decision).toBe('subscribed')
|
||||
expect(transferred).toEqual([
|
||||
expect.objectContaining({
|
||||
deliveryToken: 'token-1',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 5
|
||||
})
|
||||
])
|
||||
})
|
||||
|
||||
it('leaves a projection intact when its terminal transition cannot commit', () => {
|
||||
const ledger = new SshPtyLegacyProjectionLedger({
|
||||
onTransferred: () => {
|
||||
throw new Error('replacement unavailable')
|
||||
}
|
||||
})
|
||||
const projection = ledger.commit(reserve(ledger, { source: source(0, 3) }))
|
||||
|
||||
expect(() =>
|
||||
ledger.transfer([projection.identity.projectionSemanticsId], 'renderer-reload')
|
||||
).toThrow('replacement unavailable')
|
||||
expect(ledger.getDebugSnapshot()).toMatchObject({
|
||||
transferred: 0,
|
||||
records: 1
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,318 @@
|
||||
import { scanMode2031ReplyDecision } from '../../shared/terminal-color-scheme-protocol'
|
||||
import {
|
||||
closeProjectionPty,
|
||||
getOrCreateProjectionCursor,
|
||||
projectionDebugSnapshot,
|
||||
projectionError,
|
||||
reclaimProjectionRecord,
|
||||
resetProjectionCursorForGap,
|
||||
requireProjectionRecord,
|
||||
rollbackCommittedProjectionRecord,
|
||||
scannerSnapshot,
|
||||
type DesktopProjectionSpan,
|
||||
type LegacySshProjectionDebugSnapshot,
|
||||
type LegacySshProjectionReservation,
|
||||
type LegacySshProjectionSemantics,
|
||||
type ProjectionRecord,
|
||||
type PtyProjectionCursor
|
||||
} from './ssh-pty-legacy-projection-record'
|
||||
import {
|
||||
projectionHasOpen,
|
||||
resolveProjectionTerminality,
|
||||
SshPtyProjectionTerminality,
|
||||
unpublishedProjectionIds
|
||||
} from './ssh-pty-projection-terminality'
|
||||
import {
|
||||
hasUnpublishedLegacyProjection,
|
||||
publishLegacyProjectionPrefix,
|
||||
settlePublishedLegacyProjectionPrefix
|
||||
} from './ssh-pty-legacy-projection-publication'
|
||||
|
||||
export type {
|
||||
DesktopProjectionSpan,
|
||||
LegacySshProjectionIdentity,
|
||||
LegacySshProjectionReservation,
|
||||
LegacySshProjectionSemantics
|
||||
} from './ssh-pty-legacy-projection-record'
|
||||
|
||||
export type SshPtyLegacyProjectionLedgerOptions = {
|
||||
onSettled?: (span: DesktopProjectionSpan, reason: string) => void
|
||||
onTransferred?: (span: DesktopProjectionSpan, reason: string) => void
|
||||
}
|
||||
|
||||
export class SshPtyLegacyProjectionLedger {
|
||||
private nextId = 1
|
||||
private readonly records = new Map<string, ProjectionRecord>()
|
||||
private readonly cursorByPty = new Map<string, PtyProjectionCursor>()
|
||||
private readonly idsByPty = new Map<string, string[]>()
|
||||
private readonly terminality = new SshPtyProjectionTerminality()
|
||||
private settledCount = 0
|
||||
private transferredCount = 0
|
||||
private rolledBackCount = 0
|
||||
|
||||
constructor(private readonly options: SshPtyLegacyProjectionLedgerOptions = {}) {}
|
||||
|
||||
reserve(args: {
|
||||
ptyId: string
|
||||
providerGeneration: number
|
||||
ptyIncarnation: string
|
||||
data: string
|
||||
sequenceEnd: number
|
||||
rawLength: number
|
||||
transformed: boolean
|
||||
source?: Readonly<{
|
||||
relayPtyId?: string
|
||||
spanId: string
|
||||
clientGeneration: number
|
||||
ownerGeneration: number
|
||||
deliveryToken: string
|
||||
sourceStartSu: number
|
||||
sourceEndSu: number
|
||||
}>
|
||||
}): LegacySshProjectionReservation {
|
||||
const cursor = getOrCreateProjectionCursor(this.cursorByPty, args, (generation) =>
|
||||
this.transferGeneration(generation, 'provider-generation-replaced')
|
||||
)
|
||||
if (
|
||||
cursor.providerGeneration !== args.providerGeneration ||
|
||||
cursor.ptyIncarnation !== args.ptyIncarnation
|
||||
) {
|
||||
throw projectionError('ssh_projection_stale_generation')
|
||||
}
|
||||
const scan = scanMode2031ReplyDecision(cursor.scanner, args.data)
|
||||
const projectionSemanticsId = `ssh-projection:${args.providerGeneration}:${this.nextId++}`
|
||||
const identity = Object.freeze({
|
||||
projectionSemanticsId,
|
||||
ptyId: args.ptyId,
|
||||
providerGeneration: args.providerGeneration,
|
||||
ptyIncarnation: args.ptyIncarnation,
|
||||
displayStart: cursor.displayEnd,
|
||||
displayEnd: cursor.displayEnd + args.data.length,
|
||||
sequenceEnd: args.sequenceEnd,
|
||||
rawLength: args.rawLength,
|
||||
transformed: args.transformed
|
||||
})
|
||||
const semantics = Object.freeze({
|
||||
identity,
|
||||
...(args.source
|
||||
? {
|
||||
desktopSpan: Object.freeze({
|
||||
...args.source,
|
||||
id: args.source.relayPtyId ?? args.ptyId,
|
||||
projectionSemanticsId,
|
||||
providerGeneration: args.providerGeneration,
|
||||
ptyIncarnation: args.ptyIncarnation,
|
||||
displayStart: identity.displayStart,
|
||||
displayEnd: identity.displayEnd,
|
||||
splittable: !args.transformed,
|
||||
transform: Object.freeze({
|
||||
transformed: args.transformed,
|
||||
rawLengthSu: args.rawLength,
|
||||
scalarSafe: !args.transformed
|
||||
})
|
||||
})
|
||||
}
|
||||
: {}),
|
||||
beforeScanner: scannerSnapshot(cursor.scanner),
|
||||
afterScanner: scannerSnapshot(scan.state),
|
||||
decision: scan.decision
|
||||
})
|
||||
this.records.set(projectionSemanticsId, {
|
||||
semantics,
|
||||
state: 'reserved',
|
||||
publishedDisplay: 0,
|
||||
publishedAccounting: 0,
|
||||
settledAccounting: 0
|
||||
})
|
||||
return Object.freeze({ semantics })
|
||||
}
|
||||
|
||||
commit(reservation: LegacySshProjectionReservation): LegacySshProjectionSemantics {
|
||||
const record = requireProjectionRecord(
|
||||
this.records,
|
||||
reservation.semantics.identity.projectionSemanticsId
|
||||
)
|
||||
if (record.state !== 'reserved') {
|
||||
throw projectionError('ssh_projection_commit_invalid')
|
||||
}
|
||||
const { identity, afterScanner } = record.semantics
|
||||
const cursor = this.cursorByPty.get(identity.ptyId)
|
||||
if (
|
||||
!cursor ||
|
||||
cursor.providerGeneration !== identity.providerGeneration ||
|
||||
cursor.ptyIncarnation !== identity.ptyIncarnation ||
|
||||
cursor.displayEnd !== identity.displayStart
|
||||
) {
|
||||
this.records.delete(identity.projectionSemanticsId)
|
||||
throw projectionError('ssh_projection_commit_stale')
|
||||
}
|
||||
cursor.displayEnd = identity.displayEnd
|
||||
cursor.scanner = { ...afterScanner }
|
||||
record.state = 'committed'
|
||||
const ids = this.idsByPty.get(identity.ptyId) ?? []
|
||||
ids.push(identity.projectionSemanticsId)
|
||||
this.idsByPty.set(identity.ptyId, ids)
|
||||
return record.semantics
|
||||
}
|
||||
|
||||
rollback(reservation: LegacySshProjectionReservation): boolean {
|
||||
const id = reservation.semantics.identity.projectionSemanticsId
|
||||
const record = this.records.get(id)
|
||||
if (!record || record.state !== 'reserved') {
|
||||
return false
|
||||
}
|
||||
this.records.delete(id)
|
||||
this.rolledBackCount++
|
||||
return true
|
||||
}
|
||||
|
||||
rollbackCommitted(reservation: LegacySshProjectionReservation): boolean {
|
||||
const ptyId = rollbackCommittedProjectionRecord(
|
||||
this.records,
|
||||
this.idsByPty,
|
||||
this.cursorByPty,
|
||||
reservation
|
||||
)
|
||||
if (!ptyId) {
|
||||
return false
|
||||
}
|
||||
this.rolledBackCount++
|
||||
resolveProjectionTerminality(this.terminality, this.records, this.idsByPty, ptyId)
|
||||
return true
|
||||
}
|
||||
|
||||
publishPrefix(ids: readonly string[], displayChars: number, accountingChars: number): void {
|
||||
publishLegacyProjectionPrefix(this.records, ids, displayChars, accountingChars)
|
||||
}
|
||||
|
||||
settlePublishedPrefix(ptyId: string, accountingChars: number): number {
|
||||
const result = settlePublishedLegacyProjectionPrefix(
|
||||
this.records,
|
||||
this.idsByPty,
|
||||
ptyId,
|
||||
accountingChars,
|
||||
this.options.onSettled
|
||||
)
|
||||
this.settledCount += result.completed
|
||||
resolveProjectionTerminality(this.terminality, this.records, this.idsByPty, ptyId)
|
||||
return result.settled
|
||||
}
|
||||
|
||||
transfer(ids: readonly string[], reason: string): number {
|
||||
let transferred = 0
|
||||
const touchedPtys = new Set<string>()
|
||||
for (const id of ids.slice()) {
|
||||
const record = this.records.get(id)
|
||||
if (!record || record.state === 'reserved') {
|
||||
continue
|
||||
}
|
||||
const ptyId = record.semantics.identity.ptyId
|
||||
if (record.semantics.desktopSpan) {
|
||||
this.options.onTransferred?.(record.semantics.desktopSpan, reason)
|
||||
}
|
||||
this.transferredCount++
|
||||
reclaimProjectionRecord(this.records, this.idsByPty, id, ptyId)
|
||||
touchedPtys.add(ptyId)
|
||||
transferred++
|
||||
}
|
||||
for (const ptyId of touchedPtys) {
|
||||
resolveProjectionTerminality(this.terminality, this.records, this.idsByPty, ptyId)
|
||||
}
|
||||
return transferred
|
||||
}
|
||||
|
||||
whenPtyTerminal(
|
||||
ptyId: string,
|
||||
providerGeneration: number,
|
||||
ptyIncarnation: string
|
||||
): Promise<void> {
|
||||
return this.terminality.whenTerminal(
|
||||
ptyId,
|
||||
providerGeneration,
|
||||
ptyIncarnation,
|
||||
projectionHasOpen(this.records, this.idsByPty, ptyId)
|
||||
)
|
||||
}
|
||||
|
||||
transferGeneration(providerGeneration: number, reason: string): number {
|
||||
const ids: string[] = []
|
||||
for (const [id, record] of this.records) {
|
||||
if (record.semantics.identity.providerGeneration === providerGeneration) {
|
||||
ids.push(id)
|
||||
}
|
||||
}
|
||||
return this.transfer(ids, reason)
|
||||
}
|
||||
|
||||
transferPty(ptyId: string, reason: string): number {
|
||||
return this.transfer(this.idsByPty.get(ptyId) ?? [], reason)
|
||||
}
|
||||
|
||||
transferUnpublishedPty(
|
||||
ptyId: string,
|
||||
providerGeneration: number,
|
||||
ptyIncarnation: string,
|
||||
reason: string
|
||||
): number {
|
||||
const ids = unpublishedProjectionIds(
|
||||
this.records,
|
||||
this.idsByPty.get(ptyId) ?? [],
|
||||
providerGeneration,
|
||||
ptyIncarnation
|
||||
)
|
||||
return this.transfer(ids, reason)
|
||||
}
|
||||
|
||||
closePty(
|
||||
ptyId: string,
|
||||
providerGeneration: number,
|
||||
ptyIncarnation: string,
|
||||
reason: string
|
||||
): void {
|
||||
closeProjectionPty(this.cursorByPty, ptyId, providerGeneration, ptyIncarnation, () => {
|
||||
this.transferPty(ptyId, reason)
|
||||
this.idsByPty.delete(ptyId)
|
||||
})
|
||||
}
|
||||
|
||||
closeGeneration(providerGeneration: number, reason: string): void {
|
||||
for (const [id, record] of this.records) {
|
||||
if (record.semantics.identity.providerGeneration !== providerGeneration) {
|
||||
continue
|
||||
}
|
||||
if (record.state === 'reserved') {
|
||||
this.records.delete(id)
|
||||
this.rolledBackCount++
|
||||
} else {
|
||||
this.transfer([id], reason)
|
||||
}
|
||||
}
|
||||
for (const [ptyId, cursor] of this.cursorByPty) {
|
||||
if (cursor.providerGeneration === providerGeneration) {
|
||||
this.cursorByPty.delete(ptyId)
|
||||
this.idsByPty.delete(ptyId)
|
||||
}
|
||||
}
|
||||
this.terminality.closeGeneration(providerGeneration)
|
||||
}
|
||||
|
||||
resetForGap(ptyId: string): void {
|
||||
resetProjectionCursorForGap(this.cursorByPty, ptyId)
|
||||
}
|
||||
|
||||
get(id: string): LegacySshProjectionSemantics | undefined {
|
||||
return this.records.get(id)?.semantics
|
||||
}
|
||||
|
||||
hasUnpublished(id: string): boolean {
|
||||
return hasUnpublishedLegacyProjection(this.records, id)
|
||||
}
|
||||
|
||||
getDebugSnapshot(): LegacySshProjectionDebugSnapshot {
|
||||
return projectionDebugSnapshot(this.records, this.cursorByPty, {
|
||||
settled: this.settledCount,
|
||||
transferred: this.transferredCount,
|
||||
rolledBack: this.rolledBackCount
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
export type SshPtyModelAdmissionKey = Readonly<{
|
||||
ptyId: string
|
||||
providerGeneration: number
|
||||
}>
|
||||
|
||||
export type SshPtyModelAdmissionReceipt = Readonly<{
|
||||
ptyId: string
|
||||
providerGeneration: number
|
||||
sequence: number
|
||||
}>
|
||||
|
||||
export type SshPtyModelAdmissionDebugSnapshot = Readonly<{
|
||||
sourceUnits: number
|
||||
bytes: number
|
||||
pressureFrames: number
|
||||
pressureBytes: number
|
||||
pausedPtys: number
|
||||
migratingPtys: number
|
||||
}>
|
||||
|
||||
export type SshPtyModelAdmissionOptions = {
|
||||
perPtyHighSourceUnits?: number
|
||||
perPtyHighBytes?: number
|
||||
perPtyLowSourceUnits?: number
|
||||
perPtyLowBytes?: number
|
||||
globalHighSourceUnits?: number
|
||||
globalHighBytes?: number
|
||||
globalLowSourceUnits?: number
|
||||
globalLowBytes?: number
|
||||
pressureMaxFrames?: number
|
||||
pressureMaxBytes?: number
|
||||
pauseProvider?: (key: SshPtyModelAdmissionKey) => boolean
|
||||
resumeProvider?: (key: SshPtyModelAdmissionKey) => void
|
||||
closeProvider?: (providerGeneration: number, reason: string) => void
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
import type {
|
||||
SshPtyModelAdmissionKey,
|
||||
SshPtyModelAdmissionReceipt
|
||||
} from './ssh-pty-model-admission-contract'
|
||||
import type { SshPtyModelAdmissionLimits } from './ssh-pty-model-admission-limits'
|
||||
|
||||
export type AdmissionCharge = { sourceUnits: number; bytes: number }
|
||||
|
||||
export type AdmissionEntry = {
|
||||
key: SshPtyModelAdmissionKey
|
||||
charge: AdmissionCharge
|
||||
run: () => { sequence: number; completion: Promise<void> }
|
||||
resolve: (receipt: SshPtyModelAdmissionReceipt) => void
|
||||
reject: (error: Error) => void
|
||||
state: 'queued' | 'running' | 'pressure' | 'settled'
|
||||
}
|
||||
|
||||
export type PtyUsage = AdmissionCharge & {
|
||||
queued: AdmissionEntry[]
|
||||
running: AdmissionEntry | null
|
||||
}
|
||||
|
||||
export function admissionError(code: string): Error {
|
||||
return Object.assign(new Error(code), { code })
|
||||
}
|
||||
|
||||
export function retainedBytes(data: string): number {
|
||||
return Math.max(Buffer.byteLength(data, 'utf8'), 2 * data.length) + 128
|
||||
}
|
||||
|
||||
export function admissionKeyId(key: SshPtyModelAdmissionKey): string {
|
||||
return `${key.providerGeneration}\0${key.ptyId}`
|
||||
}
|
||||
|
||||
export function canReserveAdmission(args: {
|
||||
key: SshPtyModelAdmissionKey
|
||||
charge: AdmissionCharge
|
||||
limits: SshPtyModelAdmissionLimits
|
||||
usageByPty: ReadonlyMap<string, PtyUsage>
|
||||
closingGenerations: ReadonlySet<number>
|
||||
globalSourceUnits: number
|
||||
globalBytes: number
|
||||
}): boolean {
|
||||
if (args.closingGenerations.has(args.key.providerGeneration)) {
|
||||
return false
|
||||
}
|
||||
const usage = args.usageByPty.get(admissionKeyId(args.key))
|
||||
return (
|
||||
(usage?.sourceUnits ?? 0) + args.charge.sourceUnits <= args.limits.perPtyHighSourceUnits &&
|
||||
(usage?.bytes ?? 0) + args.charge.bytes <= args.limits.perPtyHighBytes &&
|
||||
args.globalSourceUnits + args.charge.sourceUnits <= args.limits.globalHighSourceUnits &&
|
||||
args.globalBytes + args.charge.bytes <= args.limits.globalHighBytes
|
||||
)
|
||||
}
|
||||
|
||||
export function pressureHasAdmissionKey(
|
||||
entries: readonly AdmissionEntry[],
|
||||
key: SshPtyModelAdmissionKey
|
||||
): boolean {
|
||||
const id = admissionKeyId(key)
|
||||
return entries.some((entry) => admissionKeyId(entry.key) === id)
|
||||
}
|
||||
|
||||
export function takePressureEntriesForGeneration(
|
||||
entries: AdmissionEntry[],
|
||||
providerGeneration: number
|
||||
): AdmissionEntry[] {
|
||||
const removed: AdmissionEntry[] = []
|
||||
for (let index = entries.length - 1; index >= 0; index--) {
|
||||
if (entries[index]!.key.providerGeneration === providerGeneration) {
|
||||
removed.unshift(entries.splice(index, 1)[0]!)
|
||||
}
|
||||
}
|
||||
return removed
|
||||
}
|
||||
|
||||
export function cancelAdmissionGeneration(args: {
|
||||
pressure: AdmissionEntry[]
|
||||
usageByPty: Map<string, PtyUsage>
|
||||
idleWaiters: Map<string, Set<() => void>>
|
||||
providerGeneration: number
|
||||
error: Error
|
||||
release: (key: SshPtyModelAdmissionKey, charge: AdmissionCharge) => void
|
||||
}): number {
|
||||
let pressureBytes = 0
|
||||
for (const entry of takePressureEntriesForGeneration(args.pressure, args.providerGeneration)) {
|
||||
pressureBytes += entry.charge.bytes
|
||||
entry.state = 'settled'
|
||||
entry.reject(args.error)
|
||||
}
|
||||
for (const [id, usage] of args.usageByPty) {
|
||||
const canceled = usage.queued.filter(
|
||||
(entry) => entry.key.providerGeneration === args.providerGeneration
|
||||
)
|
||||
usage.queued = usage.queued.filter(
|
||||
(entry) => entry.key.providerGeneration !== args.providerGeneration
|
||||
)
|
||||
if (usage.running?.key.providerGeneration === args.providerGeneration) {
|
||||
canceled.push(usage.running)
|
||||
usage.running = null
|
||||
}
|
||||
for (const entry of canceled) {
|
||||
if (entry.state === 'settled') {
|
||||
continue
|
||||
}
|
||||
entry.state = 'settled'
|
||||
args.release(entry.key, entry.charge)
|
||||
entry.reject(args.error)
|
||||
}
|
||||
if (!usage.running && usage.queued.length === 0 && usage.sourceUnits === 0) {
|
||||
args.usageByPty.delete(id)
|
||||
}
|
||||
resolveAdmissionIdleWaiters(args.usageByPty, args.pressure, args.idleWaiters, id)
|
||||
}
|
||||
return pressureBytes
|
||||
}
|
||||
|
||||
export function takePausedGeneration(
|
||||
paused: Map<string, SshPtyModelAdmissionKey>,
|
||||
providerGeneration: number
|
||||
): SshPtyModelAdmissionKey[] {
|
||||
const removed: SshPtyModelAdmissionKey[] = []
|
||||
for (const [id, key] of paused) {
|
||||
if (key.providerGeneration === providerGeneration) {
|
||||
paused.delete(id)
|
||||
removed.push(key)
|
||||
}
|
||||
}
|
||||
return removed
|
||||
}
|
||||
|
||||
export function resolveAdmissionIdleWaiters(
|
||||
usageByPty: ReadonlyMap<string, PtyUsage>,
|
||||
pressure: readonly AdmissionEntry[],
|
||||
waitersById: Map<string, Set<() => void>>,
|
||||
id: string
|
||||
): void {
|
||||
const usage = usageByPty.get(id)
|
||||
const stillPressured = pressure.some((entry) => admissionKeyId(entry.key) === id)
|
||||
if (usage?.running || (usage?.queued.length ?? 0) > 0 || stillPressured) {
|
||||
return
|
||||
}
|
||||
const waiters = waitersById.get(id)
|
||||
if (!waiters) {
|
||||
return
|
||||
}
|
||||
waitersById.delete(id)
|
||||
for (const resolve of waiters) {
|
||||
resolve()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import type { SshPtyModelAdmissionOptions } from './ssh-pty-model-admission-contract'
|
||||
|
||||
export type SshPtyModelAdmissionLimits = Readonly<{
|
||||
perPtyHighSourceUnits: number
|
||||
perPtyHighBytes: number
|
||||
perPtyLowSourceUnits: number
|
||||
perPtyLowBytes: number
|
||||
globalHighSourceUnits: number
|
||||
globalHighBytes: number
|
||||
globalLowSourceUnits: number
|
||||
globalLowBytes: number
|
||||
pressureMaxFrames: number
|
||||
pressureMaxBytes: number
|
||||
}>
|
||||
|
||||
const DEFAULT_PER_PTY_HIGH_SOURCE_UNITS = 256 * 1024
|
||||
const DEFAULT_PER_PTY_HIGH_BYTES = 2 * 1024 * 1024
|
||||
const DEFAULT_GLOBAL_HIGH_SOURCE_UNITS = 50 * DEFAULT_PER_PTY_HIGH_SOURCE_UNITS
|
||||
const DEFAULT_GLOBAL_HIGH_BYTES = 64 * 1024 * 1024
|
||||
|
||||
export function resolveSshPtyModelAdmissionLimits(
|
||||
options: SshPtyModelAdmissionOptions
|
||||
): SshPtyModelAdmissionLimits {
|
||||
const perPtyHighSourceUnits = options.perPtyHighSourceUnits ?? DEFAULT_PER_PTY_HIGH_SOURCE_UNITS
|
||||
const perPtyHighBytes = options.perPtyHighBytes ?? DEFAULT_PER_PTY_HIGH_BYTES
|
||||
return {
|
||||
perPtyHighSourceUnits,
|
||||
perPtyHighBytes,
|
||||
perPtyLowSourceUnits: options.perPtyLowSourceUnits ?? Math.floor(perPtyHighSourceUnits / 2),
|
||||
perPtyLowBytes: options.perPtyLowBytes ?? Math.floor(perPtyHighBytes / 2),
|
||||
globalHighSourceUnits: options.globalHighSourceUnits ?? DEFAULT_GLOBAL_HIGH_SOURCE_UNITS,
|
||||
globalHighBytes: options.globalHighBytes ?? DEFAULT_GLOBAL_HIGH_BYTES,
|
||||
globalLowSourceUnits: options.globalLowSourceUnits ?? 8 * 1024 * 1024,
|
||||
globalLowBytes: options.globalLowBytes ?? 48 * 1024 * 1024,
|
||||
pressureMaxFrames: options.pressureMaxFrames ?? 64,
|
||||
pressureMaxBytes: options.pressureMaxBytes ?? 1024 * 1024
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
import type { SshPtyModelAdmissionKey } from './ssh-pty-model-admission-contract'
|
||||
import type { AdmissionCharge, AdmissionEntry, PtyUsage } from './ssh-pty-model-admission-entry'
|
||||
import { admissionError, admissionKeyId } from './ssh-pty-model-admission-entry'
|
||||
import type { SshPtyModelAdmissionPressure } from './ssh-pty-model-admission-pressure'
|
||||
|
||||
export function beginSshPtyModelAdmissionMigration(args: {
|
||||
key: SshPtyModelAdmissionKey
|
||||
migratingPtys: Set<string>
|
||||
pressure: SshPtyModelAdmissionPressure
|
||||
usageByPty: Map<string, PtyUsage>
|
||||
release: (key: SshPtyModelAdmissionKey, charge: AdmissionCharge) => void
|
||||
cleanup: (id: string, usage: PtyUsage) => void
|
||||
}): void {
|
||||
const id = admissionKeyId(args.key)
|
||||
if (args.migratingPtys.has(id)) {
|
||||
return
|
||||
}
|
||||
args.migratingPtys.add(id)
|
||||
const error = admissionError('ssh_model_migration_queued_canceled')
|
||||
args.pressure.cancelQueuedPty(args.key, error)
|
||||
const usage = args.usageByPty.get(id)
|
||||
if (!usage) {
|
||||
return
|
||||
}
|
||||
const queued = usage.queued
|
||||
usage.queued = []
|
||||
for (const entry of queued) {
|
||||
cancelQueuedEntry(entry, error, args.release)
|
||||
}
|
||||
args.cleanup(id, usage)
|
||||
}
|
||||
|
||||
export function closeSshPtyModelAdmissionMigrations(
|
||||
migratingPtys: Set<string>,
|
||||
providerGeneration: number
|
||||
): void {
|
||||
const prefix = `${providerGeneration}\0`
|
||||
for (const id of migratingPtys) {
|
||||
if (id.startsWith(prefix)) {
|
||||
migratingPtys.delete(id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function settleSshPtyModelAdmissionFailure(args: {
|
||||
id: string
|
||||
usage: PtyUsage
|
||||
entry: AdmissionEntry
|
||||
error: Error
|
||||
migratingPtys: ReadonlySet<string>
|
||||
closingGenerations: Set<number>
|
||||
release: (key: SshPtyModelAdmissionKey, charge: AdmissionCharge) => void
|
||||
closeGeneration: (providerGeneration: number) => void
|
||||
cleanup: (id: string, usage: PtyUsage) => void
|
||||
}): void {
|
||||
if (args.entry.state !== 'running' || args.usage.running !== args.entry) {
|
||||
return
|
||||
}
|
||||
const migrationOwnsFailure = args.migratingPtys.has(args.id)
|
||||
if (!migrationOwnsFailure) {
|
||||
args.closingGenerations.add(args.entry.key.providerGeneration)
|
||||
}
|
||||
args.usage.running = null
|
||||
args.entry.state = 'settled'
|
||||
args.release(args.entry.key, args.entry.charge)
|
||||
args.entry.reject(migrationOwnsFailure ? migrationCompletionError(args.error) : args.error)
|
||||
if (!migrationOwnsFailure) {
|
||||
args.closeGeneration(args.entry.key.providerGeneration)
|
||||
}
|
||||
args.cleanup(args.id, args.usage)
|
||||
}
|
||||
|
||||
function migrationCompletionError(cause: Error): Error {
|
||||
return Object.assign(new Error(cause.message), {
|
||||
code: 'ssh_model_migration_completion_failed',
|
||||
cause
|
||||
})
|
||||
}
|
||||
|
||||
function cancelQueuedEntry(
|
||||
entry: AdmissionEntry,
|
||||
error: Error,
|
||||
release: (key: SshPtyModelAdmissionKey, charge: AdmissionCharge) => void
|
||||
): void {
|
||||
if (entry.state === 'settled') {
|
||||
return
|
||||
}
|
||||
entry.state = 'settled'
|
||||
release(entry.key, entry.charge)
|
||||
entry.reject(error)
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
import type { SshPtyModelAdmissionKey } from './ssh-pty-model-admission-contract'
|
||||
import {
|
||||
admissionKeyId,
|
||||
pressureHasAdmissionKey,
|
||||
takePausedGeneration,
|
||||
type AdmissionEntry,
|
||||
type PtyUsage
|
||||
} from './ssh-pty-model-admission-entry'
|
||||
import type { SshPtyModelAdmissionLimits } from './ssh-pty-model-admission-limits'
|
||||
|
||||
type PressureOptions = {
|
||||
limits: SshPtyModelAdmissionLimits
|
||||
pauseProvider: (key: SshPtyModelAdmissionKey) => boolean
|
||||
resumeProvider: (key: SshPtyModelAdmissionKey) => void
|
||||
}
|
||||
|
||||
type PromotionOptions = {
|
||||
usageByPty: ReadonlyMap<string, PtyUsage>
|
||||
disposed: boolean
|
||||
canReserve: (entry: AdmissionEntry) => boolean
|
||||
reserve: (entry: AdmissionEntry) => void
|
||||
isBelowGlobalLowWatermark: () => boolean
|
||||
}
|
||||
|
||||
export class SshPtyModelAdmissionPressure {
|
||||
private readonly entries: AdmissionEntry[] = []
|
||||
private readonly pausedKeys = new Map<string, SshPtyModelAdmissionKey>()
|
||||
private retainedBytes = 0
|
||||
|
||||
constructor(private readonly options: PressureOptions) {}
|
||||
|
||||
get values(): readonly AdmissionEntry[] {
|
||||
return this.entries
|
||||
}
|
||||
|
||||
get frameCount(): number {
|
||||
return this.entries.length
|
||||
}
|
||||
|
||||
get bytes(): number {
|
||||
return this.retainedBytes
|
||||
}
|
||||
|
||||
get pausedPtyCount(): number {
|
||||
return this.pausedKeys.size
|
||||
}
|
||||
|
||||
get pausedProviderGenerations(): ReadonlySet<number> {
|
||||
return new Set(Array.from(this.pausedKeys.values(), (key) => key.providerGeneration))
|
||||
}
|
||||
|
||||
has(key: SshPtyModelAdmissionKey): boolean {
|
||||
return pressureHasAdmissionKey(this.entries, key)
|
||||
}
|
||||
|
||||
admit(entry: AdmissionEntry): boolean {
|
||||
const id = admissionKeyId(entry.key)
|
||||
const paused = this.pausedKeys.has(id) || this.options.pauseProvider(entry.key)
|
||||
if (paused) {
|
||||
this.pausedKeys.set(id, entry.key)
|
||||
}
|
||||
if (
|
||||
!paused ||
|
||||
this.entries.length >= this.options.limits.pressureMaxFrames ||
|
||||
this.retainedBytes + entry.charge.bytes > this.options.limits.pressureMaxBytes
|
||||
) {
|
||||
return false
|
||||
}
|
||||
entry.state = 'pressure'
|
||||
this.entries.push(entry)
|
||||
this.retainedBytes += entry.charge.bytes
|
||||
return true
|
||||
}
|
||||
|
||||
cancelGeneration(
|
||||
providerGeneration: number,
|
||||
cancelPressureAndReserved: (entries: AdmissionEntry[]) => number
|
||||
): void {
|
||||
this.retainedBytes -= cancelPressureAndReserved(this.entries)
|
||||
for (const key of takePausedGeneration(this.pausedKeys, providerGeneration)) {
|
||||
this.resume(key)
|
||||
}
|
||||
}
|
||||
|
||||
cancelPty(key: SshPtyModelAdmissionKey, error: Error, cancelReserved: () => void): void {
|
||||
this.cancelQueuedPty(key, error)
|
||||
cancelReserved()
|
||||
}
|
||||
|
||||
cancelQueuedPty(key: SshPtyModelAdmissionKey, error: Error): void {
|
||||
const id = admissionKeyId(key)
|
||||
const canceled: AdmissionEntry[] = []
|
||||
for (let index = this.entries.length - 1; index >= 0; index--) {
|
||||
if (admissionKeyId(this.entries[index]!.key) === id) {
|
||||
canceled.push(this.entries.splice(index, 1)[0]!)
|
||||
}
|
||||
}
|
||||
this.rejectPressureEntries(canceled, error)
|
||||
const paused = this.pausedKeys.get(id)
|
||||
if (paused) {
|
||||
this.pausedKeys.delete(id)
|
||||
this.resume(paused)
|
||||
}
|
||||
}
|
||||
|
||||
promoteAndResume(options: PromotionOptions): void {
|
||||
if (options.disposed) {
|
||||
return
|
||||
}
|
||||
for (let index = 0; index < this.entries.length; ) {
|
||||
const entry = this.entries[index]!
|
||||
const hasEarlierEntryForPty = this.entries
|
||||
.slice(0, index)
|
||||
.some((earlier) => admissionKeyId(earlier.key) === admissionKeyId(entry.key))
|
||||
if (hasEarlierEntryForPty || !options.canReserve(entry)) {
|
||||
index++
|
||||
continue
|
||||
}
|
||||
this.entries.splice(index, 1)
|
||||
this.retainedBytes -= entry.charge.bytes
|
||||
options.reserve(entry)
|
||||
}
|
||||
if (!options.isBelowGlobalLowWatermark()) {
|
||||
return
|
||||
}
|
||||
for (const [id, key] of this.pausedKeys) {
|
||||
const usage = options.usageByPty.get(id)
|
||||
if (
|
||||
this.has(key) ||
|
||||
(usage?.sourceUnits ?? 0) > this.options.limits.perPtyLowSourceUnits ||
|
||||
(usage?.bytes ?? 0) > this.options.limits.perPtyLowBytes
|
||||
) {
|
||||
continue
|
||||
}
|
||||
this.pausedKeys.delete(id)
|
||||
this.resume(key)
|
||||
}
|
||||
}
|
||||
|
||||
private rejectPressureEntries(entries: readonly AdmissionEntry[], error: Error): void {
|
||||
for (const entry of entries) {
|
||||
this.retainedBytes -= entry.charge.bytes
|
||||
entry.state = 'settled'
|
||||
entry.reject(error)
|
||||
}
|
||||
}
|
||||
|
||||
private resume(key: SshPtyModelAdmissionKey): void {
|
||||
try {
|
||||
this.options.resumeProvider(key)
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import type { SshPtyModelAdmissionDebugSnapshot } from './ssh-pty-model-admission-contract'
|
||||
import type { SshPtyModelAdmissionPressure } from './ssh-pty-model-admission-pressure'
|
||||
|
||||
export function sshPtyModelAdmissionSnapshot(
|
||||
sourceUnits: number,
|
||||
bytes: number,
|
||||
pressure: SshPtyModelAdmissionPressure,
|
||||
migratingPtys: ReadonlySet<string>
|
||||
): SshPtyModelAdmissionDebugSnapshot {
|
||||
return {
|
||||
sourceUnits,
|
||||
bytes,
|
||||
pressureFrames: pressure.frameCount,
|
||||
pressureBytes: pressure.bytes,
|
||||
pausedPtys: pressure.pausedPtyCount,
|
||||
migratingPtys: migratingPtys.size
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { SshPtyModelAdmission } from './ssh-pty-model-admission'
|
||||
|
||||
function deferred() {
|
||||
let resolve!: () => void
|
||||
let reject!: (error: Error) => void
|
||||
const promise = new Promise<void>((promiseResolve, promiseReject) => {
|
||||
resolve = promiseResolve
|
||||
reject = promiseReject
|
||||
})
|
||||
return { promise, resolve, reject }
|
||||
}
|
||||
|
||||
function accept(admission: SshPtyModelAdmission, completion: Promise<void>) {
|
||||
return admission.accept({ ptyId: 'pty-1', providerGeneration: 7 }, 'data', 4, () => ({
|
||||
sequence: 4,
|
||||
completion
|
||||
}))
|
||||
}
|
||||
|
||||
describe('SshPtyModelAdmission', () => {
|
||||
it('freezes migration while retaining the running raw completion', async () => {
|
||||
const runningCompletion = deferred()
|
||||
const admission = new SshPtyModelAdmission()
|
||||
const running = accept(admission, runningCompletion.promise)
|
||||
const queued = accept(admission, Promise.resolve())
|
||||
|
||||
admission.beginMigration({ ptyId: 'pty-1', providerGeneration: 7 })
|
||||
|
||||
await expect(queued).rejects.toThrow('ssh_model_migration_queued_canceled')
|
||||
expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 4 })
|
||||
await expect(
|
||||
admission.accept({ ptyId: 'pty-1', providerGeneration: 7 }, 'late', 4, () => ({
|
||||
sequence: 8,
|
||||
completion: Promise.resolve()
|
||||
}))
|
||||
).rejects.toThrow('ssh_model_admission_migrating')
|
||||
|
||||
runningCompletion.resolve()
|
||||
await expect(running).resolves.toMatchObject({ sequence: 4 })
|
||||
expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 0, bytes: 0 })
|
||||
})
|
||||
|
||||
it('cancels a never-settling running entry when its generation closes', async () => {
|
||||
const admission = new SshPtyModelAdmission()
|
||||
const receipt = accept(admission, new Promise<void>(() => {}))
|
||||
const idle = admission.whenIdle({ ptyId: 'pty-1', providerGeneration: 7 })
|
||||
|
||||
admission.closeGeneration(7, 'provider-closed')
|
||||
|
||||
await expect(receipt).rejects.toThrow('provider-closed')
|
||||
await expect(idle).resolves.toBeUndefined()
|
||||
expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 0, bytes: 0 })
|
||||
})
|
||||
|
||||
it('cancels a never-settling running entry on disposal', async () => {
|
||||
const admission = new SshPtyModelAdmission()
|
||||
const receipt = accept(admission, new Promise<void>(() => {}))
|
||||
const idle = admission.whenIdle({ ptyId: 'pty-1', providerGeneration: 7 })
|
||||
|
||||
admission.dispose()
|
||||
|
||||
await expect(receipt).rejects.toThrow('ssh_model_admission_disposed')
|
||||
await expect(idle).resolves.toBeUndefined()
|
||||
expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 0, bytes: 0 })
|
||||
})
|
||||
|
||||
it('keeps non-migrating callback failure generation-fatal across sibling PTYs', async () => {
|
||||
const admission = new SshPtyModelAdmission()
|
||||
const failedCompletion = deferred()
|
||||
const siblingCompletion = deferred()
|
||||
const failed = accept(admission, failedCompletion.promise)
|
||||
const sibling = admission.accept({ ptyId: 'pty-2', providerGeneration: 7 }, 'data', 4, () => ({
|
||||
sequence: 4,
|
||||
completion: siblingCompletion.promise
|
||||
}))
|
||||
|
||||
failedCompletion.reject(new Error('emulator failed'))
|
||||
|
||||
await expect(failed).rejects.toThrow('emulator failed')
|
||||
await expect(sibling).rejects.toThrow('ssh_model_admission_completion_failed')
|
||||
await expect(
|
||||
admission.accept({ ptyId: 'pty-3', providerGeneration: 7 }, 'data', 4, () => ({
|
||||
sequence: 4,
|
||||
completion: Promise.resolve()
|
||||
}))
|
||||
).rejects.toThrow('ssh_model_admission_generation_closed')
|
||||
expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 0, bytes: 0 })
|
||||
siblingCompletion.resolve()
|
||||
await Promise.resolve()
|
||||
expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 0, bytes: 0 })
|
||||
})
|
||||
|
||||
it('resumes every paused provider generation exactly once on disposal', async () => {
|
||||
const resumeProvider = vi.fn()
|
||||
const admission = new SshPtyModelAdmission({
|
||||
perPtyHighSourceUnits: 4,
|
||||
perPtyHighBytes: 1024,
|
||||
globalHighSourceUnits: 4,
|
||||
globalHighBytes: 1024,
|
||||
pressureMaxFrames: 1,
|
||||
pressureMaxBytes: 1024,
|
||||
pauseProvider: () => true,
|
||||
resumeProvider
|
||||
})
|
||||
const running = accept(admission, new Promise<void>(() => {}))
|
||||
const pressured = accept(admission, Promise.resolve())
|
||||
const rejected = admission.accept({ ptyId: 'pty-2', providerGeneration: 8 }, 'data', 4, () => ({
|
||||
sequence: 4,
|
||||
completion: Promise.resolve()
|
||||
}))
|
||||
|
||||
await expect(rejected).rejects.toThrow('ssh_model_admission_pressure_exhausted')
|
||||
admission.dispose()
|
||||
admission.dispose()
|
||||
|
||||
await expect(running).rejects.toThrow('ssh_model_admission_disposed')
|
||||
await expect(pressured).rejects.toThrow('ssh_model_admission_disposed')
|
||||
expect(resumeProvider.mock.calls).toEqual([
|
||||
[{ ptyId: 'pty-1', providerGeneration: 7 }],
|
||||
[{ ptyId: 'pty-2', providerGeneration: 8 }]
|
||||
])
|
||||
})
|
||||
|
||||
it.each(['resolve', 'reject'] as const)(
|
||||
'ignores a late completion %s after cancellation',
|
||||
async (settle) => {
|
||||
const completion = deferred()
|
||||
const admission = new SshPtyModelAdmission()
|
||||
const onResolve = vi.fn()
|
||||
const onReject = vi.fn()
|
||||
const receipt = accept(admission, completion.promise)
|
||||
const observed = receipt.then(onResolve, onReject)
|
||||
|
||||
admission.closeGeneration(7, 'provider-closed')
|
||||
await observed
|
||||
if (settle === 'resolve') {
|
||||
completion.resolve()
|
||||
} else {
|
||||
completion.reject(new Error('late emulator failure'))
|
||||
}
|
||||
await Promise.resolve()
|
||||
|
||||
expect(onResolve).not.toHaveBeenCalled()
|
||||
expect(onReject).toHaveBeenCalledTimes(1)
|
||||
expect(onReject.mock.calls[0]?.[0]).toMatchObject({ message: 'provider-closed' })
|
||||
expect(admission.getDebugSnapshot()).toMatchObject({ sourceUnits: 0, bytes: 0 })
|
||||
}
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,315 @@
|
||||
import type {
|
||||
SshPtyModelAdmissionKey,
|
||||
SshPtyModelAdmissionOptions,
|
||||
SshPtyModelAdmissionReceipt
|
||||
} from './ssh-pty-model-admission-contract'
|
||||
import {
|
||||
admissionError,
|
||||
admissionKeyId,
|
||||
canReserveAdmission,
|
||||
cancelAdmissionGeneration,
|
||||
retainedBytes,
|
||||
resolveAdmissionIdleWaiters,
|
||||
type AdmissionCharge,
|
||||
type AdmissionEntry,
|
||||
type PtyUsage
|
||||
} from './ssh-pty-model-admission-entry'
|
||||
import { resolveSshPtyModelAdmissionLimits } from './ssh-pty-model-admission-limits'
|
||||
import * as modelAdmissionMigration from './ssh-pty-model-admission-migration'
|
||||
import { SshPtyModelAdmissionPressure } from './ssh-pty-model-admission-pressure'
|
||||
import { sshPtyModelAdmissionSnapshot } from './ssh-pty-model-admission-snapshot'
|
||||
|
||||
export type * from './ssh-pty-model-admission-contract'
|
||||
|
||||
export class SshPtyModelAdmission {
|
||||
private readonly limits: ReturnType<typeof resolveSshPtyModelAdmissionLimits>
|
||||
private readonly closeProvider: (providerGeneration: number, reason: string) => void
|
||||
private readonly usageByPty = new Map<string, PtyUsage>()
|
||||
private readonly pressure: SshPtyModelAdmissionPressure
|
||||
private readonly idleWaiters = new Map<string, Set<() => void>>()
|
||||
private readonly closingGenerations = new Set<number>()
|
||||
private readonly migratingPtys = new Set<string>()
|
||||
private globalSourceUnits = 0
|
||||
private globalBytes = 0
|
||||
private disposed = false
|
||||
|
||||
constructor(options: SshPtyModelAdmissionOptions = {}) {
|
||||
this.limits = resolveSshPtyModelAdmissionLimits(options)
|
||||
this.pressure = new SshPtyModelAdmissionPressure({
|
||||
limits: this.limits,
|
||||
pauseProvider: options.pauseProvider ?? (() => false),
|
||||
resumeProvider: options.resumeProvider ?? (() => {})
|
||||
})
|
||||
this.closeProvider = options.closeProvider ?? (() => {})
|
||||
}
|
||||
|
||||
accept(
|
||||
key: SshPtyModelAdmissionKey,
|
||||
data: string,
|
||||
sourceUnits: number,
|
||||
run: () => { sequence: number; completion: Promise<void> }
|
||||
): Promise<SshPtyModelAdmissionReceipt> {
|
||||
if (this.disposed) {
|
||||
return Promise.reject(admissionError('ssh_model_admission_disposed'))
|
||||
}
|
||||
if (this.closingGenerations.has(key.providerGeneration)) {
|
||||
return Promise.reject(admissionError('ssh_model_admission_generation_closed'))
|
||||
}
|
||||
if (this.migratingPtys.has(admissionKeyId(key))) {
|
||||
return Promise.reject(admissionError('ssh_model_admission_migrating'))
|
||||
}
|
||||
const charge = { sourceUnits, bytes: retainedBytes(data) }
|
||||
return new Promise((resolve, reject) => {
|
||||
const entry: AdmissionEntry = {
|
||||
key: { ...key },
|
||||
charge,
|
||||
run,
|
||||
resolve,
|
||||
reject,
|
||||
state: 'queued'
|
||||
}
|
||||
if (this.canReserve(key, charge) && !this.pressure.has(key)) {
|
||||
this.reserveAndQueue(entry)
|
||||
return
|
||||
}
|
||||
if (!this.pressure.admit(entry)) {
|
||||
entry.reject(admissionError('ssh_model_admission_pressure_exhausted'))
|
||||
this.closeProvider(entry.key.providerGeneration, 'model-admission-pressure')
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
closeGeneration(providerGeneration: number, reason = 'provider-generation-closed'): void {
|
||||
this.closingGenerations.add(providerGeneration)
|
||||
const error = admissionError(reason)
|
||||
this.pressure.cancelGeneration(providerGeneration, (pressure) =>
|
||||
cancelAdmissionGeneration({
|
||||
pressure,
|
||||
usageByPty: this.usageByPty,
|
||||
idleWaiters: this.idleWaiters,
|
||||
providerGeneration,
|
||||
error,
|
||||
release: (key, charge) => this.release(key, charge)
|
||||
})
|
||||
)
|
||||
modelAdmissionMigration.closeSshPtyModelAdmissionMigrations(
|
||||
this.migratingPtys,
|
||||
providerGeneration
|
||||
)
|
||||
const generationPrefix = `${providerGeneration}\0`
|
||||
for (const id of this.idleWaiters.keys()) {
|
||||
if (id.startsWith(generationPrefix)) {
|
||||
resolveAdmissionIdleWaiters(this.usageByPty, this.pressure.values, this.idleWaiters, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
beginMigration(key: SshPtyModelAdmissionKey): void {
|
||||
modelAdmissionMigration.beginSshPtyModelAdmissionMigration({
|
||||
key,
|
||||
migratingPtys: this.migratingPtys,
|
||||
pressure: this.pressure,
|
||||
usageByPty: this.usageByPty,
|
||||
release: (entryKey, charge) => this.release(entryKey, charge),
|
||||
cleanup: (id, usage) => this.cleanupUsage(id, usage)
|
||||
})
|
||||
const id = admissionKeyId(key)
|
||||
resolveAdmissionIdleWaiters(this.usageByPty, this.pressure.values, this.idleWaiters, id)
|
||||
}
|
||||
|
||||
cancelPty(key: SshPtyModelAdmissionKey, reason: string): void {
|
||||
const id = admissionKeyId(key)
|
||||
const error = admissionError(reason)
|
||||
this.pressure.cancelPty(key, error, () => {
|
||||
const usage = this.usageByPty.get(id)
|
||||
if (usage) {
|
||||
const canceled = [...usage.queued, ...(usage.running ? [usage.running] : [])]
|
||||
usage.queued = []
|
||||
usage.running = null
|
||||
for (const entry of canceled) {
|
||||
if (entry.state === 'settled') {
|
||||
continue
|
||||
}
|
||||
entry.state = 'settled'
|
||||
this.release(entry.key, entry.charge)
|
||||
entry.reject(error)
|
||||
}
|
||||
this.usageByPty.delete(id)
|
||||
}
|
||||
})
|
||||
resolveAdmissionIdleWaiters(this.usageByPty, this.pressure.values, this.idleWaiters, id)
|
||||
}
|
||||
|
||||
whenIdle(key: SshPtyModelAdmissionKey): Promise<void> {
|
||||
const id = admissionKeyId(key)
|
||||
const usage = this.usageByPty.get(id)
|
||||
const hasPressure = this.pressure.has(key)
|
||||
if ((!usage || (!usage.running && usage.queued.length === 0)) && !hasPressure) {
|
||||
return Promise.resolve()
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
const waiters = this.idleWaiters.get(id) ?? new Set<() => void>()
|
||||
waiters.add(resolve)
|
||||
this.idleWaiters.set(id, waiters)
|
||||
})
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
if (this.disposed) {
|
||||
return
|
||||
}
|
||||
this.disposed = true
|
||||
const generations = new Set<number>()
|
||||
for (const usage of this.usageByPty.values()) {
|
||||
if (usage.running) {
|
||||
generations.add(usage.running.key.providerGeneration)
|
||||
}
|
||||
for (const entry of usage.queued) {
|
||||
generations.add(entry.key.providerGeneration)
|
||||
}
|
||||
}
|
||||
for (const entry of this.pressure.values) {
|
||||
generations.add(entry.key.providerGeneration)
|
||||
}
|
||||
for (const generation of this.pressure.pausedProviderGenerations) {
|
||||
generations.add(generation)
|
||||
}
|
||||
for (const generation of generations) {
|
||||
this.closeGeneration(generation, 'ssh_model_admission_disposed')
|
||||
}
|
||||
}
|
||||
|
||||
getDebugSnapshot() {
|
||||
return sshPtyModelAdmissionSnapshot(
|
||||
this.globalSourceUnits,
|
||||
this.globalBytes,
|
||||
this.pressure,
|
||||
this.migratingPtys
|
||||
)
|
||||
}
|
||||
|
||||
private canReserve(key: SshPtyModelAdmissionKey, charge: AdmissionCharge): boolean {
|
||||
if (this.migratingPtys.has(admissionKeyId(key))) {
|
||||
return false
|
||||
}
|
||||
return canReserveAdmission({
|
||||
key,
|
||||
charge,
|
||||
limits: this.limits,
|
||||
usageByPty: this.usageByPty,
|
||||
closingGenerations: this.closingGenerations,
|
||||
globalSourceUnits: this.globalSourceUnits,
|
||||
globalBytes: this.globalBytes
|
||||
})
|
||||
}
|
||||
|
||||
private reserveAndQueue(entry: AdmissionEntry): void {
|
||||
const id = admissionKeyId(entry.key)
|
||||
let usage = this.usageByPty.get(id)
|
||||
if (!usage) {
|
||||
usage = { sourceUnits: 0, bytes: 0, queued: [], running: null }
|
||||
this.usageByPty.set(id, usage)
|
||||
}
|
||||
usage.sourceUnits += entry.charge.sourceUnits
|
||||
usage.bytes += entry.charge.bytes
|
||||
this.globalSourceUnits += entry.charge.sourceUnits
|
||||
this.globalBytes += entry.charge.bytes
|
||||
entry.state = 'queued'
|
||||
usage.queued.push(entry)
|
||||
this.startNext(id, usage)
|
||||
}
|
||||
|
||||
private startNext(id: string, usage: PtyUsage): void {
|
||||
if (usage.running) {
|
||||
return
|
||||
}
|
||||
const entry = usage.queued.shift()
|
||||
if (!entry) {
|
||||
this.promotePressureAndResumeProvider()
|
||||
return
|
||||
}
|
||||
usage.running = entry
|
||||
entry.state = 'running'
|
||||
let execution: { sequence: number; completion: Promise<void> }
|
||||
try {
|
||||
execution = entry.run()
|
||||
} catch (error) {
|
||||
if (this.finishEntry(id, usage, entry)) {
|
||||
entry.reject(error instanceof Error ? error : new Error(String(error)))
|
||||
}
|
||||
return
|
||||
}
|
||||
void execution.completion.then(
|
||||
() => {
|
||||
if (!this.finishEntry(id, usage, entry)) {
|
||||
return
|
||||
}
|
||||
entry.resolve({
|
||||
ptyId: entry.key.ptyId,
|
||||
providerGeneration: entry.key.providerGeneration,
|
||||
sequence: execution.sequence
|
||||
})
|
||||
},
|
||||
(error) => {
|
||||
this.failEntry(id, usage, entry, error instanceof Error ? error : new Error(String(error)))
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
private failEntry(id: string, usage: PtyUsage, entry: AdmissionEntry, error: Error): void {
|
||||
modelAdmissionMigration.settleSshPtyModelAdmissionFailure({
|
||||
id,
|
||||
usage,
|
||||
entry,
|
||||
error,
|
||||
migratingPtys: this.migratingPtys,
|
||||
closingGenerations: this.closingGenerations,
|
||||
release: (key, charge) => this.release(key, charge),
|
||||
closeGeneration: (providerGeneration) =>
|
||||
this.closeGeneration(providerGeneration, 'ssh_model_admission_completion_failed'),
|
||||
cleanup: (entryId, entryUsage) => this.cleanupUsage(entryId, entryUsage)
|
||||
})
|
||||
}
|
||||
|
||||
private finishEntry(id: string, usage: PtyUsage, entry: AdmissionEntry): boolean {
|
||||
if (entry.state !== 'running' || usage.running !== entry) {
|
||||
return false
|
||||
}
|
||||
usage.running = null
|
||||
entry.state = 'settled'
|
||||
this.release(entry.key, entry.charge)
|
||||
this.cleanupUsage(id, usage)
|
||||
return true
|
||||
}
|
||||
|
||||
private cleanupUsage(id: string, usage: PtyUsage): void {
|
||||
this.startNext(id, usage)
|
||||
if (!usage.running && usage.queued.length === 0 && usage.sourceUnits === 0) {
|
||||
this.usageByPty.delete(id)
|
||||
}
|
||||
resolveAdmissionIdleWaiters(this.usageByPty, this.pressure.values, this.idleWaiters, id)
|
||||
}
|
||||
|
||||
private release(key: SshPtyModelAdmissionKey, charge: AdmissionCharge): void {
|
||||
const usage = this.usageByPty.get(admissionKeyId(key))
|
||||
if (usage) {
|
||||
usage.sourceUnits = Math.max(0, usage.sourceUnits - charge.sourceUnits)
|
||||
usage.bytes = Math.max(0, usage.bytes - charge.bytes)
|
||||
}
|
||||
this.globalSourceUnits = Math.max(0, this.globalSourceUnits - charge.sourceUnits)
|
||||
this.globalBytes = Math.max(0, this.globalBytes - charge.bytes)
|
||||
this.promotePressureAndResumeProvider()
|
||||
}
|
||||
|
||||
private promotePressureAndResumeProvider(): void {
|
||||
this.pressure.promoteAndResume({
|
||||
usageByPty: this.usageByPty,
|
||||
disposed: this.disposed,
|
||||
canReserve: (entry) => this.canReserve(entry.key, entry.charge),
|
||||
reserve: (entry) => this.reserveAndQueue(entry),
|
||||
isBelowGlobalLowWatermark: () =>
|
||||
this.globalSourceUnits <= this.limits.globalLowSourceUnits &&
|
||||
this.globalBytes <= this.limits.globalLowBytes
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { RemoteTerminalSourceRangeStreamIdentity } from '../runtime/remote-terminal-source-range-consumer'
|
||||
import {
|
||||
SshPtyOutputIntake,
|
||||
type SshPtyOutputIntakeDependencies,
|
||||
type SshPtyOutputReceipt
|
||||
} from './ssh-pty-output-intake'
|
||||
import type { SshPtySourceCancellationProof } from './ssh-pty-output-intake-contract'
|
||||
|
||||
type ExitDeadlineHarness = Readonly<{
|
||||
intake: SshPtyOutputIntake
|
||||
dependencies: SshPtyOutputIntakeDependencies
|
||||
cancelSourceDelivery: ReturnType<typeof vi.fn>
|
||||
releaseExit: ReturnType<typeof vi.fn>
|
||||
exits: string[]
|
||||
}>
|
||||
|
||||
const stream: RemoteTerminalSourceRangeStreamIdentity = {
|
||||
ptyId: 'pty-1',
|
||||
consumerId: 'remote-1',
|
||||
streamGeneration: 'stream-1'
|
||||
}
|
||||
|
||||
function createHarness(
|
||||
cancelSourceDelivery: NonNullable<SshPtyOutputIntakeDependencies['cancelSourceDelivery']>
|
||||
): ExitDeadlineHarness {
|
||||
const exits: string[] = []
|
||||
const releaseExit = vi.fn()
|
||||
const cancellation = vi.fn(cancelSourceDelivery)
|
||||
const dependencies: SshPtyOutputIntakeDependencies = {
|
||||
getModelSequence: () => 0,
|
||||
acceptModel: (event) => ({ sequence: event.rawLength, completion: Promise.resolve() }),
|
||||
project: vi.fn(),
|
||||
prepareExit: vi.fn(() => releaseExit),
|
||||
finalizeExit: () => exits.push('exit'),
|
||||
pauseProvider: vi.fn(() => true),
|
||||
resumeProvider: vi.fn(),
|
||||
closeProvider: vi.fn(),
|
||||
cancelSourceDelivery: cancellation
|
||||
}
|
||||
return {
|
||||
intake: new SshPtyOutputIntake(dependencies, {
|
||||
exitBarrierMs: 10,
|
||||
exitCancellationProofMs: 100
|
||||
}),
|
||||
dependencies,
|
||||
cancelSourceDelivery: cancellation,
|
||||
releaseExit,
|
||||
exits
|
||||
}
|
||||
}
|
||||
|
||||
async function publishSource(harness: ExitDeadlineHarness): Promise<SshPtyOutputReceipt> {
|
||||
const remote = harness.intake.getRemoteSourceRangeConsumerHooks()
|
||||
expect(remote.attach(stream)).toBe(true)
|
||||
const receipt = await harness.intake.acceptData({
|
||||
id: 'pty-1',
|
||||
data: 'aaaa',
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
rawLength: 4,
|
||||
transformed: false,
|
||||
source: {
|
||||
spanId: 'span-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-1',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4
|
||||
}
|
||||
})
|
||||
harness.intake.publishProjectionPrefix([receipt.projection.identity.projectionSemanticsId], 4, 4)
|
||||
return receipt
|
||||
}
|
||||
|
||||
function acceptExit(harness: ExitDeadlineHarness) {
|
||||
return harness.intake.acceptExit({
|
||||
id: 'pty-1',
|
||||
code: 0,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
}
|
||||
|
||||
describe('SshPtyOutputExitDeadline', () => {
|
||||
it('transfers published projections before cancellation proof reclaims their spans', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const harness = createHarness(async () => ({ sentEndSu: 4, creditedEndSu: 0 }))
|
||||
const receipt = await publishSource(harness)
|
||||
const exitResult = acceptExit(harness).then(
|
||||
() => ({ ok: true as const }),
|
||||
(error: Error) => ({ ok: false as const, error })
|
||||
)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(10)
|
||||
|
||||
expect(await exitResult).toEqual({ ok: true })
|
||||
expect(harness.cancelSourceDelivery).toHaveBeenCalledOnce()
|
||||
expect(harness.dependencies.closeProvider).not.toHaveBeenCalled()
|
||||
expect(harness.dependencies.prepareExit).toHaveBeenCalledOnce()
|
||||
expect(harness.releaseExit).toHaveBeenCalledOnce()
|
||||
expect(harness.exits).toEqual(['exit'])
|
||||
expect(harness.intake.getDebugSnapshot()).toMatchObject({
|
||||
projection: { records: 0 },
|
||||
source: { openedTokens: 0, ptyIdentities: 0 },
|
||||
exitBarriers: 0
|
||||
})
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(harness.exits).toEqual(['exit'])
|
||||
expect(() =>
|
||||
harness.intake
|
||||
.getRemoteSourceRangeConsumerHooks()
|
||||
.settle(stream, [receipt.projection.desktopSpan!])
|
||||
).not.toThrow()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('generation close fences a pending cancellation proof from final exit', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
let resolveCancellation!: (proof: SshPtySourceCancellationProof) => void
|
||||
const cancellation = new Promise<SshPtySourceCancellationProof>((resolve) => {
|
||||
resolveCancellation = resolve
|
||||
})
|
||||
const harness = createHarness(() => cancellation)
|
||||
await publishSource(harness)
|
||||
const exitResult = acceptExit(harness).then(
|
||||
() => ({ ok: true as const }),
|
||||
(error: Error) => ({ ok: false as const, error })
|
||||
)
|
||||
await vi.advanceTimersByTimeAsync(10)
|
||||
expect(harness.cancelSourceDelivery).toHaveBeenCalledOnce()
|
||||
|
||||
harness.intake.closeGeneration(1, 'provider-closed')
|
||||
const closed = await exitResult
|
||||
expect(closed.ok).toBe(false)
|
||||
expect(closed.ok ? '' : closed.error.message).toContain('provider-closed')
|
||||
resolveCancellation({ sentEndSu: 4, creditedEndSu: 0 })
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
|
||||
expect(harness.exits).toEqual([])
|
||||
expect(harness.dependencies.closeProvider).not.toHaveBeenCalled()
|
||||
expect(harness.intake.getDebugSnapshot()).toMatchObject({
|
||||
projection: { records: 0 },
|
||||
source: { openedTokens: 0, ptyIdentities: 0 },
|
||||
exitBarriers: 0
|
||||
})
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,239 @@
|
||||
import type { SshPtyLegacyProjectionLedger } from './ssh-pty-legacy-projection'
|
||||
import type { SshPtyModelAdmission } from './ssh-pty-model-admission'
|
||||
import type {
|
||||
SshPtyOutputExitEvent,
|
||||
SshPtyOutputIntakeDependencies
|
||||
} from './ssh-pty-output-intake-contract'
|
||||
import { outputIntakeError, type SshPtyExitBarrier } from './ssh-pty-output-intake-validation'
|
||||
import type {
|
||||
SshPtyOutputSourceObligations,
|
||||
SshPtySourceCancellationProofCommit
|
||||
} from './ssh-pty-output-source-obligations'
|
||||
|
||||
type SshPtyOutputExitDeadlineDependencies = Readonly<{
|
||||
admission: SshPtyModelAdmission
|
||||
projections: SshPtyLegacyProjectionLedger
|
||||
sourceObligations: SshPtyOutputSourceObligations
|
||||
intake: SshPtyOutputIntakeDependencies
|
||||
barrierMs?: number
|
||||
cancellationProofMs?: number
|
||||
}>
|
||||
|
||||
export class SshPtyOutputExitDeadline {
|
||||
private readonly barriersByGeneration = new Map<number, Set<SshPtyExitBarrier>>()
|
||||
private readonly preparedExits = new Set<string>()
|
||||
private readonly preparedExitReleases = new Map<string, () => void>()
|
||||
private readonly barrierMs: number
|
||||
private readonly cancellationProofMs: number
|
||||
|
||||
constructor(private readonly dependencies: SshPtyOutputExitDeadlineDependencies) {
|
||||
this.barrierMs = dependencies.barrierMs ?? 30_000
|
||||
this.cancellationProofMs = dependencies.cancellationProofMs ?? 10_000
|
||||
}
|
||||
|
||||
wait(
|
||||
event: SshPtyOutputExitEvent,
|
||||
start: (validateNormalExit: () => void) => Promise<void>
|
||||
): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
let timeoutStarted = false
|
||||
let settled = false
|
||||
const validateNormalExit = (): void => {
|
||||
if (timeoutStarted) {
|
||||
throw outputIntakeError('ssh_exit_delivery_canceled')
|
||||
}
|
||||
}
|
||||
const settle = (result: { ok: true } | { ok: false; error: Error }): void => {
|
||||
if (settled) {
|
||||
return
|
||||
}
|
||||
settled = true
|
||||
clearTimeout(barrier.timer)
|
||||
this.releasePreparedExit(event)
|
||||
this.remove(event.providerGeneration, barrier)
|
||||
if (result.ok) {
|
||||
resolve()
|
||||
} else {
|
||||
reject(result.error)
|
||||
}
|
||||
}
|
||||
const barrier: SshPtyExitBarrier = {
|
||||
timer: setTimeout(() => {
|
||||
timeoutStarted = true
|
||||
void this.cancelTimedOutExit(event, barrier).then(
|
||||
() => settle({ ok: true }),
|
||||
(error) => {
|
||||
this.dependencies.intake.closeProvider?.(
|
||||
event.providerGeneration,
|
||||
'ssh-exit-cancellation-proof-failed'
|
||||
)
|
||||
settle({
|
||||
ok: false,
|
||||
error: error instanceof Error ? error : outputIntakeError(String(error))
|
||||
})
|
||||
}
|
||||
)
|
||||
}, this.barrierMs),
|
||||
reject: (error) => settle({ ok: false, error })
|
||||
}
|
||||
barrier.timer.unref?.()
|
||||
let barriers = this.barriersByGeneration.get(event.providerGeneration)
|
||||
if (!barriers) {
|
||||
barriers = new Set()
|
||||
this.barriersByGeneration.set(event.providerGeneration, barriers)
|
||||
}
|
||||
barriers.add(barrier)
|
||||
const promise = start(validateNormalExit)
|
||||
void promise.then(
|
||||
() => {
|
||||
if (!timeoutStarted) {
|
||||
settle({ ok: true })
|
||||
}
|
||||
},
|
||||
(error) => {
|
||||
if (!timeoutStarted) {
|
||||
settle({ ok: false, error })
|
||||
}
|
||||
}
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
closeGeneration(providerGeneration: number, error: Error): void {
|
||||
const barriers = this.barriersByGeneration.get(providerGeneration)
|
||||
if (barriers) {
|
||||
for (const barrier of barriers) {
|
||||
clearTimeout(barrier.timer)
|
||||
barrier.reject(error)
|
||||
}
|
||||
this.barriersByGeneration.delete(providerGeneration)
|
||||
}
|
||||
const prefix = `${providerGeneration}\0`
|
||||
for (const key of this.preparedExits) {
|
||||
if (key.startsWith(prefix)) {
|
||||
this.releasePreparedExitKey(key)
|
||||
this.preparedExits.delete(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
prepareExitOnce(event: SshPtyOutputExitEvent): void {
|
||||
const key = this.exitKey(event)
|
||||
if (this.preparedExits.has(key)) {
|
||||
return
|
||||
}
|
||||
const release = this.dependencies.intake.prepareExit(event)
|
||||
this.preparedExits.add(key)
|
||||
if (release) {
|
||||
this.preparedExitReleases.set(key, release)
|
||||
}
|
||||
}
|
||||
|
||||
get activeBarriers(): number {
|
||||
return Array.from(this.barriersByGeneration.values()).reduce(
|
||||
(total, barriers) => total + barriers.size,
|
||||
0
|
||||
)
|
||||
}
|
||||
|
||||
private async cancelTimedOutExit(
|
||||
event: SshPtyOutputExitEvent,
|
||||
barrier: SshPtyExitBarrier
|
||||
): Promise<void> {
|
||||
const cancel = this.dependencies.intake.cancelSourceDelivery
|
||||
if (!cancel) {
|
||||
throw outputIntakeError('ssh_source_cancellation_publisher_unavailable')
|
||||
}
|
||||
this.dependencies.admission.cancelPty(
|
||||
{ ptyId: event.id, providerGeneration: event.providerGeneration },
|
||||
'ssh_exit_delivery_canceled'
|
||||
)
|
||||
this.dependencies.sourceObligations.sealPty(event)
|
||||
const cancellation = this.dependencies.sourceObligations.requestPtyCancellationProof(
|
||||
event,
|
||||
(request) => cancel(event.providerGeneration, request)
|
||||
)
|
||||
let commit: SshPtySourceCancellationProofCommit | null
|
||||
try {
|
||||
commit = await this.withCancellationProofDeadline(cancellation)
|
||||
} catch (error) {
|
||||
if (!this.isActive(event.providerGeneration, barrier)) {
|
||||
return
|
||||
}
|
||||
throw error
|
||||
}
|
||||
if (!this.isActive(event.providerGeneration, barrier)) {
|
||||
return
|
||||
}
|
||||
if (!commit) {
|
||||
throw outputIntakeError('ssh_source_cancellation_identity_unavailable')
|
||||
}
|
||||
this.dependencies.projections.transferPty(event.id, 'ssh-exit-delivery-canceled')
|
||||
this.dependencies.sourceObligations.commitPtyCancellationProof(commit)
|
||||
this.prepareExitOnce(event)
|
||||
if (!this.isActive(event.providerGeneration, barrier)) {
|
||||
return
|
||||
}
|
||||
this.dependencies.intake.finalizeExit(event)
|
||||
if (!this.isActive(event.providerGeneration, barrier)) {
|
||||
return
|
||||
}
|
||||
this.dependencies.projections.closePty(
|
||||
event.id,
|
||||
event.providerGeneration,
|
||||
event.ptyIncarnation,
|
||||
'ssh-exit-delivery-canceled'
|
||||
)
|
||||
}
|
||||
|
||||
private isActive(providerGeneration: number, barrier: SshPtyExitBarrier): boolean {
|
||||
return this.barriersByGeneration.get(providerGeneration)?.has(barrier) ?? false
|
||||
}
|
||||
|
||||
private exitKey(event: SshPtyOutputExitEvent): string {
|
||||
return `${event.providerGeneration}\0${event.id}\0${event.ptyIncarnation}`
|
||||
}
|
||||
|
||||
private releasePreparedExit(event: SshPtyOutputExitEvent): void {
|
||||
const key = this.exitKey(event)
|
||||
this.preparedExits.delete(key)
|
||||
this.releasePreparedExitKey(key)
|
||||
}
|
||||
|
||||
private releasePreparedExitKey(key: string): void {
|
||||
const release = this.preparedExitReleases.get(key)
|
||||
if (!release) {
|
||||
return
|
||||
}
|
||||
this.preparedExitReleases.delete(key)
|
||||
release()
|
||||
}
|
||||
|
||||
private withCancellationProofDeadline<T>(promise: Promise<T>): Promise<T> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(
|
||||
() => reject(outputIntakeError('ssh_source_cancellation_proof_timeout')),
|
||||
this.cancellationProofMs
|
||||
)
|
||||
timer.unref?.()
|
||||
void promise.then(
|
||||
(value) => {
|
||||
clearTimeout(timer)
|
||||
resolve(value)
|
||||
},
|
||||
(error) => {
|
||||
clearTimeout(timer)
|
||||
reject(error)
|
||||
}
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
private remove(providerGeneration: number, barrier: SshPtyExitBarrier): void {
|
||||
const barriers = this.barriersByGeneration.get(providerGeneration)
|
||||
barriers?.delete(barrier)
|
||||
if (barriers?.size === 0) {
|
||||
this.barriersByGeneration.delete(providerGeneration)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
import type { SshPtyLegacyProjectionLedger } from './ssh-pty-legacy-projection'
|
||||
import type { SshPtyModelAdmission } from './ssh-pty-model-admission'
|
||||
import type {
|
||||
SshPtyOutputExitEvent,
|
||||
SshPtyOutputIntakeDependencies
|
||||
} from './ssh-pty-output-intake-contract'
|
||||
|
||||
export async function settleSshPtyOutputExit(args: {
|
||||
event: SshPtyOutputExitEvent
|
||||
admission: SshPtyModelAdmission
|
||||
projections: SshPtyLegacyProjectionLedger
|
||||
dependencies: SshPtyOutputIntakeDependencies
|
||||
validateGeneration: () => void
|
||||
prepareExit?: () => void
|
||||
afterAdmissionIdle?: () => void
|
||||
waitForSourceTerminal?: () => Promise<void>
|
||||
beforeFinalize?: () => void
|
||||
}): Promise<void> {
|
||||
const {
|
||||
event,
|
||||
admission,
|
||||
projections,
|
||||
dependencies,
|
||||
validateGeneration,
|
||||
prepareExit,
|
||||
afterAdmissionIdle,
|
||||
waitForSourceTerminal,
|
||||
beforeFinalize
|
||||
} = args
|
||||
await admission.whenIdle({
|
||||
ptyId: event.id,
|
||||
providerGeneration: event.providerGeneration
|
||||
})
|
||||
validateGeneration()
|
||||
afterAdmissionIdle?.()
|
||||
try {
|
||||
if (prepareExit) {
|
||||
prepareExit()
|
||||
} else {
|
||||
dependencies.prepareExit(event)
|
||||
}
|
||||
} catch (error) {
|
||||
projections.closePty(
|
||||
event.id,
|
||||
event.providerGeneration,
|
||||
event.ptyIncarnation,
|
||||
'pty-exit-finalize-failed'
|
||||
)
|
||||
dependencies.closeProvider?.(event.providerGeneration, 'pty-exit-finalize-failed')
|
||||
throw error
|
||||
}
|
||||
projections.transferUnpublishedPty(
|
||||
event.id,
|
||||
event.providerGeneration,
|
||||
event.ptyIncarnation,
|
||||
'pty-exit-unpublished'
|
||||
)
|
||||
await projections.whenPtyTerminal(event.id, event.providerGeneration, event.ptyIncarnation)
|
||||
await waitForSourceTerminal?.()
|
||||
validateGeneration()
|
||||
try {
|
||||
beforeFinalize?.()
|
||||
dependencies.finalizeExit(event)
|
||||
projections.closePty(event.id, event.providerGeneration, event.ptyIncarnation, 'pty-exit')
|
||||
} catch (error) {
|
||||
projections.closePty(
|
||||
event.id,
|
||||
event.providerGeneration,
|
||||
event.ptyIncarnation,
|
||||
'pty-exit-finalize-failed'
|
||||
)
|
||||
dependencies.closeProvider?.(event.providerGeneration, 'pty-exit-finalize-failed')
|
||||
throw error
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { SshPtyOutputGenerationGuard } from './ssh-pty-output-generation-guard'
|
||||
|
||||
const event = (providerGeneration: number) => ({
|
||||
id: 'pty-1',
|
||||
providerGeneration,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
|
||||
describe('SshPtyOutputGenerationGuard', () => {
|
||||
it('compacts sequential closures without weakening stale rejection', () => {
|
||||
const guard = new SshPtyOutputGenerationGuard(() => false)
|
||||
for (let generation = 1; generation <= 2_048; generation++) {
|
||||
guard.closeGeneration(generation)
|
||||
}
|
||||
|
||||
expect(guard.getDebugSnapshot()).toEqual({
|
||||
closedRanges: 1,
|
||||
activeGaps: 0,
|
||||
activePtys: 0,
|
||||
sealedPtys: 0
|
||||
})
|
||||
expect(() => guard.validate(event(1_024))).toThrow('ssh_output_stale_generation')
|
||||
expect(() => guard.validate(event(2_049))).not.toThrow()
|
||||
})
|
||||
|
||||
it('counts and merges out-of-order live generation gaps exactly', () => {
|
||||
const guard = new SshPtyOutputGenerationGuard(() => false)
|
||||
guard.closeGeneration(4)
|
||||
guard.closeGeneration(2)
|
||||
|
||||
expect(guard.getDebugSnapshot()).toMatchObject({ closedRanges: 2, activeGaps: 2 })
|
||||
guard.closeGeneration(3)
|
||||
expect(guard.getDebugSnapshot()).toMatchObject({ closedRanges: 1, activeGaps: 1 })
|
||||
expect(() => guard.validate(event(1))).not.toThrow()
|
||||
expect(() => guard.validate(event(2))).toThrow('ssh_output_stale_generation')
|
||||
|
||||
guard.closeGeneration(1)
|
||||
expect(guard.getDebugSnapshot()).toEqual({
|
||||
closedRanges: 1,
|
||||
activeGaps: 0,
|
||||
activePtys: 0,
|
||||
sealedPtys: 0
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,107 @@
|
||||
import type { SshPtyOutputDataEvent, SshPtyOutputExitEvent } from './ssh-pty-output-intake-contract'
|
||||
import {
|
||||
outputIntakeError,
|
||||
sshPtyGenerationKey,
|
||||
validOutputLength
|
||||
} from './ssh-pty-output-intake-validation'
|
||||
import { SshPtyClosedGenerationRanges } from './ssh-pty-closed-generation-ranges'
|
||||
|
||||
export class SshPtyOutputGenerationGuard {
|
||||
private readonly latestGenerationByPty = new Map<string, number>()
|
||||
private readonly incarnationByPty = new Map<string, string>()
|
||||
private readonly sealedPtys = new Set<string>()
|
||||
private readonly closedGenerations = new SshPtyClosedGenerationRanges()
|
||||
|
||||
constructor(private readonly isDisposed: () => boolean) {}
|
||||
|
||||
validateData(event: SshPtyOutputDataEvent): void {
|
||||
if (
|
||||
!event.id ||
|
||||
!event.ptyIncarnation ||
|
||||
!Number.isSafeInteger(event.providerGeneration) ||
|
||||
event.providerGeneration <= 0 ||
|
||||
!validOutputLength(event.rawLength) ||
|
||||
(event.source !== undefined &&
|
||||
(!event.source.spanId ||
|
||||
!event.source.deliveryToken ||
|
||||
!Number.isSafeInteger(event.source.clientGeneration) ||
|
||||
event.source.clientGeneration <= 0 ||
|
||||
!Number.isSafeInteger(event.source.ownerGeneration) ||
|
||||
event.source.ownerGeneration <= 0 ||
|
||||
!Number.isSafeInteger(event.source.sourceStartSu) ||
|
||||
event.source.sourceStartSu < 0 ||
|
||||
!Number.isSafeInteger(event.source.sourceEndSu) ||
|
||||
event.source.sourceEndSu - event.source.sourceStartSu !== event.rawLength))
|
||||
) {
|
||||
throw outputIntakeError('ssh_output_invalid_event')
|
||||
}
|
||||
this.validate(event)
|
||||
if (this.sealedPtys.has(sshPtyGenerationKey(event.id, event.providerGeneration))) {
|
||||
throw outputIntakeError('ssh_output_after_exit')
|
||||
}
|
||||
}
|
||||
|
||||
sealExit(event: SshPtyOutputExitEvent): void {
|
||||
this.validate(event)
|
||||
const key = sshPtyGenerationKey(event.id, event.providerGeneration)
|
||||
if (this.sealedPtys.has(key)) {
|
||||
throw outputIntakeError('ssh_output_duplicate_exit')
|
||||
}
|
||||
this.sealedPtys.add(key)
|
||||
}
|
||||
|
||||
validate(event: { id: string; providerGeneration: number; ptyIncarnation: string }): void {
|
||||
if (this.isDisposed()) {
|
||||
throw outputIntakeError('ssh_output_intake_disposed')
|
||||
}
|
||||
if (this.closedGenerations.has(event.providerGeneration)) {
|
||||
throw outputIntakeError('ssh_output_stale_generation')
|
||||
}
|
||||
const generation = this.latestGenerationByPty.get(event.id)
|
||||
if (generation !== undefined && event.providerGeneration < generation) {
|
||||
throw outputIntakeError('ssh_output_stale_generation')
|
||||
}
|
||||
const incarnation = this.incarnationByPty.get(event.id)
|
||||
if (
|
||||
generation === event.providerGeneration &&
|
||||
incarnation !== undefined &&
|
||||
incarnation !== event.ptyIncarnation
|
||||
) {
|
||||
throw outputIntakeError('ssh_output_stale_incarnation')
|
||||
}
|
||||
if (generation === undefined || event.providerGeneration > generation) {
|
||||
this.latestGenerationByPty.set(event.id, event.providerGeneration)
|
||||
this.incarnationByPty.set(event.id, event.ptyIncarnation)
|
||||
}
|
||||
}
|
||||
|
||||
closeGeneration(providerGeneration: number): void {
|
||||
this.closedGenerations.add(providerGeneration)
|
||||
for (const [ptyId, generation] of this.latestGenerationByPty) {
|
||||
if (generation === providerGeneration) {
|
||||
this.latestGenerationByPty.delete(ptyId)
|
||||
this.incarnationByPty.delete(ptyId)
|
||||
this.sealedPtys.delete(sshPtyGenerationKey(ptyId, generation))
|
||||
}
|
||||
}
|
||||
const prefix = `${providerGeneration}\0`
|
||||
for (const key of this.sealedPtys) {
|
||||
if (key.startsWith(prefix)) {
|
||||
this.sealedPtys.delete(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
activeGenerations(): ReadonlySet<number> {
|
||||
return new Set(this.latestGenerationByPty.values())
|
||||
}
|
||||
|
||||
getDebugSnapshot() {
|
||||
return {
|
||||
closedRanges: this.closedGenerations.size,
|
||||
activeGaps: this.closedGenerations.activeGaps,
|
||||
activePtys: this.latestGenerationByPty.size,
|
||||
sealedPtys: this.sealedPtys.size
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
import type { LegacySshProjectionSemantics } from './ssh-pty-legacy-projection'
|
||||
import type {
|
||||
SshPtyModelAdmissionOptions,
|
||||
SshPtyModelAdmissionReceipt
|
||||
} from './ssh-pty-model-admission-contract'
|
||||
import type { PtySourceCreditAckBatch } from '../../shared/pty-source-credit-contract'
|
||||
|
||||
export type SshPtySourceCancellationRequest = Readonly<{
|
||||
id: string
|
||||
clientGeneration: number
|
||||
ownerGeneration: number
|
||||
deliveryToken: string
|
||||
}>
|
||||
|
||||
export type SshPtySourceCancellationProof = Readonly<{
|
||||
sentEndSu: number
|
||||
creditedEndSu: number
|
||||
}>
|
||||
|
||||
export type SshPtyOutputDataEvent = Readonly<{
|
||||
id: string
|
||||
data: string
|
||||
providerGeneration: number
|
||||
ptyIncarnation: string
|
||||
rawLength: number
|
||||
transformed: boolean
|
||||
sequence?: number
|
||||
source?: Readonly<{
|
||||
relayPtyId?: string
|
||||
spanId: string
|
||||
clientGeneration: number
|
||||
ownerGeneration: number
|
||||
deliveryToken: string
|
||||
sourceStartSu: number
|
||||
sourceEndSu: number
|
||||
}>
|
||||
}>
|
||||
|
||||
export type SshPtyOutputExitEvent = Readonly<{
|
||||
id: string
|
||||
code: number
|
||||
providerGeneration: number
|
||||
ptyIncarnation: string
|
||||
}>
|
||||
|
||||
export type SshPtyOutputReceipt = SshPtyModelAdmissionReceipt &
|
||||
Readonly<{ projection: LegacySshProjectionSemantics }>
|
||||
|
||||
export type SshPtyOutputIntakeDependencies = {
|
||||
getModelSequence: (id: string) => number
|
||||
acceptModel: (
|
||||
event: SshPtyOutputDataEvent,
|
||||
projection: LegacySshProjectionSemantics
|
||||
) => { sequence: number; completion: Promise<void> }
|
||||
project: (event: SshPtyOutputDataEvent, projection: LegacySshProjectionSemantics) => void
|
||||
prepareExit: (event: SshPtyOutputExitEvent) => void | (() => void)
|
||||
finalizeExit: (event: SshPtyOutputExitEvent) => void
|
||||
pauseProvider?: (providerGeneration: number, id: string) => boolean
|
||||
resumeProvider?: (providerGeneration: number, id: string) => void
|
||||
closeProvider?: (providerGeneration: number, reason: string) => void
|
||||
resetModelForMigration?: (providerGeneration: number, id: string) => void
|
||||
onGenerationClosed?: (providerGeneration: number, reason: string) => void
|
||||
publishSourceAck?: (
|
||||
providerGeneration: number,
|
||||
batch: PtySourceCreditAckBatch,
|
||||
onSettled: (result: { ok: true } | { ok: false; error: Error }) => void
|
||||
) => void
|
||||
cancelSourceDelivery?: (
|
||||
providerGeneration: number,
|
||||
request: SshPtySourceCancellationRequest
|
||||
) => Promise<SshPtySourceCancellationProof>
|
||||
}
|
||||
|
||||
export type SshPtyOutputIntakeOptions = SshPtyModelAdmissionOptions & {
|
||||
exitBarrierMs?: number
|
||||
exitCancellationProofMs?: number
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
import type { SshPtyOutputIntake } from './ssh-pty-output-intake'
|
||||
import type {
|
||||
SshPtyOutputDataEvent,
|
||||
SshPtyOutputExitEvent,
|
||||
SshPtyOutputReceipt,
|
||||
SshPtySourceCancellationProof,
|
||||
SshPtySourceCancellationRequest
|
||||
} from './ssh-pty-output-intake-contract'
|
||||
import type { SshPtyAcceptedSourceCheckpoint } from './ssh-pty-output-source-obligations'
|
||||
import type { PtySourceCreditAckBatch } from '../../shared/pty-source-credit-contract'
|
||||
import type { SshPtyOutputGenerationMigration } from './ssh-pty-output-model-migration'
|
||||
|
||||
let installedIntake: SshPtyOutputIntake | null = null
|
||||
let nextProviderGeneration = 1
|
||||
const sourceAckPublishers = new Map<number, SshPtySourceAckPublisher>()
|
||||
const sourceCancellationPublishers = new Map<number, SshPtySourceCancellationPublisher>()
|
||||
|
||||
type SshPtySourceAckPublisher = (
|
||||
batch: PtySourceCreditAckBatch,
|
||||
onSettled: (result: { ok: true } | { ok: false; error: Error }) => void
|
||||
) => void
|
||||
|
||||
type SshPtySourceCancellationPublisher = (
|
||||
request: SshPtySourceCancellationRequest
|
||||
) => Promise<SshPtySourceCancellationProof>
|
||||
|
||||
export function allocateSshPtyProviderGeneration(): number {
|
||||
return nextProviderGeneration++
|
||||
}
|
||||
|
||||
export function installSshPtyOutputIntake(intake: SshPtyOutputIntake): () => void {
|
||||
const previous = installedIntake
|
||||
installedIntake = intake
|
||||
previous?.dispose()
|
||||
return () => {
|
||||
if (installedIntake === intake) {
|
||||
installedIntake = null
|
||||
intake.dispose()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function acceptSshPtyOutputData(event: SshPtyOutputDataEvent): Promise<SshPtyOutputReceipt> {
|
||||
return installedIntake
|
||||
? installedIntake.acceptData(event)
|
||||
: Promise.reject(outputIntakeUnavailableError())
|
||||
}
|
||||
|
||||
export function acceptSshPtyOutputExit(event: SshPtyOutputExitEvent): Promise<void> {
|
||||
return installedIntake
|
||||
? installedIntake.acceptExit(event)
|
||||
: Promise.reject(outputIntakeUnavailableError())
|
||||
}
|
||||
|
||||
export function closeSshPtyOutputGeneration(providerGeneration: number, reason: string): void {
|
||||
installedIntake?.closeGeneration(providerGeneration, reason)
|
||||
}
|
||||
|
||||
export function getSshPtyAcceptedSourceCheckpoints(
|
||||
providerGeneration: number
|
||||
): readonly SshPtyAcceptedSourceCheckpoint[] {
|
||||
return installedIntake?.getAcceptedSourceCheckpoints(providerGeneration) ?? []
|
||||
}
|
||||
|
||||
export function beginSshPtyOutputGenerationMigration(
|
||||
providerGeneration: number
|
||||
): SshPtyOutputGenerationMigration {
|
||||
return (
|
||||
installedIntake?.beginGenerationMigration(providerGeneration) ?? {
|
||||
byPty: new Map(),
|
||||
completion: Promise.resolve()
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
export function installSshPtySourceAckPublisher(
|
||||
providerGeneration: number,
|
||||
publish: SshPtySourceAckPublisher
|
||||
): () => void {
|
||||
if (sourceAckPublishers.has(providerGeneration)) {
|
||||
throw new Error('ssh_source_ack_publisher_duplicate_generation')
|
||||
}
|
||||
sourceAckPublishers.set(providerGeneration, publish)
|
||||
return () => {
|
||||
if (sourceAckPublishers.get(providerGeneration) === publish) {
|
||||
sourceAckPublishers.delete(providerGeneration)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function publishSshPtySourceAck(
|
||||
providerGeneration: number,
|
||||
batch: PtySourceCreditAckBatch,
|
||||
onSettled: (result: { ok: true } | { ok: false; error: Error }) => void
|
||||
): void {
|
||||
const publisher = sourceAckPublishers.get(providerGeneration)
|
||||
if (!publisher) {
|
||||
onSettled({ ok: false, error: new Error('ssh_source_ack_publisher_unavailable') })
|
||||
return
|
||||
}
|
||||
publisher(batch, onSettled)
|
||||
}
|
||||
|
||||
export function installSshPtySourceCancellationPublisher(
|
||||
providerGeneration: number,
|
||||
cancel: SshPtySourceCancellationPublisher
|
||||
): () => void {
|
||||
if (sourceCancellationPublishers.has(providerGeneration)) {
|
||||
throw new Error('ssh_source_cancellation_publisher_duplicate_generation')
|
||||
}
|
||||
sourceCancellationPublishers.set(providerGeneration, cancel)
|
||||
return () => {
|
||||
if (sourceCancellationPublishers.get(providerGeneration) === cancel) {
|
||||
sourceCancellationPublishers.delete(providerGeneration)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function cancelSshPtySourceDelivery(
|
||||
providerGeneration: number,
|
||||
request: SshPtySourceCancellationRequest
|
||||
): Promise<SshPtySourceCancellationProof> {
|
||||
const publisher = sourceCancellationPublishers.get(providerGeneration)
|
||||
return publisher
|
||||
? publisher(request)
|
||||
: Promise.reject(new Error('ssh_source_cancellation_publisher_unavailable'))
|
||||
}
|
||||
|
||||
export function applySshPtySourceCancellationProof(
|
||||
event: SshPtyOutputExitEvent,
|
||||
proof: SshPtySourceCancellationProof
|
||||
): boolean {
|
||||
return installedIntake?.applySourceCancellationProof(event, proof) ?? false
|
||||
}
|
||||
|
||||
export function applySshPtySourceRecoveryCancellationProof(
|
||||
event: SshPtyOutputExitEvent,
|
||||
proof: SshPtySourceCancellationProof
|
||||
): boolean {
|
||||
if (!installedIntake) {
|
||||
return false
|
||||
}
|
||||
installedIntake.applySourceRecoveryCancellationProof(event, proof)
|
||||
return true
|
||||
}
|
||||
|
||||
function outputIntakeUnavailableError(): Error {
|
||||
return Object.assign(new Error('ssh_output_intake_unavailable'), {
|
||||
code: 'ssh_output_intake_unavailable'
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
import { vi } from 'vitest'
|
||||
import {
|
||||
SshPtyOutputIntake,
|
||||
type SshPtyOutputDataEvent,
|
||||
type SshPtyOutputIntakeDependencies
|
||||
} from './ssh-pty-output-intake'
|
||||
|
||||
export function sshPtyOutputEvent(
|
||||
overrides: Partial<SshPtyOutputDataEvent> = {}
|
||||
): SshPtyOutputDataEvent {
|
||||
return {
|
||||
id: 'pty-1',
|
||||
data: 'aaaa',
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
rawLength: 4,
|
||||
transformed: false,
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
export function createSshPtyOutputIntakeHarness(
|
||||
overrides: Partial<SshPtyOutputIntakeDependencies> = {},
|
||||
options: ConstructorParameters<typeof SshPtyOutputIntake>[1] = {}
|
||||
) {
|
||||
let sequence = 0
|
||||
const completions: ReturnType<typeof deferred>[] = []
|
||||
const order: string[] = []
|
||||
const dependencies: SshPtyOutputIntakeDependencies = {
|
||||
getModelSequence: () => sequence,
|
||||
acceptModel: (input) => {
|
||||
order.push(`model:${input.data}`)
|
||||
sequence += input.rawLength
|
||||
const completion = deferred()
|
||||
completions.push(completion)
|
||||
return { sequence, completion: completion.promise }
|
||||
},
|
||||
project: (input) => order.push(`project:${input.data}`),
|
||||
prepareExit: vi.fn(),
|
||||
finalizeExit: () => order.push('exit'),
|
||||
pauseProvider: vi.fn(() => true),
|
||||
resumeProvider: vi.fn(),
|
||||
closeProvider: vi.fn(),
|
||||
...overrides
|
||||
}
|
||||
return {
|
||||
intake: new SshPtyOutputIntake(dependencies, options),
|
||||
dependencies,
|
||||
completions,
|
||||
order
|
||||
}
|
||||
}
|
||||
|
||||
function deferred() {
|
||||
let resolve!: () => void
|
||||
let reject!: (error: Error) => void
|
||||
const promise = new Promise<void>((promiseResolve, promiseReject) => {
|
||||
resolve = promiseResolve
|
||||
reject = promiseReject
|
||||
})
|
||||
return { promise, resolve, reject }
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
export type SshPtyExitBarrier = {
|
||||
timer: ReturnType<typeof setTimeout>
|
||||
reject: (error: Error) => void
|
||||
}
|
||||
|
||||
export function outputIntakeError(code: string): Error {
|
||||
return Object.assign(new Error(code), { code })
|
||||
}
|
||||
|
||||
export function validOutputLength(value: number): boolean {
|
||||
return Number.isSafeInteger(value) && value >= 0
|
||||
}
|
||||
|
||||
export function sshPtyGenerationKey(ptyId: string, providerGeneration: number): string {
|
||||
return `${providerGeneration}\0${ptyId}`
|
||||
}
|
||||
@@ -0,0 +1,760 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { LegacySshProjectionSemantics } from './ssh-pty-legacy-projection'
|
||||
import {
|
||||
createSshPtyOutputIntakeHarness as createHarness,
|
||||
sshPtyOutputEvent as event
|
||||
} from './ssh-pty-output-intake-test-harness'
|
||||
|
||||
describe('SshPtyOutputIntake', () => {
|
||||
it('plateaus at the model and pressure budgets, then resumes below low water', async () => {
|
||||
const harness = createHarness(
|
||||
{},
|
||||
{
|
||||
perPtyHighSourceUnits: 4,
|
||||
perPtyHighBytes: 1024,
|
||||
perPtyLowSourceUnits: 1,
|
||||
perPtyLowBytes: 256,
|
||||
globalHighSourceUnits: 4,
|
||||
globalHighBytes: 1024,
|
||||
globalLowSourceUnits: 1,
|
||||
globalLowBytes: 256,
|
||||
pressureMaxFrames: 2,
|
||||
pressureMaxBytes: 1024
|
||||
}
|
||||
)
|
||||
const first = harness.intake.acceptData(event())
|
||||
const second = harness.intake.acceptData(event({ data: 'bbbb' }))
|
||||
const third = harness.intake.acceptData(event({ data: 'cccc' }))
|
||||
const rejected = harness.intake.acceptData(event({ data: 'dddd' }))
|
||||
|
||||
expect(harness.intake.getDebugSnapshot().model).toMatchObject({
|
||||
sourceUnits: 4,
|
||||
pressureFrames: 2
|
||||
})
|
||||
await expect(rejected).rejects.toThrow('ssh_model_admission_pressure_exhausted')
|
||||
expect(harness.dependencies.closeProvider).toHaveBeenCalledWith(1, expect.any(String))
|
||||
|
||||
harness.completions[0]!.resolve()
|
||||
await first
|
||||
harness.completions[1]!.resolve()
|
||||
await second
|
||||
harness.completions[2]!.resolve()
|
||||
await third
|
||||
|
||||
expect(harness.intake.getDebugSnapshot().model).toMatchObject({
|
||||
sourceUnits: 0,
|
||||
bytes: 0,
|
||||
pressureFrames: 0
|
||||
})
|
||||
expect(harness.dependencies.resumeProvider).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('preserves per-PTY FIFO while differently sized pressure entries wait', async () => {
|
||||
const harness = createHarness(
|
||||
{},
|
||||
{
|
||||
perPtyHighSourceUnits: 4,
|
||||
perPtyHighBytes: 4096,
|
||||
globalHighSourceUnits: 4,
|
||||
globalHighBytes: 4096,
|
||||
pressureMaxFrames: 4,
|
||||
pressureMaxBytes: 4096
|
||||
}
|
||||
)
|
||||
const receipts = [
|
||||
harness.intake.acceptData(event({ data: 'a', rawLength: 1 })),
|
||||
harness.intake.acceptData(event({ data: 'bbb', rawLength: 3 })),
|
||||
harness.intake.acceptData(event({ data: 'cc', rawLength: 2 })),
|
||||
harness.intake.acceptData(event({ data: 'd', rawLength: 1 }))
|
||||
]
|
||||
|
||||
harness.completions[0]!.resolve()
|
||||
await receipts[0]
|
||||
harness.completions[1]!.resolve()
|
||||
await receipts[1]
|
||||
harness.completions[2]!.resolve()
|
||||
await receipts[2]
|
||||
harness.completions[3]!.resolve()
|
||||
await receipts[3]
|
||||
|
||||
expect(harness.order).toEqual([
|
||||
'model:a',
|
||||
'project:a',
|
||||
'model:bbb',
|
||||
'project:bbb',
|
||||
'model:cc',
|
||||
'project:cc',
|
||||
'model:d',
|
||||
'project:d'
|
||||
])
|
||||
})
|
||||
|
||||
it('assigns projection sequence ends when queued model capture begins', async () => {
|
||||
const harness = createHarness(
|
||||
{},
|
||||
{
|
||||
perPtyHighSourceUnits: 12,
|
||||
perPtyHighBytes: 4096,
|
||||
globalHighSourceUnits: 12,
|
||||
globalHighBytes: 4096
|
||||
}
|
||||
)
|
||||
const first = harness.intake.acceptData(event({ data: 'aaaa' }))
|
||||
const second = harness.intake.acceptData(event({ data: 'bbbb' }))
|
||||
const third = harness.intake.acceptData(event({ data: 'cccc' }))
|
||||
|
||||
harness.completions[0]!.resolve()
|
||||
expect((await first).projection.identity.sequenceEnd).toBe(4)
|
||||
harness.completions[1]!.resolve()
|
||||
expect((await second).projection.identity.sequenceEnd).toBe(8)
|
||||
harness.completions[2]!.resolve()
|
||||
expect((await third).projection.identity.sequenceEnd).toBe(12)
|
||||
})
|
||||
|
||||
it('transfers projection state and closes the provider on model failure', async () => {
|
||||
const harness = createHarness()
|
||||
const receipt = harness.intake.acceptData(event())
|
||||
harness.completions[0]!.reject(new Error('emulator failed'))
|
||||
|
||||
await expect(receipt).rejects.toThrow('emulator failed')
|
||||
expect(harness.intake.getDebugSnapshot().projection.transferred).toBe(1)
|
||||
expect(harness.dependencies.closeProvider).toHaveBeenCalledWith(1, 'model-admission-failed')
|
||||
})
|
||||
|
||||
it('transfers a committed projection when desktop admission throws', async () => {
|
||||
const projections: LegacySshProjectionSemantics[] = []
|
||||
const harness = createHarness({
|
||||
project: (_event, projection) => {
|
||||
projections.push(projection)
|
||||
if (projections.length === 1) {
|
||||
throw new Error('send failed')
|
||||
}
|
||||
}
|
||||
})
|
||||
const receipt = harness.intake.acceptData(event({ data: '\x1b[?20', rawLength: 5 }))
|
||||
harness.completions[0]!.resolve()
|
||||
|
||||
await expect(receipt).resolves.toMatchObject({ sequence: 5 })
|
||||
expect(harness.intake.getDebugSnapshot().projection.transferred).toBe(1)
|
||||
|
||||
const next = harness.intake.acceptData(event({ data: '31h', rawLength: 3 }))
|
||||
harness.completions[1]!.resolve()
|
||||
await next
|
||||
expect(projections[1]).toMatchObject({
|
||||
identity: { displayStart: 5 },
|
||||
beforeScanner: { tail: '\x1b[?20', pendingSubscribe: false },
|
||||
decision: 'subscribed'
|
||||
})
|
||||
})
|
||||
|
||||
it('commits an immutable desktop source identity through projection admission', async () => {
|
||||
const projections: LegacySshProjectionSemantics[] = []
|
||||
const harness = createHarness({
|
||||
project: (_event, projection) => projections.push(projection)
|
||||
})
|
||||
const receipt = harness.intake.acceptData(
|
||||
event({
|
||||
source: {
|
||||
spanId: 'span-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-1',
|
||||
sourceStartSu: 10,
|
||||
sourceEndSu: 14
|
||||
}
|
||||
})
|
||||
)
|
||||
harness.completions[0]!.resolve()
|
||||
|
||||
await receipt
|
||||
expect(projections[0]?.desktopSpan).toMatchObject({
|
||||
spanId: 'span-1',
|
||||
projectionSemanticsId: projections[0]?.identity.projectionSemanticsId,
|
||||
providerGeneration: 1,
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-1',
|
||||
sourceStartSu: 10,
|
||||
sourceEndSu: 14,
|
||||
displayStart: 0,
|
||||
displayEnd: 4,
|
||||
transform: { transformed: false, rawLengthSu: 4, scalarSafe: true }
|
||||
})
|
||||
expect(Object.isFrozen(projections[0]?.desktopSpan)).toBe(true)
|
||||
})
|
||||
|
||||
it('exports only the model-settled source boundary before generation close', async () => {
|
||||
const harness = createHarness()
|
||||
const receipt = harness.intake.acceptData(
|
||||
event({
|
||||
source: {
|
||||
spanId: 'recovery-span-1',
|
||||
clientGeneration: 3,
|
||||
ownerGeneration: 4,
|
||||
deliveryToken: 'delivery-token-1',
|
||||
sourceStartSu: 10,
|
||||
sourceEndSu: 14
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
expect(harness.intake.getAcceptedSourceCheckpoints(1)[0]?.acceptedSourceEndSu).toBe(10)
|
||||
harness.completions[0]!.resolve()
|
||||
await receipt
|
||||
expect(harness.intake.getAcceptedSourceCheckpoints(1)).toEqual([
|
||||
{
|
||||
id: 'pty-1',
|
||||
providerGeneration: 1,
|
||||
clientGeneration: 3,
|
||||
ownerGeneration: 4,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'delivery-token-1',
|
||||
acceptedSourceEndSu: 14
|
||||
}
|
||||
])
|
||||
})
|
||||
|
||||
it('rolls back projection staging when model capture throws synchronously', async () => {
|
||||
const project = vi.fn()
|
||||
const harness = createHarness({
|
||||
acceptModel: () => {
|
||||
throw new Error('model reservation failed')
|
||||
},
|
||||
project
|
||||
})
|
||||
|
||||
await expect(harness.intake.acceptData(event())).rejects.toThrow('model reservation failed')
|
||||
expect(project).not.toHaveBeenCalled()
|
||||
expect(harness.intake.getDebugSnapshot().projection).toMatchObject({
|
||||
rolledBack: 1,
|
||||
records: 0
|
||||
})
|
||||
expect(harness.dependencies.closeProvider).toHaveBeenCalledWith(1, 'model-admission-failed')
|
||||
})
|
||||
|
||||
it('rolls back projection staging when source reservation validation fails', async () => {
|
||||
const harness = createHarness()
|
||||
const source = {
|
||||
spanId: 'duplicate',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-1',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4
|
||||
}
|
||||
const first = harness.intake.acceptData(event({ source }))
|
||||
harness.completions[0]!.resolve()
|
||||
await first
|
||||
await expect(
|
||||
harness.intake.acceptData(
|
||||
event({
|
||||
source: { ...source, sourceStartSu: 4, sourceEndSu: 8 }
|
||||
})
|
||||
)
|
||||
).rejects.toThrow('duplicate')
|
||||
expect(harness.intake.getDebugSnapshot().projection).toMatchObject({
|
||||
rolledBack: 1,
|
||||
records: 1
|
||||
})
|
||||
})
|
||||
|
||||
it('rolls back committed source and scanner facts when model capture throws', async () => {
|
||||
let attempts = 0
|
||||
const harness = createHarness({
|
||||
acceptModel: (accepted) => {
|
||||
attempts++
|
||||
if (attempts === 1) {
|
||||
throw new Error('model reservation failed')
|
||||
}
|
||||
return {
|
||||
sequence: accepted.rawLength,
|
||||
completion: Promise.resolve()
|
||||
}
|
||||
}
|
||||
})
|
||||
const source = {
|
||||
spanId: 'span-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-1',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 5
|
||||
}
|
||||
await expect(
|
||||
harness.intake.acceptData(event({ data: '\x1b[?20', rawLength: 5, source }))
|
||||
).rejects.toThrow('model reservation failed')
|
||||
|
||||
const retry = await harness.intake.acceptData(
|
||||
event({ data: '\x1b[?20', rawLength: 5, source: { ...source, spanId: 'span-2' } })
|
||||
)
|
||||
expect(retry.projection.identity.displayStart).toBe(0)
|
||||
expect(retry.projection.beforeScanner).toEqual({ tail: '', pendingSubscribe: false })
|
||||
})
|
||||
|
||||
it('rejects stale provider generations without model capture', async () => {
|
||||
const harness = createHarness()
|
||||
const current = harness.intake.acceptData(event({ providerGeneration: 2 }))
|
||||
harness.completions[0]!.resolve()
|
||||
await current
|
||||
|
||||
await expect(harness.intake.acceptData(event({ providerGeneration: 1 }))).rejects.toThrow(
|
||||
'ssh_output_stale_generation'
|
||||
)
|
||||
expect(harness.completions).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('keeps exit behind accepted model and projection work', async () => {
|
||||
const harness = createHarness({}, { exitBarrierMs: 1000 })
|
||||
const dataReceipt = harness.intake.acceptData(event())
|
||||
const exitReceipt = harness.intake.acceptExit({
|
||||
id: 'pty-1',
|
||||
code: 0,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
await Promise.resolve()
|
||||
expect(harness.order).toEqual(['model:aaaa', 'project:aaaa'])
|
||||
|
||||
harness.completions[0]!.resolve()
|
||||
await Promise.all([dataReceipt, exitReceipt])
|
||||
expect(harness.order).toEqual(['model:aaaa', 'project:aaaa', 'exit'])
|
||||
})
|
||||
|
||||
it('admits queued pre-exit source spans before sealing the token', async () => {
|
||||
const harness = createHarness({}, { exitBarrierMs: 1000 })
|
||||
const first = harness.intake.acceptData(
|
||||
event({
|
||||
source: {
|
||||
spanId: 'span-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-1',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4
|
||||
}
|
||||
})
|
||||
)
|
||||
const second = harness.intake.acceptData(
|
||||
event({
|
||||
data: 'bbbb',
|
||||
source: {
|
||||
spanId: 'span-2',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-1',
|
||||
sourceStartSu: 4,
|
||||
sourceEndSu: 8
|
||||
}
|
||||
})
|
||||
)
|
||||
const exit = harness.intake.acceptExit({
|
||||
id: 'pty-1',
|
||||
code: 0,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
|
||||
harness.completions[0]!.resolve()
|
||||
await first
|
||||
harness.completions[1]!.resolve()
|
||||
await Promise.all([second, exit])
|
||||
expect(harness.order).toEqual([
|
||||
'model:aaaa',
|
||||
'project:aaaa',
|
||||
'model:bbbb',
|
||||
'project:bbbb',
|
||||
'exit'
|
||||
])
|
||||
})
|
||||
|
||||
it('does not pump queued model work after a running completion fails', async () => {
|
||||
const harness = createHarness()
|
||||
const first = harness.intake.acceptData(event({ data: 'first' }))
|
||||
const queued = harness.intake.acceptData(event({ data: 'queued' }))
|
||||
|
||||
harness.completions[0]!.reject(new Error('emulator failed'))
|
||||
await expect(first).rejects.toThrow('emulator failed')
|
||||
await expect(queued).rejects.toThrow('ssh_model_admission_completion_failed')
|
||||
expect(harness.order).toEqual(['model:first', 'project:first'])
|
||||
expect(harness.completions).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('retains exited delivery until published renderer projections settle', async () => {
|
||||
const harness = createHarness({}, { exitBarrierMs: 1000 })
|
||||
const dataReceipt = harness.intake.acceptData(event())
|
||||
harness.completions[0]!.resolve()
|
||||
const receipt = await dataReceipt
|
||||
harness.intake.publishProjectionPrefix(
|
||||
[receipt.projection.identity.projectionSemanticsId],
|
||||
4,
|
||||
4
|
||||
)
|
||||
|
||||
let exited = false
|
||||
const exitReceipt = harness.intake
|
||||
.acceptExit({
|
||||
id: 'pty-1',
|
||||
code: 0,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
.then(() => {
|
||||
exited = true
|
||||
})
|
||||
await Promise.resolve()
|
||||
|
||||
expect(exited).toBe(false)
|
||||
expect(harness.intake.getDebugSnapshot().projection.records).toBe(1)
|
||||
expect(harness.intake.settleProjectionPrefix('pty-1', 4)).toBe(4)
|
||||
await exitReceipt
|
||||
expect(harness.order.at(-1)).toBe('exit')
|
||||
expect(harness.intake.getDebugSnapshot().projection.records).toBe(0)
|
||||
})
|
||||
|
||||
it('owns renderer exit preparation through finalization and duplicate rejection', async () => {
|
||||
const releaseRendererExit = vi.fn()
|
||||
const harness = createHarness({
|
||||
prepareExit: vi.fn(() => releaseRendererExit)
|
||||
})
|
||||
const dataReceipt = harness.intake.acceptData(event())
|
||||
harness.completions[0]!.resolve()
|
||||
const receipt = await dataReceipt
|
||||
harness.intake.publishProjectionPrefix(
|
||||
[receipt.projection.identity.projectionSemanticsId],
|
||||
4,
|
||||
4
|
||||
)
|
||||
const exitEvent = {
|
||||
id: 'pty-1',
|
||||
code: 0,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
}
|
||||
const exit = harness.intake.acceptExit(exitEvent)
|
||||
await Promise.resolve()
|
||||
|
||||
expect(releaseRendererExit).not.toHaveBeenCalled()
|
||||
await expect(harness.intake.acceptExit(exitEvent)).rejects.toThrow('ssh_output_duplicate_exit')
|
||||
expect(releaseRendererExit).not.toHaveBeenCalled()
|
||||
|
||||
harness.intake.settleProjectionPrefix('pty-1', 4)
|
||||
await exit
|
||||
expect(releaseRendererExit).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('releases renderer exit preparation when generation close aborts finalization', async () => {
|
||||
const releaseRendererExit = vi.fn()
|
||||
const harness = createHarness({
|
||||
prepareExit: vi.fn(() => releaseRendererExit)
|
||||
})
|
||||
const dataReceipt = harness.intake.acceptData(event())
|
||||
harness.completions[0]!.resolve()
|
||||
const receipt = await dataReceipt
|
||||
harness.intake.publishProjectionPrefix(
|
||||
[receipt.projection.identity.projectionSemanticsId],
|
||||
4,
|
||||
4
|
||||
)
|
||||
const exit = harness.intake.acceptExit({
|
||||
id: 'pty-1',
|
||||
code: 0,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
await Promise.resolve()
|
||||
|
||||
harness.intake.closeGeneration(1, 'provider-replaced')
|
||||
|
||||
await expect(exit).rejects.toThrow('provider-replaced')
|
||||
expect(releaseRendererExit).toHaveBeenCalledOnce()
|
||||
expect(harness.order).not.toContain('exit')
|
||||
})
|
||||
|
||||
it('retains exit until a required remote source consumer settles', async () => {
|
||||
const harness = createHarness({}, { exitBarrierMs: 1000 })
|
||||
const remote = harness.intake.getRemoteSourceRangeConsumerHooks()
|
||||
const stream = {
|
||||
ptyId: 'pty-1',
|
||||
consumerId: 'remote-1',
|
||||
streamGeneration: 'stream-1'
|
||||
}
|
||||
expect(remote.attach(stream)).toBe(true)
|
||||
const dataReceipt = harness.intake.acceptData(
|
||||
event({
|
||||
source: {
|
||||
spanId: 'span-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-1',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4
|
||||
}
|
||||
})
|
||||
)
|
||||
harness.completions[0]!.resolve()
|
||||
const receipt = await dataReceipt
|
||||
|
||||
let exited = false
|
||||
const exitReceipt = harness.intake
|
||||
.acceptExit({
|
||||
id: 'pty-1',
|
||||
code: 0,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
.then(() => {
|
||||
exited = true
|
||||
})
|
||||
await Promise.resolve()
|
||||
expect(exited).toBe(false)
|
||||
|
||||
remote.settle(stream, [receipt.projection.desktopSpan!])
|
||||
await exitReceipt
|
||||
expect(harness.order.at(-1)).toBe('exit')
|
||||
expect(harness.dependencies.closeProvider).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('keeps timed-out exit projections until generation-close proof', async () => {
|
||||
const harness = createHarness(
|
||||
{
|
||||
cancelSourceDelivery: () => Promise.reject(new Error('cancel transport failed'))
|
||||
},
|
||||
{ exitBarrierMs: 1, exitCancellationProofMs: 10 }
|
||||
)
|
||||
const dataReceipt = harness.intake.acceptData(event())
|
||||
harness.completions[0]!.resolve()
|
||||
const receipt = await dataReceipt
|
||||
harness.intake.publishProjectionPrefix(
|
||||
[receipt.projection.identity.projectionSemanticsId],
|
||||
4,
|
||||
4
|
||||
)
|
||||
|
||||
await expect(
|
||||
harness.intake.acceptExit({
|
||||
id: 'pty-1',
|
||||
code: 0,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
).rejects.toThrow('ssh_source_cancellation_identity_unavailable')
|
||||
expect(harness.dependencies.closeProvider).toHaveBeenCalledWith(
|
||||
1,
|
||||
'ssh-exit-cancellation-proof-failed'
|
||||
)
|
||||
expect(harness.intake.getDebugSnapshot().projection.records).toBe(1)
|
||||
|
||||
harness.intake.closeGeneration(1, 'provider-closed')
|
||||
expect(harness.intake.getDebugSnapshot().projection.records).toBe(0)
|
||||
})
|
||||
|
||||
it('cancels only the timed-out source delivery and keeps the provider usable', async () => {
|
||||
const cancelSourceDelivery = vi.fn(async () => ({ sentEndSu: 4, creditedEndSu: 0 }))
|
||||
const harness = createHarness(
|
||||
{ cancelSourceDelivery },
|
||||
{ exitBarrierMs: 1, exitCancellationProofMs: 100 }
|
||||
)
|
||||
const remote = harness.intake.getRemoteSourceRangeConsumerHooks()
|
||||
const stream = {
|
||||
ptyId: 'pty-1',
|
||||
consumerId: 'remote-1',
|
||||
streamGeneration: 'stream-1'
|
||||
}
|
||||
remote.attach(stream)
|
||||
const dataReceipt = harness.intake.acceptData(
|
||||
event({
|
||||
source: {
|
||||
spanId: 'span-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-1',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4
|
||||
}
|
||||
})
|
||||
)
|
||||
harness.completions[0]!.resolve()
|
||||
await dataReceipt
|
||||
|
||||
await harness.intake.acceptExit({
|
||||
id: 'pty-1',
|
||||
code: 0,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
expect(cancelSourceDelivery).toHaveBeenCalledWith(1, {
|
||||
id: 'pty-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-1'
|
||||
})
|
||||
expect(harness.dependencies.closeProvider).not.toHaveBeenCalled()
|
||||
expect(harness.order.at(-1)).toBe('exit')
|
||||
|
||||
const sibling = harness.intake.acceptData(
|
||||
event({ id: 'pty-2', ptyIncarnation: 'incarnation-2' })
|
||||
)
|
||||
harness.completions[1]!.resolve()
|
||||
await expect(sibling).resolves.toMatchObject({ ptyId: 'pty-2' })
|
||||
})
|
||||
|
||||
it('accepts recovery cancellation proof before any replacement span is admitted', () => {
|
||||
const harness = createHarness()
|
||||
|
||||
expect(() =>
|
||||
harness.intake.applySourceRecoveryCancellationProof(
|
||||
{
|
||||
id: 'pty-1',
|
||||
code: -1,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
},
|
||||
{ sentEndSu: 8, creditedEndSu: 4 }
|
||||
)
|
||||
).not.toThrow()
|
||||
expect(harness.intake.getDebugSnapshot().source).toEqual({
|
||||
openedTokens: 0,
|
||||
ptyIdentities: 0
|
||||
})
|
||||
})
|
||||
|
||||
it('reclaims a partially admitted recovery prefix from authoritative proof', async () => {
|
||||
const harness = createHarness()
|
||||
const receipt = harness.intake.acceptData(
|
||||
event({
|
||||
source: {
|
||||
spanId: 'recovery-span',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'recovery-token',
|
||||
sourceStartSu: 4,
|
||||
sourceEndSu: 8
|
||||
}
|
||||
})
|
||||
)
|
||||
harness.completions[0]!.resolve()
|
||||
await receipt
|
||||
|
||||
harness.intake.applySourceRecoveryCancellationProof(
|
||||
{
|
||||
id: 'pty-1',
|
||||
code: -1,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
},
|
||||
{ sentEndSu: 12, creditedEndSu: 4 }
|
||||
)
|
||||
|
||||
expect(harness.intake.getDebugSnapshot().source).toEqual({
|
||||
openedTokens: 0,
|
||||
ptyIdentities: 0
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects late same-generation data after ordered exit cleanup', async () => {
|
||||
const harness = createHarness({}, { exitBarrierMs: 1000 })
|
||||
const first = harness.intake.acceptData(event())
|
||||
harness.completions[0]!.resolve()
|
||||
await first
|
||||
await harness.intake.acceptExit({
|
||||
id: 'pty-1',
|
||||
code: 0,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
|
||||
await expect(harness.intake.acceptData(event())).rejects.toThrow('ssh_output_after_exit')
|
||||
await expect(
|
||||
harness.intake.acceptExit({
|
||||
id: 'pty-1',
|
||||
code: 0,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
).rejects.toThrow('ssh_output_duplicate_exit')
|
||||
|
||||
const next = harness.intake.acceptData(
|
||||
event({ providerGeneration: 2, ptyIncarnation: 'incarnation-2' })
|
||||
)
|
||||
harness.completions[1]!.resolve()
|
||||
await expect(next).resolves.toMatchObject({
|
||||
projection: { identity: { ptyIncarnation: 'incarnation-2', displayStart: 0 } }
|
||||
})
|
||||
})
|
||||
|
||||
it('closes the provider when exit finalization fails', async () => {
|
||||
const releaseRendererExit = vi.fn()
|
||||
const harness = createHarness({
|
||||
prepareExit: () => releaseRendererExit,
|
||||
finalizeExit: () => {
|
||||
throw new Error('renderer exit send failed')
|
||||
}
|
||||
})
|
||||
|
||||
await expect(
|
||||
harness.intake.acceptExit({
|
||||
id: 'pty-1',
|
||||
code: 0,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
).rejects.toThrow('renderer exit send failed')
|
||||
expect(harness.dependencies.closeProvider).toHaveBeenCalledWith(1, 'pty-exit-finalize-failed')
|
||||
expect(releaseRendererExit).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('cancels queued work and exit waiters on generation cleanup', async () => {
|
||||
const harness = createHarness(
|
||||
{},
|
||||
{
|
||||
perPtyHighSourceUnits: 8,
|
||||
perPtyHighBytes: 2048,
|
||||
globalHighSourceUnits: 8,
|
||||
globalHighBytes: 2048
|
||||
}
|
||||
)
|
||||
const running = harness.intake.acceptData(event())
|
||||
const queued = harness.intake.acceptData(event({ data: 'bbbb' }))
|
||||
const exit = harness.intake.acceptExit({
|
||||
id: 'pty-1',
|
||||
code: 0,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
harness.intake.closeGeneration(1, 'provider-closed')
|
||||
|
||||
await expect(queued).rejects.toThrow('provider-closed')
|
||||
await expect(exit).rejects.toThrow('provider-closed')
|
||||
await expect(harness.intake.acceptData(event())).rejects.toThrow('ssh_output_stale_generation')
|
||||
await expect(running).rejects.toThrow('provider-closed')
|
||||
expect(harness.intake.getDebugSnapshot().model).toMatchObject({ sourceUnits: 0, bytes: 0 })
|
||||
harness.completions[0]!.resolve()
|
||||
await Promise.resolve()
|
||||
expect(harness.intake.getDebugSnapshot().exitBarriers).toBe(0)
|
||||
})
|
||||
|
||||
it('resumes a paused provider during generation cleanup', async () => {
|
||||
const harness = createHarness(
|
||||
{},
|
||||
{
|
||||
perPtyHighSourceUnits: 4,
|
||||
perPtyHighBytes: 1024,
|
||||
globalHighSourceUnits: 4,
|
||||
globalHighBytes: 1024,
|
||||
pressureMaxFrames: 2,
|
||||
pressureMaxBytes: 1024
|
||||
}
|
||||
)
|
||||
const running = harness.intake.acceptData(event())
|
||||
const pressured = harness.intake.acceptData(event({ data: 'bbbb' }))
|
||||
|
||||
harness.intake.closeGeneration(1, 'provider-closed')
|
||||
|
||||
await expect(pressured).rejects.toThrow('provider-closed')
|
||||
expect(harness.dependencies.resumeProvider).toHaveBeenCalledWith(1, 'pty-1')
|
||||
await expect(running).rejects.toThrow('provider-closed')
|
||||
harness.completions[0]!.resolve()
|
||||
await Promise.resolve()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,296 @@
|
||||
import {
|
||||
SshPtyLegacyProjectionLedger,
|
||||
type LegacySshProjectionSemantics
|
||||
} from './ssh-pty-legacy-projection'
|
||||
import { SshPtyModelAdmission } from './ssh-pty-model-admission'
|
||||
import { SshPtyOutputExitDeadline } from './ssh-pty-output-exit-deadline'
|
||||
import { settleSshPtyOutputExit } from './ssh-pty-output-exit'
|
||||
import { SshPtyOutputGenerationGuard } from './ssh-pty-output-generation-guard'
|
||||
import {
|
||||
SshPtyOutputModelMigration,
|
||||
type SshPtyOutputGenerationMigration,
|
||||
type SshPtyTrackedModelAdmission
|
||||
} from './ssh-pty-output-model-migration'
|
||||
import {
|
||||
SshPtyOutputSourceObligations,
|
||||
type SshPtyOutputSourceReservation
|
||||
} from './ssh-pty-output-source-obligations'
|
||||
import type {
|
||||
SshPtyOutputDataEvent,
|
||||
SshPtyOutputExitEvent,
|
||||
SshPtyOutputIntakeDependencies,
|
||||
SshPtyOutputIntakeOptions,
|
||||
SshPtyOutputReceipt
|
||||
} from './ssh-pty-output-intake-contract'
|
||||
import { outputIntakeError } from './ssh-pty-output-intake-validation'
|
||||
|
||||
export type {
|
||||
SshPtyOutputDataEvent,
|
||||
SshPtyOutputExitEvent,
|
||||
SshPtyOutputIntakeDependencies,
|
||||
SshPtyOutputIntakeOptions,
|
||||
SshPtyOutputReceipt
|
||||
} from './ssh-pty-output-intake-contract'
|
||||
|
||||
export class SshPtyOutputIntake {
|
||||
private readonly projections: SshPtyLegacyProjectionLedger
|
||||
private readonly sourceObligations: SshPtyOutputSourceObligations
|
||||
private readonly generationGuard = new SshPtyOutputGenerationGuard(() => this.disposed)
|
||||
private readonly admission: SshPtyModelAdmission
|
||||
private readonly modelMigration: SshPtyOutputModelMigration
|
||||
private readonly exitDeadline: SshPtyOutputExitDeadline
|
||||
private disposed = false
|
||||
|
||||
constructor(
|
||||
private readonly dependencies: SshPtyOutputIntakeDependencies,
|
||||
options: SshPtyOutputIntakeOptions = {}
|
||||
) {
|
||||
this.sourceObligations = new SshPtyOutputSourceObligations(dependencies.publishSourceAck)
|
||||
this.projections = new SshPtyLegacyProjectionLedger({
|
||||
onSettled: (span) => this.sourceObligations.settleDesktop(span, 'renderer-parse'),
|
||||
onTransferred: (span, reason) => this.sourceObligations.transferDesktop(span, reason)
|
||||
})
|
||||
this.admission = new SshPtyModelAdmission({
|
||||
...options,
|
||||
pauseProvider: (key) =>
|
||||
this.dependencies.pauseProvider?.(key.providerGeneration, key.ptyId) ?? false,
|
||||
resumeProvider: (key) =>
|
||||
this.dependencies.resumeProvider?.(key.providerGeneration, key.ptyId),
|
||||
closeProvider: (providerGeneration, reason) =>
|
||||
this.dependencies.closeProvider?.(providerGeneration, reason)
|
||||
})
|
||||
this.modelMigration = new SshPtyOutputModelMigration(
|
||||
this.admission,
|
||||
this.sourceObligations,
|
||||
(providerGeneration, ptyId) =>
|
||||
this.dependencies.resetModelForMigration?.(providerGeneration, ptyId)
|
||||
)
|
||||
this.exitDeadline = new SshPtyOutputExitDeadline({
|
||||
admission: this.admission,
|
||||
projections: this.projections,
|
||||
sourceObligations: this.sourceObligations,
|
||||
intake: this.dependencies,
|
||||
barrierMs: options.exitBarrierMs,
|
||||
cancellationProofMs: options.exitCancellationProofMs
|
||||
})
|
||||
}
|
||||
|
||||
acceptData(event: SshPtyOutputDataEvent): Promise<SshPtyOutputReceipt> {
|
||||
try {
|
||||
this.generationGuard.validateData(event)
|
||||
} catch (error) {
|
||||
return Promise.reject(error)
|
||||
}
|
||||
let projection: LegacySshProjectionSemantics | undefined
|
||||
let sourceReservation: SshPtyOutputSourceReservation | undefined
|
||||
const key = { ptyId: event.id, providerGeneration: event.providerGeneration }
|
||||
const tracked: SshPtyTrackedModelAdmission = { key, started: false }
|
||||
const receipt = this.admission.accept(key, event.data, event.rawLength, () => {
|
||||
tracked.started = true
|
||||
const expectedSequence = this.dependencies.getModelSequence(event.id) + event.rawLength
|
||||
const reservation = this.projections.reserve({
|
||||
ptyId: event.id,
|
||||
providerGeneration: event.providerGeneration,
|
||||
ptyIncarnation: event.ptyIncarnation,
|
||||
data: event.data,
|
||||
sequenceEnd: expectedSequence,
|
||||
rawLength: event.rawLength,
|
||||
transformed: event.transformed,
|
||||
source: event.source
|
||||
})
|
||||
try {
|
||||
if (reservation.semantics.desktopSpan) {
|
||||
sourceReservation = this.sourceObligations.reserve(
|
||||
event,
|
||||
reservation.semantics.desktopSpan
|
||||
)
|
||||
}
|
||||
} catch (error) {
|
||||
this.projections.rollback(reservation)
|
||||
throw error
|
||||
}
|
||||
try {
|
||||
projection = this.projections.commit(reservation)
|
||||
if (sourceReservation) {
|
||||
this.sourceObligations.commit(
|
||||
sourceReservation,
|
||||
event.id,
|
||||
projection.identity.sequenceEnd
|
||||
)
|
||||
}
|
||||
} catch (error) {
|
||||
if (sourceReservation) {
|
||||
this.sourceObligations.rollback(sourceReservation)
|
||||
}
|
||||
if (!this.projections.rollbackCommitted(reservation)) {
|
||||
this.projections.rollback(reservation)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
let model: { sequence: number; completion: Promise<void> }
|
||||
try {
|
||||
model = this.dependencies.acceptModel(event, projection)
|
||||
} catch (error) {
|
||||
if (sourceReservation) {
|
||||
this.sourceObligations.rollback(sourceReservation)
|
||||
}
|
||||
this.projections.rollbackCommitted(reservation)
|
||||
throw error
|
||||
}
|
||||
try {
|
||||
this.dependencies.project(event, projection)
|
||||
} catch {
|
||||
const id = projection.identity.projectionSemanticsId
|
||||
this.projections.transfer([id], 'projection-admission-failed')
|
||||
}
|
||||
return model
|
||||
})
|
||||
const completion = receipt.then(
|
||||
(modelReceipt) => {
|
||||
if (!projection) {
|
||||
throw outputIntakeError('ssh_projection_receipt_missing')
|
||||
}
|
||||
if (sourceReservation) {
|
||||
this.sourceObligations.settleModel(sourceReservation.span)
|
||||
}
|
||||
return Object.freeze({ ...modelReceipt, projection })
|
||||
},
|
||||
(error) => {
|
||||
if (projection) {
|
||||
this.projections.transfer(
|
||||
[projection.identity.projectionSemanticsId],
|
||||
'model-admission-failed'
|
||||
)
|
||||
}
|
||||
const code = (error as { code?: unknown }).code
|
||||
if (
|
||||
code !== 'ssh_exit_delivery_canceled' &&
|
||||
!(typeof code === 'string' && code.startsWith('ssh_model_migration_'))
|
||||
) {
|
||||
this.dependencies.closeProvider?.(event.providerGeneration, 'model-admission-failed')
|
||||
}
|
||||
throw error
|
||||
}
|
||||
)
|
||||
tracked.completion = completion
|
||||
this.modelMigration.track(tracked)
|
||||
return completion
|
||||
}
|
||||
|
||||
async acceptExit(event: SshPtyOutputExitEvent): Promise<void> {
|
||||
this.generationGuard.sealExit(event)
|
||||
await this.exitDeadline.wait(event, (validateNormalExit) =>
|
||||
this.finishExit(event, validateNormalExit)
|
||||
)
|
||||
}
|
||||
|
||||
private async finishExit(
|
||||
event: SshPtyOutputExitEvent,
|
||||
validateNormalExit: () => void
|
||||
): Promise<void> {
|
||||
await settleSshPtyOutputExit({
|
||||
event,
|
||||
admission: this.admission,
|
||||
projections: this.projections,
|
||||
dependencies: this.dependencies,
|
||||
validateGeneration: () => {
|
||||
this.generationGuard.validate(event)
|
||||
validateNormalExit()
|
||||
},
|
||||
prepareExit: () => this.exitDeadline.prepareExitOnce(event),
|
||||
afterAdmissionIdle: () => this.sourceObligations.sealPty(event),
|
||||
waitForSourceTerminal: () => this.sourceObligations.whenPtyTerminal(event),
|
||||
beforeFinalize: () => this.sourceObligations.markExitPublished(event)
|
||||
})
|
||||
}
|
||||
|
||||
publishProjectionPrefix(
|
||||
ids: readonly string[],
|
||||
displayChars: number,
|
||||
accountingChars: number
|
||||
): void {
|
||||
this.projections.publishPrefix(ids, displayChars, accountingChars)
|
||||
}
|
||||
|
||||
settleProjectionPrefix(ptyId: string, accountingChars: number): number {
|
||||
return this.projections.settlePublishedPrefix(ptyId, accountingChars)
|
||||
}
|
||||
|
||||
transferProjections(ids: readonly string[], reason: string): number {
|
||||
return this.projections.transfer(ids, reason)
|
||||
}
|
||||
|
||||
transferPtyProjections(ptyId: string, reason: string): number {
|
||||
return this.projections.transferPty(ptyId, reason)
|
||||
}
|
||||
|
||||
hasProjectionFromGeneration(ids: readonly string[], providerGeneration: number): boolean {
|
||||
return ids.some(
|
||||
(id) => this.projections.get(id)?.identity.providerGeneration === providerGeneration
|
||||
)
|
||||
}
|
||||
|
||||
hasUnpublishedProjection(id: string): boolean {
|
||||
return this.projections.hasUnpublished(id)
|
||||
}
|
||||
|
||||
closeGeneration(providerGeneration: number, reason: string): void {
|
||||
this.generationGuard.closeGeneration(providerGeneration)
|
||||
this.admission.closeGeneration(providerGeneration, reason)
|
||||
this.dependencies.onGenerationClosed?.(providerGeneration, reason)
|
||||
this.projections.closeGeneration(providerGeneration, reason)
|
||||
this.sourceObligations.closeGeneration(providerGeneration, reason)
|
||||
this.exitDeadline.closeGeneration(providerGeneration, outputIntakeError(reason))
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
if (this.disposed) {
|
||||
return
|
||||
}
|
||||
this.disposed = true
|
||||
for (const generation of this.generationGuard.activeGenerations()) {
|
||||
this.closeGeneration(generation, 'ssh_output_intake_disposed')
|
||||
}
|
||||
this.admission.dispose()
|
||||
this.sourceObligations.dispose()
|
||||
}
|
||||
|
||||
getRemoteSourceRangeConsumerHooks() {
|
||||
return this.sourceObligations.remoteHooks
|
||||
}
|
||||
|
||||
getAcceptedSourceCheckpoints(providerGeneration: number) {
|
||||
return this.sourceObligations.acceptedCheckpoints(providerGeneration)
|
||||
}
|
||||
|
||||
beginGenerationMigration(
|
||||
providerGeneration: number,
|
||||
timeoutMs?: number
|
||||
): SshPtyOutputGenerationMigration {
|
||||
return this.modelMigration.beginGeneration(providerGeneration, timeoutMs)
|
||||
}
|
||||
|
||||
applySourceCancellationProof(
|
||||
event: SshPtyOutputExitEvent,
|
||||
proof: Readonly<{ sentEndSu: number; creditedEndSu: number }>
|
||||
): boolean {
|
||||
return this.sourceObligations.applyCancellationProof(event, proof)
|
||||
}
|
||||
|
||||
applySourceRecoveryCancellationProof(
|
||||
event: SshPtyOutputExitEvent,
|
||||
proof: Readonly<{ sentEndSu: number; creditedEndSu: number }>
|
||||
): void {
|
||||
this.sourceObligations.applyRecoveryCancellationProof(event, proof)
|
||||
}
|
||||
|
||||
getDebugSnapshot() {
|
||||
return {
|
||||
model: this.admission.getDebugSnapshot(),
|
||||
projection: this.projections.getDebugSnapshot(),
|
||||
source: this.sourceObligations.getDebugSnapshot(),
|
||||
generation: this.generationGuard.getDebugSnapshot(),
|
||||
exitBarriers: this.exitDeadline.activeBarriers
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
createSshPtyOutputIntakeHarness as createHarness,
|
||||
sshPtyOutputEvent as event
|
||||
} from './ssh-pty-output-intake-test-harness'
|
||||
|
||||
describe('SshPtyOutputModelMigration', () => {
|
||||
it('fences a running source span before exporting its migration checkpoint', async () => {
|
||||
const harness = createHarness()
|
||||
const first = harness.intake.acceptData(
|
||||
event({
|
||||
data: 'aaaa',
|
||||
source: {
|
||||
spanId: 'span-a',
|
||||
clientGeneration: 3,
|
||||
ownerGeneration: 4,
|
||||
deliveryToken: 'delivery-token-1',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4
|
||||
}
|
||||
})
|
||||
)
|
||||
harness.completions[0]!.resolve()
|
||||
await first
|
||||
const second = harness.intake.acceptData(
|
||||
event({
|
||||
data: 'bbbb',
|
||||
source: {
|
||||
spanId: 'span-b',
|
||||
clientGeneration: 3,
|
||||
ownerGeneration: 4,
|
||||
deliveryToken: 'delivery-token-1',
|
||||
sourceStartSu: 4,
|
||||
sourceEndSu: 8
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
const migration = harness.intake.beginGenerationMigration(1)
|
||||
const result = migration.byPty.get('pty-1')
|
||||
expect(result).toBeDefined()
|
||||
await expect(Promise.race([result, Promise.resolve('pending')])).resolves.toBe('pending')
|
||||
expect(harness.intake.getAcceptedSourceCheckpoints(1)[0]?.acceptedSourceEndSu).toBe(4)
|
||||
|
||||
harness.completions[1]!.resolve()
|
||||
await expect(second).resolves.toMatchObject({ sequence: 8 })
|
||||
await expect(result).resolves.toMatchObject({
|
||||
status: 'settled',
|
||||
checkpoint: { acceptedSourceEndSu: 8 }
|
||||
})
|
||||
expect(harness.order.filter((entry) => entry === 'project:bbbb')).toHaveLength(1)
|
||||
await migration.completion
|
||||
})
|
||||
|
||||
it('times out one migration, resets its model, and releases retained admission once', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const resetModelForMigration = vi.fn()
|
||||
const harness = createHarness({ resetModelForMigration })
|
||||
const receipt = harness.intake.acceptData(
|
||||
event({
|
||||
source: {
|
||||
spanId: 'span-b',
|
||||
clientGeneration: 3,
|
||||
ownerGeneration: 4,
|
||||
deliveryToken: 'delivery-token-1',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4
|
||||
}
|
||||
})
|
||||
)
|
||||
const migration = harness.intake.beginGenerationMigration(1, 10_000)
|
||||
const result = migration.byPty.get('pty-1')
|
||||
|
||||
await vi.advanceTimersByTimeAsync(10_000)
|
||||
|
||||
await expect(result).resolves.toMatchObject({ status: 'checkpoint-unavailable' })
|
||||
await expect(receipt).rejects.toThrow('ssh_model_migration_timeout')
|
||||
expect(resetModelForMigration).toHaveBeenCalledOnce()
|
||||
expect(resetModelForMigration).toHaveBeenCalledWith(1, 'pty-1')
|
||||
expect(harness.intake.getDebugSnapshot().model).toMatchObject({
|
||||
sourceUnits: 0,
|
||||
bytes: 0,
|
||||
migratingPtys: 1
|
||||
})
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
harness.completions[0]!.resolve()
|
||||
await Promise.resolve()
|
||||
expect(resetModelForMigration).toHaveBeenCalledOnce()
|
||||
harness.intake.closeGeneration(1, 'connection_lost')
|
||||
expect(harness.intake.getDebugSnapshot().model.migratingPtys).toBe(0)
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('contains a running callback failure to its migrating PTY', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const resetModelForMigration = vi.fn()
|
||||
const harness = createHarness({ resetModelForMigration })
|
||||
const sibling = harness.intake.acceptData(
|
||||
event({
|
||||
id: 'pty-sibling',
|
||||
ptyIncarnation: 'incarnation-sibling',
|
||||
source: {
|
||||
spanId: 'span-sibling',
|
||||
clientGeneration: 3,
|
||||
ownerGeneration: 4,
|
||||
deliveryToken: 'delivery-sibling',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4
|
||||
}
|
||||
})
|
||||
)
|
||||
harness.completions[0]!.resolve()
|
||||
await sibling
|
||||
const failed = harness.intake.acceptData(
|
||||
event({
|
||||
id: 'pty-failed',
|
||||
ptyIncarnation: 'incarnation-failed',
|
||||
source: {
|
||||
spanId: 'span-failed',
|
||||
clientGeneration: 3,
|
||||
ownerGeneration: 4,
|
||||
deliveryToken: 'delivery-failed',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4
|
||||
}
|
||||
})
|
||||
)
|
||||
const migration = harness.intake.beginGenerationMigration(1)
|
||||
|
||||
harness.completions[1]!.reject(new Error('emulator failed'))
|
||||
|
||||
await expect(failed).rejects.toMatchObject({
|
||||
message: 'emulator failed',
|
||||
code: 'ssh_model_migration_completion_failed'
|
||||
})
|
||||
await expect(migration.byPty.get('pty-failed')).resolves.toEqual({
|
||||
status: 'checkpoint-unavailable',
|
||||
reason: 'completion-failed'
|
||||
})
|
||||
await expect(migration.byPty.get('pty-sibling')).resolves.toMatchObject({
|
||||
status: 'settled',
|
||||
checkpoint: { id: 'pty-sibling', acceptedSourceEndSu: 4 }
|
||||
})
|
||||
await migration.completion
|
||||
expect(resetModelForMigration).toHaveBeenCalledTimes(1)
|
||||
expect(resetModelForMigration).toHaveBeenCalledWith(1, 'pty-failed')
|
||||
expect(harness.dependencies.closeProvider).not.toHaveBeenCalled()
|
||||
expect(harness.intake.getAcceptedSourceCheckpoints(1)).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ id: 'pty-failed', acceptedSourceEndSu: 0 }),
|
||||
expect.objectContaining({ id: 'pty-sibling', acceptedSourceEndSu: 4 })
|
||||
])
|
||||
)
|
||||
expect(harness.intake.getDebugSnapshot().model).toMatchObject({
|
||||
sourceUnits: 0,
|
||||
bytes: 0,
|
||||
pressureFrames: 0,
|
||||
migratingPtys: 2
|
||||
})
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
|
||||
harness.completions[1]!.resolve()
|
||||
await Promise.resolve()
|
||||
expect(resetModelForMigration).toHaveBeenCalledTimes(1)
|
||||
expect(harness.dependencies.closeProvider).not.toHaveBeenCalled()
|
||||
expect(
|
||||
harness.intake
|
||||
.getAcceptedSourceCheckpoints(1)
|
||||
.find((checkpoint) => checkpoint.id === 'pty-failed')?.acceptedSourceEndSu
|
||||
).toBe(0)
|
||||
|
||||
const unrelated = harness.intake.acceptData(
|
||||
event({
|
||||
id: 'pty-unrelated',
|
||||
providerGeneration: 2,
|
||||
ptyIncarnation: 'incarnation-unrelated'
|
||||
})
|
||||
)
|
||||
harness.completions[2]!.resolve()
|
||||
await expect(unrelated).resolves.toMatchObject({ providerGeneration: 2 })
|
||||
expect(harness.dependencies.closeProvider).not.toHaveBeenCalled()
|
||||
harness.intake.closeGeneration(1, 'connection_lost')
|
||||
harness.intake.closeGeneration(2, 'connection_lost')
|
||||
expect(harness.intake.getDebugSnapshot().model).toMatchObject({
|
||||
sourceUnits: 0,
|
||||
bytes: 0,
|
||||
migratingPtys: 0
|
||||
})
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,134 @@
|
||||
import type {
|
||||
SshPtyAcceptedSourceCheckpoint,
|
||||
SshPtyOutputSourceObligations
|
||||
} from './ssh-pty-output-source-obligations'
|
||||
import type { SshPtyModelAdmissionKey } from './ssh-pty-model-admission-contract'
|
||||
import type { SshPtyModelAdmission } from './ssh-pty-model-admission'
|
||||
|
||||
export const SSH_PTY_MODEL_MIGRATION_TIMEOUT_MS = 10_000
|
||||
|
||||
export type SshPtyOutputMigrationResult =
|
||||
| Readonly<{ status: 'settled'; checkpoint: SshPtyAcceptedSourceCheckpoint }>
|
||||
| Readonly<{ status: 'checkpoint-unavailable'; reason: 'completion-failed' | 'timeout' }>
|
||||
|
||||
export type SshPtyOutputGenerationMigration = Readonly<{
|
||||
byPty: ReadonlyMap<string, Promise<SshPtyOutputMigrationResult>>
|
||||
completion: Promise<void>
|
||||
}>
|
||||
|
||||
export type SshPtyTrackedModelAdmission = {
|
||||
readonly key: SshPtyModelAdmissionKey
|
||||
started: boolean
|
||||
completion?: Promise<unknown>
|
||||
}
|
||||
|
||||
export class SshPtyOutputModelMigration {
|
||||
private readonly pendingByPty = new Map<string, Set<SshPtyTrackedModelAdmission>>()
|
||||
|
||||
constructor(
|
||||
private readonly admission: SshPtyModelAdmission,
|
||||
private readonly sourceObligations: SshPtyOutputSourceObligations,
|
||||
private readonly resetModel: (providerGeneration: number, ptyId: string) => void
|
||||
) {}
|
||||
|
||||
track(record: SshPtyTrackedModelAdmission): void {
|
||||
const id = migrationKey(record.key)
|
||||
const records = this.pendingByPty.get(id) ?? new Set<SshPtyTrackedModelAdmission>()
|
||||
records.add(record)
|
||||
this.pendingByPty.set(id, records)
|
||||
record.completion?.then(
|
||||
() => this.remove(id, record),
|
||||
() => this.remove(id, record)
|
||||
)
|
||||
}
|
||||
|
||||
beginGeneration(
|
||||
providerGeneration: number,
|
||||
timeoutMs = SSH_PTY_MODEL_MIGRATION_TIMEOUT_MS
|
||||
): SshPtyOutputGenerationMigration {
|
||||
const checkpoints = this.sourceObligations.acceptedCheckpoints(providerGeneration)
|
||||
const keys = checkpoints.map((checkpoint) => ({
|
||||
ptyId: checkpoint.id,
|
||||
providerGeneration
|
||||
}))
|
||||
for (const key of keys) {
|
||||
this.admission.beginMigration(key)
|
||||
}
|
||||
const byPty = new Map<string, Promise<SshPtyOutputMigrationResult>>()
|
||||
for (const key of keys) {
|
||||
byPty.set(key.ptyId, this.settlePty(key, timeoutMs))
|
||||
}
|
||||
const completion = Promise.allSettled(byPty.values()).then(() => {})
|
||||
return Object.freeze({ byPty, completion })
|
||||
}
|
||||
|
||||
private async settlePty(
|
||||
key: SshPtyModelAdmissionKey,
|
||||
timeoutMs: number
|
||||
): Promise<SshPtyOutputMigrationResult> {
|
||||
const running = Array.from(this.pendingByPty.get(migrationKey(key)) ?? []).find(
|
||||
(record) => record.started
|
||||
)
|
||||
if (!running?.completion) {
|
||||
return this.settledCheckpoint(key)
|
||||
}
|
||||
let timer: ReturnType<typeof setTimeout> | undefined
|
||||
const timeout = new Promise<'timeout'>((resolve) => {
|
||||
timer = setTimeout(() => resolve('timeout'), normalizedTimeout(timeoutMs))
|
||||
timer.unref?.()
|
||||
})
|
||||
try {
|
||||
const outcome = await Promise.race([
|
||||
running.completion.then(
|
||||
() => 'settled' as const,
|
||||
() => 'failed' as const
|
||||
),
|
||||
timeout
|
||||
])
|
||||
if (outcome === 'settled') {
|
||||
return this.settledCheckpoint(key)
|
||||
}
|
||||
const reason = outcome === 'timeout' ? 'timeout' : 'completion-failed'
|
||||
this.resetModel(key.providerGeneration, key.ptyId)
|
||||
this.admission.cancelPty(key, `ssh_model_migration_${reason}`)
|
||||
if (outcome === 'timeout') {
|
||||
await running.completion.catch(() => {})
|
||||
}
|
||||
return Object.freeze({ status: 'checkpoint-unavailable', reason })
|
||||
} finally {
|
||||
if (timer) {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private settledCheckpoint(key: SshPtyModelAdmissionKey): SshPtyOutputMigrationResult {
|
||||
const checkpoint = this.sourceObligations.acceptedCheckpoint(key)
|
||||
if (!checkpoint) {
|
||||
this.resetModel(key.providerGeneration, key.ptyId)
|
||||
return Object.freeze({
|
||||
status: 'checkpoint-unavailable',
|
||||
reason: 'completion-failed'
|
||||
})
|
||||
}
|
||||
return Object.freeze({ status: 'settled', checkpoint })
|
||||
}
|
||||
|
||||
private remove(id: string, record: SshPtyTrackedModelAdmission): void {
|
||||
const records = this.pendingByPty.get(id)
|
||||
records?.delete(record)
|
||||
if (records?.size === 0) {
|
||||
this.pendingByPty.delete(id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function migrationKey(key: SshPtyModelAdmissionKey): string {
|
||||
return `${key.providerGeneration}\0${key.ptyId}`
|
||||
}
|
||||
|
||||
function normalizedTimeout(timeoutMs: number): number {
|
||||
return Number.isFinite(timeoutMs) && timeoutMs >= 0
|
||||
? Math.floor(timeoutMs)
|
||||
: SSH_PTY_MODEL_MIGRATION_TIMEOUT_MS
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
import {
|
||||
ptySourceDeliveryKey,
|
||||
samePtySourceDelivery,
|
||||
type PtySourceDeliveryIdentity,
|
||||
type PtySourceSpan
|
||||
} from '../../shared/pty-source-credit-contract'
|
||||
import type { RemoteTerminalSourceRangeConsumerHooks } from '../runtime/remote-terminal-source-range-consumer'
|
||||
import type { DesktopProjectionSpan } from './ssh-pty-legacy-projection'
|
||||
import type {
|
||||
SshPtyOutputDataEvent,
|
||||
SshPtyOutputExitEvent,
|
||||
SshPtyOutputIntakeDependencies,
|
||||
SshPtySourceCancellationProof,
|
||||
SshPtySourceCancellationRequest
|
||||
} from './ssh-pty-output-intake-contract'
|
||||
import { SshPtyRemoteSourceRangeConsumers } from './ssh-pty-remote-source-range-consumers'
|
||||
import type { SshPtySourceAdmissionReservation } from './ssh-pty-source-obligation-contract'
|
||||
import { SshPtySourceObligationCoordinator } from './ssh-pty-source-obligation-coordinator'
|
||||
|
||||
export type SshPtyOutputSourceReservation = Readonly<{
|
||||
admission: SshPtySourceAdmissionReservation
|
||||
span: PtySourceSpan
|
||||
}>
|
||||
|
||||
export type SshPtySourceCancellationProofCommit = Readonly<{
|
||||
identity: PtySourceDeliveryIdentity
|
||||
proof: SshPtySourceCancellationProof
|
||||
}>
|
||||
|
||||
export type SshPtyAcceptedSourceCheckpoint = Readonly<{
|
||||
id: string
|
||||
providerGeneration: number
|
||||
clientGeneration: number
|
||||
ownerGeneration: number
|
||||
ptyIncarnation: string
|
||||
deliveryToken: string
|
||||
acceptedSourceEndSu: number
|
||||
}>
|
||||
|
||||
export class SshPtyOutputSourceObligations {
|
||||
private readonly coordinator: SshPtySourceObligationCoordinator
|
||||
private readonly remoteConsumers: SshPtyRemoteSourceRangeConsumers
|
||||
private readonly openedTokens = new Set<string>()
|
||||
private readonly identityByPty = new Map<string, PtySourceDeliveryIdentity>()
|
||||
|
||||
constructor(publish: SshPtyOutputDataEventPublisher | undefined) {
|
||||
this.coordinator = new SshPtySourceObligationCoordinator({
|
||||
onTokenClosed: (identity) => this.removeIdentity(identity),
|
||||
publish:
|
||||
publish ??
|
||||
((_providerGeneration, _batch, onSettled) =>
|
||||
onSettled({ ok: false, error: new Error('SSH PTY source ACK publisher unavailable') }))
|
||||
})
|
||||
this.remoteConsumers = new SshPtyRemoteSourceRangeConsumers(this.coordinator)
|
||||
}
|
||||
|
||||
get remoteHooks(): RemoteTerminalSourceRangeConsumerHooks {
|
||||
return this.remoteConsumers.hooks
|
||||
}
|
||||
|
||||
reserve(
|
||||
event: SshPtyOutputDataEvent,
|
||||
projection: DesktopProjectionSpan
|
||||
): SshPtyOutputSourceReservation {
|
||||
const span = this.toSourceSpan(event, projection)
|
||||
const identity = this.sourceIdentity(span)
|
||||
const tokenKey = ptySourceDeliveryKey(identity)
|
||||
if (!this.openedTokens.has(tokenKey)) {
|
||||
this.coordinator.open(identity, span.sourceStartSu)
|
||||
this.openedTokens.add(tokenKey)
|
||||
this.identityByPty.set(this.ptyKey(event), identity)
|
||||
}
|
||||
return Object.freeze({
|
||||
span,
|
||||
admission: this.coordinator.reserve(identity, span, [
|
||||
'model',
|
||||
'desktop',
|
||||
...this.remoteConsumers.requiredConsumers(event.id)
|
||||
])
|
||||
})
|
||||
}
|
||||
|
||||
commit(
|
||||
reservation: SshPtyOutputSourceReservation,
|
||||
ptyId: string,
|
||||
modelSequenceEnd: number
|
||||
): void {
|
||||
this.coordinator.commit(reservation.admission)
|
||||
this.remoteConsumers.trackSpan(
|
||||
ptyId,
|
||||
reservation.span.spanId,
|
||||
reservation.admission.requiredConsumers,
|
||||
modelSequenceEnd
|
||||
)
|
||||
}
|
||||
|
||||
rollback(reservation: SshPtyOutputSourceReservation): boolean {
|
||||
return (
|
||||
this.coordinator.rollback(reservation.admission) ||
|
||||
this.coordinator.rollbackCommitted(reservation.admission)
|
||||
)
|
||||
}
|
||||
|
||||
settleModel(span: PtySourceSpan): void {
|
||||
this.coordinator.settle({
|
||||
identity: span,
|
||||
spanId: span.spanId,
|
||||
consumer: 'model',
|
||||
reason: 'model-accepted'
|
||||
})
|
||||
}
|
||||
|
||||
settleDesktop(span: DesktopProjectionSpan, reason: string): void {
|
||||
this.coordinator.settle({
|
||||
identity: span,
|
||||
spanId: span.spanId,
|
||||
consumer: 'desktop',
|
||||
reason
|
||||
})
|
||||
}
|
||||
|
||||
transferDesktop(span: DesktopProjectionSpan, reason: string): void {
|
||||
const transition = {
|
||||
identity: span,
|
||||
spanId: span.spanId,
|
||||
consumer: 'desktop' as const,
|
||||
reason
|
||||
}
|
||||
if (this.coordinator.beginTransfer(transition, 'model')) {
|
||||
this.coordinator.commitTransfer(transition)
|
||||
}
|
||||
}
|
||||
|
||||
sealPty(event: SshPtyOutputExitEvent): void {
|
||||
const identity = this.identityByPty.get(this.ptyKey(event))
|
||||
if (identity) {
|
||||
this.coordinator.seal(identity)
|
||||
}
|
||||
}
|
||||
|
||||
markExitPublished(event: SshPtyOutputExitEvent): void {
|
||||
const identity = this.identityByPty.get(this.ptyKey(event))
|
||||
if (identity) {
|
||||
this.coordinator.markExitPublished(identity)
|
||||
}
|
||||
}
|
||||
|
||||
whenPtyTerminal(event: SshPtyOutputExitEvent): Promise<void> {
|
||||
const identity = this.identityByPty.get(this.ptyKey(event))
|
||||
return identity ? this.coordinator.whenTerminal(identity) : Promise.resolve()
|
||||
}
|
||||
|
||||
async requestPtyCancellationProof(
|
||||
event: SshPtyOutputExitEvent,
|
||||
cancel: (request: SshPtySourceCancellationRequest) => Promise<SshPtySourceCancellationProof>
|
||||
): Promise<SshPtySourceCancellationProofCommit | null> {
|
||||
const identity = this.identityByPty.get(this.ptyKey(event))
|
||||
if (!identity) {
|
||||
return null
|
||||
}
|
||||
const request = this.coordinator.beginExitTimeout(identity)
|
||||
const proof = await cancel(request)
|
||||
return Object.freeze({ identity, proof })
|
||||
}
|
||||
|
||||
commitPtyCancellationProof(commit: SshPtySourceCancellationProofCommit): void {
|
||||
this.coordinator.applyCancellationProof(commit.identity, commit.proof)
|
||||
}
|
||||
|
||||
applyCancellationProof(
|
||||
event: SshPtyOutputExitEvent,
|
||||
proof: SshPtySourceCancellationProof
|
||||
): boolean {
|
||||
const identity = this.identityByPty.get(this.ptyKey(event))
|
||||
if (!identity) {
|
||||
return false
|
||||
}
|
||||
this.coordinator.applyCancellationProof(identity, proof)
|
||||
return true
|
||||
}
|
||||
|
||||
applyRecoveryCancellationProof(
|
||||
event: SshPtyOutputExitEvent,
|
||||
proof: SshPtySourceCancellationProof
|
||||
): void {
|
||||
const identity = this.identityByPty.get(this.ptyKey(event))
|
||||
if (identity) {
|
||||
this.coordinator.applyRecoveryCancellationProof(identity, proof)
|
||||
}
|
||||
}
|
||||
|
||||
closeGeneration(providerGeneration: number, reason: string): void {
|
||||
this.remoteConsumers.closeGeneration(providerGeneration, reason)
|
||||
this.coordinator.closeGeneration(providerGeneration, reason)
|
||||
const prefix = `${providerGeneration}\0`
|
||||
for (const key of this.openedTokens) {
|
||||
if (key.startsWith(prefix)) {
|
||||
this.openedTokens.delete(key)
|
||||
}
|
||||
}
|
||||
for (const [key, identity] of this.identityByPty) {
|
||||
if (identity.providerGeneration === providerGeneration) {
|
||||
this.identityByPty.delete(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
acceptedCheckpoints(providerGeneration: number): readonly SshPtyAcceptedSourceCheckpoint[] {
|
||||
const checkpoints: SshPtyAcceptedSourceCheckpoint[] = []
|
||||
for (const identity of this.identityByPty.values()) {
|
||||
if (identity.providerGeneration !== providerGeneration) {
|
||||
continue
|
||||
}
|
||||
checkpoints.push(
|
||||
Object.freeze({
|
||||
id: identity.id,
|
||||
providerGeneration: identity.providerGeneration,
|
||||
clientGeneration: identity.clientGeneration,
|
||||
ownerGeneration: identity.ownerGeneration,
|
||||
ptyIncarnation: identity.ptyIncarnation,
|
||||
deliveryToken: identity.deliveryToken,
|
||||
acceptedSourceEndSu: this.coordinator.modelAcceptedEnd(identity)
|
||||
})
|
||||
)
|
||||
}
|
||||
return Object.freeze(checkpoints)
|
||||
}
|
||||
|
||||
acceptedCheckpoint(key: {
|
||||
ptyId: string
|
||||
providerGeneration: number
|
||||
}): SshPtyAcceptedSourceCheckpoint | null {
|
||||
return (
|
||||
this.acceptedCheckpoints(key.providerGeneration).find(
|
||||
(checkpoint) => checkpoint.id === key.ptyId
|
||||
) ?? null
|
||||
)
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
this.coordinator.dispose()
|
||||
}
|
||||
|
||||
getDebugSnapshot(): Readonly<{ openedTokens: number; ptyIdentities: number }> {
|
||||
return Object.freeze({
|
||||
openedTokens: this.openedTokens.size,
|
||||
ptyIdentities: this.identityByPty.size
|
||||
})
|
||||
}
|
||||
|
||||
private toSourceSpan(
|
||||
event: SshPtyOutputDataEvent,
|
||||
projection: DesktopProjectionSpan
|
||||
): PtySourceSpan {
|
||||
return Object.freeze({
|
||||
id: projection.id,
|
||||
providerGeneration: event.providerGeneration,
|
||||
clientGeneration: projection.clientGeneration,
|
||||
ownerGeneration: projection.ownerGeneration,
|
||||
ptyIncarnation: event.ptyIncarnation,
|
||||
deliveryToken: projection.deliveryToken,
|
||||
spanId: projection.spanId,
|
||||
sourceStartSu: projection.sourceStartSu,
|
||||
sourceEndSu: projection.sourceEndSu,
|
||||
displayStart: projection.displayStart,
|
||||
displayEnd: projection.displayEnd,
|
||||
splittable: projection.splittable,
|
||||
transform: projection.transform,
|
||||
data: event.data
|
||||
})
|
||||
}
|
||||
|
||||
private sourceIdentity(source: PtySourceSpan): PtySourceDeliveryIdentity {
|
||||
return source
|
||||
}
|
||||
|
||||
private ptyKey(event: {
|
||||
id: string
|
||||
providerGeneration: number
|
||||
ptyIncarnation: string
|
||||
}): string {
|
||||
return `${event.providerGeneration}\0${event.id}\0${event.ptyIncarnation}`
|
||||
}
|
||||
|
||||
private removeIdentity(identity: PtySourceDeliveryIdentity): void {
|
||||
this.openedTokens.delete(ptySourceDeliveryKey(identity))
|
||||
for (const [key, candidate] of this.identityByPty) {
|
||||
if (samePtySourceDelivery(candidate, identity)) {
|
||||
this.identityByPty.delete(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type SshPtyOutputDataEventPublisher = NonNullable<
|
||||
SshPtyOutputIntakeDependencies['publishSourceAck']
|
||||
>
|
||||
@@ -0,0 +1,112 @@
|
||||
import type { ProjectionRecord } from './ssh-pty-legacy-projection-record'
|
||||
|
||||
type ProjectionTerminalWaiter = {
|
||||
providerGeneration: number
|
||||
ptyIncarnation: string
|
||||
resolve: () => void
|
||||
}
|
||||
|
||||
function matchesProjection(
|
||||
record: ProjectionRecord | undefined,
|
||||
providerGeneration: number,
|
||||
ptyIncarnation: string
|
||||
): boolean {
|
||||
const identity = record?.semantics.identity
|
||||
return (
|
||||
identity?.providerGeneration === providerGeneration &&
|
||||
identity.ptyIncarnation === ptyIncarnation
|
||||
)
|
||||
}
|
||||
|
||||
export function unpublishedProjectionIds(
|
||||
records: ReadonlyMap<string, ProjectionRecord>,
|
||||
ids: readonly string[],
|
||||
providerGeneration: number,
|
||||
ptyIncarnation: string
|
||||
): string[] {
|
||||
return ids.filter((id) => {
|
||||
const record = records.get(id)
|
||||
return (
|
||||
record?.state === 'committed' && matchesProjection(record, providerGeneration, ptyIncarnation)
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
export function projectionHasOpen(
|
||||
records: ReadonlyMap<string, ProjectionRecord>,
|
||||
idsByPty: ReadonlyMap<string, readonly string[]>,
|
||||
ptyId: string
|
||||
): (providerGeneration: number, ptyIncarnation: string) => boolean {
|
||||
return (providerGeneration, ptyIncarnation) =>
|
||||
(idsByPty.get(ptyId) ?? []).some((id) =>
|
||||
matchesProjection(records.get(id), providerGeneration, ptyIncarnation)
|
||||
)
|
||||
}
|
||||
|
||||
export function resolveProjectionTerminality(
|
||||
terminality: SshPtyProjectionTerminality,
|
||||
records: ReadonlyMap<string, ProjectionRecord>,
|
||||
idsByPty: ReadonlyMap<string, readonly string[]>,
|
||||
ptyId: string
|
||||
): void {
|
||||
terminality.resolve(ptyId, projectionHasOpen(records, idsByPty, ptyId))
|
||||
}
|
||||
|
||||
export class SshPtyProjectionTerminality {
|
||||
private readonly waitersByPty = new Map<string, ProjectionTerminalWaiter[]>()
|
||||
|
||||
whenTerminal(
|
||||
ptyId: string,
|
||||
providerGeneration: number,
|
||||
ptyIncarnation: string,
|
||||
hasOpen: (providerGeneration: number, ptyIncarnation: string) => boolean
|
||||
): Promise<void> {
|
||||
if (!hasOpen(providerGeneration, ptyIncarnation)) {
|
||||
return Promise.resolve()
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
const waiters = this.waitersByPty.get(ptyId) ?? []
|
||||
waiters.push({ providerGeneration, ptyIncarnation, resolve })
|
||||
this.waitersByPty.set(ptyId, waiters)
|
||||
})
|
||||
}
|
||||
|
||||
resolve(
|
||||
ptyId: string,
|
||||
hasOpen: (providerGeneration: number, ptyIncarnation: string) => boolean
|
||||
): void {
|
||||
const waiters = this.waitersByPty.get(ptyId)
|
||||
if (!waiters) {
|
||||
return
|
||||
}
|
||||
const pending = waiters.filter((waiter) =>
|
||||
hasOpen(waiter.providerGeneration, waiter.ptyIncarnation)
|
||||
)
|
||||
for (const waiter of waiters) {
|
||||
if (!pending.includes(waiter)) {
|
||||
waiter.resolve()
|
||||
}
|
||||
}
|
||||
if (pending.length > 0) {
|
||||
this.waitersByPty.set(ptyId, pending)
|
||||
} else {
|
||||
this.waitersByPty.delete(ptyId)
|
||||
}
|
||||
}
|
||||
|
||||
closeGeneration(providerGeneration: number): void {
|
||||
for (const [ptyId, waiters] of this.waitersByPty) {
|
||||
const pending = waiters.filter((waiter) => waiter.providerGeneration !== providerGeneration)
|
||||
for (const waiter of waiters) {
|
||||
if (waiter.providerGeneration === providerGeneration) {
|
||||
waiter.resolve()
|
||||
}
|
||||
}
|
||||
if (pending.length > 0) {
|
||||
this.waitersByPty.set(ptyId, pending)
|
||||
} else {
|
||||
this.waitersByPty.delete(ptyId)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,451 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type {
|
||||
PtySourceDeliveryIdentity,
|
||||
PtySourceSpan
|
||||
} from '../../shared/pty-source-credit-contract'
|
||||
import type { TerminalOutputSourceRange } from '../../shared/terminal-output-source-range'
|
||||
import { SshPtyRemoteSourceRangeConsumers } from './ssh-pty-remote-source-range-consumers'
|
||||
import { SshPtySourceObligationCoordinator } from './ssh-pty-source-obligation-coordinator'
|
||||
|
||||
const identity: PtySourceDeliveryIdentity = {
|
||||
id: 'pty-1',
|
||||
providerGeneration: 1,
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-1'
|
||||
}
|
||||
|
||||
function span(spanId: string, overrides: Partial<PtySourceSpan> = {}): PtySourceSpan {
|
||||
return {
|
||||
...identity,
|
||||
spanId,
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4,
|
||||
displayStart: 0,
|
||||
displayEnd: 4,
|
||||
data: 'data',
|
||||
splittable: true,
|
||||
transform: { transformed: false, rawLengthSu: 4, scalarSafe: true },
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
function range(
|
||||
spanId: string,
|
||||
overrides: Partial<TerminalOutputSourceRange> = {}
|
||||
): TerminalOutputSourceRange {
|
||||
return {
|
||||
id: 'pty-1',
|
||||
spanId,
|
||||
providerGeneration: 1,
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-1',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4,
|
||||
displayStart: 0,
|
||||
displayEnd: 4,
|
||||
splittable: true,
|
||||
transform: { transformed: false, rawLengthSu: 4, scalarSafe: true },
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
function createCoordinator(): SshPtySourceObligationCoordinator {
|
||||
return new SshPtySourceObligationCoordinator({
|
||||
publish: vi.fn(),
|
||||
schedule: vi.fn(() => 1 as unknown as ReturnType<typeof setTimeout>),
|
||||
cancelSchedule: vi.fn()
|
||||
})
|
||||
}
|
||||
|
||||
describe('SshPtyRemoteSourceRangeConsumers', () => {
|
||||
it('snapshots membership and settles only the current stream generation', () => {
|
||||
const ledger = createCoordinator()
|
||||
const progress = vi.fn()
|
||||
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger, progress)
|
||||
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
|
||||
ledger.open(identity)
|
||||
expect(consumers.hooks.attach(stream)).toBe(true)
|
||||
const sourceSpan = span('span-1')
|
||||
const reservation = ledger.reserve(identity, sourceSpan, [
|
||||
'model',
|
||||
...consumers.requiredConsumers('pty-1')
|
||||
])
|
||||
ledger.commit(reservation)
|
||||
consumers.trackSpan('pty-1', sourceSpan.spanId, reservation.requiredConsumers, 4)
|
||||
|
||||
consumers.hooks.settle({ ...stream, streamGeneration: 'stale' }, [range('span-1')])
|
||||
expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('open')
|
||||
consumers.hooks.settle(stream, [range('span-1')])
|
||||
expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('settled')
|
||||
expect(progress).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it.each(['headless', 'renderer'] as const)(
|
||||
'commits remaining mappings only after a %s snapshot publication',
|
||||
(source) => {
|
||||
const ledger = createCoordinator()
|
||||
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
|
||||
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
|
||||
ledger.open(identity)
|
||||
consumers.hooks.attach(stream)
|
||||
const reservation = ledger.reserve(identity, span('span-1'), [
|
||||
'model',
|
||||
...consumers.requiredConsumers('pty-1')
|
||||
])
|
||||
ledger.commit(reservation)
|
||||
consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4)
|
||||
|
||||
const replacement = consumers.hooks.reserveReplacement(stream, 4, 'initial-snapshot')
|
||||
|
||||
expect(replacement).not.toBeNull()
|
||||
expect(ledger.obligation('span-1', 'remote:consumer-1')).toMatchObject({
|
||||
state: 'transferring',
|
||||
to: 'remote:snapshot:consumer-1'
|
||||
})
|
||||
expect(consumers.hooks.commitReplacement(replacement!, { source, seq: 3 })).toBe(false)
|
||||
expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('transferring')
|
||||
expect(consumers.hooks.commitReplacement(replacement!, { source, seq: 4 })).toBe(true)
|
||||
expect(ledger.obligation('span-1', 'remote:consumer-1')).toMatchObject({
|
||||
state: 'transferred',
|
||||
to: 'remote:snapshot:consumer-1'
|
||||
})
|
||||
expect(consumers.requiredConsumers('pty-1')).toEqual(['remote:consumer-1'])
|
||||
}
|
||||
)
|
||||
|
||||
it('reserves only spans covered by the authoritative snapshot sequence', () => {
|
||||
const ledger = createCoordinator()
|
||||
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
|
||||
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
|
||||
ledger.open(identity)
|
||||
consumers.hooks.attach(stream)
|
||||
const covered = ledger.reserve(identity, span('span-covered'), [
|
||||
'model',
|
||||
...consumers.requiredConsumers('pty-1')
|
||||
])
|
||||
ledger.commit(covered)
|
||||
consumers.trackSpan('pty-1', 'span-covered', covered.requiredConsumers, 4)
|
||||
const trailingSpan = {
|
||||
...span('span-trailing'),
|
||||
sourceStartSu: 4,
|
||||
sourceEndSu: 8,
|
||||
displayStart: 4,
|
||||
displayEnd: 8
|
||||
}
|
||||
const trailing = ledger.reserve(identity, trailingSpan, [
|
||||
'model',
|
||||
...consumers.requiredConsumers('pty-1')
|
||||
])
|
||||
ledger.commit(trailing)
|
||||
consumers.trackSpan('pty-1', 'span-trailing', trailing.requiredConsumers, 8)
|
||||
|
||||
const replacement = consumers.hooks.reserveReplacement(stream, 4, 'initial-snapshot')
|
||||
|
||||
expect(replacement).not.toBeNull()
|
||||
expect(ledger.obligation('span-covered', 'remote:consumer-1')).toMatchObject({
|
||||
state: 'transferring'
|
||||
})
|
||||
expect(ledger.obligation('span-trailing', 'remote:consumer-1')).toMatchObject({
|
||||
state: 'open'
|
||||
})
|
||||
})
|
||||
|
||||
it('rolls a failed replacement publication back to the live stream obligation', () => {
|
||||
const ledger = createCoordinator()
|
||||
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
|
||||
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
|
||||
ledger.open(identity)
|
||||
consumers.hooks.attach(stream)
|
||||
const reservation = ledger.reserve(identity, span('span-1'), [
|
||||
'model',
|
||||
...consumers.requiredConsumers('pty-1')
|
||||
])
|
||||
ledger.commit(reservation)
|
||||
consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4)
|
||||
|
||||
const replacement = consumers.hooks.reserveReplacement(stream, 4, 'initial-snapshot')
|
||||
expect(consumers.hooks.rollbackReplacement(replacement!, 'snapshot-write-failed')).toBe(true)
|
||||
|
||||
expect(ledger.obligation('span-1', 'remote:consumer-1')).toMatchObject({
|
||||
state: 'open'
|
||||
})
|
||||
})
|
||||
|
||||
it('rolls back every transfer when replacement reservation fails partway', () => {
|
||||
const ledger = createCoordinator()
|
||||
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
|
||||
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
|
||||
ledger.open(identity)
|
||||
consumers.hooks.attach(stream)
|
||||
const first = ledger.reserve(identity, span('span-1'), [
|
||||
'model',
|
||||
...consumers.requiredConsumers('pty-1')
|
||||
])
|
||||
ledger.commit(first)
|
||||
consumers.trackSpan('pty-1', 'span-1', first.requiredConsumers, 4)
|
||||
const secondSpan = span('span-2', {
|
||||
sourceStartSu: 4,
|
||||
sourceEndSu: 8,
|
||||
displayStart: 4,
|
||||
displayEnd: 8
|
||||
})
|
||||
const second = ledger.reserve(identity, secondSpan, [
|
||||
'model',
|
||||
...consumers.requiredConsumers('pty-1')
|
||||
])
|
||||
ledger.commit(second)
|
||||
consumers.trackSpan('pty-1', 'span-2', second.requiredConsumers, 8)
|
||||
const beginTransfer = ledger.beginTransfer.bind(ledger)
|
||||
vi.spyOn(ledger, 'beginTransfer').mockImplementation((transition, replacement) => {
|
||||
if (transition.spanId === 'span-2') {
|
||||
throw new Error('injected partial reserve failure')
|
||||
}
|
||||
return beginTransfer(transition, replacement)
|
||||
})
|
||||
|
||||
expect(() => consumers.hooks.reserveReplacement(stream, 8, 'initial-snapshot')).toThrowError(
|
||||
'injected partial reserve failure'
|
||||
)
|
||||
expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('open')
|
||||
expect(ledger.obligation('span-2', 'remote:consumer-1').state).toBe('open')
|
||||
})
|
||||
|
||||
it('rejects a replacement whose exact transfer state changed before commit', () => {
|
||||
const ledger = createCoordinator()
|
||||
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
|
||||
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
|
||||
ledger.open(identity)
|
||||
consumers.hooks.attach(stream)
|
||||
const sourceSpan = span('span-1')
|
||||
const admission = ledger.reserve(identity, sourceSpan, [
|
||||
'model',
|
||||
...consumers.requiredConsumers('pty-1')
|
||||
])
|
||||
ledger.commit(admission)
|
||||
consumers.trackSpan('pty-1', 'span-1', admission.requiredConsumers, 4)
|
||||
const replacement = consumers.hooks.reserveReplacement(stream, 4, 'initial-snapshot')!
|
||||
const transition = {
|
||||
identity: sourceSpan,
|
||||
spanId: sourceSpan.spanId,
|
||||
consumer: 'remote:consumer-1' as const,
|
||||
reason: 'concurrent-replacement'
|
||||
}
|
||||
expect(ledger.rollbackTransfer(transition)).toBe(true)
|
||||
expect(ledger.beginTransfer(transition, 'remote:snapshot:consumer-1')).toBe(true)
|
||||
|
||||
expect(consumers.hooks.commitReplacement(replacement, { source: 'headless', seq: 4 })).toBe(
|
||||
false
|
||||
)
|
||||
expect(consumers.hooks.rollbackReplacement(replacement, 'commit-rejected')).toBe(false)
|
||||
expect(ledger.obligation('span-1', 'remote:consumer-1')).toMatchObject({
|
||||
state: 'transferring',
|
||||
reason: 'concurrent-replacement'
|
||||
})
|
||||
})
|
||||
|
||||
it('settles a split source span only after its complete ordered range is acknowledged', () => {
|
||||
const ledger = createCoordinator()
|
||||
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
|
||||
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
|
||||
ledger.open(identity)
|
||||
consumers.hooks.attach(stream)
|
||||
const reservation = ledger.reserve(identity, span('span-1'), [
|
||||
'model',
|
||||
...consumers.requiredConsumers('pty-1')
|
||||
])
|
||||
ledger.commit(reservation)
|
||||
consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4)
|
||||
|
||||
consumers.hooks.settle(stream, [
|
||||
range('span-1', {
|
||||
sourceEndSu: 2,
|
||||
displayEnd: 2,
|
||||
transform: { transformed: false, rawLengthSu: 2, scalarSafe: true }
|
||||
})
|
||||
])
|
||||
expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('open')
|
||||
|
||||
consumers.hooks.settle(stream, [
|
||||
range('span-1', {
|
||||
sourceStartSu: 2,
|
||||
sourceEndSu: 4,
|
||||
displayStart: 2,
|
||||
displayEnd: 4,
|
||||
transform: { transformed: false, rawLengthSu: 2, scalarSafe: true }
|
||||
})
|
||||
])
|
||||
expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('settled')
|
||||
})
|
||||
|
||||
it('cancels an admitted span on detach without minting a snapshot owner', () => {
|
||||
const ledger = createCoordinator()
|
||||
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
|
||||
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
|
||||
ledger.open(identity)
|
||||
consumers.hooks.attach(stream)
|
||||
const reservation = ledger.reserve(identity, span('span-1'), [
|
||||
'model',
|
||||
...consumers.requiredConsumers('pty-1')
|
||||
])
|
||||
ledger.commit(reservation)
|
||||
consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4)
|
||||
|
||||
consumers.hooks.cancel(stream, [], 'stream-detached')
|
||||
|
||||
expect(ledger.obligation('span-1', 'remote:consumer-1')).toMatchObject({
|
||||
state: 'canceled',
|
||||
reason: 'stream-detached'
|
||||
})
|
||||
expect(consumers.requiredConsumers('pty-1')).toEqual([])
|
||||
})
|
||||
|
||||
it('rolls back a pending replacement before disconnect cancellation', () => {
|
||||
const ledger = createCoordinator()
|
||||
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
|
||||
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
|
||||
ledger.open(identity)
|
||||
consumers.hooks.attach(stream)
|
||||
const reservation = ledger.reserve(identity, span('span-1'), [
|
||||
'model',
|
||||
...consumers.requiredConsumers('pty-1')
|
||||
])
|
||||
ledger.commit(reservation)
|
||||
consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4)
|
||||
const replacement = consumers.hooks.reserveReplacement(stream, 4, 'initial-snapshot')
|
||||
|
||||
consumers.hooks.cancel(stream, [], 'connection-closed')
|
||||
|
||||
expect(consumers.hooks.commitReplacement(replacement!, { source: 'headless', seq: 4 })).toBe(
|
||||
false
|
||||
)
|
||||
expect(ledger.obligation('span-1', 'remote:consumer-1')).toMatchObject({
|
||||
state: 'canceled',
|
||||
reason: 'connection-closed'
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects stale stream generations without changing the current obligation', () => {
|
||||
const ledger = createCoordinator()
|
||||
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
|
||||
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
|
||||
ledger.open(identity)
|
||||
consumers.hooks.attach(stream)
|
||||
const reservation = ledger.reserve(identity, span('span-1'), [
|
||||
'model',
|
||||
...consumers.requiredConsumers('pty-1')
|
||||
])
|
||||
ledger.commit(reservation)
|
||||
consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4)
|
||||
|
||||
expect(() =>
|
||||
consumers.hooks.reserveReplacement(
|
||||
{ ...stream, streamGeneration: 'stale' },
|
||||
4,
|
||||
'initial-snapshot'
|
||||
)
|
||||
).toThrow('stale')
|
||||
expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('open')
|
||||
})
|
||||
|
||||
it('rolls back replacement admission before provider-generation close', () => {
|
||||
const ledger = createCoordinator()
|
||||
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
|
||||
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
|
||||
ledger.open(identity)
|
||||
consumers.hooks.attach(stream)
|
||||
const reservation = ledger.reserve(identity, span('span-1'), [
|
||||
'model',
|
||||
...consumers.requiredConsumers('pty-1')
|
||||
])
|
||||
ledger.commit(reservation)
|
||||
consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4)
|
||||
const replacement = consumers.hooks.reserveReplacement(stream, 4, 'initial-snapshot')
|
||||
|
||||
consumers.closeGeneration(identity.providerGeneration, 'provider-replaced')
|
||||
|
||||
expect(ledger.obligation('span-1', 'remote:consumer-1').state).toBe('open')
|
||||
expect(consumers.hooks.commitReplacement(replacement!, { source: 'headless', seq: 4 })).toBe(
|
||||
false
|
||||
)
|
||||
})
|
||||
|
||||
it('detaches cleanly after cancellation proof reclaims tracked spans', () => {
|
||||
const ledger = createCoordinator()
|
||||
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
|
||||
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
|
||||
ledger.open(identity)
|
||||
consumers.hooks.attach(stream)
|
||||
const reservation = ledger.reserve(identity, span('span-1'), [
|
||||
'model',
|
||||
...consumers.requiredConsumers('pty-1')
|
||||
])
|
||||
ledger.commit(reservation)
|
||||
consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4)
|
||||
ledger.seal(identity)
|
||||
ledger.beginExitTimeout(identity)
|
||||
ledger.applyCancellationProof(identity, { sentEndSu: 4, creditedEndSu: 0 })
|
||||
|
||||
expect(() => consumers.hooks.settle(stream, [range('span-1')])).not.toThrow()
|
||||
expect(() => consumers.hooks.cancel(stream, [], 'stream-detached')).not.toThrow()
|
||||
expect(consumers.requiredConsumers('pty-1')).toEqual([])
|
||||
})
|
||||
|
||||
it('rejects and rolls back a replacement after proof reclaims its covered spans', () => {
|
||||
const ledger = createCoordinator()
|
||||
const consumers = new SshPtyRemoteSourceRangeConsumers(ledger)
|
||||
const stream = { ptyId: 'pty-1', consumerId: 'consumer-1', streamGeneration: 'stream-1' }
|
||||
ledger.open(identity)
|
||||
consumers.hooks.attach(stream)
|
||||
const reservation = ledger.reserve(identity, span('span-1'), [
|
||||
'model',
|
||||
...consumers.requiredConsumers('pty-1')
|
||||
])
|
||||
ledger.commit(reservation)
|
||||
consumers.trackSpan('pty-1', 'span-1', reservation.requiredConsumers, 4)
|
||||
const recoveredIdentity = {
|
||||
...identity,
|
||||
clientGeneration: 3,
|
||||
ownerGeneration: 4,
|
||||
deliveryToken: 'token-2'
|
||||
}
|
||||
ledger.open(recoveredIdentity, 4)
|
||||
const recovered = ledger.reserve(
|
||||
recoveredIdentity,
|
||||
span('span-2', {
|
||||
...recoveredIdentity,
|
||||
sourceStartSu: 4,
|
||||
sourceEndSu: 8,
|
||||
displayStart: 4,
|
||||
displayEnd: 8
|
||||
}),
|
||||
['model', ...consumers.requiredConsumers('pty-1')]
|
||||
)
|
||||
ledger.commit(recovered)
|
||||
consumers.trackSpan('pty-1', 'span-2', recovered.requiredConsumers, 8)
|
||||
const replacement = consumers.hooks.reserveReplacement(stream, 8, 'initial-snapshot')
|
||||
ledger.seal(identity)
|
||||
ledger.beginExitTimeout(identity)
|
||||
ledger.applyCancellationProof(identity, { sentEndSu: 4, creditedEndSu: 0 })
|
||||
|
||||
const queuedOutput = ['covered-output', 'surviving-output']
|
||||
const committed = consumers.hooks.commitReplacement(replacement!, {
|
||||
source: 'headless',
|
||||
seq: 8
|
||||
})
|
||||
if (committed) {
|
||||
queuedOutput.splice(0)
|
||||
}
|
||||
|
||||
expect(committed).toBe(false)
|
||||
expect(queuedOutput).toEqual(['covered-output', 'surviving-output'])
|
||||
expect(consumers.hooks.rollbackReplacement(replacement!, 'commit-rejected')).toBe(true)
|
||||
expect(ledger.obligation('span-2', 'remote:consumer-1').state).toBe('open')
|
||||
expect(consumers.hooks.rollbackReplacement(replacement!, 'commit-rejected')).toBe(false)
|
||||
expect(() => consumers.hooks.cancel(stream, [], 'stream-detached')).not.toThrow()
|
||||
expect(consumers.requiredConsumers('pty-1')).toEqual([])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,285 @@
|
||||
import {
|
||||
sameTerminalOutputSourceIdentity,
|
||||
type TerminalOutputSourceRange
|
||||
} from '../../shared/terminal-output-source-range'
|
||||
import type {
|
||||
RemoteTerminalSourceRangeConsumerHooks,
|
||||
RemoteTerminalSourceRangeStreamIdentity
|
||||
} from '../runtime/remote-terminal-source-range-consumer'
|
||||
import type { PtySourceSpan } from '../../shared/pty-source-credit-contract'
|
||||
import type { SshPtySourceConsumerId } from './ssh-pty-source-obligation-contract'
|
||||
import type { SshPtySourceObligationCoordinator } from './ssh-pty-source-obligation-coordinator'
|
||||
import { SshPtyRemoteSourceRangeReplacements } from './ssh-pty-remote-source-range-replacement'
|
||||
|
||||
function remoteConsumerId(
|
||||
identity: RemoteTerminalSourceRangeStreamIdentity
|
||||
): SshPtySourceConsumerId {
|
||||
return `remote:${identity.consumerId}`
|
||||
}
|
||||
|
||||
function uniqueSpanIds(ranges: readonly TerminalOutputSourceRange[]): string[] {
|
||||
return Array.from(new Set(ranges.map((range) => range.spanId)))
|
||||
}
|
||||
|
||||
type RemoteConsumerState = {
|
||||
streamGeneration: string
|
||||
spans: Map<
|
||||
string,
|
||||
Readonly<{
|
||||
identity: PtySourceSpan
|
||||
modelSequenceEnd: number
|
||||
}>
|
||||
>
|
||||
ackedEndBySpan: Map<string, number>
|
||||
}
|
||||
|
||||
export class SshPtyRemoteSourceRangeConsumers {
|
||||
private readonly consumersByPty = new Map<string, Map<string, RemoteConsumerState>>()
|
||||
private readonly replacements: SshPtyRemoteSourceRangeReplacements
|
||||
|
||||
constructor(
|
||||
private readonly coordinator: SshPtySourceObligationCoordinator,
|
||||
private readonly onProgress: (range: TerminalOutputSourceRange) => void = () => {}
|
||||
) {
|
||||
this.replacements = new SshPtyRemoteSourceRangeReplacements(coordinator)
|
||||
}
|
||||
|
||||
readonly hooks: RemoteTerminalSourceRangeConsumerHooks = {
|
||||
attach: (identity) => this.attach(identity),
|
||||
settle: (identity, ranges) => this.settle(identity, ranges),
|
||||
reserveReplacement: (identity, requiredSeq, reason) =>
|
||||
this.reserveReplacement(identity, requiredSeq, reason),
|
||||
commitReplacement: (reservation, publication) =>
|
||||
this.commitReplacement(reservation, publication),
|
||||
rollbackReplacement: (reservation, reason) => this.rollbackReplacement(reservation, reason),
|
||||
cancel: (identity, ranges, reason) => this.cancel(identity, ranges, reason)
|
||||
}
|
||||
|
||||
requiredConsumers(ptyId: string): readonly SshPtySourceConsumerId[] {
|
||||
return Object.freeze(
|
||||
Array.from(this.consumersByPty.get(ptyId)?.keys() ?? []).map(
|
||||
(consumerId) => `remote:${consumerId}` as const
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
trackSpan(
|
||||
ptyId: string,
|
||||
spanId: string,
|
||||
requiredConsumers: readonly SshPtySourceConsumerId[],
|
||||
modelSequenceEnd: number
|
||||
): void {
|
||||
if (!Number.isSafeInteger(modelSequenceEnd) || modelSequenceEnd < 0) {
|
||||
throw new Error('ssh_remote_source_range_model_sequence_invalid')
|
||||
}
|
||||
for (const [consumerId, state] of this.consumersByPty.get(ptyId) ?? []) {
|
||||
if (requiredConsumers.includes(`remote:${consumerId}`)) {
|
||||
state.spans.set(
|
||||
spanId,
|
||||
Object.freeze({
|
||||
identity: this.coordinator.spanIdentity(spanId),
|
||||
modelSequenceEnd
|
||||
})
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
closeGeneration(providerGeneration: number, reason: string): void {
|
||||
this.replacements.closeGeneration(providerGeneration, reason)
|
||||
for (const consumers of this.consumersByPty.values()) {
|
||||
for (const state of consumers.values()) {
|
||||
for (const [spanId, tracked] of state.spans) {
|
||||
if (tracked.identity.providerGeneration === providerGeneration) {
|
||||
state.spans.delete(spanId)
|
||||
state.ackedEndBySpan.delete(spanId)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private attach(identity: RemoteTerminalSourceRangeStreamIdentity): boolean {
|
||||
const consumers =
|
||||
this.consumersByPty.get(identity.ptyId) ?? new Map<string, RemoteConsumerState>()
|
||||
const current = consumers.get(identity.consumerId)
|
||||
if (current && current.streamGeneration !== identity.streamGeneration) {
|
||||
return false
|
||||
}
|
||||
consumers.set(
|
||||
identity.consumerId,
|
||||
current ?? {
|
||||
streamGeneration: identity.streamGeneration,
|
||||
spans: new Map(),
|
||||
ackedEndBySpan: new Map()
|
||||
}
|
||||
)
|
||||
this.consumersByPty.set(identity.ptyId, consumers)
|
||||
return true
|
||||
}
|
||||
|
||||
private settle(
|
||||
identity: RemoteTerminalSourceRangeStreamIdentity,
|
||||
ranges: readonly TerminalOutputSourceRange[]
|
||||
): void {
|
||||
if (!this.isCurrent(identity)) {
|
||||
return
|
||||
}
|
||||
const state = this.requireState(identity)
|
||||
const consumer = remoteConsumerId(identity)
|
||||
const nextEnds = new Map(state.ackedEndBySpan)
|
||||
const completed = new Set<string>()
|
||||
for (const range of ranges) {
|
||||
const tracked = state.spans.get(range.spanId)
|
||||
if (!tracked) {
|
||||
continue
|
||||
}
|
||||
const source = tracked.identity
|
||||
if (!this.coordinator.hasRetainedSpan(range.spanId)) {
|
||||
state.spans.delete(range.spanId)
|
||||
nextEnds.delete(range.spanId)
|
||||
continue
|
||||
}
|
||||
const currentEnd = nextEnds.get(range.spanId) ?? source.sourceStartSu
|
||||
if (
|
||||
!sameTerminalOutputSourceIdentity(source, range) ||
|
||||
range.sourceStartSu !== currentEnd ||
|
||||
range.sourceEndSu > source.sourceEndSu
|
||||
) {
|
||||
throw new Error('ssh_remote_source_range_settlement_invalid')
|
||||
}
|
||||
nextEnds.set(range.spanId, range.sourceEndSu)
|
||||
if (range.sourceEndSu === source.sourceEndSu) {
|
||||
completed.add(range.spanId)
|
||||
}
|
||||
}
|
||||
state.ackedEndBySpan = nextEnds
|
||||
for (const spanId of completed) {
|
||||
const tracked = state.spans.get(spanId)
|
||||
if (!tracked || !this.coordinator.hasRetainedSpan(spanId)) {
|
||||
state.spans.delete(spanId)
|
||||
state.ackedEndBySpan.delete(spanId)
|
||||
continue
|
||||
}
|
||||
const source = tracked.identity
|
||||
this.coordinator.settle({
|
||||
identity: source,
|
||||
spanId,
|
||||
consumer,
|
||||
reason: 'remote-frame-ack'
|
||||
})
|
||||
state.spans.delete(spanId)
|
||||
state.ackedEndBySpan.delete(spanId)
|
||||
}
|
||||
for (const range of ranges) {
|
||||
this.onProgress(range)
|
||||
}
|
||||
}
|
||||
|
||||
private reserveReplacement(
|
||||
identity: RemoteTerminalSourceRangeStreamIdentity,
|
||||
requiredSeq: number,
|
||||
reason: string
|
||||
): ReturnType<SshPtyRemoteSourceRangeReplacements['reserve']> {
|
||||
if (!this.isCurrent(identity)) {
|
||||
throw new Error('ssh_remote_source_range_stale_generation')
|
||||
}
|
||||
const state = this.requireState(identity)
|
||||
for (const spanId of state.spans.keys()) {
|
||||
if (!this.coordinator.hasRetainedSpan(spanId)) {
|
||||
state.spans.delete(spanId)
|
||||
state.ackedEndBySpan.delete(spanId)
|
||||
}
|
||||
}
|
||||
const spanIds = Array.from(state.spans)
|
||||
.filter(([, tracked]) => tracked.modelSequenceEnd <= requiredSeq)
|
||||
.map(([spanId]) => spanId)
|
||||
return this.replacements.reserve(identity, spanIds, requiredSeq, reason)
|
||||
}
|
||||
|
||||
private commitReplacement(
|
||||
reservation: Parameters<SshPtyRemoteSourceRangeReplacements['commit']>[0],
|
||||
publication: Parameters<SshPtyRemoteSourceRangeReplacements['commit']>[1]
|
||||
): boolean {
|
||||
return this.replacements.commit(
|
||||
reservation,
|
||||
publication,
|
||||
this.isCurrent(reservation.identity),
|
||||
(spanIds) => {
|
||||
const state = this.requireState(reservation.identity)
|
||||
for (const spanId of spanIds) {
|
||||
state.spans.delete(spanId)
|
||||
state.ackedEndBySpan.delete(spanId)
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
private rollbackReplacement(
|
||||
reservation: Parameters<SshPtyRemoteSourceRangeReplacements['rollback']>[0],
|
||||
reason: string
|
||||
): boolean {
|
||||
return this.replacements.rollback(reservation, reason)
|
||||
}
|
||||
|
||||
private cancel(
|
||||
identity: RemoteTerminalSourceRangeStreamIdentity,
|
||||
ranges: readonly TerminalOutputSourceRange[],
|
||||
reason: string
|
||||
): void {
|
||||
if (!this.isCurrent(identity)) {
|
||||
return
|
||||
}
|
||||
this.replacements.rollbackIdentity(identity, `${reason}-replacement-aborted`)
|
||||
const consumer = remoteConsumerId(identity)
|
||||
const state = this.requireState(identity)
|
||||
const spanIds = new Set([
|
||||
...state.spans.keys(),
|
||||
...uniqueSpanIds(ranges).filter((spanId) => state.spans.has(spanId))
|
||||
])
|
||||
for (const spanId of spanIds) {
|
||||
const tracked = state.spans.get(spanId)
|
||||
if (!tracked || !this.coordinator.hasRetainedSpan(spanId)) {
|
||||
continue
|
||||
}
|
||||
const source = tracked.identity
|
||||
const transition = { identity: source, spanId, consumer, reason }
|
||||
if (this.coordinator.beginTransfer(transition, consumer)) {
|
||||
this.coordinator.cancelTransfer(transition)
|
||||
}
|
||||
}
|
||||
this.detachIdentity(identity)
|
||||
for (const range of ranges) {
|
||||
this.onProgress(range)
|
||||
}
|
||||
}
|
||||
|
||||
private isCurrent(identity: RemoteTerminalSourceRangeStreamIdentity): boolean {
|
||||
return (
|
||||
this.consumersByPty.get(identity.ptyId)?.get(identity.consumerId)?.streamGeneration ===
|
||||
identity.streamGeneration
|
||||
)
|
||||
}
|
||||
|
||||
private requireState(identity: RemoteTerminalSourceRangeStreamIdentity): RemoteConsumerState {
|
||||
const state = this.consumersByPty.get(identity.ptyId)?.get(identity.consumerId)
|
||||
if (!state || state.streamGeneration !== identity.streamGeneration) {
|
||||
throw new Error('ssh_remote_source_range_stale_generation')
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
private detachIdentity(identity: RemoteTerminalSourceRangeStreamIdentity): void {
|
||||
const consumers = this.consumersByPty.get(identity.ptyId)
|
||||
if (
|
||||
!consumers ||
|
||||
consumers.get(identity.consumerId)?.streamGeneration !== identity.streamGeneration
|
||||
) {
|
||||
return
|
||||
}
|
||||
consumers.delete(identity.consumerId)
|
||||
if (consumers.size === 0) {
|
||||
this.consumersByPty.delete(identity.ptyId)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
import type {
|
||||
RemoteTerminalSourceRangeReplacementPublication,
|
||||
RemoteTerminalSourceRangeReplacementReservation,
|
||||
RemoteTerminalSourceRangeStreamIdentity
|
||||
} from '../runtime/remote-terminal-source-range-consumer'
|
||||
import type { PtySourceSpan } from '../../shared/pty-source-credit-contract'
|
||||
import type {
|
||||
SshPtySourceConsumerId,
|
||||
SshPtySourceObligationState
|
||||
} from './ssh-pty-source-obligation-contract'
|
||||
import type { SshPtySourceObligationCoordinator } from './ssh-pty-source-obligation-coordinator'
|
||||
|
||||
type ReplacementSpanRecord = Readonly<{
|
||||
source: PtySourceSpan
|
||||
transferState: SshPtySourceObligationState
|
||||
}>
|
||||
|
||||
type ReplacementReservationRecord = {
|
||||
reservation: RemoteTerminalSourceRangeReplacementReservation
|
||||
spans: readonly ReplacementSpanRecord[]
|
||||
consumer: SshPtySourceConsumerId
|
||||
replacement: SshPtySourceConsumerId
|
||||
reason: string
|
||||
}
|
||||
|
||||
function remoteConsumerId(
|
||||
identity: RemoteTerminalSourceRangeStreamIdentity
|
||||
): SshPtySourceConsumerId {
|
||||
return `remote:${identity.consumerId}`
|
||||
}
|
||||
|
||||
export class SshPtyRemoteSourceRangeReplacements {
|
||||
private readonly reservations = new Map<string, ReplacementReservationRecord>()
|
||||
private nextReservationId = 1
|
||||
|
||||
constructor(private readonly coordinator: SshPtySourceObligationCoordinator) {}
|
||||
|
||||
reserve(
|
||||
identity: RemoteTerminalSourceRangeStreamIdentity,
|
||||
spanIds: readonly string[],
|
||||
requiredSeq: number,
|
||||
reason: string
|
||||
): RemoteTerminalSourceRangeReplacementReservation | null {
|
||||
if (spanIds.length === 0) {
|
||||
return null
|
||||
}
|
||||
if (!Number.isSafeInteger(requiredSeq) || requiredSeq < 0) {
|
||||
throw new Error('ssh_remote_source_range_replacement_sequence_invalid')
|
||||
}
|
||||
const consumer = remoteConsumerId(identity)
|
||||
const replacement = `remote:snapshot:${identity.consumerId}` as const
|
||||
const spans = spanIds.map((spanId) => this.coordinator.spanIdentity(spanId))
|
||||
for (const { spanId } of spans) {
|
||||
if (this.coordinator.obligation(spanId, consumer).state !== 'open') {
|
||||
throw new Error('ssh_remote_source_range_transfer_invalid')
|
||||
}
|
||||
}
|
||||
const transferred: ReplacementSpanRecord[] = []
|
||||
try {
|
||||
for (const source of spans) {
|
||||
const { spanId } = source
|
||||
const transition = { identity: source, spanId, consumer, reason }
|
||||
if (!this.coordinator.beginTransfer(transition, replacement)) {
|
||||
throw new Error('ssh_remote_source_range_transfer_invalid')
|
||||
}
|
||||
const transferState = this.coordinator.obligation(spanId, consumer)
|
||||
if (transferState.state !== 'transferring' || transferState.to !== replacement) {
|
||||
this.coordinator.rollbackTransfer(transition)
|
||||
throw new Error('ssh_remote_source_range_transfer_invalid')
|
||||
}
|
||||
transferred.push(Object.freeze({ source, transferState }))
|
||||
}
|
||||
} catch (error) {
|
||||
for (const span of transferred.toReversed()) {
|
||||
this.rollbackExactSpan(span, consumer, reason)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
const reservation = Object.freeze({
|
||||
reservationId: `remote-source-replacement:${this.nextReservationId++}`,
|
||||
identity: Object.freeze({ ...identity }),
|
||||
requiredSeq
|
||||
})
|
||||
this.reservations.set(reservation.reservationId, {
|
||||
reservation,
|
||||
spans: Object.freeze(transferred),
|
||||
consumer,
|
||||
replacement,
|
||||
reason
|
||||
})
|
||||
return reservation
|
||||
}
|
||||
|
||||
commit(
|
||||
reservation: RemoteTerminalSourceRangeReplacementReservation,
|
||||
publication: RemoteTerminalSourceRangeReplacementPublication,
|
||||
isCurrent: boolean,
|
||||
onCommitted: (spanIds: readonly string[]) => void
|
||||
): boolean {
|
||||
const record = this.reservations.get(reservation.reservationId)
|
||||
if (
|
||||
!record ||
|
||||
record.reservation !== reservation ||
|
||||
!isCurrent ||
|
||||
!Number.isSafeInteger(publication.seq) ||
|
||||
publication.seq < reservation.requiredSeq ||
|
||||
(publication.source !== 'headless' && publication.source !== 'renderer')
|
||||
) {
|
||||
return false
|
||||
}
|
||||
if (
|
||||
record.spans.some(({ source, transferState }) => {
|
||||
const { spanId } = source
|
||||
if (!this.coordinator.hasRetainedSpan(spanId)) {
|
||||
return true
|
||||
}
|
||||
return this.coordinator.obligation(spanId, record.consumer) !== transferState
|
||||
})
|
||||
) {
|
||||
return false
|
||||
}
|
||||
for (const { source } of record.spans) {
|
||||
const { spanId } = source
|
||||
if (!this.coordinator.hasRetainedSpan(spanId)) {
|
||||
continue
|
||||
}
|
||||
if (
|
||||
!this.coordinator.commitTransfer({ identity: source, spanId, consumer: record.consumer })
|
||||
) {
|
||||
throw new Error('ssh_remote_source_range_replacement_commit_invalid')
|
||||
}
|
||||
}
|
||||
this.reservations.delete(reservation.reservationId)
|
||||
onCommitted(record.spans.map(({ source }) => source.spanId))
|
||||
return true
|
||||
}
|
||||
|
||||
rollback(reservation: RemoteTerminalSourceRangeReplacementReservation, reason: string): boolean {
|
||||
const record = this.reservations.get(reservation.reservationId)
|
||||
if (!record || record.reservation !== reservation) {
|
||||
return false
|
||||
}
|
||||
this.reservations.delete(reservation.reservationId)
|
||||
let rolledBack = true
|
||||
for (const span of record.spans) {
|
||||
const { spanId } = span.source
|
||||
if (!this.coordinator.hasRetainedSpan(spanId)) {
|
||||
continue
|
||||
}
|
||||
rolledBack = this.rollbackExactSpan(span, record.consumer, reason) && rolledBack
|
||||
}
|
||||
return rolledBack
|
||||
}
|
||||
|
||||
rollbackIdentity(identity: RemoteTerminalSourceRangeStreamIdentity, reason: string): void {
|
||||
for (const record of Array.from(this.reservations.values())) {
|
||||
if (
|
||||
record.reservation.identity.ptyId === identity.ptyId &&
|
||||
record.reservation.identity.consumerId === identity.consumerId &&
|
||||
record.reservation.identity.streamGeneration === identity.streamGeneration
|
||||
) {
|
||||
this.rollback(record.reservation, reason)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
closeGeneration(providerGeneration: number, reason: string): void {
|
||||
for (const record of Array.from(this.reservations.values())) {
|
||||
if (record.spans.some(({ source }) => source.providerGeneration === providerGeneration)) {
|
||||
this.rollback(record.reservation, `${reason}-replacement-aborted`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private rollbackExactSpan(
|
||||
span: ReplacementSpanRecord,
|
||||
consumer: SshPtySourceConsumerId,
|
||||
reason: string
|
||||
): boolean {
|
||||
const { source, transferState } = span
|
||||
if (
|
||||
!this.coordinator.hasRetainedSpan(source.spanId) ||
|
||||
this.coordinator.obligation(source.spanId, consumer) !== transferState
|
||||
) {
|
||||
return false
|
||||
}
|
||||
return this.coordinator.rollbackTransfer({
|
||||
identity: source,
|
||||
spanId: source.spanId,
|
||||
consumer,
|
||||
reason
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { PtySourceCreditAckBatch } from '../../shared/pty-source-credit-contract'
|
||||
import { SshPtySourceAckCoalescer } from './ssh-pty-source-ack-coalescer'
|
||||
|
||||
function publication(token: number, endSu: number, settled = vi.fn(), providerGeneration = 1) {
|
||||
const identity = {
|
||||
id: `pty-${token}`,
|
||||
providerGeneration,
|
||||
clientGeneration: 1,
|
||||
ownerGeneration: 1,
|
||||
ptyIncarnation: `incarnation-${token}`,
|
||||
deliveryToken: `token-${token}`
|
||||
}
|
||||
return {
|
||||
identity,
|
||||
ack: {
|
||||
id: identity.id,
|
||||
clientGeneration: 1,
|
||||
ownerGeneration: 1,
|
||||
deliveryToken: identity.deliveryToken,
|
||||
creditedEndSu: endSu
|
||||
},
|
||||
onSettled: settled
|
||||
}
|
||||
}
|
||||
|
||||
describe('SshPtySourceAckCoalescer', () => {
|
||||
it('coalesces cumulative values and advances them only from the write callback', () => {
|
||||
const writes: {
|
||||
batch: PtySourceCreditAckBatch
|
||||
settle: (result: { ok: true } | { ok: false; error: Error }) => void
|
||||
}[] = []
|
||||
const firstSettled = vi.fn()
|
||||
const latestSettled = vi.fn()
|
||||
const coalescer = new SshPtySourceAckCoalescer({
|
||||
publish: (_providerGeneration, batch, settle) => writes.push({ batch, settle }),
|
||||
schedule: vi.fn(() => 1 as unknown as ReturnType<typeof setTimeout>),
|
||||
cancelSchedule: vi.fn()
|
||||
})
|
||||
coalescer.enqueue(publication(1, 10, firstSettled))
|
||||
coalescer.enqueue(publication(1, 20, latestSettled))
|
||||
coalescer.flush()
|
||||
|
||||
expect(writes[0].batch.acknowledgements).toEqual([
|
||||
expect.objectContaining({ deliveryToken: 'token-1', creditedEndSu: 20 })
|
||||
])
|
||||
expect(latestSettled).not.toHaveBeenCalled()
|
||||
writes[0].settle({ ok: true })
|
||||
expect(latestSettled).toHaveBeenCalledWith({ ok: true })
|
||||
expect(firstSettled).toHaveBeenCalledWith({ ok: true })
|
||||
})
|
||||
|
||||
it('limits one batch to 64 tokens and gives the remainder another turn', () => {
|
||||
const batches: PtySourceCreditAckBatch[] = []
|
||||
const scheduled: (() => void)[] = []
|
||||
const coalescer = new SshPtySourceAckCoalescer({
|
||||
publish: (_providerGeneration, batch, settle) => {
|
||||
batches.push(batch)
|
||||
settle({ ok: true })
|
||||
},
|
||||
schedule: (callback) => {
|
||||
scheduled.push(callback)
|
||||
return scheduled.length as unknown as ReturnType<typeof setTimeout>
|
||||
},
|
||||
cancelSchedule: vi.fn()
|
||||
})
|
||||
for (let token = 0; token < 70; token++) {
|
||||
coalescer.enqueue(publication(token, 1))
|
||||
}
|
||||
|
||||
coalescer.flush()
|
||||
expect(batches[0].acknowledgements).toHaveLength(64)
|
||||
scheduled.at(-1)!()
|
||||
expect(batches[1].acknowledgements).toHaveLength(6)
|
||||
})
|
||||
|
||||
it('never mixes provider generations in one transport batch', () => {
|
||||
const generations: number[] = []
|
||||
const coalescer = new SshPtySourceAckCoalescer({
|
||||
publish: (providerGeneration, _batch, settle) => {
|
||||
generations.push(providerGeneration)
|
||||
settle({ ok: true })
|
||||
},
|
||||
schedule: vi.fn(() => 1 as unknown as ReturnType<typeof setTimeout>),
|
||||
cancelSchedule: vi.fn()
|
||||
})
|
||||
coalescer.enqueue(publication(1, 1, vi.fn(), 1))
|
||||
coalescer.enqueue(publication(2, 1, vi.fn(), 2))
|
||||
|
||||
coalescer.flush()
|
||||
coalescer.flush()
|
||||
|
||||
expect(generations).toEqual([1, 2])
|
||||
})
|
||||
|
||||
it('settles every entry as failed on a synchronous send error', () => {
|
||||
const settled = vi.fn()
|
||||
const coalescer = new SshPtySourceAckCoalescer({
|
||||
publish: () => {
|
||||
throw new Error('send failed')
|
||||
},
|
||||
schedule: vi.fn(() => 1 as unknown as ReturnType<typeof setTimeout>),
|
||||
cancelSchedule: vi.fn()
|
||||
})
|
||||
coalescer.enqueue(publication(1, 10, settled))
|
||||
coalescer.flush()
|
||||
|
||||
expect(settled).toHaveBeenCalledWith({
|
||||
ok: false,
|
||||
error: expect.objectContaining({ message: 'send failed' })
|
||||
})
|
||||
})
|
||||
|
||||
it('promotes a pending interval flush to immediate at the source threshold', () => {
|
||||
const delays: number[] = []
|
||||
const cancelSchedule = vi.fn()
|
||||
const coalescer = new SshPtySourceAckCoalescer({
|
||||
publish: vi.fn(),
|
||||
schedule: (_callback, delayMs) => {
|
||||
delays.push(delayMs)
|
||||
return delays.length as unknown as ReturnType<typeof setTimeout>
|
||||
},
|
||||
cancelSchedule
|
||||
})
|
||||
|
||||
coalescer.enqueue(publication(1, 1))
|
||||
coalescer.enqueue(publication(1, 64 * 1024))
|
||||
|
||||
expect(delays).toEqual([8, 0])
|
||||
expect(cancelSchedule).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('fails queued callbacks exactly once during cleanup', () => {
|
||||
const settled = vi.fn()
|
||||
const coalescer = new SshPtySourceAckCoalescer({
|
||||
publish: vi.fn(),
|
||||
schedule: vi.fn(() => 1 as unknown as ReturnType<typeof setTimeout>),
|
||||
cancelSchedule: vi.fn()
|
||||
})
|
||||
coalescer.enqueue(publication(1, 10, settled))
|
||||
|
||||
coalescer.dispose()
|
||||
coalescer.dispose()
|
||||
expect(settled).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('owns an in-flight callback until dispose and ignores its late transport callback', () => {
|
||||
let transportSettle!: (result: { ok: true } | { ok: false; error: Error }) => void
|
||||
const settled = vi.fn()
|
||||
const coalescer = new SshPtySourceAckCoalescer({
|
||||
publish: (_providerGeneration, _batch, settle) => {
|
||||
transportSettle = settle
|
||||
},
|
||||
schedule: vi.fn(() => 1 as unknown as ReturnType<typeof setTimeout>),
|
||||
cancelSchedule: vi.fn()
|
||||
})
|
||||
coalescer.enqueue(publication(1, 10, settled))
|
||||
coalescer.flush()
|
||||
|
||||
coalescer.dispose('generation closed')
|
||||
transportSettle({ ok: true })
|
||||
expect(settled).toHaveBeenCalledOnce()
|
||||
expect(settled).toHaveBeenCalledWith({
|
||||
ok: false,
|
||||
error: expect.objectContaining({ message: 'generation closed' })
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,230 @@
|
||||
import {
|
||||
MAX_PTY_ACK_ENTRIES,
|
||||
type PtySourceCreditAckBatch
|
||||
} from '../../shared/pty-source-credit-contract'
|
||||
import type { SshPtySourceAckPublication } from './ssh-pty-source-obligation-contract'
|
||||
|
||||
export const SSH_PTY_ACK_FLUSH_MS = 8
|
||||
export const SSH_PTY_ACK_EAGER_ADVANCE_SU = 64 * 1024
|
||||
const ACK_PUBLICATION_WATERMARK_LIMIT = 1024
|
||||
|
||||
type AckSettlement = { ok: true } | { ok: false; error: Error }
|
||||
|
||||
type CoalescedEntry = {
|
||||
publication: SshPtySourceAckPublication
|
||||
members: SshPtySourceAckPublication[]
|
||||
}
|
||||
|
||||
type InFlightBatch = {
|
||||
entries: CoalescedEntry[]
|
||||
settled: boolean
|
||||
}
|
||||
|
||||
export type SshPtySourceAckCoalescerOptions = {
|
||||
publish: (
|
||||
providerGeneration: number,
|
||||
batch: PtySourceCreditAckBatch,
|
||||
onSettled: (result: AckSettlement) => void
|
||||
) => void
|
||||
schedule?: (callback: () => void, delayMs: number) => ReturnType<typeof setTimeout>
|
||||
cancelSchedule?: (timer: ReturnType<typeof setTimeout>) => void
|
||||
onTokenClosed?: (identity: SshPtySourceAckPublication['identity']) => void
|
||||
}
|
||||
|
||||
function ackKey(publication: SshPtySourceAckPublication): string {
|
||||
const { identity } = publication
|
||||
return `${identity.providerGeneration}\0${identity.clientGeneration}\0${identity.ownerGeneration}\0${identity.id}\0${identity.ptyIncarnation}\0${identity.deliveryToken}`
|
||||
}
|
||||
|
||||
export class SshPtySourceAckCoalescer {
|
||||
private readonly pending = new Map<string, CoalescedEntry>()
|
||||
private readonly lastPublishedEndByToken = new Map<string, number>()
|
||||
private readonly schedule: NonNullable<SshPtySourceAckCoalescerOptions['schedule']>
|
||||
private readonly cancelSchedule: NonNullable<SshPtySourceAckCoalescerOptions['cancelSchedule']>
|
||||
private timer: ReturnType<typeof setTimeout> | null = null
|
||||
private timerDelayMs: number | null = null
|
||||
private inFlight: InFlightBatch | null = null
|
||||
private disposed = false
|
||||
|
||||
constructor(private readonly options: SshPtySourceAckCoalescerOptions) {
|
||||
this.schedule = options.schedule ?? ((callback, delayMs) => setTimeout(callback, delayMs))
|
||||
this.cancelSchedule = options.cancelSchedule ?? clearTimeout
|
||||
}
|
||||
|
||||
enqueue(publication: SshPtySourceAckPublication): void {
|
||||
if (this.disposed) {
|
||||
publication.onSettled({ ok: false, error: new Error('SSH PTY ACK coalescer disposed') })
|
||||
return
|
||||
}
|
||||
const key = ackKey(publication)
|
||||
const current = this.pending.get(key)
|
||||
if (!current) {
|
||||
this.pending.set(key, { publication, members: [publication] })
|
||||
} else {
|
||||
current.members.push(publication)
|
||||
if (publication.ack.creditedEndSu > current.publication.ack.creditedEndSu) {
|
||||
current.publication = publication
|
||||
}
|
||||
}
|
||||
const lastPublished = this.lastPublishedEndByToken.get(key) ?? 0
|
||||
const eager = publication.ack.creditedEndSu - lastPublished >= SSH_PTY_ACK_EAGER_ADVANCE_SU
|
||||
this.requestFlush(eager ? 0 : SSH_PTY_ACK_FLUSH_MS)
|
||||
}
|
||||
|
||||
flush(): void {
|
||||
if (this.disposed || this.inFlight || this.pending.size === 0) {
|
||||
return
|
||||
}
|
||||
if (this.timer) {
|
||||
this.cancelSchedule(this.timer)
|
||||
this.timer = null
|
||||
this.timerDelayMs = null
|
||||
}
|
||||
const providerGeneration = this.pending.values().next().value!.publication
|
||||
.identity.providerGeneration
|
||||
const selected = Array.from(this.pending.entries())
|
||||
.filter(([, entry]) => entry.publication.identity.providerGeneration === providerGeneration)
|
||||
.slice(0, MAX_PTY_ACK_ENTRIES)
|
||||
for (const [key] of selected) {
|
||||
this.pending.delete(key)
|
||||
}
|
||||
const batch: InFlightBatch = {
|
||||
entries: selected.map(([, entry]) => entry),
|
||||
settled: false
|
||||
}
|
||||
this.inFlight = batch
|
||||
const settle = (result: AckSettlement): void => this.settleBatch(batch, result)
|
||||
try {
|
||||
this.options.publish(
|
||||
providerGeneration,
|
||||
Object.freeze({
|
||||
acknowledgements: Object.freeze(batch.entries.map((entry) => entry.publication.ack))
|
||||
}),
|
||||
settle
|
||||
)
|
||||
} catch (error) {
|
||||
settle({
|
||||
ok: false,
|
||||
error: error instanceof Error ? error : new Error(String(error))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
dispose(reason = 'SSH PTY ACK coalescer disposed'): void {
|
||||
if (this.disposed) {
|
||||
return
|
||||
}
|
||||
this.disposed = true
|
||||
if (this.timer) {
|
||||
this.cancelSchedule(this.timer)
|
||||
this.timer = null
|
||||
this.timerDelayMs = null
|
||||
}
|
||||
const result = { ok: false as const, error: new Error(reason) }
|
||||
for (const entry of this.pending.values()) {
|
||||
this.settleMembers(entry, result)
|
||||
}
|
||||
this.pending.clear()
|
||||
if (this.inFlight && !this.inFlight.settled) {
|
||||
const batch = this.inFlight
|
||||
batch.settled = true
|
||||
this.inFlight = null
|
||||
for (const entry of batch.entries) {
|
||||
this.settleMembers(entry, result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cancelGeneration(providerGeneration: number, reason: string): void {
|
||||
const result = { ok: false as const, error: new Error(reason) }
|
||||
for (const [key, entry] of this.pending) {
|
||||
if (entry.publication.identity.providerGeneration === providerGeneration) {
|
||||
this.pending.delete(key)
|
||||
this.settleMembers(entry, result)
|
||||
}
|
||||
}
|
||||
const batch = this.inFlight
|
||||
if (batch && !batch.settled) {
|
||||
const retained: CoalescedEntry[] = []
|
||||
for (const entry of batch.entries) {
|
||||
if (entry.publication.identity.providerGeneration === providerGeneration) {
|
||||
this.settleMembers(entry, result)
|
||||
} else {
|
||||
retained.push(entry)
|
||||
}
|
||||
}
|
||||
batch.entries = retained
|
||||
if (retained.length === 0) {
|
||||
batch.settled = true
|
||||
this.inFlight = null
|
||||
}
|
||||
}
|
||||
const prefix = `${providerGeneration}\0`
|
||||
for (const key of this.lastPublishedEndByToken.keys()) {
|
||||
if (key.startsWith(prefix)) {
|
||||
this.lastPublishedEndByToken.delete(key)
|
||||
}
|
||||
}
|
||||
if (!this.inFlight && this.pending.size === 0 && this.timer) {
|
||||
this.cancelSchedule(this.timer)
|
||||
this.timer = null
|
||||
this.timerDelayMs = null
|
||||
} else if (!this.inFlight && this.pending.size > 0) {
|
||||
this.requestFlush(0)
|
||||
}
|
||||
}
|
||||
|
||||
get pendingCount(): number {
|
||||
return this.pending.size
|
||||
}
|
||||
|
||||
private settleBatch(batch: InFlightBatch, result: AckSettlement): void {
|
||||
if (batch.settled) {
|
||||
return
|
||||
}
|
||||
batch.settled = true
|
||||
if (this.inFlight === batch) {
|
||||
this.inFlight = null
|
||||
}
|
||||
for (const entry of batch.entries) {
|
||||
if (result.ok) {
|
||||
const key = ackKey(entry.publication)
|
||||
this.lastPublishedEndByToken.delete(key)
|
||||
this.lastPublishedEndByToken.set(key, entry.publication.ack.creditedEndSu)
|
||||
while (this.lastPublishedEndByToken.size > ACK_PUBLICATION_WATERMARK_LIMIT) {
|
||||
this.lastPublishedEndByToken.delete(this.lastPublishedEndByToken.keys().next().value!)
|
||||
}
|
||||
}
|
||||
this.settleMembers(entry, result)
|
||||
}
|
||||
if (this.pending.size > 0) {
|
||||
this.requestFlush(0)
|
||||
}
|
||||
}
|
||||
|
||||
private settleMembers(entry: CoalescedEntry, result: AckSettlement): void {
|
||||
for (const publication of entry.members.splice(0)) {
|
||||
publication.onSettled(result)
|
||||
}
|
||||
}
|
||||
|
||||
private requestFlush(delayMs: number): void {
|
||||
if (this.inFlight || this.disposed) {
|
||||
return
|
||||
}
|
||||
if (this.timer) {
|
||||
if (this.timerDelayMs !== null && this.timerDelayMs <= delayMs) {
|
||||
return
|
||||
}
|
||||
this.cancelSchedule(this.timer)
|
||||
this.timer = null
|
||||
}
|
||||
this.timerDelayMs = delayMs
|
||||
this.timer = this.schedule(() => {
|
||||
this.timer = null
|
||||
this.timerDelayMs = null
|
||||
this.flush()
|
||||
}, delayMs)
|
||||
this.timer.unref?.()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import type { SshPtySourceAckPublication } from './ssh-pty-source-obligation-contract'
|
||||
import { reclaimPublishedSourcePrefix, type TokenRecord } from './ssh-pty-source-obligation-state'
|
||||
|
||||
export function createSshPtySourceAckPublication(
|
||||
token: TokenRecord,
|
||||
endSu: number,
|
||||
spanOwners: Map<string, TokenRecord>,
|
||||
onPublished: () => void
|
||||
): SshPtySourceAckPublication {
|
||||
let settled = false
|
||||
return Object.freeze({
|
||||
identity: token.identity,
|
||||
ack: Object.freeze({
|
||||
id: token.identity.id,
|
||||
clientGeneration: token.identity.clientGeneration,
|
||||
ownerGeneration: token.identity.ownerGeneration,
|
||||
deliveryToken: token.identity.deliveryToken,
|
||||
creditedEndSu: endSu
|
||||
}),
|
||||
onSettled: (result) => {
|
||||
if (settled) {
|
||||
return
|
||||
}
|
||||
settled = true
|
||||
if (!result.ok || token.state === 'closed' || endSu > token.ackQueuedEndSu) {
|
||||
return
|
||||
}
|
||||
token.ackPublishedEndSu = Math.max(token.ackPublishedEndSu, endSu)
|
||||
reclaimPublishedSourcePrefix(token, spanOwners)
|
||||
onPublished()
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { PtySourceCreditAckBatch } from '../../shared/pty-source-credit-contract'
|
||||
import { SshPtyOutputIntake } from './ssh-pty-output-intake'
|
||||
import {
|
||||
installSshPtySourceAckPublisher,
|
||||
publishSshPtySourceAck
|
||||
} from './ssh-pty-output-intake-registry'
|
||||
|
||||
afterEach(() => vi.useRealTimers())
|
||||
|
||||
describe('SSH PTY intake to relay ACK contract', () => {
|
||||
it('publishes one cumulative relay-ID ACK only after model and desktop settlement', async () => {
|
||||
vi.useFakeTimers()
|
||||
const batches: PtySourceCreditAckBatch[] = []
|
||||
const cleanup = installSshPtySourceAckPublisher(7, (batch, onSettled) => {
|
||||
batches.push(batch)
|
||||
onSettled({ ok: true })
|
||||
})
|
||||
let sequence = 0
|
||||
const intake = new SshPtyOutputIntake({
|
||||
getModelSequence: () => sequence,
|
||||
acceptModel: (event) => {
|
||||
sequence += event.rawLength
|
||||
return { sequence, completion: Promise.resolve() }
|
||||
},
|
||||
project: () => {},
|
||||
prepareExit: () => {},
|
||||
finalizeExit: () => {},
|
||||
publishSourceAck: publishSshPtySourceAck
|
||||
})
|
||||
|
||||
try {
|
||||
const receipt = await intake.acceptData({
|
||||
id: 'ssh:target@@relay-pty-1',
|
||||
data: 'data',
|
||||
providerGeneration: 7,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
rawLength: 4,
|
||||
transformed: false,
|
||||
source: {
|
||||
relayPtyId: 'relay-pty-1',
|
||||
spanId: 'token-1:0:4',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-1',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4
|
||||
}
|
||||
})
|
||||
await vi.advanceTimersByTimeAsync(8)
|
||||
expect(batches).toHaveLength(0)
|
||||
|
||||
const projectionId = receipt.projection.identity.projectionSemanticsId
|
||||
intake.publishProjectionPrefix([projectionId], 4, 4)
|
||||
intake.settleProjectionPrefix('ssh:target@@relay-pty-1', 4)
|
||||
await vi.advanceTimersByTimeAsync(8)
|
||||
|
||||
expect(batches).toEqual([
|
||||
{
|
||||
acknowledgements: [
|
||||
{
|
||||
id: 'relay-pty-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-1',
|
||||
creditedEndSu: 4
|
||||
}
|
||||
]
|
||||
}
|
||||
])
|
||||
await intake.acceptExit({
|
||||
id: 'ssh:target@@relay-pty-1',
|
||||
code: 0,
|
||||
providerGeneration: 7,
|
||||
ptyIncarnation: 'incarnation-1'
|
||||
})
|
||||
expect(intake.getDebugSnapshot().source).toEqual({
|
||||
openedTokens: 0,
|
||||
ptyIdentities: 0
|
||||
})
|
||||
} finally {
|
||||
intake.dispose()
|
||||
cleanup()
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,38 @@
|
||||
import type {
|
||||
PtySourceCreditAck,
|
||||
PtySourceDeliveryIdentity,
|
||||
PtySourceSpan
|
||||
} from '../../shared/pty-source-credit-contract'
|
||||
|
||||
export type SshPtySourceConsumerId = 'model' | 'desktop' | `remote:${string}`
|
||||
|
||||
export type SshPtySourceObligationState =
|
||||
| Readonly<{ state: 'open' }>
|
||||
| Readonly<{ state: 'transferring'; to: SshPtySourceConsumerId; reason: string }>
|
||||
| Readonly<{ state: 'settled'; reason: string }>
|
||||
| Readonly<{ state: 'transferred'; to: SshPtySourceConsumerId; reason: string }>
|
||||
| Readonly<{ state: 'canceled'; reason: string }>
|
||||
|
||||
export type SshPtySourceAdmissionReservation = Readonly<{
|
||||
reservationId: string
|
||||
span: PtySourceSpan
|
||||
requiredConsumers: readonly SshPtySourceConsumerId[]
|
||||
}>
|
||||
|
||||
export type SshPtySourceTokenSnapshot = PtySourceDeliveryIdentity &
|
||||
Readonly<{
|
||||
state: 'active' | 'sealed-unsettled' | 'canceling' | 'closed'
|
||||
receivedEndSu: number
|
||||
obligationsTerminalEndSu: number
|
||||
ackQueuedEndSu: number
|
||||
ackPublishedEndSu: number
|
||||
openSpans: number
|
||||
exitPublished: boolean
|
||||
generationClosed: boolean
|
||||
}>
|
||||
|
||||
export type SshPtySourceAckPublication = Readonly<{
|
||||
identity: PtySourceDeliveryIdentity
|
||||
ack: PtySourceCreditAck
|
||||
onSettled: (result: { ok: true } | { ok: false; error: Error }) => void
|
||||
}>
|
||||
@@ -0,0 +1,137 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type {
|
||||
PtySourceCreditAckBatch,
|
||||
PtySourceDeliveryIdentity,
|
||||
PtySourceSpan
|
||||
} from '../../shared/pty-source-credit-contract'
|
||||
import { SshPtySourceObligationCoordinator } from './ssh-pty-source-obligation-coordinator'
|
||||
|
||||
const identity: PtySourceDeliveryIdentity = {
|
||||
id: 'pty-1',
|
||||
providerGeneration: 1,
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-1'
|
||||
}
|
||||
|
||||
const span: PtySourceSpan = {
|
||||
...identity,
|
||||
spanId: 'span-1',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4,
|
||||
displayStart: 0,
|
||||
displayEnd: 4,
|
||||
data: 'data',
|
||||
splittable: true,
|
||||
transform: { transformed: false, rawLengthSu: 4, scalarSafe: true }
|
||||
}
|
||||
|
||||
describe('SshPtySourceObligationCoordinator', () => {
|
||||
it('is the single boundary from exact consumer settlement to upstream ACK publication', () => {
|
||||
let written:
|
||||
| {
|
||||
batch: PtySourceCreditAckBatch
|
||||
settle: (result: { ok: true } | { ok: false; error: Error }) => void
|
||||
}
|
||||
| undefined
|
||||
const coordinator = new SshPtySourceObligationCoordinator({
|
||||
publish: (_providerGeneration, batch, settle) => {
|
||||
written = { batch, settle }
|
||||
},
|
||||
schedule: vi.fn(() => 1 as unknown as ReturnType<typeof setTimeout>),
|
||||
cancelSchedule: vi.fn()
|
||||
})
|
||||
coordinator.open(identity)
|
||||
const reservation = coordinator.reserve(identity, span, ['model', 'desktop'])
|
||||
coordinator.commit(reservation)
|
||||
coordinator.settle({
|
||||
identity,
|
||||
spanId: span.spanId,
|
||||
consumer: 'model',
|
||||
reason: 'emulator-receipt'
|
||||
})
|
||||
coordinator.settle({
|
||||
identity,
|
||||
spanId: span.spanId,
|
||||
consumer: 'desktop',
|
||||
reason: 'renderer-parse'
|
||||
})
|
||||
coordinator.flushAcknowledgements()
|
||||
|
||||
expect(written?.batch.acknowledgements).toEqual([
|
||||
expect.objectContaining({ deliveryToken: 'token-1', creditedEndSu: 4 })
|
||||
])
|
||||
expect(coordinator.snapshot(identity).ackPublishedEndSu).toBe(0)
|
||||
written?.settle({ ok: true })
|
||||
expect(coordinator.snapshot(identity).ackPublishedEndSu).toBe(4)
|
||||
})
|
||||
|
||||
it('rejects an adapter transition carrying a stale full delivery identity', () => {
|
||||
const coordinator = new SshPtySourceObligationCoordinator({
|
||||
publish: vi.fn(),
|
||||
schedule: vi.fn(() => 1 as unknown as ReturnType<typeof setTimeout>),
|
||||
cancelSchedule: vi.fn()
|
||||
})
|
||||
coordinator.open(identity)
|
||||
coordinator.commit(coordinator.reserve(identity, span, ['model']))
|
||||
|
||||
expect(() =>
|
||||
coordinator.settle({
|
||||
identity: { ...identity, ownerGeneration: 99 },
|
||||
spanId: span.spanId,
|
||||
consumer: 'model',
|
||||
reason: 'stale'
|
||||
})
|
||||
).toThrow('stale')
|
||||
})
|
||||
|
||||
it('keeps later generations publishable after closing one generation', () => {
|
||||
const publish = vi.fn()
|
||||
const coordinator = new SshPtySourceObligationCoordinator({
|
||||
publish,
|
||||
schedule: vi.fn(() => 1 as unknown as ReturnType<typeof setTimeout>),
|
||||
cancelSchedule: vi.fn()
|
||||
})
|
||||
coordinator.open(identity)
|
||||
coordinator.closeGeneration(1, 'replaced')
|
||||
const nextIdentity = { ...identity, providerGeneration: 2, deliveryToken: 'token-2' }
|
||||
const nextSpan = { ...span, ...nextIdentity, spanId: 'span-2' }
|
||||
coordinator.open(nextIdentity)
|
||||
coordinator.commit(coordinator.reserve(nextIdentity, nextSpan, ['model']))
|
||||
coordinator.settle({
|
||||
identity: nextIdentity,
|
||||
spanId: nextSpan.spanId,
|
||||
consumer: 'model',
|
||||
reason: 'accepted'
|
||||
})
|
||||
coordinator.flushAcknowledgements()
|
||||
|
||||
expect(publish).toHaveBeenCalledWith(
|
||||
2,
|
||||
{
|
||||
acknowledgements: [expect.objectContaining({ deliveryToken: 'token-2', creditedEndSu: 4 })]
|
||||
},
|
||||
expect.any(Function)
|
||||
)
|
||||
})
|
||||
|
||||
it('generation-closes every retained token on coordinator disposal', () => {
|
||||
const coordinator = new SshPtySourceObligationCoordinator({
|
||||
publish: vi.fn(),
|
||||
schedule: vi.fn(() => 1 as unknown as ReturnType<typeof setTimeout>),
|
||||
cancelSchedule: vi.fn()
|
||||
})
|
||||
coordinator.open(identity)
|
||||
coordinator.commit(coordinator.reserve(identity, span, ['model']))
|
||||
|
||||
coordinator.dispose('provider disposed')
|
||||
|
||||
expect(coordinator.snapshot(identity)).toMatchObject({
|
||||
state: 'closed',
|
||||
generationClosed: true,
|
||||
openSpans: 0
|
||||
})
|
||||
expect(() => coordinator.open({ ...identity, deliveryToken: 'late-token' })).toThrow('disposed')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,267 @@
|
||||
import {
|
||||
ptySourceDeliveryKey,
|
||||
samePtySourceDelivery,
|
||||
type PtySourceDeliveryIdentity,
|
||||
type PtySourceSpan
|
||||
} from '../../shared/pty-source-credit-contract'
|
||||
import {
|
||||
SshPtySourceAckCoalescer,
|
||||
type SshPtySourceAckCoalescerOptions
|
||||
} from './ssh-pty-source-ack-coalescer'
|
||||
import {
|
||||
SshPtySourceObligationLedger,
|
||||
type SshPtySourceAdmissionReservation,
|
||||
type SshPtySourceConsumerId,
|
||||
type SshPtySourceObligationState,
|
||||
type SshPtySourceTokenSnapshot
|
||||
} from './ssh-pty-source-obligation-ledger'
|
||||
|
||||
export type SshPtySourceObligationTransition = Readonly<{
|
||||
identity: PtySourceDeliveryIdentity
|
||||
spanId: string
|
||||
consumer: SshPtySourceConsumerId
|
||||
reason: string
|
||||
}>
|
||||
|
||||
type TerminalWaiter = {
|
||||
resolve: () => void
|
||||
reject: (error: Error) => void
|
||||
}
|
||||
|
||||
type TerminalWaiterGroup = {
|
||||
identity: PtySourceDeliveryIdentity
|
||||
waiters: Set<TerminalWaiter>
|
||||
}
|
||||
|
||||
export class SshPtySourceObligationCoordinator {
|
||||
private readonly ledger: SshPtySourceObligationLedger
|
||||
private readonly acknowledgements: SshPtySourceAckCoalescer
|
||||
private readonly terminalWaiters = new Map<string, TerminalWaiterGroup>()
|
||||
private disposed = false
|
||||
|
||||
constructor(options: SshPtySourceAckCoalescerOptions) {
|
||||
this.ledger = new SshPtySourceObligationLedger(options.onTokenClosed)
|
||||
this.acknowledgements = new SshPtySourceAckCoalescer(options)
|
||||
}
|
||||
|
||||
open(identity: PtySourceDeliveryIdentity, checkpointSourceEndSu = 0): void {
|
||||
if (this.disposed) {
|
||||
throw new Error('SSH PTY source obligation coordinator is disposed')
|
||||
}
|
||||
this.ledger.open(identity, checkpointSourceEndSu)
|
||||
}
|
||||
|
||||
reserve(
|
||||
identity: PtySourceDeliveryIdentity,
|
||||
span: PtySourceSpan,
|
||||
requiredConsumers: readonly SshPtySourceConsumerId[]
|
||||
): SshPtySourceAdmissionReservation {
|
||||
return this.ledger.reserve(identity, span, requiredConsumers)
|
||||
}
|
||||
|
||||
commit(reservation: SshPtySourceAdmissionReservation): void {
|
||||
this.ledger.commit(reservation)
|
||||
}
|
||||
|
||||
rollback(reservation: SshPtySourceAdmissionReservation): boolean {
|
||||
return this.ledger.rollback(reservation)
|
||||
}
|
||||
|
||||
rollbackCommitted(reservation: SshPtySourceAdmissionReservation): boolean {
|
||||
const rolledBack = this.ledger.rollbackCommitted(reservation)
|
||||
if (rolledBack) {
|
||||
this.maybeResolveTerminal(reservation.span)
|
||||
}
|
||||
return rolledBack
|
||||
}
|
||||
|
||||
settle(transition: SshPtySourceObligationTransition): boolean {
|
||||
this.requireSpanIdentity(transition)
|
||||
const changed = this.ledger.settle(transition.spanId, transition.consumer, transition.reason)
|
||||
this.queueEligibleAck(transition.identity)
|
||||
return changed
|
||||
}
|
||||
|
||||
beginTransfer(transition: SshPtySourceObligationTransition, to: SshPtySourceConsumerId): boolean {
|
||||
this.requireSpanIdentity(transition)
|
||||
return this.ledger.beginTransfer(transition.spanId, transition.consumer, to, transition.reason)
|
||||
}
|
||||
|
||||
commitTransfer(transition: Omit<SshPtySourceObligationTransition, 'reason'>): boolean {
|
||||
this.requireSpanIdentity(transition)
|
||||
const changed = this.ledger.commitTransfer(transition.spanId, transition.consumer)
|
||||
this.queueEligibleAck(transition.identity)
|
||||
return changed
|
||||
}
|
||||
|
||||
cancelTransfer(transition: SshPtySourceObligationTransition): boolean {
|
||||
this.requireSpanIdentity(transition)
|
||||
const changed = this.ledger.cancelTransfer(
|
||||
transition.spanId,
|
||||
transition.consumer,
|
||||
transition.reason
|
||||
)
|
||||
this.queueEligibleAck(transition.identity)
|
||||
return changed
|
||||
}
|
||||
|
||||
rollbackTransfer(transition: SshPtySourceObligationTransition): boolean {
|
||||
this.requireSpanIdentity(transition)
|
||||
return this.ledger.rollbackTransfer(transition.spanId, transition.consumer)
|
||||
}
|
||||
|
||||
seal(identity: PtySourceDeliveryIdentity): void {
|
||||
if (this.ledger.snapshot(identity).state === 'sealed-unsettled') {
|
||||
return
|
||||
}
|
||||
this.ledger.seal(identity)
|
||||
}
|
||||
|
||||
markExitPublished(identity: PtySourceDeliveryIdentity): void {
|
||||
this.queueEligibleAck(identity)
|
||||
this.ledger.markExitPublished(identity)
|
||||
}
|
||||
|
||||
whenTerminal(identity: PtySourceDeliveryIdentity): Promise<void> {
|
||||
if (this.isTerminal(identity)) {
|
||||
return Promise.resolve()
|
||||
}
|
||||
const key = ptySourceDeliveryKey(identity)
|
||||
let group = this.terminalWaiters.get(key)
|
||||
if (!group) {
|
||||
group = { identity: Object.freeze({ ...identity }), waiters: new Set() }
|
||||
this.terminalWaiters.set(key, group)
|
||||
}
|
||||
return new Promise((resolve, reject) => {
|
||||
group!.waiters.add({ resolve, reject })
|
||||
})
|
||||
}
|
||||
|
||||
beginExitTimeout(identity: PtySourceDeliveryIdentity) {
|
||||
return this.ledger.beginExitTimeout(identity)
|
||||
}
|
||||
|
||||
applyCancellationProof(
|
||||
identity: PtySourceDeliveryIdentity,
|
||||
proof: Readonly<{ sentEndSu: number; creditedEndSu: number }>
|
||||
): void {
|
||||
const snapshot = this.ledger.snapshot(identity)
|
||||
if (
|
||||
snapshot.state === 'closed' &&
|
||||
snapshot.receivedEndSu === proof.sentEndSu &&
|
||||
snapshot.ackPublishedEndSu === proof.creditedEndSu
|
||||
) {
|
||||
return
|
||||
}
|
||||
this.ledger.applyCancellationProof(identity, proof)
|
||||
this.rejectWaiters(
|
||||
(group) => samePtySourceDelivery(group.identity, identity),
|
||||
new Error('ssh_source_delivery_canceled')
|
||||
)
|
||||
}
|
||||
|
||||
applyRecoveryCancellationProof(
|
||||
identity: PtySourceDeliveryIdentity,
|
||||
proof: Readonly<{ sentEndSu: number; creditedEndSu: number }>
|
||||
): void {
|
||||
this.ledger.applyRecoveryCancellationProof(identity, proof)
|
||||
this.rejectWaiters(
|
||||
(group) => samePtySourceDelivery(group.identity, identity),
|
||||
new Error('ssh_source_delivery_canceled')
|
||||
)
|
||||
}
|
||||
|
||||
closeGeneration(providerGeneration: number, reason: string): number {
|
||||
this.rejectWaiters(
|
||||
(group) => group.identity.providerGeneration === providerGeneration,
|
||||
new Error(reason)
|
||||
)
|
||||
const closed = this.ledger.closeGeneration(providerGeneration, reason)
|
||||
this.acknowledgements.cancelGeneration(providerGeneration, reason)
|
||||
return closed
|
||||
}
|
||||
|
||||
snapshot(identity: PtySourceDeliveryIdentity): SshPtySourceTokenSnapshot {
|
||||
return this.ledger.snapshot(identity)
|
||||
}
|
||||
|
||||
modelAcceptedEnd(identity: PtySourceDeliveryIdentity): number {
|
||||
return this.ledger.modelAcceptedEnd(identity)
|
||||
}
|
||||
|
||||
obligation(spanId: string, consumer: SshPtySourceConsumerId): SshPtySourceObligationState {
|
||||
return this.ledger.obligation(spanId, consumer)
|
||||
}
|
||||
|
||||
spanIdentity(spanId: string): PtySourceSpan {
|
||||
return this.ledger.spanIdentity(spanId)
|
||||
}
|
||||
|
||||
hasRetainedSpan(spanId: string): boolean {
|
||||
return this.ledger.hasRetainedSpan(spanId)
|
||||
}
|
||||
|
||||
flushAcknowledgements(): void {
|
||||
this.acknowledgements.flush()
|
||||
}
|
||||
|
||||
dispose(reason?: string): void {
|
||||
if (this.disposed) {
|
||||
return
|
||||
}
|
||||
this.disposed = true
|
||||
this.rejectWaiters(() => true, new Error(reason ?? 'SSH PTY source obligations disposed'))
|
||||
this.ledger.closeAll(reason ?? 'SSH PTY source obligation coordinator disposed')
|
||||
this.acknowledgements.dispose(reason)
|
||||
}
|
||||
|
||||
private queueEligibleAck(identity: PtySourceDeliveryIdentity): void {
|
||||
const publication = this.ledger.queueAck(identity)
|
||||
if (publication) {
|
||||
this.acknowledgements.enqueue(publication)
|
||||
}
|
||||
this.maybeResolveTerminal(identity)
|
||||
}
|
||||
|
||||
private isTerminal(identity: PtySourceDeliveryIdentity): boolean {
|
||||
const snapshot = this.ledger.snapshot(identity)
|
||||
return (
|
||||
snapshot.obligationsTerminalEndSu === snapshot.receivedEndSu &&
|
||||
snapshot.ackQueuedEndSu === snapshot.receivedEndSu
|
||||
)
|
||||
}
|
||||
|
||||
private maybeResolveTerminal(identity: PtySourceDeliveryIdentity): void {
|
||||
if (!this.isTerminal(identity)) {
|
||||
return
|
||||
}
|
||||
const group = this.terminalWaiters.get(ptySourceDeliveryKey(identity))
|
||||
if (!group) {
|
||||
return
|
||||
}
|
||||
this.terminalWaiters.delete(ptySourceDeliveryKey(identity))
|
||||
for (const waiter of group.waiters) {
|
||||
waiter.resolve()
|
||||
}
|
||||
}
|
||||
|
||||
private rejectWaiters(predicate: (group: TerminalWaiterGroup) => boolean, error: Error): void {
|
||||
for (const [key, group] of this.terminalWaiters) {
|
||||
if (!predicate(group)) {
|
||||
continue
|
||||
}
|
||||
this.terminalWaiters.delete(key)
|
||||
for (const waiter of group.waiters) {
|
||||
waiter.reject(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private requireSpanIdentity(
|
||||
transition: Pick<SshPtySourceObligationTransition, 'identity' | 'spanId'>
|
||||
): void {
|
||||
if (!samePtySourceDelivery(this.ledger.spanIdentity(transition.spanId), transition.identity)) {
|
||||
throw new Error('SSH PTY source obligation transition has a stale delivery identity')
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import type {
|
||||
PtySourceDeliveryIdentity,
|
||||
PtySourceSpan
|
||||
} from '../../shared/pty-source-credit-contract'
|
||||
import { SshPtySourceObligationLedger } from './ssh-pty-source-obligation-ledger'
|
||||
|
||||
function identity(
|
||||
deliveryToken = 'token-1',
|
||||
overrides: Partial<PtySourceDeliveryIdentity> = {}
|
||||
): PtySourceDeliveryIdentity {
|
||||
return {
|
||||
id: 'pty-1',
|
||||
providerGeneration: 1,
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken,
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
function span(
|
||||
owner: PtySourceDeliveryIdentity,
|
||||
spanId: string,
|
||||
sourceStartSu: number,
|
||||
data: string
|
||||
): PtySourceSpan {
|
||||
return Object.freeze({
|
||||
...owner,
|
||||
spanId,
|
||||
sourceStartSu,
|
||||
sourceEndSu: sourceStartSu + data.length,
|
||||
displayStart: sourceStartSu,
|
||||
displayEnd: sourceStartSu + data.length,
|
||||
data,
|
||||
splittable: true,
|
||||
transform: Object.freeze({
|
||||
transformed: false,
|
||||
rawLengthSu: data.length,
|
||||
scalarSafe: true
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function commitSpan(
|
||||
ledger: SshPtySourceObligationLedger,
|
||||
owner: PtySourceDeliveryIdentity,
|
||||
sourceSpan: PtySourceSpan,
|
||||
consumers: ('model' | 'desktop')[] = ['model', 'desktop']
|
||||
) {
|
||||
const reservation = ledger.reserve(owner, sourceSpan, consumers)
|
||||
ledger.commit(reservation)
|
||||
return reservation
|
||||
}
|
||||
|
||||
describe('SshPtySourceObligationLedger', () => {
|
||||
it('rolls back an uncommitted admission without consuming its source coordinate', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owner = identity()
|
||||
ledger.open(owner)
|
||||
const first = ledger.reserve(owner, span(owner, 'span-1', 0, 'abc'), ['model'])
|
||||
|
||||
expect(ledger.rollback(first)).toBe(true)
|
||||
const retry = ledger.reserve(owner, span(owner, 'span-2', 0, 'abc'), ['model'])
|
||||
ledger.commit(retry)
|
||||
expect(ledger.snapshot(owner).receivedEndSu).toBe(3)
|
||||
})
|
||||
|
||||
it('rolls back a committed tail while every obligation is still open', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owner = identity()
|
||||
ledger.open(owner)
|
||||
const reservation = commitSpan(ledger, owner, span(owner, 'span-1', 0, 'abc'))
|
||||
|
||||
expect(ledger.rollbackCommitted(reservation)).toBe(true)
|
||||
expect(ledger.snapshot(owner)).toMatchObject({ receivedEndSu: 0, openSpans: 0 })
|
||||
expect(() => ledger.spanIdentity('span-1')).toThrow('Unknown or reclaimed')
|
||||
expect(
|
||||
ledger.commit(ledger.reserve(owner, span(owner, 'span-2', 0, 'abc'), ['model']))
|
||||
).toBeUndefined()
|
||||
})
|
||||
|
||||
it('keeps terminal, queued, and successfully published ACK ends independent', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owner = identity()
|
||||
ledger.open(owner)
|
||||
commitSpan(ledger, owner, span(owner, 'span-1', 0, 'abcd'))
|
||||
|
||||
ledger.settle('span-1', 'model', 'emulator-receipt')
|
||||
expect(ledger.snapshot(owner)).toMatchObject({
|
||||
obligationsTerminalEndSu: 0,
|
||||
ackQueuedEndSu: 0,
|
||||
ackPublishedEndSu: 0
|
||||
})
|
||||
ledger.settle('span-1', 'desktop', 'renderer-parse')
|
||||
expect(ledger.snapshot(owner)).toMatchObject({
|
||||
obligationsTerminalEndSu: 4,
|
||||
ackQueuedEndSu: 0,
|
||||
ackPublishedEndSu: 0
|
||||
})
|
||||
const publication = ledger.queueAck(owner)!
|
||||
expect(ledger.snapshot(owner)).toMatchObject({
|
||||
obligationsTerminalEndSu: 4,
|
||||
ackQueuedEndSu: 4,
|
||||
ackPublishedEndSu: 0
|
||||
})
|
||||
publication.onSettled({ ok: false, error: new Error('write callback failed') })
|
||||
expect(ledger.snapshot(owner)).toMatchObject({ ackQueuedEndSu: 4, ackPublishedEndSu: 0 })
|
||||
ledger.retryQueuedAck(owner)!.onSettled({ ok: true })
|
||||
expect(ledger.snapshot(owner)).toMatchObject({ ackPublishedEndSu: 4, openSpans: 0 })
|
||||
})
|
||||
|
||||
it('checkpoints only the contiguous model-settled prefix', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owner = identity()
|
||||
ledger.open(owner)
|
||||
commitSpan(ledger, owner, span(owner, 'span-1', 0, 'abcd'))
|
||||
commitSpan(ledger, owner, span(owner, 'span-2', 4, 'efgh'))
|
||||
|
||||
expect(ledger.modelAcceptedEnd(owner)).toBe(0)
|
||||
ledger.settle('span-2', 'model', 'out-of-order')
|
||||
expect(ledger.modelAcceptedEnd(owner)).toBe(0)
|
||||
ledger.settle('span-1', 'model', 'emulator-receipt')
|
||||
expect(ledger.modelAcceptedEnd(owner)).toBe(8)
|
||||
})
|
||||
|
||||
it('continues checkpoints from the reclaimed ACK-published prefix', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owner = identity()
|
||||
ledger.open(owner)
|
||||
commitSpan(ledger, owner, span(owner, 'span-1', 0, 'abcd'))
|
||||
commitSpan(ledger, owner, span(owner, 'span-2', 4, 'efgh'))
|
||||
ledger.settle('span-1', 'model', 'emulator-receipt')
|
||||
ledger.settle('span-1', 'desktop', 'renderer-parse')
|
||||
ledger.queueAck(owner)!.onSettled({ ok: true })
|
||||
|
||||
expect(ledger.snapshot(owner)).toMatchObject({ ackPublishedEndSu: 4, openSpans: 1 })
|
||||
expect(ledger.modelAcceptedEnd(owner)).toBe(4)
|
||||
ledger.settle('span-2', 'model', 'emulator-receipt')
|
||||
expect(ledger.modelAcceptedEnd(owner)).toBe(8)
|
||||
})
|
||||
|
||||
it('requires an exact transfer fence before a desktop obligation becomes terminal', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owner = identity()
|
||||
ledger.open(owner)
|
||||
commitSpan(ledger, owner, span(owner, 'span-1', 0, 'abc'))
|
||||
ledger.settle('span-1', 'model', 'emulator-receipt')
|
||||
|
||||
expect(ledger.beginTransfer('span-1', 'desktop', 'model', 'renderer-send-failed')).toBe(true)
|
||||
expect(ledger.snapshot(owner).obligationsTerminalEndSu).toBe(0)
|
||||
expect(ledger.commitTransfer('span-1', 'desktop')).toBe(true)
|
||||
expect(ledger.snapshot(owner).obligationsTerminalEndSu).toBe(3)
|
||||
expect(ledger.obligation('span-1', 'desktop')).toMatchObject({
|
||||
state: 'transferred',
|
||||
to: 'model'
|
||||
})
|
||||
})
|
||||
|
||||
it('keeps sealed exit state until final ACK publication succeeds', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owner = identity()
|
||||
ledger.open(owner)
|
||||
commitSpan(ledger, owner, span(owner, 'tail', 0, 'tail'), ['model'])
|
||||
ledger.seal(owner)
|
||||
expect(() => ledger.markExitPublished(owner)).toThrow('terminal ACK')
|
||||
ledger.settle('tail', 'model', 'emulator-receipt')
|
||||
const publication = ledger.queueAck(owner)!
|
||||
ledger.markExitPublished(owner)
|
||||
expect(ledger.snapshot(owner).state).toBe('sealed-unsettled')
|
||||
publication.onSettled({ ok: true })
|
||||
expect(ledger.snapshot(owner).state).toBe('closed')
|
||||
})
|
||||
|
||||
it('publishes token cancellation intent before accepting timeout cleanup proof', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owner = identity()
|
||||
ledger.open(owner)
|
||||
commitSpan(ledger, owner, span(owner, 'tail', 0, 'tail'), ['model'])
|
||||
ledger.seal(owner)
|
||||
|
||||
expect(() => ledger.applyCancellationProof(owner, { sentEndSu: 4, creditedEndSu: 0 })).toThrow()
|
||||
expect(ledger.beginExitTimeout(owner)).toEqual({
|
||||
id: owner.id,
|
||||
deliveryToken: owner.deliveryToken,
|
||||
clientGeneration: owner.clientGeneration,
|
||||
ownerGeneration: owner.ownerGeneration
|
||||
})
|
||||
ledger.applyCancellationProof(owner, { sentEndSu: 4, creditedEndSu: 0 })
|
||||
expect(ledger.snapshot(owner)).toMatchObject({ state: 'closed', openSpans: 0 })
|
||||
})
|
||||
|
||||
it('ignores a late successful write callback after generation-close proof', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owner = identity()
|
||||
ledger.open(owner)
|
||||
commitSpan(ledger, owner, span(owner, 'span-1', 0, 'abc'), ['model'])
|
||||
ledger.settle('span-1', 'model', 'accepted')
|
||||
const publication = ledger.queueAck(owner)!
|
||||
|
||||
expect(ledger.closeGeneration(1, 'provider-closed')).toBe(1)
|
||||
publication.onSettled({ ok: true })
|
||||
expect(ledger.snapshot(owner)).toMatchObject({
|
||||
state: 'closed',
|
||||
generationClosed: true,
|
||||
ackPublishedEndSu: 0
|
||||
})
|
||||
expect(() => ledger.obligation('span-1', 'model')).toThrow('reclaimed')
|
||||
})
|
||||
|
||||
it('rejects stale generations, tokens, and non-contiguous source spans', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owner = identity()
|
||||
ledger.open(owner)
|
||||
expect(() =>
|
||||
ledger.reserve(identity('stale'), span(identity('stale'), 'stale', 0, 'x'), ['model'])
|
||||
).toThrow('stale')
|
||||
expect(() => ledger.reserve(owner, span(owner, 'gap', 1, 'x'), ['model'])).toThrow(
|
||||
'non-contiguous'
|
||||
)
|
||||
})
|
||||
|
||||
it('requires cancellation proof to match the exact local received and published ends', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owner = identity()
|
||||
ledger.open(owner)
|
||||
commitSpan(ledger, owner, span(owner, 'tail', 0, 'tail'), ['model'])
|
||||
ledger.seal(owner)
|
||||
ledger.beginExitTimeout(owner)
|
||||
|
||||
expect(() => ledger.applyCancellationProof(owner, { sentEndSu: 3, creditedEndSu: 0 })).toThrow(
|
||||
'invalid'
|
||||
)
|
||||
expect(() => ledger.applyCancellationProof(owner, { sentEndSu: 4, creditedEndSu: 1 })).toThrow(
|
||||
'invalid'
|
||||
)
|
||||
})
|
||||
|
||||
it('applies recovery cancellation proof over a locally admitted prefix', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owner = identity()
|
||||
ledger.open(owner, 4)
|
||||
commitSpan(ledger, owner, span(owner, 'recovery', 4, 'tail'), ['model'])
|
||||
|
||||
ledger.applyRecoveryCancellationProof(owner, { sentEndSu: 12, creditedEndSu: 4 })
|
||||
|
||||
expect(ledger.snapshot(owner)).toMatchObject({ state: 'closed', openSpans: 0 })
|
||||
})
|
||||
|
||||
it('rejects recovery cancellation proof that misses local intake state', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owner = identity()
|
||||
ledger.open(owner, 4)
|
||||
commitSpan(ledger, owner, span(owner, 'recovery', 4, 'tail'), ['model'])
|
||||
|
||||
expect(() =>
|
||||
ledger.applyRecoveryCancellationProof(owner, { sentEndSu: 7, creditedEndSu: 4 })
|
||||
).toThrow('invalid')
|
||||
expect(() =>
|
||||
ledger.applyRecoveryCancellationProof(owner, { sentEndSu: 8, creditedEndSu: 5 })
|
||||
).toThrow('invalid')
|
||||
})
|
||||
|
||||
it('bounds closed-token tombstones and removes committed reservation indexes', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owners = Array.from({ length: 300 }, (_, index) =>
|
||||
identity(`token-${index}`, {
|
||||
id: `pty-${index}`,
|
||||
ptyIncarnation: `incarnation-${index}`
|
||||
})
|
||||
)
|
||||
for (const [index, owner] of owners.entries()) {
|
||||
ledger.open(owner)
|
||||
commitSpan(ledger, owner, span(owner, `span-${index}`, 0, 'x'), ['model'])
|
||||
ledger.closeGeneration(1, 'generation-closed')
|
||||
}
|
||||
|
||||
expect(() => ledger.snapshot(owners[0])).toThrow('stale')
|
||||
expect(ledger.snapshot(owners.at(-1)!)).toMatchObject({ state: 'closed' })
|
||||
})
|
||||
|
||||
it('reclaims uncommitted reservations on exact cancellation proof', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owner = identity()
|
||||
ledger.open(owner)
|
||||
const reservation = ledger.reserve(owner, span(owner, 'pending', 0, 'x'), ['model'])
|
||||
ledger.seal(owner)
|
||||
ledger.beginExitTimeout(owner)
|
||||
|
||||
ledger.applyCancellationProof(owner, { sentEndSu: 0, creditedEndSu: 0 })
|
||||
|
||||
expect(ledger.rollback(reservation)).toBe(false)
|
||||
expect(ledger.snapshot(owner)).toMatchObject({ state: 'closed', openSpans: 0 })
|
||||
})
|
||||
|
||||
it('rejects reopening a recently closed one-use token', () => {
|
||||
const ledger = new SshPtySourceObligationLedger()
|
||||
const owner = identity()
|
||||
ledger.open(owner)
|
||||
ledger.closeGeneration(owner.providerGeneration, 'closed')
|
||||
|
||||
expect(() => ledger.open(owner)).toThrow('already used')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,329 @@
|
||||
import {
|
||||
ptySourceDeliveryKey,
|
||||
samePtySourceDelivery,
|
||||
type PtySourceDeliveryIdentity,
|
||||
type PtySourceSpan
|
||||
} from '../../shared/pty-source-credit-contract'
|
||||
import {
|
||||
assertNonNegativeSafeInteger,
|
||||
assertPtySourceIdentity,
|
||||
assertPtySourceSpan
|
||||
} from '../../shared/pty-source-credit-validation'
|
||||
import type {
|
||||
SshPtySourceAckPublication,
|
||||
SshPtySourceAdmissionReservation,
|
||||
SshPtySourceConsumerId,
|
||||
SshPtySourceObligationState,
|
||||
SshPtySourceTokenSnapshot
|
||||
} from './ssh-pty-source-obligation-contract'
|
||||
import { createSshPtySourceAckPublication } from './ssh-pty-source-ack-publication'
|
||||
import {
|
||||
beginSourceExitTimeout,
|
||||
cancelOpenSourceObligations,
|
||||
closeAllSourceTokens,
|
||||
closeSourceGeneration,
|
||||
closeSourceToken,
|
||||
createSourceSpanRecord,
|
||||
createSourceToken,
|
||||
markSourceExitPublished,
|
||||
requireSourceSpan,
|
||||
rollbackCommittedSourceSpan,
|
||||
sealSourceToken,
|
||||
snapshotSourceToken,
|
||||
type ReservationRecord,
|
||||
type TokenRecord
|
||||
} from './ssh-pty-source-obligation-state'
|
||||
import {
|
||||
applySourceRecoveryCancellationProof,
|
||||
cancelSourceObligationTransfer,
|
||||
commitSourceObligationTransfer,
|
||||
modelAcceptedSourceEnd,
|
||||
rollbackSourceObligationTransfer,
|
||||
transitionOpenSourceObligation
|
||||
} from './ssh-pty-source-obligation-transitions'
|
||||
|
||||
export type {
|
||||
SshPtySourceAckPublication,
|
||||
SshPtySourceAdmissionReservation,
|
||||
SshPtySourceConsumerId,
|
||||
SshPtySourceObligationState,
|
||||
SshPtySourceTokenSnapshot
|
||||
} from './ssh-pty-source-obligation-contract'
|
||||
|
||||
export class SshPtySourceObligationLedger {
|
||||
private readonly tokens = new Map<string, TokenRecord>()
|
||||
private readonly closedSnapshots = new Map<string, SshPtySourceTokenSnapshot>()
|
||||
private readonly reservations = new Map<string, ReservationRecord>()
|
||||
private readonly spanOwners = new Map<string, TokenRecord>()
|
||||
private nextReservationId = 1
|
||||
|
||||
constructor(
|
||||
private readonly onTokenClosed: (identity: PtySourceDeliveryIdentity) => void = () => {}
|
||||
) {}
|
||||
|
||||
open(identity: PtySourceDeliveryIdentity, checkpointSourceEndSu = 0): void {
|
||||
assertPtySourceIdentity(identity)
|
||||
assertNonNegativeSafeInteger(checkpointSourceEndSu, 'checkpointSourceEndSu')
|
||||
const key = ptySourceDeliveryKey(identity)
|
||||
if (this.tokens.has(key) || this.closedSnapshots.has(key)) {
|
||||
throw new Error('SSH PTY source token was already used')
|
||||
}
|
||||
this.tokens.set(key, createSourceToken(identity, checkpointSourceEndSu))
|
||||
}
|
||||
|
||||
reserve(
|
||||
identity: PtySourceDeliveryIdentity,
|
||||
span: PtySourceSpan,
|
||||
requiredConsumers: readonly SshPtySourceConsumerId[]
|
||||
): SshPtySourceAdmissionReservation {
|
||||
const token = this.requireToken(identity)
|
||||
if (token.state !== 'active') {
|
||||
throw new Error('SSH PTY source token no longer admits data')
|
||||
}
|
||||
assertPtySourceSpan(span)
|
||||
if (
|
||||
!samePtySourceDelivery(token.identity, span) ||
|
||||
span.sourceStartSu !== token.receivedEndSu ||
|
||||
this.spanOwners.has(span.spanId)
|
||||
) {
|
||||
throw new Error('SSH PTY source span is stale, duplicate, or non-contiguous')
|
||||
}
|
||||
const uniqueConsumers = Array.from(new Set(requiredConsumers))
|
||||
if (!uniqueConsumers.includes('model')) {
|
||||
throw new Error('SSH PTY source span requires the terminal model obligation')
|
||||
}
|
||||
const reservation = Object.freeze({
|
||||
reservationId: `ssh-source-admission:${this.nextReservationId++}`,
|
||||
span,
|
||||
requiredConsumers: Object.freeze(uniqueConsumers)
|
||||
})
|
||||
this.reservations.set(reservation.reservationId, {
|
||||
reservation,
|
||||
state: 'reserved'
|
||||
})
|
||||
return reservation
|
||||
}
|
||||
|
||||
commit(reservation: SshPtySourceAdmissionReservation): void {
|
||||
const record = this.requireReservation(reservation)
|
||||
if (record.state !== 'reserved') {
|
||||
throw new Error('SSH PTY source admission reservation is not pending')
|
||||
}
|
||||
const token = this.requireToken(reservation.span)
|
||||
if (token.state !== 'active' || token.receivedEndSu !== reservation.span.sourceStartSu) {
|
||||
throw new Error('SSH PTY source admission reservation became stale')
|
||||
}
|
||||
const spanRecord = createSourceSpanRecord(reservation.span, reservation.requiredConsumers)
|
||||
token.spans.push(spanRecord)
|
||||
token.receivedEndSu = reservation.span.sourceEndSu
|
||||
this.spanOwners.set(reservation.span.spanId, token)
|
||||
this.reservations.delete(reservation.reservationId)
|
||||
}
|
||||
|
||||
rollback(reservation: SshPtySourceAdmissionReservation): boolean {
|
||||
const record = this.reservations.get(reservation.reservationId)
|
||||
if (!record || record.reservation !== reservation || record.state !== 'reserved') {
|
||||
return false
|
||||
}
|
||||
this.reservations.delete(reservation.reservationId)
|
||||
return true
|
||||
}
|
||||
|
||||
rollbackCommitted(reservation: SshPtySourceAdmissionReservation): boolean {
|
||||
const token = this.tokens.get(ptySourceDeliveryKey(reservation.span))
|
||||
if (!token || !samePtySourceDelivery(token.identity, reservation.span)) {
|
||||
return false
|
||||
}
|
||||
return rollbackCommittedSourceSpan(token, reservation, this.spanOwners)
|
||||
}
|
||||
|
||||
settle(spanId: string, consumer: SshPtySourceConsumerId, reason: string): boolean {
|
||||
return this.transitionOpen(spanId, consumer, Object.freeze({ state: 'settled', reason }))
|
||||
}
|
||||
|
||||
beginTransfer(
|
||||
spanId: string,
|
||||
consumer: SshPtySourceConsumerId,
|
||||
to: SshPtySourceConsumerId,
|
||||
reason: string
|
||||
): boolean {
|
||||
return this.transitionOpen(
|
||||
spanId,
|
||||
consumer,
|
||||
Object.freeze({ state: 'transferring', to, reason })
|
||||
)
|
||||
}
|
||||
|
||||
commitTransfer(spanId: string, consumer: SshPtySourceConsumerId): boolean {
|
||||
return commitSourceObligationTransfer(this.spanOwners, spanId, consumer)
|
||||
}
|
||||
|
||||
cancelTransfer(spanId: string, consumer: SshPtySourceConsumerId, reason: string): boolean {
|
||||
return cancelSourceObligationTransfer(this.spanOwners, spanId, consumer, reason)
|
||||
}
|
||||
|
||||
rollbackTransfer(spanId: string, consumer: SshPtySourceConsumerId): boolean {
|
||||
return rollbackSourceObligationTransfer(this.spanOwners, spanId, consumer)
|
||||
}
|
||||
|
||||
queueAck(identity: PtySourceDeliveryIdentity): SshPtySourceAckPublication | null {
|
||||
const token = this.requireToken(identity)
|
||||
if (token.obligationsTerminalEndSu <= token.ackQueuedEndSu) {
|
||||
return null
|
||||
}
|
||||
const endSu = token.obligationsTerminalEndSu
|
||||
token.ackQueuedEndSu = endSu
|
||||
return createSshPtySourceAckPublication(token, endSu, this.spanOwners, () =>
|
||||
this.maybeClose(token)
|
||||
)
|
||||
}
|
||||
|
||||
retryQueuedAck(identity: PtySourceDeliveryIdentity): SshPtySourceAckPublication | null {
|
||||
const token = this.requireToken(identity)
|
||||
if (token.ackQueuedEndSu <= token.ackPublishedEndSu) {
|
||||
return null
|
||||
}
|
||||
const endSu = token.ackQueuedEndSu
|
||||
return createSshPtySourceAckPublication(token, endSu, this.spanOwners, () =>
|
||||
this.maybeClose(token)
|
||||
)
|
||||
}
|
||||
|
||||
seal(identity: PtySourceDeliveryIdentity): void {
|
||||
sealSourceToken(this.requireToken(identity))
|
||||
}
|
||||
|
||||
markExitPublished(identity: PtySourceDeliveryIdentity): void {
|
||||
const token = this.requireToken(identity)
|
||||
markSourceExitPublished(token)
|
||||
this.maybeClose(token)
|
||||
}
|
||||
|
||||
beginExitTimeout(identity: PtySourceDeliveryIdentity): Readonly<{
|
||||
id: string
|
||||
deliveryToken: string
|
||||
clientGeneration: number
|
||||
ownerGeneration: number
|
||||
}> {
|
||||
return beginSourceExitTimeout(this.requireToken(identity))
|
||||
}
|
||||
|
||||
applyCancellationProof(
|
||||
identity: PtySourceDeliveryIdentity,
|
||||
proof: Readonly<{ sentEndSu: number; creditedEndSu: number }>
|
||||
): void {
|
||||
const token = this.requireToken(identity)
|
||||
if (
|
||||
token.state !== 'canceling' ||
|
||||
proof.sentEndSu !== token.receivedEndSu ||
|
||||
proof.creditedEndSu !== token.ackPublishedEndSu
|
||||
) {
|
||||
throw new Error('SSH PTY source cancellation proof is stale or invalid')
|
||||
}
|
||||
cancelOpenSourceObligations(token, 'relay-cancellation-proof')
|
||||
this.closeToken(token)
|
||||
}
|
||||
|
||||
applyRecoveryCancellationProof(
|
||||
identity: PtySourceDeliveryIdentity,
|
||||
proof: Readonly<{ sentEndSu: number; creditedEndSu: number }>
|
||||
): void {
|
||||
const token = this.requireToken(identity)
|
||||
applySourceRecoveryCancellationProof(token, proof)
|
||||
this.closeToken(token)
|
||||
}
|
||||
|
||||
closeGeneration(providerGeneration: number, reason: string): number {
|
||||
return closeSourceGeneration(
|
||||
this.tokens,
|
||||
this.reservations,
|
||||
providerGeneration,
|
||||
reason,
|
||||
(token) => this.closeToken(token)
|
||||
)
|
||||
}
|
||||
|
||||
closeAll(reason: string): number {
|
||||
return closeAllSourceTokens(this.tokens, this.reservations, reason, (token) =>
|
||||
this.closeToken(token)
|
||||
)
|
||||
}
|
||||
|
||||
snapshot(identity: PtySourceDeliveryIdentity): SshPtySourceTokenSnapshot {
|
||||
const key = ptySourceDeliveryKey(identity)
|
||||
const token = this.tokens.get(key)
|
||||
if (token && samePtySourceDelivery(token.identity, identity)) {
|
||||
return snapshotSourceToken(token)
|
||||
}
|
||||
const closed = this.closedSnapshots.get(key)
|
||||
if (closed && samePtySourceDelivery(closed, identity)) {
|
||||
return closed
|
||||
}
|
||||
throw new Error('Unknown or stale SSH PTY source token')
|
||||
}
|
||||
|
||||
modelAcceptedEnd(identity: PtySourceDeliveryIdentity): number {
|
||||
return modelAcceptedSourceEnd(this.requireToken(identity))
|
||||
}
|
||||
|
||||
obligation(spanId: string, consumer: SshPtySourceConsumerId): SshPtySourceObligationState {
|
||||
const obligation = requireSourceSpan(this.spanOwners, spanId).span.obligations.get(consumer)
|
||||
if (!obligation) {
|
||||
throw new Error('SSH PTY source consumer obligation does not exist')
|
||||
}
|
||||
return obligation
|
||||
}
|
||||
|
||||
spanIdentity(spanId: string): PtySourceSpan {
|
||||
return requireSourceSpan(this.spanOwners, spanId).span.span
|
||||
}
|
||||
|
||||
hasRetainedSpan(spanId: string): boolean {
|
||||
return this.spanOwners.has(spanId)
|
||||
}
|
||||
|
||||
private transitionOpen(
|
||||
spanId: string,
|
||||
consumer: SshPtySourceConsumerId,
|
||||
next: SshPtySourceObligationState
|
||||
): boolean {
|
||||
return transitionOpenSourceObligation(this.spanOwners, spanId, consumer, next)
|
||||
}
|
||||
|
||||
private maybeClose(token: TokenRecord): void {
|
||||
if (
|
||||
token.state === 'sealed-unsettled' &&
|
||||
token.exitPublished &&
|
||||
token.ackPublishedEndSu === token.receivedEndSu
|
||||
) {
|
||||
this.closeToken(token)
|
||||
}
|
||||
}
|
||||
|
||||
private closeToken(token: TokenRecord): void {
|
||||
closeSourceToken(
|
||||
token,
|
||||
this.tokens,
|
||||
this.closedSnapshots,
|
||||
this.reservations,
|
||||
this.spanOwners,
|
||||
this.onTokenClosed
|
||||
)
|
||||
}
|
||||
|
||||
private requireReservation(reservation: SshPtySourceAdmissionReservation): ReservationRecord {
|
||||
const record = this.reservations.get(reservation.reservationId)
|
||||
if (!record || record.reservation !== reservation) {
|
||||
throw new Error('Unknown SSH PTY source admission reservation')
|
||||
}
|
||||
return record
|
||||
}
|
||||
|
||||
private requireToken(identity: PtySourceDeliveryIdentity): TokenRecord {
|
||||
const token = this.tokens.get(ptySourceDeliveryKey(identity))
|
||||
if (!token || !samePtySourceDelivery(token.identity, identity)) {
|
||||
throw new Error('Unknown or stale SSH PTY source token')
|
||||
}
|
||||
return token
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
import type {
|
||||
PtySourceDeliveryIdentity,
|
||||
PtySourceSpan
|
||||
} from '../../shared/pty-source-credit-contract'
|
||||
import {
|
||||
ptySourceDeliveryKey,
|
||||
samePtySourceDelivery
|
||||
} from '../../shared/pty-source-credit-contract'
|
||||
import type {
|
||||
SshPtySourceAdmissionReservation,
|
||||
SshPtySourceConsumerId,
|
||||
SshPtySourceObligationState,
|
||||
SshPtySourceTokenSnapshot
|
||||
} from './ssh-pty-source-obligation-contract'
|
||||
|
||||
export const CLOSED_SOURCE_TOKEN_TOMBSTONE_LIMIT = 256
|
||||
|
||||
export type SpanRecord = {
|
||||
span: PtySourceSpan
|
||||
obligations: Map<SshPtySourceConsumerId, SshPtySourceObligationState>
|
||||
}
|
||||
|
||||
export type ReservationRecord = {
|
||||
reservation: SshPtySourceAdmissionReservation
|
||||
state: 'reserved' | 'committed' | 'rolled-back'
|
||||
}
|
||||
|
||||
export type TokenRecord = {
|
||||
identity: PtySourceDeliveryIdentity
|
||||
state: 'active' | 'sealed-unsettled' | 'canceling' | 'closed'
|
||||
checkpointSourceEndSu: number
|
||||
receivedEndSu: number
|
||||
obligationsTerminalEndSu: number
|
||||
ackQueuedEndSu: number
|
||||
ackPublishedEndSu: number
|
||||
spans: SpanRecord[]
|
||||
exitPublished: boolean
|
||||
generationClosed: boolean
|
||||
}
|
||||
|
||||
export function createSourceToken(
|
||||
identity: PtySourceDeliveryIdentity,
|
||||
checkpointSourceEndSu: number
|
||||
): TokenRecord {
|
||||
return {
|
||||
identity: Object.freeze({ ...identity }),
|
||||
state: 'active',
|
||||
checkpointSourceEndSu,
|
||||
receivedEndSu: checkpointSourceEndSu,
|
||||
obligationsTerminalEndSu: checkpointSourceEndSu,
|
||||
ackQueuedEndSu: checkpointSourceEndSu,
|
||||
ackPublishedEndSu: checkpointSourceEndSu,
|
||||
spans: [],
|
||||
exitPublished: false,
|
||||
generationClosed: false
|
||||
}
|
||||
}
|
||||
|
||||
export function createSourceSpanRecord(
|
||||
span: PtySourceSpan,
|
||||
consumers: readonly SshPtySourceConsumerId[]
|
||||
): SpanRecord {
|
||||
return {
|
||||
span,
|
||||
obligations: new Map(
|
||||
consumers.map((consumer) => [consumer, Object.freeze({ state: 'open' as const })])
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export function sealSourceToken(token: TokenRecord): void {
|
||||
if (token.state !== 'active') {
|
||||
throw new Error('SSH PTY source token cannot be sealed from its current state')
|
||||
}
|
||||
token.state = 'sealed-unsettled'
|
||||
}
|
||||
|
||||
export function markSourceExitPublished(token: TokenRecord): void {
|
||||
if (token.state !== 'sealed-unsettled') {
|
||||
throw new Error('SSH PTY source exit publication requires a sealed token')
|
||||
}
|
||||
if (
|
||||
token.obligationsTerminalEndSu !== token.receivedEndSu ||
|
||||
token.ackQueuedEndSu !== token.receivedEndSu
|
||||
) {
|
||||
throw new Error('SSH PTY source exit cannot publish before terminal ACK queueing')
|
||||
}
|
||||
token.exitPublished = true
|
||||
}
|
||||
|
||||
export function beginSourceExitTimeout(token: TokenRecord): Readonly<{
|
||||
id: string
|
||||
deliveryToken: string
|
||||
clientGeneration: number
|
||||
ownerGeneration: number
|
||||
}> {
|
||||
if (token.state !== 'sealed-unsettled') {
|
||||
throw new Error('SSH PTY source exit timeout requires a sealed token')
|
||||
}
|
||||
token.state = 'canceling'
|
||||
return Object.freeze({
|
||||
id: token.identity.id,
|
||||
deliveryToken: token.identity.deliveryToken,
|
||||
clientGeneration: token.identity.clientGeneration,
|
||||
ownerGeneration: token.identity.ownerGeneration
|
||||
})
|
||||
}
|
||||
|
||||
function obligationIsTerminal(obligation: SshPtySourceObligationState): boolean {
|
||||
return (
|
||||
obligation.state === 'settled' ||
|
||||
obligation.state === 'transferred' ||
|
||||
obligation.state === 'canceled'
|
||||
)
|
||||
}
|
||||
|
||||
export function snapshotSourceToken(token: TokenRecord): SshPtySourceTokenSnapshot {
|
||||
return Object.freeze({
|
||||
...token.identity,
|
||||
state: token.state,
|
||||
receivedEndSu: token.receivedEndSu,
|
||||
obligationsTerminalEndSu: token.obligationsTerminalEndSu,
|
||||
ackQueuedEndSu: token.ackQueuedEndSu,
|
||||
ackPublishedEndSu: token.ackPublishedEndSu,
|
||||
openSpans: token.spans.length,
|
||||
exitPublished: token.exitPublished,
|
||||
generationClosed: token.generationClosed
|
||||
})
|
||||
}
|
||||
|
||||
export function advanceSourceTerminalEnd(token: TokenRecord): void {
|
||||
let endSu = token.obligationsTerminalEndSu
|
||||
for (const record of token.spans) {
|
||||
if (record.span.sourceEndSu <= endSu) {
|
||||
continue
|
||||
}
|
||||
if (
|
||||
record.span.sourceStartSu !== endSu ||
|
||||
!Array.from(record.obligations.values()).every(obligationIsTerminal)
|
||||
) {
|
||||
break
|
||||
}
|
||||
endSu = record.span.sourceEndSu
|
||||
}
|
||||
token.obligationsTerminalEndSu = endSu
|
||||
}
|
||||
|
||||
export function cancelOpenSourceObligations(token: TokenRecord, reason: string): void {
|
||||
for (const record of token.spans) {
|
||||
for (const [consumer, obligation] of record.obligations) {
|
||||
if (obligation.state === 'open' || obligation.state === 'transferring') {
|
||||
record.obligations.set(consumer, Object.freeze({ state: 'canceled', reason }))
|
||||
}
|
||||
}
|
||||
}
|
||||
advanceSourceTerminalEnd(token)
|
||||
}
|
||||
|
||||
export function reclaimPublishedSourcePrefix(
|
||||
token: TokenRecord,
|
||||
spanOwners: Map<string, TokenRecord>
|
||||
): void {
|
||||
while (token.spans[0]?.span.sourceEndSu <= token.ackPublishedEndSu) {
|
||||
const record = token.spans.shift()!
|
||||
spanOwners.delete(record.span.spanId)
|
||||
}
|
||||
}
|
||||
|
||||
export function releaseSourceTokenSpans(
|
||||
token: TokenRecord,
|
||||
spanOwners: Map<string, TokenRecord>
|
||||
): void {
|
||||
for (const record of token.spans) {
|
||||
spanOwners.delete(record.span.spanId)
|
||||
}
|
||||
token.spans = []
|
||||
}
|
||||
|
||||
export function releaseSourceTokenReservations(
|
||||
token: TokenRecord,
|
||||
reservations: Map<string, ReservationRecord>
|
||||
): void {
|
||||
for (const [reservationId, record] of reservations) {
|
||||
if (samePtySourceDelivery(record.reservation.span, token.identity)) {
|
||||
record.state = 'rolled-back'
|
||||
reservations.delete(reservationId)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function rollbackCommittedSourceSpan(
|
||||
token: TokenRecord,
|
||||
reservation: SshPtySourceAdmissionReservation,
|
||||
spanOwners: Map<string, TokenRecord>
|
||||
): boolean {
|
||||
const last = token.spans.at(-1)
|
||||
if (
|
||||
last?.span !== reservation.span ||
|
||||
token.receivedEndSu !== reservation.span.sourceEndSu ||
|
||||
token.obligationsTerminalEndSu > reservation.span.sourceStartSu ||
|
||||
token.ackQueuedEndSu > reservation.span.sourceStartSu ||
|
||||
Array.from(last.obligations.values()).some((obligation) => obligation.state !== 'open')
|
||||
) {
|
||||
return false
|
||||
}
|
||||
token.spans.pop()
|
||||
token.receivedEndSu = reservation.span.sourceStartSu
|
||||
spanOwners.delete(reservation.span.spanId)
|
||||
return true
|
||||
}
|
||||
|
||||
export function requireSourceSpan(
|
||||
spanOwners: ReadonlyMap<string, TokenRecord>,
|
||||
spanId: string
|
||||
): { token: TokenRecord; span: SpanRecord } {
|
||||
const token = spanOwners.get(spanId)
|
||||
const span = token?.spans.find((candidate) => candidate.span.spanId === spanId)
|
||||
if (!token || !span) {
|
||||
throw new Error('Unknown or reclaimed SSH PTY source span')
|
||||
}
|
||||
return { token, span }
|
||||
}
|
||||
|
||||
export function closeSourceGeneration(
|
||||
tokens: Map<string, TokenRecord>,
|
||||
reservations: Map<string, ReservationRecord>,
|
||||
providerGeneration: number,
|
||||
reason: string,
|
||||
closeToken: (token: TokenRecord) => void
|
||||
): number {
|
||||
let closed = 0
|
||||
for (const token of Array.from(tokens.values())) {
|
||||
if (token.identity.providerGeneration !== providerGeneration || token.state === 'closed') {
|
||||
continue
|
||||
}
|
||||
token.generationClosed = true
|
||||
cancelOpenSourceObligations(token, reason)
|
||||
closeToken(token)
|
||||
closed++
|
||||
}
|
||||
for (const [id, record] of reservations) {
|
||||
if (
|
||||
record.state === 'reserved' &&
|
||||
record.reservation.span.providerGeneration === providerGeneration
|
||||
) {
|
||||
record.state = 'rolled-back'
|
||||
reservations.delete(id)
|
||||
}
|
||||
}
|
||||
return closed
|
||||
}
|
||||
|
||||
export function closeAllSourceTokens(
|
||||
tokens: Map<string, TokenRecord>,
|
||||
reservations: Map<string, ReservationRecord>,
|
||||
reason: string,
|
||||
closeToken: (token: TokenRecord) => void
|
||||
): number {
|
||||
let closed = 0
|
||||
const generations = new Set(
|
||||
Array.from(tokens.values(), (token) => token.identity.providerGeneration)
|
||||
)
|
||||
for (const providerGeneration of generations) {
|
||||
closed += closeSourceGeneration(tokens, reservations, providerGeneration, reason, closeToken)
|
||||
}
|
||||
return closed
|
||||
}
|
||||
|
||||
export function closeSourceToken(
|
||||
token: TokenRecord,
|
||||
tokens: Map<string, TokenRecord>,
|
||||
closedSnapshots: Map<string, SshPtySourceTokenSnapshot>,
|
||||
reservations: Map<string, ReservationRecord>,
|
||||
spanOwners: Map<string, TokenRecord>,
|
||||
onTokenClosed: (identity: PtySourceDeliveryIdentity) => void
|
||||
): void {
|
||||
token.state = 'closed'
|
||||
releaseSourceTokenSpans(token, spanOwners)
|
||||
releaseSourceTokenReservations(token, reservations)
|
||||
const key = ptySourceDeliveryKey(token.identity)
|
||||
tokens.delete(key)
|
||||
closedSnapshots.set(key, snapshotSourceToken(token))
|
||||
while (closedSnapshots.size > CLOSED_SOURCE_TOKEN_TOMBSTONE_LIMIT) {
|
||||
closedSnapshots.delete(closedSnapshots.keys().next().value!)
|
||||
}
|
||||
onTokenClosed(token.identity)
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
import type {
|
||||
SshPtySourceConsumerId,
|
||||
SshPtySourceObligationState
|
||||
} from './ssh-pty-source-obligation-contract'
|
||||
import {
|
||||
advanceSourceTerminalEnd,
|
||||
cancelOpenSourceObligations,
|
||||
requireSourceSpan,
|
||||
type TokenRecord
|
||||
} from './ssh-pty-source-obligation-state'
|
||||
|
||||
export function applySourceRecoveryCancellationProof(
|
||||
token: TokenRecord,
|
||||
proof: Readonly<{ sentEndSu: number; creditedEndSu: number }>
|
||||
): void {
|
||||
if (
|
||||
token.state !== 'active' ||
|
||||
proof.sentEndSu < token.receivedEndSu ||
|
||||
proof.creditedEndSu !== token.ackPublishedEndSu ||
|
||||
proof.creditedEndSu > token.receivedEndSu
|
||||
) {
|
||||
throw new Error('SSH PTY source recovery cancellation proof is stale or invalid')
|
||||
}
|
||||
cancelOpenSourceObligations(token, 'relay-recovery-cancellation-proof')
|
||||
}
|
||||
|
||||
export function transitionOpenSourceObligation(
|
||||
spanOwners: ReadonlyMap<string, TokenRecord>,
|
||||
spanId: string,
|
||||
consumer: SshPtySourceConsumerId,
|
||||
next: SshPtySourceObligationState
|
||||
): boolean {
|
||||
const { token, span } = requireSourceSpan(spanOwners, spanId)
|
||||
if (span.obligations.get(consumer)?.state !== 'open') {
|
||||
return false
|
||||
}
|
||||
span.obligations.set(consumer, next)
|
||||
advanceSourceTerminalEnd(token)
|
||||
return true
|
||||
}
|
||||
|
||||
export function commitSourceObligationTransfer(
|
||||
spanOwners: ReadonlyMap<string, TokenRecord>,
|
||||
spanId: string,
|
||||
consumer: SshPtySourceConsumerId
|
||||
): boolean {
|
||||
const { token, span } = requireSourceSpan(spanOwners, spanId)
|
||||
const current = span.obligations.get(consumer)
|
||||
if (current?.state !== 'transferring') {
|
||||
return false
|
||||
}
|
||||
span.obligations.set(
|
||||
consumer,
|
||||
Object.freeze({ state: 'transferred', to: current.to, reason: current.reason })
|
||||
)
|
||||
advanceSourceTerminalEnd(token)
|
||||
return true
|
||||
}
|
||||
|
||||
export function cancelSourceObligationTransfer(
|
||||
spanOwners: ReadonlyMap<string, TokenRecord>,
|
||||
spanId: string,
|
||||
consumer: SshPtySourceConsumerId,
|
||||
reason: string
|
||||
): boolean {
|
||||
const { token, span } = requireSourceSpan(spanOwners, spanId)
|
||||
if (span.obligations.get(consumer)?.state !== 'transferring') {
|
||||
return false
|
||||
}
|
||||
span.obligations.set(consumer, Object.freeze({ state: 'canceled', reason }))
|
||||
advanceSourceTerminalEnd(token)
|
||||
return true
|
||||
}
|
||||
|
||||
export function rollbackSourceObligationTransfer(
|
||||
spanOwners: ReadonlyMap<string, TokenRecord>,
|
||||
spanId: string,
|
||||
consumer: SshPtySourceConsumerId
|
||||
): boolean {
|
||||
const { span } = requireSourceSpan(spanOwners, spanId)
|
||||
if (span.obligations.get(consumer)?.state !== 'transferring') {
|
||||
return false
|
||||
}
|
||||
span.obligations.set(consumer, Object.freeze({ state: 'open' }))
|
||||
return true
|
||||
}
|
||||
|
||||
export function modelAcceptedSourceEnd(token: TokenRecord): number {
|
||||
let acceptedEndSu = token.ackPublishedEndSu
|
||||
for (const record of token.spans) {
|
||||
if (
|
||||
record.span.sourceStartSu !== acceptedEndSu ||
|
||||
record.obligations.get('model')?.state !== 'settled'
|
||||
) {
|
||||
break
|
||||
}
|
||||
acceptedEndSu = record.span.sourceEndSu
|
||||
}
|
||||
return acceptedEndSu
|
||||
}
|
||||
+132
-45
@@ -11,6 +11,8 @@ const {
|
||||
mockConnectionManager,
|
||||
mockDeployAndLaunchRelay,
|
||||
mockForceStopRelayForTarget,
|
||||
mockAcceptSshPtyOutputData,
|
||||
mockAcceptSshPtyOutputExit,
|
||||
mockMux,
|
||||
mockPtyProvider,
|
||||
mockFsProvider,
|
||||
@@ -49,10 +51,13 @@ const {
|
||||
},
|
||||
mockDeployAndLaunchRelay: vi.fn(),
|
||||
mockForceStopRelayForTarget: vi.fn(),
|
||||
mockAcceptSshPtyOutputData: vi.fn().mockResolvedValue({}),
|
||||
mockAcceptSshPtyOutputExit: vi.fn().mockResolvedValue(undefined),
|
||||
mockMux: {
|
||||
dispose: vi.fn(),
|
||||
isDisposed: vi.fn().mockReturnValue(false),
|
||||
onNotification: vi.fn(),
|
||||
onNotificationByMethod: vi.fn().mockReturnValue(() => {}),
|
||||
onRequest: vi.fn().mockReturnValue(() => {}),
|
||||
onDispose: vi.fn().mockReturnValue(() => {}),
|
||||
request: vi.fn().mockResolvedValue({}),
|
||||
@@ -65,7 +70,8 @@ const {
|
||||
onReplay: vi.fn(),
|
||||
attach: vi.fn(),
|
||||
attachForReconnect: vi.fn().mockResolvedValue({}),
|
||||
shutdown: vi.fn()
|
||||
shutdown: vi.fn(),
|
||||
providerGeneration: 0
|
||||
},
|
||||
mockFsProvider: {},
|
||||
mockGitProvider: {},
|
||||
@@ -100,6 +106,25 @@ vi.mock('electron', () => ({
|
||||
}
|
||||
}))
|
||||
|
||||
vi.mock('./ssh-pty-output-intake-registry', () => ({
|
||||
acceptSshPtyOutputData: mockAcceptSshPtyOutputData,
|
||||
acceptSshPtyOutputExit: mockAcceptSshPtyOutputExit,
|
||||
allocateSshPtyProviderGeneration: (() => {
|
||||
let generation = 0
|
||||
return () => ++generation
|
||||
})(),
|
||||
beginSshPtyOutputGenerationMigration: vi.fn(() => ({
|
||||
byPty: new Map(),
|
||||
completion: Promise.resolve()
|
||||
})),
|
||||
applySshPtySourceCancellationProof: vi.fn().mockReturnValue(false),
|
||||
applySshPtySourceRecoveryCancellationProof: vi.fn().mockReturnValue(false),
|
||||
closeSshPtyOutputGeneration: vi.fn(),
|
||||
getSshPtyAcceptedSourceCheckpoints: vi.fn().mockReturnValue([]),
|
||||
installSshPtySourceAckPublisher: vi.fn().mockReturnValue(() => {}),
|
||||
installSshPtySourceCancellationPublisher: vi.fn().mockReturnValue(() => {})
|
||||
}))
|
||||
|
||||
vi.mock('../ssh/ssh-connection-store', () => ({
|
||||
SshConnectionStore: class MockSshConnectionStore {
|
||||
constructor() {
|
||||
@@ -142,7 +167,8 @@ vi.mock('../providers/ssh-pty-provider', () => ({
|
||||
isSshPtyNotFoundError: (err: unknown) =>
|
||||
(err instanceof Error ? err.message : String(err)).includes('not found'),
|
||||
SshPtyProvider: class MockSshPtyProvider {
|
||||
constructor() {
|
||||
constructor(_targetId: unknown, _mux: unknown, _env: unknown, providerGeneration: number) {
|
||||
mockPtyProvider.providerGeneration = providerGeneration
|
||||
return mockPtyProvider
|
||||
}
|
||||
}
|
||||
@@ -222,6 +248,9 @@ import {
|
||||
type SshConnectionState,
|
||||
type SshTarget
|
||||
} from '../../shared/ssh-types'
|
||||
import { PTY_CONSUMER_SESSION_PROTOCOL_VERSION } from '../../shared/pty-consumer-session'
|
||||
import { DEFAULT_PTY_SOURCE_WINDOW_SU } from '../../shared/pty-source-credit-contract'
|
||||
import type { SshPtyDataCallback } from '../providers/ssh-pty-provider-contract'
|
||||
import {
|
||||
clearProviderPtyState,
|
||||
deletePtyOwnership,
|
||||
@@ -231,6 +260,16 @@ import {
|
||||
import { assertSshMutationExpectation } from '../ssh/ssh-connection-generation'
|
||||
|
||||
describe('SSH IPC handlers', () => {
|
||||
const relayBuildId = '0.1.0+ipc-test'
|
||||
const ipcTestSource = {
|
||||
relayPtyId: 'remote-pty',
|
||||
spanId: 'ipc-test-delivery:0:5',
|
||||
clientGeneration: 1,
|
||||
ownerGeneration: 1,
|
||||
deliveryToken: 'ipc-test-delivery',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 5
|
||||
} as const
|
||||
const handlers = new Map<string, (_event: unknown, args: unknown) => unknown>()
|
||||
const mockStore = {
|
||||
getRepos: () => [],
|
||||
@@ -272,7 +311,8 @@ describe('SSH IPC handlers', () => {
|
||||
const relayLostStabilizedMs = 5_000
|
||||
const createRelayLaunchResult = () => ({
|
||||
transport: { write: vi.fn(), onData: vi.fn(), onClose: vi.fn() },
|
||||
platform: 'linux-x64'
|
||||
platform: 'linux-x64',
|
||||
serverBuildId: relayBuildId
|
||||
})
|
||||
const getLatestRelayDisposeCallback = (): RelayDisposeCallback => {
|
||||
const calls = mockMux.onDispose.mock.calls
|
||||
@@ -322,21 +362,43 @@ describe('SSH IPC handlers', () => {
|
||||
mockConnectionManager.setCallbacks.mockReset()
|
||||
mockConnectionManager.callbacksRef.current = null
|
||||
mockForceStopRelayForTarget.mockReset().mockResolvedValue(undefined)
|
||||
mockAcceptSshPtyOutputData.mockReset().mockResolvedValue({})
|
||||
mockAcceptSshPtyOutputExit.mockReset().mockResolvedValue(undefined)
|
||||
|
||||
mockDeployAndLaunchRelay.mockReset().mockResolvedValue({
|
||||
transport: { write: vi.fn(), onData: vi.fn(), onClose: vi.fn() },
|
||||
platform: 'linux-x64'
|
||||
platform: 'linux-x64',
|
||||
serverBuildId: relayBuildId
|
||||
})
|
||||
mockMux.dispose.mockReset()
|
||||
mockMux.isDisposed.mockReset().mockReturnValue(false)
|
||||
mockMux.onNotification.mockReset()
|
||||
mockMux.onNotificationByMethod.mockReset().mockReturnValue(() => {})
|
||||
mockMux.onDispose.mockReset().mockReturnValue(() => {})
|
||||
mockMux.request.mockReset().mockImplementation((method: string) =>
|
||||
Promise.resolve(
|
||||
method === 'pty.openClient'
|
||||
? {
|
||||
protocolVersion: PTY_CONSUMER_SESSION_PROTOCOL_VERSION,
|
||||
serverBuildId: relayBuildId,
|
||||
clientGeneration: 1,
|
||||
role: 'session-owner',
|
||||
ownerGeneration: 1,
|
||||
ownerLease: 'ipc-test-owner',
|
||||
capabilities: {
|
||||
outputFlowControl: { version: 1, windowSu: DEFAULT_PTY_SOURCE_WINDOW_SU }
|
||||
}
|
||||
}
|
||||
: {}
|
||||
)
|
||||
)
|
||||
mockMux.probeLiveness.mockReset().mockResolvedValue(false)
|
||||
mockPtyProvider.onData.mockReset()
|
||||
mockPtyProvider.onExit.mockReset()
|
||||
mockPtyProvider.onReplay.mockReset()
|
||||
mockPtyProvider.attachForReconnect.mockReset().mockResolvedValue({})
|
||||
mockPtyProvider.shutdown.mockReset()
|
||||
mockPtyProvider.providerGeneration = 0
|
||||
mockRegisterSshGitProvider.mockReset()
|
||||
mockPortForwardManager.addForward.mockReset()
|
||||
mockPortForwardManager.updateForward.mockReset()
|
||||
@@ -565,6 +627,7 @@ describe('SSH IPC handlers', () => {
|
||||
}
|
||||
mockDeployAndLaunchRelay.mockResolvedValueOnce({
|
||||
transport: { write: vi.fn(), onData: vi.fn(), onClose: vi.fn() },
|
||||
serverBuildId: relayBuildId,
|
||||
hostPlatform
|
||||
})
|
||||
mockSshStore.getTarget.mockReturnValue(target)
|
||||
@@ -1050,7 +1113,7 @@ describe('SSH IPC handlers', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('forwards remote PTY events into the runtime', async () => {
|
||||
it('forwards remote PTY events through the output intake authority', async () => {
|
||||
const runtime = {
|
||||
onPtyData: vi.fn(),
|
||||
onPtyExit: vi.fn()
|
||||
@@ -1073,24 +1136,47 @@ describe('SSH IPC handlers', () => {
|
||||
})
|
||||
|
||||
await handlers.get('ssh:connect')!(null, { targetId: 'ssh-1' })
|
||||
const onData = mockPtyProvider.onData.mock.calls[0]?.[0] as
|
||||
| ((payload: { id: string; data: string }) => void)
|
||||
| undefined
|
||||
const onData = mockPtyProvider.onData.mock.calls[0]?.[0] as SshPtyDataCallback | undefined
|
||||
const onExit = mockPtyProvider.onExit.mock.calls[0]?.[0] as
|
||||
| ((payload: { id: string; code: number }) => void)
|
||||
| ((payload: {
|
||||
id: string
|
||||
code: number
|
||||
providerGeneration: number
|
||||
ptyIncarnation: string
|
||||
}) => void)
|
||||
| undefined
|
||||
|
||||
onData?.({ id: 'remote-pty', data: 'hello' })
|
||||
onExit?.({ id: 'remote-pty', code: 7 })
|
||||
onData?.({
|
||||
id: 'remote-pty',
|
||||
data: 'hello',
|
||||
providerGeneration: mockPtyProvider.providerGeneration,
|
||||
ptyIncarnation: 'ipc-test-pty',
|
||||
source: ipcTestSource
|
||||
})
|
||||
onExit?.({
|
||||
id: 'remote-pty',
|
||||
code: 7,
|
||||
providerGeneration: mockPtyProvider.providerGeneration,
|
||||
ptyIncarnation: 'ipc-test-pty'
|
||||
})
|
||||
|
||||
expect(runtime.onPtyData).toHaveBeenCalledWith(
|
||||
'remote-pty',
|
||||
'hello',
|
||||
expect.any(Number),
|
||||
'hello'.length,
|
||||
undefined
|
||||
)
|
||||
expect(runtime.onPtyExit).toHaveBeenCalledWith('remote-pty', 7, undefined)
|
||||
expect(mockAcceptSshPtyOutputData).toHaveBeenCalledWith({
|
||||
id: 'remote-pty',
|
||||
data: 'hello',
|
||||
providerGeneration: mockPtyProvider.providerGeneration,
|
||||
ptyIncarnation: 'ipc-test-pty',
|
||||
rawLength: 'hello'.length,
|
||||
transformed: false,
|
||||
source: ipcTestSource
|
||||
})
|
||||
expect(mockAcceptSshPtyOutputExit).toHaveBeenCalledWith({
|
||||
id: 'remote-pty',
|
||||
code: 7,
|
||||
providerGeneration: mockPtyProvider.providerGeneration,
|
||||
ptyIncarnation: 'ipc-test-pty'
|
||||
})
|
||||
expect(runtime.onPtyData).not.toHaveBeenCalled()
|
||||
expect(runtime.onPtyExit).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('mirrors SSH state broadcasts onto the runtime client-event stream', async () => {
|
||||
@@ -1380,7 +1466,7 @@ describe('SSH IPC handlers', () => {
|
||||
expect(replacementConnectionManager.disconnect).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('refreshes live session callbacks to the newest window, store, and runtime', async () => {
|
||||
it('refreshes live session callbacks to the newest window and output authorities', async () => {
|
||||
const firstWindow = createMockWindow()
|
||||
const secondWindow = createMockWindow()
|
||||
const firstRuntime = {
|
||||
@@ -1411,11 +1497,14 @@ describe('SSH IPC handlers', () => {
|
||||
})
|
||||
|
||||
await handlers.get('ssh:connect')!(null, { targetId: 'ssh-1' })
|
||||
const onData = mockPtyProvider.onData.mock.calls[0]?.[0] as
|
||||
| ((payload: { id: string; data: string }) => void)
|
||||
| undefined
|
||||
const onData = mockPtyProvider.onData.mock.calls[0]?.[0] as SshPtyDataCallback | undefined
|
||||
const onExit = mockPtyProvider.onExit.mock.calls[0]?.[0] as
|
||||
| ((payload: { id: string; code: number }) => void)
|
||||
| ((payload: {
|
||||
id: string
|
||||
code: number
|
||||
providerGeneration: number
|
||||
ptyIncarnation: string
|
||||
}) => void)
|
||||
| undefined
|
||||
const onDetectedPorts = mockPortScannerCallbacks.get('ssh-1') as
|
||||
| ((targetId: string, ports: unknown[], platform: string) => void)
|
||||
@@ -1434,8 +1523,19 @@ describe('SSH IPC handlers', () => {
|
||||
error: 'network down',
|
||||
reconnectAttempt: 0
|
||||
})
|
||||
onData?.({ id: 'remote-pty', data: 'hello' })
|
||||
onExit?.({ id: 'remote-pty', code: 9 })
|
||||
onData?.({
|
||||
id: 'remote-pty',
|
||||
data: 'hello',
|
||||
providerGeneration: mockPtyProvider.providerGeneration,
|
||||
ptyIncarnation: 'ipc-test-pty',
|
||||
source: ipcTestSource
|
||||
})
|
||||
onExit?.({
|
||||
id: 'remote-pty',
|
||||
code: 9,
|
||||
providerGeneration: mockPtyProvider.providerGeneration,
|
||||
ptyIncarnation: 'ipc-test-pty'
|
||||
})
|
||||
onDetectedPorts?.(
|
||||
'ssh-1',
|
||||
[{ host: '127.0.0.1', port: 3000, pid: 12, processName: 'node' }],
|
||||
@@ -1454,33 +1554,20 @@ describe('SSH IPC handlers', () => {
|
||||
connectionGeneration: 1
|
||||
}
|
||||
})
|
||||
expect(secondWindow.webContents.send).toHaveBeenCalledWith(
|
||||
'pty:data',
|
||||
expect(mockAcceptSshPtyOutputData).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ id: 'remote-pty', data: 'hello' })
|
||||
)
|
||||
expect(secondWindow.webContents.send).toHaveBeenCalledWith('pty:exit', {
|
||||
id: 'remote-pty',
|
||||
code: 9
|
||||
})
|
||||
expect(mockAcceptSshPtyOutputExit).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ id: 'remote-pty', code: 9 })
|
||||
)
|
||||
expect(secondWindow.webContents.send).toHaveBeenCalledWith('ssh:detected-ports-changed', {
|
||||
targetId: 'ssh-1',
|
||||
ports: expect.arrayContaining([expect.objectContaining({ port: 3000 })])
|
||||
})
|
||||
expect(secondRuntime.onPtyData).toHaveBeenCalledWith(
|
||||
'remote-pty',
|
||||
'hello',
|
||||
expect.any(Number),
|
||||
'hello'.length,
|
||||
undefined
|
||||
)
|
||||
expect(secondRuntime.onPtyExit).toHaveBeenCalledWith('remote-pty', 9, undefined)
|
||||
expect(secondRuntime.onPtyData).not.toHaveBeenCalled()
|
||||
expect(secondRuntime.onPtyExit).not.toHaveBeenCalled()
|
||||
expect(firstRuntime.onPtyData).not.toHaveBeenCalled()
|
||||
expect(firstRuntime.onPtyExit).not.toHaveBeenCalled()
|
||||
expect(mockStore.markSshRemotePtyLease).toHaveBeenCalledWith(
|
||||
'ssh-1',
|
||||
'remote-pty',
|
||||
'terminated'
|
||||
)
|
||||
})
|
||||
|
||||
it('re-registers without replacing managers when no targets are connected', () => {
|
||||
|
||||
@@ -1548,6 +1548,26 @@ describe('Store', () => {
|
||||
expect(updatedTarget).not.toHaveProperty('systemSshConnectionReuse')
|
||||
})
|
||||
|
||||
it('drops retired per-target SSH terminal source-credit selections', async () => {
|
||||
const store = await createStore()
|
||||
store.addSshTarget({
|
||||
id: 'ssh-source-credit-on',
|
||||
label: 'Noisy build host',
|
||||
host: 'build.example.com',
|
||||
port: 22,
|
||||
username: 'dev',
|
||||
experimentalPtySourceCreditV1: true
|
||||
} as never)
|
||||
|
||||
expect(store.getSshTarget('ssh-source-credit-on')).not.toHaveProperty(
|
||||
'experimentalPtySourceCreditV1'
|
||||
)
|
||||
store.flush()
|
||||
const persisted = readDataFile() as { sshTargets?: Record<string, unknown>[] }
|
||||
const target = persisted.sshTargets?.find((entry) => entry.id === 'ssh-source-credit-on')
|
||||
expect(target).not.toHaveProperty('experimentalPtySourceCreditV1')
|
||||
})
|
||||
|
||||
it('upserts ~/.ssh/config through the real store: rotated port updates in place and persists', async () => {
|
||||
loadUserSshConfigMock.mockReturnValue([{ host: 'cluster' }])
|
||||
const candidate = (port: number, id: string) => [
|
||||
@@ -1563,7 +1583,6 @@ describe('Store', () => {
|
||||
expect(inserted).toHaveLength(1)
|
||||
expect(inserted[0]?.source).toBe('ssh-config')
|
||||
expect(inserted[0]?.port).toBe(2200)
|
||||
|
||||
// Rotated port: upsert updates the same target in place and normalizeSshTarget must keep `source` (no false re-derive into a permanently-dirty state).
|
||||
sshConfigHostsToTargetsMock.mockReturnValue(candidate(2222, 'ssh-cfg-2'))
|
||||
const changed = sshStore.importFromSshConfig()
|
||||
|
||||
@@ -1113,6 +1113,7 @@ function backfillLegacyAutomationContexts(
|
||||
type LegacySshTarget = SshTarget & {
|
||||
remoteWorkspaceSyncEnabled?: unknown
|
||||
remoteWorkspaceSyncGracePeriodSeconds?: unknown
|
||||
experimentalPtySourceCreditV1?: unknown
|
||||
}
|
||||
|
||||
// Why: old targets predate configHost; default to label-based lookup so imported SSH aliases still resolve via ssh -G.
|
||||
@@ -1127,6 +1128,7 @@ function normalizeSshTarget(t: SshTarget): SshTarget {
|
||||
delete target.remoteWorkspaceSyncGracePeriodSeconds
|
||||
delete target.relayGracePeriodSeconds
|
||||
delete target.systemSshConnectionReuse
|
||||
delete target.experimentalPtySourceCreditV1
|
||||
// Why: prefer the synced grace over stale relayGracePeriodSeconds so a user's "unlimited" (0) survives migration.
|
||||
const relayGracePeriodSeconds =
|
||||
legacySyncEnabled === true && typeof legacyGracePeriodSeconds === 'number'
|
||||
|
||||
@@ -2,6 +2,7 @@ import type { TerminalOscLinkRange } from '../../shared/terminal-osc-link-ranges
|
||||
import type { TuiAgent } from '../../shared/types'
|
||||
import type { AgentSessionClaimedSpawnResult } from '../../shared/agent-session-host-authority'
|
||||
import type { PtyIncarnationId } from '../../shared/pty-incarnation'
|
||||
import type { PtySourceReceivingActivation } from '../../shared/pty-source-receiving-activation'
|
||||
|
||||
export type PtySpawnResult = {
|
||||
agentSessionEnsure?: AgentSessionClaimedSpawnResult
|
||||
@@ -10,6 +11,8 @@ export type PtySpawnResult = {
|
||||
id: string
|
||||
/** Opaque provider-owned identity for this process behind a reusable PTY id. */
|
||||
incarnationId?: PtyIncarnationId
|
||||
/** Relay source identity installed before adjacent source frames are decoded. */
|
||||
sourceActivation?: PtySourceReceivingActivation
|
||||
/** The provider observed this exact spawn exit before its control reply settled. */
|
||||
exitedBeforeSpawnReply?: true
|
||||
/** OS-level pid of the shell process, when available at spawn time.
|
||||
|
||||
@@ -2,6 +2,14 @@ import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer'
|
||||
import { AGENT_SESSION_CREATE_OPERATION_PROTOCOL_VERSION } from '../../shared/agent-session-host-authority'
|
||||
import { isPtyIncarnationId } from '../../shared/pty-incarnation'
|
||||
import type { PtySpawnResult } from './pty-spawn-result'
|
||||
import type { PtySpawnOptions } from './types'
|
||||
import type { SshPtySpawnExitRaceTracker } from './ssh-pty-spawn-exit-race'
|
||||
import type { SshPtyReceivingActivationLease } from './ssh-pty-notification-routing'
|
||||
import {
|
||||
parsePtySourceReceivingActivation,
|
||||
type PtySourceReceivingActivation
|
||||
} from '../../shared/pty-source-receiving-activation'
|
||||
import { validateClaimedSshSpawn } from './ssh-agent-session-claim-validation'
|
||||
|
||||
export const SSH_AGENT_SESSION_CAPABILITY_PROBE_TIMEOUT_MS = 5_000
|
||||
|
||||
@@ -49,11 +57,14 @@ export async function requestSshAgentSessionCreate(args: {
|
||||
params: Record<string, unknown>
|
||||
operationId?: string
|
||||
signal?: AbortSignal
|
||||
beforeResolve?: (result: unknown) => void
|
||||
}): Promise<unknown> {
|
||||
try {
|
||||
return await (args.signal
|
||||
? args.mux.request('pty.spawn', args.params, { signal: args.signal })
|
||||
: args.mux.request('pty.spawn', args.params))
|
||||
const options =
|
||||
args.signal || args.beforeResolve
|
||||
? { signal: args.signal, beforeResolve: args.beforeResolve }
|
||||
: undefined
|
||||
return await args.mux.request('pty.spawn', args.params, options)
|
||||
} catch (error) {
|
||||
if (!args.operationId) {
|
||||
throw error
|
||||
@@ -63,3 +74,111 @@ export async function requestSshAgentSessionCreate(args: {
|
||||
throw Object.assign(spawnError, { agentSessionOperationOutcome: 'unknown' as const })
|
||||
}
|
||||
}
|
||||
|
||||
export async function spawnFreshSshPty(args: {
|
||||
mux: SshChannelMultiplexer
|
||||
options: PtySpawnOptions
|
||||
params: Record<string, unknown>
|
||||
exitRaceTracker: SshPtySpawnExitRaceTracker
|
||||
installSourceActivation: (
|
||||
relayPtyId: string,
|
||||
activation: PtySourceReceivingActivation
|
||||
) => SshPtyReceivingActivationLease
|
||||
rememberPtyIncarnation: (relayPtyId: string, incarnationId: unknown) => void
|
||||
acceptLivePty: (appPtyId: string) => void
|
||||
toAppPtyId: (relayPtyId: string) => string
|
||||
}): Promise<PtySpawnResult> {
|
||||
const operation = args.exitRaceTracker.begin()
|
||||
let sourceActivationLease: SshPtyReceivingActivationLease | undefined
|
||||
try {
|
||||
const result = await requestSshAgentSessionCreate({
|
||||
mux: args.mux,
|
||||
operationId: args.options.agentSessionCreateOperationId,
|
||||
signal: args.options.signal,
|
||||
params: args.params,
|
||||
beforeResolve: (value) => {
|
||||
sourceActivationLease = installSpawnSourceActivation(value, args.installSourceActivation)
|
||||
}
|
||||
})
|
||||
if (args.options.agentSessionCreateOperationId) {
|
||||
assertSshAgentSessionCreateResult(result)
|
||||
}
|
||||
const spawnResult = parseSshPtySpawnResult(result)
|
||||
if (args.exitRaceTracker.didMatchingExitArrive(operation, spawnResult)) {
|
||||
throw Object.assign(new Error('agent_session_exited_during_start'), {
|
||||
agentSessionOperationOutcome: 'unknown' as const
|
||||
})
|
||||
}
|
||||
const claimed = spawnResult.agentSessionEnsure
|
||||
if (args.options.agentSessionEnsure) {
|
||||
const validation = validateClaimedSshSpawn(spawnResult, args.options.agentSessionEnsure)
|
||||
if (!validation.valid) {
|
||||
if (validation.cleanup === 'created' && typeof spawnResult.id === 'string') {
|
||||
try {
|
||||
await args.mux.request('pty.shutdown', { id: spawnResult.id, immediate: true })
|
||||
} catch {
|
||||
throw new Error('execution_owner_unavailable')
|
||||
}
|
||||
}
|
||||
throw new Error(validation.error)
|
||||
}
|
||||
}
|
||||
const id = args.toAppPtyId(spawnResult.id)
|
||||
args.rememberPtyIncarnation(spawnResult.id, spawnResult.incarnationId)
|
||||
args.acceptLivePty(id)
|
||||
const mappedResult = {
|
||||
...spawnResult,
|
||||
id,
|
||||
...(claimed
|
||||
? {
|
||||
agentSessionEnsure: {
|
||||
...claimed,
|
||||
owner: { ...claimed.owner, ptyId: args.toAppPtyId(claimed.owner.ptyId) }
|
||||
}
|
||||
}
|
||||
: {})
|
||||
}
|
||||
sourceActivationLease?.commit()
|
||||
return mappedResult
|
||||
} catch (error) {
|
||||
if (sourceActivationLease && !(await sourceActivationLease.rollback())) {
|
||||
throw new Error('execution_owner_unavailable')
|
||||
}
|
||||
throw error
|
||||
} finally {
|
||||
args.exitRaceTracker.finish(operation)
|
||||
}
|
||||
}
|
||||
|
||||
function installSpawnSourceActivation(
|
||||
value: unknown,
|
||||
install: (
|
||||
relayPtyId: string,
|
||||
activation: PtySourceReceivingActivation
|
||||
) => SshPtyReceivingActivationLease
|
||||
): SshPtyReceivingActivationLease | undefined {
|
||||
const result = parseSshPtySpawnResult(value)
|
||||
const activation = result.sourceActivation
|
||||
if (!activation) {
|
||||
return undefined
|
||||
}
|
||||
return install(result.id, activation)
|
||||
}
|
||||
|
||||
function parseSshPtySpawnResult(value: unknown): PtySpawnResult {
|
||||
const result =
|
||||
typeof value === 'object' && value !== null && !Array.isArray(value)
|
||||
? (value as PtySpawnResult)
|
||||
: ({} as PtySpawnResult)
|
||||
const activation = parsePtySourceReceivingActivation(result.sourceActivation)
|
||||
if (
|
||||
activation &&
|
||||
(typeof result.id !== 'string' ||
|
||||
result.id.length === 0 ||
|
||||
!isPtyIncarnationId(result.incarnationId) ||
|
||||
activation.ptyIncarnation !== result.incarnationId)
|
||||
) {
|
||||
throw new Error('Invalid SSH PTY source activation identity')
|
||||
}
|
||||
return activation ? { ...result, sourceActivation: activation } : result
|
||||
}
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { subscribeSshPtyNotifications } from './ssh-pty-notification-routing'
|
||||
import type { PtySourceReceivingActivation } from '../../shared/pty-source-receiving-activation'
|
||||
|
||||
type MockMux = {
|
||||
onNotification: ReturnType<typeof vi.fn>
|
||||
request: ReturnType<typeof vi.fn>
|
||||
}
|
||||
|
||||
function createSubscription() {
|
||||
const mux: MockMux = {
|
||||
onNotification: vi.fn()
|
||||
onNotification: vi.fn(),
|
||||
request: vi.fn(async () => ({ canceled: true, sentEndSu: 0, creditedEndSu: 0 }))
|
||||
}
|
||||
const dataListeners = new Set<(payload: { id: string; data: string }) => void>()
|
||||
const replayListeners = new Set<(payload: { id: string; data: string }) => void>()
|
||||
@@ -15,15 +18,18 @@ function createSubscription() {
|
||||
const livePtyIds = new Set<string>()
|
||||
const recordExit = vi.fn()
|
||||
const toAppPtyId = vi.fn((id: string) => `ssh:conn@@${id}`)
|
||||
const resolvePtyIncarnation = vi.fn((id: string) => `incarnation:${id}`)
|
||||
|
||||
subscribeSshPtyNotifications({
|
||||
const subscription = subscribeSshPtyNotifications({
|
||||
mux: mux as never,
|
||||
toAppPtyId,
|
||||
dataListeners: dataListeners as never,
|
||||
replayListeners: replayListeners as never,
|
||||
exitListeners: exitListeners as never,
|
||||
livePtyIds,
|
||||
recordExit
|
||||
recordExit,
|
||||
providerGeneration: 7,
|
||||
resolvePtyIncarnation
|
||||
})
|
||||
|
||||
const handler = mux.onNotification.mock.calls[0]?.[0] as (
|
||||
@@ -36,15 +42,33 @@ function createSubscription() {
|
||||
|
||||
return {
|
||||
handler,
|
||||
mux,
|
||||
toAppPtyId,
|
||||
dataListeners,
|
||||
replayListeners,
|
||||
exitListeners,
|
||||
livePtyIds,
|
||||
recordExit
|
||||
recordExit,
|
||||
resolvePtyIncarnation,
|
||||
installReceivingActivation: subscription.installReceivingActivation
|
||||
}
|
||||
}
|
||||
|
||||
function sourceActivation(
|
||||
overrides: Partial<PtySourceReceivingActivation> = {}
|
||||
): PtySourceReceivingActivation {
|
||||
return Object.freeze({
|
||||
status: 'pending',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-1',
|
||||
checkpointSourceEndSu: 0,
|
||||
recoveryEndSu: 0,
|
||||
...overrides
|
||||
})
|
||||
}
|
||||
|
||||
describe('subscribeSshPtyNotifications', () => {
|
||||
it('ignores non-PTY notifications without mapping params.id', () => {
|
||||
const { handler, toAppPtyId } = createSubscription()
|
||||
@@ -70,6 +94,8 @@ describe('subscribeSshPtyNotifications', () => {
|
||||
expect(onData).toHaveBeenCalledWith({
|
||||
id: 'ssh:conn@@pty-1',
|
||||
data: 'hello',
|
||||
providerGeneration: 7,
|
||||
ptyIncarnation: 'incarnation:pty-1',
|
||||
sequenceChars: 5,
|
||||
seq: 9
|
||||
})
|
||||
@@ -92,10 +118,689 @@ describe('subscribeSshPtyNotifications', () => {
|
||||
expect(onExit).toHaveBeenCalledWith({
|
||||
id: 'ssh:conn@@pty-1',
|
||||
code: 0,
|
||||
providerGeneration: 7,
|
||||
ptyIncarnation: 'incarnation:pty-1',
|
||||
incarnationId: 'incarnation-1'
|
||||
})
|
||||
})
|
||||
|
||||
it('derives exact immutable source ranges and cancels malformed frames without side effects', () => {
|
||||
const {
|
||||
handler,
|
||||
mux,
|
||||
dataListeners,
|
||||
livePtyIds,
|
||||
toAppPtyId,
|
||||
resolvePtyIncarnation,
|
||||
installReceivingActivation
|
||||
} = createSubscription()
|
||||
const onData = vi.fn()
|
||||
dataListeners.add(onData)
|
||||
livePtyIds.add('ssh:conn@@unrelated')
|
||||
installReceivingActivation(
|
||||
'pty-1',
|
||||
sourceActivation({ checkpointSourceEndSu: 10, recoveryEndSu: 14 })
|
||||
).commit()
|
||||
|
||||
handler('pty.data', {
|
||||
id: 'pty-1',
|
||||
data: 'data',
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
sourceEndSu: 14,
|
||||
sourceLengthSu: 4
|
||||
})
|
||||
const acceptedSource = onData.mock.calls[0]?.[0].source
|
||||
expect(Object.isFrozen(acceptedSource)).toBe(true)
|
||||
const liveBeforeMalformed = new Set(livePtyIds)
|
||||
toAppPtyId.mockClear()
|
||||
resolvePtyIncarnation.mockClear()
|
||||
handler('pty.data', {
|
||||
id: 'pty-1',
|
||||
data: 'bad',
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
sourceEndSu: 17,
|
||||
sourceLengthSu: 4
|
||||
})
|
||||
|
||||
expect(onData.mock.calls[0]?.[0]).toMatchObject({
|
||||
source: {
|
||||
relayPtyId: 'pty-1',
|
||||
spanId: 'token-1:10:14',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-1',
|
||||
sourceStartSu: 10,
|
||||
sourceEndSu: 14
|
||||
}
|
||||
})
|
||||
expect(onData).toHaveBeenCalledTimes(1)
|
||||
expect(livePtyIds).toEqual(liveBeforeMalformed)
|
||||
expect(toAppPtyId).not.toHaveBeenCalled()
|
||||
expect(resolvePtyIncarnation).not.toHaveBeenCalled()
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', {
|
||||
id: 'pty-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-1'
|
||||
})
|
||||
})
|
||||
|
||||
it('keeps exact source incarnation independent without mutating legacy delivery state', () => {
|
||||
const { handler, dataListeners, resolvePtyIncarnation, installReceivingActivation } =
|
||||
createSubscription()
|
||||
const onData = vi.fn()
|
||||
dataListeners.add(onData)
|
||||
handler('pty.data', { id: 'pty-1', data: 'legacy' })
|
||||
installReceivingActivation(
|
||||
'pty-1',
|
||||
sourceActivation({ checkpointSourceEndSu: 0, recoveryEndSu: 4 })
|
||||
).commit()
|
||||
handler('pty.data', {
|
||||
id: 'pty-1',
|
||||
data: 'data',
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
sourceEndSu: 4,
|
||||
sourceLengthSu: 4
|
||||
})
|
||||
|
||||
expect(onData.mock.calls.map(([payload]) => payload.ptyIncarnation)).toEqual([
|
||||
'incarnation:pty-1',
|
||||
'incarnation-1'
|
||||
])
|
||||
expect(resolvePtyIncarnation).toHaveBeenCalledTimes(1)
|
||||
expect(resolvePtyIncarnation).toHaveBeenCalledWith('pty-1', undefined)
|
||||
})
|
||||
|
||||
it('drops stale delivery generations without touching their PTY or unrelated PTYs', () => {
|
||||
const {
|
||||
handler,
|
||||
mux,
|
||||
dataListeners,
|
||||
livePtyIds,
|
||||
toAppPtyId,
|
||||
resolvePtyIncarnation,
|
||||
installReceivingActivation
|
||||
} = createSubscription()
|
||||
const onData = vi.fn()
|
||||
dataListeners.add(onData)
|
||||
livePtyIds.add('ssh:conn@@unrelated')
|
||||
installReceivingActivation(
|
||||
'pty-1',
|
||||
sourceActivation({
|
||||
clientGeneration: 4,
|
||||
ownerGeneration: 5,
|
||||
deliveryToken: 'token-new',
|
||||
recoveryEndSu: 3
|
||||
})
|
||||
).commit()
|
||||
|
||||
handler('pty.data', {
|
||||
id: 'pty-1',
|
||||
data: 'new',
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-new',
|
||||
clientGeneration: 4,
|
||||
ownerGeneration: 5,
|
||||
sourceEndSu: 3,
|
||||
sourceLengthSu: 3
|
||||
})
|
||||
const liveBeforeStale = new Set(livePtyIds)
|
||||
toAppPtyId.mockClear()
|
||||
resolvePtyIncarnation.mockClear()
|
||||
|
||||
handler('pty.data', {
|
||||
id: 'pty-1',
|
||||
data: 'old',
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-old',
|
||||
clientGeneration: 3,
|
||||
ownerGeneration: 4,
|
||||
sourceEndSu: 6,
|
||||
sourceLengthSu: 3
|
||||
})
|
||||
|
||||
expect(onData).toHaveBeenCalledTimes(1)
|
||||
expect(livePtyIds).toEqual(liveBeforeStale)
|
||||
expect(toAppPtyId).not.toHaveBeenCalled()
|
||||
expect(resolvePtyIncarnation).not.toHaveBeenCalled()
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', {
|
||||
id: 'pty-1',
|
||||
clientGeneration: 3,
|
||||
ownerGeneration: 4,
|
||||
deliveryToken: 'token-old'
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects same-generation token changes and source discontinuities', () => {
|
||||
const { handler, mux, dataListeners, installReceivingActivation } = createSubscription()
|
||||
const onData = vi.fn()
|
||||
dataListeners.add(onData)
|
||||
const sourceParams = {
|
||||
id: 'pty-1',
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
sourceLengthSu: 3
|
||||
}
|
||||
installReceivingActivation('pty-1', sourceActivation({ recoveryEndSu: 3 })).commit()
|
||||
|
||||
handler('pty.data', {
|
||||
...sourceParams,
|
||||
data: 'one',
|
||||
deliveryToken: 'token-1',
|
||||
sourceEndSu: 3
|
||||
})
|
||||
handler('pty.data', {
|
||||
...sourceParams,
|
||||
data: 'two',
|
||||
deliveryToken: 'token-2',
|
||||
sourceEndSu: 6
|
||||
})
|
||||
handler('pty.data', {
|
||||
...sourceParams,
|
||||
data: 'gap',
|
||||
deliveryToken: 'token-1',
|
||||
sourceEndSu: 9
|
||||
})
|
||||
handler('pty.data', {
|
||||
...sourceParams,
|
||||
data: 'two',
|
||||
deliveryToken: 'token-1',
|
||||
sourceEndSu: 6
|
||||
})
|
||||
|
||||
expect(onData.mock.calls.map((call) => call[0].data)).toEqual(['one', 'two'])
|
||||
expect(mux.request).toHaveBeenCalledTimes(2)
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', {
|
||||
id: 'pty-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-2'
|
||||
})
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', {
|
||||
id: 'pty-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-1'
|
||||
})
|
||||
})
|
||||
|
||||
it('accepts a strictly newer rotation, rejects late old data, and preserves new continuity', () => {
|
||||
const { handler, mux, dataListeners, installReceivingActivation } = createSubscription()
|
||||
const onData = vi.fn()
|
||||
dataListeners.add(onData)
|
||||
const frame = (
|
||||
data: string,
|
||||
deliveryToken: string,
|
||||
clientGeneration: number,
|
||||
ownerGeneration: number,
|
||||
sourceEndSu: number
|
||||
) => ({
|
||||
id: 'pty-1',
|
||||
data,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken,
|
||||
clientGeneration,
|
||||
ownerGeneration,
|
||||
sourceEndSu,
|
||||
sourceLengthSu: data.length
|
||||
})
|
||||
|
||||
installReceivingActivation(
|
||||
'pty-1',
|
||||
sourceActivation({ deliveryToken: 'token-old', recoveryEndSu: 3 })
|
||||
).commit()
|
||||
handler('pty.data', frame('old', 'token-old', 2, 3, 3))
|
||||
installReceivingActivation(
|
||||
'pty-1',
|
||||
sourceActivation({
|
||||
clientGeneration: 3,
|
||||
ownerGeneration: 4,
|
||||
deliveryToken: 'token-new',
|
||||
checkpointSourceEndSu: 10,
|
||||
recoveryEndSu: 13
|
||||
})
|
||||
).commit()
|
||||
handler('pty.data', frame('new', 'token-new', 3, 4, 13))
|
||||
handler('pty.data', frame('old', 'token-old', 2, 3, 6))
|
||||
handler('pty.data', frame('next', 'token-new', 3, 4, 17))
|
||||
|
||||
expect(onData.mock.calls.map((call) => call[0].data)).toEqual(['old', 'new', 'next'])
|
||||
expect(mux.request).toHaveBeenCalledTimes(1)
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', {
|
||||
id: 'pty-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-old'
|
||||
})
|
||||
})
|
||||
|
||||
it.each([
|
||||
['client-only advance', 3, 3, 'token-client'],
|
||||
['owner-only advance', 2, 4, 'token-owner'],
|
||||
['crossed generations', 3, 2, 'token-crossed'],
|
||||
['replayed client generation', 1, 4, 'token-replayed'],
|
||||
['reused token on newer generations', 3, 4, 'token-current']
|
||||
])(
|
||||
'rejects a %s without replacing the accepted continuity record',
|
||||
(_case, clientGeneration, ownerGeneration, deliveryToken) => {
|
||||
const {
|
||||
handler,
|
||||
mux,
|
||||
dataListeners,
|
||||
livePtyIds,
|
||||
toAppPtyId,
|
||||
resolvePtyIncarnation,
|
||||
installReceivingActivation
|
||||
} = createSubscription()
|
||||
const onData = vi.fn()
|
||||
dataListeners.add(onData)
|
||||
const base = {
|
||||
id: 'pty-1',
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
sourceLengthSu: 3
|
||||
}
|
||||
installReceivingActivation(
|
||||
'pty-1',
|
||||
sourceActivation({ deliveryToken: 'token-current', recoveryEndSu: 3 })
|
||||
).commit()
|
||||
handler('pty.data', {
|
||||
...base,
|
||||
data: 'one',
|
||||
deliveryToken: 'token-current',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
sourceEndSu: 3
|
||||
})
|
||||
const liveBeforeInvalid = new Set(livePtyIds)
|
||||
toAppPtyId.mockClear()
|
||||
resolvePtyIncarnation.mockClear()
|
||||
|
||||
handler('pty.data', {
|
||||
...base,
|
||||
data: 'bad',
|
||||
deliveryToken,
|
||||
clientGeneration,
|
||||
ownerGeneration,
|
||||
sourceEndSu: 6
|
||||
})
|
||||
handler('pty.data', {
|
||||
...base,
|
||||
data: 'two',
|
||||
deliveryToken: 'token-current',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
sourceEndSu: 6
|
||||
})
|
||||
|
||||
expect(onData.mock.calls.map((call) => call[0].data)).toEqual(['one', 'two'])
|
||||
expect(livePtyIds).toEqual(liveBeforeInvalid)
|
||||
expect(toAppPtyId).toHaveBeenCalledTimes(1)
|
||||
expect(resolvePtyIncarnation).not.toHaveBeenCalled()
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', {
|
||||
id: 'pty-1',
|
||||
clientGeneration,
|
||||
ownerGeneration,
|
||||
deliveryToken
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
it('does not cancel an incomplete malformed identity or mutate provider state', () => {
|
||||
const { handler, mux, dataListeners, livePtyIds, toAppPtyId, resolvePtyIncarnation } =
|
||||
createSubscription()
|
||||
const onData = vi.fn()
|
||||
dataListeners.add(onData)
|
||||
livePtyIds.add('ssh:conn@@unrelated')
|
||||
|
||||
handler('pty.data', {
|
||||
id: 'pty-1',
|
||||
data: 'bad',
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
sourceEndSu: 3,
|
||||
sourceLengthSu: 3
|
||||
})
|
||||
|
||||
expect(onData).not.toHaveBeenCalled()
|
||||
expect(livePtyIds).toEqual(new Set(['ssh:conn@@unrelated']))
|
||||
expect(toAppPtyId).not.toHaveBeenCalled()
|
||||
expect(resolvePtyIncarnation).not.toHaveBeenCalled()
|
||||
expect(mux.request).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('accepts non-empty recovery from the activation checkpoint', () => {
|
||||
const { handler, dataListeners, installReceivingActivation } = createSubscription()
|
||||
const onData = vi.fn()
|
||||
dataListeners.add(onData)
|
||||
const lease = installReceivingActivation(
|
||||
'pty-1',
|
||||
sourceActivation({ checkpointSourceEndSu: 4, recoveryEndSu: 8 })
|
||||
)
|
||||
|
||||
handler('pty.data', {
|
||||
id: 'pty-1',
|
||||
data: 'next',
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
sourceEndSu: 8,
|
||||
sourceLengthSu: 4
|
||||
})
|
||||
|
||||
expect(onData).not.toHaveBeenCalled()
|
||||
lease.commit()
|
||||
expect(onData).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
data: 'next',
|
||||
source: expect.objectContaining({ sourceStartSu: 4, sourceEndSu: 8 })
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
it('routes held and later recovery frames only to the private sink until commit', () => {
|
||||
const { handler, dataListeners, livePtyIds, installReceivingActivation } = createSubscription()
|
||||
const onData = vi.fn()
|
||||
const onRecoveryData = vi.fn()
|
||||
dataListeners.add(onData)
|
||||
const lease = installReceivingActivation(
|
||||
'pty-1',
|
||||
sourceActivation({ checkpointSourceEndSu: 4, recoveryEndSu: 12 })
|
||||
)
|
||||
const publishSource = (data: string, sourceEndSu: number): void => {
|
||||
handler('pty.data', {
|
||||
id: 'pty-1',
|
||||
data,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
sourceEndSu,
|
||||
sourceLengthSu: 4
|
||||
})
|
||||
}
|
||||
|
||||
publishSource('held', 8)
|
||||
const recoveryLease = lease.transferToRecovery(onRecoveryData)
|
||||
publishSource('next', 12)
|
||||
|
||||
expect(onRecoveryData.mock.calls.map(([payload]) => payload.data)).toEqual(['held', 'next'])
|
||||
expect(onData).not.toHaveBeenCalled()
|
||||
expect(livePtyIds).not.toContain('ssh:conn@@pty-1')
|
||||
|
||||
recoveryLease.commit()
|
||||
expect(onData).not.toHaveBeenCalled()
|
||||
publishSource('live', 16)
|
||||
|
||||
expect(onRecoveryData).toHaveBeenCalledTimes(2)
|
||||
expect(onData).toHaveBeenCalledWith(expect.objectContaining({ data: 'live' }))
|
||||
expect(livePtyIds).toContain('ssh:conn@@pty-1')
|
||||
})
|
||||
|
||||
it('retires an exited private recovery when its activation commits', () => {
|
||||
const { handler, mux, dataListeners, livePtyIds, installReceivingActivation } =
|
||||
createSubscription()
|
||||
const onData = vi.fn()
|
||||
const onRecoveryData = vi.fn()
|
||||
dataListeners.add(onData)
|
||||
const lease = installReceivingActivation('pty-1', sourceActivation({ recoveryEndSu: 4 }))
|
||||
handler('pty.data', {
|
||||
id: 'pty-1',
|
||||
data: 'held',
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
sourceEndSu: 4,
|
||||
sourceLengthSu: 4
|
||||
})
|
||||
const recoveryLease = lease.transferToRecovery(onRecoveryData)
|
||||
|
||||
handler('pty.exit', { id: 'pty-1', code: 0, incarnationId: 'incarnation-1' })
|
||||
recoveryLease.commit()
|
||||
handler('pty.data', {
|
||||
id: 'pty-1',
|
||||
data: 'late',
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
sourceEndSu: 8,
|
||||
sourceLengthSu: 4
|
||||
})
|
||||
|
||||
expect(onRecoveryData).toHaveBeenCalledOnce()
|
||||
expect(onData).not.toHaveBeenCalled()
|
||||
expect(livePtyIds).not.toContain('ssh:conn@@pty-1')
|
||||
expect(mux.request).toHaveBeenCalledWith(
|
||||
'pty.cancelDelivery',
|
||||
expect.objectContaining({ id: 'pty-1', deliveryToken: 'token-1' })
|
||||
)
|
||||
})
|
||||
|
||||
it('retires private recovery locally and restores the exact predecessor', () => {
|
||||
const { handler, mux, dataListeners, installReceivingActivation } = createSubscription()
|
||||
const onData = vi.fn()
|
||||
const onRecoveryData = vi.fn()
|
||||
dataListeners.add(onData)
|
||||
installReceivingActivation(
|
||||
'pty-1',
|
||||
sourceActivation({ deliveryToken: 'token-old', recoveryEndSu: 3 })
|
||||
).commit()
|
||||
handler('pty.data', {
|
||||
id: 'pty-1',
|
||||
data: 'pre',
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-old',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
sourceEndSu: 3,
|
||||
sourceLengthSu: 3
|
||||
})
|
||||
const replacement = installReceivingActivation(
|
||||
'pty-1',
|
||||
sourceActivation({
|
||||
clientGeneration: 3,
|
||||
ownerGeneration: 4,
|
||||
deliveryToken: 'token-new',
|
||||
checkpointSourceEndSu: 3,
|
||||
recoveryEndSu: 6
|
||||
})
|
||||
)
|
||||
handler('pty.data', {
|
||||
id: 'pty-1',
|
||||
data: 'new',
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-new',
|
||||
clientGeneration: 3,
|
||||
ownerGeneration: 4,
|
||||
sourceEndSu: 6,
|
||||
sourceLengthSu: 3
|
||||
})
|
||||
|
||||
replacement.transferToRecovery(onRecoveryData).retire()
|
||||
handler('pty.data', {
|
||||
id: 'pty-1',
|
||||
data: 'old',
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-old',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
sourceEndSu: 6,
|
||||
sourceLengthSu: 3
|
||||
})
|
||||
|
||||
expect(onRecoveryData).toHaveBeenCalledWith(expect.objectContaining({ data: 'new' }))
|
||||
expect(onData.mock.calls.map(([payload]) => payload.data)).toEqual(['pre', 'old'])
|
||||
expect(mux.request).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rejects a stale activation without disturbing current continuity', () => {
|
||||
const { handler, dataListeners, installReceivingActivation } = createSubscription()
|
||||
const onData = vi.fn()
|
||||
dataListeners.add(onData)
|
||||
installReceivingActivation('pty-1', sourceActivation({ recoveryEndSu: 3 })).commit()
|
||||
|
||||
expect(() =>
|
||||
installReceivingActivation(
|
||||
'pty-1',
|
||||
sourceActivation({
|
||||
clientGeneration: 1,
|
||||
ownerGeneration: 4,
|
||||
deliveryToken: 'token-stale',
|
||||
checkpointSourceEndSu: 3,
|
||||
recoveryEndSu: 3
|
||||
})
|
||||
)
|
||||
).toThrow('ssh_source_receiving_activation_stale')
|
||||
|
||||
handler('pty.data', {
|
||||
id: 'pty-1',
|
||||
data: 'one',
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
sourceEndSu: 3,
|
||||
sourceLengthSu: 3
|
||||
})
|
||||
expect(onData).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('drops provisional frames and settles cancellation before rollback completes', async () => {
|
||||
const { handler, mux, dataListeners, livePtyIds, installReceivingActivation } =
|
||||
createSubscription()
|
||||
const onData = vi.fn()
|
||||
dataListeners.add(onData)
|
||||
const lease = installReceivingActivation(
|
||||
'pty-1',
|
||||
sourceActivation({ checkpointSourceEndSu: 4, recoveryEndSu: 8 })
|
||||
)
|
||||
handler('pty.data', {
|
||||
id: 'pty-1',
|
||||
data: 'next',
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
sourceEndSu: 8,
|
||||
sourceLengthSu: 4
|
||||
})
|
||||
|
||||
await expect(lease.rollback()).resolves.toBe(true)
|
||||
|
||||
expect(onData).not.toHaveBeenCalled()
|
||||
expect(livePtyIds).not.toContain('ssh:conn@@pty-1')
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', {
|
||||
id: 'pty-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-1'
|
||||
})
|
||||
})
|
||||
|
||||
it('restores the exact prior cursor when a replacement rolls back after frames', async () => {
|
||||
const { handler, dataListeners, installReceivingActivation } = createSubscription()
|
||||
const onData = vi.fn()
|
||||
dataListeners.add(onData)
|
||||
installReceivingActivation('pty-1', sourceActivation({ deliveryToken: 'token-old' })).commit()
|
||||
handler('pty.data', {
|
||||
id: 'pty-1',
|
||||
data: 'pre',
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-old',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
sourceEndSu: 3,
|
||||
sourceLengthSu: 3
|
||||
})
|
||||
const replacement = installReceivingActivation(
|
||||
'pty-1',
|
||||
sourceActivation({
|
||||
clientGeneration: 3,
|
||||
ownerGeneration: 4,
|
||||
deliveryToken: 'token-new',
|
||||
checkpointSourceEndSu: 3,
|
||||
recoveryEndSu: 3
|
||||
})
|
||||
)
|
||||
handler('pty.data', {
|
||||
id: 'pty-1',
|
||||
data: 'new',
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-new',
|
||||
clientGeneration: 3,
|
||||
ownerGeneration: 4,
|
||||
sourceEndSu: 6,
|
||||
sourceLengthSu: 3
|
||||
})
|
||||
|
||||
await replacement.rollback()
|
||||
handler('pty.data', {
|
||||
id: 'pty-1',
|
||||
data: 'old',
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-old',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
sourceEndSu: 6,
|
||||
sourceLengthSu: 3
|
||||
})
|
||||
|
||||
expect(onData.mock.calls.map(([payload]) => payload.data)).toEqual(['pre', 'old'])
|
||||
})
|
||||
|
||||
it('does not let an older lease rollback replace a newer activation', async () => {
|
||||
const { handler, mux, dataListeners, installReceivingActivation } = createSubscription()
|
||||
const onData = vi.fn()
|
||||
dataListeners.add(onData)
|
||||
const older = installReceivingActivation('pty-1', sourceActivation())
|
||||
const newer = installReceivingActivation(
|
||||
'pty-1',
|
||||
sourceActivation({
|
||||
clientGeneration: 3,
|
||||
ownerGeneration: 4,
|
||||
deliveryToken: 'token-new'
|
||||
})
|
||||
)
|
||||
|
||||
await older.rollback()
|
||||
handler('pty.data', {
|
||||
id: 'pty-1',
|
||||
data: 'new',
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-new',
|
||||
clientGeneration: 3,
|
||||
ownerGeneration: 4,
|
||||
sourceEndSu: 3,
|
||||
sourceLengthSu: 3
|
||||
})
|
||||
newer.commit()
|
||||
|
||||
expect(onData).toHaveBeenCalledWith(expect.objectContaining({ data: 'new' }))
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', {
|
||||
id: 'pty-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-1'
|
||||
})
|
||||
expect(mux.request).not.toHaveBeenCalledWith(
|
||||
'pty.cancelDelivery',
|
||||
expect.objectContaining({ deliveryToken: 'token-new' })
|
||||
)
|
||||
})
|
||||
|
||||
it('ignores PTY methods with missing ids', () => {
|
||||
const { handler, toAppPtyId, dataListeners } = createSubscription()
|
||||
const onData = vi.fn()
|
||||
@@ -105,4 +810,49 @@ describe('subscribeSshPtyNotifications', () => {
|
||||
expect(toAppPtyId).not.toHaveBeenCalled()
|
||||
expect(onData).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('leaves recovery and cancellation control methods to their dedicated handlers', () => {
|
||||
const {
|
||||
handler,
|
||||
mux,
|
||||
toAppPtyId,
|
||||
dataListeners,
|
||||
replayListeners,
|
||||
exitListeners,
|
||||
livePtyIds,
|
||||
recordExit,
|
||||
resolvePtyIncarnation
|
||||
} = createSubscription()
|
||||
const onData = vi.fn()
|
||||
const onReplay = vi.fn()
|
||||
const onExit = vi.fn()
|
||||
dataListeners.add(onData)
|
||||
replayListeners.add(onReplay)
|
||||
exitListeners.add(onExit)
|
||||
livePtyIds.add('ssh:conn@@unrelated')
|
||||
|
||||
for (const method of [
|
||||
'pty.recoveryData',
|
||||
'pty.recoveryComplete',
|
||||
'pty.restoreRequired',
|
||||
'pty.deliveryCanceled'
|
||||
]) {
|
||||
handler(method, {
|
||||
id: 'pty-1',
|
||||
data: 'control',
|
||||
deliveryToken: 'token-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3
|
||||
})
|
||||
}
|
||||
|
||||
expect(toAppPtyId).not.toHaveBeenCalled()
|
||||
expect(resolvePtyIncarnation).not.toHaveBeenCalled()
|
||||
expect(recordExit).not.toHaveBeenCalled()
|
||||
expect(onData).not.toHaveBeenCalled()
|
||||
expect(onReplay).not.toHaveBeenCalled()
|
||||
expect(onExit).not.toHaveBeenCalled()
|
||||
expect(livePtyIds).toEqual(new Set(['ssh:conn@@unrelated']))
|
||||
expect(mux.request).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,12 +1,35 @@
|
||||
import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer'
|
||||
import { isPtyIncarnationId } from '../../shared/pty-incarnation'
|
||||
import type { PtySourceReceivingActivation } from '../../shared/pty-source-receiving-activation'
|
||||
import type {
|
||||
SshPtyDataCallback,
|
||||
SshPtyExitCallback,
|
||||
SshPtyReplayCallback
|
||||
} from './ssh-pty-provider-contract'
|
||||
import { parseSshPtySourceFrame } from './ssh-pty-source-frame'
|
||||
import {
|
||||
SshPtySourceDeliveryLedger,
|
||||
type PendingSshPtySourceData
|
||||
} from './ssh-pty-source-delivery-ledger'
|
||||
|
||||
export type { SshPtyDataCallback, SshPtyExitCallback, SshPtyReplayCallback }
|
||||
export type SshPtyRecoveryActivationLease = Readonly<{
|
||||
commit: () => void
|
||||
retire: () => void
|
||||
}>
|
||||
export type SshPtyReceivingActivationLease = Readonly<{
|
||||
commit: () => void
|
||||
rollback: () => Promise<boolean>
|
||||
transferToRecovery: (sink: SshPtyDataCallback) => SshPtyRecoveryActivationLease
|
||||
}>
|
||||
|
||||
export type SshPtyNotificationSubscription = Readonly<{
|
||||
dispose: () => void
|
||||
installReceivingActivation: (
|
||||
relayPtyId: string,
|
||||
activation: PtySourceReceivingActivation
|
||||
) => SshPtyReceivingActivationLease
|
||||
}>
|
||||
|
||||
export function subscribeSshPtyNotifications(args: {
|
||||
mux: SshChannelMultiplexer
|
||||
@@ -16,8 +39,36 @@ export function subscribeSshPtyNotifications(args: {
|
||||
exitListeners: Set<SshPtyExitCallback>
|
||||
livePtyIds: Set<string>
|
||||
recordExit: (relayPtyId: string, incarnationId: unknown) => void
|
||||
}): () => void {
|
||||
return args.mux.onNotification((method, params) => {
|
||||
providerGeneration: number
|
||||
resolvePtyIncarnation: (relayPtyId: string, incarnationId?: unknown) => string
|
||||
}): SshPtyNotificationSubscription {
|
||||
const toDataPayload = (pending: PendingSshPtySourceData): Parameters<SshPtyDataCallback>[0] => {
|
||||
const id = args.toAppPtyId(pending.relayPtyId)
|
||||
const ptyIncarnation = pending.source
|
||||
? (pending.params.ptyIncarnation as string)
|
||||
: args.resolvePtyIncarnation(pending.relayPtyId, pending.params.incarnationId)
|
||||
return {
|
||||
id,
|
||||
data: pending.data,
|
||||
providerGeneration: args.providerGeneration,
|
||||
ptyIncarnation,
|
||||
...(typeof pending.params.rawLength === 'number'
|
||||
? { sequenceChars: pending.params.rawLength }
|
||||
: {}),
|
||||
...(pending.params.transformed === true ? { transformed: true } : {}),
|
||||
...(typeof pending.params.seq === 'number' ? { seq: pending.params.seq } : {}),
|
||||
...(pending.source ? { source: pending.source } : {})
|
||||
}
|
||||
}
|
||||
const publishData = (pending: PendingSshPtySourceData): void => {
|
||||
const payload = toDataPayload(pending)
|
||||
args.livePtyIds.add(payload.id)
|
||||
for (const listener of args.dataListeners) {
|
||||
listener(payload)
|
||||
}
|
||||
}
|
||||
const sourceDeliveries = new SshPtySourceDeliveryLedger(args.mux, publishData)
|
||||
const dispose = args.mux.onNotification((method, params) => {
|
||||
// Why: mux delivers every method to generic handlers; non-PTY payloads
|
||||
// (workspace.changed, fs.changed, …) have no `id` and must not reach
|
||||
// toAppPtyId → startsWith.
|
||||
@@ -28,14 +79,18 @@ export function subscribeSshPtyNotifications(args: {
|
||||
return
|
||||
}
|
||||
const relayPtyId = params.id
|
||||
const id = args.toAppPtyId(relayPtyId)
|
||||
if (method === 'pty.exit') {
|
||||
const id = args.toAppPtyId(relayPtyId)
|
||||
const ptyIncarnation = args.resolvePtyIncarnation(relayPtyId, params.incarnationId)
|
||||
args.recordExit(relayPtyId, params.incarnationId)
|
||||
args.livePtyIds.delete(id)
|
||||
sourceDeliveries.recordExit(relayPtyId)
|
||||
for (const listener of args.exitListeners) {
|
||||
listener({
|
||||
id,
|
||||
code: params.code as number,
|
||||
providerGeneration: args.providerGeneration,
|
||||
ptyIncarnation,
|
||||
...(isPtyIncarnationId(params.incarnationId)
|
||||
? { incarnationId: params.incarnationId }
|
||||
: {})
|
||||
@@ -43,21 +98,77 @@ export function subscribeSshPtyNotifications(args: {
|
||||
}
|
||||
return
|
||||
}
|
||||
args.livePtyIds.add(id)
|
||||
if (method === 'pty.replay') {
|
||||
const id = args.toAppPtyId(relayPtyId)
|
||||
args.livePtyIds.add(id)
|
||||
for (const listener of args.replayListeners) {
|
||||
listener({ id, data: params.data as string })
|
||||
}
|
||||
return
|
||||
}
|
||||
for (const listener of args.dataListeners) {
|
||||
listener({
|
||||
id,
|
||||
data: params.data as string,
|
||||
...(typeof params.rawLength === 'number' ? { sequenceChars: params.rawLength } : {}),
|
||||
...(params.transformed === true ? { transformed: true } : {}),
|
||||
...(typeof params.seq === 'number' ? { seq: params.seq } : {})
|
||||
const data = typeof params.data === 'string' ? params.data : ''
|
||||
const sourceFrame = parseSshPtySourceFrame(params, data, relayPtyId)
|
||||
if (sourceFrame.malformed) {
|
||||
cancelExactSourceDelivery(args.mux, relayPtyId, params)
|
||||
return
|
||||
}
|
||||
const pending = Object.freeze({
|
||||
relayPtyId,
|
||||
params,
|
||||
data,
|
||||
source: sourceFrame.source
|
||||
})
|
||||
if (sourceFrame.source) {
|
||||
if (!sourceDeliveries.admit({ ...pending, source: sourceFrame.source })) {
|
||||
cancelExactSourceDelivery(args.mux, relayPtyId, params)
|
||||
}
|
||||
return
|
||||
}
|
||||
publishData(pending)
|
||||
})
|
||||
return Object.freeze({
|
||||
dispose,
|
||||
installReceivingActivation: (relayPtyId, activation) => {
|
||||
const lease = sourceDeliveries.install(relayPtyId, activation)
|
||||
return Object.freeze({
|
||||
commit: lease.commit,
|
||||
rollback: lease.rollback,
|
||||
transferToRecovery: (sink: SshPtyDataCallback) =>
|
||||
lease.transferToRecovery((pending) => sink(toDataPayload(pending)))
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function cancelExactSourceDelivery(
|
||||
mux: SshChannelMultiplexer,
|
||||
relayPtyId: string,
|
||||
params: {
|
||||
deliveryToken?: unknown
|
||||
clientGeneration?: unknown
|
||||
ownerGeneration?: unknown
|
||||
}
|
||||
): void {
|
||||
if (
|
||||
typeof params.deliveryToken !== 'string' ||
|
||||
params.deliveryToken.length === 0 ||
|
||||
!positiveSafeInteger(params.clientGeneration) ||
|
||||
!positiveSafeInteger(params.ownerGeneration)
|
||||
) {
|
||||
return
|
||||
}
|
||||
try {
|
||||
void mux
|
||||
.request('pty.cancelDelivery', {
|
||||
id: relayPtyId,
|
||||
clientGeneration: params.clientGeneration,
|
||||
ownerGeneration: params.ownerGeneration,
|
||||
deliveryToken: params.deliveryToken
|
||||
})
|
||||
.catch(() => {})
|
||||
} catch {}
|
||||
}
|
||||
|
||||
function positiveSafeInteger(value: unknown): value is number {
|
||||
return Number.isSafeInteger(value) && (value as number) > 0
|
||||
}
|
||||
|
||||
@@ -1,7 +1,76 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { AGENT_SESSION_CREATE_OPERATION_PROTOCOL_VERSION } from '../../shared/agent-session-host-authority'
|
||||
import {
|
||||
AGENT_SESSION_CREATE_OPERATION_PROTOCOL_VERSION,
|
||||
AGENT_SESSION_EXECUTION_OWNER_PROTOCOL_VERSION
|
||||
} from '../../shared/agent-session-host-authority'
|
||||
import { SshChannelMultiplexer, type MultiplexerTransport } from '../ssh/ssh-channel-multiplexer'
|
||||
import { encodeFrame, HEADER_LENGTH, MessageType } from '../ssh/relay-protocol'
|
||||
import { SshPtyProvider } from './ssh-pty-provider'
|
||||
|
||||
function createTransport(): MultiplexerTransport & {
|
||||
deliver: (data: Buffer) => void
|
||||
written: Buffer[]
|
||||
} {
|
||||
let deliver = (_data: Buffer): void => {}
|
||||
const written: Buffer[] = []
|
||||
return {
|
||||
write: (data) => {
|
||||
written.push(data)
|
||||
},
|
||||
onData: (callback) => {
|
||||
deliver = callback
|
||||
},
|
||||
onClose: () => {},
|
||||
deliver: (data) => deliver(data),
|
||||
written
|
||||
}
|
||||
}
|
||||
|
||||
function rpcFrame(payload: Record<string, unknown>, sequence: number): Buffer {
|
||||
return encodeFrame(MessageType.Regular, sequence, 0, Buffer.from(JSON.stringify(payload)))
|
||||
}
|
||||
|
||||
function responseFrame(id: number, result: unknown, sequence: number): Buffer {
|
||||
return rpcFrame({ jsonrpc: '2.0', id, result }, sequence)
|
||||
}
|
||||
|
||||
function notificationFrame(
|
||||
method: string,
|
||||
params: Record<string, unknown>,
|
||||
sequence: number
|
||||
): Buffer {
|
||||
return rpcFrame({ jsonrpc: '2.0', method, params }, sequence)
|
||||
}
|
||||
|
||||
function requestPayloads(transport: ReturnType<typeof createTransport>): Record<string, unknown>[] {
|
||||
return transport.written.flatMap((frame) => {
|
||||
if (frame[0] !== MessageType.Regular) {
|
||||
return []
|
||||
}
|
||||
const payloadLength = frame.readUInt32BE(9)
|
||||
return [
|
||||
JSON.parse(frame.subarray(HEADER_LENGTH, HEADER_LENGTH + payloadLength).toString()) as Record<
|
||||
string,
|
||||
unknown
|
||||
>
|
||||
]
|
||||
})
|
||||
}
|
||||
|
||||
async function waitForRequest(
|
||||
transport: ReturnType<typeof createTransport>,
|
||||
method: string
|
||||
): Promise<Record<string, unknown>> {
|
||||
for (let turn = 0; turn < 10; turn += 1) {
|
||||
const request = requestPayloads(transport).find((payload) => payload.method === method)
|
||||
if (request) {
|
||||
return request
|
||||
}
|
||||
await Promise.resolve()
|
||||
}
|
||||
throw new Error(`request not dispatched: ${method}`)
|
||||
}
|
||||
|
||||
describe('SSH fresh agent-session create operations', () => {
|
||||
const request = vi.fn()
|
||||
let provider: SshPtyProvider
|
||||
@@ -35,14 +104,19 @@ describe('SSH fresh agent-session create operations', () => {
|
||||
signal: undefined,
|
||||
timeoutMs: 5_000
|
||||
})
|
||||
expect(request).toHaveBeenNthCalledWith(2, 'pty.spawn', {
|
||||
cols: 80,
|
||||
rows: 24,
|
||||
cwd: undefined,
|
||||
env: { POWERLEVEL9K_DISABLE_CONFIGURATION_WIZARD: 'true' },
|
||||
command: 'codex',
|
||||
agentSessionCreateOperationId: 'a'.repeat(43)
|
||||
})
|
||||
expect(request).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
'pty.spawn',
|
||||
{
|
||||
cols: 80,
|
||||
rows: 24,
|
||||
cwd: undefined,
|
||||
env: { POWERLEVEL9K_DISABLE_CONFIGURATION_WIZARD: 'true' },
|
||||
command: 'codex',
|
||||
agentSessionCreateOperationId: 'a'.repeat(43)
|
||||
},
|
||||
expect.objectContaining({ beforeResolve: expect.any(Function) })
|
||||
)
|
||||
})
|
||||
|
||||
it('does not downgrade after structured dispatch reaches an old relay', async () => {
|
||||
@@ -70,13 +144,18 @@ describe('SSH fresh agent-session create operations', () => {
|
||||
})
|
||||
).resolves.toMatchObject({ id: 'ssh:conn-1@@pty-legacy' })
|
||||
|
||||
expect(request).toHaveBeenNthCalledWith(1, 'pty.spawn', {
|
||||
cols: 80,
|
||||
rows: 24,
|
||||
cwd: undefined,
|
||||
env: { POWERLEVEL9K_DISABLE_CONFIGURATION_WIZARD: 'true' },
|
||||
command: 'codex'
|
||||
})
|
||||
expect(request).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
'pty.spawn',
|
||||
{
|
||||
cols: 80,
|
||||
rows: 24,
|
||||
cwd: undefined,
|
||||
env: { POWERLEVEL9K_DISABLE_CONFIGURATION_WIZARD: 'true' },
|
||||
command: 'codex'
|
||||
},
|
||||
expect.objectContaining({ beforeResolve: expect.any(Function) })
|
||||
)
|
||||
})
|
||||
|
||||
it('re-probes a negative capability after an in-place relay upgrade', async () => {
|
||||
@@ -155,4 +234,163 @@ describe('SSH fresh agent-session create operations', () => {
|
||||
})
|
||||
expect(request).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('withholds same-turn source data until claim validation and isolates rollback', async () => {
|
||||
const transport = createTransport()
|
||||
const mux = new SshChannelMultiplexer(transport)
|
||||
const exactProvider = new SshPtyProvider('conn-1', mux)
|
||||
const onData = vi.fn()
|
||||
exactProvider.onData(onData)
|
||||
const claim = {
|
||||
digestVersion: 1 as const,
|
||||
keyId: 'key',
|
||||
identityDigest: 'a'.repeat(43),
|
||||
worktreeScopeDigest: 'b'.repeat(43),
|
||||
agent: 'codex' as const
|
||||
}
|
||||
const surface = {
|
||||
worktreeId: 'worktree',
|
||||
tabId: 'tab',
|
||||
leafId: '11111111-1111-4111-8111-111111111111',
|
||||
terminalHandle: 'term_claimed'
|
||||
}
|
||||
const spawn = exactProvider.spawn({
|
||||
cols: 80,
|
||||
rows: 24,
|
||||
agentSessionEnsure: { claim, surface }
|
||||
})
|
||||
const capabilityRequest = await waitForRequest(transport, 'pty.getCapabilities')
|
||||
transport.deliver(
|
||||
responseFrame(
|
||||
capabilityRequest.id as number,
|
||||
{ agentSessionClaimVersion: AGENT_SESSION_EXECUTION_OWNER_PROTOCOL_VERSION },
|
||||
1
|
||||
)
|
||||
)
|
||||
const spawnRequest = await waitForRequest(transport, 'pty.spawn')
|
||||
const oldActivation = {
|
||||
status: 'pending',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
ptyIncarnation: 'incarnation-old',
|
||||
deliveryToken: 'token-old',
|
||||
checkpointSourceEndSu: 0,
|
||||
recoveryEndSu: 3
|
||||
}
|
||||
transport.deliver(
|
||||
Buffer.concat([
|
||||
responseFrame(
|
||||
spawnRequest.id as number,
|
||||
{
|
||||
id: 'pty-1',
|
||||
incarnationId: 'incarnation-old',
|
||||
sourceActivation: oldActivation,
|
||||
agentSessionEnsure: {
|
||||
disposition: 'created',
|
||||
owner: {
|
||||
claim: { ...claim, identityDigest: 'c'.repeat(43) },
|
||||
generation: 'generation-old',
|
||||
phase: 'live',
|
||||
ptyId: 'pty-1',
|
||||
surface
|
||||
}
|
||||
}
|
||||
},
|
||||
2
|
||||
),
|
||||
notificationFrame(
|
||||
'pty.data',
|
||||
{
|
||||
id: 'pty-1',
|
||||
data: 'old',
|
||||
ptyIncarnation: 'incarnation-old',
|
||||
deliveryToken: 'token-old',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
sourceEndSu: 3,
|
||||
sourceLengthSu: 3
|
||||
},
|
||||
3
|
||||
)
|
||||
])
|
||||
)
|
||||
|
||||
expect(onData).not.toHaveBeenCalled()
|
||||
expect(exactProvider.hasPty('ssh:conn-1@@pty-1')).toBe(false)
|
||||
const shutdownRequest = await waitForRequest(transport, 'pty.shutdown')
|
||||
transport.deliver(responseFrame(shutdownRequest.id as number, null, 4))
|
||||
const cancelRequest = await waitForRequest(transport, 'pty.cancelDelivery')
|
||||
let spawnSettled = false
|
||||
void spawn.then(
|
||||
() => {
|
||||
spawnSettled = true
|
||||
},
|
||||
() => {
|
||||
spawnSettled = true
|
||||
}
|
||||
)
|
||||
await Promise.resolve()
|
||||
expect(spawnSettled).toBe(false)
|
||||
expect(cancelRequest.params).toEqual({
|
||||
id: 'pty-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-old'
|
||||
})
|
||||
|
||||
const replacement = exactProvider.spawn({ cols: 80, rows: 24 })
|
||||
const replacementRequest = requestPayloads(transport).findLast(
|
||||
(payload) => payload.method === 'pty.spawn'
|
||||
)
|
||||
expect(replacementRequest).toBeDefined()
|
||||
transport.deliver(
|
||||
Buffer.concat([
|
||||
responseFrame(
|
||||
replacementRequest!.id as number,
|
||||
{
|
||||
id: 'pty-1',
|
||||
incarnationId: 'incarnation-new',
|
||||
sourceActivation: {
|
||||
...oldActivation,
|
||||
clientGeneration: 4,
|
||||
ownerGeneration: 5,
|
||||
ptyIncarnation: 'incarnation-new',
|
||||
deliveryToken: 'token-new'
|
||||
}
|
||||
},
|
||||
5
|
||||
),
|
||||
notificationFrame(
|
||||
'pty.data',
|
||||
{
|
||||
id: 'pty-1',
|
||||
data: 'new',
|
||||
ptyIncarnation: 'incarnation-new',
|
||||
deliveryToken: 'token-new',
|
||||
clientGeneration: 4,
|
||||
ownerGeneration: 5,
|
||||
sourceEndSu: 3,
|
||||
sourceLengthSu: 3
|
||||
},
|
||||
6
|
||||
)
|
||||
])
|
||||
)
|
||||
await expect(replacement).resolves.toMatchObject({ incarnationId: 'incarnation-new' })
|
||||
expect(onData.mock.calls.map(([payload]) => payload.data)).toEqual(['new'])
|
||||
|
||||
transport.deliver(
|
||||
responseFrame(
|
||||
cancelRequest.id as number,
|
||||
{ canceled: true, sentEndSu: 3, creditedEndSu: 0 },
|
||||
7
|
||||
)
|
||||
)
|
||||
await expect(spawn).rejects.toThrow('agent_session_ownership_unknown')
|
||||
expect(exactProvider.hasPty('ssh:conn-1@@pty-1')).toBe(true)
|
||||
expect(
|
||||
requestPayloads(transport).filter((payload) => payload.method === 'pty.cancelDelivery')
|
||||
).toHaveLength(1)
|
||||
mux.dispose()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -5,19 +5,40 @@ export type RemoteCliBridgeEnv = {
|
||||
relayDir: string
|
||||
nodePath: string
|
||||
sockPath: string
|
||||
credentialFile?: string
|
||||
pathDelimiter?: ':' | ';'
|
||||
}
|
||||
|
||||
export type SshPtyDataCallback = (payload: {
|
||||
id: string
|
||||
data: string
|
||||
providerGeneration: number
|
||||
ptyIncarnation: string
|
||||
sequenceChars?: number
|
||||
transformed?: boolean
|
||||
seq?: number
|
||||
source?: Readonly<{
|
||||
relayPtyId: string
|
||||
spanId: string
|
||||
clientGeneration: number
|
||||
ownerGeneration: number
|
||||
deliveryToken: string
|
||||
sourceStartSu: number
|
||||
sourceEndSu: number
|
||||
}>
|
||||
sourceMalformed?: boolean
|
||||
}) => void
|
||||
export type SshPtyReplayCallback = (payload: { id: string; data: string }) => void
|
||||
export type SshPtyExitCallback = (payload: {
|
||||
id: string
|
||||
code: number
|
||||
providerGeneration: number
|
||||
ptyIncarnation: string
|
||||
incarnationId?: PtyIncarnationId
|
||||
}) => void
|
||||
|
||||
export type SshPtyDeliveryPauseAdapter = (args: {
|
||||
id: string
|
||||
providerGeneration: number
|
||||
paused: boolean
|
||||
}) => void
|
||||
|
||||
@@ -1,6 +1,18 @@
|
||||
import { expect, it, vi } from 'vitest'
|
||||
import { SshPtyProvider } from './ssh-pty-provider'
|
||||
|
||||
function sourceActivation(ptyIncarnation: string) {
|
||||
return {
|
||||
status: 'pending' as const,
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
ptyIncarnation,
|
||||
deliveryToken: `token:${ptyIncarnation}`,
|
||||
checkpointSourceEndSu: 0,
|
||||
recoveryEndSu: 0
|
||||
}
|
||||
}
|
||||
|
||||
it('rejects a fresh SSH PTY whose exit shares the spawn response batch', async () => {
|
||||
const mux = {
|
||||
request: vi.fn(),
|
||||
@@ -10,17 +22,38 @@ it('rejects a fresh SSH PTY whose exit shares the spawn response batch', async (
|
||||
isDisposed: vi.fn().mockReturnValue(false)
|
||||
}
|
||||
const provider = new SshPtyProvider('conn-1', mux as never)
|
||||
const dataListener = vi.fn()
|
||||
provider.onData(dataListener)
|
||||
const exitListener = vi.fn()
|
||||
provider.onExit(exitListener)
|
||||
mux.request.mockImplementation(async (method: string) => {
|
||||
mux.request.mockImplementation(async (method: string, _params, options) => {
|
||||
if (method === 'pty.spawn') {
|
||||
const result = {
|
||||
id: 'pty-raced',
|
||||
incarnationId: 'incarnation-raced',
|
||||
sourceActivation: sourceActivation('incarnation-raced')
|
||||
}
|
||||
options?.beforeResolve?.(result)
|
||||
const notify = mux.onNotification.mock.calls[0]?.[0]
|
||||
notify?.('pty.data', {
|
||||
id: 'pty-raced',
|
||||
data: 'data',
|
||||
ptyIncarnation: 'incarnation-raced',
|
||||
deliveryToken: 'token:incarnation-raced',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
sourceEndSu: 4,
|
||||
sourceLengthSu: 4
|
||||
})
|
||||
notify?.('pty.exit', {
|
||||
id: 'pty-raced',
|
||||
code: 0,
|
||||
incarnationId: 'incarnation-raced'
|
||||
})
|
||||
return { id: 'pty-raced', incarnationId: 'incarnation-raced' }
|
||||
return result
|
||||
}
|
||||
if (method === 'pty.cancelDelivery') {
|
||||
return { canceled: true, sentEndSu: 4, creditedEndSu: 0 }
|
||||
}
|
||||
return undefined
|
||||
})
|
||||
@@ -32,7 +65,16 @@ it('rejects a fresh SSH PTY whose exit shares the spawn response batch', async (
|
||||
expect(exitListener).toHaveBeenCalledWith({
|
||||
id: 'ssh:conn-1@@pty-raced',
|
||||
code: 0,
|
||||
incarnationId: 'incarnation-raced'
|
||||
incarnationId: 'incarnation-raced',
|
||||
providerGeneration: expect.any(Number),
|
||||
ptyIncarnation: 'incarnation-raced'
|
||||
})
|
||||
expect(dataListener).not.toHaveBeenCalled()
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', {
|
||||
id: 'pty-raced',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token:incarnation-raced'
|
||||
})
|
||||
mux.request.mockResolvedValue({ id: 'pty-next', incarnationId: 'incarnation-next' })
|
||||
await expect(provider.spawn({ cols: 80, rows: 24 })).resolves.toMatchObject({
|
||||
@@ -50,15 +92,20 @@ it('rejects an SSH reattach whose matching exit shares the attach reply batch',
|
||||
isDisposed: vi.fn().mockReturnValue(false)
|
||||
}
|
||||
const provider = new SshPtyProvider('conn-1', mux as never)
|
||||
mux.request.mockImplementation(async (method: string) => {
|
||||
mux.request.mockImplementation(async (method: string, _params, options) => {
|
||||
if (method === 'pty.attach') {
|
||||
const result = {
|
||||
incarnationId: 'incarnation-existing',
|
||||
sourceActivation: sourceActivation('incarnation-existing')
|
||||
}
|
||||
options?.beforeResolve?.(result)
|
||||
const notify = mux.onNotification.mock.calls[0]?.[0]
|
||||
notify?.('pty.exit', {
|
||||
id: 'pty-existing',
|
||||
code: 0,
|
||||
incarnationId: 'incarnation-existing'
|
||||
})
|
||||
return { incarnationId: 'incarnation-existing' }
|
||||
return result
|
||||
}
|
||||
return undefined
|
||||
})
|
||||
@@ -66,6 +113,12 @@ it('rejects an SSH reattach whose matching exit shares the attach reply batch',
|
||||
await expect(
|
||||
provider.spawn({ cols: 80, rows: 24, sessionId: 'ssh:conn-1@@pty-existing' })
|
||||
).rejects.toThrow('agent_session_exited_during_start')
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', {
|
||||
id: 'pty-existing',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token:incarnation-existing'
|
||||
})
|
||||
|
||||
mux.request.mockResolvedValue({ incarnationId: 'incarnation-next' })
|
||||
await expect(
|
||||
@@ -76,3 +129,44 @@ it('rejects an SSH reattach whose matching exit shares the attach reply batch',
|
||||
isReattach: true
|
||||
})
|
||||
})
|
||||
|
||||
it('returns a provisional source activation lease to reconnect authority', async () => {
|
||||
const mux = {
|
||||
request: vi.fn(),
|
||||
notify: vi.fn(),
|
||||
onNotification: vi.fn(),
|
||||
dispose: vi.fn(),
|
||||
isDisposed: vi.fn().mockReturnValue(false)
|
||||
}
|
||||
const provider = new SshPtyProvider('conn-1', mux as never)
|
||||
const response = {
|
||||
incarnationId: 'incarnation-reconnect',
|
||||
sourceActivation: {
|
||||
...sourceActivation('incarnation-reconnect'),
|
||||
deliveryToken: 'token-reconnect',
|
||||
checkpointSourceEndSu: 4,
|
||||
recoveryEndSu: 8
|
||||
}
|
||||
}
|
||||
mux.request.mockImplementation(async (method: string, _params, options) => {
|
||||
if (method === 'pty.cancelDelivery') {
|
||||
return { canceled: true, sentEndSu: 8, creditedEndSu: 4 }
|
||||
}
|
||||
if (method !== 'pty.attach') {
|
||||
return undefined
|
||||
}
|
||||
options?.beforeResolve?.(response)
|
||||
return response
|
||||
})
|
||||
|
||||
const result = await provider.attachForReconnect('ssh:conn-1@@pty-1')
|
||||
await result.sourceActivationLease?.rollback()
|
||||
|
||||
expect(Object.isFrozen(result.sourceActivation)).toBe(true)
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.cancelDelivery', {
|
||||
id: 'pty-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-reconnect'
|
||||
})
|
||||
})
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer'
|
||||
import type {
|
||||
SshPtyDataCallback,
|
||||
SshPtyDeliveryPauseAdapter,
|
||||
SshPtyExitCallback,
|
||||
SshPtyReplayCallback
|
||||
} from './ssh-pty-provider-contract'
|
||||
import {
|
||||
subscribeSshPtyNotifications,
|
||||
type SshPtyNotificationSubscription,
|
||||
type SshPtyReceivingActivationLease
|
||||
} from './ssh-pty-notification-routing'
|
||||
import type { PtySourceReceivingActivation } from '../../shared/pty-source-receiving-activation'
|
||||
|
||||
export class SshPtyProviderOutputState {
|
||||
private readonly dataListeners = new Set<SshPtyDataCallback>()
|
||||
private readonly replayListeners = new Set<SshPtyReplayCallback>()
|
||||
private readonly exitListeners = new Set<SshPtyExitCallback>()
|
||||
private readonly incarnationByRelayPtyId = new Map<string, string>()
|
||||
private readonly pausedRelayPtyIds = new Set<string>()
|
||||
private deliveryPauseAdapter: SshPtyDeliveryPauseAdapter | null = null
|
||||
private legacyIncarnationSerial = 1
|
||||
private subscription: SshPtyNotificationSubscription | null
|
||||
|
||||
constructor(
|
||||
private readonly providerGeneration: number,
|
||||
args: {
|
||||
mux: SshChannelMultiplexer
|
||||
toAppPtyId: (id: string) => string
|
||||
livePtyIds: Set<string>
|
||||
recordExit: (relayPtyId: string, incarnationId: unknown) => void
|
||||
}
|
||||
) {
|
||||
this.subscription = subscribeSshPtyNotifications({
|
||||
...args,
|
||||
dataListeners: this.dataListeners,
|
||||
replayListeners: this.replayListeners,
|
||||
exitListeners: this.exitListeners,
|
||||
providerGeneration,
|
||||
resolvePtyIncarnation: (relayPtyId, incarnationId) =>
|
||||
this.resolvePtyIncarnation(relayPtyId, incarnationId),
|
||||
recordExit: (relayPtyId, incarnationId) => {
|
||||
args.recordExit(relayPtyId, incarnationId)
|
||||
this.incarnationByRelayPtyId.delete(relayPtyId)
|
||||
this.pausedRelayPtyIds.delete(relayPtyId)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
this.resumePausedDeliveries()
|
||||
this.subscription?.dispose()
|
||||
this.subscription = null
|
||||
this.dataListeners.clear()
|
||||
this.replayListeners.clear()
|
||||
this.exitListeners.clear()
|
||||
this.incarnationByRelayPtyId.clear()
|
||||
this.deliveryPauseAdapter = null
|
||||
}
|
||||
|
||||
onData(callback: SshPtyDataCallback): () => void {
|
||||
this.dataListeners.add(callback)
|
||||
return () => this.dataListeners.delete(callback)
|
||||
}
|
||||
|
||||
onReplay(callback: SshPtyReplayCallback): () => void {
|
||||
this.replayListeners.add(callback)
|
||||
return () => this.replayListeners.delete(callback)
|
||||
}
|
||||
|
||||
onExit(callback: SshPtyExitCallback): () => void {
|
||||
this.exitListeners.add(callback)
|
||||
return () => this.exitListeners.delete(callback)
|
||||
}
|
||||
|
||||
setDeliveryPauseAdapter(adapter: SshPtyDeliveryPauseAdapter | null): void {
|
||||
if (adapter !== this.deliveryPauseAdapter) {
|
||||
this.resumePausedDeliveries()
|
||||
}
|
||||
this.deliveryPauseAdapter = adapter
|
||||
}
|
||||
|
||||
hasDeliveryPauseAdapter(): boolean {
|
||||
return this.deliveryPauseAdapter !== null
|
||||
}
|
||||
|
||||
pause(id: string): void {
|
||||
if (!this.deliveryPauseAdapter || this.pausedRelayPtyIds.has(id)) {
|
||||
return
|
||||
}
|
||||
this.pausedRelayPtyIds.add(id)
|
||||
this.deliveryPauseAdapter({ id, providerGeneration: this.providerGeneration, paused: true })
|
||||
}
|
||||
|
||||
resume(id: string): void {
|
||||
if (!this.deliveryPauseAdapter || !this.pausedRelayPtyIds.delete(id)) {
|
||||
return
|
||||
}
|
||||
this.deliveryPauseAdapter({ id, providerGeneration: this.providerGeneration, paused: false })
|
||||
}
|
||||
|
||||
installReceivingActivation(
|
||||
relayPtyId: string,
|
||||
activation: PtySourceReceivingActivation
|
||||
): SshPtyReceivingActivationLease {
|
||||
if (!this.subscription) {
|
||||
throw new Error('ssh_source_receiving_activation_disposed')
|
||||
}
|
||||
return this.subscription.installReceivingActivation(relayPtyId, activation)
|
||||
}
|
||||
|
||||
rememberPtyIncarnation(relayPtyId: string, incarnationId: unknown): void {
|
||||
if (
|
||||
!this.incarnationByRelayPtyId.has(relayPtyId) &&
|
||||
typeof incarnationId === 'string' &&
|
||||
incarnationId.length > 0
|
||||
) {
|
||||
this.incarnationByRelayPtyId.set(relayPtyId, incarnationId)
|
||||
}
|
||||
}
|
||||
|
||||
private resolvePtyIncarnation(relayPtyId: string, incarnationId: unknown): string {
|
||||
this.rememberPtyIncarnation(relayPtyId, incarnationId)
|
||||
let resolved = this.incarnationByRelayPtyId.get(relayPtyId)
|
||||
if (!resolved) {
|
||||
resolved = `legacy:${this.providerGeneration}:${this.legacyIncarnationSerial++}:${relayPtyId}`
|
||||
this.incarnationByRelayPtyId.set(relayPtyId, resolved)
|
||||
}
|
||||
return resolved
|
||||
}
|
||||
|
||||
private resumePausedDeliveries(): void {
|
||||
const adapter = this.deliveryPauseAdapter
|
||||
if (!adapter) {
|
||||
this.pausedRelayPtyIds.clear()
|
||||
return
|
||||
}
|
||||
for (const id of this.pausedRelayPtyIds) {
|
||||
try {
|
||||
adapter({ id, providerGeneration: this.providerGeneration, paused: false })
|
||||
} catch {
|
||||
/* Generation close is the fallback cleanup proof. */
|
||||
}
|
||||
}
|
||||
this.pausedRelayPtyIds.clear()
|
||||
}
|
||||
}
|
||||
@@ -13,7 +13,7 @@ function createMockMux(): MockMultiplexer {
|
||||
return {
|
||||
request: vi.fn().mockResolvedValue(undefined),
|
||||
notify: vi.fn(),
|
||||
onNotification: vi.fn(),
|
||||
onNotification: vi.fn().mockReturnValue(vi.fn()),
|
||||
dispose: vi.fn(),
|
||||
isDisposed: vi.fn().mockReturnValue(false)
|
||||
}
|
||||
@@ -135,10 +135,17 @@ describe('SshPtyProvider process listings and events', () => {
|
||||
notify('pty.replay', { id: 'pty-1', data: 'buffered output' })
|
||||
notify('pty.exit', { id: 'pty-1', code: 0, incarnationId: 'incarnation-1' })
|
||||
|
||||
expect(dataHandler).toHaveBeenNthCalledWith(1, { id: scopedPty1, data: 'output' })
|
||||
expect(dataHandler).toHaveBeenNthCalledWith(1, {
|
||||
id: scopedPty1,
|
||||
data: 'output',
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'legacy:1:1:pty-1'
|
||||
})
|
||||
expect(dataHandler).toHaveBeenNthCalledWith(2, {
|
||||
id: scopedPty1,
|
||||
data: '',
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'legacy:1:1:pty-1',
|
||||
sequenceChars: 9,
|
||||
seq: 9,
|
||||
transformed: true
|
||||
@@ -147,10 +154,28 @@ describe('SshPtyProvider process listings and events', () => {
|
||||
expect(exitHandler).toHaveBeenCalledWith({
|
||||
id: scopedPty1,
|
||||
code: 0,
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'legacy:1:1:pty-1',
|
||||
incarnationId: 'incarnation-1'
|
||||
})
|
||||
})
|
||||
|
||||
it('keeps fallback admission identity stable when spawn metadata arrives later', async () => {
|
||||
const dataHandler = vi.fn()
|
||||
provider.onData(dataHandler)
|
||||
const notify = mux.onNotification.mock.calls[0][0]
|
||||
|
||||
notify('pty.data', { id: 'pty-1', data: 'before-response' })
|
||||
mux.request.mockResolvedValue({ id: 'pty-1', incarnationId: 'incarnation-1' })
|
||||
await provider.spawn({ cols: 80, rows: 24 })
|
||||
notify('pty.data', { id: 'pty-1', data: 'after-response' })
|
||||
|
||||
expect(dataHandler.mock.calls.map(([payload]) => payload.ptyIncarnation)).toEqual([
|
||||
'legacy:1:1:pty-1',
|
||||
'legacy:1:1:pty-1'
|
||||
])
|
||||
})
|
||||
|
||||
it('supports listener removal, fanout, and connection namespaces', () => {
|
||||
const removed = vi.fn()
|
||||
const first = vi.fn()
|
||||
@@ -170,6 +195,29 @@ describe('SshPtyProvider process listings and events', () => {
|
||||
const other = vi.fn()
|
||||
otherProvider.onData(other)
|
||||
otherMux.onNotification.mock.calls[0][0]('pty.data', { id: 'pty-1', data: 'second' })
|
||||
expect(other).toHaveBeenCalledWith({ id: 'ssh:conn-2@@pty-1', data: 'second' })
|
||||
expect(other).toHaveBeenCalledWith({
|
||||
id: 'ssh:conn-2@@pty-1',
|
||||
data: 'second',
|
||||
providerGeneration: 1,
|
||||
ptyIncarnation: 'legacy:1:1:pty-1'
|
||||
})
|
||||
})
|
||||
|
||||
it('scopes pause delivery adapters and resumes them during cleanup', () => {
|
||||
const adapter = vi.fn()
|
||||
provider.setPtyDeliveryPauseAdapter(adapter)
|
||||
|
||||
provider.pauseProducer(scopedPty1)
|
||||
provider.pauseProducer(scopedPty1)
|
||||
provider.resumeProducer(scopedPty1)
|
||||
provider.pauseProducer(scopedPty1)
|
||||
provider.dispose()
|
||||
|
||||
expect(adapter.mock.calls).toEqual([
|
||||
[{ id: 'pty-1', providerGeneration: 1, paused: true }],
|
||||
[{ id: 'pty-1', providerGeneration: 1, paused: false }],
|
||||
[{ id: 'pty-1', providerGeneration: 1, paused: true }],
|
||||
[{ id: 'pty-1', providerGeneration: 1, paused: false }]
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { SSH_SESSION_EXPIRED_ERROR } from './ssh-pty-errors'
|
||||
import { SshPtyProvider } from './ssh-pty-provider'
|
||||
|
||||
describe('SSH PTY provider session reattach incarnation', () => {
|
||||
it('remembers the authoritative incarnation before a legacy exit arrives', async () => {
|
||||
let notify: ((method: string, params: Record<string, unknown>) => void) | undefined
|
||||
const mux = {
|
||||
request: vi.fn().mockResolvedValue({ incarnationId: 'incarnation-reattached' }),
|
||||
notify: vi.fn(),
|
||||
onNotification: vi.fn(
|
||||
(callback: (method: string, params: Record<string, unknown>) => void) => {
|
||||
notify = callback
|
||||
return vi.fn()
|
||||
}
|
||||
)
|
||||
}
|
||||
const provider = new SshPtyProvider('conn-1', mux as never)
|
||||
const onExit = vi.fn()
|
||||
provider.onExit(onExit)
|
||||
|
||||
await provider.spawn({ cols: 80, rows: 24, sessionId: 'pty-old' })
|
||||
notify?.('pty.exit', { id: 'pty-old', code: 0 })
|
||||
|
||||
expect(onExit).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
id: 'ssh:conn-1@@pty-old',
|
||||
ptyIncarnation: 'incarnation-reattached'
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
it('fails closed when generic reattach requires source restoration', async () => {
|
||||
const mux = {
|
||||
request: vi.fn().mockResolvedValue({
|
||||
incarnationId: 'incarnation-reattached',
|
||||
sourceRecovery: {
|
||||
status: 'restoreRequired',
|
||||
reason: 'checkpointUnavailable'
|
||||
}
|
||||
}),
|
||||
notify: vi.fn(),
|
||||
onNotification: vi.fn().mockReturnValue(vi.fn())
|
||||
}
|
||||
const provider = new SshPtyProvider('conn-1', mux as never)
|
||||
|
||||
await expect(provider.spawn({ cols: 80, rows: 24, sessionId: 'pty-old' })).rejects.toThrow(
|
||||
`${SSH_SESSION_EXPIRED_ERROR}: pty-old`
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -22,6 +22,14 @@ function createMockMux(): MockMultiplexer {
|
||||
}
|
||||
}
|
||||
|
||||
const sourceActivationRequestOptions = expect.objectContaining({
|
||||
beforeResolve: expect.any(Function)
|
||||
})
|
||||
|
||||
function expectRequest(request: ReturnType<typeof vi.fn>, ...expected: unknown[]): void {
|
||||
expect(request.mock.calls.map((call) => call.slice(0, expected.length))).toContainEqual(expected)
|
||||
}
|
||||
|
||||
describe('SshPtyProvider', () => {
|
||||
let mux: MockMultiplexer
|
||||
let provider: SshPtyProvider
|
||||
@@ -184,7 +192,7 @@ describe('SshPtyProvider', () => {
|
||||
await expect(
|
||||
provider.spawn({ cols: 80, rows: 24, agentSessionEnsure: { claim, surface } })
|
||||
).rejects.toThrow('agent_session_ownership_unknown')
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.shutdown', {
|
||||
expectRequest(mux.request, 'pty.shutdown', {
|
||||
id: 'pty-malformed',
|
||||
immediate: true
|
||||
})
|
||||
@@ -261,7 +269,7 @@ describe('SshPtyProvider', () => {
|
||||
|
||||
const result = await provider.spawn({ cols: 80, rows: 24 })
|
||||
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.spawn', {
|
||||
expectRequest(mux.request, 'pty.spawn', {
|
||||
cols: 80,
|
||||
rows: 24,
|
||||
cwd: undefined,
|
||||
@@ -289,7 +297,8 @@ describe('SshPtyProvider', () => {
|
||||
|
||||
await provider.spawn({ cols: 80, rows: 24, startupIngress })
|
||||
|
||||
expect(mux.request).toHaveBeenCalledWith(
|
||||
expectRequest(
|
||||
mux.request,
|
||||
'pty.spawn',
|
||||
expect.objectContaining({
|
||||
startupIngressVersion: PTY_STARTUP_INGRESS_VERSION,
|
||||
@@ -308,7 +317,7 @@ describe('SshPtyProvider', () => {
|
||||
env: { FOO: 'bar' }
|
||||
})
|
||||
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.spawn', {
|
||||
expectRequest(mux.request, 'pty.spawn', {
|
||||
cols: 120,
|
||||
rows: 40,
|
||||
cwd: '/home/user',
|
||||
@@ -326,7 +335,8 @@ describe('SshPtyProvider', () => {
|
||||
launchAgent: 'claude'
|
||||
})
|
||||
|
||||
expect(mux.request).toHaveBeenCalledWith(
|
||||
expectRequest(
|
||||
mux.request,
|
||||
'pty.spawn',
|
||||
expect.objectContaining({
|
||||
command: 'cd /repo && custom-agent-wrapper',
|
||||
@@ -345,7 +355,7 @@ describe('SshPtyProvider', () => {
|
||||
tabId: 'tab-a'
|
||||
})
|
||||
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.spawn', {
|
||||
expectRequest(mux.request, 'pty.spawn', {
|
||||
cols: 120,
|
||||
rows: 40,
|
||||
cwd: undefined,
|
||||
@@ -365,7 +375,7 @@ describe('SshPtyProvider', () => {
|
||||
terminalWindowsWslDistro: 'Ubuntu'
|
||||
})
|
||||
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.spawn', {
|
||||
expectRequest(mux.request, 'pty.spawn', {
|
||||
cols: 120,
|
||||
rows: 40,
|
||||
cwd: undefined,
|
||||
@@ -384,7 +394,7 @@ describe('SshPtyProvider', () => {
|
||||
env: { [POWERLEVEL10K_WIZARD_DISABLE_ENV]: 'already-set' }
|
||||
})
|
||||
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.spawn', {
|
||||
expectRequest(mux.request, 'pty.spawn', {
|
||||
cols: 120,
|
||||
rows: 40,
|
||||
cwd: undefined,
|
||||
@@ -402,7 +412,7 @@ describe('SshPtyProvider', () => {
|
||||
envToDelete: [POWERLEVEL10K_WIZARD_DISABLE_ENV]
|
||||
})
|
||||
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.spawn', {
|
||||
expectRequest(mux.request, 'pty.spawn', {
|
||||
cols: 120,
|
||||
rows: 40,
|
||||
cwd: undefined,
|
||||
@@ -426,7 +436,7 @@ describe('SshPtyProvider', () => {
|
||||
envToDelete
|
||||
})
|
||||
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.spawn', {
|
||||
expectRequest(mux.request, 'pty.spawn', {
|
||||
cols: 120,
|
||||
rows: 40,
|
||||
cwd: undefined,
|
||||
@@ -452,7 +462,7 @@ describe('SshPtyProvider', () => {
|
||||
startupCommandDelivery: 'shell-ready'
|
||||
})
|
||||
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.spawn', {
|
||||
expectRequest(mux.request, 'pty.spawn', {
|
||||
cols: 120,
|
||||
rows: 40,
|
||||
cwd: undefined,
|
||||
@@ -478,7 +488,7 @@ describe('SshPtyProvider', () => {
|
||||
env: { PATH: '/usr/bin', ORCA_TERMINAL_HANDLE: 'term_ssh' }
|
||||
})
|
||||
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.spawn', {
|
||||
expectRequest(mux.request, 'pty.spawn', {
|
||||
cols: 120,
|
||||
rows: 40,
|
||||
cwd: undefined,
|
||||
@@ -509,7 +519,7 @@ describe('SshPtyProvider', () => {
|
||||
env: { ORCA_TERMINAL_HANDLE: 'term_ssh' }
|
||||
})
|
||||
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.spawn', {
|
||||
expectRequest(mux.request, 'pty.spawn', {
|
||||
cols: 120,
|
||||
rows: 40,
|
||||
cwd: undefined,
|
||||
@@ -540,7 +550,7 @@ describe('SshPtyProvider', () => {
|
||||
env: { Path: 'C:/Windows/System32;C:/Tools' }
|
||||
})
|
||||
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.spawn', {
|
||||
expectRequest(mux.request, 'pty.spawn', {
|
||||
cols: 120,
|
||||
rows: 40,
|
||||
cwd: undefined,
|
||||
@@ -563,12 +573,17 @@ describe('SshPtyProvider', () => {
|
||||
|
||||
const result = await provider.spawn({ cols: 80, rows: 24, sessionId: 'pty-old' })
|
||||
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.attach', {
|
||||
id: 'pty-old',
|
||||
cols: 80,
|
||||
rows: 24,
|
||||
suppressReplayNotification: true
|
||||
})
|
||||
expectRequest(
|
||||
mux.request,
|
||||
'pty.attach',
|
||||
{
|
||||
id: 'pty-old',
|
||||
cols: 80,
|
||||
rows: 24,
|
||||
suppressReplayNotification: true
|
||||
},
|
||||
sourceActivationRequestOptions
|
||||
)
|
||||
expect(result).toEqual({
|
||||
id: 'ssh:conn-1@@pty-old',
|
||||
isReattach: true,
|
||||
@@ -590,7 +605,8 @@ describe('SshPtyProvider', () => {
|
||||
}
|
||||
})
|
||||
|
||||
expect(mux.request).toHaveBeenCalledWith(
|
||||
expectRequest(
|
||||
mux.request,
|
||||
'pty.attach',
|
||||
expect.not.objectContaining({ startupIngress: expect.anything() })
|
||||
)
|
||||
@@ -606,7 +622,7 @@ describe('SshPtyProvider', () => {
|
||||
sessionId: 'ssh:conn-1@@pty-old'
|
||||
})
|
||||
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.attach', {
|
||||
expectRequest(mux.request, 'pty.attach', {
|
||||
id: 'pty-old',
|
||||
cols: 80,
|
||||
rows: 24,
|
||||
@@ -630,7 +646,7 @@ describe('SshPtyProvider', () => {
|
||||
tabId: 'tab-a'
|
||||
})
|
||||
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.attach', {
|
||||
expectRequest(mux.request, 'pty.attach', {
|
||||
id: 'pty-old',
|
||||
cols: 80,
|
||||
rows: 24,
|
||||
@@ -647,12 +663,17 @@ describe('SshPtyProvider', () => {
|
||||
'SSH_SESSION_EXPIRED: pty-old'
|
||||
)
|
||||
|
||||
expect(mux.request).toHaveBeenNthCalledWith(1, 'pty.attach', {
|
||||
id: 'pty-old',
|
||||
cols: 80,
|
||||
rows: 24,
|
||||
suppressReplayNotification: true
|
||||
})
|
||||
expect(mux.request).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
'pty.attach',
|
||||
{
|
||||
id: 'pty-old',
|
||||
cols: 80,
|
||||
rows: 24,
|
||||
suppressReplayNotification: true
|
||||
},
|
||||
sourceActivationRequestOptions
|
||||
)
|
||||
expect(mux.request).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
@@ -669,7 +690,7 @@ describe('SshPtyProvider', () => {
|
||||
|
||||
it('attach sends pty.attach request', async () => {
|
||||
await provider.attach(scopedPty1)
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.attach', { id: 'pty-1' })
|
||||
expectRequest(mux.request, 'pty.attach', { id: 'pty-1' })
|
||||
})
|
||||
|
||||
it('attachForReconnect returns replay without relay notification', async () => {
|
||||
@@ -684,10 +705,18 @@ describe('SshPtyProvider', () => {
|
||||
replay: 'restored output',
|
||||
incarnationId: 'incarnation-reconnect'
|
||||
})
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.attach', {
|
||||
id: 'pty-1',
|
||||
suppressReplayNotification: true
|
||||
})
|
||||
expectRequest(
|
||||
mux.request,
|
||||
'pty.attach',
|
||||
{
|
||||
id: 'pty-1',
|
||||
suppressReplayNotification: true
|
||||
},
|
||||
expect.objectContaining({
|
||||
timeoutMs: 10_000,
|
||||
beforeResolve: expect.any(Function)
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
it('keeps missing incarnation compatible with an old relay', async () => {
|
||||
@@ -712,12 +741,20 @@ describe('SshPtyProvider', () => {
|
||||
tabId: 'tab-a'
|
||||
})
|
||||
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.attach', {
|
||||
id: 'pty-1',
|
||||
suppressReplayNotification: true,
|
||||
expectedPaneKey: 'tab-a:leaf-a',
|
||||
expectedTabId: 'tab-a'
|
||||
})
|
||||
expectRequest(
|
||||
mux.request,
|
||||
'pty.attach',
|
||||
{
|
||||
id: 'pty-1',
|
||||
suppressReplayNotification: true,
|
||||
expectedPaneKey: 'tab-a:leaf-a',
|
||||
expectedTabId: 'tab-a'
|
||||
},
|
||||
expect.objectContaining({
|
||||
timeoutMs: 10_000,
|
||||
beforeResolve: expect.any(Function)
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
it('write sends pty.data notification', () => {
|
||||
@@ -734,7 +771,7 @@ describe('SshPtyProvider', () => {
|
||||
mux.request.mockResolvedValue({ cols: 120, rows: 40 })
|
||||
|
||||
await expect(provider.getAppliedSize(scopedPty1)).resolves.toEqual({ cols: 120, rows: 40 })
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.getSize', { id: 'pty-1' }, { timeoutMs: 1_000 })
|
||||
expectRequest(mux.request, 'pty.getSize', { id: 'pty-1' }, { timeoutMs: 1_000 })
|
||||
})
|
||||
|
||||
it('caches only an old relay method-not-found response', async () => {
|
||||
@@ -759,7 +796,8 @@ describe('SshPtyProvider', () => {
|
||||
|
||||
it('shutdown sends pty.shutdown request', async () => {
|
||||
await provider.shutdown(scopedPty1, { immediate: true })
|
||||
expect(mux.request).toHaveBeenCalledWith(
|
||||
expectRequest(
|
||||
mux.request,
|
||||
'pty.shutdown',
|
||||
{
|
||||
id: 'pty-1',
|
||||
@@ -772,7 +810,8 @@ describe('SshPtyProvider', () => {
|
||||
|
||||
it('shutdown forwards keepHistory: true over the relay', async () => {
|
||||
await provider.shutdown(scopedPty1, { immediate: true, keepHistory: true })
|
||||
expect(mux.request).toHaveBeenCalledWith(
|
||||
expectRequest(
|
||||
mux.request,
|
||||
'pty.shutdown',
|
||||
{
|
||||
id: 'pty-1',
|
||||
@@ -789,7 +828,8 @@ describe('SshPtyProvider', () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
await provider.shutdown(scopedPty1, { immediate: true, deadlineMs: Date.now() + 4321 })
|
||||
expect(mux.request).toHaveBeenCalledWith(
|
||||
expectRequest(
|
||||
mux.request,
|
||||
'pty.shutdown',
|
||||
{ id: 'pty-1', immediate: true, keepHistory: false },
|
||||
{ timeoutMs: 4321 }
|
||||
@@ -801,19 +841,19 @@ describe('SshPtyProvider', () => {
|
||||
|
||||
it('sendSignal sends pty.sendSignal request', async () => {
|
||||
await provider.sendSignal(scopedPty1, 'SIGINT')
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.sendSignal', { id: 'pty-1', signal: 'SIGINT' })
|
||||
expectRequest(mux.request, 'pty.sendSignal', { id: 'pty-1', signal: 'SIGINT' })
|
||||
})
|
||||
|
||||
it('getCwd sends pty.getCwd request', async () => {
|
||||
mux.request.mockResolvedValue('/home/user/project')
|
||||
const cwd = await provider.getCwd(scopedPty1)
|
||||
expect(cwd).toBe('/home/user/project')
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.getCwd', { id: 'pty-1' })
|
||||
expectRequest(mux.request, 'pty.getCwd', { id: 'pty-1' })
|
||||
})
|
||||
|
||||
it('clearBuffer sends pty.clearBuffer request', async () => {
|
||||
await provider.clearBuffer(scopedPty1)
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.clearBuffer', { id: 'pty-1' })
|
||||
expectRequest(mux.request, 'pty.clearBuffer', { id: 'pty-1' })
|
||||
})
|
||||
|
||||
it('acknowledgeDataEvent sends pty.ackData notification', () => {
|
||||
@@ -825,14 +865,14 @@ describe('SshPtyProvider', () => {
|
||||
mux.request.mockResolvedValue(true)
|
||||
const result = await provider.hasChildProcesses(scopedPty1)
|
||||
expect(result).toBe(true)
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.hasChildProcesses', { id: 'pty-1' })
|
||||
expectRequest(mux.request, 'pty.hasChildProcesses', { id: 'pty-1' })
|
||||
})
|
||||
|
||||
it('getForegroundProcess returns process name', async () => {
|
||||
mux.request.mockResolvedValue('node')
|
||||
const result = await provider.getForegroundProcess(scopedPty1)
|
||||
expect(result).toBe('node')
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.getForegroundProcess', { id: 'pty-1' })
|
||||
expectRequest(mux.request, 'pty.getForegroundProcess', { id: 'pty-1' })
|
||||
})
|
||||
|
||||
it('preserves unavailable process inspection', async () => {
|
||||
@@ -844,7 +884,7 @@ describe('SshPtyProvider', () => {
|
||||
mux.request.mockResolvedValue(inspection)
|
||||
|
||||
await expect(provider.inspectProcess(scopedPty1)).resolves.toEqual(inspection)
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.inspectProcess', { id: 'pty-1' })
|
||||
expectRequest(mux.request, 'pty.inspectProcess', { id: 'pty-1' })
|
||||
})
|
||||
|
||||
it('serializes scoped app ids using raw relay ids', async () => {
|
||||
@@ -853,7 +893,7 @@ describe('SshPtyProvider', () => {
|
||||
const result = await provider.serialize([scopedPty1])
|
||||
|
||||
expect(result).toBe('serialized')
|
||||
expect(mux.request).toHaveBeenCalledWith('pty.serialize', { ids: ['pty-1'] })
|
||||
expectRequest(mux.request, 'pty.serialize', { ids: ['pty-1'] })
|
||||
})
|
||||
|
||||
it('rejects scoped ids owned by another SSH connection', async () => {
|
||||
|
||||
@@ -5,25 +5,24 @@ import { createSshPtyAppliedSizeReader } from './ssh-pty-applied-size'
|
||||
import type {
|
||||
RemoteCliBridgeEnv,
|
||||
SshPtyDataCallback,
|
||||
SshPtyDeliveryPauseAdapter,
|
||||
SshPtyExitCallback,
|
||||
SshPtyReplayCallback
|
||||
} from './ssh-pty-provider-contract'
|
||||
import { subscribeSshPtyNotifications } from './ssh-pty-notification-routing'
|
||||
import { validateClaimedSshSpawn } from './ssh-agent-session-claim-validation'
|
||||
import {
|
||||
assertSshAgentSessionCreateResult,
|
||||
requestSshAgentSessionCreate
|
||||
} from './ssh-agent-session-create-operation'
|
||||
import { SshPtyProviderOutputState } from './ssh-pty-provider-output-state'
|
||||
import { spawnFreshSshPty } from './ssh-agent-session-create-operation'
|
||||
import { mapSshPtyProcessList } from './ssh-agent-session-process-list'
|
||||
import {
|
||||
parseSshPtyAttachResult,
|
||||
requestSshPtyAttach,
|
||||
reattachSshPtySessionWithExitFence,
|
||||
type PtySourceRecoveryRequest,
|
||||
type SshPtyAttachResult
|
||||
} from './ssh-pty-session-reattach'
|
||||
import { buildSshPtySpawnRequest } from './ssh-pty-spawn-request'
|
||||
import { SshPtySpawnExitRaceTracker } from './ssh-pty-spawn-exit-race'
|
||||
import { SshAgentSessionCapabilities } from './ssh-agent-session-capabilities'
|
||||
import type { PtyProcessInspection } from './pty-process-inspection'
|
||||
import { SSH_SESSION_EXPIRED_ERROR } from './ssh-pty-errors'
|
||||
|
||||
// Why: sequential relay teardown calls share one absolute budget; convert to the mux-relative timeout only at dispatch.
|
||||
function relayTimeoutOptions(deadlineMs: number | undefined): { timeoutMs: number } | undefined {
|
||||
@@ -34,58 +33,43 @@ function relayTimeoutOptions(deadlineMs: number | undefined): { timeoutMs: numbe
|
||||
export class SshPtyProvider implements IPtyProvider {
|
||||
private mux: SshChannelMultiplexer
|
||||
private connectionId: string
|
||||
private dataListeners = new Set<SshPtyDataCallback>()
|
||||
private replayListeners = new Set<SshPtyReplayCallback>()
|
||||
private exitListeners = new Set<SshPtyExitCallback>()
|
||||
private livePtyIds = new Set<string>()
|
||||
// Why: stale notification callbacks must not outlive a disconnected provider.
|
||||
private unsubscribeNotifications: (() => void) | null = null
|
||||
readonly getAppliedSize: NonNullable<IPtyProvider['getAppliedSize']>
|
||||
private readonly agentSessionCapabilities: SshAgentSessionCapabilities
|
||||
private spawnExitRaces = new SshPtySpawnExitRaceTracker()
|
||||
private readonly outputState: SshPtyProviderOutputState
|
||||
|
||||
constructor(
|
||||
connectionId: string,
|
||||
mux: SshChannelMultiplexer,
|
||||
private readonly remoteCliBridgeEnv?: RemoteCliBridgeEnv
|
||||
private readonly remoteCliBridgeEnv?: RemoteCliBridgeEnv,
|
||||
readonly providerGeneration = 1
|
||||
) {
|
||||
this.connectionId = connectionId
|
||||
this.mux = mux
|
||||
this.agentSessionCapabilities = new SshAgentSessionCapabilities(mux)
|
||||
this.getAppliedSize = createSshPtyAppliedSizeReader(mux, connectionId)
|
||||
|
||||
this.unsubscribeNotifications = subscribeSshPtyNotifications({
|
||||
this.outputState = new SshPtyProviderOutputState(providerGeneration, {
|
||||
mux,
|
||||
toAppPtyId: (id) => this.toAppPtyId(id),
|
||||
dataListeners: this.dataListeners,
|
||||
replayListeners: this.replayListeners,
|
||||
exitListeners: this.exitListeners,
|
||||
livePtyIds: this.livePtyIds,
|
||||
recordExit: (relayPtyId, incarnationId) =>
|
||||
recordExit: (relayPtyId, incarnationId) => {
|
||||
this.spawnExitRaces.recordExit(relayPtyId, incarnationId)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
if (this.unsubscribeNotifications) {
|
||||
this.unsubscribeNotifications()
|
||||
this.unsubscribeNotifications = null
|
||||
}
|
||||
this.dataListeners.clear()
|
||||
this.replayListeners.clear()
|
||||
this.exitListeners.clear()
|
||||
this.outputState.dispose()
|
||||
this.livePtyIds.clear()
|
||||
}
|
||||
|
||||
getConnectionId = (): string => this.connectionId
|
||||
|
||||
private toRelayPtyId(id: string): string {
|
||||
return toRelaySshPtyId(this.connectionId, id)
|
||||
}
|
||||
private toRelayPtyId = (id: string): string => toRelaySshPtyId(this.connectionId, id)
|
||||
|
||||
private toAppPtyId(id: string): string {
|
||||
return toAppSshPtyId(this.connectionId, id)
|
||||
}
|
||||
private toAppPtyId = (id: string): string => toAppSshPtyId(this.connectionId, id)
|
||||
|
||||
async spawn(opts: PtySpawnOptions): Promise<PtySpawnResult> {
|
||||
if (opts.agentSessionEnsure && opts.sessionId) {
|
||||
@@ -101,15 +85,36 @@ export class SshPtyProvider implements IPtyProvider {
|
||||
}
|
||||
}
|
||||
if (opts.sessionId) {
|
||||
const result = await reattachSshPtySessionWithExitFence({
|
||||
mux: this.mux,
|
||||
connectionId: this.connectionId,
|
||||
sessionId: opts.sessionId,
|
||||
options: opts,
|
||||
exitRaceTracker: this.spawnExitRaces
|
||||
})
|
||||
this.livePtyIds.add(result.id)
|
||||
return result
|
||||
let result: Awaited<ReturnType<typeof reattachSshPtySessionWithExitFence>> | undefined
|
||||
try {
|
||||
result = await reattachSshPtySessionWithExitFence({
|
||||
mux: this.mux,
|
||||
connectionId: this.connectionId,
|
||||
sessionId: opts.sessionId,
|
||||
options: opts,
|
||||
exitRaceTracker: this.spawnExitRaces,
|
||||
installSourceActivation: (relayPtyId, activation) =>
|
||||
this.outputState.installReceivingActivation(relayPtyId, activation),
|
||||
rememberPtyIncarnation: (relayPtyId, incarnationId) =>
|
||||
this.outputState.rememberPtyIncarnation(relayPtyId, incarnationId)
|
||||
})
|
||||
if (result.sourceRecovery?.status === 'restoreRequired') {
|
||||
throw new Error(
|
||||
`${SSH_SESSION_EXPIRED_ERROR}: ${toRelaySshPtyId(this.connectionId, result.id)}`
|
||||
)
|
||||
}
|
||||
this.livePtyIds.add(result.id)
|
||||
result.sourceActivationLease?.commit()
|
||||
const {
|
||||
sourceActivationLease: _lease,
|
||||
sourceRecovery: _sourceRecovery,
|
||||
...spawnResult
|
||||
} = result
|
||||
return spawnResult
|
||||
} catch (error) {
|
||||
result?.sourceActivationLease?.rollback()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
const supportsCreateOperation = opts.agentSessionCreateOperationId
|
||||
@@ -122,65 +127,22 @@ export class SshPtyProvider implements IPtyProvider {
|
||||
// Why: host routing owns legacy selection; a changed relay must not downgrade after dispatch.
|
||||
throw new Error('execution_owner_unavailable')
|
||||
}
|
||||
const operation = this.spawnExitRaces.begin()
|
||||
try {
|
||||
const result = await requestSshAgentSessionCreate({
|
||||
mux: this.mux,
|
||||
operationId: opts.agentSessionCreateOperationId,
|
||||
signal: opts.signal,
|
||||
params: buildSshPtySpawnRequest({
|
||||
options: opts,
|
||||
remoteCliBridgeEnv: this.remoteCliBridgeEnv,
|
||||
supportsCreateOperation
|
||||
})
|
||||
})
|
||||
if (opts.agentSessionCreateOperationId) {
|
||||
assertSshAgentSessionCreateResult(result)
|
||||
}
|
||||
const spawnResult = result as PtySpawnResult
|
||||
if (this.spawnExitRaces.didMatchingExitArrive(operation, spawnResult)) {
|
||||
// Why: relay notification can share the response batch; no controller registration may follow.
|
||||
throw Object.assign(new Error('agent_session_exited_during_start'), {
|
||||
agentSessionOperationOutcome: 'unknown' as const
|
||||
})
|
||||
}
|
||||
const claimed = spawnResult.agentSessionEnsure
|
||||
if (opts.agentSessionEnsure) {
|
||||
const validation = validateClaimedSshSpawn(spawnResult, opts.agentSessionEnsure)
|
||||
if (!validation.valid) {
|
||||
if (validation.cleanup === 'created' && typeof spawnResult.id === 'string') {
|
||||
try {
|
||||
// Why: immediate relay shutdown resolves only after physical exit;
|
||||
// a best-effort graceful request cannot prove the duplicate is gone.
|
||||
await this.mux.request('pty.shutdown', { id: spawnResult.id, immediate: true })
|
||||
} catch {
|
||||
throw new Error('execution_owner_unavailable')
|
||||
}
|
||||
}
|
||||
throw new Error(validation.error)
|
||||
}
|
||||
}
|
||||
const id = this.toAppPtyId(spawnResult.id)
|
||||
this.livePtyIds.add(id)
|
||||
return {
|
||||
...spawnResult,
|
||||
id,
|
||||
...(claimed
|
||||
? {
|
||||
agentSessionEnsure: {
|
||||
...claimed,
|
||||
owner: {
|
||||
...claimed.owner,
|
||||
ptyId: this.toAppPtyId(claimed.owner.ptyId)
|
||||
}
|
||||
}
|
||||
}
|
||||
: {}),
|
||||
...(opts.sessionId ? { sessionExpired: true } : {})
|
||||
}
|
||||
} finally {
|
||||
this.spawnExitRaces.finish(operation)
|
||||
}
|
||||
return await spawnFreshSshPty({
|
||||
mux: this.mux,
|
||||
options: opts,
|
||||
params: buildSshPtySpawnRequest({
|
||||
options: opts,
|
||||
remoteCliBridgeEnv: this.remoteCliBridgeEnv,
|
||||
supportsCreateOperation
|
||||
}),
|
||||
exitRaceTracker: this.spawnExitRaces,
|
||||
installSourceActivation: (id, activation) =>
|
||||
this.outputState.installReceivingActivation(id, activation),
|
||||
rememberPtyIncarnation: (id, incarnation) =>
|
||||
this.outputState.rememberPtyIncarnation(id, incarnation),
|
||||
acceptLivePty: (id) => this.livePtyIds.add(id),
|
||||
toAppPtyId: this.toAppPtyId
|
||||
})
|
||||
}
|
||||
|
||||
async supportsAgentSessionClaims(options: { signal?: AbortSignal } = {}): Promise<boolean> {
|
||||
@@ -198,25 +160,46 @@ export class SshPtyProvider implements IPtyProvider {
|
||||
}
|
||||
|
||||
async attach(id: string): Promise<void> {
|
||||
await this.mux.request('pty.attach', { id: this.toRelayPtyId(id) })
|
||||
const relayPtyId = this.toRelayPtyId(id)
|
||||
await requestSshPtyAttach({
|
||||
mux: this.mux,
|
||||
relayPtyId,
|
||||
params: { id: relayPtyId },
|
||||
commitSourceActivation: true,
|
||||
installSourceActivation: (ptyId, activation) =>
|
||||
this.outputState.installReceivingActivation(ptyId, activation),
|
||||
rememberPtyIncarnation: (ptyId, incarnationId) =>
|
||||
this.outputState.rememberPtyIncarnation(ptyId, incarnationId)
|
||||
})
|
||||
}
|
||||
|
||||
async attachForReconnect(
|
||||
id: string,
|
||||
expected?: { paneKey?: string; tabId?: string }
|
||||
expected?: { paneKey?: string; tabId?: string },
|
||||
sourceRecovery?: PtySourceRecoveryRequest
|
||||
): Promise<SshPtyAttachResult> {
|
||||
// Why: reconnect owns replay delivery so stale/duplicate attach results can
|
||||
// be filtered before they reach the renderer. The expected identity lets the
|
||||
// relay reject a cross-generation id collision instead of reattaching this
|
||||
// lease to a different pane's freshly spawned PTY.
|
||||
return parseSshPtyAttachResult(
|
||||
await this.mux.request('pty.attach', {
|
||||
id: this.toRelayPtyId(id),
|
||||
suppressReplayNotification: true,
|
||||
...(expected?.paneKey ? { expectedPaneKey: expected.paneKey } : {}),
|
||||
...(expected?.tabId ? { expectedTabId: expected.tabId } : {})
|
||||
})
|
||||
)
|
||||
const params = {
|
||||
id: this.toRelayPtyId(id),
|
||||
suppressReplayNotification: true,
|
||||
...(sourceRecovery ? { sourceRecovery } : {}),
|
||||
...(expected?.paneKey ? { expectedPaneKey: expected.paneKey } : {}),
|
||||
...(expected?.tabId ? { expectedTabId: expected.tabId } : {})
|
||||
}
|
||||
const relayPtyId = this.toRelayPtyId(id)
|
||||
return await requestSshPtyAttach({
|
||||
mux: this.mux,
|
||||
relayPtyId,
|
||||
params,
|
||||
timeoutMs: 10_000,
|
||||
installSourceActivation: (ptyId, activation) =>
|
||||
this.outputState.installReceivingActivation(ptyId, activation),
|
||||
rememberPtyIncarnation: (ptyId, incarnationId) =>
|
||||
this.outputState.rememberPtyIncarnation(ptyId, incarnationId)
|
||||
})
|
||||
}
|
||||
|
||||
write(id: string, data: string): void {
|
||||
@@ -308,6 +291,8 @@ export class SshPtyProvider implements IPtyProvider {
|
||||
const processes = mapSshPtyProcessList(result as PtyProcessInfo[], (id) => this.toAppPtyId(id))
|
||||
for (const process of processes) {
|
||||
this.livePtyIds.add(process.id)
|
||||
const relayPtyId = this.toRelayPtyId(process.id)
|
||||
this.outputState.rememberPtyIncarnation(relayPtyId, process.incarnationId)
|
||||
}
|
||||
return processes
|
||||
}
|
||||
@@ -326,18 +311,28 @@ export class SshPtyProvider implements IPtyProvider {
|
||||
return result as { name: string; path: string }[]
|
||||
}
|
||||
|
||||
onData(callback: SshPtyDataCallback): () => void {
|
||||
this.dataListeners.add(callback)
|
||||
return () => this.dataListeners.delete(callback)
|
||||
onData = (callback: SshPtyDataCallback): (() => void) => this.outputState.onData(callback)
|
||||
onReplay = (callback: SshPtyReplayCallback): (() => void) => this.outputState.onReplay(callback)
|
||||
onExit = (callback: SshPtyExitCallback): (() => void) => this.outputState.onExit(callback)
|
||||
|
||||
setPtyDeliveryPauseAdapter(adapter: SshPtyDeliveryPauseAdapter | null): void {
|
||||
this.outputState.setDeliveryPauseAdapter(adapter)
|
||||
}
|
||||
|
||||
onReplay(callback: SshPtyReplayCallback): () => void {
|
||||
this.replayListeners.add(callback)
|
||||
return () => this.replayListeners.delete(callback)
|
||||
hasPtyDeliveryPauseAdapter(): boolean {
|
||||
return this.outputState.hasDeliveryPauseAdapter()
|
||||
}
|
||||
|
||||
onExit(callback: SshPtyExitCallback): () => void {
|
||||
this.exitListeners.add(callback)
|
||||
return () => this.exitListeners.delete(callback)
|
||||
pauseProducer(id: string): void {
|
||||
this.outputState.pause(this.toRelayPtyId(id))
|
||||
}
|
||||
|
||||
resumeProducer(id: string): void {
|
||||
this.outputState.resume(this.toRelayPtyId(id))
|
||||
}
|
||||
|
||||
closeOutputIntake(reason: string): void {
|
||||
this.mux.dispose('connection_lost')
|
||||
console.error('[ssh-pty-provider] closed after bounded output intake failure', { reason })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,10 +9,27 @@ import {
|
||||
import { toAppSshPtyId, toRelaySshPtyId } from './ssh-pty-id'
|
||||
import type { PtySpawnOptions, PtySpawnResult } from './types'
|
||||
import type { SshPtySpawnExitRaceTracker } from './ssh-pty-spawn-exit-race'
|
||||
import type {
|
||||
PtySourceRecoveryRequest,
|
||||
PtySourceRecoveryResult
|
||||
} from '../../shared/pty-source-recovery-contract'
|
||||
import {
|
||||
parsePtySourceReceivingActivation,
|
||||
type PtySourceReceivingActivation
|
||||
} from '../../shared/pty-source-receiving-activation'
|
||||
import type { SshPtyReceivingActivationLease } from './ssh-pty-notification-routing'
|
||||
|
||||
export type SshPtyAttachResult = {
|
||||
replay?: string
|
||||
incarnationId?: PtyIncarnationId
|
||||
sourceRecovery?: PtySourceRecoveryResult
|
||||
sourceActivation?: PtySourceReceivingActivation
|
||||
sourceActivationLease?: SshPtyReceivingActivationLease
|
||||
}
|
||||
|
||||
type SshPtyReattachResult = PtySpawnResult & {
|
||||
sourceRecovery?: PtySourceRecoveryResult
|
||||
sourceActivationLease?: SshPtyReceivingActivationLease
|
||||
}
|
||||
|
||||
export function parseSshPtyAttachResult(value: unknown): SshPtyAttachResult {
|
||||
@@ -22,7 +39,12 @@ export function parseSshPtyAttachResult(value: unknown): SshPtyAttachResult {
|
||||
if (typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new Error('Invalid SSH PTY attach response')
|
||||
}
|
||||
const result = value as { replay?: unknown; incarnationId?: unknown }
|
||||
const result = value as {
|
||||
replay?: unknown
|
||||
incarnationId?: unknown
|
||||
sourceRecovery?: unknown
|
||||
sourceActivation?: unknown
|
||||
}
|
||||
if (result.replay !== undefined && typeof result.replay !== 'string') {
|
||||
throw new Error('Invalid SSH PTY attach replay')
|
||||
}
|
||||
@@ -30,34 +52,156 @@ export function parseSshPtyAttachResult(value: unknown): SshPtyAttachResult {
|
||||
// Why: a present-but-invalid identity cannot safely fence delayed exits from a reused relay id.
|
||||
throw new Error('Invalid SSH PTY attach incarnation')
|
||||
}
|
||||
const sourceRecovery = parseSourceRecoveryResult(result.sourceRecovery)
|
||||
const sourceActivation = parsePtySourceReceivingActivation(result.sourceActivation)
|
||||
const activation =
|
||||
sourceActivation ?? (sourceRecovery?.status === 'pending' ? sourceRecovery : undefined)
|
||||
if (
|
||||
activation &&
|
||||
(!isPtyIncarnationId(result.incarnationId) ||
|
||||
activation.ptyIncarnation !== result.incarnationId ||
|
||||
(sourceRecovery?.status === 'pending' && !sameSourceActivation(activation, sourceRecovery)))
|
||||
) {
|
||||
throw new Error('Invalid SSH PTY source activation identity')
|
||||
}
|
||||
return {
|
||||
...(typeof result.replay === 'string' ? { replay: result.replay } : {}),
|
||||
...(isPtyIncarnationId(result.incarnationId) ? { incarnationId: result.incarnationId } : {})
|
||||
...(isPtyIncarnationId(result.incarnationId) ? { incarnationId: result.incarnationId } : {}),
|
||||
...(sourceRecovery ? { sourceRecovery } : {}),
|
||||
...(activation ? { sourceActivation: activation } : {})
|
||||
}
|
||||
}
|
||||
|
||||
export async function requestSshPtyAttach(args: {
|
||||
mux: SshChannelMultiplexer
|
||||
relayPtyId: string
|
||||
params: Record<string, unknown>
|
||||
timeoutMs?: number
|
||||
commitSourceActivation?: boolean
|
||||
installSourceActivation?: (
|
||||
relayPtyId: string,
|
||||
activation: PtySourceReceivingActivation
|
||||
) => SshPtyReceivingActivationLease
|
||||
rememberPtyIncarnation?: (relayPtyId: string, incarnationId: unknown) => void
|
||||
}): Promise<SshPtyAttachResult> {
|
||||
let activationLease: SshPtyReceivingActivationLease | undefined
|
||||
const installFromResult = (result: SshPtyAttachResult): void => {
|
||||
if (!activationLease && result.sourceActivation && args.installSourceActivation) {
|
||||
activationLease = args.installSourceActivation(args.relayPtyId, result.sourceActivation)
|
||||
}
|
||||
}
|
||||
try {
|
||||
const rawResult = await args.mux.request('pty.attach', args.params, {
|
||||
...(args.timeoutMs === undefined ? {} : { timeoutMs: args.timeoutMs }),
|
||||
beforeResolve: (value) => installFromResult(parseSshPtyAttachResult(value))
|
||||
})
|
||||
const result = parseSshPtyAttachResult(rawResult)
|
||||
installFromResult(result)
|
||||
args.rememberPtyIncarnation?.(args.relayPtyId, result.incarnationId)
|
||||
if (args.commitSourceActivation) {
|
||||
activationLease?.commit()
|
||||
}
|
||||
return {
|
||||
...result,
|
||||
...(activationLease ? { sourceActivationLease: activationLease } : {})
|
||||
}
|
||||
} catch (error) {
|
||||
activationLease?.rollback()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
function parseSourceRecoveryResult(value: unknown): PtySourceRecoveryResult | undefined {
|
||||
if (value === undefined) {
|
||||
return undefined
|
||||
}
|
||||
if (typeof value !== 'object' || value === null || Array.isArray(value)) {
|
||||
throw new Error('Invalid SSH PTY source recovery response')
|
||||
}
|
||||
const input = value as Record<string, unknown>
|
||||
if (input.status === 'restoreRequired' && typeof input.reason === 'string') {
|
||||
return Object.freeze({ status: 'restoreRequired', reason: input.reason })
|
||||
}
|
||||
if (
|
||||
input.status !== 'pending' ||
|
||||
typeof input.deliveryToken !== 'string' ||
|
||||
input.deliveryToken.length === 0 ||
|
||||
typeof input.ptyIncarnation !== 'string' ||
|
||||
input.ptyIncarnation.length === 0 ||
|
||||
!positiveInteger(input.clientGeneration) ||
|
||||
!positiveInteger(input.ownerGeneration) ||
|
||||
!nonNegativeInteger(input.checkpointSourceEndSu) ||
|
||||
!nonNegativeInteger(input.recoveryEndSu) ||
|
||||
Number(input.recoveryEndSu) < Number(input.checkpointSourceEndSu)
|
||||
) {
|
||||
throw new Error('Invalid SSH PTY source recovery response')
|
||||
}
|
||||
return Object.freeze({
|
||||
status: 'pending',
|
||||
deliveryToken: input.deliveryToken,
|
||||
ptyIncarnation: input.ptyIncarnation,
|
||||
clientGeneration: Number(input.clientGeneration),
|
||||
ownerGeneration: Number(input.ownerGeneration),
|
||||
checkpointSourceEndSu: Number(input.checkpointSourceEndSu),
|
||||
recoveryEndSu: Number(input.recoveryEndSu)
|
||||
})
|
||||
}
|
||||
|
||||
function positiveInteger(value: unknown): boolean {
|
||||
return Number.isSafeInteger(value) && Number(value) > 0
|
||||
}
|
||||
|
||||
function nonNegativeInteger(value: unknown): boolean {
|
||||
return Number.isSafeInteger(value) && Number(value) >= 0
|
||||
}
|
||||
|
||||
function sameSourceActivation(
|
||||
left: PtySourceReceivingActivation,
|
||||
right: PtySourceReceivingActivation
|
||||
): boolean {
|
||||
return (
|
||||
left.clientGeneration === right.clientGeneration &&
|
||||
left.ownerGeneration === right.ownerGeneration &&
|
||||
left.ptyIncarnation === right.ptyIncarnation &&
|
||||
left.deliveryToken === right.deliveryToken &&
|
||||
left.checkpointSourceEndSu === right.checkpointSourceEndSu &&
|
||||
left.recoveryEndSu === right.recoveryEndSu
|
||||
)
|
||||
}
|
||||
|
||||
export type { PtySourceRecoveryRequest }
|
||||
|
||||
export async function reattachSshPtySession(args: {
|
||||
mux: SshChannelMultiplexer
|
||||
connectionId: string
|
||||
sessionId: string
|
||||
options: PtySpawnOptions
|
||||
}): Promise<PtySpawnResult> {
|
||||
rememberPtyIncarnation?: (relayPtyId: string, incarnationId: unknown) => void
|
||||
installSourceActivation?: (
|
||||
relayPtyId: string,
|
||||
activation: PtySourceReceivingActivation
|
||||
) => SshPtyReceivingActivationLease
|
||||
}): Promise<SshPtyReattachResult> {
|
||||
const relaySessionId = toRelaySshPtyId(args.connectionId, args.sessionId)
|
||||
console.warn(`[ssh-pty] spawn() called with sessionId=${args.sessionId}, attempting pty.attach`)
|
||||
try {
|
||||
// Why: expected pane identity prevents a reused relay id from attaching the wrong shell.
|
||||
const expectedPaneKey = args.options.paneKey ?? args.options.env?.ORCA_PANE_KEY
|
||||
const expectedTabId = args.options.tabId ?? args.options.env?.ORCA_TAB_ID
|
||||
const attachResult = parseSshPtyAttachResult(
|
||||
await args.mux.request('pty.attach', {
|
||||
const attachResult = await requestSshPtyAttach({
|
||||
mux: args.mux,
|
||||
relayPtyId: relaySessionId,
|
||||
params: {
|
||||
id: relaySessionId,
|
||||
cols: args.options.cols,
|
||||
rows: args.options.rows,
|
||||
suppressReplayNotification: true,
|
||||
...(expectedPaneKey ? { expectedPaneKey } : {}),
|
||||
...(expectedTabId ? { expectedTabId } : {})
|
||||
})
|
||||
)
|
||||
},
|
||||
installSourceActivation: args.installSourceActivation,
|
||||
rememberPtyIncarnation: args.rememberPtyIncarnation
|
||||
})
|
||||
console.warn(
|
||||
`[ssh-pty] pty.attach succeeded for ${args.sessionId}, replay=${!!attachResult.replay}`
|
||||
)
|
||||
@@ -65,7 +209,12 @@ export async function reattachSshPtySession(args: {
|
||||
id: toAppSshPtyId(args.connectionId, relaySessionId),
|
||||
isReattach: true,
|
||||
...(attachResult.replay ? { replay: attachResult.replay } : {}),
|
||||
...(attachResult.incarnationId ? { incarnationId: attachResult.incarnationId } : {})
|
||||
...(attachResult.incarnationId ? { incarnationId: attachResult.incarnationId } : {}),
|
||||
...(attachResult.sourceRecovery ? { sourceRecovery: attachResult.sourceRecovery } : {}),
|
||||
...(attachResult.sourceActivation ? { sourceActivation: attachResult.sourceActivation } : {}),
|
||||
...(attachResult.sourceActivationLease
|
||||
? { sourceActivationLease: attachResult.sourceActivationLease }
|
||||
: {})
|
||||
}
|
||||
} catch (error) {
|
||||
// Why: an expired relay lease must be surfaced distinctly so the renderer clears its binding.
|
||||
@@ -84,10 +233,11 @@ export async function reattachSshPtySessionWithExitFence(
|
||||
args: Parameters<typeof reattachSshPtySession>[0] & {
|
||||
exitRaceTracker: SshPtySpawnExitRaceTracker
|
||||
}
|
||||
): Promise<PtySpawnResult> {
|
||||
): Promise<SshPtyReattachResult> {
|
||||
const operation = args.exitRaceTracker.begin()
|
||||
let result: SshPtyReattachResult | undefined
|
||||
try {
|
||||
const result = await reattachSshPtySession(args)
|
||||
result = await reattachSshPtySession(args)
|
||||
const relayPtyId = toRelaySshPtyId(args.connectionId, result.id)
|
||||
if (
|
||||
args.exitRaceTracker.didMatchingExitArrive(operation, {
|
||||
@@ -98,6 +248,9 @@ export async function reattachSshPtySessionWithExitFence(
|
||||
throw new Error('agent_session_exited_during_start')
|
||||
}
|
||||
return result
|
||||
} catch (error) {
|
||||
result?.sourceActivationLease?.rollback()
|
||||
throw error
|
||||
} finally {
|
||||
args.exitRaceTracker.finish(operation)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { SshPtySourceDeliveryLedger } from './ssh-pty-source-delivery-ledger'
|
||||
|
||||
describe('SshPtySourceDeliveryLedger', () => {
|
||||
it('retains cancellation ownership when recovery transfer is superseded', async () => {
|
||||
const request = vi.fn(async () => ({ canceled: true, sentEndSu: 0, creditedEndSu: 0 }))
|
||||
const ledger = new SshPtySourceDeliveryLedger({ request } as never, vi.fn())
|
||||
const older = ledger.install(
|
||||
'pty-1',
|
||||
Object.freeze({
|
||||
status: 'pending',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-old',
|
||||
checkpointSourceEndSu: 0,
|
||||
recoveryEndSu: 0
|
||||
})
|
||||
)
|
||||
ledger.install(
|
||||
'pty-1',
|
||||
Object.freeze({
|
||||
status: 'pending',
|
||||
clientGeneration: 3,
|
||||
ownerGeneration: 4,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-new',
|
||||
checkpointSourceEndSu: 0,
|
||||
recoveryEndSu: 0
|
||||
})
|
||||
)
|
||||
|
||||
expect(() => older.transferToRecovery(vi.fn())).toThrow('ssh_source_receiving_activation_stale')
|
||||
await expect(older.rollback()).resolves.toBe(true)
|
||||
|
||||
expect(request).toHaveBeenCalledOnce()
|
||||
expect(request).toHaveBeenCalledWith('pty.cancelDelivery', {
|
||||
id: 'pty-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-old'
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,320 @@
|
||||
import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer'
|
||||
import type { PtySourceReceivingActivation } from '../../shared/pty-source-receiving-activation'
|
||||
import type { SshPtySourceFrame } from './ssh-pty-source-frame'
|
||||
|
||||
export type PendingSshPtySourceData = Readonly<{
|
||||
relayPtyId: string
|
||||
params: Record<string, unknown>
|
||||
data: string
|
||||
source?: SshPtySourceFrame
|
||||
}>
|
||||
|
||||
type SourceDeliveryLeaseState = {
|
||||
phase: 'provisional' | 'recovery' | 'committing' | 'committed' | 'retired'
|
||||
pendingData: PendingSshPtySourceData[]
|
||||
recoverySink?: (pending: PendingSshPtySourceData) => void
|
||||
exited: boolean
|
||||
}
|
||||
|
||||
type SourceDeliveryState = Readonly<{
|
||||
activation: PtySourceReceivingActivation
|
||||
sourceEndSu: number
|
||||
lease: SourceDeliveryLeaseState
|
||||
previous?: SourceDeliveryState
|
||||
}>
|
||||
|
||||
export type SshPtyRecoveryActivationLease = Readonly<{
|
||||
commit: () => void
|
||||
retire: () => void
|
||||
}>
|
||||
|
||||
export type SshPtySourceDeliveryLease = Readonly<{
|
||||
commit: () => void
|
||||
rollback: () => Promise<boolean>
|
||||
transferToRecovery: (
|
||||
sink: (pending: PendingSshPtySourceData) => void
|
||||
) => SshPtyRecoveryActivationLease
|
||||
}>
|
||||
|
||||
export class SshPtySourceDeliveryLedger {
|
||||
private readonly deliveryByPty = new Map<string, SourceDeliveryState>()
|
||||
|
||||
constructor(
|
||||
private readonly mux: SshChannelMultiplexer,
|
||||
private readonly publishData: (pending: PendingSshPtySourceData) => void
|
||||
) {}
|
||||
|
||||
install(relayPtyId: string, activation: PtySourceReceivingActivation): SshPtySourceDeliveryLease {
|
||||
if (!relayPtyId || activation.ptyIncarnation.length === 0) {
|
||||
throw new Error('ssh_source_receiving_activation_invalid')
|
||||
}
|
||||
const previous = this.deliveryByPty.get(relayPtyId)
|
||||
if (previous && sameReceivingActivation(previous.activation, activation)) {
|
||||
if (previous.lease.phase !== 'committed') {
|
||||
throw new Error('ssh_source_receiving_activation_stale')
|
||||
}
|
||||
return settledReceivingActivationLease()
|
||||
}
|
||||
if (
|
||||
previous &&
|
||||
(activation.clientGeneration <= previous.activation.clientGeneration ||
|
||||
activation.ownerGeneration <= previous.activation.ownerGeneration ||
|
||||
activation.deliveryToken === previous.activation.deliveryToken)
|
||||
) {
|
||||
throw new Error('ssh_source_receiving_activation_stale')
|
||||
}
|
||||
return this.installProvisional(relayPtyId, activation, previous)
|
||||
}
|
||||
|
||||
admit(pending: PendingSshPtySourceData & { source: SshPtySourceFrame }): boolean {
|
||||
const current = this.deliveryByPty.get(pending.relayPtyId)
|
||||
if (!acceptsSourceFrame(current, pending.params, pending.source)) {
|
||||
return false
|
||||
}
|
||||
const accepted = Object.freeze({
|
||||
...current,
|
||||
sourceEndSu: pending.source.sourceEndSu
|
||||
}) as SourceDeliveryState
|
||||
this.deliveryByPty.set(pending.relayPtyId, accepted)
|
||||
if (accepted.lease.phase === 'recovery') {
|
||||
accepted.lease.recoverySink?.(pending)
|
||||
} else if (accepted.lease.phase !== 'committed') {
|
||||
accepted.lease.pendingData.push(pending)
|
||||
} else {
|
||||
this.publishData(pending)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
recordExit(relayPtyId: string): void {
|
||||
const current = this.deliveryByPty.get(relayPtyId)
|
||||
if (current?.lease.phase === 'committed') {
|
||||
this.deliveryByPty.delete(relayPtyId)
|
||||
} else if (current) {
|
||||
current.lease.exited = true
|
||||
}
|
||||
}
|
||||
|
||||
private installProvisional(
|
||||
relayPtyId: string,
|
||||
activation: PtySourceReceivingActivation,
|
||||
previous: SourceDeliveryState | undefined
|
||||
): SshPtySourceDeliveryLease {
|
||||
const leaseState: SourceDeliveryLeaseState = {
|
||||
phase: 'provisional',
|
||||
pendingData: [],
|
||||
exited: false
|
||||
}
|
||||
this.deliveryByPty.set(
|
||||
relayPtyId,
|
||||
Object.freeze({
|
||||
activation,
|
||||
sourceEndSu: activation.checkpointSourceEndSu,
|
||||
lease: leaseState,
|
||||
...(previous ? { previous } : {})
|
||||
})
|
||||
)
|
||||
let settled = false
|
||||
let transferInProgress = false
|
||||
let rollbackSettlement: Promise<boolean> | undefined
|
||||
return Object.freeze({
|
||||
commit: () => {
|
||||
if (settled || transferInProgress) {
|
||||
return
|
||||
}
|
||||
settled = true
|
||||
this.commit(relayPtyId, leaseState)
|
||||
},
|
||||
rollback: () => {
|
||||
if (rollbackSettlement) {
|
||||
return rollbackSettlement
|
||||
}
|
||||
if (settled || transferInProgress) {
|
||||
return Promise.resolve(false)
|
||||
}
|
||||
settled = true
|
||||
this.retire(relayPtyId, previous, leaseState)
|
||||
rollbackSettlement = settleExactSourceDeliveryCancellation(this.mux, relayPtyId, activation)
|
||||
return rollbackSettlement
|
||||
},
|
||||
transferToRecovery: (sink) => {
|
||||
if (settled || transferInProgress || leaseState.phase !== 'provisional') {
|
||||
throw new Error('ssh_source_receiving_activation_stale')
|
||||
}
|
||||
transferInProgress = true
|
||||
try {
|
||||
const recoveryLease = this.transferToRecovery(relayPtyId, leaseState, previous, sink)
|
||||
settled = true
|
||||
return recoveryLease
|
||||
} finally {
|
||||
transferInProgress = false
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
private transferToRecovery(
|
||||
relayPtyId: string,
|
||||
lease: SourceDeliveryLeaseState,
|
||||
previous: SourceDeliveryState | undefined,
|
||||
sink: (pending: PendingSshPtySourceData) => void
|
||||
): SshPtyRecoveryActivationLease {
|
||||
if (this.deliveryByPty.get(relayPtyId)?.lease !== lease) {
|
||||
this.retire(relayPtyId, previous, lease)
|
||||
throw new Error('ssh_source_receiving_activation_stale')
|
||||
}
|
||||
lease.phase = 'recovery'
|
||||
lease.recoverySink = sink
|
||||
try {
|
||||
while (lease.pendingData.length > 0) {
|
||||
sink(lease.pendingData.shift()!)
|
||||
}
|
||||
} catch (error) {
|
||||
this.retire(relayPtyId, previous, lease)
|
||||
throw error
|
||||
}
|
||||
let settled = false
|
||||
return Object.freeze({
|
||||
commit: () => {
|
||||
if (settled) {
|
||||
return
|
||||
}
|
||||
settled = true
|
||||
lease.recoverySink = undefined
|
||||
this.commit(relayPtyId, lease)
|
||||
},
|
||||
retire: () => {
|
||||
if (settled) {
|
||||
return
|
||||
}
|
||||
settled = true
|
||||
lease.recoverySink = undefined
|
||||
this.retire(relayPtyId, previous, lease)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
private commit(relayPtyId: string, lease: SourceDeliveryLeaseState): void {
|
||||
if (this.deliveryByPty.get(relayPtyId)?.lease !== lease) {
|
||||
lease.phase = 'retired'
|
||||
lease.pendingData.splice(0)
|
||||
return
|
||||
}
|
||||
lease.phase = 'committing'
|
||||
while (lease.pendingData.length > 0) {
|
||||
this.publishData(lease.pendingData.shift()!)
|
||||
}
|
||||
lease.phase = 'committed'
|
||||
const current = this.deliveryByPty.get(relayPtyId)
|
||||
if (lease.exited && current?.lease === lease) {
|
||||
this.deliveryByPty.delete(relayPtyId)
|
||||
return
|
||||
}
|
||||
if (current?.lease === lease && current.previous) {
|
||||
this.deliveryByPty.set(
|
||||
relayPtyId,
|
||||
Object.freeze({
|
||||
activation: current.activation,
|
||||
sourceEndSu: current.sourceEndSu,
|
||||
lease: current.lease
|
||||
})
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private retire(
|
||||
relayPtyId: string,
|
||||
previous: SourceDeliveryState | undefined,
|
||||
lease: SourceDeliveryLeaseState
|
||||
): void {
|
||||
lease.phase = 'retired'
|
||||
lease.recoverySink = undefined
|
||||
lease.pendingData.splice(0)
|
||||
if (this.deliveryByPty.get(relayPtyId)?.lease !== lease) {
|
||||
return
|
||||
}
|
||||
if (lease.exited) {
|
||||
this.deliveryByPty.delete(relayPtyId)
|
||||
return
|
||||
}
|
||||
const predecessor = activePredecessor(previous)
|
||||
if (predecessor) {
|
||||
this.deliveryByPty.set(relayPtyId, predecessor)
|
||||
} else {
|
||||
this.deliveryByPty.delete(relayPtyId)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function settledReceivingActivationLease(): SshPtySourceDeliveryLease {
|
||||
return Object.freeze({
|
||||
commit: () => {},
|
||||
rollback: async () => true,
|
||||
transferToRecovery: () => Object.freeze({ commit: () => {}, retire: () => {} })
|
||||
})
|
||||
}
|
||||
|
||||
function activePredecessor(previous?: SourceDeliveryState): SourceDeliveryState | undefined {
|
||||
while (previous?.lease.phase === 'retired') {
|
||||
previous = previous.previous
|
||||
}
|
||||
return previous
|
||||
}
|
||||
|
||||
function sameReceivingActivation(
|
||||
left: PtySourceReceivingActivation,
|
||||
right: PtySourceReceivingActivation
|
||||
): boolean {
|
||||
return (
|
||||
left.clientGeneration === right.clientGeneration &&
|
||||
left.ownerGeneration === right.ownerGeneration &&
|
||||
left.ptyIncarnation === right.ptyIncarnation &&
|
||||
left.deliveryToken === right.deliveryToken &&
|
||||
left.checkpointSourceEndSu === right.checkpointSourceEndSu &&
|
||||
left.recoveryEndSu === right.recoveryEndSu
|
||||
)
|
||||
}
|
||||
|
||||
function acceptsSourceFrame(
|
||||
current: SourceDeliveryState | undefined,
|
||||
params: Record<string, unknown>,
|
||||
source: SshPtySourceFrame
|
||||
): current is SourceDeliveryState {
|
||||
return Boolean(
|
||||
current &&
|
||||
current.lease.phase !== 'retired' &&
|
||||
!current.lease.exited &&
|
||||
current.activation.ptyIncarnation === params.ptyIncarnation &&
|
||||
current.activation.deliveryToken === source.deliveryToken &&
|
||||
current.activation.clientGeneration === source.clientGeneration &&
|
||||
current.activation.ownerGeneration === source.ownerGeneration &&
|
||||
current.sourceEndSu === source.sourceStartSu
|
||||
)
|
||||
}
|
||||
|
||||
async function settleExactSourceDeliveryCancellation(
|
||||
mux: SshChannelMultiplexer,
|
||||
relayPtyId: string,
|
||||
activation: PtySourceReceivingActivation
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
const result = (await mux.request('pty.cancelDelivery', {
|
||||
id: relayPtyId,
|
||||
clientGeneration: activation.clientGeneration,
|
||||
ownerGeneration: activation.ownerGeneration,
|
||||
deliveryToken: activation.deliveryToken
|
||||
})) as Record<string, unknown>
|
||||
return (
|
||||
result.canceled === true &&
|
||||
nonNegativeSafeInteger(result.sentEndSu) &&
|
||||
nonNegativeSafeInteger(result.creditedEndSu) &&
|
||||
result.creditedEndSu <= result.sentEndSu
|
||||
)
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
function nonNegativeSafeInteger(value: unknown): value is number {
|
||||
return Number.isSafeInteger(value) && (value as number) >= 0
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import type { SshPtyDataCallback } from './ssh-pty-provider-contract'
|
||||
|
||||
export type SshPtySourceFrame = NonNullable<Parameters<SshPtyDataCallback>[0]['source']>
|
||||
|
||||
const SOURCE_KEYS = [
|
||||
'deliveryToken',
|
||||
'clientGeneration',
|
||||
'ownerGeneration',
|
||||
'sourceEndSu',
|
||||
'sourceLengthSu',
|
||||
'ptyIncarnation'
|
||||
] as const
|
||||
|
||||
export function parseSshPtySourceFrame(
|
||||
params: Record<string, unknown>,
|
||||
data: string,
|
||||
relayPtyId: string
|
||||
): Readonly<{ source?: SshPtySourceFrame; malformed: boolean }> {
|
||||
if (!SOURCE_KEYS.some((key) => params[key] !== undefined)) {
|
||||
return Object.freeze({ malformed: false })
|
||||
}
|
||||
const sourceEndSu = params.sourceEndSu
|
||||
const sourceLengthSu = params.sourceLengthSu
|
||||
const rawLength = params.rawLength
|
||||
const transformed = params.transformed === true
|
||||
if (
|
||||
typeof params.data !== 'string' ||
|
||||
typeof params.deliveryToken !== 'string' ||
|
||||
params.deliveryToken.length === 0 ||
|
||||
!positiveSafeInteger(params.clientGeneration) ||
|
||||
!positiveSafeInteger(params.ownerGeneration) ||
|
||||
!nonNegativeSafeInteger(sourceEndSu) ||
|
||||
!nonNegativeSafeInteger(sourceLengthSu) ||
|
||||
sourceEndSu < sourceLengthSu ||
|
||||
typeof params.ptyIncarnation !== 'string' ||
|
||||
params.ptyIncarnation.length === 0 ||
|
||||
(params.seq !== undefined && !nonNegativeSafeInteger(params.seq)) ||
|
||||
(transformed ? rawLength !== sourceLengthSu : data.length !== sourceLengthSu) ||
|
||||
(rawLength !== undefined && rawLength !== sourceLengthSu)
|
||||
) {
|
||||
return Object.freeze({ malformed: true })
|
||||
}
|
||||
const sourceStartSu = sourceEndSu - sourceLengthSu
|
||||
return Object.freeze({
|
||||
malformed: false,
|
||||
source: Object.freeze({
|
||||
relayPtyId,
|
||||
spanId: `${params.deliveryToken}:${sourceStartSu}:${sourceEndSu}`,
|
||||
clientGeneration: params.clientGeneration,
|
||||
ownerGeneration: params.ownerGeneration,
|
||||
deliveryToken: params.deliveryToken,
|
||||
sourceStartSu,
|
||||
sourceEndSu
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function positiveSafeInteger(value: unknown): value is number {
|
||||
return Number.isSafeInteger(value) && (value as number) > 0
|
||||
}
|
||||
|
||||
function nonNegativeSafeInteger(value: unknown): value is number {
|
||||
return Number.isSafeInteger(value) && (value as number) >= 0
|
||||
}
|
||||
@@ -22,6 +22,9 @@ export function buildSshPtySpawnEnv(args: {
|
||||
merged.ORCA_RELAY_DIR = args.remoteCliBridgeEnv.relayDir
|
||||
merged.ORCA_RELAY_NODE_PATH = args.remoteCliBridgeEnv.nodePath
|
||||
merged.ORCA_RELAY_SOCKET_PATH = args.remoteCliBridgeEnv.sockPath
|
||||
if (args.remoteCliBridgeEnv.credentialFile) {
|
||||
merged.ORCA_RELAY_CREDENTIAL_FILE = args.remoteCliBridgeEnv.credentialFile
|
||||
}
|
||||
}
|
||||
// Why: match local/daemon precedence—managed defaults cannot restore explicitly removed values.
|
||||
for (const key of args.envToDelete ?? []) {
|
||||
|
||||
@@ -16,6 +16,13 @@ import { parseFileUriPathParts } from '../daemon/osc7-file-uri'
|
||||
import type { AgentStatus } from '../../shared/agent-detection'
|
||||
import type { TerminalOscLinkRange } from '../../shared/terminal-osc-link-ranges'
|
||||
import type { TerminalOscColorQueryReplyColors } from '../../shared/terminal-osc-color-reply'
|
||||
import type { TerminalOutputSourceRange } from '../../shared/terminal-output-source-range'
|
||||
import type {
|
||||
RemoteTerminalSourceRangeConsumerHooks,
|
||||
RemoteTerminalSourceRangeReplacementPublication,
|
||||
RemoteTerminalSourceRangeReplacementReservation,
|
||||
RemoteTerminalSourceRangeStreamIdentity
|
||||
} from './remote-terminal-source-range-consumer'
|
||||
import {
|
||||
createTerminalTitleTracker,
|
||||
stripBrailleSpinnerGlyphs,
|
||||
@@ -1431,6 +1438,19 @@ type RuntimeHeadlessTerminal = {
|
||||
writeChain: Promise<void>
|
||||
}
|
||||
|
||||
export type RuntimePtyDataAdmission = Readonly<{
|
||||
sequence: number
|
||||
completion: Promise<void>
|
||||
}>
|
||||
|
||||
export type RuntimeTerminalDataMeta = Readonly<{
|
||||
seq?: number
|
||||
rawLength?: number
|
||||
transformed?: boolean
|
||||
cwd?: string
|
||||
sourceRanges?: readonly TerminalOutputSourceRange[]
|
||||
}>
|
||||
|
||||
type RuntimeVisibleTerminalState = {
|
||||
lines: string[]
|
||||
isAlternateScreen: boolean
|
||||
@@ -2669,13 +2689,10 @@ export class OrcaRuntimeService {
|
||||
// without polling. Keyed by ptyId for O(1) lookup per data event.
|
||||
private dataListeners = new Map<
|
||||
string,
|
||||
Set<
|
||||
(
|
||||
data: string,
|
||||
meta?: { seq?: number; rawLength?: number; transformed?: boolean; cwd?: string }
|
||||
) => void
|
||||
>
|
||||
Set<(data: string, meta?: RuntimeTerminalDataMeta) => void>
|
||||
>()
|
||||
private remoteTerminalSourceRangeConsumerHooks: RemoteTerminalSourceRangeConsumerHooks | null =
|
||||
null
|
||||
// Why: startup draft paste can subscribe after the agent already emitted its
|
||||
// ready marker. Keep a bounded raw buffer so fast startup output is replayed.
|
||||
private recentPtyOutputById = new Map<string, RecentPtyOutputBuffer>()
|
||||
@@ -8616,16 +8633,49 @@ export class OrcaRuntimeService {
|
||||
}
|
||||
}
|
||||
|
||||
resetPtyModelAfterMigrationFailure(ptyId: string): void {
|
||||
this.providerSnapshotPreferredPtys.add(ptyId)
|
||||
this.disposeHeadlessTerminal(ptyId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles incoming data from a PTY process, running agent detection,
|
||||
* updating terminal tail buffers, and triggering foreground agent refreshes.
|
||||
*/
|
||||
acceptPtyDataBounded(
|
||||
ptyId: string,
|
||||
data: string,
|
||||
at: number,
|
||||
sequenceChars = data.length,
|
||||
transformed = false,
|
||||
sourceRanges?: readonly TerminalOutputSourceRange[]
|
||||
): RuntimePtyDataAdmission {
|
||||
let completion: Promise<void> | null = null
|
||||
const sequence = this.onPtyData(
|
||||
ptyId,
|
||||
data,
|
||||
at,
|
||||
sequenceChars,
|
||||
transformed,
|
||||
(receipt) => {
|
||||
completion = receipt
|
||||
},
|
||||
sourceRanges
|
||||
)
|
||||
if (!completion) {
|
||||
throw new Error('PTY model admission receipt was not captured')
|
||||
}
|
||||
return Object.freeze({ sequence, completion })
|
||||
}
|
||||
|
||||
onPtyData(
|
||||
ptyId: string,
|
||||
data: string,
|
||||
at: number,
|
||||
sequenceChars = data.length,
|
||||
transformed = false
|
||||
transformed = false,
|
||||
captureModelReceipt?: (completion: Promise<void>) => void,
|
||||
sourceRanges?: readonly TerminalOutputSourceRange[]
|
||||
): number {
|
||||
const outputSequence = (this.ptyOutputSequenceById.get(ptyId) ?? 0) + sequenceChars
|
||||
this.ptyOutputSequenceById.set(ptyId, outputSequence)
|
||||
@@ -8664,7 +8714,13 @@ export class OrcaRuntimeService {
|
||||
// applyTrackedPtyTitle) in byte order, superseding main's inline
|
||||
// extractLastOscTitleForPty block (#7880/#7852 title/status semantics are
|
||||
// preserved via the tracker + detectAgentStatusFromTitle path).
|
||||
this.trackHeadlessTerminalData(ptyId, data, outputSequence, forwardQueryReplies)
|
||||
const modelCompletion = this.trackHeadlessTerminalData(
|
||||
ptyId,
|
||||
data,
|
||||
outputSequence,
|
||||
forwardQueryReplies
|
||||
)
|
||||
captureModelReceipt?.(modelCompletion)
|
||||
|
||||
const pty = this.getOrCreatePtyWorktreeRecord(ptyId)
|
||||
const ptyTailBefore = pty
|
||||
@@ -8846,7 +8902,8 @@ export class OrcaRuntimeService {
|
||||
seq: outputSequence,
|
||||
rawLength: sequenceChars,
|
||||
...(transformed ? { transformed: true } : {}),
|
||||
...(cwdChanged && cwd !== null ? { cwd } : {})
|
||||
...(cwdChanged && cwd !== null ? { cwd } : {}),
|
||||
...(sourceRanges && sourceRanges.length > 0 ? { sourceRanges } : {})
|
||||
}
|
||||
for (const listener of listeners) {
|
||||
try {
|
||||
@@ -9721,14 +9778,71 @@ export class OrcaRuntimeService {
|
||||
|
||||
subscribeToTerminalData(
|
||||
ptyId: string,
|
||||
listener: (
|
||||
data: string,
|
||||
meta?: { seq?: number; rawLength?: number; transformed?: boolean; cwd?: string }
|
||||
) => void
|
||||
listener: (data: string, meta?: RuntimeTerminalDataMeta) => void
|
||||
): () => void {
|
||||
return addListenerToMap(this.dataListeners, ptyId, listener)
|
||||
}
|
||||
|
||||
setRemoteTerminalSourceRangeConsumerHooks(
|
||||
hooks: RemoteTerminalSourceRangeConsumerHooks | null
|
||||
): void {
|
||||
this.remoteTerminalSourceRangeConsumerHooks = hooks
|
||||
}
|
||||
|
||||
attachRemoteTerminalSourceRangeConsumer(
|
||||
identity: RemoteTerminalSourceRangeStreamIdentity
|
||||
): boolean {
|
||||
return this.remoteTerminalSourceRangeConsumerHooks?.attach(identity) ?? false
|
||||
}
|
||||
|
||||
settleRemoteTerminalSourceRanges(
|
||||
identity: RemoteTerminalSourceRangeStreamIdentity,
|
||||
ranges: readonly TerminalOutputSourceRange[]
|
||||
): void {
|
||||
this.remoteTerminalSourceRangeConsumerHooks?.settle(identity, ranges)
|
||||
}
|
||||
|
||||
reserveRemoteTerminalSourceRangeReplacement(
|
||||
identity: RemoteTerminalSourceRangeStreamIdentity,
|
||||
requiredSeq: number,
|
||||
reason: string
|
||||
): RemoteTerminalSourceRangeReplacementReservation | null {
|
||||
return (
|
||||
this.remoteTerminalSourceRangeConsumerHooks?.reserveReplacement(
|
||||
identity,
|
||||
requiredSeq,
|
||||
reason
|
||||
) ?? null
|
||||
)
|
||||
}
|
||||
|
||||
commitRemoteTerminalSourceRangeReplacement(
|
||||
reservation: RemoteTerminalSourceRangeReplacementReservation,
|
||||
publication: RemoteTerminalSourceRangeReplacementPublication
|
||||
): boolean {
|
||||
return (
|
||||
this.remoteTerminalSourceRangeConsumerHooks?.commitReplacement(reservation, publication) ??
|
||||
false
|
||||
)
|
||||
}
|
||||
|
||||
rollbackRemoteTerminalSourceRangeReplacement(
|
||||
reservation: RemoteTerminalSourceRangeReplacementReservation,
|
||||
reason: string
|
||||
): boolean {
|
||||
return (
|
||||
this.remoteTerminalSourceRangeConsumerHooks?.rollbackReplacement(reservation, reason) ?? false
|
||||
)
|
||||
}
|
||||
|
||||
cancelRemoteTerminalSourceRanges(
|
||||
identity: RemoteTerminalSourceRangeStreamIdentity,
|
||||
ranges: readonly TerminalOutputSourceRange[],
|
||||
reason: string
|
||||
): void {
|
||||
this.remoteTerminalSourceRangeConsumerHooks?.cancel(identity, ranges, reason)
|
||||
}
|
||||
|
||||
/** Set by pty IPC: fires when a PTY gains/loses remote view subscribers so
|
||||
* the daemon background mark (keep-tail stream thinning) can resync — a
|
||||
* live mobile/web view consumes raw bytes and must never be thinned, even
|
||||
@@ -10186,19 +10300,17 @@ export class OrcaRuntimeService {
|
||||
data: string,
|
||||
outputSequence: number,
|
||||
forwardQueryReplies = false
|
||||
): void {
|
||||
): Promise<void> {
|
||||
const state = this.getOrCreateHeadlessTerminal(ptyId)
|
||||
state.writeChain = state.writeChain
|
||||
.then(async () => {
|
||||
// Why: the ingestion-time ownership decision is closed over this
|
||||
// chain link; async scheduling cannot retroactively change it.
|
||||
await state.emulator.write(data, { forwardQueryReplies })
|
||||
state.outputSequence = outputSequence
|
||||
})
|
||||
.catch(() => {
|
||||
// Best-effort state tracking; live streaming must continue even if
|
||||
// xterm rejects a malformed or raced write during shutdown.
|
||||
})
|
||||
const completion = state.writeChain.then(async () => {
|
||||
// Why: the ingestion-time ownership decision is closed over this
|
||||
// chain link; async scheduling cannot retroactively change it.
|
||||
await state.emulator.write(data, { forwardQueryReplies })
|
||||
state.outputSequence = outputSequence
|
||||
})
|
||||
// Legacy callers remain best-effort; bounded SSH admission observes the raw receipt.
|
||||
state.writeChain = completion.catch(() => {})
|
||||
return completion
|
||||
}
|
||||
|
||||
/** Shared factory for the per-PTY runtime emulators (seed, hydration, and
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
import type { TerminalOutputSourceRange } from '../../shared/terminal-output-source-range'
|
||||
|
||||
export type RemoteTerminalSourceRangeStreamIdentity = Readonly<{
|
||||
ptyId: string
|
||||
consumerId: string
|
||||
streamGeneration: string
|
||||
}>
|
||||
|
||||
export type RemoteTerminalSourceRangeReplacementReservation = Readonly<{
|
||||
reservationId: string
|
||||
identity: RemoteTerminalSourceRangeStreamIdentity
|
||||
requiredSeq: number
|
||||
}>
|
||||
|
||||
export type RemoteTerminalSourceRangeReplacementPublication = Readonly<{
|
||||
source: 'headless' | 'renderer'
|
||||
seq: number
|
||||
}>
|
||||
|
||||
export type RemoteTerminalSourceRangeConsumerHooks = {
|
||||
attach: (identity: RemoteTerminalSourceRangeStreamIdentity) => boolean
|
||||
settle: (
|
||||
identity: RemoteTerminalSourceRangeStreamIdentity,
|
||||
ranges: readonly TerminalOutputSourceRange[]
|
||||
) => void
|
||||
reserveReplacement: (
|
||||
identity: RemoteTerminalSourceRangeStreamIdentity,
|
||||
requiredSeq: number,
|
||||
reason: string
|
||||
) => RemoteTerminalSourceRangeReplacementReservation | null
|
||||
commitReplacement: (
|
||||
reservation: RemoteTerminalSourceRangeReplacementReservation,
|
||||
publication: RemoteTerminalSourceRangeReplacementPublication
|
||||
) => boolean
|
||||
rollbackReplacement: (
|
||||
reservation: RemoteTerminalSourceRangeReplacementReservation,
|
||||
reason: string
|
||||
) => boolean
|
||||
cancel: (
|
||||
identity: RemoteTerminalSourceRangeStreamIdentity,
|
||||
ranges: readonly TerminalOutputSourceRange[],
|
||||
reason: string
|
||||
) => void
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
/* oxlint-disable max-lines -- Why: terminal RPC methods are co-located for discoverability; splitting would scatter related handlers across files. */
|
||||
import { z } from 'zod'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import {
|
||||
InvalidArgumentError,
|
||||
defineMethod,
|
||||
@@ -19,6 +20,7 @@ import {
|
||||
} from '../../../../shared/terminal-stream-protocol'
|
||||
import {
|
||||
iterateTerminalOutputFrameChunks,
|
||||
sliceTerminalOutputSourceRanges,
|
||||
type TerminalOutputFrameChunk,
|
||||
type TerminalOutputMeta
|
||||
} from '../terminal-output-frame-chunks'
|
||||
@@ -61,6 +63,13 @@ import {
|
||||
TERMINAL_OUTPUT_BATCH_MAX_BYTES
|
||||
} from '../../../../shared/terminal-multiplex-flow-control'
|
||||
import { drainTerminalMultiplexRoundRobin } from '../terminal-multiplex-round-robin'
|
||||
import type { TerminalSourceRangeLedger } from '../terminal-source-range-ledger'
|
||||
import { TerminalSourceRangeRegistry } from '../terminal-source-range-registry'
|
||||
import {
|
||||
sameTerminalOutputSourceIdentity,
|
||||
type TerminalOutputSourceRange
|
||||
} from '../../../../shared/terminal-output-source-range'
|
||||
import type { RemoteTerminalSourceRangeReplacementReservation } from '../../remote-terminal-source-range-consumer'
|
||||
|
||||
const REQUESTED_SNAPSHOT_BYTE_BUDGET = 2 * 1024 * 1024
|
||||
const TERMINAL_OUTPUT_FLUSH_MS = 5
|
||||
@@ -112,6 +121,11 @@ type TerminalMultiplexStream = {
|
||||
client: TerminalViewportClient | undefined
|
||||
isMobile: boolean
|
||||
ackOutput: boolean
|
||||
ackOutputSourceRanges: boolean
|
||||
streamGeneration: string
|
||||
sourceRangeLedger: TerminalSourceRangeLedger | null
|
||||
sourceRangeConsumerAttached: boolean
|
||||
sourceRangeReplacement: RemoteTerminalSourceRangeReplacementReservation | null
|
||||
ackInFlightBytes: number
|
||||
ackWindowBytes: number
|
||||
supportsDesktopViewportClaims: boolean
|
||||
@@ -157,6 +171,7 @@ function createTerminalOutputBatcher(onFlush: (data: string, meta?: TerminalOutp
|
||||
let lastSeq: number | undefined
|
||||
let pendingCwd: string | undefined
|
||||
let pendingRawLength = 0
|
||||
let pendingSourceRanges: TerminalOutputSourceRange[] = []
|
||||
let timer: ReturnType<typeof setTimeout> | null = null
|
||||
|
||||
const clearTimer = (): void => {
|
||||
@@ -174,10 +189,13 @@ function createTerminalOutputBatcher(onFlush: (data: string, meta?: TerminalOutp
|
||||
}
|
||||
const data = chunks.length === 1 ? chunks[0]! : chunks.join('')
|
||||
const meta =
|
||||
typeof lastSeq === 'number' || pendingCwd !== undefined
|
||||
typeof lastSeq === 'number' || pendingCwd !== undefined || pendingSourceRanges.length > 0
|
||||
? {
|
||||
...(typeof lastSeq === 'number' ? { seq: lastSeq, rawLength: pendingRawLength } : {}),
|
||||
...(pendingCwd !== undefined ? { cwd: pendingCwd } : {})
|
||||
...(pendingCwd !== undefined ? { cwd: pendingCwd } : {}),
|
||||
...(pendingSourceRanges.length > 0
|
||||
? { sourceRanges: Object.freeze(pendingSourceRanges.slice()) }
|
||||
: {})
|
||||
}
|
||||
: undefined
|
||||
chunks = []
|
||||
@@ -185,6 +203,7 @@ function createTerminalOutputBatcher(onFlush: (data: string, meta?: TerminalOutp
|
||||
lastSeq = undefined
|
||||
pendingCwd = undefined
|
||||
pendingRawLength = 0
|
||||
pendingSourceRanges = []
|
||||
onFlush(data, meta)
|
||||
}
|
||||
|
||||
@@ -199,12 +218,26 @@ function createTerminalOutputBatcher(onFlush: (data: string, meta?: TerminalOutp
|
||||
onFlush(data, { ...meta, rawLength, transformed: true })
|
||||
return
|
||||
}
|
||||
const nextSourceRanges = meta?.sourceRanges ?? []
|
||||
const lastSourceRange = pendingSourceRanges.at(-1)
|
||||
const firstNextSourceRange = nextSourceRanges[0]
|
||||
if (
|
||||
chunks.length > 0 &&
|
||||
(pendingSourceRanges.length > 0 !== nextSourceRanges.length > 0 ||
|
||||
(lastSourceRange &&
|
||||
firstNextSourceRange &&
|
||||
(!sameTerminalOutputSourceIdentity(lastSourceRange, firstNextSourceRange) ||
|
||||
lastSourceRange.displayEnd !== firstNextSourceRange.displayStart)))
|
||||
) {
|
||||
flush()
|
||||
}
|
||||
if (meta?.cwd !== undefined) {
|
||||
flush()
|
||||
pendingCwd = meta.cwd
|
||||
}
|
||||
chunks.push(data)
|
||||
pendingRawLength += rawLength
|
||||
pendingSourceRanges.push(...nextSourceRanges)
|
||||
const remainingBudget = Math.max(1, TERMINAL_OUTPUT_BATCH_MAX_BYTES - bytes)
|
||||
const measurement = measureTerminalStreamByteLength(data, {
|
||||
stopAfterBytes: remainingBudget
|
||||
@@ -231,6 +264,7 @@ function createTerminalOutputBatcher(onFlush: (data: string, meta?: TerminalOutp
|
||||
chunks = []
|
||||
bytes = 0
|
||||
pendingRawLength = 0
|
||||
pendingSourceRanges = []
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -380,22 +414,38 @@ function appendPendingMultiplexOutput(
|
||||
function getOutputAfterSnapshotSeq(
|
||||
chunk: TerminalOutputChunk,
|
||||
snapshotSeq: number | undefined
|
||||
): string | null {
|
||||
): TerminalOutputChunk | null {
|
||||
if (
|
||||
typeof snapshotSeq !== 'number' ||
|
||||
typeof chunk.meta?.seq !== 'number' ||
|
||||
typeof chunk.meta.rawLength !== 'number'
|
||||
) {
|
||||
return chunk.data
|
||||
return chunk
|
||||
}
|
||||
if (chunk.meta.seq <= snapshotSeq) {
|
||||
return null
|
||||
}
|
||||
const chunkStartSeq = chunk.meta.seq - chunk.meta.rawLength
|
||||
if (chunkStartSeq >= snapshotSeq) {
|
||||
return chunk.data
|
||||
return chunk
|
||||
}
|
||||
if (chunk.meta.transformed) {
|
||||
return null
|
||||
}
|
||||
const offset = snapshotSeq - chunkStartSeq
|
||||
return {
|
||||
data: chunk.data.slice(offset),
|
||||
bytes: chunk.bytes,
|
||||
meta: {
|
||||
...chunk.meta,
|
||||
rawLength: chunk.meta.rawLength - offset,
|
||||
sourceRanges: sliceTerminalOutputSourceRanges(
|
||||
chunk.meta.sourceRanges,
|
||||
offset,
|
||||
chunk.data.length
|
||||
)
|
||||
}
|
||||
}
|
||||
return chunk.data.slice(snapshotSeq - chunkStartSeq)
|
||||
}
|
||||
|
||||
function stripSnapshotBoundaryQuerySuffixes(
|
||||
@@ -551,36 +601,45 @@ async function serializeBudgetedRequestedSnapshot(
|
||||
}
|
||||
|
||||
function sendSnapshotFrames(
|
||||
sendFrame: (opcode: TerminalStreamOpcode, payload?: Uint8Array<ArrayBufferLike>) => void,
|
||||
sendFrame: (
|
||||
opcode: TerminalStreamOpcode,
|
||||
payload?: Uint8Array<ArrayBufferLike>
|
||||
) => boolean | void,
|
||||
options: SnapshotFrameOptions
|
||||
): { bytes: number; chunks: number } {
|
||||
sendFrame(
|
||||
TerminalStreamOpcode.SnapshotStart,
|
||||
encodeTerminalStreamJson({
|
||||
kind: options.kind,
|
||||
cols: options.cols,
|
||||
rows: options.rows,
|
||||
requestId: options.requestId,
|
||||
displayMode: options.displayMode,
|
||||
reason: options.reason,
|
||||
seq: options.seq,
|
||||
cwd: options.cwd,
|
||||
source: options.source,
|
||||
oscLinks: options.oscLinks,
|
||||
pendingEscapeTailAnsi: options.pendingEscapeTailAnsi,
|
||||
truncated: options.truncated === true,
|
||||
truncatedByByteBudget: options.truncatedByByteBudget === true
|
||||
})
|
||||
)
|
||||
): { bytes: number; chunks: number; published: boolean } {
|
||||
if (
|
||||
sendFrame(
|
||||
TerminalStreamOpcode.SnapshotStart,
|
||||
encodeTerminalStreamJson({
|
||||
kind: options.kind,
|
||||
cols: options.cols,
|
||||
rows: options.rows,
|
||||
requestId: options.requestId,
|
||||
displayMode: options.displayMode,
|
||||
reason: options.reason,
|
||||
seq: options.seq,
|
||||
cwd: options.cwd,
|
||||
source: options.source,
|
||||
oscLinks: options.oscLinks,
|
||||
pendingEscapeTailAnsi: options.pendingEscapeTailAnsi,
|
||||
truncated: options.truncated === true,
|
||||
truncatedByByteBudget: options.truncatedByByteBudget === true
|
||||
})
|
||||
) === false
|
||||
) {
|
||||
return { bytes: 0, chunks: 0, published: false }
|
||||
}
|
||||
let chunks = 0
|
||||
let bytes = 0
|
||||
for (const chunk of iterateTerminalStreamTextPayloads(options.data)) {
|
||||
if (sendFrame(TerminalStreamOpcode.SnapshotChunk, chunk) === false) {
|
||||
return { bytes, chunks, published: false }
|
||||
}
|
||||
chunks++
|
||||
bytes += chunk.byteLength
|
||||
sendFrame(TerminalStreamOpcode.SnapshotChunk, chunk)
|
||||
}
|
||||
sendFrame(TerminalStreamOpcode.SnapshotEnd)
|
||||
return { bytes, chunks }
|
||||
const published = sendFrame(TerminalStreamOpcode.SnapshotEnd) !== false
|
||||
return { bytes, chunks, published }
|
||||
}
|
||||
|
||||
async function serializeBudgetedMobileSnapshot(
|
||||
@@ -953,14 +1012,24 @@ const TerminalMultiplexSubscribeFrame = TerminalHandle.extend({
|
||||
capabilities: z
|
||||
.object({
|
||||
ackOutput: z.literal(1).optional(),
|
||||
ackOutputSourceRanges: z.literal(1).optional(),
|
||||
desktopViewportClaims: z.literal(1).optional()
|
||||
})
|
||||
.optional()
|
||||
})
|
||||
|
||||
const TerminalMultiplexAckFrame = z.object({
|
||||
bytes: z.number().int().nonnegative()
|
||||
})
|
||||
const TerminalMultiplexLegacyAckFrame = z
|
||||
.object({
|
||||
bytes: z.number().int().nonnegative()
|
||||
})
|
||||
.strict()
|
||||
|
||||
const TerminalMultiplexSourceRangeAckFrame = z
|
||||
.object({
|
||||
streamGeneration: z.string().min(1),
|
||||
ackedEndByte: z.number().int().nonnegative()
|
||||
})
|
||||
.strict()
|
||||
|
||||
const TerminalMultiplexSnapshotRequestFrame = z.object({
|
||||
requestId: z.number().int().positive().optional(),
|
||||
@@ -1514,6 +1583,7 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
|
||||
let closed = false
|
||||
let cursor = 0
|
||||
const streams = new Map<number, TerminalMultiplexStream>()
|
||||
const sourceRangeRegistry = new TerminalSourceRangeRegistry()
|
||||
const pendingPtyWaitControllers = new Map<number, Set<AbortController>>()
|
||||
let ackTotalInFlightBytes = 0
|
||||
let ackTotalWindowBytes = TERMINAL_MULTIPLEX_ACK_TOTAL_INITIAL_WINDOW_BYTES
|
||||
@@ -1526,9 +1596,11 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
|
||||
streamId: number,
|
||||
opcode: TerminalStreamOpcode,
|
||||
payload: Uint8Array<ArrayBufferLike> = new Uint8Array(),
|
||||
seq?: number
|
||||
seq?: number,
|
||||
onRejected?: () => void
|
||||
): boolean => {
|
||||
if (closed) {
|
||||
onRejected?.()
|
||||
return false
|
||||
}
|
||||
// Why: a seq-less Output chunk must carry sentinel 0, not the control-frame cursor, or it poisons the client's frame-drop tracker.
|
||||
@@ -1540,10 +1612,12 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
|
||||
encodeTerminalStreamFrame({ opcode, streamId, seq: resolvedSeq, payload })
|
||||
)
|
||||
} catch {
|
||||
onRejected?.()
|
||||
closeMultiplex()
|
||||
return false
|
||||
}
|
||||
if (sent === false) {
|
||||
onRejected?.()
|
||||
// Why: false means the transport discarded this frame; reconnect is the only available retry boundary with an authoritative snapshot.
|
||||
closeMultiplex()
|
||||
return false
|
||||
@@ -1577,22 +1651,43 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
|
||||
}
|
||||
return (
|
||||
stream.ackInFlightBytes + bytes <= stream.ackWindowBytes &&
|
||||
ackTotalInFlightBytes + bytes <= ackTotalWindowBytes
|
||||
ackTotalInFlightBytes + bytes <= ackTotalWindowBytes &&
|
||||
(!stream.ackOutputSourceRanges || stream.sourceRangeLedger?.canAccept(bytes) === true)
|
||||
)
|
||||
}
|
||||
const sendAckGatedOutput = (
|
||||
stream: TerminalMultiplexStream,
|
||||
chunk: TerminalOutputFrameChunk
|
||||
): boolean => {
|
||||
const prepared = stream.ackOutputSourceRanges
|
||||
? stream.sourceRangeLedger?.prepareAccept(
|
||||
chunk.bytes.byteLength,
|
||||
chunk.displayLength,
|
||||
chunk.sourceRanges ?? [],
|
||||
chunk.seq
|
||||
)
|
||||
: undefined
|
||||
if (stream.ackOutputSourceRanges && prepared?.status !== 'ready') {
|
||||
if (prepared?.status !== 'capacity') {
|
||||
detachStream(stream.streamId, true)
|
||||
}
|
||||
return false
|
||||
}
|
||||
const admission = prepared?.status === 'ready' ? prepared.admission : undefined
|
||||
const sent = sendFrame(
|
||||
stream.streamId,
|
||||
chunk.opcode ?? TerminalStreamOpcode.Output,
|
||||
chunk.bytes,
|
||||
chunk.seq
|
||||
chunk.seq,
|
||||
admission?.rollback
|
||||
)
|
||||
if (!sent) {
|
||||
return false
|
||||
}
|
||||
if (admission && !admission.commit()) {
|
||||
detachStream(stream.streamId, true)
|
||||
return false
|
||||
}
|
||||
if (stream.ackOutput) {
|
||||
stream.ackInFlightBytes += chunk.bytes.byteLength
|
||||
ackTotalInFlightBytes += chunk.bytes.byteLength
|
||||
@@ -1625,6 +1720,7 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
|
||||
return
|
||||
}
|
||||
stream.ackRecoverySnapshotInFlight = true
|
||||
let replacement: RemoteTerminalSourceRangeReplacementReservation | null = null
|
||||
try {
|
||||
const serialized = await serializeBudgetedRequestedSnapshot(runtime, stream.ptyId, 0)
|
||||
if (closed || streams.get(stream.streamId) !== stream) {
|
||||
@@ -1633,21 +1729,75 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
|
||||
if (!serialized) {
|
||||
throw new Error('Remote terminal recovery snapshot unavailable.')
|
||||
}
|
||||
if (
|
||||
stream.ackOutputSourceRanges &&
|
||||
(serialized.source === undefined || typeof serialized.seq !== 'number')
|
||||
) {
|
||||
throw new Error('Remote terminal recovery snapshot source identity unavailable.')
|
||||
}
|
||||
if (
|
||||
stream.ackOutputSourceRanges &&
|
||||
serialized.source !== undefined &&
|
||||
typeof serialized.seq === 'number'
|
||||
) {
|
||||
replacement = runtime.reserveRemoteTerminalSourceRangeReplacement(
|
||||
{
|
||||
ptyId: stream.ptyId,
|
||||
consumerId: stream.remoteDesktopSubscriptionKey,
|
||||
streamGeneration: stream.streamGeneration
|
||||
},
|
||||
serialized.seq,
|
||||
'ack-pending-overflow'
|
||||
)
|
||||
stream.sourceRangeReplacement = replacement
|
||||
}
|
||||
const displayMode = runtime.getMobileDisplayMode(stream.ptyId)
|
||||
// Why: dropped ACK-pending output breaks live replay; send a fresh snapshot before resuming output.
|
||||
sendSnapshotFrames((opcode, payload) => sendFrame(stream.streamId, opcode, payload), {
|
||||
kind: 'scrollback',
|
||||
cols: serialized.cols,
|
||||
rows: serialized.rows,
|
||||
displayMode,
|
||||
reason: 'ack-pending-overflow',
|
||||
seq: serialized.seq,
|
||||
source: serialized.source,
|
||||
truncatedByByteBudget: serialized.truncatedByByteBudget,
|
||||
data: serialized.data
|
||||
})
|
||||
const publication = sendSnapshotFrames(
|
||||
(opcode, payload) =>
|
||||
!closed &&
|
||||
streams.get(stream.streamId) === stream &&
|
||||
sendFrame(stream.streamId, opcode, payload),
|
||||
{
|
||||
kind: 'scrollback',
|
||||
cols: serialized.cols,
|
||||
rows: serialized.rows,
|
||||
displayMode,
|
||||
reason: 'ack-pending-overflow',
|
||||
seq: serialized.seq,
|
||||
source: serialized.source,
|
||||
truncatedByByteBudget: serialized.truncatedByByteBudget,
|
||||
data: serialized.data
|
||||
}
|
||||
)
|
||||
if (!publication.published) {
|
||||
throw new Error('Remote terminal recovery snapshot was not published.')
|
||||
}
|
||||
if (closed || streams.get(stream.streamId) !== stream) {
|
||||
throw new Error('Remote terminal recovery snapshot stream detached.')
|
||||
}
|
||||
const localReplacement = replacement
|
||||
? typeof serialized.seq === 'number'
|
||||
? stream.sourceRangeLedger?.planSourceRangeReplacement(serialized.seq)
|
||||
: null
|
||||
: null
|
||||
if (replacement && !localReplacement) {
|
||||
throw new Error('Remote terminal recovery source ledger replacement unavailable.')
|
||||
}
|
||||
if (
|
||||
replacement &&
|
||||
(!serialized.source ||
|
||||
typeof serialized.seq !== 'number' ||
|
||||
!runtime.commitRemoteTerminalSourceRangeReplacement(replacement, {
|
||||
source: serialized.source,
|
||||
seq: serialized.seq
|
||||
}))
|
||||
) {
|
||||
throw new Error('Remote terminal recovery snapshot replacement was not accepted.')
|
||||
}
|
||||
localReplacement?.commit()
|
||||
stream.sourceRangeReplacement = null
|
||||
replacement = null
|
||||
if (typeof serialized.seq === 'number') {
|
||||
// Why: chunks queued before the snapshot serialized are already in it; replaying them would duplicate output.
|
||||
const snapshotSeq = serialized.seq
|
||||
const retained = stream.ackPendingOutput.filter(
|
||||
(chunk) => !(typeof chunk.seq === 'number' && chunk.seq <= snapshotSeq)
|
||||
@@ -1660,11 +1810,23 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
|
||||
}
|
||||
stream.ackPendingOutputOverflowed = false
|
||||
} catch (error) {
|
||||
if (replacement) {
|
||||
if (stream.sourceRangeReplacement === replacement) {
|
||||
stream.sourceRangeReplacement = null
|
||||
runtime.rollbackRemoteTerminalSourceRangeReplacement(
|
||||
replacement,
|
||||
'ack-pending-overflow-unpublished'
|
||||
)
|
||||
}
|
||||
replacement = null
|
||||
}
|
||||
if (closed || streams.get(stream.streamId) !== stream) {
|
||||
return
|
||||
}
|
||||
sendStreamError(
|
||||
stream.streamId,
|
||||
error instanceof Error ? error.message : 'Remote terminal recovery snapshot failed.'
|
||||
)
|
||||
// Why: retrying the same failed recovery from finally creates an unbounded error loop.
|
||||
detachStream(stream.streamId, true)
|
||||
} finally {
|
||||
if (streams.get(stream.streamId) === stream) {
|
||||
@@ -1735,6 +1897,56 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
|
||||
ackTotalInFlightBytes = Math.max(0, ackTotalInFlightBytes - acknowledged)
|
||||
flushAllAckPendingOutput()
|
||||
}
|
||||
const acknowledgeSourceRanges = (
|
||||
stream: TerminalMultiplexStream,
|
||||
streamGeneration: string,
|
||||
ackedEndByte: number
|
||||
): void => {
|
||||
if (!stream.ackOutputSourceRanges) {
|
||||
return
|
||||
}
|
||||
const result = stream.sourceRangeLedger?.acknowledge(streamGeneration, ackedEndByte)
|
||||
if (!result) {
|
||||
return
|
||||
}
|
||||
if (result.status !== 'accepted') {
|
||||
return
|
||||
}
|
||||
if (result.settled.length > 0) {
|
||||
runtime.settleRemoteTerminalSourceRanges(
|
||||
{
|
||||
ptyId: stream.ptyId,
|
||||
consumerId: stream.remoteDesktopSubscriptionKey,
|
||||
streamGeneration: stream.streamGeneration
|
||||
},
|
||||
result.settled
|
||||
)
|
||||
}
|
||||
acknowledgeOutput(stream, result.acknowledgedBytes)
|
||||
}
|
||||
const detachSourceRangeConsumer = (stream: TerminalMultiplexStream, reason: string): void => {
|
||||
if (!stream.sourceRangeConsumerAttached) {
|
||||
return
|
||||
}
|
||||
stream.sourceRangeConsumerAttached = false
|
||||
const ledger = stream.sourceRangeLedger
|
||||
stream.sourceRangeLedger = null
|
||||
if (!ledger) {
|
||||
return
|
||||
}
|
||||
const identity = {
|
||||
ptyId: stream.ptyId,
|
||||
consumerId: stream.remoteDesktopSubscriptionKey,
|
||||
streamGeneration: stream.streamGeneration
|
||||
}
|
||||
const transfer = ledger.beginTransfer()
|
||||
const ranges = transfer.frames.flatMap((frame) => frame.sourceRanges)
|
||||
try {
|
||||
runtime.cancelRemoteTerminalSourceRanges(identity, ranges, reason)
|
||||
} finally {
|
||||
transfer.commit()
|
||||
}
|
||||
}
|
||||
const detachStream = (
|
||||
streamId: number,
|
||||
emitEnd: boolean,
|
||||
@@ -1744,8 +1956,17 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
|
||||
if (!stream) {
|
||||
return
|
||||
}
|
||||
const replacement = stream.sourceRangeReplacement
|
||||
stream.sourceRangeReplacement = null
|
||||
if (replacement) {
|
||||
runtime.rollbackRemoteTerminalSourceRangeReplacement(
|
||||
replacement,
|
||||
'stream-detached-replacement-aborted'
|
||||
)
|
||||
}
|
||||
stream.outputBatcher.flush()
|
||||
stream.outputBatcher.dispose()
|
||||
detachSourceRangeConsumer(stream, 'stream-detached')
|
||||
ackTotalInFlightBytes = Math.max(0, ackTotalInFlightBytes - stream.ackInFlightBytes)
|
||||
stream.ackInFlightBytes = 0
|
||||
stream.ackPendingOutput = []
|
||||
@@ -1827,11 +2048,21 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
|
||||
return
|
||||
}
|
||||
if (frame.opcode === TerminalStreamOpcode.Ack) {
|
||||
const parsed = TerminalMultiplexAckFrame.safeParse(
|
||||
decodeTerminalStreamJson<unknown>(frame.payload) ?? {}
|
||||
)
|
||||
if (parsed.success) {
|
||||
acknowledgeOutput(stream, parsed.data.bytes)
|
||||
const payload = decodeTerminalStreamJson<unknown>(frame.payload) ?? {}
|
||||
if (stream.ackOutputSourceRanges) {
|
||||
const parsed = TerminalMultiplexSourceRangeAckFrame.safeParse(payload)
|
||||
if (parsed.success) {
|
||||
acknowledgeSourceRanges(
|
||||
stream,
|
||||
parsed.data.streamGeneration,
|
||||
parsed.data.ackedEndByte
|
||||
)
|
||||
}
|
||||
} else {
|
||||
const parsed = TerminalMultiplexLegacyAckFrame.safeParse(payload)
|
||||
if (parsed.success) {
|
||||
acknowledgeOutput(stream, parsed.data.bytes)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -2024,12 +2255,12 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
|
||||
// high-water, so covered bytes would render twice; tagged
|
||||
// snapshots feed a side consumer and the live view still
|
||||
// needs every buffered chunk.
|
||||
const uncoveredData =
|
||||
const uncovered =
|
||||
typeof requestId === 'number'
|
||||
? chunk.data
|
||||
? chunk
|
||||
: getOutputAfterSnapshotSeq(chunk, sentSnapshotOutputSeq)
|
||||
if (uncoveredData) {
|
||||
stream.outputBatcher.push(uncoveredData, chunk.meta)
|
||||
if (uncovered) {
|
||||
stream.outputBatcher.push(uncovered.data, uncovered.meta)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2140,6 +2371,23 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
|
||||
detachStream(request.streamId, false)
|
||||
|
||||
const ptyId = leaf.ptyId
|
||||
const remoteDesktopSubscriptionKey = `multiplex:${connectionId}:${request.streamId}`
|
||||
const streamGeneration = randomUUID()
|
||||
const requestedSourceRangeConsumer =
|
||||
request.capabilities?.ackOutput === 1 && request.capabilities?.ackOutputSourceRanges === 1
|
||||
const sourceRangeLedger = requestedSourceRangeConsumer
|
||||
? sourceRangeRegistry.open(streamGeneration)
|
||||
: null
|
||||
const sourceRangeConsumerAttached =
|
||||
sourceRangeLedger !== null &&
|
||||
runtime.attachRemoteTerminalSourceRangeConsumer({
|
||||
ptyId,
|
||||
consumerId: remoteDesktopSubscriptionKey,
|
||||
streamGeneration
|
||||
})
|
||||
if (!sourceRangeConsumerAttached) {
|
||||
sourceRangeLedger?.close()
|
||||
}
|
||||
const stream: TerminalMultiplexStream = {
|
||||
streamId: request.streamId,
|
||||
terminal: request.terminal,
|
||||
@@ -2147,13 +2395,18 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
|
||||
client: request.client,
|
||||
isMobile,
|
||||
ackOutput: request.capabilities?.ackOutput === 1,
|
||||
ackOutputSourceRanges: sourceRangeConsumerAttached,
|
||||
streamGeneration,
|
||||
sourceRangeLedger: sourceRangeConsumerAttached ? sourceRangeLedger : null,
|
||||
sourceRangeConsumerAttached,
|
||||
sourceRangeReplacement: null,
|
||||
ackInFlightBytes: 0,
|
||||
ackWindowBytes: TERMINAL_MULTIPLEX_ACK_STREAM_INITIAL_WINDOW_BYTES,
|
||||
supportsDesktopViewportClaims: request.capabilities?.desktopViewportClaims === 1,
|
||||
desktopClaimTail: Promise.resolve(true),
|
||||
registeredRemoteDesktopDriver: false,
|
||||
// Why: streamId is client-local, so key the width floor by connectionId or two connections sharing stream 1 for one PTY clobber each other's floor.
|
||||
remoteDesktopSubscriptionKey: `multiplex:${connectionId}:${request.streamId}`,
|
||||
remoteDesktopSubscriptionKey,
|
||||
pendingRemoteDesktopViewport: null,
|
||||
buffering: true,
|
||||
ackPendingOutput: [],
|
||||
@@ -2273,35 +2526,77 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
|
||||
rows: serialized?.rows ?? size?.rows,
|
||||
displayMode,
|
||||
seq: layoutSeq,
|
||||
...(stream.ackOutputSourceRanges
|
||||
? {
|
||||
streamGeneration: stream.streamGeneration,
|
||||
capabilities: { ackOutputSourceRanges: 1 as const }
|
||||
}
|
||||
: {}),
|
||||
truncated:
|
||||
initialOutputOverflowed ||
|
||||
(serialized ? read.truncated : isTerminalReadPayloadIncomplete(read))
|
||||
})
|
||||
sendSnapshotFrames((opcode, payload) => sendFrame(request.streamId, opcode, payload), {
|
||||
kind: 'scrollback',
|
||||
cols: serialized?.cols ?? size?.cols ?? 80,
|
||||
rows: serialized?.rows ?? size?.rows ?? 24,
|
||||
displayMode,
|
||||
seq: snapshotFrameSeq,
|
||||
cwd: serialized?.cwd,
|
||||
truncated:
|
||||
initialOutputOverflowed ||
|
||||
(serialized ? read.truncated : isTerminalReadPayloadIncomplete(read)),
|
||||
truncatedByByteBudget: serialized?.truncatedByByteBudget,
|
||||
source: serialized?.source,
|
||||
oscLinks: serialized?.oscLinks,
|
||||
pendingEscapeTailAnsi: serialized?.pendingEscapeTailAnsi,
|
||||
data: serialized?.data ?? (read.tail.length > 0 ? `${read.tail.join('\r\n')}\r\n` : '')
|
||||
})
|
||||
stream.sourceRangeReplacement =
|
||||
stream.ackOutputSourceRanges &&
|
||||
serialized?.source !== undefined &&
|
||||
typeof serialized.seq === 'number'
|
||||
? runtime.reserveRemoteTerminalSourceRangeReplacement(
|
||||
{
|
||||
ptyId,
|
||||
consumerId: stream.remoteDesktopSubscriptionKey,
|
||||
streamGeneration: stream.streamGeneration
|
||||
},
|
||||
serialized.seq,
|
||||
'initial-snapshot'
|
||||
)
|
||||
: null
|
||||
const snapshotPublication = sendSnapshotFrames(
|
||||
(opcode, payload) => sendFrame(request.streamId, opcode, payload),
|
||||
{
|
||||
kind: 'scrollback',
|
||||
cols: serialized?.cols ?? size?.cols ?? 80,
|
||||
rows: serialized?.rows ?? size?.rows ?? 24,
|
||||
displayMode,
|
||||
seq: snapshotFrameSeq,
|
||||
cwd: serialized?.cwd,
|
||||
truncated:
|
||||
initialOutputOverflowed ||
|
||||
(serialized ? read.truncated : isTerminalReadPayloadIncomplete(read)),
|
||||
truncatedByByteBudget: serialized?.truncatedByByteBudget,
|
||||
source: serialized?.source,
|
||||
oscLinks: serialized?.oscLinks,
|
||||
pendingEscapeTailAnsi: serialized?.pendingEscapeTailAnsi,
|
||||
data:
|
||||
serialized?.data ?? (read.tail.length > 0 ? `${read.tail.join('\r\n')}\r\n` : '')
|
||||
}
|
||||
)
|
||||
const replacement = stream.sourceRangeReplacement
|
||||
stream.sourceRangeReplacement = null
|
||||
if (replacement) {
|
||||
const committed =
|
||||
snapshotPublication.published &&
|
||||
serialized?.source !== undefined &&
|
||||
typeof serialized.seq === 'number' &&
|
||||
runtime.commitRemoteTerminalSourceRangeReplacement(replacement, {
|
||||
source: serialized.source,
|
||||
seq: serialized.seq
|
||||
})
|
||||
if (!committed) {
|
||||
runtime.rollbackRemoteTerminalSourceRangeReplacement(
|
||||
replacement,
|
||||
'initial-snapshot-unpublished'
|
||||
)
|
||||
}
|
||||
}
|
||||
// Why: baseline for resize re-stream gating; the client already rewrapped to these cols via the initial snapshot replay.
|
||||
stream.lastResizeCols = serialized?.cols ?? size?.cols
|
||||
stream.buffering = false
|
||||
const pendingOutput = stream.pendingOutput.splice(0)
|
||||
if (!initialOutputOverflowed) {
|
||||
for (const chunk of pendingOutput) {
|
||||
const uncoveredData = getOutputAfterSnapshotSeq(chunk, snapshotOutputSeq)
|
||||
if (uncoveredData) {
|
||||
stream.outputBatcher.push(uncoveredData, chunk.meta)
|
||||
const uncovered = getOutputAfterSnapshotSeq(chunk, snapshotOutputSeq)
|
||||
if (uncovered) {
|
||||
stream.outputBatcher.push(uncovered.data, uncovered.meta)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2982,9 +3277,9 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
|
||||
if (stableRendererSnapshot?.data.length) {
|
||||
serialized = stableRendererSnapshot
|
||||
const trailingOutput = pendingOutput.flatMap((item) => {
|
||||
const data = getOutputAfterSnapshotSeq(item, stableRendererSnapshot.seq)
|
||||
const output = getOutputAfterSnapshotSeq(item, stableRendererSnapshot.seq)
|
||||
const seq = item.meta?.seq
|
||||
return data && typeof seq === 'number' ? [{ data, seq }] : []
|
||||
return output && typeof seq === 'number' ? [{ data: output.data, seq }] : []
|
||||
})
|
||||
runtime.replaceHeadlessTerminalFromRendererSnapshotForRecovery(
|
||||
ptyId,
|
||||
@@ -3118,8 +3413,9 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
|
||||
}
|
||||
if (!initialOutputOverflowed) {
|
||||
for (const item of bufferedOutput) {
|
||||
let uncoveredData = getOutputAfterSnapshotSeq(item, snapshotOutputSeq)
|
||||
let uncoveredMeta = item.meta
|
||||
const uncovered = getOutputAfterSnapshotSeq(item, snapshotOutputSeq)
|
||||
let uncoveredData = uncovered?.data ?? null
|
||||
let uncoveredMeta = uncovered?.meta
|
||||
if (
|
||||
uncoveredData &&
|
||||
uncoveredData !== item.data &&
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -47,12 +47,13 @@ function* legacyIterateTerminalOutputFrameChunks(
|
||||
yield {
|
||||
opcode: TerminalStreamOpcode.OutputSpan,
|
||||
bytes: encodeTerminalStreamJson({ data, rawLength, transformed: true }),
|
||||
displayLength: data.length,
|
||||
seq: meta?.seq
|
||||
}
|
||||
return
|
||||
}
|
||||
if (!legacyByteLengthExceeds(data, TERMINAL_STREAM_CHUNK_BYTES)) {
|
||||
yield { bytes: encodeTerminalStreamText(data), seq: meta?.seq }
|
||||
yield { bytes: encodeTerminalStreamText(data), displayLength: data.length, seq: meta?.seq }
|
||||
return
|
||||
}
|
||||
const canPreserveChunkSeq = typeof meta?.seq === 'number' && rawLength === data.length
|
||||
@@ -83,11 +84,18 @@ function* legacyIterateTerminalOutputFrameChunks(
|
||||
if (nextChunk) {
|
||||
if (shouldDelayFinalSeq) {
|
||||
if (delayedChunk) {
|
||||
yield { bytes: encodeTerminalStreamText(delayedChunk.text) }
|
||||
yield {
|
||||
bytes: encodeTerminalStreamText(delayedChunk.text),
|
||||
displayLength: delayedChunk.text.length
|
||||
}
|
||||
}
|
||||
delayedChunk = nextChunk
|
||||
} else {
|
||||
yield { bytes: encodeTerminalStreamText(nextChunk.text), seq: nextChunk.seq }
|
||||
yield {
|
||||
bytes: encodeTerminalStreamText(nextChunk.text),
|
||||
displayLength: nextChunk.text.length,
|
||||
seq: nextChunk.seq
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -99,17 +107,28 @@ function* legacyIterateTerminalOutputFrameChunks(
|
||||
if (shouldDelayFinalSeq) {
|
||||
if (finalChunk) {
|
||||
if (delayedChunk) {
|
||||
yield { bytes: encodeTerminalStreamText(delayedChunk.text) }
|
||||
yield {
|
||||
bytes: encodeTerminalStreamText(delayedChunk.text),
|
||||
displayLength: delayedChunk.text.length
|
||||
}
|
||||
}
|
||||
delayedChunk = finalChunk
|
||||
}
|
||||
if (delayedChunk) {
|
||||
yield { bytes: encodeTerminalStreamText(delayedChunk.text), seq: meta.seq }
|
||||
yield {
|
||||
bytes: encodeTerminalStreamText(delayedChunk.text),
|
||||
displayLength: delayedChunk.text.length,
|
||||
seq: meta.seq
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
if (finalChunk) {
|
||||
yield { bytes: encodeTerminalStreamText(finalChunk.text), seq: finalChunk.seq }
|
||||
yield {
|
||||
bytes: encodeTerminalStreamText(finalChunk.text),
|
||||
displayLength: finalChunk.text.length,
|
||||
seq: finalChunk.seq
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
encodeTerminalStreamText
|
||||
} from '../../../shared/terminal-stream-protocol'
|
||||
import { TERMINAL_STREAM_CHUNK_BYTES } from '../../../shared/terminal-multiplex-flow-control'
|
||||
import type { TerminalOutputSourceRange } from '../../../shared/terminal-output-source-range'
|
||||
import { terminalStreamByteLength } from './terminal-stream-byte-length'
|
||||
|
||||
export type TerminalOutputMeta = {
|
||||
@@ -11,12 +12,15 @@ export type TerminalOutputMeta = {
|
||||
rawLength?: number
|
||||
transformed?: boolean
|
||||
cwd?: string
|
||||
sourceRanges?: readonly TerminalOutputSourceRange[]
|
||||
}
|
||||
|
||||
export type TerminalOutputFrameChunk = {
|
||||
bytes: Uint8Array<ArrayBufferLike>
|
||||
displayLength: number
|
||||
seq?: number
|
||||
opcode?: TerminalStreamOpcode
|
||||
sourceRanges?: readonly TerminalOutputSourceRange[]
|
||||
}
|
||||
|
||||
export const TERMINAL_STREAM_BYTE_PROBE_CODE_UNITS = 8 * 1024
|
||||
@@ -55,12 +59,19 @@ export function* iterateTerminalOutputFrameChunks(
|
||||
yield {
|
||||
opcode: TerminalStreamOpcode.OutputSpan,
|
||||
bytes: encodeTerminalStreamJson({ data, rawLength, transformed: true }),
|
||||
seq: meta?.seq
|
||||
displayLength: data.length,
|
||||
seq: meta?.seq,
|
||||
sourceRanges: meta?.sourceRanges
|
||||
}
|
||||
return
|
||||
}
|
||||
if (!exceedsTerminalStreamChunkBytes(data)) {
|
||||
yield { bytes: encodeTerminalStreamText(data), seq: meta?.seq }
|
||||
yield {
|
||||
bytes: encodeTerminalStreamText(data),
|
||||
displayLength: data.length,
|
||||
seq: meta?.seq,
|
||||
sourceRanges: meta?.sourceRanges
|
||||
}
|
||||
return
|
||||
}
|
||||
const canPreserveChunkSeq = typeof meta?.seq === 'number' && rawLength === data.length
|
||||
@@ -109,11 +120,23 @@ export function* iterateTerminalOutputFrameChunks(
|
||||
const nextChunk = takeChunk(index)
|
||||
if (shouldDelayFinalSeq) {
|
||||
if (delayedChunk) {
|
||||
yield { bytes: encodeTerminalStreamText(delayedChunk.text) }
|
||||
yield {
|
||||
bytes: encodeTerminalStreamText(delayedChunk.text),
|
||||
displayLength: delayedChunk.text.length
|
||||
}
|
||||
}
|
||||
delayedChunk = nextChunk
|
||||
} else {
|
||||
yield { bytes: encodeTerminalStreamText(nextChunk.text), seq: nextChunk.seq }
|
||||
yield {
|
||||
bytes: encodeTerminalStreamText(nextChunk.text),
|
||||
displayLength: nextChunk.text.length,
|
||||
seq: nextChunk.seq,
|
||||
sourceRanges: sliceTerminalOutputSourceRanges(
|
||||
meta?.sourceRanges,
|
||||
chunkStart - nextChunk.text.length,
|
||||
chunkStart
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
chunkBytes += partBytes
|
||||
@@ -124,10 +147,65 @@ export function* iterateTerminalOutputFrameChunks(
|
||||
if (shouldDelayFinalSeq) {
|
||||
// Why: only the final frame can safely carry the high-water mark when rawLength can't map back to UTF-16 offsets.
|
||||
if (delayedChunk) {
|
||||
yield { bytes: encodeTerminalStreamText(delayedChunk.text) }
|
||||
yield {
|
||||
bytes: encodeTerminalStreamText(delayedChunk.text),
|
||||
displayLength: delayedChunk.text.length
|
||||
}
|
||||
}
|
||||
yield {
|
||||
bytes: encodeTerminalStreamText(finalChunk.text),
|
||||
displayLength: finalChunk.text.length,
|
||||
seq: meta.seq
|
||||
}
|
||||
yield { bytes: encodeTerminalStreamText(finalChunk.text), seq: meta.seq }
|
||||
return
|
||||
}
|
||||
yield { bytes: encodeTerminalStreamText(finalChunk.text), seq: finalChunk.seq }
|
||||
yield {
|
||||
bytes: encodeTerminalStreamText(finalChunk.text),
|
||||
displayLength: finalChunk.text.length,
|
||||
seq: finalChunk.seq,
|
||||
sourceRanges: sliceTerminalOutputSourceRanges(
|
||||
meta?.sourceRanges,
|
||||
data.length - finalChunk.text.length,
|
||||
data.length
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export function sliceTerminalOutputSourceRanges(
|
||||
sourceRanges: readonly TerminalOutputSourceRange[] | undefined,
|
||||
displayStartOffset: number,
|
||||
displayEndOffset: number
|
||||
): readonly TerminalOutputSourceRange[] | undefined {
|
||||
if (!sourceRanges || sourceRanges.length === 0) {
|
||||
return undefined
|
||||
}
|
||||
const baseDisplayStart = sourceRanges[0]!.displayStart
|
||||
const sliceStart = baseDisplayStart + displayStartOffset
|
||||
const sliceEnd = baseDisplayStart + displayEndOffset
|
||||
const selected: TerminalOutputSourceRange[] = []
|
||||
for (const range of sourceRanges) {
|
||||
const start = Math.max(sliceStart, range.displayStart)
|
||||
const end = Math.min(sliceEnd, range.displayEnd)
|
||||
if (end <= start) {
|
||||
continue
|
||||
}
|
||||
if (!range.splittable && (start !== range.displayStart || end !== range.displayEnd)) {
|
||||
throw new Error('terminal_source_range_indivisible_split')
|
||||
}
|
||||
const sourceStartSu = range.sourceStartSu + (start - range.displayStart)
|
||||
selected.push(
|
||||
Object.freeze({
|
||||
...range,
|
||||
displayStart: start,
|
||||
displayEnd: end,
|
||||
sourceStartSu,
|
||||
sourceEndSu: sourceStartSu + (end - start),
|
||||
transform: Object.freeze({
|
||||
...range.transform,
|
||||
rawLengthSu: end - start
|
||||
})
|
||||
})
|
||||
)
|
||||
}
|
||||
return Object.freeze(selected)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { TERMINAL_STREAM_CHUNK_BYTES } from '../../../shared/terminal-multiplex-flow-control'
|
||||
import type { TerminalOutputSourceRange } from '../../../shared/terminal-output-source-range'
|
||||
import { iterateTerminalOutputFrameChunks } from './terminal-output-frame-chunks'
|
||||
|
||||
function range(overrides: Partial<TerminalOutputSourceRange>): TerminalOutputSourceRange {
|
||||
return {
|
||||
id: 'pty-1',
|
||||
spanId: 'span-1',
|
||||
providerGeneration: 2,
|
||||
clientGeneration: 3,
|
||||
ownerGeneration: 4,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-1',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 1,
|
||||
displayStart: 0,
|
||||
displayEnd: 1,
|
||||
splittable: true,
|
||||
transform: { transformed: false, rawLengthSu: 1, scalarSafe: true },
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
describe('terminal output frame source ranges', () => {
|
||||
it('maps encoded chunk boundaries to exact ordered source subranges', () => {
|
||||
const data = 'a'.repeat(TERMINAL_STREAM_CHUNK_BYTES + 2)
|
||||
const frames = Array.from(
|
||||
iterateTerminalOutputFrameChunks(data, {
|
||||
seq: data.length,
|
||||
rawLength: data.length,
|
||||
sourceRanges: [
|
||||
range({
|
||||
sourceEndSu: TERMINAL_STREAM_CHUNK_BYTES - 1,
|
||||
displayEnd: TERMINAL_STREAM_CHUNK_BYTES - 1,
|
||||
transform: {
|
||||
transformed: false,
|
||||
rawLengthSu: TERMINAL_STREAM_CHUNK_BYTES - 1,
|
||||
scalarSafe: true
|
||||
}
|
||||
}),
|
||||
range({
|
||||
spanId: 'span-2',
|
||||
sourceStartSu: TERMINAL_STREAM_CHUNK_BYTES - 1,
|
||||
sourceEndSu: data.length,
|
||||
displayStart: TERMINAL_STREAM_CHUNK_BYTES - 1,
|
||||
displayEnd: data.length,
|
||||
transform: {
|
||||
transformed: false,
|
||||
rawLengthSu: data.length - TERMINAL_STREAM_CHUNK_BYTES + 1,
|
||||
scalarSafe: true
|
||||
}
|
||||
})
|
||||
]
|
||||
})
|
||||
)
|
||||
|
||||
expect(frames).toHaveLength(2)
|
||||
expect(frames[0]?.sourceRanges).toEqual([
|
||||
expect.objectContaining({
|
||||
spanId: 'span-1',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: TERMINAL_STREAM_CHUNK_BYTES - 1
|
||||
}),
|
||||
expect.objectContaining({
|
||||
spanId: 'span-2',
|
||||
sourceStartSu: TERMINAL_STREAM_CHUNK_BYTES - 1,
|
||||
sourceEndSu: TERMINAL_STREAM_CHUNK_BYTES
|
||||
})
|
||||
])
|
||||
expect(frames[1]?.sourceRanges).toEqual([
|
||||
expect.objectContaining({
|
||||
spanId: 'span-2',
|
||||
sourceStartSu: TERMINAL_STREAM_CHUNK_BYTES,
|
||||
sourceEndSu: data.length
|
||||
})
|
||||
])
|
||||
})
|
||||
|
||||
it('keeps transformed source ranges indivisible', () => {
|
||||
const sourceRange = range({
|
||||
sourceEndSu: 9,
|
||||
displayEnd: 3,
|
||||
splittable: false,
|
||||
transform: { transformed: true, rawLengthSu: 9, scalarSafe: false }
|
||||
})
|
||||
const [frame] = iterateTerminalOutputFrameChunks('xyz', {
|
||||
seq: 9,
|
||||
rawLength: 9,
|
||||
transformed: true,
|
||||
sourceRanges: [sourceRange]
|
||||
})
|
||||
|
||||
expect(frame?.sourceRanges).toEqual([sourceRange])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,332 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { TerminalOutputSourceRange } from '../../../shared/terminal-output-source-range'
|
||||
import {
|
||||
TERMINAL_SOURCE_RANGE_STREAM_MAX_BYTES,
|
||||
TerminalSourceRangeLedger
|
||||
} from './terminal-source-range-ledger'
|
||||
import { TerminalSourceRangeRegistry } from './terminal-source-range-registry'
|
||||
|
||||
function range(overrides: Partial<TerminalOutputSourceRange> = {}): TerminalOutputSourceRange {
|
||||
return {
|
||||
id: 'pty-1',
|
||||
spanId: 'span-1',
|
||||
providerGeneration: 4,
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-1',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4,
|
||||
displayStart: 0,
|
||||
displayEnd: 4,
|
||||
splittable: true,
|
||||
transform: { transformed: false, rawLengthSu: 4, scalarSafe: true },
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
describe('TerminalSourceRangeLedger', () => {
|
||||
it('advances partial byte credit without settling its covering source frame', () => {
|
||||
const release = vi.fn()
|
||||
const ledger = new TerminalSourceRangeLedger('generation-1', {
|
||||
canReserve: () => true,
|
||||
reserve: () => true,
|
||||
release,
|
||||
close: () => {}
|
||||
})
|
||||
ledger.accept(100, 100, [
|
||||
range({
|
||||
sourceEndSu: 100,
|
||||
displayEnd: 100,
|
||||
transform: { transformed: false, rawLengthSu: 100, scalarSafe: true }
|
||||
})
|
||||
])
|
||||
|
||||
expect(ledger.acknowledge('stale-generation', 40)).toEqual({
|
||||
status: 'stale-generation',
|
||||
settled: []
|
||||
})
|
||||
expect(ledger.acknowledge('generation-1', 40)).toEqual({
|
||||
status: 'accepted',
|
||||
acknowledgedBytes: 40,
|
||||
settled: []
|
||||
})
|
||||
expect(ledger.getDebugSnapshot()).toMatchObject({
|
||||
ackedEndByte: 40,
|
||||
retainedBytes: 60,
|
||||
frames: 1
|
||||
})
|
||||
expect(release).toHaveBeenLastCalledWith(40)
|
||||
|
||||
expect(ledger.acknowledge('generation-1', 100)).toMatchObject({
|
||||
status: 'accepted',
|
||||
acknowledgedBytes: 60,
|
||||
settled: [{ spanId: 'span-1', sourceStartSu: 0, sourceEndSu: 100 }]
|
||||
})
|
||||
expect(ledger.getDebugSnapshot()).toMatchObject({
|
||||
ackedEndByte: 100,
|
||||
retainedBytes: 0,
|
||||
frames: 0
|
||||
})
|
||||
expect(release.mock.calls).toEqual([[40], [60]])
|
||||
})
|
||||
|
||||
it('admits a contiguous recovered delivery while the prior token remains unsettled', () => {
|
||||
const ledger = new TerminalSourceRangeLedger('generation-1')
|
||||
expect(ledger.accept(4, 4, [range()])).not.toBeNull()
|
||||
expect(
|
||||
ledger.accept(4, 4, [
|
||||
range({
|
||||
spanId: 'span-recovered',
|
||||
clientGeneration: 3,
|
||||
ownerGeneration: 4,
|
||||
deliveryToken: 'token-2',
|
||||
sourceStartSu: 4,
|
||||
sourceEndSu: 8,
|
||||
displayStart: 4,
|
||||
displayEnd: 8
|
||||
})
|
||||
])
|
||||
).not.toBeNull()
|
||||
|
||||
expect(ledger.acknowledge('generation-1', 4)).toMatchObject({
|
||||
status: 'accepted',
|
||||
settled: [{ spanId: 'span-1', deliveryToken: 'token-1' }]
|
||||
})
|
||||
expect(ledger.acknowledge('generation-1', 8)).toMatchObject({
|
||||
status: 'accepted',
|
||||
settled: [{ spanId: 'span-recovered', deliveryToken: 'token-2' }]
|
||||
})
|
||||
expect(
|
||||
ledger.prepareAccept(4, 4, [
|
||||
range({
|
||||
spanId: 'span-stale',
|
||||
sourceStartSu: 8,
|
||||
sourceEndSu: 12,
|
||||
displayStart: 8,
|
||||
displayEnd: 12
|
||||
})
|
||||
]).status
|
||||
).toBe('cross-generation')
|
||||
})
|
||||
|
||||
it('settles only complete frames when a cumulative ACK crosses frame boundaries', () => {
|
||||
const ledger = new TerminalSourceRangeLedger('generation-1')
|
||||
ledger.accept(5, 4, [range()])
|
||||
ledger.accept(7, 5, [
|
||||
range({
|
||||
spanId: 'span-2',
|
||||
sourceStartSu: 4,
|
||||
sourceEndSu: 9,
|
||||
displayStart: 4,
|
||||
displayEnd: 9,
|
||||
transform: { transformed: false, rawLengthSu: 5, scalarSafe: true }
|
||||
})
|
||||
])
|
||||
|
||||
expect(ledger.acknowledge('generation-1', 3)).toEqual({
|
||||
status: 'accepted',
|
||||
acknowledgedBytes: 3,
|
||||
settled: []
|
||||
})
|
||||
expect(ledger.getDebugSnapshot()).toMatchObject({
|
||||
ackedEndByte: 3,
|
||||
retainedBytes: 9,
|
||||
frames: 2
|
||||
})
|
||||
expect(ledger.acknowledge('generation-1', 12)).toMatchObject({
|
||||
status: 'accepted',
|
||||
acknowledgedBytes: 9,
|
||||
settled: [{ spanId: 'span-1' }, { spanId: 'span-2' }]
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects malformed coverage, gaps, and later cross-token mappings', () => {
|
||||
const ledger = new TerminalSourceRangeLedger('generation-1')
|
||||
expect(ledger.accept(5, 3, [range()])).toBeNull()
|
||||
expect(ledger.accept(5, 4, [range()])).not.toBeNull()
|
||||
expect(
|
||||
ledger.accept(5, 4, [
|
||||
range({
|
||||
spanId: 'gap',
|
||||
sourceStartSu: 5,
|
||||
sourceEndSu: 9,
|
||||
displayStart: 4,
|
||||
displayEnd: 8
|
||||
})
|
||||
])
|
||||
).toBeNull()
|
||||
expect(
|
||||
ledger.prepareAccept(5, 4, [
|
||||
range({
|
||||
spanId: 'other',
|
||||
deliveryToken: 'token-2',
|
||||
sourceStartSu: 4,
|
||||
sourceEndSu: 8,
|
||||
displayStart: 4,
|
||||
displayEnd: 8
|
||||
})
|
||||
]).status
|
||||
).toBe('cross-generation')
|
||||
expect(ledger.getDebugSnapshot()).toMatchObject({ retainedBytes: 5, frames: 1 })
|
||||
})
|
||||
|
||||
it('rejects mixing mapped and unmapped output in either order', () => {
|
||||
const mapped = new TerminalSourceRangeLedger('mapped')
|
||||
expect(mapped.accept(5, 4, [range()])).not.toBeNull()
|
||||
expect(mapped.prepareAccept(1, 1, []).status).toBe('invalid')
|
||||
|
||||
const unmapped = new TerminalSourceRangeLedger('unmapped')
|
||||
expect(unmapped.accept(1, 1, [])).not.toBeNull()
|
||||
expect(unmapped.prepareAccept(5, 4, [range()]).status).toBe('invalid')
|
||||
})
|
||||
|
||||
it('rejects excessive, stale, notification-shaped, and late settlement', () => {
|
||||
const ledger = new TerminalSourceRangeLedger('generation-1')
|
||||
ledger.accept(5, 4, [range()])
|
||||
|
||||
expect(ledger.acknowledge('generation-2', 5).status).toBe('stale-generation')
|
||||
expect(ledger.acknowledge('generation-1', 6).status).toBe('excessive')
|
||||
expect(ledger.acknowledge('generation-1', Number.NaN).status).toBe('invalid')
|
||||
const transfer = ledger.beginTransfer()
|
||||
transfer.commit()
|
||||
expect(ledger.acknowledge('generation-1', 5).status).toBe('invalid')
|
||||
expect(ledger.getDebugSnapshot()).toMatchObject({ ackedEndByte: 0, closed: true })
|
||||
})
|
||||
|
||||
it('restores all mappings when an atomic transfer rolls back', () => {
|
||||
const ledger = new TerminalSourceRangeLedger('generation-1')
|
||||
ledger.accept(5, 4, [range()])
|
||||
const transfer = ledger.beginTransfer()
|
||||
|
||||
expect(transfer.frames).toHaveLength(1)
|
||||
expect(ledger.canAccept(1)).toBe(false)
|
||||
transfer.rollback()
|
||||
expect(ledger.getDebugSnapshot()).toMatchObject({
|
||||
retainedBytes: 5,
|
||||
frames: 1,
|
||||
transferring: false,
|
||||
closed: false
|
||||
})
|
||||
|
||||
const retry = ledger.beginTransfer()
|
||||
retry.commit()
|
||||
expect(ledger.getDebugSnapshot()).toMatchObject({
|
||||
retainedBytes: 0,
|
||||
frames: 0,
|
||||
closed: true
|
||||
})
|
||||
})
|
||||
|
||||
it('replaces covered mappings without synthesizing encoded-byte credit', () => {
|
||||
const ledger = new TerminalSourceRangeLedger('generation-1')
|
||||
ledger.accept(5, 4, [range()], 4)
|
||||
ledger.accept(
|
||||
7,
|
||||
4,
|
||||
[
|
||||
range({
|
||||
spanId: 'span-trailing',
|
||||
sourceStartSu: 4,
|
||||
sourceEndSu: 8,
|
||||
displayStart: 4,
|
||||
displayEnd: 8
|
||||
})
|
||||
],
|
||||
8
|
||||
)
|
||||
const replacement = ledger.planSourceRangeReplacement(8)
|
||||
|
||||
expect(replacement).not.toBeNull()
|
||||
expect(() => replacement?.commit()).not.toThrow()
|
||||
expect(ledger.acknowledge('generation-1', 12)).toMatchObject({
|
||||
status: 'accepted',
|
||||
acknowledgedBytes: 12,
|
||||
settled: []
|
||||
})
|
||||
expect(
|
||||
ledger.accept(
|
||||
3,
|
||||
2,
|
||||
[
|
||||
range({
|
||||
spanId: 'span-live',
|
||||
sourceStartSu: 8,
|
||||
sourceEndSu: 10,
|
||||
displayStart: 8,
|
||||
displayEnd: 10,
|
||||
transform: { transformed: false, rawLengthSu: 2, scalarSafe: true }
|
||||
})
|
||||
],
|
||||
10
|
||||
)
|
||||
).not.toBeNull()
|
||||
})
|
||||
|
||||
it('rejects an admitted trailing mapping before authoritative commit', () => {
|
||||
const ledger = new TerminalSourceRangeLedger('generation-1')
|
||||
ledger.accept(5, 4, [range()], 4)
|
||||
ledger.accept(
|
||||
7,
|
||||
4,
|
||||
[
|
||||
range({
|
||||
spanId: 'span-trailing',
|
||||
sourceStartSu: 4,
|
||||
sourceEndSu: 8,
|
||||
displayStart: 4,
|
||||
displayEnd: 8
|
||||
})
|
||||
],
|
||||
8
|
||||
)
|
||||
|
||||
expect(ledger.planSourceRangeReplacement(4)).toBeNull()
|
||||
})
|
||||
|
||||
it('rejects an unsequenced mapped frame before replacement commit', () => {
|
||||
const ledger = new TerminalSourceRangeLedger('generation-1')
|
||||
ledger.accept(5, 4, [range()])
|
||||
|
||||
expect(ledger.planSourceRangeReplacement(4)).toBeNull()
|
||||
})
|
||||
|
||||
it('rolls back a pre-send admission without accepting a byte boundary', () => {
|
||||
const registry = new TerminalSourceRangeRegistry()
|
||||
const ledger = registry.open('generation-1')!
|
||||
const prepared = ledger.prepareAccept(5, 4, [range()])
|
||||
expect(prepared.status).toBe('ready')
|
||||
if (prepared.status !== 'ready') {
|
||||
throw new Error('expected source range admission')
|
||||
}
|
||||
expect(registry.getDebugSnapshot().retainedBytes).toBe(5)
|
||||
|
||||
prepared.admission.rollback()
|
||||
|
||||
expect(ledger.getDebugSnapshot()).toMatchObject({
|
||||
acceptedEndByte: 0,
|
||||
retainedBytes: 0,
|
||||
frames: 0
|
||||
})
|
||||
expect(registry.getDebugSnapshot().retainedBytes).toBe(0)
|
||||
})
|
||||
|
||||
it('bounds aggregate retained mapping bytes and releases them on ACK and close', () => {
|
||||
const registry = new TerminalSourceRangeRegistry()
|
||||
const ledgers = Array.from({ length: 9 }, (_, index) => registry.open(`stream-${index}`)!)
|
||||
for (const ledger of ledgers.slice(0, 8)) {
|
||||
expect(ledger.accept(TERMINAL_SOURCE_RANGE_STREAM_MAX_BYTES, 0, [])).not.toBeNull()
|
||||
}
|
||||
expect(ledgers[8]!.canAccept(1)).toBe(false)
|
||||
expect(registry.getDebugSnapshot().retainedBytes).toBe(16 * 1024 * 1024)
|
||||
|
||||
expect(ledgers[0]!.acknowledge('stream-0', TERMINAL_SOURCE_RANGE_STREAM_MAX_BYTES).status).toBe(
|
||||
'accepted'
|
||||
)
|
||||
expect(ledgers[8]!.accept(1, 0, [])).not.toBeNull()
|
||||
for (const ledger of ledgers) {
|
||||
ledger.close()
|
||||
}
|
||||
expect(registry.getDebugSnapshot()).toEqual({ streams: 0, retainedBytes: 0 })
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,310 @@
|
||||
import {
|
||||
sameTerminalOutputSourceIdentity,
|
||||
type TerminalOutputSourceRange
|
||||
} from '../../../shared/terminal-output-source-range'
|
||||
import {
|
||||
canPlanTerminalSourceRangeReplacement,
|
||||
freezeTerminalOutputSourceRanges,
|
||||
replaceTerminalSourceRangeFrames,
|
||||
type TerminalSourceRangeFrame,
|
||||
validateTerminalSourceRangeFrame
|
||||
} from './terminal-source-range-validation'
|
||||
|
||||
export type { TerminalSourceRangeFrame } from './terminal-source-range-validation'
|
||||
|
||||
export const TERMINAL_SOURCE_RANGE_STREAM_MAX_BYTES = 2 * 1024 * 1024
|
||||
|
||||
export type TerminalSourceRangeAckResult =
|
||||
| {
|
||||
status: 'accepted'
|
||||
acknowledgedBytes: number
|
||||
settled: readonly TerminalOutputSourceRange[]
|
||||
}
|
||||
| { status: 'duplicate'; settled: readonly [] }
|
||||
| {
|
||||
status: 'invalid' | 'stale-generation' | 'excessive' | 'cross-generation'
|
||||
settled: readonly []
|
||||
}
|
||||
|
||||
export type TerminalSourceRangeAdmission = Readonly<{
|
||||
frame: TerminalSourceRangeFrame
|
||||
commit: () => boolean
|
||||
rollback: () => void
|
||||
}>
|
||||
|
||||
export type TerminalSourceRangeAdmissionResult =
|
||||
| { status: 'ready'; admission: TerminalSourceRangeAdmission }
|
||||
| { status: 'capacity' | 'invalid' | 'cross-generation' }
|
||||
|
||||
export type TerminalSourceRangeTransfer = Readonly<{
|
||||
frames: readonly TerminalSourceRangeFrame[]
|
||||
commit: () => void
|
||||
rollback: () => void
|
||||
}>
|
||||
|
||||
export type TerminalSourceRangeBudget = {
|
||||
canReserve: (bytes: number) => boolean
|
||||
reserve: (bytes: number) => boolean
|
||||
release: (bytes: number) => void
|
||||
close: () => void
|
||||
}
|
||||
|
||||
const STANDALONE_BUDGET: TerminalSourceRangeBudget = {
|
||||
canReserve: () => true,
|
||||
reserve: () => true,
|
||||
release: () => {},
|
||||
close: () => {}
|
||||
}
|
||||
|
||||
function isRecoveredSourceIdentity(
|
||||
previous: TerminalOutputSourceRange,
|
||||
next: TerminalOutputSourceRange
|
||||
): boolean {
|
||||
return (
|
||||
previous.id === next.id &&
|
||||
previous.providerGeneration === next.providerGeneration &&
|
||||
previous.ptyIncarnation === next.ptyIncarnation &&
|
||||
next.clientGeneration > previous.clientGeneration &&
|
||||
next.ownerGeneration > previous.ownerGeneration &&
|
||||
next.deliveryToken !== previous.deliveryToken
|
||||
)
|
||||
}
|
||||
|
||||
export class TerminalSourceRangeLedger {
|
||||
private acceptedEndByte = 0
|
||||
private ackedEndByte = 0
|
||||
private retainedBytes = 0
|
||||
private frames: TerminalSourceRangeFrame[] = []
|
||||
private mappingMode: 'mapped' | 'unmapped' | null = null
|
||||
private boundRange: TerminalOutputSourceRange | null = null
|
||||
private mappedSourceEndSu: number | null = null
|
||||
private mappedDisplayEnd: number | null = null
|
||||
private pending: TerminalSourceRangeAdmission | null = null
|
||||
private transferring = false
|
||||
private closed = false
|
||||
|
||||
constructor(
|
||||
readonly streamGeneration: string,
|
||||
private readonly budget: TerminalSourceRangeBudget = STANDALONE_BUDGET
|
||||
) {
|
||||
if (!streamGeneration) {
|
||||
throw new Error('terminal_source_range_generation_required')
|
||||
}
|
||||
}
|
||||
|
||||
canAccept(encodedBytes: number): boolean {
|
||||
return (
|
||||
!this.closed &&
|
||||
!this.transferring &&
|
||||
!this.pending &&
|
||||
Number.isSafeInteger(encodedBytes) &&
|
||||
encodedBytes > 0 &&
|
||||
this.retainedBytes + encodedBytes <= TERMINAL_SOURCE_RANGE_STREAM_MAX_BYTES &&
|
||||
this.budget.canReserve(encodedBytes)
|
||||
)
|
||||
}
|
||||
|
||||
prepareAccept(
|
||||
encodedBytes: number,
|
||||
displayLength: number,
|
||||
sourceRanges: readonly TerminalOutputSourceRange[],
|
||||
outputSeq?: number
|
||||
): TerminalSourceRangeAdmissionResult {
|
||||
if (!this.canAccept(encodedBytes)) {
|
||||
return { status: 'capacity' }
|
||||
}
|
||||
if (!validateTerminalSourceRangeFrame(displayLength, sourceRanges)) {
|
||||
return { status: 'invalid' }
|
||||
}
|
||||
const nextMappingMode = sourceRanges.length > 0 ? 'mapped' : 'unmapped'
|
||||
if (this.mappingMode && this.mappingMode !== nextMappingMode) {
|
||||
return { status: 'invalid' }
|
||||
}
|
||||
const first = sourceRanges[0]
|
||||
if (
|
||||
first &&
|
||||
(this.boundRange
|
||||
? !sameTerminalOutputSourceIdentity(this.boundRange, first) &&
|
||||
!isRecoveredSourceIdentity(this.boundRange, first)
|
||||
: !sourceRanges.every((range) => sameTerminalOutputSourceIdentity(first, range)))
|
||||
) {
|
||||
return { status: 'cross-generation' }
|
||||
}
|
||||
if (
|
||||
first &&
|
||||
((this.mappedSourceEndSu !== null && first.sourceStartSu !== this.mappedSourceEndSu) ||
|
||||
(this.mappedDisplayEnd !== null && first.displayStart !== this.mappedDisplayEnd))
|
||||
) {
|
||||
return { status: 'invalid' }
|
||||
}
|
||||
if (!this.budget.reserve(encodedBytes)) {
|
||||
return { status: 'capacity' }
|
||||
}
|
||||
const ranges = freezeTerminalOutputSourceRanges(sourceRanges)
|
||||
const frame = Object.freeze({
|
||||
encodedStartByte: this.acceptedEndByte,
|
||||
encodedEndByte: this.acceptedEndByte + encodedBytes,
|
||||
displayLength,
|
||||
...(typeof outputSeq === 'number' ? { outputSeq } : {}),
|
||||
sourceRanges: ranges
|
||||
})
|
||||
let finished = false
|
||||
const admission: TerminalSourceRangeAdmission = Object.freeze({
|
||||
frame,
|
||||
commit: () => {
|
||||
if (finished || this.pending !== admission || this.closed || this.transferring) {
|
||||
return false
|
||||
}
|
||||
finished = true
|
||||
this.pending = null
|
||||
this.acceptedEndByte = frame.encodedEndByte
|
||||
this.retainedBytes += encodedBytes
|
||||
this.frames.push(frame)
|
||||
this.mappingMode ??= nextMappingMode
|
||||
const last = ranges.at(-1)
|
||||
if (first && last) {
|
||||
this.boundRange = first
|
||||
this.mappedSourceEndSu = last.sourceEndSu
|
||||
this.mappedDisplayEnd = last.displayEnd
|
||||
}
|
||||
return true
|
||||
},
|
||||
rollback: () => {
|
||||
if (finished) {
|
||||
return
|
||||
}
|
||||
finished = true
|
||||
if (this.pending === admission) {
|
||||
this.pending = null
|
||||
}
|
||||
this.budget.release(encodedBytes)
|
||||
}
|
||||
})
|
||||
this.pending = admission
|
||||
return { status: 'ready', admission }
|
||||
}
|
||||
|
||||
accept(
|
||||
encodedBytes: number,
|
||||
displayLength: number,
|
||||
sourceRanges: readonly TerminalOutputSourceRange[],
|
||||
outputSeq?: number
|
||||
): TerminalSourceRangeFrame | null {
|
||||
const prepared = this.prepareAccept(encodedBytes, displayLength, sourceRanges, outputSeq)
|
||||
if (prepared.status !== 'ready' || !prepared.admission.commit()) {
|
||||
return null
|
||||
}
|
||||
return prepared.admission.frame
|
||||
}
|
||||
|
||||
acknowledge(streamGeneration: string, ackedEndByte: number): TerminalSourceRangeAckResult {
|
||||
if (streamGeneration !== this.streamGeneration) {
|
||||
return { status: 'stale-generation', settled: [] }
|
||||
}
|
||||
if (this.closed || this.transferring || this.pending) {
|
||||
return { status: 'invalid', settled: [] }
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(ackedEndByte) ||
|
||||
ackedEndByte < 0 ||
|
||||
ackedEndByte < this.ackedEndByte
|
||||
) {
|
||||
return { status: 'invalid', settled: [] }
|
||||
}
|
||||
if (ackedEndByte > this.acceptedEndByte) {
|
||||
return { status: 'excessive', settled: [] }
|
||||
}
|
||||
if (ackedEndByte === this.ackedEndByte) {
|
||||
return { status: 'duplicate', settled: [] }
|
||||
}
|
||||
const acknowledgedBytes = ackedEndByte - this.ackedEndByte
|
||||
const settled: TerminalOutputSourceRange[] = []
|
||||
let frameCount = 0
|
||||
for (const frame of this.frames) {
|
||||
if (frame.encodedEndByte > ackedEndByte) {
|
||||
break
|
||||
}
|
||||
settled.push(...frame.sourceRanges)
|
||||
frameCount++
|
||||
}
|
||||
this.ackedEndByte = ackedEndByte
|
||||
this.frames.splice(0, frameCount)
|
||||
this.retainedBytes -= acknowledgedBytes
|
||||
this.budget.release(acknowledgedBytes)
|
||||
return { status: 'accepted', acknowledgedBytes, settled: Object.freeze(settled) }
|
||||
}
|
||||
|
||||
planSourceRangeReplacement(snapshotSeq: number): Readonly<{ commit: () => void }> | null {
|
||||
const unavailable = this.closed || this.transferring || this.pending
|
||||
if (unavailable || !canPlanTerminalSourceRangeReplacement(this.frames, snapshotSeq)) {
|
||||
return null
|
||||
}
|
||||
const replacement = replaceTerminalSourceRangeFrames(this.frames, snapshotSeq)
|
||||
let committed = false
|
||||
return Object.freeze({
|
||||
commit: () => {
|
||||
if (committed || this.closed) {
|
||||
return
|
||||
}
|
||||
committed = true
|
||||
Object.assign(this, replacement)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
beginTransfer(): TerminalSourceRangeTransfer {
|
||||
if (this.closed || this.transferring || this.pending) {
|
||||
throw new Error('terminal_source_range_transfer_invalid')
|
||||
}
|
||||
this.transferring = true
|
||||
let finished = false
|
||||
const frames = Object.freeze(this.frames.slice())
|
||||
return Object.freeze({
|
||||
frames,
|
||||
commit: () => {
|
||||
if (finished) {
|
||||
return
|
||||
}
|
||||
finished = true
|
||||
this.budget.release(this.retainedBytes)
|
||||
this.frames = []
|
||||
this.retainedBytes = 0
|
||||
this.transferring = false
|
||||
this.closed = true
|
||||
this.budget.close()
|
||||
},
|
||||
rollback: () => {
|
||||
if (finished) {
|
||||
return
|
||||
}
|
||||
finished = true
|
||||
this.transferring = false
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
close(): void {
|
||||
if (this.closed) {
|
||||
return
|
||||
}
|
||||
this.pending?.rollback()
|
||||
this.budget.release(this.retainedBytes)
|
||||
this.frames = []
|
||||
this.retainedBytes = 0
|
||||
this.transferring = false
|
||||
this.closed = true
|
||||
this.budget.close()
|
||||
}
|
||||
|
||||
getDebugSnapshot() {
|
||||
return {
|
||||
acceptedEndByte: this.acceptedEndByte,
|
||||
ackedEndByte: this.ackedEndByte,
|
||||
retainedBytes: this.retainedBytes,
|
||||
frames: this.frames.length,
|
||||
transferring: this.transferring,
|
||||
closed: this.closed,
|
||||
bound: this.boundRange !== null
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import { TERMINAL_MULTIPLEX_MAX_STREAMS_PER_CONNECTION } from '../../../shared/terminal-multiplex-flow-control'
|
||||
import {
|
||||
TerminalSourceRangeLedger,
|
||||
type TerminalSourceRangeBudget
|
||||
} from './terminal-source-range-ledger'
|
||||
|
||||
export const TERMINAL_SOURCE_RANGE_CONNECTION_MAX_BYTES = 16 * 1024 * 1024
|
||||
|
||||
export class TerminalSourceRangeRegistry {
|
||||
private readonly ledgers = new Set<TerminalSourceRangeLedger>()
|
||||
private retainedBytes = 0
|
||||
|
||||
open(streamGeneration: string): TerminalSourceRangeLedger | null {
|
||||
if (this.ledgers.size >= TERMINAL_MULTIPLEX_MAX_STREAMS_PER_CONNECTION) {
|
||||
return null
|
||||
}
|
||||
let ledger: TerminalSourceRangeLedger
|
||||
const budget: TerminalSourceRangeBudget = {
|
||||
canReserve: (bytes) =>
|
||||
this.retainedBytes + bytes <= TERMINAL_SOURCE_RANGE_CONNECTION_MAX_BYTES,
|
||||
reserve: (bytes) => {
|
||||
if (this.retainedBytes + bytes > TERMINAL_SOURCE_RANGE_CONNECTION_MAX_BYTES) {
|
||||
return false
|
||||
}
|
||||
this.retainedBytes += bytes
|
||||
return true
|
||||
},
|
||||
release: (bytes) => {
|
||||
this.retainedBytes = Math.max(0, this.retainedBytes - bytes)
|
||||
},
|
||||
close: () => {
|
||||
this.ledgers.delete(ledger)
|
||||
}
|
||||
}
|
||||
ledger = new TerminalSourceRangeLedger(streamGeneration, budget)
|
||||
this.ledgers.add(ledger)
|
||||
return ledger
|
||||
}
|
||||
|
||||
getDebugSnapshot() {
|
||||
return { streams: this.ledgers.size, retainedBytes: this.retainedBytes }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
import {
|
||||
assertTerminalOutputSourceRange,
|
||||
sameTerminalOutputSourceIdentity,
|
||||
type TerminalOutputSourceRange
|
||||
} from '../../../shared/terminal-output-source-range'
|
||||
|
||||
export type TerminalSourceRangeFrame = Readonly<{
|
||||
encodedStartByte: number
|
||||
encodedEndByte: number
|
||||
displayLength: number
|
||||
outputSeq?: number
|
||||
sourceRanges: readonly TerminalOutputSourceRange[]
|
||||
}>
|
||||
|
||||
export function freezeTerminalOutputSourceRanges(
|
||||
ranges: readonly TerminalOutputSourceRange[]
|
||||
): readonly TerminalOutputSourceRange[] {
|
||||
return Object.freeze(
|
||||
ranges.map((range) =>
|
||||
Object.freeze({
|
||||
...range,
|
||||
transform: Object.freeze({ ...range.transform })
|
||||
})
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
export function validateTerminalSourceRangeFrame(
|
||||
displayLength: number,
|
||||
ranges: readonly TerminalOutputSourceRange[]
|
||||
): boolean {
|
||||
if (!Number.isSafeInteger(displayLength) || displayLength < 0) {
|
||||
return false
|
||||
}
|
||||
if (ranges.length === 0) {
|
||||
return true
|
||||
}
|
||||
try {
|
||||
for (const range of ranges) {
|
||||
assertTerminalOutputSourceRange(range)
|
||||
}
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
const first = ranges[0]!
|
||||
let previous = first
|
||||
for (const range of ranges.slice(1)) {
|
||||
if (
|
||||
!sameTerminalOutputSourceIdentity(first, range) ||
|
||||
range.sourceStartSu !== previous.sourceEndSu ||
|
||||
range.displayStart !== previous.displayEnd
|
||||
) {
|
||||
return false
|
||||
}
|
||||
previous = range
|
||||
}
|
||||
return previous.displayEnd - first.displayStart === displayLength
|
||||
}
|
||||
|
||||
export function replaceTerminalSourceRangeFrames(
|
||||
frames: readonly TerminalSourceRangeFrame[],
|
||||
snapshotSeq: number
|
||||
): Readonly<{
|
||||
frames: TerminalSourceRangeFrame[]
|
||||
mappingMode: 'mapped' | null
|
||||
boundRange: TerminalOutputSourceRange | null
|
||||
mappedSourceEndSu: number | null
|
||||
mappedDisplayEnd: number | null
|
||||
}> {
|
||||
const replaced = frames.map((frame) =>
|
||||
typeof frame.outputSeq === 'number' && frame.outputSeq <= snapshotSeq
|
||||
? Object.freeze({ ...frame, sourceRanges: Object.freeze([]) })
|
||||
: frame
|
||||
)
|
||||
const remainingRanges = replaced.flatMap((frame) => frame.sourceRanges)
|
||||
const last = remainingRanges.at(-1)
|
||||
return Object.freeze({
|
||||
frames: replaced,
|
||||
mappingMode: remainingRanges.length > 0 ? 'mapped' : null,
|
||||
boundRange: last ?? null,
|
||||
mappedSourceEndSu: last?.sourceEndSu ?? null,
|
||||
mappedDisplayEnd: last?.displayEnd ?? null
|
||||
})
|
||||
}
|
||||
|
||||
export function canPlanTerminalSourceRangeReplacement(
|
||||
frames: readonly TerminalSourceRangeFrame[],
|
||||
snapshotSeq: number
|
||||
): boolean {
|
||||
return (
|
||||
Number.isSafeInteger(snapshotSeq) &&
|
||||
snapshotSeq >= 0 &&
|
||||
frames.every(
|
||||
(frame) =>
|
||||
frame.sourceRanges.length === 0 ||
|
||||
(typeof frame.outputSeq === 'number' && frame.outputSeq <= snapshotSeq)
|
||||
)
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
import {
|
||||
containFrameDecoderContinuation,
|
||||
publishFrameDecoderError,
|
||||
type DecodedFrame,
|
||||
type FrameDecoderOptions
|
||||
} from '../../shared/relay-frame-decoder-contract'
|
||||
import { RelayFrameBuffer } from '../../shared/relay-frame-buffer'
|
||||
export {
|
||||
FrameDecoderContinuationError,
|
||||
type DecodedFrame,
|
||||
type FrameDecoderOptions
|
||||
} from '../../shared/relay-frame-decoder-contract'
|
||||
|
||||
export const HEADER_LENGTH = 13
|
||||
export const MAX_MESSAGE_SIZE = 16 * 1024 * 1024
|
||||
export const FRAME_DECODER_MAX_FRAMES_PER_TURN = 64
|
||||
export const FRAME_DECODER_MAX_BYTES_PER_TURN = MAX_MESSAGE_SIZE + HEADER_LENGTH
|
||||
export const FRAME_DECODER_MAX_TURN_MS = 4,
|
||||
FRAME_DECODER_MAX_RETAINED_BYTES = MAX_MESSAGE_SIZE + HEADER_LENGTH + 1024 * 1024
|
||||
|
||||
export class FrameDecoder {
|
||||
private readonly buffer = new RelayFrameBuffer()
|
||||
private oversizedPayloadBytesRemaining = 0
|
||||
private onFrame: (frame: DecodedFrame) => void
|
||||
private onError: ((err: Error) => void) | null
|
||||
private maxFramesPerTurn: number
|
||||
private maxBytesPerTurn: number
|
||||
private maxTurnMs: number
|
||||
private now: () => number
|
||||
private schedule: (callback: () => void) => unknown
|
||||
private cancelScheduled: (handle: unknown) => void
|
||||
private pause: (() => void) | null
|
||||
private resume: (() => void) | null
|
||||
private continuationHandle: unknown
|
||||
private continuationHandleAssigned = false
|
||||
private continuationScheduled = false
|
||||
private paused = false
|
||||
private draining = false
|
||||
private generation = 0
|
||||
|
||||
constructor(
|
||||
onFrame: (frame: DecodedFrame) => void,
|
||||
onError?: (err: Error) => void,
|
||||
options: FrameDecoderOptions = {}
|
||||
) {
|
||||
this.onFrame = onFrame
|
||||
this.onError = onError ?? null
|
||||
this.maxFramesPerTurn = positiveLimit(
|
||||
options.maxFramesPerTurn,
|
||||
FRAME_DECODER_MAX_FRAMES_PER_TURN
|
||||
)
|
||||
this.maxBytesPerTurn = positiveLimit(options.maxBytesPerTurn, FRAME_DECODER_MAX_BYTES_PER_TURN)
|
||||
this.maxTurnMs = positiveLimit(options.maxTurnMs, FRAME_DECODER_MAX_TURN_MS)
|
||||
this.now = options.now ?? Date.now
|
||||
this.schedule = options.schedule ?? ((callback) => setImmediate(callback))
|
||||
this.cancelScheduled =
|
||||
options.cancelScheduled ?? ((handle) => clearImmediate(handle as NodeJS.Immediate))
|
||||
this.pause = options.pause ?? null
|
||||
this.resume = options.resume ?? null
|
||||
}
|
||||
|
||||
feed(chunk: Buffer | Uint8Array): void {
|
||||
const buf = Buffer.isBuffer(chunk)
|
||||
? chunk
|
||||
: Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength)
|
||||
const retained = this.buffer.length + buf.length
|
||||
if (retained > FRAME_DECODER_MAX_RETAINED_BYTES) {
|
||||
this.reset()
|
||||
publishFrameDecoderError(
|
||||
this.onError,
|
||||
new Error(`Frame decoder retained-input limit exceeded: ${retained}`)
|
||||
)
|
||||
return
|
||||
}
|
||||
if (buf.length > 0) {
|
||||
this.buffer.append(buf)
|
||||
}
|
||||
if (!this.draining && !this.continuationScheduled) {
|
||||
this.drainTurn()
|
||||
}
|
||||
}
|
||||
|
||||
reset(): void {
|
||||
this.generation += 1
|
||||
this.cancelContinuation()
|
||||
this.buffer.clear()
|
||||
this.oversizedPayloadBytesRemaining = 0
|
||||
this.releasePause()
|
||||
}
|
||||
|
||||
drain(): Buffer {
|
||||
const out = this.buffer.drain()
|
||||
this.reset()
|
||||
return out
|
||||
}
|
||||
|
||||
private drainTurn(): void {
|
||||
if (this.draining) {
|
||||
return
|
||||
}
|
||||
this.draining = true
|
||||
const generation = this.generation
|
||||
const startedAt = this.now()
|
||||
let frames = 0
|
||||
let bytes = 0
|
||||
|
||||
try {
|
||||
while (generation === this.generation) {
|
||||
if (
|
||||
frames >= this.maxFramesPerTurn ||
|
||||
bytes >= this.maxBytesPerTurn ||
|
||||
(frames > 0 && this.now() - startedAt >= this.maxTurnMs)
|
||||
) {
|
||||
break
|
||||
}
|
||||
const discarded = this.discardOversizedPayload(bytes)
|
||||
if (discarded > 0) {
|
||||
bytes += discarded
|
||||
continue
|
||||
}
|
||||
if (this.buffer.length < HEADER_LENGTH) {
|
||||
break
|
||||
}
|
||||
const header = this.buffer.peek(HEADER_LENGTH)
|
||||
const length = header.readUInt32BE(9)
|
||||
if (length > MAX_MESSAGE_SIZE) {
|
||||
this.buffer.discard(HEADER_LENGTH)
|
||||
this.oversizedPayloadBytesRemaining = length
|
||||
bytes += HEADER_LENGTH
|
||||
publishFrameDecoderError(
|
||||
this.onError,
|
||||
new Error(`Frame payload too large: ${length} bytes — discarded`)
|
||||
)
|
||||
continue
|
||||
}
|
||||
const totalLength = HEADER_LENGTH + length
|
||||
if (this.buffer.length < totalLength) {
|
||||
break
|
||||
}
|
||||
if (frames > 0 && bytes + totalLength > this.maxBytesPerTurn) {
|
||||
break
|
||||
}
|
||||
const framed = this.buffer.take(totalLength)
|
||||
frames += 1
|
||||
bytes += totalLength
|
||||
this.onFrame({
|
||||
type: framed[0],
|
||||
id: framed.readUInt32BE(1),
|
||||
ack: framed.readUInt32BE(5),
|
||||
payload: framed.subarray(HEADER_LENGTH, totalLength)
|
||||
})
|
||||
}
|
||||
} finally {
|
||||
this.draining = false
|
||||
}
|
||||
|
||||
if (generation !== this.generation) {
|
||||
return
|
||||
}
|
||||
if (this.hasRunnableWork()) {
|
||||
this.scheduleContinuation()
|
||||
} else {
|
||||
this.releasePause()
|
||||
}
|
||||
}
|
||||
|
||||
private discardOversizedPayload(bytes: number): number {
|
||||
if (this.oversizedPayloadBytesRemaining === 0 || this.buffer.length === 0) {
|
||||
return 0
|
||||
}
|
||||
const discarded = Math.min(
|
||||
this.oversizedPayloadBytesRemaining,
|
||||
this.buffer.length,
|
||||
Math.max(1, this.maxBytesPerTurn - bytes)
|
||||
)
|
||||
this.buffer.discard(discarded)
|
||||
this.oversizedPayloadBytesRemaining -= discarded
|
||||
return discarded
|
||||
}
|
||||
|
||||
private hasRunnableWork(): boolean {
|
||||
if (this.oversizedPayloadBytesRemaining > 0) {
|
||||
return this.buffer.length > 0
|
||||
}
|
||||
if (this.buffer.length < HEADER_LENGTH) {
|
||||
return false
|
||||
}
|
||||
const length = this.buffer.peek(HEADER_LENGTH).readUInt32BE(9)
|
||||
return length > MAX_MESSAGE_SIZE || this.buffer.length >= HEADER_LENGTH + length
|
||||
}
|
||||
|
||||
private scheduleContinuation(): void {
|
||||
if (this.continuationScheduled) {
|
||||
return
|
||||
}
|
||||
const generation = this.generation
|
||||
this.continuationScheduled = true
|
||||
try {
|
||||
this.acquirePause()
|
||||
} catch (error) {
|
||||
this.continuationScheduled = false
|
||||
throw error
|
||||
}
|
||||
if (generation !== this.generation) {
|
||||
this.continuationScheduled = false
|
||||
return
|
||||
}
|
||||
try {
|
||||
this.continuationHandle = this.schedule(() => {
|
||||
if (!this.continuationScheduled || generation !== this.generation) {
|
||||
return
|
||||
}
|
||||
this.continuationScheduled = false
|
||||
this.continuationHandleAssigned = false
|
||||
this.continuationHandle = undefined
|
||||
try {
|
||||
this.drainTurn()
|
||||
} catch (error) {
|
||||
containFrameDecoderContinuation(() => this.reset(), this.onError, error)
|
||||
}
|
||||
})
|
||||
this.continuationHandleAssigned = true
|
||||
} catch (error) {
|
||||
this.continuationScheduled = false
|
||||
this.continuationHandle = undefined
|
||||
this.releasePause()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
private cancelContinuation(): void {
|
||||
if (!this.continuationScheduled) {
|
||||
return
|
||||
}
|
||||
this.continuationScheduled = false
|
||||
if (this.continuationHandleAssigned) {
|
||||
this.cancelScheduled(this.continuationHandle)
|
||||
}
|
||||
this.continuationHandleAssigned = false
|
||||
this.continuationHandle = undefined
|
||||
}
|
||||
|
||||
private acquirePause(): void {
|
||||
if (!this.paused) {
|
||||
this.paused = true
|
||||
try {
|
||||
this.pause?.()
|
||||
} catch (error) {
|
||||
this.paused = false
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private releasePause(): void {
|
||||
if (this.paused) {
|
||||
this.paused = false
|
||||
this.resume?.()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const positiveLimit = (value: number | undefined, fallback: number): number =>
|
||||
value !== undefined && Number.isFinite(value) && value > 0 ? value : fallback
|
||||
@@ -0,0 +1,263 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
FrameDecoder,
|
||||
FrameDecoderContinuationError,
|
||||
FRAME_DECODER_MAX_RETAINED_BYTES,
|
||||
HEADER_LENGTH,
|
||||
MAX_MESSAGE_SIZE,
|
||||
MessageType,
|
||||
encodeFrame,
|
||||
type DecodedFrame
|
||||
} from './relay-protocol'
|
||||
|
||||
function createScheduler(): {
|
||||
schedule: (callback: () => void) => number
|
||||
cancel: (handle: unknown) => void
|
||||
runNext: () => void
|
||||
pending: () => number
|
||||
} {
|
||||
let nextHandle = 1
|
||||
const callbacks = new Map<number, () => void>()
|
||||
return {
|
||||
schedule: (callback) => {
|
||||
const handle = nextHandle++
|
||||
callbacks.set(handle, callback)
|
||||
return handle
|
||||
},
|
||||
cancel: (handle) => callbacks.delete(handle as number),
|
||||
runNext: () => {
|
||||
const entry = callbacks.entries().next().value as [number, () => void] | undefined
|
||||
if (!entry) {
|
||||
throw new Error('No decoder continuation scheduled')
|
||||
}
|
||||
callbacks.delete(entry[0])
|
||||
entry[1]()
|
||||
},
|
||||
pending: () => callbacks.size
|
||||
}
|
||||
}
|
||||
|
||||
function frame(id: number, payload = `${id}`): Buffer {
|
||||
return encodeFrame(MessageType.Regular, id, 0, Buffer.from(payload))
|
||||
}
|
||||
|
||||
describe('FrameDecoder bounded turns', () => {
|
||||
it('retains at most one maximum frame plus one MiB of partial input', () => {
|
||||
expect(FRAME_DECODER_MAX_RETAINED_BYTES).toBe(MAX_MESSAGE_SIZE + HEADER_LENGTH + 1024 * 1024)
|
||||
const maximumFrame = encodeFrame(MessageType.Regular, 1, 0, Buffer.alloc(MAX_MESSAGE_SIZE))
|
||||
const acceptedError = vi.fn()
|
||||
const accepted = new FrameDecoder(vi.fn(), acceptedError)
|
||||
|
||||
accepted.feed(Buffer.concat([maximumFrame, Buffer.alloc(1024 * 1024)]))
|
||||
|
||||
expect(acceptedError).not.toHaveBeenCalled()
|
||||
expect(accepted.drain()).toHaveLength(1024 * 1024)
|
||||
|
||||
const excessError = vi.fn()
|
||||
const excess = new FrameDecoder(vi.fn(), excessError)
|
||||
excess.feed(Buffer.concat([maximumFrame, Buffer.alloc(1024 * 1024 + 1)]))
|
||||
|
||||
expect(excessError).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ message: expect.stringContaining('retained-input') })
|
||||
)
|
||||
expect(excess.drain()).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('fails closed when one delivered chunk exceeds retained-input capacity', () => {
|
||||
const onError = vi.fn()
|
||||
const decoder = new FrameDecoder(vi.fn(), onError)
|
||||
|
||||
decoder.feed(Buffer.alloc(FRAME_DECODER_MAX_RETAINED_BYTES + 1))
|
||||
|
||||
expect(onError).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ message: expect.stringContaining('retained-input') })
|
||||
)
|
||||
expect(decoder.drain()).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('emits the first frame synchronously and preserves order through self-pause', () => {
|
||||
const scheduler = createScheduler()
|
||||
const seen: number[] = []
|
||||
let decoder: FrameDecoder
|
||||
const pause = vi.fn(() => decoder.feed(frame(4)))
|
||||
const resume = vi.fn()
|
||||
decoder = new FrameDecoder((decoded) => seen.push(decoded.id), undefined, {
|
||||
maxFramesPerTurn: 1,
|
||||
schedule: scheduler.schedule,
|
||||
cancelScheduled: scheduler.cancel,
|
||||
pause,
|
||||
resume
|
||||
})
|
||||
|
||||
decoder.feed(Buffer.concat([frame(1), frame(2), frame(3)]))
|
||||
|
||||
expect(seen).toEqual([1])
|
||||
expect(pause).toHaveBeenCalledTimes(1)
|
||||
expect(scheduler.pending()).toBe(1)
|
||||
|
||||
scheduler.runNext()
|
||||
scheduler.runNext()
|
||||
scheduler.runNext()
|
||||
|
||||
expect(seen).toEqual([1, 2, 3, 4])
|
||||
expect(pause).toHaveBeenCalledTimes(1)
|
||||
expect(resume).toHaveBeenCalledTimes(1)
|
||||
expect(scheduler.pending()).toBe(0)
|
||||
})
|
||||
|
||||
it('bounds decoded bytes and time independently from the frame count', () => {
|
||||
const byteScheduler = createScheduler()
|
||||
const byteSeen: number[] = []
|
||||
const first = frame(1, 'one')
|
||||
const second = frame(2, 'two')
|
||||
const byteDecoder = new FrameDecoder((decoded) => byteSeen.push(decoded.id), undefined, {
|
||||
maxFramesPerTurn: 64,
|
||||
maxBytesPerTurn: first.length,
|
||||
schedule: byteScheduler.schedule,
|
||||
cancelScheduled: byteScheduler.cancel
|
||||
})
|
||||
|
||||
byteDecoder.feed(Buffer.concat([first, second]))
|
||||
expect(byteSeen).toEqual([1])
|
||||
byteScheduler.runNext()
|
||||
expect(byteSeen).toEqual([1, 2])
|
||||
|
||||
const timeScheduler = createScheduler()
|
||||
const timeSeen: number[] = []
|
||||
let nowCalls = 0
|
||||
const timeDecoder = new FrameDecoder((decoded) => timeSeen.push(decoded.id), undefined, {
|
||||
maxFramesPerTurn: 64,
|
||||
maxBytesPerTurn: MAX_MESSAGE_SIZE + HEADER_LENGTH,
|
||||
maxTurnMs: 4,
|
||||
now: () => (nowCalls++ === 0 ? 0 : 5),
|
||||
schedule: timeScheduler.schedule,
|
||||
cancelScheduled: timeScheduler.cancel
|
||||
})
|
||||
|
||||
timeDecoder.feed(Buffer.concat([frame(3), frame(4)]))
|
||||
expect(timeSeen).toEqual([3])
|
||||
timeScheduler.runNext()
|
||||
expect(timeSeen).toEqual([3, 4])
|
||||
})
|
||||
|
||||
it('releases its pause epoch when continuation scheduling throws', () => {
|
||||
const pause = vi.fn()
|
||||
const resume = vi.fn()
|
||||
const decoder = new FrameDecoder(() => {}, undefined, {
|
||||
maxFramesPerTurn: 1,
|
||||
pause,
|
||||
resume,
|
||||
schedule: () => {
|
||||
throw new Error('scheduler unavailable')
|
||||
}
|
||||
})
|
||||
|
||||
expect(() => decoder.feed(Buffer.concat([frame(1), frame(2)]))).toThrow('scheduler unavailable')
|
||||
expect(pause).toHaveBeenCalledTimes(1)
|
||||
expect(resume).toHaveBeenCalledTimes(1)
|
||||
decoder.reset()
|
||||
expect(resume).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('contains a throwing continuation, resets residue, and reports one typed error', () => {
|
||||
const scheduler = createScheduler()
|
||||
const seen: number[] = []
|
||||
const onError = vi.fn()
|
||||
const pause = vi.fn()
|
||||
const resume = vi.fn()
|
||||
const decoder = new FrameDecoder(
|
||||
(decoded) => {
|
||||
if (decoded.id === 2) {
|
||||
throw new Error('frame owner failed')
|
||||
}
|
||||
seen.push(decoded.id)
|
||||
},
|
||||
onError,
|
||||
{
|
||||
maxFramesPerTurn: 1,
|
||||
schedule: scheduler.schedule,
|
||||
cancelScheduled: scheduler.cancel,
|
||||
pause,
|
||||
resume
|
||||
}
|
||||
)
|
||||
|
||||
decoder.feed(Buffer.concat([frame(1), frame(2), frame(3)]))
|
||||
expect(() => scheduler.runNext()).not.toThrow()
|
||||
|
||||
expect(seen).toEqual([1])
|
||||
expect(onError).toHaveBeenCalledExactlyOnceWith(expect.any(FrameDecoderContinuationError))
|
||||
expect(onError.mock.calls[0]?.[0]).toMatchObject({
|
||||
name: 'FrameDecoderContinuationError',
|
||||
cause: expect.objectContaining({ message: 'frame owner failed' })
|
||||
})
|
||||
expect(pause).toHaveBeenCalledTimes(1)
|
||||
expect(resume).toHaveBeenCalledTimes(1)
|
||||
expect(scheduler.pending()).toBe(0)
|
||||
expect(decoder.drain()).toHaveLength(0)
|
||||
|
||||
decoder.feed(frame(4))
|
||||
expect(seen).toEqual([1, 4])
|
||||
})
|
||||
|
||||
it('keeps reads active for partial frames and incrementally discards oversized payloads', () => {
|
||||
const errors: Error[] = []
|
||||
const seen: DecodedFrame[] = []
|
||||
const pause = vi.fn()
|
||||
const decoder = new FrameDecoder(
|
||||
(decoded) => seen.push(decoded),
|
||||
(error) => errors.push(error),
|
||||
{ pause }
|
||||
)
|
||||
const valid = frame(2, 'complete')
|
||||
|
||||
decoder.feed(valid.subarray(0, HEADER_LENGTH + 2))
|
||||
expect(seen).toHaveLength(0)
|
||||
expect(pause).not.toHaveBeenCalled()
|
||||
decoder.feed(valid.subarray(HEADER_LENGTH + 2))
|
||||
expect(seen.map(({ id }) => id)).toEqual([2])
|
||||
|
||||
const oversizedHeader = Buffer.alloc(HEADER_LENGTH)
|
||||
oversizedHeader[0] = MessageType.Regular
|
||||
oversizedHeader.writeUInt32BE(3, 1)
|
||||
oversizedHeader.writeUInt32BE(MAX_MESSAGE_SIZE + 1, 9)
|
||||
decoder.feed(Buffer.concat([oversizedHeader, Buffer.alloc(32)]))
|
||||
|
||||
expect(errors).toHaveLength(1)
|
||||
expect(pause).not.toHaveBeenCalled()
|
||||
decoder.reset()
|
||||
decoder.feed(frame(4, 'after-reset'))
|
||||
expect(seen.map(({ id }) => id)).toEqual([2, 4])
|
||||
})
|
||||
|
||||
it('drain and reset cancel continuation ownership without replaying residue', () => {
|
||||
const scheduler = createScheduler()
|
||||
const seen: number[] = []
|
||||
const resume = vi.fn()
|
||||
const cancel = vi.fn(scheduler.cancel)
|
||||
const decoder = new FrameDecoder((decoded) => seen.push(decoded.id), undefined, {
|
||||
maxFramesPerTurn: 1,
|
||||
schedule: scheduler.schedule,
|
||||
cancelScheduled: cancel,
|
||||
resume
|
||||
})
|
||||
const second = frame(2, 'residue')
|
||||
|
||||
decoder.feed(Buffer.concat([frame(1), second]))
|
||||
const residue = decoder.drain()
|
||||
|
||||
expect(seen).toEqual([1])
|
||||
expect(residue.equals(second)).toBe(true)
|
||||
expect(cancel).toHaveBeenCalledTimes(1)
|
||||
expect(resume).toHaveBeenCalledTimes(1)
|
||||
expect(scheduler.pending()).toBe(0)
|
||||
|
||||
decoder.feed(Buffer.concat([frame(3), frame(4)]))
|
||||
decoder.reset()
|
||||
expect(cancel).toHaveBeenCalledTimes(2)
|
||||
expect(scheduler.pending()).toBe(0)
|
||||
|
||||
decoder.feed(frame(5))
|
||||
expect(seen).toEqual([1, 3, 5])
|
||||
})
|
||||
})
|
||||
+22
-147
@@ -3,17 +3,34 @@
|
||||
// See design-ssh-support.md § JSON-RPC Protocol Specification.
|
||||
|
||||
import { DEFAULT_SSH_RELAY_GRACE_PERIOD_SECONDS } from '../../shared/ssh-types'
|
||||
import {
|
||||
FrameDecoder,
|
||||
FrameDecoderContinuationError,
|
||||
HEADER_LENGTH,
|
||||
MAX_MESSAGE_SIZE,
|
||||
FRAME_DECODER_MAX_FRAMES_PER_TURN,
|
||||
FRAME_DECODER_MAX_BYTES_PER_TURN,
|
||||
FRAME_DECODER_MAX_TURN_MS,
|
||||
FRAME_DECODER_MAX_RETAINED_BYTES
|
||||
} from './relay-frame-decoder'
|
||||
|
||||
export {
|
||||
FrameDecoder,
|
||||
FrameDecoderContinuationError,
|
||||
HEADER_LENGTH,
|
||||
MAX_MESSAGE_SIZE,
|
||||
FRAME_DECODER_MAX_FRAMES_PER_TURN,
|
||||
FRAME_DECODER_MAX_BYTES_PER_TURN,
|
||||
FRAME_DECODER_MAX_TURN_MS,
|
||||
FRAME_DECODER_MAX_RETAINED_BYTES
|
||||
}
|
||||
export type { DecodedFrame, FrameDecoderOptions } from './relay-frame-decoder'
|
||||
|
||||
export const RELAY_VERSION = '0.1.0'
|
||||
export const RELAY_SENTINEL = `ORCA-RELAY v${RELAY_VERSION} READY\n`
|
||||
export const RELAY_SENTINEL_TIMEOUT_MS = 10_000
|
||||
export const RELAY_REMOTE_DIR = '.orca-remote'
|
||||
|
||||
// ── Framing constants (VS Code ProtocolConstants) ───────────────────
|
||||
|
||||
export const HEADER_LENGTH = 13
|
||||
export const MAX_MESSAGE_SIZE = 16 * 1024 * 1024 // 16 MB
|
||||
|
||||
/** Message type byte. */
|
||||
export const MessageType = {
|
||||
Regular: 1,
|
||||
@@ -155,148 +172,6 @@ export function encodeKeepAliveFrame(id: number, ack: number): Buffer {
|
||||
return encodeFrame(MessageType.KeepAlive, id, ack, Buffer.alloc(0))
|
||||
}
|
||||
|
||||
export type DecodedFrame = {
|
||||
type: number
|
||||
id: number
|
||||
ack: number
|
||||
payload: Buffer
|
||||
}
|
||||
|
||||
/**
|
||||
* Incremental frame parser. Feed it chunks of data; it emits complete frames.
|
||||
*/
|
||||
export class FrameDecoder {
|
||||
// Why: feed() runs on the Electron main thread for every SSH channel data
|
||||
// event. Rebuilding one contiguous buffer per feed (Buffer.concat) re-copies
|
||||
// every already-buffered byte for each incoming ~32KB TCP chunk — O(n²) per
|
||||
// large frame (a 340KB fs.streamChunk frame cost ~2MB of memcpy). A chunk
|
||||
// list assembles each frame exactly once instead.
|
||||
private chunks: Buffer[] = []
|
||||
private bufferedLength = 0
|
||||
private onFrame: (frame: DecodedFrame) => void
|
||||
private onError: ((err: Error) => void) | null
|
||||
|
||||
constructor(onFrame: (frame: DecodedFrame) => void, onError?: (err: Error) => void) {
|
||||
this.onFrame = onFrame
|
||||
this.onError = onError ?? null
|
||||
}
|
||||
|
||||
feed(chunk: Buffer | Uint8Array): void {
|
||||
const buf = Buffer.isBuffer(chunk)
|
||||
? chunk
|
||||
: Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength)
|
||||
if (buf.length > 0) {
|
||||
this.chunks.push(buf)
|
||||
this.bufferedLength += buf.length
|
||||
}
|
||||
|
||||
while (this.bufferedLength >= HEADER_LENGTH) {
|
||||
const header = this.peekBytes(HEADER_LENGTH)
|
||||
const length = header.readUInt32BE(9)
|
||||
const totalLength = HEADER_LENGTH + length
|
||||
|
||||
if (this.bufferedLength < totalLength) {
|
||||
// Not fully received yet (also holds oversized frames until they can
|
||||
// be skipped whole, keeping the decoder synchronized).
|
||||
break
|
||||
}
|
||||
|
||||
// Why: throwing here would leave the buffer in a partially consumed
|
||||
// state — subsequent feed() calls would try to parse leftover payload
|
||||
// bytes as a new header, corrupting every future frame. Instead we
|
||||
// skip the entire oversized frame so the decoder stays synchronized.
|
||||
if (length > MAX_MESSAGE_SIZE) {
|
||||
this.discardBytes(totalLength)
|
||||
const err = new Error(`Frame payload too large: ${length} bytes — discarded`)
|
||||
if (this.onError) {
|
||||
this.onError(err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
const framed = this.takeBytes(totalLength)
|
||||
const frame: DecodedFrame = {
|
||||
type: framed[0],
|
||||
id: framed.readUInt32BE(1),
|
||||
ack: framed.readUInt32BE(5),
|
||||
payload: framed.subarray(HEADER_LENGTH, totalLength)
|
||||
}
|
||||
this.onFrame(frame)
|
||||
}
|
||||
}
|
||||
|
||||
reset(): void {
|
||||
this.chunks = []
|
||||
this.bufferedLength = 0
|
||||
}
|
||||
|
||||
/** View of the first `count` buffered bytes without consuming them. */
|
||||
private peekBytes(count: number): Buffer {
|
||||
const first = this.chunks[0]
|
||||
if (first.length >= count) {
|
||||
return first
|
||||
}
|
||||
const out = Buffer.allocUnsafe(count)
|
||||
let copied = 0
|
||||
for (const part of this.chunks) {
|
||||
copied += part.copy(out, copied, 0, Math.min(part.length, count - copied))
|
||||
if (copied >= count) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** Consume and return the first `count` buffered bytes (single copy). */
|
||||
private takeBytes(count: number): Buffer {
|
||||
const first = this.chunks[0]
|
||||
if (first.length === count) {
|
||||
this.chunks.shift()
|
||||
this.bufferedLength -= count
|
||||
return first
|
||||
}
|
||||
if (first.length > count) {
|
||||
this.chunks[0] = first.subarray(count)
|
||||
this.bufferedLength -= count
|
||||
return first.subarray(0, count)
|
||||
}
|
||||
const out = Buffer.allocUnsafe(count)
|
||||
let copied = 0
|
||||
while (copied < count) {
|
||||
const part = this.chunks[0]
|
||||
const take = Math.min(part.length, count - copied)
|
||||
part.copy(out, copied, 0, take)
|
||||
copied += take
|
||||
if (take === part.length) {
|
||||
this.chunks.shift()
|
||||
} else {
|
||||
this.chunks[0] = part.subarray(take)
|
||||
}
|
||||
}
|
||||
this.bufferedLength -= count
|
||||
return out
|
||||
}
|
||||
|
||||
/** Consume the first `count` buffered bytes without assembling them. */
|
||||
private discardBytes(count: number): void {
|
||||
let remaining = count
|
||||
while (remaining > 0) {
|
||||
const part = this.chunks[0]
|
||||
if (part.length <= remaining) {
|
||||
this.chunks.shift()
|
||||
remaining -= part.length
|
||||
} else {
|
||||
this.chunks[0] = part.subarray(remaining)
|
||||
remaining = 0
|
||||
}
|
||||
}
|
||||
this.bufferedLength -= count
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a JSON-RPC message from a frame payload.
|
||||
*/
|
||||
export function parseJsonRpcMessage(payload: Buffer): JsonRpcMessage {
|
||||
const text = payload.toString('utf-8')
|
||||
const msg = JSON.parse(text) as JsonRpcMessage
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { SshChannelMultiplexer, type MultiplexerTransport } from './ssh-channel-multiplexer'
|
||||
import { encodeFrame, encodeKeepAliveFrame, HEADER_LENGTH, MessageType } from './relay-protocol'
|
||||
|
||||
type MockTransport = MultiplexerTransport & {
|
||||
data: (chunk: Buffer) => void
|
||||
written: Buffer[]
|
||||
pauseReads: ReturnType<typeof vi.fn>
|
||||
resumeReads: ReturnType<typeof vi.fn>
|
||||
}
|
||||
|
||||
type MuxInternals = {
|
||||
nextOutgoingSeq: number
|
||||
highestAckedBySelf: number
|
||||
lastReceivedAt: number
|
||||
decoderReadPaused: boolean
|
||||
unackedTimestamps: Map<number, number>
|
||||
}
|
||||
|
||||
function createTransport(): MockTransport {
|
||||
let onData: (chunk: Buffer) => void = () => {}
|
||||
const written: Buffer[] = []
|
||||
const pauseReads = vi.fn()
|
||||
const resumeReads = vi.fn()
|
||||
return {
|
||||
write: (data) => {
|
||||
written.push(data)
|
||||
},
|
||||
onData: (callback) => {
|
||||
onData = callback
|
||||
},
|
||||
onClose: () => {},
|
||||
pauseReads,
|
||||
resumeReads,
|
||||
data: (chunk) => onData(chunk),
|
||||
written
|
||||
}
|
||||
}
|
||||
|
||||
function internals(mux: SshChannelMultiplexer): MuxInternals {
|
||||
return mux as unknown as MuxInternals
|
||||
}
|
||||
|
||||
describe('SshChannelMultiplexer backpressure hardening', () => {
|
||||
let transport: MockTransport
|
||||
let mux: SshChannelMultiplexer
|
||||
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers()
|
||||
vi.setSystemTime(0)
|
||||
transport = createTransport()
|
||||
mux = new SshChannelMultiplexer(transport)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
mux.dispose()
|
||||
vi.restoreAllMocks()
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('clamps ack=0xffffffff and deletes only retained sequence keys', () => {
|
||||
mux.notify('one')
|
||||
mux.notify('two')
|
||||
const state = internals(mux)
|
||||
const deleteSpy = vi.spyOn(state.unackedTimestamps, 'delete')
|
||||
|
||||
transport.data(encodeKeepAliveFrame(1, 0xffffffff))
|
||||
|
||||
expect(deleteSpy).toHaveBeenCalledTimes(2)
|
||||
expect(state.unackedTimestamps.size).toBe(0)
|
||||
expect(state.highestAckedBySelf).toBe(state.nextOutgoingSeq - 1)
|
||||
})
|
||||
|
||||
it('caps timestamps while preserving a reserved liveness entry', () => {
|
||||
for (let i = 0; i < 5000; i += 1) {
|
||||
mux.notify('bounded')
|
||||
}
|
||||
const state = internals(mux)
|
||||
|
||||
expect(state.unackedTimestamps.size).toBe(4095)
|
||||
expect(state.unackedTimestamps.has(1)).toBe(true)
|
||||
void mux.probeLiveness(60_000)
|
||||
expect(state.unackedTimestamps.size).toBe(4096)
|
||||
void mux.probeLiveness(60_000)
|
||||
expect(state.unackedTimestamps.size).toBe(4096)
|
||||
})
|
||||
|
||||
it('suppresses timeout while self-paused and rebases both clocks on resume', () => {
|
||||
mux.dispose()
|
||||
const continuations: (() => void)[] = []
|
||||
vi.spyOn(globalThis, 'setImmediate').mockImplementation(
|
||||
(callback: (...args: never[]) => void) => {
|
||||
continuations.push(callback)
|
||||
return {} as NodeJS.Immediate
|
||||
}
|
||||
)
|
||||
transport = createTransport()
|
||||
mux = new SshChannelMultiplexer(transport)
|
||||
mux.notify('tracked')
|
||||
|
||||
const incoming = Buffer.concat(
|
||||
Array.from({ length: 65 }, (_, index) => encodeKeepAliveFrame(index + 1, 0))
|
||||
)
|
||||
transport.data(incoming)
|
||||
const state = internals(mux)
|
||||
|
||||
expect(state.decoderReadPaused).toBe(true)
|
||||
expect(transport.pauseReads).toHaveBeenCalledTimes(1)
|
||||
expect(continuations).toHaveLength(1)
|
||||
|
||||
vi.advanceTimersByTime(25_000)
|
||||
expect(mux.isDisposed()).toBe(false)
|
||||
|
||||
continuations.shift()!()
|
||||
const resumedAt = Date.now()
|
||||
expect(state.decoderReadPaused).toBe(false)
|
||||
expect(transport.resumeReads).toHaveBeenCalledTimes(1)
|
||||
expect(state.lastReceivedAt).toBe(resumedAt)
|
||||
expect(new Set(state.unackedTimestamps.values())).toEqual(new Set([resumedAt]))
|
||||
})
|
||||
|
||||
it('keeps frame dispatch ordered across a decoder continuation', () => {
|
||||
const seen: number[] = []
|
||||
mux.onNotification((_method, params) => seen.push(params.index as number))
|
||||
const frames = Array.from({ length: 65 }, (_, index) => {
|
||||
const payload = Buffer.from(
|
||||
JSON.stringify({ jsonrpc: '2.0', method: 'ordered', params: { index } })
|
||||
)
|
||||
return encodeFrame(MessageType.Regular, index + 1, 0, payload)
|
||||
})
|
||||
|
||||
transport.data(Buffer.concat(frames))
|
||||
expect(seen).toEqual(Array.from({ length: 64 }, (_, index) => index))
|
||||
vi.runAllTicks()
|
||||
vi.advanceTimersByTime(0)
|
||||
expect(seen).toEqual(Array.from({ length: 65 }, (_, index) => index))
|
||||
})
|
||||
|
||||
it('prioritizes source ACK and control while preserving FIFO and ordinary progress', async () => {
|
||||
mux.dispose()
|
||||
const written: Buffer[] = []
|
||||
let drain = (): void => {}
|
||||
let deliver = (_data: Buffer): void => {}
|
||||
transport = {
|
||||
write: (data) => {
|
||||
written.push(data)
|
||||
return written.length !== 1
|
||||
},
|
||||
onDrain: (callback) => {
|
||||
drain = callback
|
||||
},
|
||||
onData: (callback) => {
|
||||
deliver = callback
|
||||
},
|
||||
onClose: vi.fn(),
|
||||
pauseReads: vi.fn<() => void>(),
|
||||
resumeReads: vi.fn<() => void>(),
|
||||
data: vi.fn<(chunk: Buffer) => void>(),
|
||||
written,
|
||||
close: vi.fn()
|
||||
}
|
||||
mux = new SshChannelMultiplexer(transport)
|
||||
mux.onRequest('client.control', () => ({ accepted: true }))
|
||||
const controller = new AbortController()
|
||||
|
||||
mux.notify('pty.data', { id: 'pty-1', data: 'ordinary-1' })
|
||||
mux.notify('pty.data', { id: 'pty-1', data: 'ordinary-2' })
|
||||
mux.notify('pty.data', { id: 'pty-1', data: 'ordinary-3' })
|
||||
mux.notify('pty.ackData', { acknowledgements: [] })
|
||||
const request = mux.request('fs.scan', {}, { signal: controller.signal })
|
||||
controller.abort()
|
||||
mux.notify('pty.exit', { id: 'pty-1', code: 0 })
|
||||
const remoteRequest = Buffer.from(
|
||||
JSON.stringify({ jsonrpc: '2.0', id: 91, method: 'client.control' })
|
||||
)
|
||||
deliver(encodeFrame(MessageType.Regular, 1, 0, remoteRequest))
|
||||
await Promise.resolve()
|
||||
await Promise.resolve()
|
||||
|
||||
expect(written).toHaveLength(1)
|
||||
drain()
|
||||
const payloads = written.map((frame) =>
|
||||
JSON.parse(frame.subarray(HEADER_LENGTH, HEADER_LENGTH + frame.readUInt32BE(9)).toString())
|
||||
)
|
||||
expect(
|
||||
payloads.map((payload) =>
|
||||
payload.method === 'pty.data'
|
||||
? `pty.data:${payload.params.data}`
|
||||
: (payload.method ?? `response:${payload.id}`)
|
||||
)
|
||||
).toEqual([
|
||||
'pty.data:ordinary-1',
|
||||
'pty.ackData',
|
||||
'fs.scan',
|
||||
'rpc.cancel',
|
||||
'pty.exit',
|
||||
'pty.data:ordinary-2',
|
||||
'response:91',
|
||||
'pty.data:ordinary-3'
|
||||
])
|
||||
await expect(request).rejects.toMatchObject({ name: 'AbortError' })
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,93 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { SshChannelMultiplexer, type MultiplexerTransport } from './ssh-channel-multiplexer'
|
||||
|
||||
function transportHarness(): {
|
||||
transport: MultiplexerTransport
|
||||
settlements: ((result: { ok: true } | { ok: false; error: Error }) => void)[]
|
||||
} {
|
||||
const settlements: ((result: { ok: true } | { ok: false; error: Error }) => void)[] = []
|
||||
return {
|
||||
transport: {
|
||||
write: (_data, onSettled) => {
|
||||
if (onSettled) {
|
||||
settlements.push(onSettled)
|
||||
}
|
||||
},
|
||||
supportsWriteSettlement: true,
|
||||
onData: vi.fn(),
|
||||
onClose: vi.fn()
|
||||
},
|
||||
settlements
|
||||
}
|
||||
}
|
||||
|
||||
describe('SshChannelMultiplexer notification settlement', () => {
|
||||
it('reports publication only from the transport write callback', () => {
|
||||
const harness = transportHarness()
|
||||
const mux = new SshChannelMultiplexer(harness.transport)
|
||||
const settled = vi.fn()
|
||||
|
||||
mux.notifyWithSettlement('pty.ackData', { acknowledgements: [] }, settled)
|
||||
expect(settled).not.toHaveBeenCalled()
|
||||
harness.settlements[0]({ ok: true })
|
||||
expect(settled).toHaveBeenCalledWith({ ok: true })
|
||||
mux.dispose()
|
||||
})
|
||||
|
||||
it('reports a synchronous write failure without publishing success', () => {
|
||||
const error = new Error('write failed')
|
||||
const mux = new SshChannelMultiplexer({
|
||||
write: () => {
|
||||
throw error
|
||||
},
|
||||
supportsWriteSettlement: true,
|
||||
onData: vi.fn(),
|
||||
onClose: vi.fn()
|
||||
})
|
||||
const settled = vi.fn()
|
||||
|
||||
mux.notifyWithSettlement('pty.ackData', { acknowledgements: [] }, settled)
|
||||
expect(settled).toHaveBeenCalledWith({ ok: false, error })
|
||||
expect(mux.isDisposed()).toBe(true)
|
||||
})
|
||||
|
||||
it('settles once when a hostile transport invokes its callback and then throws', () => {
|
||||
const mux = new SshChannelMultiplexer({
|
||||
write: (_data, onSettled) => {
|
||||
onSettled?.({ ok: true })
|
||||
throw new Error('late throw')
|
||||
},
|
||||
supportsWriteSettlement: true,
|
||||
onData: vi.fn(),
|
||||
onClose: vi.fn()
|
||||
})
|
||||
const settled = vi.fn()
|
||||
|
||||
mux.notifyWithSettlement('pty.ackData', { acknowledgements: [] }, settled)
|
||||
expect(settled).toHaveBeenCalledOnce()
|
||||
expect(settled).toHaveBeenCalledWith({ ok: true })
|
||||
})
|
||||
|
||||
it('fails an unsettled publication when the multiplexer is disposed', () => {
|
||||
const close = vi.fn()
|
||||
const mux = new SshChannelMultiplexer({
|
||||
write: () => false,
|
||||
supportsWriteSettlement: true,
|
||||
onDrain: vi.fn(),
|
||||
onData: vi.fn(),
|
||||
onClose: vi.fn(),
|
||||
close
|
||||
})
|
||||
const settled = vi.fn()
|
||||
|
||||
mux.notifyWithSettlement('pty.ackData', { acknowledgements: [] }, settled)
|
||||
expect(settled).not.toHaveBeenCalled()
|
||||
mux.dispose()
|
||||
|
||||
expect(settled).toHaveBeenCalledWith({
|
||||
ok: false,
|
||||
error: expect.objectContaining({ code: 'DISPOSED' })
|
||||
})
|
||||
expect(close).toHaveBeenCalledOnce()
|
||||
})
|
||||
})
|
||||
@@ -12,7 +12,9 @@ function createMockTransport(): MultiplexerTransport & {
|
||||
const written: Buffer[] = []
|
||||
|
||||
return {
|
||||
write: (data: Buffer) => written.push(data),
|
||||
write: (data: Buffer) => {
|
||||
written.push(data)
|
||||
},
|
||||
onData: (cb) => dataCallbacks.push(cb),
|
||||
onClose: (cb) => closeCallbacks.push(cb),
|
||||
dataCallbacks,
|
||||
@@ -69,6 +71,7 @@ type MuxInternals = {
|
||||
disposeHandlers: unknown[]
|
||||
lastReceivedAt: number
|
||||
unackedTimestamps: Map<number, number>
|
||||
writerSaturated: boolean
|
||||
}
|
||||
|
||||
function getMuxInternals(instance: SshChannelMultiplexer): MuxInternals {
|
||||
@@ -123,6 +126,28 @@ describe('SshChannelMultiplexer', () => {
|
||||
await expect(promise).rejects.toThrow('PTY allocation failed')
|
||||
})
|
||||
|
||||
it('runs beforeResolve before an adjacent notification in the same decoder turn', async () => {
|
||||
const order: string[] = []
|
||||
mux.onNotification(() => order.push('notification'))
|
||||
const promise = mux.request(
|
||||
'pty.attach',
|
||||
{ id: 'pty-1' },
|
||||
{
|
||||
beforeResolve: () => order.push('beforeResolve')
|
||||
}
|
||||
)
|
||||
|
||||
transport.dataCallbacks[0](
|
||||
Buffer.concat([
|
||||
makeResponseFrame(1, { incarnationId: 'incarnation-1' }, 1),
|
||||
makeNotificationFrame('pty.data', { id: 'pty-1', data: 'first' }, 2)
|
||||
])
|
||||
)
|
||||
|
||||
expect(order).toEqual(['beforeResolve', 'notification'])
|
||||
await expect(promise).resolves.toEqual({ incarnationId: 'incarnation-1' })
|
||||
})
|
||||
|
||||
it('times out after 30s with no response', async () => {
|
||||
const promise = mux.request('pty.spawn')
|
||||
|
||||
@@ -328,6 +353,43 @@ describe('SshChannelMultiplexer', () => {
|
||||
vi.advanceTimersByTime(25_000)
|
||||
expect(mux.isDisposed()).toBe(true)
|
||||
})
|
||||
|
||||
it('suppresses false death while locally saturated and rebases both clocks on drain', () => {
|
||||
mux.dispose()
|
||||
let drain = (): void => {}
|
||||
const written: Buffer[] = []
|
||||
const saturatedTransport: MultiplexerTransport = {
|
||||
write: (data) => {
|
||||
written.push(data)
|
||||
return false
|
||||
},
|
||||
supportsWriteSettlement: true,
|
||||
onDrain: (callback) => {
|
||||
drain = callback
|
||||
},
|
||||
onData: vi.fn(),
|
||||
onClose: vi.fn()
|
||||
}
|
||||
mux = new SshChannelMultiplexer(saturatedTransport)
|
||||
|
||||
vi.advanceTimersByTime(5_000)
|
||||
expect(getMuxInternals(mux).writerSaturated).toBe(true)
|
||||
vi.advanceTimersByTime(25_000)
|
||||
expect(mux.isDisposed()).toBe(false)
|
||||
expect(written).toHaveLength(1)
|
||||
|
||||
drain()
|
||||
const resumedAt = Date.now()
|
||||
const internals = getMuxInternals(mux)
|
||||
expect(internals.writerSaturated).toBe(false)
|
||||
expect(internals.lastReceivedAt).toBe(resumedAt)
|
||||
expect(new Set(internals.unackedTimestamps.values())).toEqual(new Set([resumedAt]))
|
||||
|
||||
vi.advanceTimersByTime(20_000)
|
||||
expect(mux.isDisposed()).toBe(false)
|
||||
vi.advanceTimersByTime(5_000)
|
||||
expect(mux.isDisposed()).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('wake guard (timer pause across system sleep, #7773)', () => {
|
||||
|
||||
@@ -14,26 +14,36 @@ import {
|
||||
type JsonRpcResponse,
|
||||
type JsonRpcNotification
|
||||
} from './relay-protocol'
|
||||
import {
|
||||
SshMultiplexerTransportWriter,
|
||||
type MultiplexerTransport,
|
||||
type MultiplexerWriteSettlement,
|
||||
type MultiplexerWriterLane
|
||||
} from './ssh-multiplexer-transport-writer'
|
||||
|
||||
export type MultiplexerTransport = {
|
||||
write: (data: Buffer) => void
|
||||
onData: (cb: (data: Buffer) => void) => void
|
||||
onClose: (cb: () => void) => void
|
||||
close?: () => void
|
||||
}
|
||||
export type { MultiplexerTransport, MultiplexerWriteSettlement }
|
||||
|
||||
type PendingRequest = {
|
||||
resolve: (result: unknown) => void
|
||||
reject: (error: Error) => void
|
||||
beforeResolve?: (result: unknown) => void
|
||||
timer: ReturnType<typeof setTimeout>
|
||||
cleanup: () => void
|
||||
}
|
||||
|
||||
export type SshMultiplexerRequestOptions = {
|
||||
signal?: AbortSignal
|
||||
timeoutMs?: number
|
||||
beforeResolve?: (result: unknown) => void
|
||||
}
|
||||
|
||||
export type NotificationHandler = (method: string, params: Record<string, unknown>) => void
|
||||
export type MethodNotificationHandler = (params: Record<string, unknown>) => void
|
||||
export type RequestHandler = (params: Record<string, unknown>) => Promise<unknown> | unknown
|
||||
|
||||
const REQUEST_TIMEOUT_MS = 30_000
|
||||
const MAX_ORDINARY_UNACKED_TIMESTAMPS = 4095
|
||||
const MAX_UNACKED_TIMESTAMPS = MAX_ORDINARY_UNACKED_TIMESTAMPS + 1
|
||||
// Why: a tick gap far beyond the interval means the process was paused
|
||||
// (system sleep, App Nap timer throttling) — not that the link is dead (#7773).
|
||||
const WAKE_GAP_MS = KEEPALIVE_SEND_MS * 3
|
||||
@@ -41,6 +51,7 @@ const WAKE_GAP_MS = KEEPALIVE_SEND_MS * 3
|
||||
export class SshChannelMultiplexer {
|
||||
private decoder: FrameDecoder
|
||||
private transport: MultiplexerTransport
|
||||
private writer: SshMultiplexerTransportWriter
|
||||
private nextRequestId = 1
|
||||
private nextOutgoingSeq = 1
|
||||
private highestReceivedSeq = 0
|
||||
@@ -56,6 +67,8 @@ export class SshChannelMultiplexer {
|
||||
private disposeHandlers: ((reason: 'shutdown' | 'connection_lost') => void)[] = []
|
||||
private connectionHealthTimer: ReturnType<typeof setInterval> | null = null
|
||||
private disposed = false
|
||||
private decoderReadPaused = false
|
||||
private writerSaturated = false
|
||||
|
||||
// Track the oldest unacked outgoing message timestamp
|
||||
private unackedTimestamps = new Map<number, number>()
|
||||
@@ -66,10 +79,19 @@ export class SshChannelMultiplexer {
|
||||
|
||||
constructor(transport: MultiplexerTransport) {
|
||||
this.transport = transport
|
||||
this.writer = new SshMultiplexerTransportWriter(
|
||||
transport,
|
||||
(error) => this.handleProtocolError(error),
|
||||
(saturated) => this.handleWriterSaturationChange(saturated)
|
||||
)
|
||||
|
||||
this.decoder = new FrameDecoder(
|
||||
(frame) => this.handleFrame(frame),
|
||||
(err) => this.handleProtocolError(err)
|
||||
(err) => this.handleProtocolError(err),
|
||||
{
|
||||
pause: () => this.pauseDecoderReads(),
|
||||
resume: () => this.resumeDecoderReads()
|
||||
}
|
||||
)
|
||||
|
||||
transport.onData((data) => {
|
||||
@@ -158,7 +180,7 @@ export class SshChannelMultiplexer {
|
||||
async request(
|
||||
method: string,
|
||||
params?: Record<string, unknown>,
|
||||
options?: { signal?: AbortSignal; timeoutMs?: number }
|
||||
options?: SshMultiplexerRequestOptions
|
||||
): Promise<unknown> {
|
||||
if (this.disposed) {
|
||||
throw new Error('Multiplexer disposed')
|
||||
@@ -215,7 +237,13 @@ export class SshChannelMultiplexer {
|
||||
if (options?.signal) {
|
||||
options.signal.addEventListener('abort', onAbort, { once: true })
|
||||
}
|
||||
this.pendingRequests.set(id, { resolve, reject, timer, cleanup })
|
||||
this.pendingRequests.set(id, {
|
||||
resolve,
|
||||
reject,
|
||||
beforeResolve: options?.beforeResolve,
|
||||
timer,
|
||||
cleanup
|
||||
})
|
||||
this.sendMessage(msg)
|
||||
})
|
||||
}
|
||||
@@ -237,6 +265,25 @@ export class SshChannelMultiplexer {
|
||||
this.sendMessage(msg)
|
||||
}
|
||||
|
||||
notifyWithSettlement(
|
||||
method: string,
|
||||
params: Record<string, unknown> | undefined,
|
||||
onSettled: (result: { ok: true } | { ok: false; error: Error }) => void
|
||||
): void {
|
||||
if (this.disposed) {
|
||||
onSettled({ ok: false, error: new Error('Multiplexer disposed') })
|
||||
return
|
||||
}
|
||||
this.sendMessage(
|
||||
{
|
||||
jsonrpc: '2.0',
|
||||
method,
|
||||
...(params !== undefined ? { params } : {})
|
||||
},
|
||||
onSettled
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a fresh keepalive and resolve true when any frame arrives before the
|
||||
* timeout. Used on system resume to distinguish a link that survived sleep
|
||||
@@ -297,6 +344,9 @@ export class SshChannelMultiplexer {
|
||||
this.pendingRequests.delete(id)
|
||||
}
|
||||
|
||||
const writerError = new Error(errorMessage) as Error & { code: string }
|
||||
writerError.code = errorCode
|
||||
this.writer.dispose(writerError)
|
||||
this.unackedTimestamps.clear()
|
||||
// Why: relay teardown can race with late provider registration; disposed
|
||||
// muxes must not retain provider/session closures through subscribers.
|
||||
@@ -321,34 +371,27 @@ export class SshChannelMultiplexer {
|
||||
|
||||
// ── Private ───────────────────────────────────────────────────────
|
||||
|
||||
private sendMessage(msg: JsonRpcMessage): void {
|
||||
private sendMessage(
|
||||
msg: JsonRpcMessage,
|
||||
onSettled?: (result: MultiplexerWriteSettlement) => void
|
||||
): void {
|
||||
const seq = this.nextOutgoingSeq++
|
||||
const frame = encodeJsonRpcFrame(msg, seq, this.highestReceivedSeq)
|
||||
this.unackedTimestamps.set(seq, Date.now())
|
||||
try {
|
||||
this.transport.write(frame)
|
||||
} catch (err) {
|
||||
// Why: a remote reboot can make the SSH channel's stdin throw EPIPE
|
||||
// from a timer/request path. Scope it to this mux instead of letting
|
||||
// the Electron main process treat it as an uncaught exception.
|
||||
this.handleProtocolError(err)
|
||||
}
|
||||
this.trackOutgoingTimestamp(seq, false)
|
||||
this.writer.enqueue(frame, messageLane(msg), onSettled)
|
||||
}
|
||||
|
||||
private sendKeepAlive(): void {
|
||||
if (this.disposed) {
|
||||
return
|
||||
}
|
||||
const seq = this.nextOutgoingSeq++
|
||||
const seq = this.nextOutgoingSeq
|
||||
const frame = encodeKeepAliveFrame(seq, this.highestReceivedSeq)
|
||||
this.unackedTimestamps.set(seq, Date.now())
|
||||
try {
|
||||
this.transport.write(frame)
|
||||
} catch (err) {
|
||||
// Why: keepalive runs on an interval; without catching transport
|
||||
// write failures here, a dead SSH host can terminate the whole app.
|
||||
this.handleProtocolError(err)
|
||||
if (!this.writer.enqueue(frame, 'liveness')) {
|
||||
return
|
||||
}
|
||||
this.nextOutgoingSeq++
|
||||
this.trackOutgoingTimestamp(seq, true)
|
||||
}
|
||||
|
||||
private handleFrame(frame: DecodedFrame): void {
|
||||
@@ -363,12 +406,16 @@ export class SshChannelMultiplexer {
|
||||
this.highestReceivedSeq = frame.id
|
||||
}
|
||||
|
||||
// Process ack from remote: discard timestamps for acked messages
|
||||
if (frame.ack > this.highestAckedBySelf) {
|
||||
for (let i = this.highestAckedBySelf + 1; i <= frame.ack; i++) {
|
||||
this.unackedTimestamps.delete(i)
|
||||
// Header ACKs are untrusted uint32 values; work stays proportional to the
|
||||
// bounded set of sequence keys we actually retained.
|
||||
const acknowledgedSeq = Math.min(frame.ack, this.nextOutgoingSeq - 1)
|
||||
if (acknowledgedSeq > this.highestAckedBySelf) {
|
||||
for (const seq of this.unackedTimestamps.keys()) {
|
||||
if (seq <= acknowledgedSeq) {
|
||||
this.unackedTimestamps.delete(seq)
|
||||
}
|
||||
}
|
||||
this.highestAckedBySelf = frame.ack
|
||||
this.highestAckedBySelf = acknowledgedSeq
|
||||
}
|
||||
|
||||
if (frame.type === MessageType.KeepAlive) {
|
||||
@@ -440,7 +487,12 @@ export class SshChannelMultiplexer {
|
||||
Object.defineProperty(err, 'data', { value: msg.error.data })
|
||||
pending.reject(err)
|
||||
} else {
|
||||
pending.resolve(msg.result)
|
||||
try {
|
||||
pending.beforeResolve?.(msg.result)
|
||||
pending.resolve(msg.result)
|
||||
} catch (error) {
|
||||
pending.reject(error instanceof Error ? error : new Error(String(error)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -495,15 +547,12 @@ export class SshChannelMultiplexer {
|
||||
// before this tick's fresh probe, then allow the next full window.
|
||||
const resumedAfterWake = sinceLastTick > WAKE_GAP_MS
|
||||
if (resumedAfterWake) {
|
||||
this.lastReceivedAt = now
|
||||
for (const seq of this.unackedTimestamps.keys()) {
|
||||
this.unackedTimestamps.set(seq, now)
|
||||
}
|
||||
this.rebaseHealthClocks(now)
|
||||
}
|
||||
|
||||
this.sendKeepAlive()
|
||||
|
||||
if (this.disposed || resumedAfterWake) {
|
||||
if (this.disposed || resumedAfterWake || this.decoderReadPaused || this.writerSaturated) {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -529,4 +578,57 @@ export class SshChannelMultiplexer {
|
||||
console.warn(`[ssh-mux] Protocol error: ${err instanceof Error ? err.message : String(err)}`)
|
||||
this.dispose('connection_lost')
|
||||
}
|
||||
|
||||
private trackOutgoingTimestamp(seq: number, liveness: boolean): void {
|
||||
const limit = liveness ? MAX_UNACKED_TIMESTAMPS : MAX_ORDINARY_UNACKED_TIMESTAMPS
|
||||
if (this.unackedTimestamps.size < limit) {
|
||||
this.unackedTimestamps.set(seq, Date.now())
|
||||
}
|
||||
}
|
||||
|
||||
private pauseDecoderReads(): void {
|
||||
if (this.disposed || this.decoderReadPaused) {
|
||||
return
|
||||
}
|
||||
this.decoderReadPaused = true
|
||||
try {
|
||||
this.transport.pauseReads?.()
|
||||
} catch (error) {
|
||||
this.handleProtocolError(error)
|
||||
}
|
||||
}
|
||||
|
||||
private resumeDecoderReads(): void {
|
||||
if (!this.decoderReadPaused) {
|
||||
return
|
||||
}
|
||||
this.decoderReadPaused = false
|
||||
if (this.disposed) {
|
||||
return
|
||||
}
|
||||
this.rebaseHealthClocks(Date.now())
|
||||
try {
|
||||
this.transport.resumeReads?.()
|
||||
} catch (error) {
|
||||
this.handleProtocolError(error)
|
||||
}
|
||||
}
|
||||
|
||||
private handleWriterSaturationChange(saturated: boolean): void {
|
||||
this.writerSaturated = saturated
|
||||
if (!saturated && !this.disposed) {
|
||||
this.rebaseHealthClocks(Date.now())
|
||||
}
|
||||
}
|
||||
|
||||
private rebaseHealthClocks(now: number): void {
|
||||
this.lastReceivedAt = now
|
||||
for (const seq of this.unackedTimestamps.keys()) {
|
||||
this.unackedTimestamps.set(seq, now)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function messageLane(msg: JsonRpcMessage): MultiplexerWriterLane {
|
||||
return 'method' in msg && msg.method === 'pty.data' ? 'ordinary' : 'control'
|
||||
}
|
||||
|
||||
@@ -0,0 +1,284 @@
|
||||
import { EventEmitter } from 'node:events'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
MULTIPLEXER_CONTROL_RESERVE_BYTES,
|
||||
MULTIPLEXER_ORDINARY_QUEUE_MAX_BYTES,
|
||||
SshMultiplexerTransportWriter,
|
||||
type MultiplexerTransport,
|
||||
type MultiplexerWriteSettlement
|
||||
} from './ssh-multiplexer-transport-writer'
|
||||
|
||||
type WriterHarness = {
|
||||
transport: MultiplexerTransport
|
||||
drain: () => void
|
||||
writes: Buffer[]
|
||||
callbacks: ((result: MultiplexerWriteSettlement) => void)[]
|
||||
removeDrain: ReturnType<typeof vi.fn>
|
||||
}
|
||||
|
||||
function transportHarness(writeResults: (boolean | void)[]): WriterHarness {
|
||||
const emitter = new EventEmitter()
|
||||
const writes: Buffer[] = []
|
||||
const callbacks: ((result: MultiplexerWriteSettlement) => void)[] = []
|
||||
const removeDrain = vi.fn()
|
||||
return {
|
||||
transport: {
|
||||
write: (data, onSettled) => {
|
||||
writes.push(data)
|
||||
callbacks.push(onSettled!)
|
||||
return writeResults.shift()
|
||||
},
|
||||
supportsWriteSettlement: true,
|
||||
onDrain: (callback) => {
|
||||
emitter.on('drain', callback)
|
||||
return () => {
|
||||
removeDrain()
|
||||
emitter.off('drain', callback)
|
||||
}
|
||||
},
|
||||
onData: vi.fn(),
|
||||
onClose: vi.fn()
|
||||
},
|
||||
drain: () => emitter.emit('drain'),
|
||||
writes,
|
||||
callbacks,
|
||||
removeDrain
|
||||
}
|
||||
}
|
||||
|
||||
describe('SshMultiplexerTransportWriter', () => {
|
||||
it('selects queued control before ordinary backlog at the drain boundary', () => {
|
||||
const harness = transportHarness([false, true, true, true])
|
||||
const writer = new SshMultiplexerTransportWriter(harness.transport, vi.fn())
|
||||
|
||||
writer.enqueue(Buffer.from('ordinary-1'), 'ordinary')
|
||||
writer.enqueue(Buffer.from('ordinary-2'), 'ordinary')
|
||||
writer.enqueue(Buffer.from('ordinary-3'), 'ordinary')
|
||||
writer.enqueue(Buffer.from('control'), 'control')
|
||||
|
||||
expect(harness.writes.map(String)).toEqual(['ordinary-1'])
|
||||
harness.drain()
|
||||
expect(harness.writes.map(String)).toEqual([
|
||||
'ordinary-1',
|
||||
'control',
|
||||
'ordinary-2',
|
||||
'ordinary-3'
|
||||
])
|
||||
writer.dispose()
|
||||
})
|
||||
|
||||
it('preserves FIFO within each lane and prevents ordinary starvation', () => {
|
||||
const harness = transportHarness([false, ...Array<boolean>(8).fill(true)])
|
||||
const writer = new SshMultiplexerTransportWriter(harness.transport, vi.fn())
|
||||
|
||||
writer.enqueue(Buffer.from('ordinary-1'), 'ordinary')
|
||||
writer.enqueue(Buffer.from('ordinary-2'), 'ordinary')
|
||||
for (let index = 1; index <= 6; index++) {
|
||||
writer.enqueue(Buffer.from(`control-${index}`), 'control')
|
||||
}
|
||||
|
||||
harness.drain()
|
||||
expect(harness.writes.map(String)).toEqual([
|
||||
'ordinary-1',
|
||||
'control-1',
|
||||
'control-2',
|
||||
'control-3',
|
||||
'control-4',
|
||||
'ordinary-2',
|
||||
'control-5',
|
||||
'control-6'
|
||||
])
|
||||
writer.dispose()
|
||||
})
|
||||
|
||||
it('waits for drain and settles each write once', () => {
|
||||
const harness = transportHarness([false, true, true])
|
||||
const failed = vi.fn()
|
||||
const writer = new SshMultiplexerTransportWriter(harness.transport, failed)
|
||||
const settlements = [vi.fn(), vi.fn(), vi.fn()]
|
||||
|
||||
writer.enqueue(Buffer.from('ordinary-1'), 'ordinary', settlements[0])
|
||||
writer.enqueue(Buffer.from('control'), 'control', settlements[1])
|
||||
writer.enqueue(Buffer.from('ordinary-2'), 'ordinary', settlements[2])
|
||||
|
||||
expect(harness.writes.map(String)).toEqual(['ordinary-1'])
|
||||
harness.callbacks[0]({ ok: true })
|
||||
expect(settlements[0]).toHaveBeenCalledWith({ ok: true })
|
||||
expect(harness.writes.map(String)).toEqual(['ordinary-1'])
|
||||
|
||||
harness.drain()
|
||||
expect(harness.writes.map(String)).toEqual(['ordinary-1', 'control', 'ordinary-2'])
|
||||
harness.callbacks[1]({ ok: true })
|
||||
harness.callbacks[2]({ ok: true })
|
||||
expect(settlements.every((settle) => settle.mock.calls.length === 1)).toBe(true)
|
||||
expect(failed).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('allows one coalesced liveness bypass per saturated write settlement', () => {
|
||||
const harness = transportHarness([false, true, true, true])
|
||||
const writer = new SshMultiplexerTransportWriter(harness.transport, vi.fn())
|
||||
|
||||
writer.enqueue(Buffer.from('ordinary-1'), 'ordinary')
|
||||
writer.enqueue(Buffer.from('ordinary-2'), 'ordinary')
|
||||
expect(writer.enqueue(Buffer.from('liveness-1'), 'liveness')).toBe(true)
|
||||
expect(writer.enqueue(Buffer.from('liveness-coalesced'), 'liveness')).toBe(false)
|
||||
expect(harness.writes.map(String)).toEqual(['ordinary-1', 'liveness-1'])
|
||||
|
||||
harness.callbacks[1]({ ok: true })
|
||||
expect(writer.enqueue(Buffer.from('liveness-2'), 'liveness')).toBe(true)
|
||||
expect(harness.writes.map(String)).toEqual(['ordinary-1', 'liveness-1', 'liveness-2'])
|
||||
|
||||
harness.drain()
|
||||
expect(harness.writes.map(String)).toEqual([
|
||||
'ordinary-1',
|
||||
'liveness-1',
|
||||
'liveness-2',
|
||||
'ordinary-2'
|
||||
])
|
||||
writer.dispose()
|
||||
})
|
||||
|
||||
it('reports saturated epochs and their drain boundary exactly once', () => {
|
||||
const harness = transportHarness([false])
|
||||
const saturation = vi.fn()
|
||||
const writer = new SshMultiplexerTransportWriter(harness.transport, vi.fn(), saturation)
|
||||
|
||||
writer.enqueue(Buffer.from('ordinary'), 'ordinary')
|
||||
writer.enqueue(Buffer.from('queued'), 'ordinary')
|
||||
expect(saturation.mock.calls).toEqual([[true]])
|
||||
|
||||
harness.drain()
|
||||
expect(saturation.mock.calls).toEqual([[true], [false]])
|
||||
writer.dispose()
|
||||
})
|
||||
|
||||
it('keeps a full ordinary lane from consuming the control reserve', () => {
|
||||
const harness = transportHarness([false, true, true])
|
||||
const writer = new SshMultiplexerTransportWriter(harness.transport, vi.fn())
|
||||
|
||||
expect(writer.enqueue(Buffer.alloc(1), 'ordinary')).toBe(true)
|
||||
expect(writer.enqueue(Buffer.alloc(MULTIPLEXER_ORDINARY_QUEUE_MAX_BYTES - 1), 'ordinary')).toBe(
|
||||
true
|
||||
)
|
||||
expect(writer.enqueue(Buffer.alloc(MULTIPLEXER_CONTROL_RESERVE_BYTES), 'control')).toBe(true)
|
||||
|
||||
expect(harness.writes).toHaveLength(1)
|
||||
harness.drain()
|
||||
expect(harness.writes).toHaveLength(3)
|
||||
writer.dispose()
|
||||
})
|
||||
|
||||
it('fails all retained writes once when a callback reports an error', () => {
|
||||
const harness = transportHarness([false])
|
||||
const failed = vi.fn()
|
||||
const writer = new SshMultiplexerTransportWriter(harness.transport, failed)
|
||||
const first = vi.fn()
|
||||
const queued = vi.fn()
|
||||
const error = new Error('broken pipe')
|
||||
|
||||
writer.enqueue(Buffer.from('first'), 'ordinary', first)
|
||||
writer.enqueue(Buffer.from('queued'), 'control', queued)
|
||||
harness.callbacks[0]({ ok: false, error })
|
||||
harness.callbacks[0]({ ok: true })
|
||||
|
||||
expect(first).toHaveBeenCalledOnce()
|
||||
expect(first).toHaveBeenCalledWith({ ok: false, error })
|
||||
expect(queued).toHaveBeenCalledWith({ ok: false, error })
|
||||
expect(failed).toHaveBeenCalledWith(error)
|
||||
expect(harness.removeDrain).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('closes on queue overflow and rejects the overflowing write', () => {
|
||||
const harness = transportHarness([false])
|
||||
const failed = vi.fn()
|
||||
const writer = new SshMultiplexerTransportWriter(harness.transport, failed)
|
||||
const retained = vi.fn()
|
||||
const overflow = vi.fn()
|
||||
|
||||
writer.enqueue(Buffer.alloc(MULTIPLEXER_ORDINARY_QUEUE_MAX_BYTES), 'ordinary', retained)
|
||||
expect(writer.enqueue(Buffer.alloc(1), 'ordinary', overflow)).toBe(false)
|
||||
|
||||
expect(overflow).toHaveBeenCalledWith({
|
||||
ok: false,
|
||||
error: expect.objectContaining({ message: expect.stringContaining('bounded capacity') })
|
||||
})
|
||||
expect(retained).toHaveBeenCalledWith({
|
||||
ok: false,
|
||||
error: expect.objectContaining({ message: expect.stringContaining('bounded capacity') })
|
||||
})
|
||||
expect(failed).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('settles legacy callback-less transports on acceptance and drain', () => {
|
||||
const emitter = new EventEmitter()
|
||||
const write = vi.fn().mockReturnValueOnce(false).mockReturnValueOnce(undefined)
|
||||
const transport: MultiplexerTransport = {
|
||||
write,
|
||||
onDrain: (callback) => {
|
||||
emitter.on('drain', callback)
|
||||
},
|
||||
onData: vi.fn(),
|
||||
onClose: vi.fn()
|
||||
}
|
||||
const writer = new SshMultiplexerTransportWriter(transport, vi.fn())
|
||||
const first = vi.fn()
|
||||
const second = vi.fn()
|
||||
|
||||
writer.enqueue(Buffer.from('first'), 'ordinary', first)
|
||||
writer.enqueue(Buffer.from('second'), 'control', second)
|
||||
expect(first).not.toHaveBeenCalled()
|
||||
expect(second).not.toHaveBeenCalled()
|
||||
|
||||
emitter.emit('drain')
|
||||
expect(first).toHaveBeenCalledWith({ ok: true })
|
||||
expect(second).toHaveBeenCalledWith({ ok: true })
|
||||
})
|
||||
|
||||
it('does not miss a drain emitted synchronously by a hostile transport', () => {
|
||||
let drain = (): void => {}
|
||||
const write = vi.fn(() => {
|
||||
drain()
|
||||
return false
|
||||
})
|
||||
const writer = new SshMultiplexerTransportWriter(
|
||||
{
|
||||
write,
|
||||
onDrain: (callback) => {
|
||||
drain = callback
|
||||
},
|
||||
onData: vi.fn(),
|
||||
onClose: vi.fn()
|
||||
},
|
||||
vi.fn()
|
||||
)
|
||||
const first = vi.fn()
|
||||
const second = vi.fn()
|
||||
|
||||
writer.enqueue(Buffer.from('first'), 'ordinary', first)
|
||||
writer.enqueue(Buffer.from('second'), 'control', second)
|
||||
|
||||
expect(write).toHaveBeenCalledTimes(2)
|
||||
expect(first).toHaveBeenCalledWith({ ok: true })
|
||||
expect(second).toHaveBeenCalledWith({ ok: true })
|
||||
})
|
||||
|
||||
it('fails deterministically when write(false) has no drain source', () => {
|
||||
const failed = vi.fn()
|
||||
const writer = new SshMultiplexerTransportWriter(
|
||||
{
|
||||
write: () => false,
|
||||
onData: vi.fn(),
|
||||
onClose: vi.fn()
|
||||
},
|
||||
failed
|
||||
)
|
||||
const settled = vi.fn()
|
||||
|
||||
expect(writer.enqueue(Buffer.from('data'), 'ordinary', settled)).toBe(true)
|
||||
expect(settled).toHaveBeenCalledWith({
|
||||
ok: false,
|
||||
error: expect.objectContaining({ message: expect.stringContaining('without drain support') })
|
||||
})
|
||||
expect(failed).toHaveBeenCalledOnce()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,270 @@
|
||||
import { HEADER_LENGTH, MAX_MESSAGE_SIZE } from './relay-protocol'
|
||||
import { SshMultiplexerWriterLaneScheduler } from './ssh-multiplexer-writer-lane-scheduler'
|
||||
|
||||
export type MultiplexerWriteSettlement = { ok: true } | { ok: false; error: Error }
|
||||
|
||||
export type MultiplexerTransport = {
|
||||
write: (data: Buffer, onSettled?: (result: MultiplexerWriteSettlement) => void) => boolean | void
|
||||
onData: (cb: (data: Buffer) => void) => void
|
||||
onClose: (cb: () => void) => void
|
||||
onDrain?: (cb: () => void) => void | (() => void)
|
||||
supportsWriteSettlement?: boolean
|
||||
pauseReads?: () => void
|
||||
resumeReads?: () => void
|
||||
close?: () => void
|
||||
}
|
||||
|
||||
export type MultiplexerWriterLane = 'ordinary' | 'control' | 'liveness'
|
||||
|
||||
type WriterEntry = {
|
||||
data: Buffer
|
||||
lane: MultiplexerWriterLane
|
||||
onSettled: (result: MultiplexerWriteSettlement) => void
|
||||
settled: boolean
|
||||
}
|
||||
|
||||
export const MULTIPLEXER_ORDINARY_QUEUE_MAX_BYTES = 2 * 1024 * 1024
|
||||
export const MULTIPLEXER_CONTROL_RESERVE_BYTES = MAX_MESSAGE_SIZE + HEADER_LENGTH
|
||||
const ORDINARY_QUEUE_MAX_FRAMES = 2048
|
||||
const CONTROL_QUEUE_MAX_FRAMES = 512
|
||||
|
||||
function onceSettlement(
|
||||
callback: (result: MultiplexerWriteSettlement) => void
|
||||
): (result: MultiplexerWriteSettlement) => void {
|
||||
let settled = false
|
||||
return (result) => {
|
||||
if (settled) {
|
||||
return
|
||||
}
|
||||
settled = true
|
||||
callback(result)
|
||||
}
|
||||
}
|
||||
|
||||
export class SshMultiplexerTransportWriter {
|
||||
private readonly scheduler = new SshMultiplexerWriterLaneScheduler<WriterEntry>()
|
||||
private readonly inFlight = new Set<WriterEntry>()
|
||||
private readonly settleOnDrain = new Set<WriterEntry>()
|
||||
private ordinaryBytes = 0
|
||||
private controlBytes = 0
|
||||
private ordinaryFrames = 0
|
||||
private controlFrames = 0
|
||||
private saturated = false
|
||||
private writing = false
|
||||
private drainObservedDuringWrite = false
|
||||
private pumping = false
|
||||
private closed = false
|
||||
private livenessOutstanding = false
|
||||
private removeDrainListener: (() => void) | null = null
|
||||
|
||||
constructor(
|
||||
private readonly transport: MultiplexerTransport,
|
||||
private readonly onFailure: (error: Error) => void,
|
||||
private readonly onSaturationChange: (saturated: boolean) => void = () => {}
|
||||
) {
|
||||
if (transport.onDrain) {
|
||||
const remove = transport.onDrain(() => this.handleDrain())
|
||||
this.removeDrainListener = typeof remove === 'function' ? remove : null
|
||||
}
|
||||
}
|
||||
|
||||
enqueue(
|
||||
data: Buffer,
|
||||
lane: MultiplexerWriterLane,
|
||||
onSettled: (result: MultiplexerWriteSettlement) => void = () => {}
|
||||
): boolean {
|
||||
const settle = onceSettlement(onSettled)
|
||||
if (this.closed) {
|
||||
settle({ ok: false, error: new Error('Multiplexer writer is closed') })
|
||||
return false
|
||||
}
|
||||
if (lane === 'liveness' && this.livenessOutstanding) {
|
||||
return false
|
||||
}
|
||||
const admissionError = this.admissionError(data.length, lane)
|
||||
if (admissionError) {
|
||||
settle({ ok: false, error: admissionError })
|
||||
this.fail(admissionError)
|
||||
return false
|
||||
}
|
||||
const entry = { data, lane, onSettled: settle, settled: false }
|
||||
this.retain(entry)
|
||||
if (lane === 'liveness' && this.saturated) {
|
||||
this.writeEntry(entry)
|
||||
} else {
|
||||
this.scheduler.enqueue(entry, lane)
|
||||
this.pump()
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
dispose(error = new Error('Multiplexer writer disposed')): void {
|
||||
if (this.closed) {
|
||||
return
|
||||
}
|
||||
this.closed = true
|
||||
this.saturated = false
|
||||
this.removeDrainListener?.()
|
||||
this.removeDrainListener = null
|
||||
for (const entry of this.scheduler.clear()) {
|
||||
this.release(entry, { ok: false, error })
|
||||
}
|
||||
for (const entry of Array.from(this.inFlight)) {
|
||||
this.release(entry, { ok: false, error })
|
||||
}
|
||||
this.settleOnDrain.clear()
|
||||
}
|
||||
|
||||
private admissionError(bytes: number, lane: MultiplexerWriterLane): Error | null {
|
||||
const byteLimit =
|
||||
lane === 'ordinary' ? MULTIPLEXER_ORDINARY_QUEUE_MAX_BYTES : MULTIPLEXER_CONTROL_RESERVE_BYTES
|
||||
const retainedBytes = lane === 'ordinary' ? this.ordinaryBytes : this.controlBytes
|
||||
const frameLimit = lane === 'ordinary' ? ORDINARY_QUEUE_MAX_FRAMES : CONTROL_QUEUE_MAX_FRAMES
|
||||
const retainedFrames = lane === 'ordinary' ? this.ordinaryFrames : this.controlFrames
|
||||
if (retainedBytes + bytes <= byteLimit && retainedFrames < frameLimit) {
|
||||
return null
|
||||
}
|
||||
return new Error(`Multiplexer ${lane} write queue exceeded its bounded capacity`)
|
||||
}
|
||||
|
||||
private pump(): void {
|
||||
if (this.pumping || this.closed || this.saturated) {
|
||||
return
|
||||
}
|
||||
this.pumping = true
|
||||
try {
|
||||
while (!this.closed && !this.saturated) {
|
||||
const entry = this.scheduler.select()
|
||||
if (!entry) {
|
||||
return
|
||||
}
|
||||
this.writeEntry(entry)
|
||||
}
|
||||
} finally {
|
||||
this.pumping = false
|
||||
}
|
||||
}
|
||||
|
||||
private writeEntry(entry: WriterEntry): void {
|
||||
this.inFlight.add(entry)
|
||||
let callbackResult: MultiplexerWriteSettlement | undefined
|
||||
let writeReturned = false
|
||||
const onWriteSettled = (result: MultiplexerWriteSettlement): void => {
|
||||
if (!writeReturned) {
|
||||
callbackResult = result
|
||||
return
|
||||
}
|
||||
this.handleWriteSettlement(entry, result)
|
||||
}
|
||||
try {
|
||||
this.writing = true
|
||||
this.drainObservedDuringWrite = false
|
||||
const accepted = this.transport.write(entry.data, onWriteSettled)
|
||||
this.writing = false
|
||||
writeReturned = true
|
||||
if (accepted === false) {
|
||||
if (!this.transport.onDrain) {
|
||||
throw new Error('Multiplexer transport returned write(false) without drain support')
|
||||
}
|
||||
this.setSaturated(!this.drainObservedDuringWrite)
|
||||
if (this.transport.supportsWriteSettlement !== true && this.saturated) {
|
||||
this.settleOnDrain.add(entry)
|
||||
} else if (this.transport.supportsWriteSettlement !== true) {
|
||||
this.handleWriteSettlement(entry, { ok: true })
|
||||
}
|
||||
} else if (this.transport.supportsWriteSettlement !== true) {
|
||||
this.handleWriteSettlement(entry, { ok: true })
|
||||
}
|
||||
if (callbackResult) {
|
||||
this.handleWriteSettlement(entry, callbackResult)
|
||||
}
|
||||
} catch (error) {
|
||||
this.writing = false
|
||||
writeReturned = true
|
||||
if (callbackResult) {
|
||||
this.handleWriteSettlement(entry, callbackResult)
|
||||
}
|
||||
this.fail(error instanceof Error ? error : new Error(String(error)))
|
||||
}
|
||||
}
|
||||
|
||||
private handleWriteSettlement(entry: WriterEntry, result: MultiplexerWriteSettlement): void {
|
||||
if (entry.settled) {
|
||||
return
|
||||
}
|
||||
this.release(entry, result)
|
||||
if (!result.ok) {
|
||||
this.fail(result.error)
|
||||
return
|
||||
}
|
||||
this.pump()
|
||||
}
|
||||
|
||||
private handleDrain(): void {
|
||||
if (this.closed) {
|
||||
return
|
||||
}
|
||||
if (this.writing) {
|
||||
this.drainObservedDuringWrite = true
|
||||
return
|
||||
}
|
||||
if (!this.saturated) {
|
||||
return
|
||||
}
|
||||
this.setSaturated(false)
|
||||
for (const entry of Array.from(this.settleOnDrain)) {
|
||||
this.release(entry, { ok: true })
|
||||
}
|
||||
this.settleOnDrain.clear()
|
||||
this.pump()
|
||||
}
|
||||
|
||||
private retain(entry: WriterEntry): void {
|
||||
if (entry.lane === 'ordinary') {
|
||||
this.ordinaryBytes += entry.data.length
|
||||
this.ordinaryFrames++
|
||||
} else {
|
||||
this.controlBytes += entry.data.length
|
||||
this.controlFrames++
|
||||
}
|
||||
if (entry.lane === 'liveness') {
|
||||
this.livenessOutstanding = true
|
||||
}
|
||||
}
|
||||
|
||||
private release(entry: WriterEntry, result: MultiplexerWriteSettlement): void {
|
||||
if (entry.settled) {
|
||||
return
|
||||
}
|
||||
entry.settled = true
|
||||
this.inFlight.delete(entry)
|
||||
this.settleOnDrain.delete(entry)
|
||||
if (entry.lane === 'ordinary') {
|
||||
this.ordinaryBytes -= entry.data.length
|
||||
this.ordinaryFrames--
|
||||
} else {
|
||||
this.controlBytes -= entry.data.length
|
||||
this.controlFrames--
|
||||
}
|
||||
if (entry.lane === 'liveness') {
|
||||
this.livenessOutstanding = false
|
||||
}
|
||||
entry.onSettled(result)
|
||||
}
|
||||
|
||||
private setSaturated(saturated: boolean): void {
|
||||
if (this.saturated === saturated) {
|
||||
return
|
||||
}
|
||||
this.saturated = saturated
|
||||
this.onSaturationChange(saturated)
|
||||
}
|
||||
|
||||
private fail(error: Error): void {
|
||||
if (this.closed) {
|
||||
return
|
||||
}
|
||||
this.dispose(error)
|
||||
this.onFailure(error)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
import type { MultiplexerWriterLane } from './ssh-multiplexer-transport-writer'
|
||||
|
||||
const CONTROL_WRITES_BEFORE_ORDINARY = 4
|
||||
|
||||
type LaneQueue<T> = {
|
||||
entries: T[]
|
||||
head: number
|
||||
}
|
||||
|
||||
function createLaneQueue<T>(): LaneQueue<T> {
|
||||
return { entries: [], head: 0 }
|
||||
}
|
||||
|
||||
function hasEntries<T>(queue: LaneQueue<T>): boolean {
|
||||
return queue.head < queue.entries.length
|
||||
}
|
||||
|
||||
function shift<T>(queue: LaneQueue<T>): T | undefined {
|
||||
const entry = queue.entries[queue.head]
|
||||
if (entry === undefined) {
|
||||
return undefined
|
||||
}
|
||||
queue.head += 1
|
||||
if (queue.head === queue.entries.length) {
|
||||
queue.entries.length = 0
|
||||
queue.head = 0
|
||||
}
|
||||
return entry
|
||||
}
|
||||
|
||||
function clear<T>(queue: LaneQueue<T>): T[] {
|
||||
const entries = queue.entries.slice(queue.head)
|
||||
queue.entries.length = 0
|
||||
queue.head = 0
|
||||
return entries
|
||||
}
|
||||
|
||||
export class SshMultiplexerWriterLaneScheduler<T extends object> {
|
||||
private readonly ordinary = createLaneQueue<T>()
|
||||
private readonly control = createLaneQueue<T>()
|
||||
private readonly liveness = createLaneQueue<T>()
|
||||
private controlWritesSinceOrdinary = 0
|
||||
|
||||
enqueue(entry: T, lane: MultiplexerWriterLane): void {
|
||||
this[lane].entries.push(entry)
|
||||
}
|
||||
|
||||
select(): T | undefined {
|
||||
const liveness = shift(this.liveness)
|
||||
if (liveness) {
|
||||
return liveness
|
||||
}
|
||||
if (
|
||||
hasEntries(this.control) &&
|
||||
(!hasEntries(this.ordinary) ||
|
||||
this.controlWritesSinceOrdinary < CONTROL_WRITES_BEFORE_ORDINARY)
|
||||
) {
|
||||
this.controlWritesSinceOrdinary += 1
|
||||
return shift(this.control)
|
||||
}
|
||||
const ordinary = shift(this.ordinary)
|
||||
if (ordinary) {
|
||||
this.controlWritesSinceOrdinary = 0
|
||||
return ordinary
|
||||
}
|
||||
return shift(this.control)
|
||||
}
|
||||
|
||||
clear(): T[] {
|
||||
this.controlWritesSinceOrdinary = 0
|
||||
return [...clear(this.liveness), ...clear(this.control), ...clear(this.ordinary)]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { SshChannelMultiplexer } from './ssh-channel-multiplexer'
|
||||
import { openSshPtyConsumerSession } from './ssh-pty-consumer-session'
|
||||
|
||||
function muxReturning(result: unknown): {
|
||||
mux: SshChannelMultiplexer
|
||||
request: ReturnType<typeof vi.fn>
|
||||
} {
|
||||
const request = vi.fn().mockResolvedValue(result)
|
||||
return { mux: { request } as unknown as SshChannelMultiplexer, request }
|
||||
}
|
||||
|
||||
function legacyOwnerGrant(overrides: Record<string, unknown> = {}): Record<string, unknown> {
|
||||
return {
|
||||
protocolVersion: 1,
|
||||
serverBuildId: 'build-a',
|
||||
clientGeneration: 3,
|
||||
role: 'session-owner',
|
||||
ownerGeneration: 7,
|
||||
ownerLease: 'lease-a',
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
describe('openSshPtyConsumerSession', () => {
|
||||
it('makes openClient the one request needed for token-free legacy readiness', async () => {
|
||||
const { mux, request } = muxReturning(legacyOwnerGrant())
|
||||
|
||||
await expect(
|
||||
openSshPtyConsumerSession(mux, {
|
||||
clientInstanceId: 'client-a',
|
||||
expectedServerBuildId: 'build-a'
|
||||
})
|
||||
).resolves.toEqual({
|
||||
mode: 'negotiated',
|
||||
clientInstanceId: 'client-a',
|
||||
clientGeneration: 3,
|
||||
ownerGeneration: 7,
|
||||
ownerLease: 'lease-a'
|
||||
})
|
||||
expect(request).toHaveBeenCalledWith(
|
||||
'pty.openClient',
|
||||
{
|
||||
protocolVersion: 1,
|
||||
clientInstanceId: 'client-a',
|
||||
requestedRole: 'session-owner'
|
||||
},
|
||||
{ timeoutMs: 10_000 }
|
||||
)
|
||||
})
|
||||
|
||||
it('carries recovery generation and lease on reconnect', async () => {
|
||||
const { mux, request } = muxReturning(
|
||||
legacyOwnerGrant({ ownerGeneration: 8, ownerLease: 'lease-b' })
|
||||
)
|
||||
await openSshPtyConsumerSession(mux, {
|
||||
clientInstanceId: 'client-a',
|
||||
expectedServerBuildId: 'build-a',
|
||||
resume: { ownerGeneration: 7, ownerLease: 'lease-a' }
|
||||
})
|
||||
|
||||
expect(request.mock.calls[0][1]).toMatchObject({
|
||||
resume: { ownerGeneration: 7, ownerLease: 'lease-a' }
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects a prior or mismatched relay build', async () => {
|
||||
const { mux } = muxReturning(legacyOwnerGrant({ serverBuildId: 'old-build' }))
|
||||
|
||||
await expect(
|
||||
openSshPtyConsumerSession(mux, {
|
||||
clientInstanceId: 'client-a',
|
||||
expectedServerBuildId: 'build-a'
|
||||
})
|
||||
).rejects.toThrow('session contract mismatch')
|
||||
})
|
||||
|
||||
it('does not silently downgrade when V1 was offered', async () => {
|
||||
const { mux } = muxReturning(legacyOwnerGrant())
|
||||
|
||||
await expect(
|
||||
openSshPtyConsumerSession(mux, {
|
||||
clientInstanceId: 'client-a',
|
||||
expectedServerBuildId: 'build-a',
|
||||
outputFlowControl: { requestedWindowSu: 64 }
|
||||
})
|
||||
).rejects.toThrow('did not grant')
|
||||
})
|
||||
|
||||
it('rejects an unoffered V1 capability in a legacy session', async () => {
|
||||
const { mux } = muxReturning(
|
||||
legacyOwnerGrant({
|
||||
capabilities: { outputFlowControl: { version: 1, windowSu: 64 } }
|
||||
})
|
||||
)
|
||||
|
||||
await expect(
|
||||
openSshPtyConsumerSession(mux, {
|
||||
clientInstanceId: 'client-a',
|
||||
expectedServerBuildId: 'build-a'
|
||||
})
|
||||
).rejects.toThrow('unoffered')
|
||||
})
|
||||
|
||||
it('uses explicit token-free fallback only for same-build method-not-found', async () => {
|
||||
const error = Object.assign(new Error('Method not found: pty.openClient'), { code: -32601 })
|
||||
const request = vi.fn().mockRejectedValue(error)
|
||||
const mux = { request } as unknown as SshChannelMultiplexer
|
||||
|
||||
await expect(
|
||||
openSshPtyConsumerSession(mux, {
|
||||
clientInstanceId: 'client-a',
|
||||
expectedServerBuildId: 'build-a',
|
||||
allowSameBuildLegacyFallback: true,
|
||||
outputFlowControl: { requestedWindowSu: 64 }
|
||||
})
|
||||
).resolves.toEqual({
|
||||
mode: 'legacy-fallback',
|
||||
clientInstanceId: 'client-a',
|
||||
serverBuildId: 'build-a'
|
||||
})
|
||||
})
|
||||
|
||||
it.each([
|
||||
Object.assign(new Error('timeout'), { code: 'TIMEOUT' }),
|
||||
Object.assign(new Error('auth failed'), { code: -32000 }),
|
||||
Object.assign(new Error('method missing'), { code: -32601 })
|
||||
])('does not downgrade an unproved or non-method-not-found error', async (error) => {
|
||||
const request = vi.fn().mockRejectedValue(error)
|
||||
const mux = { request } as unknown as SshChannelMultiplexer
|
||||
|
||||
await expect(
|
||||
openSshPtyConsumerSession(mux, {
|
||||
clientInstanceId: 'client-a',
|
||||
expectedServerBuildId: 'build-a',
|
||||
allowSameBuildLegacyFallback: error.code !== -32601
|
||||
})
|
||||
).rejects.toBe(error)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,143 @@
|
||||
import {
|
||||
PTY_CONSUMER_SESSION_PROTOCOL_VERSION,
|
||||
type PtyConsumerSessionGrant
|
||||
} from '../../shared/pty-consumer-session'
|
||||
import { DEFAULT_PTY_SOURCE_WINDOW_SU } from '../../shared/pty-source-credit-contract'
|
||||
import type { SshChannelMultiplexer } from './ssh-channel-multiplexer'
|
||||
|
||||
export const SSH_PTY_OPEN_CLIENT_METHOD = 'pty.openClient'
|
||||
export const SSH_PTY_OPEN_CLIENT_TIMEOUT_MS = 10_000
|
||||
|
||||
export type SshPtyConsumerOwnerState = {
|
||||
mode: 'negotiated'
|
||||
clientInstanceId: string
|
||||
clientGeneration: number
|
||||
ownerGeneration: number
|
||||
ownerLease: string
|
||||
outputFlowControl?: {
|
||||
version: 1
|
||||
windowSu: number
|
||||
}
|
||||
}
|
||||
|
||||
export type SshPtyLegacyFallbackState = {
|
||||
mode: 'legacy-fallback'
|
||||
clientInstanceId: string
|
||||
serverBuildId: string
|
||||
}
|
||||
|
||||
export type SshPtyConsumerSessionState = SshPtyConsumerOwnerState | SshPtyLegacyFallbackState
|
||||
|
||||
export type OpenSshPtyConsumerSessionOptions = {
|
||||
clientInstanceId: string
|
||||
expectedServerBuildId: string | undefined
|
||||
resume?: Pick<SshPtyConsumerOwnerState, 'ownerGeneration' | 'ownerLease'>
|
||||
outputFlowControl?: {
|
||||
requestedWindowSu: number
|
||||
}
|
||||
allowSameBuildLegacyFallback?: boolean
|
||||
}
|
||||
|
||||
function validateGrant(
|
||||
value: unknown,
|
||||
options: OpenSshPtyConsumerSessionOptions
|
||||
): PtyConsumerSessionGrant {
|
||||
if (typeof value !== 'object' || value === null) {
|
||||
throw new Error('Remote relay returned an invalid pty.openClient grant')
|
||||
}
|
||||
if (!options.expectedServerBuildId) {
|
||||
throw new Error('Local relay build identity is unavailable')
|
||||
}
|
||||
const grant = value as Partial<PtyConsumerSessionGrant>
|
||||
if (
|
||||
grant.protocolVersion !== PTY_CONSUMER_SESSION_PROTOCOL_VERSION ||
|
||||
grant.serverBuildId !== options.expectedServerBuildId
|
||||
) {
|
||||
throw new Error(
|
||||
`Remote relay session contract mismatch — expected build ${options.expectedServerBuildId}, got ${grant.serverBuildId ?? 'unknown'}`
|
||||
)
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(grant.clientGeneration) ||
|
||||
grant.clientGeneration! <= 0 ||
|
||||
grant.role !== 'session-owner' ||
|
||||
!Number.isSafeInteger(grant.ownerGeneration) ||
|
||||
grant.ownerGeneration! <= 0 ||
|
||||
typeof grant.ownerLease !== 'string' ||
|
||||
grant.ownerLease.length === 0
|
||||
) {
|
||||
throw new Error('Remote relay did not grant an authenticated PTY session owner')
|
||||
}
|
||||
const requestedFlow = options.outputFlowControl
|
||||
const grantedFlow = grant.capabilities?.outputFlowControl
|
||||
if (requestedFlow) {
|
||||
if (
|
||||
grantedFlow?.version !== 1 ||
|
||||
!Number.isSafeInteger(grantedFlow.windowSu) ||
|
||||
grantedFlow.windowSu <= 0 ||
|
||||
grantedFlow.windowSu > requestedFlow.requestedWindowSu
|
||||
) {
|
||||
throw new Error('Remote relay did not grant the offered PTY output-flow-control capability')
|
||||
}
|
||||
} else if (grantedFlow) {
|
||||
throw new Error('Remote relay granted an unoffered PTY output-flow-control capability')
|
||||
}
|
||||
return grant as PtyConsumerSessionGrant
|
||||
}
|
||||
|
||||
export async function openSshPtyConsumerSession(
|
||||
mux: SshChannelMultiplexer,
|
||||
options: OpenSshPtyConsumerSessionOptions
|
||||
): Promise<SshPtyConsumerSessionState> {
|
||||
let result: unknown
|
||||
try {
|
||||
result = await mux.request(
|
||||
SSH_PTY_OPEN_CLIENT_METHOD,
|
||||
{
|
||||
protocolVersion: PTY_CONSUMER_SESSION_PROTOCOL_VERSION,
|
||||
clientInstanceId: options.clientInstanceId,
|
||||
requestedRole: 'session-owner',
|
||||
...(options.resume ? { resume: options.resume } : {}),
|
||||
...(options.outputFlowControl
|
||||
? {
|
||||
capabilities: {
|
||||
outputFlowControl: {
|
||||
versions: [1],
|
||||
requestedWindowSu: options.outputFlowControl.requestedWindowSu
|
||||
}
|
||||
}
|
||||
}
|
||||
: {})
|
||||
},
|
||||
{ timeoutMs: SSH_PTY_OPEN_CLIENT_TIMEOUT_MS }
|
||||
)
|
||||
} catch (error) {
|
||||
const code = (error as { code?: unknown })?.code
|
||||
if (
|
||||
code === -32601 &&
|
||||
options.allowSameBuildLegacyFallback === true &&
|
||||
typeof options.expectedServerBuildId === 'string' &&
|
||||
options.expectedServerBuildId.length > 0
|
||||
) {
|
||||
return Object.freeze({
|
||||
mode: 'legacy-fallback',
|
||||
clientInstanceId: options.clientInstanceId,
|
||||
serverBuildId: options.expectedServerBuildId
|
||||
})
|
||||
}
|
||||
throw error
|
||||
}
|
||||
const grant = validateGrant(result, options)
|
||||
return {
|
||||
mode: 'negotiated',
|
||||
clientInstanceId: options.clientInstanceId,
|
||||
clientGeneration: grant.clientGeneration,
|
||||
ownerGeneration: grant.ownerGeneration!,
|
||||
ownerLease: grant.ownerLease!,
|
||||
...(grant.capabilities?.outputFlowControl
|
||||
? { outputFlowControl: grant.capabilities.outputFlowControl }
|
||||
: {})
|
||||
}
|
||||
}
|
||||
|
||||
export const SSH_PTY_SOURCE_WINDOW_SU = DEFAULT_PTY_SOURCE_WINDOW_SU
|
||||
@@ -0,0 +1,43 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { SshPtyRecoveryRetentionBudget } from './ssh-pty-recovery-retention-budget'
|
||||
|
||||
describe('SshPtyRecoveryRetentionBudget', () => {
|
||||
it('bounds fragmented recovery by per-PTY and session source, charged bytes, and frames', () => {
|
||||
const budget = new SshPtyRecoveryRetentionBudget({
|
||||
perPtySourceSu: 4,
|
||||
perPtyBytes: 300,
|
||||
perPtyFrames: 2,
|
||||
sessionSourceSu: 6,
|
||||
sessionBytes: 500,
|
||||
sessionFrames: 3
|
||||
})
|
||||
|
||||
expect(budget.tryRetain('pty-1', 'aa', 2)).toBe(true)
|
||||
expect(budget.tryRetain('pty-1', 'bb', 2)).toBe(true)
|
||||
expect(budget.tryRetain('pty-1', 'c', 0)).toBe(false)
|
||||
expect(budget.tryRetain('pty-2', 'cc', 2)).toBe(true)
|
||||
expect(budget.snapshot()).toEqual({ sourceSu: 6, bytes: 396, frames: 3, ptys: 2 })
|
||||
expect(budget.tryRetain('pty-2', 'd', 1)).toBe(false)
|
||||
|
||||
budget.release('pty-1')
|
||||
|
||||
expect(budget.snapshot()).toEqual({ sourceSu: 2, bytes: 132, frames: 1, ptys: 1 })
|
||||
expect(budget.tryRetain('pty-2', 'dd', 2)).toBe(true)
|
||||
})
|
||||
|
||||
it('charges UTF-16 storage plus one record before aggregate admission', () => {
|
||||
const budget = new SshPtyRecoveryRetentionBudget({
|
||||
perPtySourceSu: 100,
|
||||
perPtyBytes: 1_000,
|
||||
perPtyFrames: 10,
|
||||
sessionSourceSu: 100,
|
||||
sessionBytes: 500,
|
||||
sessionFrames: 10
|
||||
})
|
||||
|
||||
expect(budget.tryRetain('pty-1', '\u0000'.repeat(64), 1)).toBe(true)
|
||||
expect(budget.snapshot().bytes).toBe(256)
|
||||
expect(budget.tryRetain('pty-2', '\u0000'.repeat(64), 1)).toBe(false)
|
||||
expect(budget.snapshot()).toEqual({ sourceSu: 1, bytes: 256, frames: 1, ptys: 1 })
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,95 @@
|
||||
import { DEFAULT_PTY_SOURCE_WINDOW_SU } from '../../shared/pty-source-credit-contract'
|
||||
import { chargedPtyRetainedStringBytes } from '../../shared/pty-retained-string-memory'
|
||||
|
||||
export const SSH_PTY_RECOVERY_PER_PTY_MAX_SOURCE_SU = DEFAULT_PTY_SOURCE_WINDOW_SU
|
||||
export const SSH_PTY_RECOVERY_PER_PTY_MAX_BYTES = 2 * 1024 * 1024
|
||||
export const SSH_PTY_RECOVERY_PER_PTY_MAX_FRAMES = 1_024
|
||||
export const SSH_PTY_RECOVERY_SESSION_MAX_SOURCE_SU = 50 * DEFAULT_PTY_SOURCE_WINDOW_SU
|
||||
export const SSH_PTY_RECOVERY_SESSION_MAX_BYTES = 64 * 1024 * 1024
|
||||
export const SSH_PTY_RECOVERY_SESSION_MAX_FRAMES = 64 * 1_024
|
||||
|
||||
export type SshPtyRecoveryRetentionLimits = Readonly<{
|
||||
perPtySourceSu: number
|
||||
perPtyBytes: number
|
||||
perPtyFrames: number
|
||||
sessionSourceSu: number
|
||||
sessionBytes: number
|
||||
sessionFrames: number
|
||||
}>
|
||||
|
||||
type RetainedPty = {
|
||||
sourceSu: number
|
||||
bytes: number
|
||||
frames: number
|
||||
}
|
||||
|
||||
const DEFAULT_LIMITS: SshPtyRecoveryRetentionLimits = Object.freeze({
|
||||
perPtySourceSu: SSH_PTY_RECOVERY_PER_PTY_MAX_SOURCE_SU,
|
||||
perPtyBytes: SSH_PTY_RECOVERY_PER_PTY_MAX_BYTES,
|
||||
perPtyFrames: SSH_PTY_RECOVERY_PER_PTY_MAX_FRAMES,
|
||||
sessionSourceSu: SSH_PTY_RECOVERY_SESSION_MAX_SOURCE_SU,
|
||||
sessionBytes: SSH_PTY_RECOVERY_SESSION_MAX_BYTES,
|
||||
sessionFrames: SSH_PTY_RECOVERY_SESSION_MAX_FRAMES
|
||||
})
|
||||
|
||||
export class SshPtyRecoveryRetentionBudget {
|
||||
private readonly retainedByPty = new Map<string, RetainedPty>()
|
||||
private sourceSu = 0
|
||||
private bytes = 0
|
||||
private frames = 0
|
||||
|
||||
constructor(private readonly limits: SshPtyRecoveryRetentionLimits = DEFAULT_LIMITS) {}
|
||||
|
||||
tryRetain(ptyId: string, data: string, sourceSu: number): boolean {
|
||||
if (!Number.isSafeInteger(sourceSu) || sourceSu < 0) {
|
||||
return false
|
||||
}
|
||||
const retained = this.retainedByPty.get(ptyId) ?? { sourceSu: 0, bytes: 0, frames: 0 }
|
||||
const chargedBytes = chargedPtyRetainedStringBytes(data)
|
||||
if (
|
||||
retained.sourceSu + sourceSu > this.limits.perPtySourceSu ||
|
||||
retained.bytes + chargedBytes > this.limits.perPtyBytes ||
|
||||
retained.frames + 1 > this.limits.perPtyFrames ||
|
||||
this.sourceSu + sourceSu > this.limits.sessionSourceSu ||
|
||||
this.bytes + chargedBytes > this.limits.sessionBytes ||
|
||||
this.frames + 1 > this.limits.sessionFrames
|
||||
) {
|
||||
return false
|
||||
}
|
||||
retained.sourceSu += sourceSu
|
||||
retained.bytes += chargedBytes
|
||||
retained.frames++
|
||||
this.retainedByPty.set(ptyId, retained)
|
||||
this.sourceSu += sourceSu
|
||||
this.bytes += chargedBytes
|
||||
this.frames++
|
||||
return true
|
||||
}
|
||||
|
||||
release(ptyId: string): void {
|
||||
const retained = this.retainedByPty.get(ptyId)
|
||||
if (!retained) {
|
||||
return
|
||||
}
|
||||
this.retainedByPty.delete(ptyId)
|
||||
this.sourceSu -= retained.sourceSu
|
||||
this.bytes -= retained.bytes
|
||||
this.frames -= retained.frames
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
this.retainedByPty.clear()
|
||||
this.sourceSu = 0
|
||||
this.bytes = 0
|
||||
this.frames = 0
|
||||
}
|
||||
|
||||
snapshot(): Readonly<{ sourceSu: number; bytes: number; frames: number; ptys: number }> {
|
||||
return Object.freeze({
|
||||
sourceSu: this.sourceSu,
|
||||
bytes: this.bytes,
|
||||
frames: this.frames,
|
||||
ptys: this.retainedByPty.size
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { SshPtyRetiredSourceDeliveries } from './ssh-pty-retired-source-deliveries'
|
||||
|
||||
const source = (deliveryToken: string, relayPtyId = 'pty-1') => ({
|
||||
relayPtyId,
|
||||
deliveryToken,
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3
|
||||
})
|
||||
|
||||
describe('SshPtyRetiredSourceDeliveries', () => {
|
||||
it('retains only the latest canceled token for each ordered PTY stream', () => {
|
||||
const retired = new SshPtyRetiredSourceDeliveries()
|
||||
|
||||
for (let token = 0; token < 10_000; token++) {
|
||||
retired.retire(1, source(`token-${token}`))
|
||||
}
|
||||
|
||||
expect(retired.size).toBe(1)
|
||||
expect(retired.has(1, source('token-9999'))).toBe(true)
|
||||
expect(retired.has(1, source('token-9998'))).toBe(false)
|
||||
})
|
||||
|
||||
it('retires state at the next activation or PTY exit boundary', () => {
|
||||
const retired = new SshPtyRetiredSourceDeliveries()
|
||||
retired.retire(1, source('old-token'))
|
||||
retired.retire(1, source('other-token', 'pty-2'))
|
||||
|
||||
retired.activate('pty-1')
|
||||
|
||||
expect(retired.has(1, source('old-token'))).toBe(false)
|
||||
expect(retired.has(1, source('other-token', 'pty-2'))).toBe(true)
|
||||
expect(retired.size).toBe(1)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,36 @@
|
||||
import type { SshPtySourceFrame } from '../providers/ssh-pty-source-frame'
|
||||
|
||||
type SourceDeliveryIdentity = Pick<
|
||||
SshPtySourceFrame,
|
||||
'relayPtyId' | 'deliveryToken' | 'clientGeneration' | 'ownerGeneration'
|
||||
>
|
||||
|
||||
export class SshPtyRetiredSourceDeliveries {
|
||||
private readonly keyByRelayPtyId = new Map<string, string>()
|
||||
|
||||
has(providerGeneration: number, source: SourceDeliveryIdentity): boolean {
|
||||
return (
|
||||
this.keyByRelayPtyId.get(source.relayPtyId) === sourceDeliveryKey(providerGeneration, source)
|
||||
)
|
||||
}
|
||||
|
||||
retire(providerGeneration: number, source: SourceDeliveryIdentity): void {
|
||||
this.keyByRelayPtyId.set(source.relayPtyId, sourceDeliveryKey(providerGeneration, source))
|
||||
}
|
||||
|
||||
activate(relayPtyId: string): void {
|
||||
this.keyByRelayPtyId.delete(relayPtyId)
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
this.keyByRelayPtyId.clear()
|
||||
}
|
||||
|
||||
get size(): number {
|
||||
return this.keyByRelayPtyId.size
|
||||
}
|
||||
}
|
||||
|
||||
function sourceDeliveryKey(providerGeneration: number, source: SourceDeliveryIdentity): string {
|
||||
return `${providerGeneration}\0${source.clientGeneration}\0${source.ownerGeneration}\0${source.deliveryToken}`
|
||||
}
|
||||
@@ -104,7 +104,7 @@ describe('cross-version isolation', () => {
|
||||
//
|
||||
// We feed enough exec results to walk through the deploy: platform,
|
||||
// $HOME, isRelayAlreadyInstalled probe, lock acquire, upload (no exec),
|
||||
// npm install, finalize, socket probe, socket poll, then GC scan.
|
||||
// npm install, finalize, socket probe, credential publication, socket poll, then GC scan.
|
||||
const responses: string[] = [
|
||||
'__ORCA_REMOTE_PLATFORM__ Linux x86_64', // tagged POSIX platform probe
|
||||
'/home/u', // echo $HOME
|
||||
@@ -122,6 +122,7 @@ describe('cross-version isolation', () => {
|
||||
'', // rm -f probe-stderr (best-effort cleanup after probe resolved)
|
||||
'', // touch .install-complete (finalizeInstall)
|
||||
'DEAD', // launch socket probe
|
||||
'', // publish the per-launch credential
|
||||
'READY', // socket poll
|
||||
'', // release .install-lock after relay liveness is observable
|
||||
// GC scan begins here
|
||||
|
||||
@@ -9,13 +9,17 @@ import {
|
||||
RELAY_EXIT_CODE_VERSION_MISMATCH
|
||||
} from './ssh-relay-version-mismatch-error'
|
||||
|
||||
function createMockChannel(): ClientChannel {
|
||||
type MockChannel = ClientChannel & {
|
||||
stdin: EventEmitter & { write: ReturnType<typeof vi.fn> }
|
||||
}
|
||||
|
||||
function createMockChannel(): MockChannel {
|
||||
return Object.assign(new EventEmitter(), {
|
||||
stderr: Object.assign(new EventEmitter(), { resume: vi.fn() }),
|
||||
stdin: { write: vi.fn() },
|
||||
stdin: Object.assign(new EventEmitter(), { write: vi.fn(() => true) }),
|
||||
close: vi.fn(),
|
||||
resume: vi.fn()
|
||||
}) as unknown as ClientChannel
|
||||
}) as unknown as MockChannel
|
||||
}
|
||||
|
||||
// execCommand only rejects with Error; narrow the caught reason (its resolve
|
||||
@@ -204,6 +208,35 @@ describe('waitForSentinel', () => {
|
||||
expect(Buffer.concat(chunks)).toEqual(postSentinelPayload)
|
||||
expect(channel.close).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it.each(['ssh2 channel', 'system-SSH child stdio'])(
|
||||
'forwards write(false), callback settlement, and drain for a %s',
|
||||
async (shape) => {
|
||||
const channel = createMockChannel()
|
||||
if (shape.startsWith('system')) {
|
||||
Object.assign(channel, { _process: new EventEmitter() })
|
||||
}
|
||||
const callback = vi.fn()
|
||||
const drain = vi.fn()
|
||||
channel.stdin.write.mockImplementation((...args: unknown[]) => {
|
||||
const onWritten = args.find((arg) => typeof arg === 'function') as (
|
||||
error?: Error | null
|
||||
) => void
|
||||
onWritten(null)
|
||||
return false
|
||||
})
|
||||
const transportPromise = waitForSentinel(channel)
|
||||
channel.emit('data', Buffer.from(RELAY_SENTINEL))
|
||||
const transport = await transportPromise
|
||||
|
||||
transport.onDrain?.(drain)
|
||||
expect(transport.write(Buffer.from('frame'), callback)).toBe(false)
|
||||
expect(callback).toHaveBeenCalledWith({ ok: true })
|
||||
channel.stdin.emit('drain')
|
||||
expect(drain).toHaveBeenCalledOnce()
|
||||
expect(transport.supportsWriteSettlement).toBe(true)
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
describe('execCommand', () => {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user