mirror of
https://github.com/stablyai/orca.git
synced 2026-10-22 08:03:37 +00:00
fix(ssh): handle rejected PTY deliveries with targeted recovery (#12746)
Add targeted recovery for rejected PTY source frames instead of terminating the relay channel. Classify rejection reasons (malformed, generation mismatch, range invalid) and attempt recovery based on the rejection type. Implement admission control at publication time to ensure frames aren't delivered after ownership changes. Bound recovery attempts and retry with backoff to prevent exhaustion. Diagnose and log rejection reasons to aid debugging.
This commit is contained in:
@@ -0,0 +1,183 @@
|
||||
import { expect, it, vi } from 'vitest'
|
||||
import { subscribeSshPtyNotifications } from './ssh-pty-notification-routing'
|
||||
|
||||
it('routes malformed and unadmitted source frames only to rejection diagnostics', async () => {
|
||||
const mux = {
|
||||
onNotification: vi.fn(),
|
||||
request: vi.fn(async (_method: string, params: Record<string, unknown>) => {
|
||||
if (params.deliveryToken !== 'token-1') {
|
||||
throw new Error('Unknown or stale PTY source delivery cancellation')
|
||||
}
|
||||
return { canceled: true, sentEndSu: 0, creditedEndSu: 0 }
|
||||
})
|
||||
}
|
||||
const dataListeners = new Set()
|
||||
const rejectedDataListeners = new Set()
|
||||
const subscription = subscribeSshPtyNotifications({
|
||||
mux: mux as never,
|
||||
toAppPtyId: (id) => `ssh:conn@@${id}`,
|
||||
dataListeners: dataListeners as never,
|
||||
rejectedDataListeners: rejectedDataListeners as never,
|
||||
replayListeners: new Set() as never,
|
||||
exitListeners: new Set() as never,
|
||||
livePtyIds: new Set(),
|
||||
recordExit: vi.fn(),
|
||||
providerGeneration: 7,
|
||||
resolvePtyIncarnation: (id) => `incarnation:${id}`,
|
||||
peekPtyIncarnation: () => undefined
|
||||
})
|
||||
const handler = mux.onNotification.mock.calls[0]?.[0] as (
|
||||
method: string,
|
||||
params: Record<string, unknown>
|
||||
) => void
|
||||
const onData = vi.fn()
|
||||
const onRejectedData = vi.fn()
|
||||
dataListeners.add(onData)
|
||||
rejectedDataListeners.add(onRejectedData)
|
||||
subscription
|
||||
.installReceivingActivation('pty-1', {
|
||||
status: 'pending',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-1',
|
||||
checkpointSourceEndSu: 0,
|
||||
recoveryEndSu: 4
|
||||
})
|
||||
.commit()
|
||||
|
||||
handler('pty.data', {
|
||||
id: 'pty-1',
|
||||
data: 'bad',
|
||||
deliveryToken: 'token-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
sourceEndSu: 4,
|
||||
sourceLengthSu: 4
|
||||
})
|
||||
handler('pty.data', {
|
||||
id: 'pty-2',
|
||||
data: 'old',
|
||||
deliveryToken: 'token-old',
|
||||
clientGeneration: 1,
|
||||
ownerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
sourceEndSu: 3,
|
||||
sourceLengthSu: 3
|
||||
})
|
||||
|
||||
await vi.waitFor(() => expect(onRejectedData).toHaveBeenCalledTimes(2))
|
||||
|
||||
expect(onData).not.toHaveBeenCalled()
|
||||
expect(onRejectedData).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({ sourceMalformed: true })
|
||||
)
|
||||
expect(onRejectedData).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.objectContaining({
|
||||
sourceRejected: true,
|
||||
source: expect.objectContaining({
|
||||
deliveryToken: 'token-old',
|
||||
clientGeneration: 1,
|
||||
ownerGeneration: 1
|
||||
})
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
it('never resolves an incarnation for a rejected frame', async () => {
|
||||
const mux = {
|
||||
onNotification: vi.fn(),
|
||||
request: vi.fn(async () => ({ canceled: true, sentEndSu: 0, creditedEndSu: 0 }))
|
||||
}
|
||||
const rejectedDataListeners = new Set()
|
||||
const resolvePtyIncarnation = vi.fn((id: string) => `incarnation:${id}`)
|
||||
subscribeSshPtyNotifications({
|
||||
mux: mux as never,
|
||||
toAppPtyId: (id) => `ssh:conn@@${id}`,
|
||||
dataListeners: new Set() as never,
|
||||
rejectedDataListeners: rejectedDataListeners as never,
|
||||
replayListeners: new Set() as never,
|
||||
exitListeners: new Set() as never,
|
||||
livePtyIds: new Set(),
|
||||
recordExit: vi.fn(),
|
||||
providerGeneration: 7,
|
||||
resolvePtyIncarnation,
|
||||
peekPtyIncarnation: () => undefined
|
||||
})
|
||||
const handler = mux.onNotification.mock.calls[0]?.[0] as (
|
||||
method: string,
|
||||
params: Record<string, unknown>
|
||||
) => void
|
||||
const onRejectedData = vi.fn()
|
||||
rejectedDataListeners.add(onRejectedData)
|
||||
|
||||
// Why this matters: resolvePtyIncarnation caches what it mints and rememberPtyIncarnation is
|
||||
// first-write-wins, so a synthetic id minted here would fence the PTY's real incarnation off for
|
||||
// the rest of the generation.
|
||||
handler('pty.data', { id: 'pty-1', data: 'bad', deliveryToken: 'token-1', sourceEndSu: 4 })
|
||||
|
||||
await vi.waitFor(() => expect(onRejectedData).toHaveBeenCalledOnce())
|
||||
|
||||
expect(onRejectedData).toHaveBeenCalledOnce()
|
||||
expect(onRejectedData).toHaveBeenCalledWith(expect.objectContaining({ sourceMalformed: true }))
|
||||
expect(resolvePtyIncarnation).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('coalesces repeated exact rejections into one fresh-activation recovery', async () => {
|
||||
const mux = {
|
||||
onNotification: vi.fn(),
|
||||
request: vi.fn(async () => ({ canceled: true, sentEndSu: 3, creditedEndSu: 0 }))
|
||||
}
|
||||
const rejectedDataListeners = new Set()
|
||||
const subscription = subscribeSshPtyNotifications({
|
||||
mux: mux as never,
|
||||
toAppPtyId: (id) => `ssh:conn@@${id}`,
|
||||
dataListeners: new Set() as never,
|
||||
rejectedDataListeners: rejectedDataListeners as never,
|
||||
replayListeners: new Set() as never,
|
||||
exitListeners: new Set() as never,
|
||||
livePtyIds: new Set(),
|
||||
recordExit: vi.fn(),
|
||||
providerGeneration: 7,
|
||||
resolvePtyIncarnation: (id) => `incarnation:${id}`,
|
||||
peekPtyIncarnation: () => 'incarnation-1'
|
||||
})
|
||||
subscription
|
||||
.installReceivingActivation('pty-1', {
|
||||
status: 'pending',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken: 'token-1',
|
||||
checkpointSourceEndSu: 0,
|
||||
recoveryEndSu: 0
|
||||
})
|
||||
.commit()
|
||||
const handler = mux.onNotification.mock.calls[0]?.[0] as (
|
||||
method: string,
|
||||
params: Record<string, unknown>
|
||||
) => void
|
||||
const onRejectedData = vi.fn()
|
||||
rejectedDataListeners.add(onRejectedData)
|
||||
const rejected = {
|
||||
id: 'pty-1',
|
||||
data: 'gap',
|
||||
deliveryToken: 'token-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
sourceEndSu: 6,
|
||||
sourceLengthSu: 3
|
||||
}
|
||||
|
||||
handler('pty.data', rejected)
|
||||
handler('pty.data', rejected)
|
||||
|
||||
await vi.waitFor(() => expect(onRejectedData).toHaveBeenCalledOnce())
|
||||
expect(onRejectedData).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ rejectedSourceRecovery: 'fresh-activation' })
|
||||
)
|
||||
})
|
||||
@@ -29,7 +29,8 @@ function createSubscription() {
|
||||
livePtyIds,
|
||||
recordExit,
|
||||
providerGeneration: 7,
|
||||
resolvePtyIncarnation
|
||||
resolvePtyIncarnation,
|
||||
peekPtyIncarnation: () => undefined
|
||||
})
|
||||
|
||||
const handler = mux.onNotification.mock.calls[0]?.[0] as (
|
||||
|
||||
@@ -7,10 +7,11 @@ import type {
|
||||
SshPtyReplayCallback
|
||||
} from './ssh-pty-provider-contract'
|
||||
import { parseSshPtySourceFrame } from './ssh-pty-source-frame'
|
||||
import {
|
||||
SshPtySourceDeliveryLedger,
|
||||
type PendingSshPtySourceData
|
||||
} from './ssh-pty-source-delivery-ledger'
|
||||
import { SshPtySourceDeliveryLedger } from './ssh-pty-source-delivery-ledger'
|
||||
import type {
|
||||
PendingSshPtySourceData,
|
||||
SshPtyRejectedSourceRecovery
|
||||
} from './ssh-pty-source-delivery-state'
|
||||
|
||||
export type { SshPtyDataCallback, SshPtyExitCallback, SshPtyReplayCallback }
|
||||
export type SshPtyRecoveryActivationLease = Readonly<{
|
||||
@@ -35,18 +36,25 @@ export function subscribeSshPtyNotifications(args: {
|
||||
mux: SshChannelMultiplexer
|
||||
toAppPtyId: (id: string) => string
|
||||
dataListeners: Set<SshPtyDataCallback>
|
||||
rejectedDataListeners?: Set<SshPtyDataCallback>
|
||||
replayListeners: Set<SshPtyReplayCallback>
|
||||
exitListeners: Set<SshPtyExitCallback>
|
||||
livePtyIds: Set<string>
|
||||
recordExit: (relayPtyId: string, incarnationId: unknown) => void
|
||||
providerGeneration: number
|
||||
resolvePtyIncarnation: (relayPtyId: string, incarnationId?: unknown) => string
|
||||
peekPtyIncarnation: (relayPtyId: string) => string | undefined
|
||||
}): SshPtyNotificationSubscription {
|
||||
const toDataPayload = (pending: PendingSshPtySourceData): Parameters<SshPtyDataCallback>[0] => {
|
||||
const toDataPayload = (
|
||||
pending: PendingSshPtySourceData,
|
||||
incarnationOverride?: string
|
||||
): Parameters<SshPtyDataCallback>[0] => {
|
||||
const id = args.toAppPtyId(pending.relayPtyId)
|
||||
const ptyIncarnation = pending.source
|
||||
? (pending.params.ptyIncarnation as string)
|
||||
: args.resolvePtyIncarnation(pending.relayPtyId, pending.params.incarnationId)
|
||||
const ptyIncarnation =
|
||||
incarnationOverride ??
|
||||
(pending.source
|
||||
? (pending.params.ptyIncarnation as string)
|
||||
: args.resolvePtyIncarnation(pending.relayPtyId, pending.params.incarnationId))
|
||||
return {
|
||||
id,
|
||||
data: pending.data,
|
||||
@@ -67,7 +75,80 @@ export function subscribeSshPtyNotifications(args: {
|
||||
listener(payload)
|
||||
}
|
||||
}
|
||||
// Why: a rejected frame is diagnostic and must never mint an incarnation. resolvePtyIncarnation
|
||||
// caches what it synthesizes and rememberPtyIncarnation is first-write-wins, so a malformed frame
|
||||
// that lands before the PTY's first good one would pin a `legacy:` id the real attach can never
|
||||
// displace — fencing every later frame of that generation off as a mismatch.
|
||||
const rejectedPtyIncarnation = (pending: PendingSshPtySourceData): string => {
|
||||
const offered = pending.params.ptyIncarnation
|
||||
if (typeof offered === 'string' && offered.length > 0) {
|
||||
return offered
|
||||
}
|
||||
return args.peekPtyIncarnation(pending.relayPtyId) ?? ''
|
||||
}
|
||||
const publishRejectedData = (
|
||||
pending: PendingSshPtySourceData,
|
||||
rejection: 'malformed' | 'unadmitted',
|
||||
recovery: SshPtyRejectedSourceRecovery
|
||||
): void => {
|
||||
const listeners = args.rejectedDataListeners
|
||||
if (!listeners || listeners.size === 0) {
|
||||
return
|
||||
}
|
||||
const payload = {
|
||||
...toDataPayload(pending, rejectedPtyIncarnation(pending)),
|
||||
...(rejection === 'malformed' ? { sourceMalformed: true } : {}),
|
||||
...(rejection === 'unadmitted' ? { sourceRejected: true } : {}),
|
||||
rejectedSourceRecovery: recovery
|
||||
}
|
||||
for (const listener of listeners) {
|
||||
listener(payload)
|
||||
}
|
||||
}
|
||||
const sourceDeliveries = new SshPtySourceDeliveryLedger(args.mux, publishData)
|
||||
const rejectedPublications = new Map<
|
||||
string,
|
||||
{
|
||||
pending: number
|
||||
payload: PendingSshPtySourceData
|
||||
rejection: 'malformed' | 'unadmitted'
|
||||
recovery?: SshPtyRejectedSourceRecovery
|
||||
}
|
||||
>()
|
||||
const rejectSourceData = (
|
||||
pending: PendingSshPtySourceData,
|
||||
rejection: 'malformed' | 'unadmitted'
|
||||
): void => {
|
||||
const batch = rejectedPublications.get(pending.relayPtyId) ?? {
|
||||
pending: 0,
|
||||
payload: pending,
|
||||
rejection
|
||||
}
|
||||
batch.pending++
|
||||
rejectedPublications.set(pending.relayPtyId, batch)
|
||||
void sourceDeliveries
|
||||
.reject(pending.relayPtyId, rejectedSourceIdentity(pending.params))
|
||||
.then((recovery) => {
|
||||
if (
|
||||
!batch.recovery ||
|
||||
rejectedRecoveryPriority(recovery) > rejectedRecoveryPriority(batch.recovery)
|
||||
) {
|
||||
batch.payload = pending
|
||||
batch.rejection = rejection
|
||||
batch.recovery = recovery
|
||||
}
|
||||
})
|
||||
.finally(() => {
|
||||
batch.pending--
|
||||
queueMicrotask(() => {
|
||||
if (batch.pending > 0 || rejectedPublications.get(pending.relayPtyId) !== batch) {
|
||||
return
|
||||
}
|
||||
rejectedPublications.delete(pending.relayPtyId)
|
||||
publishRejectedData(batch.payload, batch.rejection, batch.recovery ?? 'reconnect-channel')
|
||||
})
|
||||
})
|
||||
}
|
||||
const dispose = args.mux.onNotification((method, params) => {
|
||||
// Why: mux delivers every method to generic handlers; non-PTY payloads
|
||||
// (workspace.changed, fs.changed, …) have no `id` and must not reach
|
||||
@@ -82,6 +163,7 @@ export function subscribeSshPtyNotifications(args: {
|
||||
if (method === 'pty.exit') {
|
||||
const id = args.toAppPtyId(relayPtyId)
|
||||
const ptyIncarnation = args.resolvePtyIncarnation(relayPtyId, params.incarnationId)
|
||||
rejectedPublications.delete(relayPtyId)
|
||||
args.recordExit(relayPtyId, params.incarnationId)
|
||||
args.livePtyIds.delete(id)
|
||||
sourceDeliveries.recordExit(relayPtyId)
|
||||
@@ -109,7 +191,8 @@ export function subscribeSshPtyNotifications(args: {
|
||||
const data = typeof params.data === 'string' ? params.data : ''
|
||||
const sourceFrame = parseSshPtySourceFrame(params, data, relayPtyId)
|
||||
if (sourceFrame.malformed) {
|
||||
cancelExactSourceDelivery(args.mux, relayPtyId, params)
|
||||
const pending = Object.freeze({ relayPtyId, params, data })
|
||||
rejectSourceData(pending, 'malformed')
|
||||
return
|
||||
}
|
||||
const pending = Object.freeze({
|
||||
@@ -120,14 +203,17 @@ export function subscribeSshPtyNotifications(args: {
|
||||
})
|
||||
if (sourceFrame.source) {
|
||||
if (!sourceDeliveries.admit({ ...pending, source: sourceFrame.source })) {
|
||||
cancelExactSourceDelivery(args.mux, relayPtyId, params)
|
||||
rejectSourceData(pending, 'unadmitted')
|
||||
}
|
||||
return
|
||||
}
|
||||
publishData(pending)
|
||||
})
|
||||
return Object.freeze({
|
||||
dispose,
|
||||
dispose: () => {
|
||||
rejectedPublications.clear()
|
||||
dispose()
|
||||
},
|
||||
installReceivingActivation: (relayPtyId, activation) => {
|
||||
const lease = sourceDeliveries.install(relayPtyId, activation)
|
||||
return Object.freeze({
|
||||
@@ -140,33 +226,42 @@ export function subscribeSshPtyNotifications(args: {
|
||||
})
|
||||
}
|
||||
|
||||
function cancelExactSourceDelivery(
|
||||
mux: SshChannelMultiplexer,
|
||||
relayPtyId: string,
|
||||
params: {
|
||||
deliveryToken?: unknown
|
||||
clientGeneration?: unknown
|
||||
ownerGeneration?: unknown
|
||||
function rejectedRecoveryPriority(recovery: SshPtyRejectedSourceRecovery): number {
|
||||
if (recovery === 'reconnect-channel') {
|
||||
return 3
|
||||
}
|
||||
): void {
|
||||
return recovery === 'fresh-activation' ? 2 : 1
|
||||
}
|
||||
|
||||
function rejectedSourceIdentity(params: {
|
||||
deliveryToken?: unknown
|
||||
clientGeneration?: unknown
|
||||
ownerGeneration?: unknown
|
||||
ptyIncarnation?: unknown
|
||||
}):
|
||||
| Readonly<{
|
||||
deliveryToken: string
|
||||
clientGeneration: number
|
||||
ownerGeneration: number
|
||||
ptyIncarnation: string
|
||||
}>
|
||||
| undefined {
|
||||
if (
|
||||
typeof params.deliveryToken !== 'string' ||
|
||||
params.deliveryToken.length === 0 ||
|
||||
!positiveSafeInteger(params.clientGeneration) ||
|
||||
!positiveSafeInteger(params.ownerGeneration)
|
||||
!positiveSafeInteger(params.ownerGeneration) ||
|
||||
typeof params.ptyIncarnation !== 'string' ||
|
||||
params.ptyIncarnation.length === 0
|
||||
) {
|
||||
return
|
||||
return undefined
|
||||
}
|
||||
try {
|
||||
void mux
|
||||
.request('pty.cancelDelivery', {
|
||||
id: relayPtyId,
|
||||
clientGeneration: params.clientGeneration,
|
||||
ownerGeneration: params.ownerGeneration,
|
||||
deliveryToken: params.deliveryToken
|
||||
})
|
||||
.catch(() => {})
|
||||
} catch {}
|
||||
return Object.freeze({
|
||||
deliveryToken: params.deliveryToken,
|
||||
clientGeneration: params.clientGeneration,
|
||||
ownerGeneration: params.ownerGeneration,
|
||||
ptyIncarnation: params.ptyIncarnation
|
||||
})
|
||||
}
|
||||
|
||||
function positiveSafeInteger(value: unknown): value is number {
|
||||
|
||||
@@ -27,6 +27,8 @@ export type SshPtyDataCallback = (payload: {
|
||||
sourceEndSu: number
|
||||
}>
|
||||
sourceMalformed?: boolean
|
||||
sourceRejected?: boolean
|
||||
rejectedSourceRecovery?: 'confirm-existing' | 'fresh-activation' | 'reconnect-channel'
|
||||
}) => void
|
||||
export type SshPtyReplayCallback = (payload: { id: string; data: string }) => void
|
||||
export type SshPtyExitCallback = (payload: {
|
||||
|
||||
@@ -14,6 +14,7 @@ import type { PtySourceReceivingActivation } from '../../shared/pty-source-recei
|
||||
|
||||
export class SshPtyProviderOutputState {
|
||||
private readonly dataListeners = new Set<SshPtyDataCallback>()
|
||||
private readonly rejectedDataListeners = new Set<SshPtyDataCallback>()
|
||||
private readonly replayListeners = new Set<SshPtyReplayCallback>()
|
||||
private readonly exitListeners = new Set<SshPtyExitCallback>()
|
||||
private readonly incarnationByRelayPtyId = new Map<string, string>()
|
||||
@@ -34,11 +35,13 @@ export class SshPtyProviderOutputState {
|
||||
this.subscription = subscribeSshPtyNotifications({
|
||||
...args,
|
||||
dataListeners: this.dataListeners,
|
||||
rejectedDataListeners: this.rejectedDataListeners,
|
||||
replayListeners: this.replayListeners,
|
||||
exitListeners: this.exitListeners,
|
||||
providerGeneration,
|
||||
resolvePtyIncarnation: (relayPtyId, incarnationId) =>
|
||||
this.resolvePtyIncarnation(relayPtyId, incarnationId),
|
||||
peekPtyIncarnation: (relayPtyId) => this.incarnationByRelayPtyId.get(relayPtyId),
|
||||
recordExit: (relayPtyId, incarnationId) => {
|
||||
args.recordExit(relayPtyId, incarnationId)
|
||||
this.incarnationByRelayPtyId.delete(relayPtyId)
|
||||
@@ -52,6 +55,7 @@ export class SshPtyProviderOutputState {
|
||||
this.subscription?.dispose()
|
||||
this.subscription = null
|
||||
this.dataListeners.clear()
|
||||
this.rejectedDataListeners.clear()
|
||||
this.replayListeners.clear()
|
||||
this.exitListeners.clear()
|
||||
this.incarnationByRelayPtyId.clear()
|
||||
@@ -63,6 +67,11 @@ export class SshPtyProviderOutputState {
|
||||
return () => this.dataListeners.delete(callback)
|
||||
}
|
||||
|
||||
onRejectedData(callback: SshPtyDataCallback): () => void {
|
||||
this.rejectedDataListeners.add(callback)
|
||||
return () => this.rejectedDataListeners.delete(callback)
|
||||
}
|
||||
|
||||
onReplay(callback: SshPtyReplayCallback): () => void {
|
||||
this.replayListeners.add(callback)
|
||||
return () => this.replayListeners.delete(callback)
|
||||
|
||||
@@ -314,6 +314,8 @@ export class SshPtyProvider implements IPtyProvider {
|
||||
}
|
||||
|
||||
onData = (callback: SshPtyDataCallback): (() => void) => this.outputState.onData(callback)
|
||||
onRejectedData = (callback: SshPtyDataCallback): (() => void) =>
|
||||
this.outputState.onRejectedData(callback)
|
||||
onReplay = (callback: SshPtyReplayCallback): (() => void) => this.outputState.onReplay(callback)
|
||||
onExit = (callback: SshPtyExitCallback): (() => void) => this.outputState.onExit(callback)
|
||||
|
||||
|
||||
@@ -2,6 +2,17 @@ import { describe, expect, it, vi } from 'vitest'
|
||||
import { SshPtySourceDeliveryLedger } from './ssh-pty-source-delivery-ledger'
|
||||
|
||||
describe('SshPtySourceDeliveryLedger', () => {
|
||||
const activation = (deliveryToken: string) =>
|
||||
Object.freeze({
|
||||
status: 'pending' as const,
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
ptyIncarnation: 'incarnation-1',
|
||||
deliveryToken,
|
||||
checkpointSourceEndSu: 0,
|
||||
recoveryEndSu: 0
|
||||
})
|
||||
|
||||
it('retains cancellation ownership when recovery transfer is superseded', async () => {
|
||||
const request = vi.fn(async () => ({ canceled: true, sentEndSu: 0, creditedEndSu: 0 }))
|
||||
const ledger = new SshPtySourceDeliveryLedger({ request } as never, vi.fn())
|
||||
@@ -41,4 +52,44 @@ describe('SshPtySourceDeliveryLedger', () => {
|
||||
deliveryToken: 'token-old'
|
||||
})
|
||||
})
|
||||
|
||||
it('retires an exact rejected activation only after cancellation is proven', async () => {
|
||||
const request = vi.fn(async () => ({ canceled: true, sentEndSu: 4, creditedEndSu: 0 }))
|
||||
const ledger = new SshPtySourceDeliveryLedger({ request } as never, vi.fn())
|
||||
ledger.install('pty-1', activation('token-old')).commit()
|
||||
|
||||
await expect(ledger.reject('pty-1', activation('token-old'))).resolves.toBe('fresh-activation')
|
||||
|
||||
expect(() => ledger.install('pty-1', activation('token-new')).commit()).not.toThrow()
|
||||
})
|
||||
|
||||
it('keeps the current activation when a stale rejected token cannot cancel it', async () => {
|
||||
const publish = vi.fn()
|
||||
const request = vi.fn(async () => {
|
||||
throw new Error('Unknown or stale PTY source delivery cancellation')
|
||||
})
|
||||
const ledger = new SshPtySourceDeliveryLedger({ request } as never, publish)
|
||||
ledger.install('pty-1', activation('token-current')).commit()
|
||||
|
||||
await expect(ledger.reject('pty-1', activation('token-stale'))).resolves.toBe(
|
||||
'confirm-existing'
|
||||
)
|
||||
expect(
|
||||
ledger.admit({
|
||||
relayPtyId: 'pty-1',
|
||||
params: { ptyIncarnation: 'incarnation-1' },
|
||||
data: 'healthy',
|
||||
source: {
|
||||
relayPtyId: 'pty-1',
|
||||
spanId: 'span-1',
|
||||
clientGeneration: 2,
|
||||
ownerGeneration: 3,
|
||||
deliveryToken: 'token-current',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 7
|
||||
}
|
||||
})
|
||||
).toBe(true)
|
||||
expect(publish).toHaveBeenCalledWith(expect.objectContaining({ data: 'healthy' }))
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,40 +1,21 @@
|
||||
import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer'
|
||||
import type { PtySourceReceivingActivation } from '../../shared/pty-source-receiving-activation'
|
||||
import type { SshPtySourceFrame } from './ssh-pty-source-frame'
|
||||
|
||||
export type PendingSshPtySourceData = Readonly<{
|
||||
relayPtyId: string
|
||||
params: Record<string, unknown>
|
||||
data: string
|
||||
source?: SshPtySourceFrame
|
||||
}>
|
||||
|
||||
type SourceDeliveryLeaseState = {
|
||||
phase: 'provisional' | 'recovery' | 'committing' | 'committed' | 'retired'
|
||||
pendingData: PendingSshPtySourceData[]
|
||||
recoverySink?: (pending: PendingSshPtySourceData) => void
|
||||
exited: boolean
|
||||
}
|
||||
|
||||
type SourceDeliveryState = Readonly<{
|
||||
activation: PtySourceReceivingActivation
|
||||
sourceEndSu: number
|
||||
lease: SourceDeliveryLeaseState
|
||||
previous?: SourceDeliveryState
|
||||
}>
|
||||
|
||||
export type SshPtyRecoveryActivationLease = Readonly<{
|
||||
commit: () => void
|
||||
retire: () => void
|
||||
}>
|
||||
|
||||
export type SshPtySourceDeliveryLease = Readonly<{
|
||||
commit: () => void
|
||||
rollback: () => Promise<boolean>
|
||||
transferToRecovery: (
|
||||
sink: (pending: PendingSshPtySourceData) => void
|
||||
) => SshPtyRecoveryActivationLease
|
||||
}>
|
||||
import {
|
||||
acceptsSourceFrame,
|
||||
activePredecessor,
|
||||
sameReceivingActivation,
|
||||
sameRejectedSourceIdentity,
|
||||
settleExactSourceDeliveryCancellation,
|
||||
settledReceivingActivationLease,
|
||||
type PendingSshPtySourceData,
|
||||
type RejectedSourceIdentity,
|
||||
type SourceDeliveryLeaseState,
|
||||
type SourceDeliveryState,
|
||||
type SshPtyRecoveryActivationLease,
|
||||
type SshPtyRejectedSourceRecovery,
|
||||
type SshPtySourceDeliveryLease
|
||||
} from './ssh-pty-source-delivery-state'
|
||||
|
||||
export class SshPtySourceDeliveryLedger {
|
||||
private readonly deliveryByPty = new Map<string, SourceDeliveryState>()
|
||||
@@ -95,6 +76,32 @@ export class SshPtySourceDeliveryLedger {
|
||||
}
|
||||
}
|
||||
|
||||
async reject(
|
||||
relayPtyId: string,
|
||||
identity: RejectedSourceIdentity | undefined
|
||||
): Promise<SshPtyRejectedSourceRecovery> {
|
||||
if (!identity) {
|
||||
return 'reconnect-channel'
|
||||
}
|
||||
const offeredCurrent = this.deliveryByPty.get(relayPtyId)
|
||||
const matched = Boolean(
|
||||
offeredCurrent && sameRejectedSourceIdentity(offeredCurrent.activation, identity)
|
||||
)
|
||||
const canceled = await settleExactSourceDeliveryCancellation(this.mux, relayPtyId, identity)
|
||||
if (matched && canceled) {
|
||||
const current = this.deliveryByPty.get(relayPtyId)
|
||||
if (current && sameRejectedSourceIdentity(current.activation, identity)) {
|
||||
this.retire(relayPtyId, current.previous, current.lease)
|
||||
return 'fresh-activation'
|
||||
}
|
||||
return current ? 'reconnect-channel' : 'fresh-activation'
|
||||
}
|
||||
if (!matched && !canceled) {
|
||||
return 'confirm-existing'
|
||||
}
|
||||
return 'reconnect-channel'
|
||||
}
|
||||
|
||||
private installProvisional(
|
||||
relayPtyId: string,
|
||||
activation: PtySourceReceivingActivation,
|
||||
@@ -245,76 +252,3 @@ export class SshPtySourceDeliveryLedger {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function settledReceivingActivationLease(): SshPtySourceDeliveryLease {
|
||||
return Object.freeze({
|
||||
commit: () => {},
|
||||
rollback: async () => true,
|
||||
transferToRecovery: () => Object.freeze({ commit: () => {}, retire: () => {} })
|
||||
})
|
||||
}
|
||||
|
||||
function activePredecessor(previous?: SourceDeliveryState): SourceDeliveryState | undefined {
|
||||
while (previous?.lease.phase === 'retired') {
|
||||
previous = previous.previous
|
||||
}
|
||||
return previous
|
||||
}
|
||||
|
||||
function sameReceivingActivation(
|
||||
left: PtySourceReceivingActivation,
|
||||
right: PtySourceReceivingActivation
|
||||
): boolean {
|
||||
return (
|
||||
left.clientGeneration === right.clientGeneration &&
|
||||
left.ownerGeneration === right.ownerGeneration &&
|
||||
left.ptyIncarnation === right.ptyIncarnation &&
|
||||
left.deliveryToken === right.deliveryToken &&
|
||||
left.checkpointSourceEndSu === right.checkpointSourceEndSu &&
|
||||
left.recoveryEndSu === right.recoveryEndSu
|
||||
)
|
||||
}
|
||||
|
||||
function acceptsSourceFrame(
|
||||
current: SourceDeliveryState | undefined,
|
||||
params: Record<string, unknown>,
|
||||
source: SshPtySourceFrame
|
||||
): current is SourceDeliveryState {
|
||||
return Boolean(
|
||||
current &&
|
||||
current.lease.phase !== 'retired' &&
|
||||
!current.lease.exited &&
|
||||
current.activation.ptyIncarnation === params.ptyIncarnation &&
|
||||
current.activation.deliveryToken === source.deliveryToken &&
|
||||
current.activation.clientGeneration === source.clientGeneration &&
|
||||
current.activation.ownerGeneration === source.ownerGeneration &&
|
||||
current.sourceEndSu === source.sourceStartSu
|
||||
)
|
||||
}
|
||||
|
||||
async function settleExactSourceDeliveryCancellation(
|
||||
mux: SshChannelMultiplexer,
|
||||
relayPtyId: string,
|
||||
activation: PtySourceReceivingActivation
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
const result = (await mux.request('pty.cancelDelivery', {
|
||||
id: relayPtyId,
|
||||
clientGeneration: activation.clientGeneration,
|
||||
ownerGeneration: activation.ownerGeneration,
|
||||
deliveryToken: activation.deliveryToken
|
||||
})) as Record<string, unknown>
|
||||
return (
|
||||
result.canceled === true &&
|
||||
nonNegativeSafeInteger(result.sentEndSu) &&
|
||||
nonNegativeSafeInteger(result.creditedEndSu) &&
|
||||
result.creditedEndSu <= result.sentEndSu
|
||||
)
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
function nonNegativeSafeInteger(value: unknown): value is number {
|
||||
return Number.isSafeInteger(value) && (value as number) >= 0
|
||||
}
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
import type { SshChannelMultiplexer } from '../ssh/ssh-channel-multiplexer'
|
||||
import type { PtySourceReceivingActivation } from '../../shared/pty-source-receiving-activation'
|
||||
import type { SshPtySourceFrame } from './ssh-pty-source-frame'
|
||||
|
||||
export type PendingSshPtySourceData = Readonly<{
|
||||
relayPtyId: string
|
||||
params: Record<string, unknown>
|
||||
data: string
|
||||
source?: SshPtySourceFrame
|
||||
}>
|
||||
|
||||
export type SourceDeliveryLeaseState = {
|
||||
phase: 'provisional' | 'recovery' | 'committing' | 'committed' | 'retired'
|
||||
pendingData: PendingSshPtySourceData[]
|
||||
recoverySink?: (pending: PendingSshPtySourceData) => void
|
||||
exited: boolean
|
||||
}
|
||||
|
||||
export type SourceDeliveryState = Readonly<{
|
||||
activation: PtySourceReceivingActivation
|
||||
sourceEndSu: number
|
||||
lease: SourceDeliveryLeaseState
|
||||
previous?: SourceDeliveryState
|
||||
}>
|
||||
|
||||
export type SshPtyRecoveryActivationLease = Readonly<{
|
||||
commit: () => void
|
||||
retire: () => void
|
||||
}>
|
||||
|
||||
export type SshPtySourceDeliveryLease = Readonly<{
|
||||
commit: () => void
|
||||
rollback: () => Promise<boolean>
|
||||
transferToRecovery: (
|
||||
sink: (pending: PendingSshPtySourceData) => void
|
||||
) => SshPtyRecoveryActivationLease
|
||||
}>
|
||||
|
||||
export type SshPtyRejectedSourceRecovery =
|
||||
| 'confirm-existing'
|
||||
| 'fresh-activation'
|
||||
| 'reconnect-channel'
|
||||
|
||||
export type RejectedSourceIdentity = Readonly<{
|
||||
clientGeneration: number
|
||||
ownerGeneration: number
|
||||
ptyIncarnation: string
|
||||
deliveryToken: string
|
||||
}>
|
||||
|
||||
export function settledReceivingActivationLease(): SshPtySourceDeliveryLease {
|
||||
return Object.freeze({
|
||||
commit: () => {},
|
||||
rollback: async () => true,
|
||||
transferToRecovery: () => Object.freeze({ commit: () => {}, retire: () => {} })
|
||||
})
|
||||
}
|
||||
|
||||
export function activePredecessor(previous?: SourceDeliveryState): SourceDeliveryState | undefined {
|
||||
while (previous?.lease.phase === 'retired') {
|
||||
previous = previous.previous
|
||||
}
|
||||
return previous
|
||||
}
|
||||
|
||||
export function sameReceivingActivation(
|
||||
left: PtySourceReceivingActivation,
|
||||
right: PtySourceReceivingActivation
|
||||
): boolean {
|
||||
return (
|
||||
left.clientGeneration === right.clientGeneration &&
|
||||
left.ownerGeneration === right.ownerGeneration &&
|
||||
left.ptyIncarnation === right.ptyIncarnation &&
|
||||
left.deliveryToken === right.deliveryToken &&
|
||||
left.checkpointSourceEndSu === right.checkpointSourceEndSu &&
|
||||
left.recoveryEndSu === right.recoveryEndSu
|
||||
)
|
||||
}
|
||||
|
||||
export function sameRejectedSourceIdentity(
|
||||
left: PtySourceReceivingActivation,
|
||||
right: RejectedSourceIdentity
|
||||
): boolean {
|
||||
return (
|
||||
left.clientGeneration === right.clientGeneration &&
|
||||
left.ownerGeneration === right.ownerGeneration &&
|
||||
left.ptyIncarnation === right.ptyIncarnation &&
|
||||
left.deliveryToken === right.deliveryToken
|
||||
)
|
||||
}
|
||||
|
||||
export function acceptsSourceFrame(
|
||||
current: SourceDeliveryState | undefined,
|
||||
params: Record<string, unknown>,
|
||||
source: SshPtySourceFrame
|
||||
): current is SourceDeliveryState {
|
||||
return Boolean(
|
||||
current &&
|
||||
current.lease.phase !== 'retired' &&
|
||||
!current.lease.exited &&
|
||||
current.activation.ptyIncarnation === params.ptyIncarnation &&
|
||||
current.activation.deliveryToken === source.deliveryToken &&
|
||||
current.activation.clientGeneration === source.clientGeneration &&
|
||||
current.activation.ownerGeneration === source.ownerGeneration &&
|
||||
current.sourceEndSu === source.sourceStartSu
|
||||
)
|
||||
}
|
||||
|
||||
export async function settleExactSourceDeliveryCancellation(
|
||||
mux: SshChannelMultiplexer,
|
||||
relayPtyId: string,
|
||||
activation: RejectedSourceIdentity
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
const result = (await mux.request('pty.cancelDelivery', {
|
||||
id: relayPtyId,
|
||||
clientGeneration: activation.clientGeneration,
|
||||
ownerGeneration: activation.ownerGeneration,
|
||||
deliveryToken: activation.deliveryToken
|
||||
})) as Record<string, unknown>
|
||||
return (
|
||||
result.canceled === true &&
|
||||
nonNegativeSafeInteger(result.sentEndSu) &&
|
||||
nonNegativeSafeInteger(result.creditedEndSu) &&
|
||||
result.creditedEndSu <= result.sentEndSu
|
||||
)
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
function nonNegativeSafeInteger(value: unknown): value is number {
|
||||
return Number.isSafeInteger(value) && (value as number) >= 0
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
import type { SshPtyDataCallback } from '../providers/ssh-pty-provider-contract'
|
||||
import type { SshPtyConsumerOwnerState } from './ssh-pty-consumer-session'
|
||||
|
||||
type SshPtyDataPayload = Parameters<SshPtyDataCallback>[0]
|
||||
|
||||
const SSH_PTY_FRAME_REJECTION_LOG_KEY_LIMIT = 12
|
||||
|
||||
export type SshPtyFrameRejectionReason =
|
||||
| 'source-missing'
|
||||
| 'source-malformed'
|
||||
| 'client-generation-stale'
|
||||
| 'owner-generation-stale'
|
||||
| 'client-and-owner-generation-stale'
|
||||
| 'source-range-invalid'
|
||||
|
||||
export type SshPtyFrameRejection = Readonly<{
|
||||
reason: SshPtyFrameRejectionReason
|
||||
action: 'quarantine-legacy-frame' | 'retire-and-reattach-delivery'
|
||||
}>
|
||||
|
||||
// Why a source-less frame is quarantined rather than reattached: it is the signature of legacy
|
||||
// output already in flight when the flow-control grant landed, not of a broken delivery. The relay's
|
||||
// own admission stops publishing those once the grant is live, and source recovery replays from
|
||||
// sourceStartSu, so dropping them loses nothing. Reattaching instead would restart the PTY on every
|
||||
// openClient handshake.
|
||||
export function classifySshPtyFrameRejection(
|
||||
payload: SshPtyDataPayload,
|
||||
owner: SshPtyConsumerOwnerState | null
|
||||
): SshPtyFrameRejection | null {
|
||||
if (payload.sourceMalformed) {
|
||||
return Object.freeze({
|
||||
reason: 'source-malformed',
|
||||
action: 'retire-and-reattach-delivery'
|
||||
})
|
||||
}
|
||||
const source = payload.source
|
||||
if (!source) {
|
||||
return owner?.outputFlowControl
|
||||
? Object.freeze({
|
||||
reason: 'source-missing',
|
||||
action: 'quarantine-legacy-frame'
|
||||
})
|
||||
: null
|
||||
}
|
||||
if (!owner?.outputFlowControl) {
|
||||
return payload.sourceRejected
|
||||
? Object.freeze({
|
||||
reason: 'source-range-invalid',
|
||||
action: 'retire-and-reattach-delivery'
|
||||
})
|
||||
: null
|
||||
}
|
||||
const staleClient = source.clientGeneration !== owner.clientGeneration
|
||||
const staleOwner = source.ownerGeneration !== owner.ownerGeneration
|
||||
if (staleClient || staleOwner) {
|
||||
return Object.freeze({
|
||||
reason:
|
||||
staleClient && staleOwner
|
||||
? 'client-and-owner-generation-stale'
|
||||
: staleClient
|
||||
? 'client-generation-stale'
|
||||
: 'owner-generation-stale',
|
||||
action: 'retire-and-reattach-delivery'
|
||||
})
|
||||
}
|
||||
if (payload.sourceRejected) {
|
||||
return Object.freeze({
|
||||
reason: 'source-range-invalid',
|
||||
action: 'retire-and-reattach-delivery'
|
||||
})
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Why bounded: a quarantined frame carries no delivery token to retire, so nothing dedupes it — a
|
||||
* relay that keeps publishing legacy output to a flow-control owner would log once per frame and
|
||||
* bury every other diagnostic. One line per PTY and reason per generation names the fault; the
|
||||
* generation is itself the counter for how often it recurs.
|
||||
*/
|
||||
export class SshPtyFrameRejectionLog {
|
||||
private generation: number | null = null
|
||||
private readonly loggedKeys = new Set<string>()
|
||||
|
||||
record(
|
||||
payload: SshPtyDataPayload,
|
||||
owner: SshPtyConsumerOwnerState | null,
|
||||
rejection: SshPtyFrameRejection
|
||||
): void {
|
||||
if (this.generation !== payload.providerGeneration) {
|
||||
this.generation = payload.providerGeneration
|
||||
this.loggedKeys.clear()
|
||||
}
|
||||
const key = `${payload.id}\0${rejection.reason}`
|
||||
if (this.loggedKeys.has(key) || this.loggedKeys.size >= SSH_PTY_FRAME_REJECTION_LOG_KEY_LIMIT) {
|
||||
return
|
||||
}
|
||||
this.loggedKeys.add(key)
|
||||
console.warn('[ssh-relay-session] Rejected PTY delivery', {
|
||||
ptyId: payload.id,
|
||||
providerGeneration: payload.providerGeneration,
|
||||
expectedClientGeneration: owner?.clientGeneration ?? null,
|
||||
expectedOwnerGeneration: owner?.ownerGeneration ?? null,
|
||||
offeredClientGeneration: payload.source?.clientGeneration ?? null,
|
||||
offeredOwnerGeneration: payload.source?.ownerGeneration ?? null,
|
||||
sourceState: rejection.reason,
|
||||
recoveryAction: rejection.action
|
||||
})
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
this.generation = null
|
||||
this.loggedKeys.clear()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
type TargetedReattach = Readonly<{ start: () => void }>
|
||||
|
||||
// Why bounded: every rejected frame asks for its own attach round trip, so a relay that starts
|
||||
// rejecting across many PTYs at once fans out one concurrent reattach per PTY. The bulk reconnect
|
||||
// path caps the identical work, and each attach also costs a lease read and a lease write.
|
||||
export class SshPtyTargetedReattachQueue {
|
||||
private readonly running = new Map<string, TargetedReattach>()
|
||||
private readonly waiting: TargetedReattach[] = []
|
||||
private active = 0
|
||||
|
||||
constructor(private readonly maxConcurrency: number) {}
|
||||
|
||||
has(key: string): boolean {
|
||||
return this.running.has(key)
|
||||
}
|
||||
|
||||
run(key: string, task: () => Promise<boolean>): Promise<boolean> {
|
||||
return new Promise<boolean>((resolve, reject) => {
|
||||
const entry: TargetedReattach = {
|
||||
start: () => {
|
||||
this.active++
|
||||
task().then(
|
||||
(recovered) => {
|
||||
this.settle(key, entry)
|
||||
resolve(recovered)
|
||||
},
|
||||
(error: unknown) => {
|
||||
this.settle(key, entry)
|
||||
reject(error instanceof Error ? error : new Error(String(error)))
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
this.running.set(key, entry)
|
||||
if (this.active < this.maxConcurrency) {
|
||||
entry.start()
|
||||
return
|
||||
}
|
||||
this.waiting.push(entry)
|
||||
})
|
||||
}
|
||||
|
||||
// Why queued entries are dropped rather than started: each one is keyed to the provider generation
|
||||
// the teardown just ended, so running it would attach onto a mux that is already gone.
|
||||
clear(): void {
|
||||
this.waiting.length = 0
|
||||
this.running.clear()
|
||||
}
|
||||
|
||||
private settle(key: string, entry: TargetedReattach): void {
|
||||
if (this.running.get(key) === entry) {
|
||||
this.running.delete(key)
|
||||
}
|
||||
this.active--
|
||||
this.waiting.shift()?.start()
|
||||
}
|
||||
}
|
||||
@@ -519,7 +519,7 @@ describe('SshRelaySession data delivery', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('rejects missing negotiated source identity before main admission', async () => {
|
||||
it('quarantines missing negotiated source identity before main admission', async () => {
|
||||
const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps()
|
||||
const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward)
|
||||
await session.establish(mockConn)
|
||||
@@ -536,10 +536,8 @@ describe('SshRelaySession data delivery', () => {
|
||||
})
|
||||
|
||||
expect(acceptOutputDataMock).not.toHaveBeenCalled()
|
||||
expect(closeSshPtyOutputGeneration).toHaveBeenCalledWith(
|
||||
23,
|
||||
'ssh_source_frame_malformed_or_missing'
|
||||
)
|
||||
expect(closeSshPtyOutputGeneration).not.toHaveBeenCalled()
|
||||
expect(muxDisposeMock).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('keeps unoffered source metadata out of legacy intake', async () => {
|
||||
|
||||
@@ -420,6 +420,7 @@ describe('SshRelaySession recovery race fencing', () => {
|
||||
})
|
||||
await session.reconnect(deps.mockConn)
|
||||
const closeCount = vi.mocked(closeSshPtyOutputGeneration).mock.calls.length
|
||||
const muxDisposeCount = muxDisposeMock.mock.calls.length
|
||||
|
||||
emitSourceFrame({
|
||||
targetId,
|
||||
@@ -431,11 +432,9 @@ describe('SshRelaySession recovery race fencing', () => {
|
||||
})
|
||||
|
||||
expect(acceptOutputDataMock).not.toHaveBeenCalled()
|
||||
expect(closeSshPtyOutputGeneration).toHaveBeenCalledTimes(closeCount + 1)
|
||||
expect(closeSshPtyOutputGeneration).toHaveBeenLastCalledWith(
|
||||
23,
|
||||
'ssh_source_frame_stale_or_non_contiguous'
|
||||
)
|
||||
expect(closeSshPtyOutputGeneration).toHaveBeenCalledTimes(closeCount)
|
||||
expect(muxDisposeMock).toHaveBeenCalledTimes(muxDisposeCount)
|
||||
await vi.waitFor(() => expect(attachForReconnectMock).toHaveBeenCalledTimes(2))
|
||||
})
|
||||
|
||||
it('drops late frames from a token after its cancellation proof is validated', async () => {
|
||||
|
||||
@@ -0,0 +1,496 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { SshPtyDataCallback } from '../providers/ssh-pty-provider-contract'
|
||||
import type { SshPtyProvider } from '../providers/ssh-pty-provider'
|
||||
import type { SshChannelMultiplexer } from './ssh-channel-multiplexer'
|
||||
import type { SshPtyConsumerSessionState } from './ssh-pty-consumer-session'
|
||||
import { SshRelaySession } from './ssh-relay-session'
|
||||
import { createMockDeps } from './ssh-relay-session-test-fixtures'
|
||||
|
||||
const { acceptOutputDataMock, getSshPtyProviderMock } = vi.hoisted(() => ({
|
||||
acceptOutputDataMock: vi.fn().mockResolvedValue(undefined),
|
||||
getSshPtyProviderMock: vi.fn()
|
||||
}))
|
||||
|
||||
vi.mock('../ipc/ssh-pty-output-intake-registry', async (importOriginal) => {
|
||||
const original = (await importOriginal()) as object
|
||||
return { ...original, acceptSshPtyOutputData: acceptOutputDataMock }
|
||||
})
|
||||
|
||||
vi.mock('../ipc/pty', async (importOriginal) => {
|
||||
const original = (await importOriginal()) as object
|
||||
return { ...original, getSshPtyProvider: getSshPtyProviderMock }
|
||||
})
|
||||
|
||||
type SshPtyDataPayload = Parameters<SshPtyDataCallback>[0]
|
||||
|
||||
type RejectedDeliverySession = {
|
||||
mux: SshChannelMultiplexer | null
|
||||
activePtyProviderGeneration: number | null
|
||||
ptyConsumerSessionState: SshPtyConsumerSessionState | null
|
||||
acceptPtyData: (payload: SshPtyDataPayload) => Promise<unknown>
|
||||
reattachKnownPty: (args: {
|
||||
ptyId: string
|
||||
activeLeaseByPtyId: Map<string, unknown>
|
||||
expectedIdentityByPtyId: Map<string, unknown>
|
||||
attachedLeaseIds: Set<string>
|
||||
targetedDeliveryRecovery?: 'confirm-existing' | 'fresh-activation'
|
||||
}) => Promise<void>
|
||||
reattachRejectedPty: (
|
||||
relayPtyId: string,
|
||||
mux: SshChannelMultiplexer,
|
||||
providerGeneration: number,
|
||||
targetedDeliveryRecovery: 'confirm-existing' | 'fresh-activation'
|
||||
) => Promise<boolean>
|
||||
sourceRecoveryRequest: (appPtyId: string) => Promise<
|
||||
| {
|
||||
status: 'checkpoint'
|
||||
clientGeneration: number
|
||||
ownerGeneration: number
|
||||
ptyIncarnation: string
|
||||
deliveryToken: string
|
||||
acceptedSourceEndSu: number
|
||||
}
|
||||
| undefined
|
||||
>
|
||||
rejectedPtyRecoveryAttempts: Map<string, unknown>
|
||||
sourceIdentityByRelayPtyId: Map<string, unknown>
|
||||
retireExitedPty: (payload: {
|
||||
id: string
|
||||
code: number
|
||||
providerGeneration: number
|
||||
ptyIncarnation: string
|
||||
}) => void
|
||||
}
|
||||
|
||||
function source(overrides: Partial<NonNullable<SshPtyDataPayload['source']>> = {}) {
|
||||
return {
|
||||
relayPtyId: 'pty-bad',
|
||||
spanId: 'token-bad:0:4',
|
||||
clientGeneration: 1,
|
||||
ownerGeneration: 1,
|
||||
deliveryToken: 'token-bad',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4,
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
function rejectedPayload(overrides: Partial<SshPtyDataPayload> = {}): SshPtyDataPayload {
|
||||
return {
|
||||
id: 'ssh:target-1@@pty-bad',
|
||||
data: 'rejected',
|
||||
providerGeneration: 23,
|
||||
ptyIncarnation: 'incarnation-bad',
|
||||
source: source(),
|
||||
sourceRejected: true,
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
function prepareSession() {
|
||||
const deps = createMockDeps()
|
||||
const mux = {
|
||||
isDisposed: vi.fn(() => false),
|
||||
dispose: vi.fn()
|
||||
} as unknown as SshChannelMultiplexer
|
||||
const session = new SshRelaySession(
|
||||
'target-1',
|
||||
deps.getMainWindow,
|
||||
deps.mockStore,
|
||||
deps.mockPortForward
|
||||
)
|
||||
const internals = session as unknown as RejectedDeliverySession
|
||||
internals.mux = mux
|
||||
internals.activePtyProviderGeneration = 23
|
||||
internals.ptyConsumerSessionState = {
|
||||
mode: 'negotiated',
|
||||
clientInstanceId: 'client-1',
|
||||
clientGeneration: 1,
|
||||
ownerGeneration: 1,
|
||||
ownerLease: 'owner-lease',
|
||||
outputFlowControl: { version: 1, windowSu: 64 }
|
||||
}
|
||||
return { deps, internals, mux, session }
|
||||
}
|
||||
|
||||
describe('SshRelaySession rejected PTY delivery recovery', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
it('quarantines source-less output queued before a flow-control grant', async () => {
|
||||
const { internals, mux } = prepareSession()
|
||||
const reattach = vi.fn().mockResolvedValue(true)
|
||||
internals.reattachRejectedPty = reattach
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
|
||||
await internals.acceptPtyData(rejectedPayload({ source: undefined, sourceRejected: undefined }))
|
||||
|
||||
expect(warn).toHaveBeenCalledWith(
|
||||
'[ssh-relay-session] Rejected PTY delivery',
|
||||
expect.objectContaining({
|
||||
sourceState: 'source-missing',
|
||||
recoveryAction: 'quarantine-legacy-frame'
|
||||
})
|
||||
)
|
||||
expect(reattach).not.toHaveBeenCalled()
|
||||
expect(acceptOutputDataMock).not.toHaveBeenCalled()
|
||||
expect(mux.dispose).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it.each([
|
||||
['client', 9, 1],
|
||||
['owner', 1, 9]
|
||||
] as const)(
|
||||
'retires a superseded %s generation without disposing the mux',
|
||||
async (_generation, clientGeneration, ownerGeneration) => {
|
||||
const { internals, mux } = prepareSession()
|
||||
const reattach = vi.fn().mockResolvedValue(true)
|
||||
internals.reattachRejectedPty = reattach
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
|
||||
await internals.acceptPtyData(
|
||||
rejectedPayload({ source: source({ clientGeneration, ownerGeneration }) })
|
||||
)
|
||||
|
||||
expect(reattach).toHaveBeenCalledWith('pty-bad', mux, 23, 'confirm-existing')
|
||||
expect(acceptOutputDataMock).not.toHaveBeenCalled()
|
||||
expect(mux.dispose).not.toHaveBeenCalled()
|
||||
}
|
||||
)
|
||||
|
||||
it('logs malformed source metadata and reattaches from the exact outer PTY identity', async () => {
|
||||
const { internals } = prepareSession()
|
||||
const reattach = vi.fn().mockResolvedValue(true)
|
||||
internals.reattachRejectedPty = reattach
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
|
||||
await internals.acceptPtyData(
|
||||
rejectedPayload({ source: undefined, sourceMalformed: true, sourceRejected: undefined })
|
||||
)
|
||||
|
||||
expect(warn).toHaveBeenCalledWith(
|
||||
'[ssh-relay-session] Rejected PTY delivery',
|
||||
expect.objectContaining({
|
||||
ptyId: 'ssh:target-1@@pty-bad',
|
||||
providerGeneration: 23,
|
||||
expectedClientGeneration: 1,
|
||||
expectedOwnerGeneration: 1,
|
||||
sourceState: 'source-malformed',
|
||||
recoveryAction: 'retire-and-reattach-delivery'
|
||||
})
|
||||
)
|
||||
expect(reattach).toHaveBeenCalledWith('pty-bad', expect.anything(), 23, 'confirm-existing')
|
||||
expect(acceptOutputDataMock).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('reattaches only the rejected PTY while a healthy sibling keeps delivering', async () => {
|
||||
const { deps, internals, mux } = prepareSession()
|
||||
const provider = {} as SshPtyProvider
|
||||
getSshPtyProviderMock.mockReturnValue(provider)
|
||||
vi.mocked(deps.mockStore.getSshRemotePtyLeases).mockReturnValue([
|
||||
{
|
||||
targetId: 'target-1',
|
||||
ptyId: 'pty-bad',
|
||||
state: 'detached',
|
||||
createdAt: 1,
|
||||
updatedAt: 1
|
||||
},
|
||||
{
|
||||
targetId: 'target-1',
|
||||
ptyId: 'pty-healthy',
|
||||
state: 'attached',
|
||||
createdAt: 1,
|
||||
updatedAt: 1
|
||||
}
|
||||
])
|
||||
const reattachKnownPty = vi.fn(
|
||||
async (args: Parameters<RejectedDeliverySession['reattachKnownPty']>[0]) => {
|
||||
args.attachedLeaseIds.add(args.ptyId)
|
||||
}
|
||||
)
|
||||
internals.reattachKnownPty = reattachKnownPty
|
||||
|
||||
await internals.acceptPtyData(rejectedPayload({ rejectedSourceRecovery: 'fresh-activation' }))
|
||||
await vi.waitFor(() => expect(reattachKnownPty).toHaveBeenCalledOnce())
|
||||
await internals.acceptPtyData(
|
||||
rejectedPayload({
|
||||
id: 'ssh:target-1@@pty-healthy',
|
||||
data: 'healthy',
|
||||
ptyIncarnation: 'incarnation-healthy',
|
||||
source: source({
|
||||
relayPtyId: 'pty-healthy',
|
||||
spanId: 'token-healthy:0:7',
|
||||
deliveryToken: 'token-healthy',
|
||||
sourceEndSu: 7
|
||||
}),
|
||||
sourceRejected: undefined
|
||||
})
|
||||
)
|
||||
|
||||
const recovery = reattachKnownPty.mock.calls[0]?.[0]
|
||||
expect(recovery?.ptyId).toBe('pty-bad')
|
||||
expect(Array.from(recovery?.activeLeaseByPtyId.keys() ?? [])).toEqual(['pty-bad'])
|
||||
expect(Array.from(recovery?.expectedIdentityByPtyId.keys() ?? [])).toEqual([])
|
||||
expect(recovery?.targetedDeliveryRecovery).toBe('fresh-activation')
|
||||
expect(deps.mockStore.markSshRemotePtyLeasesAttachedAsync).toHaveBeenCalledWith('target-1', [
|
||||
'pty-bad'
|
||||
])
|
||||
expect(acceptOutputDataMock).toHaveBeenCalledOnce()
|
||||
expect(acceptOutputDataMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ id: 'ssh:target-1@@pty-healthy', data: 'healthy' })
|
||||
)
|
||||
expect(mux.dispose).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('reconnects instead of canceling an unprovable malformed delivery', async () => {
|
||||
const { internals, mux } = prepareSession()
|
||||
const reattach = vi.fn().mockResolvedValue(true)
|
||||
internals.reattachRejectedPty = reattach
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
|
||||
await internals.acceptPtyData(
|
||||
rejectedPayload({
|
||||
source: undefined,
|
||||
sourceMalformed: true,
|
||||
sourceRejected: undefined,
|
||||
rejectedSourceRecovery: 'reconnect-channel'
|
||||
})
|
||||
)
|
||||
|
||||
expect(mux.dispose).toHaveBeenCalledWith('connection_lost')
|
||||
expect(reattach).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('opens a fresh activation after exact rejected-delivery cancellation', async () => {
|
||||
const { deps, internals, mux } = prepareSession()
|
||||
internals.sourceIdentityByRelayPtyId.set('pty-bad', {
|
||||
clientGeneration: 1,
|
||||
ownerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-bad',
|
||||
deliveryToken: 'token-old',
|
||||
nextSourceSu: 4
|
||||
})
|
||||
const commit = vi.fn()
|
||||
const attachForReconnect = vi.fn(async () => ({
|
||||
incarnationId: 'incarnation-bad',
|
||||
sourceActivation: {
|
||||
status: 'pending' as const,
|
||||
clientGeneration: 1,
|
||||
ownerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-bad',
|
||||
deliveryToken: 'token-fresh',
|
||||
checkpointSourceEndSu: 0,
|
||||
recoveryEndSu: 0
|
||||
},
|
||||
sourceActivationLease: { commit, rollback: vi.fn(async () => true) }
|
||||
}))
|
||||
getSshPtyProviderMock.mockReturnValue({ attachForReconnect } as unknown as SshPtyProvider)
|
||||
|
||||
await expect(
|
||||
internals.reattachRejectedPty('pty-bad', mux, 23, 'fresh-activation')
|
||||
).resolves.toBe(true)
|
||||
|
||||
expect(attachForReconnect).toHaveBeenCalledWith('pty-bad')
|
||||
expect(commit).toHaveBeenCalledOnce()
|
||||
expect(deps.mockStore.markSshRemotePtyLeasesAttachedAsync).toHaveBeenCalledWith('target-1', [
|
||||
'pty-bad'
|
||||
])
|
||||
await internals.acceptPtyData(
|
||||
rejectedPayload({
|
||||
data: 'fresh',
|
||||
sourceRejected: undefined,
|
||||
source: source({ deliveryToken: 'token-fresh' })
|
||||
})
|
||||
)
|
||||
expect(acceptOutputDataMock).toHaveBeenCalledWith(expect.objectContaining({ data: 'fresh' }))
|
||||
expect(mux.dispose).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('accepts an exact existing activation as stale-frame confirmation', async () => {
|
||||
const { internals, mux } = prepareSession()
|
||||
const checkpoint = {
|
||||
status: 'checkpoint' as const,
|
||||
clientGeneration: 1,
|
||||
ownerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-bad',
|
||||
deliveryToken: 'token-current',
|
||||
acceptedSourceEndSu: 7
|
||||
}
|
||||
internals.sourceRecoveryRequest = vi.fn(async () => checkpoint)
|
||||
const commit = vi.fn()
|
||||
const attachForReconnect = vi.fn(async () => ({
|
||||
incarnationId: 'incarnation-bad',
|
||||
sourceActivation: {
|
||||
status: 'pending' as const,
|
||||
clientGeneration: 1,
|
||||
ownerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-bad',
|
||||
deliveryToken: 'token-current',
|
||||
checkpointSourceEndSu: 0,
|
||||
recoveryEndSu: 0
|
||||
},
|
||||
sourceActivationLease: { commit, rollback: vi.fn(async () => true) }
|
||||
}))
|
||||
getSshPtyProviderMock.mockReturnValue({ attachForReconnect } as unknown as SshPtyProvider)
|
||||
|
||||
await expect(
|
||||
internals.reattachRejectedPty('pty-bad', mux, 23, 'confirm-existing')
|
||||
).resolves.toBe(true)
|
||||
|
||||
expect(attachForReconnect).toHaveBeenCalledWith('pty-bad', undefined, checkpoint)
|
||||
expect(commit).toHaveBeenCalledOnce()
|
||||
expect(mux.dispose).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('forgets rejected-delivery recovery history when the PTY exits', () => {
|
||||
const { internals } = prepareSession()
|
||||
internals.rejectedPtyRecoveryAttempts.set('ssh:target-1@@pty-bad', {})
|
||||
|
||||
internals.retireExitedPty({
|
||||
id: 'ssh:target-1@@pty-bad',
|
||||
code: 0,
|
||||
providerGeneration: 23,
|
||||
ptyIncarnation: 'incarnation-bad'
|
||||
})
|
||||
|
||||
expect(internals.rejectedPtyRecoveryAttempts).toHaveLength(0)
|
||||
})
|
||||
|
||||
// Why a channel drop rather than a terminal relay error: a terminal error clears the reconnect
|
||||
// backoff and rotates provider authority, aborting every fs and git request on the target, so one
|
||||
// PTY's undeliverable output would strand the whole connection in manual recovery.
|
||||
it('bounds failed targeted recovery and escalates to a recoverable relay reconnect', async () => {
|
||||
const { internals, mux, session } = prepareSession()
|
||||
const reattach = vi.fn().mockResolvedValue(false)
|
||||
internals.reattachRejectedPty = reattach
|
||||
getSshPtyProviderMock.mockReturnValue({ hasPty: () => true } as unknown as SshPtyProvider)
|
||||
const onTerminalError = vi.fn()
|
||||
session.setOnTerminalRelayError(onTerminalError)
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
|
||||
await internals.acceptPtyData(rejectedPayload())
|
||||
await vi.waitFor(() => expect(mux.dispose).toHaveBeenCalledOnce(), { timeout: 2000 })
|
||||
|
||||
expect(reattach).toHaveBeenCalledTimes(2)
|
||||
expect(reattach.mock.calls).toEqual([
|
||||
['pty-bad', mux, 23, 'confirm-existing'],
|
||||
['pty-bad', mux, 23, 'confirm-existing']
|
||||
])
|
||||
expect(mux.dispose).toHaveBeenCalledWith('connection_lost')
|
||||
expect(warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining('PTY pty-bad delivery recovery exhausted')
|
||||
)
|
||||
expect(onTerminalError).not.toHaveBeenCalled()
|
||||
expect(acceptOutputDataMock).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('stops without an error when the rejected PTY exited during recovery', async () => {
|
||||
const { internals, mux, session } = prepareSession()
|
||||
const reattach = vi.fn().mockResolvedValue(false)
|
||||
internals.reattachRejectedPty = reattach
|
||||
// Why: reattachKnownPty resolves without claiming the lease when the PTY exits mid-attach, so a
|
||||
// plain "not recovered" is indistinguishable from a failure until liveness is checked.
|
||||
getSshPtyProviderMock.mockReturnValue({ hasPty: () => false } as unknown as SshPtyProvider)
|
||||
const onTerminalError = vi.fn()
|
||||
session.setOnTerminalRelayError(onTerminalError)
|
||||
|
||||
await internals.acceptPtyData(rejectedPayload())
|
||||
await vi.waitFor(() => expect(reattach).toHaveBeenCalledOnce())
|
||||
await new Promise((resolve) => setTimeout(resolve, 400))
|
||||
|
||||
expect(reattach).toHaveBeenCalledOnce()
|
||||
expect(onTerminalError).not.toHaveBeenCalled()
|
||||
expect(mux.dispose).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
// Why bounded: the bulk reconnect path caps the identical attach work at 8, and a relay that
|
||||
// starts rejecting across every PTY at once would otherwise open one attach round trip per PTY.
|
||||
it('caps concurrent targeted reattaches and coalesces repeats for one PTY', async () => {
|
||||
const { internals } = prepareSession()
|
||||
getSshPtyProviderMock.mockReturnValue({ hasPty: () => true } as unknown as SshPtyProvider)
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
const release: (() => void)[] = []
|
||||
let active = 0
|
||||
let peak = 0
|
||||
const reattach = vi.fn(async () => {
|
||||
active++
|
||||
peak = Math.max(peak, active)
|
||||
await new Promise<void>((resolve) => release.push(resolve))
|
||||
active--
|
||||
return true
|
||||
})
|
||||
internals.reattachRejectedPty = reattach
|
||||
|
||||
for (let index = 0; index < 20; index++) {
|
||||
const relayPtyId = `pty-${index}`
|
||||
await internals.acceptPtyData(
|
||||
rejectedPayload({
|
||||
id: `ssh:target-1@@${relayPtyId}`,
|
||||
source: source({ relayPtyId, deliveryToken: `token-${index}` })
|
||||
})
|
||||
)
|
||||
// Why twice: a repeat for a PTY already recovering must not consume a second slot.
|
||||
await internals.acceptPtyData(
|
||||
rejectedPayload({
|
||||
id: `ssh:target-1@@${relayPtyId}`,
|
||||
source: source({ relayPtyId, deliveryToken: `token-${index}-repeat` })
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
expect(peak).toBe(8)
|
||||
expect(reattach).toHaveBeenCalledTimes(8)
|
||||
for (const resolve of release.splice(0)) {
|
||||
resolve()
|
||||
}
|
||||
await vi.waitFor(() => expect(reattach).toHaveBeenCalledTimes(16))
|
||||
expect(peak).toBe(8)
|
||||
for (const resolve of release.splice(0)) {
|
||||
resolve()
|
||||
}
|
||||
})
|
||||
|
||||
it('does not let accepted frames refill the recovery budget indefinitely', async () => {
|
||||
const { internals, mux, session } = prepareSession()
|
||||
const reattach = vi.fn().mockResolvedValue(true)
|
||||
internals.reattachRejectedPty = reattach
|
||||
getSshPtyProviderMock.mockReturnValue({ hasPty: () => true } as unknown as SshPtyProvider)
|
||||
const onTerminalError = vi.fn()
|
||||
session.setOnTerminalRelayError(onTerminalError)
|
||||
|
||||
// Why alternating, with a fresh bad token each round: every rejection retires its own delivery,
|
||||
// so a flapping PTY only keeps asking for recovery by moving onto new ones, and the accepted
|
||||
// frame in between is what used to clear the budget outright. Each reattach here succeeds, so
|
||||
// the consecutive budget is cleared legitimately too — only the per-generation ceiling can stop
|
||||
// it.
|
||||
for (let round = 0; round < 40; round++) {
|
||||
await internals.acceptPtyData(
|
||||
rejectedPayload({
|
||||
source: source({
|
||||
spanId: `token-bad-${round}:0:4`,
|
||||
deliveryToken: `token-bad-${round}`,
|
||||
clientGeneration: 9
|
||||
})
|
||||
})
|
||||
)
|
||||
await internals.acceptPtyData(
|
||||
rejectedPayload({
|
||||
sourceRejected: undefined,
|
||||
source: source({
|
||||
spanId: `token-good:${round * 4}:${round * 4 + 4}`,
|
||||
deliveryToken: 'token-good',
|
||||
sourceStartSu: round * 4,
|
||||
sourceEndSu: round * 4 + 4
|
||||
})
|
||||
})
|
||||
)
|
||||
await Promise.resolve()
|
||||
}
|
||||
await vi.waitFor(() => expect(mux.dispose).toHaveBeenCalledOnce(), { timeout: 2000 })
|
||||
|
||||
expect(onTerminalError).not.toHaveBeenCalled()
|
||||
expect(acceptOutputDataMock).toHaveBeenCalledTimes(40)
|
||||
expect(reattach.mock.calls.length).toBeLessThanOrEqual(12)
|
||||
})
|
||||
})
|
||||
@@ -1,5 +1,6 @@
|
||||
import { vi, type Mock } from 'vitest'
|
||||
import type { BrowserWindow } from 'electron'
|
||||
import { PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR } from '../../shared/pty-consumer-session'
|
||||
import type { SshConnection } from './ssh-connection'
|
||||
import type { Store } from '../persistence'
|
||||
import type { SshPortForwardManager } from './ssh-port-forward'
|
||||
@@ -53,3 +54,9 @@ export function mockDeploySuccess(): void {
|
||||
platform: 'linux-x64'
|
||||
})
|
||||
}
|
||||
|
||||
export function createMismatchedOwnerRecoveryError(): unknown {
|
||||
return Object.assign(new Error('Owner recovery lease is stale'), {
|
||||
code: PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR
|
||||
})
|
||||
}
|
||||
|
||||
@@ -79,6 +79,7 @@ import {
|
||||
import type { Store } from '../persistence'
|
||||
import type { OrcaRuntimeService } from '../runtime/orca-runtime'
|
||||
import { DEFAULT_PTY_SOURCE_WINDOW_SU } from '../../shared/pty-source-credit-contract'
|
||||
import { PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR } from '../../shared/pty-consumer-session'
|
||||
import {
|
||||
isSshOwnerAdmissionBlocked,
|
||||
retrySshOwnerRecoveryWhileBlocked
|
||||
@@ -121,6 +122,8 @@ import {
|
||||
getSshPtyConsumerRecovery,
|
||||
rememberSshPtyConsumerRecovery
|
||||
} from './ssh-pty-consumer-recovery'
|
||||
import { classifySshPtyFrameRejection, SshPtyFrameRejectionLog } from './ssh-pty-frame-rejection'
|
||||
import { SshPtyTargetedReattachQueue } from './ssh-pty-targeted-reattach-queue'
|
||||
|
||||
export type RelaySessionState = 'idle' | 'deploying' | 'ready' | 'reconnecting' | 'disposed'
|
||||
|
||||
@@ -131,6 +134,12 @@ const SSH_PTY_REATTACH_MAX_CONCURRENCY = 8
|
||||
const SSH_PTY_REATTACH_ATTEMPT_TIMEOUT_MS = 10_000
|
||||
const SSH_PTY_REATTACH_RETRY_MIN_DELAY_MS = 50
|
||||
const SSH_PTY_REATTACH_RETRY_JITTER_MS = 200
|
||||
const SSH_REJECTED_PTY_RECOVERY_MAX_ATTEMPTS = 2
|
||||
// Why a second ceiling: the consecutive budget resets whenever a reattach succeeds, so a PTY that
|
||||
// alternates recovered and rejected frames would otherwise reattach forever — each one costs a
|
||||
// store read, an attach round trip and a store write.
|
||||
const SSH_REJECTED_PTY_RECOVERY_MAX_GENERATION_ATTEMPTS = 12
|
||||
const SSH_REJECTED_PTY_RECOVERY_RETRY_DELAY_MS = 150
|
||||
const SSH_SOURCE_RECOVERY_CANCELLATION_FAILED = 'ssh_source_recovery_cancellation_failed'
|
||||
|
||||
// Why: superseded attempts stop quietly; a dead mux still owned by this attempt must enter recovery.
|
||||
@@ -179,6 +188,7 @@ type RemoteCliBridgeEnv = {
|
||||
}
|
||||
|
||||
type ExpectedPtyIdentity = { paneKey?: string; tabId?: string }
|
||||
type TargetedDeliveryRecovery = 'confirm-existing' | 'fresh-activation'
|
||||
|
||||
function expectedIdentityForLease(lease: {
|
||||
tabId?: string
|
||||
@@ -324,6 +334,20 @@ export class SshRelaySession {
|
||||
}>
|
||||
>()
|
||||
private readonly retiredSourceDeliveries = new SshPtyRetiredSourceDeliveries()
|
||||
private readonly rejectedPtyRecoveryAttempts = new Map<
|
||||
string,
|
||||
{
|
||||
providerGeneration: number
|
||||
attempts: number
|
||||
generationAttempts: number
|
||||
reported: boolean
|
||||
}
|
||||
>()
|
||||
private readonly rejectedPtyRecoveryRetries = new Set<ReturnType<typeof setTimeout>>()
|
||||
private readonly rejectedPtyReattaches = new SshPtyTargetedReattachQueue(
|
||||
SSH_PTY_REATTACH_MAX_CONCURRENCY
|
||||
)
|
||||
private readonly ptyFrameRejectionLog = new SshPtyFrameRejectionLog()
|
||||
private readonly ptyConsumerClientInstanceId: string
|
||||
private ptyConsumerSessionState: SshPtyConsumerSessionState | null = null
|
||||
private activeCompatibilityAttachmentIds = new Set<string>()
|
||||
@@ -1072,13 +1096,28 @@ export class SshRelaySession {
|
||||
ownsAttempt: () => boolean
|
||||
): Promise<SshPtyConsumerSessionState> {
|
||||
const previousOwner = this.recoverablePtyConsumerOwner(serverBuildId)
|
||||
const options = {
|
||||
const options: OpenSshPtyConsumerSessionOptions = {
|
||||
clientInstanceId: this.ptyConsumerClientInstanceId,
|
||||
expectedServerBuildId: serverBuildId,
|
||||
allowSameBuildLegacyFallback: true,
|
||||
outputFlowControl: { requestedWindowSu: DEFAULT_PTY_SOURCE_WINDOW_SU }
|
||||
}
|
||||
const admission = await this.admitPtyConsumerOwner(mux, previousOwner, options, ownsAttempt)
|
||||
let admission: SshPtyConsumerAdmission
|
||||
try {
|
||||
admission = await this.admitPtyConsumerOwner(mux, previousOwner, options, ownsAttempt)
|
||||
} catch (error) {
|
||||
if (
|
||||
!previousOwner ||
|
||||
(error as { code?: unknown }).code !== PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR
|
||||
) {
|
||||
throw error
|
||||
}
|
||||
this.voidPtyConsumerCheckpoints(previousOwner, ownsAttempt)
|
||||
if (!ownsAttempt()) {
|
||||
throw new Error('Session disposed during owner recovery')
|
||||
}
|
||||
admission = await openSshPtyConsumerSession(mux, options)
|
||||
}
|
||||
if (previousOwner && !admission.resumed) {
|
||||
this.voidPtyConsumerCheckpoints(previousOwner, ownsAttempt)
|
||||
}
|
||||
@@ -1535,6 +1574,13 @@ export class SshRelaySession {
|
||||
unregisterSshGitProvider(this.targetId)
|
||||
this.sourceIdentityByRelayPtyId.clear()
|
||||
this.retiredSourceDeliveries.clear()
|
||||
this.rejectedPtyRecoveryAttempts.clear()
|
||||
for (const timer of this.rejectedPtyRecoveryRetries) {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
this.rejectedPtyRecoveryRetries.clear()
|
||||
this.rejectedPtyReattaches.clear()
|
||||
this.ptyFrameRejectionLog.clear()
|
||||
for (const pending of this.pendingPtyReattaches.values()) {
|
||||
for (const resolve of pending.recoveryWaiters) {
|
||||
resolve()
|
||||
@@ -1637,6 +1683,24 @@ export class SshRelaySession {
|
||||
}
|
||||
void this.acceptPtyData(payload).catch(() => {})
|
||||
})
|
||||
ptyProvider.onRejectedData?.((payload) => {
|
||||
if (
|
||||
this.mux !== mux ||
|
||||
this.activePtyProviderGeneration !== providerGeneration ||
|
||||
payload.providerGeneration !== providerGeneration
|
||||
) {
|
||||
return
|
||||
}
|
||||
const pending = this.pendingPtyReattaches.get(payload.id)
|
||||
if (pending) {
|
||||
pending.restoreRequired = payload.sourceMalformed
|
||||
? 'recoverySourceMalformed'
|
||||
: 'recoverySourceUnadmitted'
|
||||
this.wakeRecovery(pending)
|
||||
return
|
||||
}
|
||||
void this.acceptPtyData(payload).catch(() => {})
|
||||
})
|
||||
ptyProvider.onReplay((payload) => {
|
||||
if (this.mux !== mux || this.activePtyProviderGeneration !== providerGeneration) {
|
||||
return
|
||||
@@ -1677,22 +1741,19 @@ export class SshRelaySession {
|
||||
) {
|
||||
return Promise.resolve()
|
||||
}
|
||||
if (
|
||||
consumerOwner?.outputFlowControl &&
|
||||
(!offeredSource ||
|
||||
payload.sourceMalformed ||
|
||||
offeredSource.clientGeneration !== consumerOwner.clientGeneration ||
|
||||
offeredSource.ownerGeneration !== consumerOwner.ownerGeneration)
|
||||
) {
|
||||
closeSshPtyOutputGeneration(
|
||||
payload.providerGeneration,
|
||||
'ssh_source_frame_malformed_or_missing'
|
||||
)
|
||||
this.mux?.dispose('connection_lost')
|
||||
return Promise.reject(new Error('ssh_source_frame_malformed_or_missing'))
|
||||
const rejection = classifySshPtyFrameRejection(payload, consumerOwner)
|
||||
if (rejection) {
|
||||
if (offeredSource) {
|
||||
this.retiredSourceDeliveries.retire(payload.providerGeneration, offeredSource)
|
||||
}
|
||||
this.ptyFrameRejectionLog.record(payload, consumerOwner, rejection)
|
||||
if (rejection.action === 'retire-and-reattach-delivery') {
|
||||
this.recoverRejectedPtyDelivery(payload, offeredSource)
|
||||
}
|
||||
return Promise.resolve()
|
||||
}
|
||||
const source = consumerOwner?.outputFlowControl ? offeredSource : undefined
|
||||
if (source) {
|
||||
if (source && consumerOwner) {
|
||||
const current = this.sourceIdentityByRelayPtyId.get(source.relayPtyId)
|
||||
if (
|
||||
source.sourceEndSu <= source.sourceStartSu ||
|
||||
@@ -1703,12 +1764,14 @@ export class SshRelaySession {
|
||||
current.ptyIncarnation !== payload.ptyIncarnation ||
|
||||
(current.nextSourceSu !== undefined && current.nextSourceSu !== source.sourceStartSu)))
|
||||
) {
|
||||
closeSshPtyOutputGeneration(
|
||||
payload.providerGeneration,
|
||||
'ssh_source_frame_stale_or_non_contiguous'
|
||||
)
|
||||
this.mux?.dispose('connection_lost')
|
||||
return Promise.reject(new Error('ssh_source_frame_stale_or_non_contiguous'))
|
||||
const rejection = {
|
||||
reason: 'source-range-invalid',
|
||||
action: 'retire-and-reattach-delivery'
|
||||
} as const
|
||||
this.retiredSourceDeliveries.retire(payload.providerGeneration, source)
|
||||
this.ptyFrameRejectionLog.record(payload, consumerOwner, rejection)
|
||||
this.recoverRejectedPtyDelivery(payload, source)
|
||||
return Promise.resolve()
|
||||
}
|
||||
this.sourceIdentityByRelayPtyId.set(source.relayPtyId, {
|
||||
deliveryToken: source.deliveryToken,
|
||||
@@ -1731,6 +1794,163 @@ export class SshRelaySession {
|
||||
})
|
||||
}
|
||||
|
||||
private recoverRejectedPtyDelivery(
|
||||
payload: SshPtyDataPayload,
|
||||
source: SshPtyDataPayload['source']
|
||||
): void {
|
||||
const mux = this.mux
|
||||
const providerGeneration = this.activePtyProviderGeneration
|
||||
let relayPtyId: string
|
||||
try {
|
||||
relayPtyId = source?.relayPtyId ?? toRelaySshPtyId(this.targetId, payload.id)
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
const appPtyId = toAppSshPtyId(this.targetId, relayPtyId)
|
||||
if (
|
||||
payload.id !== appPtyId ||
|
||||
!mux ||
|
||||
mux.isDisposed() ||
|
||||
providerGeneration !== payload.providerGeneration ||
|
||||
this.pendingPtyReattaches.has(appPtyId) ||
|
||||
this.rejectedPtyReattaches.has(appPtyId)
|
||||
) {
|
||||
return
|
||||
}
|
||||
if (payload.rejectedSourceRecovery === 'reconnect-channel') {
|
||||
console.warn(
|
||||
`[ssh-relay-session] PTY ${relayPtyId} delivery identity could not be retired safely for ${this.targetId}; dropping the relay channel to reconnect`
|
||||
)
|
||||
mux.dispose('connection_lost')
|
||||
return
|
||||
}
|
||||
const previous = this.rejectedPtyRecoveryAttempts.get(appPtyId)
|
||||
const attempt =
|
||||
previous?.providerGeneration === providerGeneration
|
||||
? previous
|
||||
: { providerGeneration, attempts: 0, generationAttempts: 0, reported: false }
|
||||
if (
|
||||
attempt.attempts >= SSH_REJECTED_PTY_RECOVERY_MAX_ATTEMPTS ||
|
||||
attempt.generationAttempts >= SSH_REJECTED_PTY_RECOVERY_MAX_GENERATION_ATTEMPTS
|
||||
) {
|
||||
if (!attempt.reported) {
|
||||
attempt.reported = true
|
||||
console.warn(
|
||||
`[ssh-relay-session] PTY ${relayPtyId} delivery recovery exhausted for ${this.targetId}; dropping the relay channel to reconnect`
|
||||
)
|
||||
// Why a channel drop and not a terminal relay error: a terminal error clears the reconnect
|
||||
// backoff, rotates provider authority (aborting every in-flight fs and git request on the
|
||||
// target) and parks the target in a manual-recovery state — over one PTY's delivery. Losing
|
||||
// the channel is the recoverable escalation, and it is what this path did before targeted
|
||||
// recovery existed.
|
||||
mux.dispose('connection_lost')
|
||||
}
|
||||
return
|
||||
}
|
||||
attempt.attempts++
|
||||
attempt.generationAttempts++
|
||||
this.rejectedPtyRecoveryAttempts.set(appPtyId, attempt)
|
||||
void this.rejectedPtyReattaches
|
||||
.run(appPtyId, () =>
|
||||
this.reattachRejectedPty(
|
||||
relayPtyId,
|
||||
mux,
|
||||
providerGeneration,
|
||||
payload.rejectedSourceRecovery === 'fresh-activation'
|
||||
? 'fresh-activation'
|
||||
: 'confirm-existing'
|
||||
)
|
||||
)
|
||||
.then(
|
||||
(recovered) => {
|
||||
if (recovered) {
|
||||
// Why only a completed reattach clears this: an accepted frame proves nothing about the
|
||||
// delivery that was rejected, and resetting on one lets a flapping PTY reattach forever.
|
||||
attempt.attempts = 0
|
||||
return
|
||||
}
|
||||
this.retryRejectedPtyDelivery(payload, source, appPtyId)
|
||||
},
|
||||
(error: unknown) => {
|
||||
console.warn(`[ssh-relay-session] PTY ${relayPtyId} targeted delivery recovery failed`, {
|
||||
providerGeneration,
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
})
|
||||
this.retryRejectedPtyDelivery(payload, source, appPtyId)
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
// Why liveness is checked before retrying: reattachKnownPty resolves without claiming the lease
|
||||
// when the PTY exited mid-attach, which is indistinguishable from a failed reattach at the call
|
||||
// site. Retrying that race twice would drop the relay channel over an ordinary PTY exit.
|
||||
private retryRejectedPtyDelivery(
|
||||
payload: SshPtyDataPayload,
|
||||
source: SshPtyDataPayload['source'],
|
||||
appPtyId: string
|
||||
): void {
|
||||
const ptyProvider = getSshPtyProvider(this.targetId) as SshPtyProvider | undefined
|
||||
if (!ptyProvider || typeof ptyProvider.hasPty !== 'function' || !ptyProvider.hasPty(appPtyId)) {
|
||||
this.rejectedPtyRecoveryAttempts.delete(appPtyId)
|
||||
return
|
||||
}
|
||||
const timer = setTimeout(() => {
|
||||
this.rejectedPtyRecoveryRetries.delete(timer)
|
||||
this.recoverRejectedPtyDelivery(payload, source)
|
||||
}, SSH_REJECTED_PTY_RECOVERY_RETRY_DELAY_MS)
|
||||
timer.unref?.()
|
||||
this.rejectedPtyRecoveryRetries.add(timer)
|
||||
}
|
||||
|
||||
private async reattachRejectedPty(
|
||||
relayPtyId: string,
|
||||
mux: SshChannelMultiplexer,
|
||||
providerGeneration: number,
|
||||
targetedDeliveryRecovery: TargetedDeliveryRecovery
|
||||
): Promise<boolean> {
|
||||
const shouldContinue = () =>
|
||||
this.mux === mux &&
|
||||
!mux.isDisposed() &&
|
||||
this.activePtyProviderGeneration === providerGeneration
|
||||
const ptyProvider = getSshPtyProvider(this.targetId) as SshPtyProvider | undefined
|
||||
// Why re-checked here: this can have waited for a queue slot, and a superseded generation must
|
||||
// not pay for a lease read or an attach round trip.
|
||||
if (!ptyProvider || !shouldContinue()) {
|
||||
return false
|
||||
}
|
||||
const activeLease = this.store
|
||||
.getSshRemotePtyLeases(this.targetId)
|
||||
.find(
|
||||
(lease) =>
|
||||
lease.ptyId === relayPtyId && lease.state !== 'terminated' && lease.state !== 'expired'
|
||||
)
|
||||
const activeLeaseByPtyId = activeLease
|
||||
? new Map<string, SshPtyLease>([[relayPtyId, activeLease]])
|
||||
: new Map<string, SshPtyLease>()
|
||||
const expectedIdentity = activeLease ? expectedIdentityForLease(activeLease) : undefined
|
||||
const attachedLeaseIds = new Set<string>()
|
||||
await this.reattachKnownPty({
|
||||
ptyProvider,
|
||||
ptyId: relayPtyId,
|
||||
activeLeaseByPtyId,
|
||||
expectedIdentityByPtyId: expectedIdentity
|
||||
? new Map([[relayPtyId, expectedIdentity]])
|
||||
: new Map(),
|
||||
attachedLeaseIds,
|
||||
mux,
|
||||
providerGeneration,
|
||||
shouldContinue,
|
||||
targetedDeliveryRecovery
|
||||
})
|
||||
if (attachedLeaseIds.size > 0 && shouldContinue()) {
|
||||
await this.store.markSshRemotePtyLeasesAttachedAsync(
|
||||
this.targetId,
|
||||
Array.from(attachedLeaseIds)
|
||||
)
|
||||
}
|
||||
return attachedLeaseIds.has(relayPtyId)
|
||||
}
|
||||
|
||||
private quarantineReattachData(pending: PendingPtyReattach, payload: SshPtyDataPayload): void {
|
||||
this.observePrivateRecoveryFrame(pending, payload)
|
||||
if (pending.restoreRequired) {
|
||||
@@ -1943,6 +2163,7 @@ export class SshRelaySession {
|
||||
this.retiredSourceDeliveries.activate(relayPtyId)
|
||||
clearProviderPtyState(payload.id)
|
||||
deletePtyOwnership(payload.id)
|
||||
this.rejectedPtyRecoveryAttempts.delete(payload.id)
|
||||
getSshPtyConsumerRecovery(this.targetId)?.checkpointsByAppPtyId.delete(payload.id)
|
||||
getSshPtyConsumerRecovery(this.targetId)?.checkpointsByAppPtyId.delete(
|
||||
toRelaySshPtyId(this.targetId, payload.id)
|
||||
@@ -2051,6 +2272,7 @@ export class SshRelaySession {
|
||||
mux: SshChannelMultiplexer
|
||||
providerGeneration: number
|
||||
shouldContinue: () => boolean
|
||||
targetedDeliveryRecovery?: TargetedDeliveryRecovery
|
||||
}): Promise<void> {
|
||||
const {
|
||||
ptyProvider,
|
||||
@@ -2060,7 +2282,8 @@ export class SshRelaySession {
|
||||
attachedLeaseIds,
|
||||
mux,
|
||||
providerGeneration,
|
||||
shouldContinue
|
||||
shouldContinue,
|
||||
targetedDeliveryRecovery
|
||||
} = args
|
||||
const appPtyId = toAppSshPtyId(this.targetId, ptyId)
|
||||
const pendingReattach: PendingPtyReattach = {
|
||||
@@ -2079,7 +2302,10 @@ export class SshRelaySession {
|
||||
let sourceActivationLease: SshPtyAttachResult['sourceActivationLease']
|
||||
let recoveryActivationLease: SshPtyRecoveryActivationLease | undefined
|
||||
try {
|
||||
const recoveryRequest = await this.sourceRecoveryRequest(appPtyId)
|
||||
const recoveryRequest =
|
||||
targetedDeliveryRecovery === 'fresh-activation'
|
||||
? undefined
|
||||
: await this.sourceRecoveryRequest(appPtyId)
|
||||
const attachResult = await this.attachPtyWithRetry(
|
||||
ptyProvider,
|
||||
ptyId,
|
||||
@@ -2105,7 +2331,27 @@ export class SshRelaySession {
|
||||
await this.acceptPtyExit(exitDuringAttach)
|
||||
return
|
||||
}
|
||||
if (recoveryRequest) {
|
||||
const existingDeliveryConfirmed =
|
||||
targetedDeliveryRecovery === 'confirm-existing' &&
|
||||
recoveryRequest?.status === 'checkpoint' &&
|
||||
!attachResult.sourceRecovery &&
|
||||
Boolean(
|
||||
attachResult.sourceActivation &&
|
||||
this.sameSourceDelivery(attachResult.sourceActivation, recoveryRequest)
|
||||
)
|
||||
if (targetedDeliveryRecovery) {
|
||||
const owner = this.activePtyConsumerOwner()
|
||||
const activation = attachResult.sourceActivation
|
||||
if (
|
||||
!owner?.outputFlowControl ||
|
||||
!activation ||
|
||||
activation.clientGeneration !== owner.clientGeneration ||
|
||||
activation.ownerGeneration !== owner.ownerGeneration
|
||||
) {
|
||||
return
|
||||
}
|
||||
}
|
||||
if (recoveryRequest && !existingDeliveryConfirmed) {
|
||||
const recovered = await this.finishSourceRecovery(
|
||||
ptyId,
|
||||
appPtyId,
|
||||
@@ -2177,6 +2423,18 @@ export class SshRelaySession {
|
||||
pendingReattach.activated = true
|
||||
recoveryActivationLease?.commit()
|
||||
recoveryActivationLease = undefined
|
||||
if (targetedDeliveryRecovery) {
|
||||
if (targetedDeliveryRecovery === 'fresh-activation') {
|
||||
this.retiredSourceDeliveries.activate(ptyId)
|
||||
this.sourceIdentityByRelayPtyId.delete(ptyId)
|
||||
getSshPtyConsumerRecovery(this.targetId)?.checkpointsByAppPtyId.delete(appPtyId)
|
||||
getSshPtyConsumerRecovery(this.targetId)?.checkpointsByAppPtyId.delete(ptyId)
|
||||
}
|
||||
while (pendingReattach.queuedData.length > 0) {
|
||||
await this.acceptPtyData(pendingReattach.queuedData.shift()!)
|
||||
}
|
||||
pendingReattach.livePassthrough = true
|
||||
}
|
||||
const exitAfterActivation = pendingReattach.exits.find(
|
||||
(exit) =>
|
||||
!exit.incarnationId ||
|
||||
@@ -2187,7 +2445,7 @@ export class SshRelaySession {
|
||||
await this.acceptPtyExit(exitAfterActivation)
|
||||
return
|
||||
}
|
||||
if (!recoveryRequest) {
|
||||
if (!recoveryRequest && !targetedDeliveryRecovery) {
|
||||
this.forwardReattachReplay(appPtyId, attachResult.replay ?? '')
|
||||
}
|
||||
sourceActivationLease?.commit()
|
||||
|
||||
Reference in New Issue
Block a user