mirror of
https://github.com/stablyai/orca.git
synced 2026-10-01 00:02:10 +00:00
fix(ssh): handle rejected PTY deliveries with targeted recovery (#12746)
Add targeted recovery for rejected PTY source frames instead of terminating the relay channel. Classify rejection reasons (malformed, generation mismatch, range invalid) and attempt recovery based on the rejection type. Implement admission control at publication time to ensure frames aren't delivered after ownership changes. Bound recovery attempts and retry with backoff to prevent exhaustion. Diagnose and log rejection reasons to aid debugging.
This commit is contained in:
Jinjing
@@ -0,0 +1,115 @@
|
||||
import type { SshPtyDataCallback } from '../providers/ssh-pty-provider-contract'
|
||||
import type { SshPtyConsumerOwnerState } from './ssh-pty-consumer-session'
|
||||
|
||||
type SshPtyDataPayload = Parameters<SshPtyDataCallback>[0]
|
||||
|
||||
const SSH_PTY_FRAME_REJECTION_LOG_KEY_LIMIT = 12
|
||||
|
||||
export type SshPtyFrameRejectionReason =
|
||||
| 'source-missing'
|
||||
| 'source-malformed'
|
||||
| 'client-generation-stale'
|
||||
| 'owner-generation-stale'
|
||||
| 'client-and-owner-generation-stale'
|
||||
| 'source-range-invalid'
|
||||
|
||||
export type SshPtyFrameRejection = Readonly<{
|
||||
reason: SshPtyFrameRejectionReason
|
||||
action: 'quarantine-legacy-frame' | 'retire-and-reattach-delivery'
|
||||
}>
|
||||
|
||||
// Why a source-less frame is quarantined rather than reattached: it is the signature of legacy
|
||||
// output already in flight when the flow-control grant landed, not of a broken delivery. The relay's
|
||||
// own admission stops publishing those once the grant is live, and source recovery replays from
|
||||
// sourceStartSu, so dropping them loses nothing. Reattaching instead would restart the PTY on every
|
||||
// openClient handshake.
|
||||
export function classifySshPtyFrameRejection(
|
||||
payload: SshPtyDataPayload,
|
||||
owner: SshPtyConsumerOwnerState | null
|
||||
): SshPtyFrameRejection | null {
|
||||
if (payload.sourceMalformed) {
|
||||
return Object.freeze({
|
||||
reason: 'source-malformed',
|
||||
action: 'retire-and-reattach-delivery'
|
||||
})
|
||||
}
|
||||
const source = payload.source
|
||||
if (!source) {
|
||||
return owner?.outputFlowControl
|
||||
? Object.freeze({
|
||||
reason: 'source-missing',
|
||||
action: 'quarantine-legacy-frame'
|
||||
})
|
||||
: null
|
||||
}
|
||||
if (!owner?.outputFlowControl) {
|
||||
return payload.sourceRejected
|
||||
? Object.freeze({
|
||||
reason: 'source-range-invalid',
|
||||
action: 'retire-and-reattach-delivery'
|
||||
})
|
||||
: null
|
||||
}
|
||||
const staleClient = source.clientGeneration !== owner.clientGeneration
|
||||
const staleOwner = source.ownerGeneration !== owner.ownerGeneration
|
||||
if (staleClient || staleOwner) {
|
||||
return Object.freeze({
|
||||
reason:
|
||||
staleClient && staleOwner
|
||||
? 'client-and-owner-generation-stale'
|
||||
: staleClient
|
||||
? 'client-generation-stale'
|
||||
: 'owner-generation-stale',
|
||||
action: 'retire-and-reattach-delivery'
|
||||
})
|
||||
}
|
||||
if (payload.sourceRejected) {
|
||||
return Object.freeze({
|
||||
reason: 'source-range-invalid',
|
||||
action: 'retire-and-reattach-delivery'
|
||||
})
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Why bounded: a quarantined frame carries no delivery token to retire, so nothing dedupes it — a
|
||||
* relay that keeps publishing legacy output to a flow-control owner would log once per frame and
|
||||
* bury every other diagnostic. One line per PTY and reason per generation names the fault; the
|
||||
* generation is itself the counter for how often it recurs.
|
||||
*/
|
||||
export class SshPtyFrameRejectionLog {
|
||||
private generation: number | null = null
|
||||
private readonly loggedKeys = new Set<string>()
|
||||
|
||||
record(
|
||||
payload: SshPtyDataPayload,
|
||||
owner: SshPtyConsumerOwnerState | null,
|
||||
rejection: SshPtyFrameRejection
|
||||
): void {
|
||||
if (this.generation !== payload.providerGeneration) {
|
||||
this.generation = payload.providerGeneration
|
||||
this.loggedKeys.clear()
|
||||
}
|
||||
const key = `${payload.id}\0${rejection.reason}`
|
||||
if (this.loggedKeys.has(key) || this.loggedKeys.size >= SSH_PTY_FRAME_REJECTION_LOG_KEY_LIMIT) {
|
||||
return
|
||||
}
|
||||
this.loggedKeys.add(key)
|
||||
console.warn('[ssh-relay-session] Rejected PTY delivery', {
|
||||
ptyId: payload.id,
|
||||
providerGeneration: payload.providerGeneration,
|
||||
expectedClientGeneration: owner?.clientGeneration ?? null,
|
||||
expectedOwnerGeneration: owner?.ownerGeneration ?? null,
|
||||
offeredClientGeneration: payload.source?.clientGeneration ?? null,
|
||||
offeredOwnerGeneration: payload.source?.ownerGeneration ?? null,
|
||||
sourceState: rejection.reason,
|
||||
recoveryAction: rejection.action
|
||||
})
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
this.generation = null
|
||||
this.loggedKeys.clear()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
type TargetedReattach = Readonly<{ start: () => void }>
|
||||
|
||||
// Why bounded: every rejected frame asks for its own attach round trip, so a relay that starts
|
||||
// rejecting across many PTYs at once fans out one concurrent reattach per PTY. The bulk reconnect
|
||||
// path caps the identical work, and each attach also costs a lease read and a lease write.
|
||||
export class SshPtyTargetedReattachQueue {
|
||||
private readonly running = new Map<string, TargetedReattach>()
|
||||
private readonly waiting: TargetedReattach[] = []
|
||||
private active = 0
|
||||
|
||||
constructor(private readonly maxConcurrency: number) {}
|
||||
|
||||
has(key: string): boolean {
|
||||
return this.running.has(key)
|
||||
}
|
||||
|
||||
run(key: string, task: () => Promise<boolean>): Promise<boolean> {
|
||||
return new Promise<boolean>((resolve, reject) => {
|
||||
const entry: TargetedReattach = {
|
||||
start: () => {
|
||||
this.active++
|
||||
task().then(
|
||||
(recovered) => {
|
||||
this.settle(key, entry)
|
||||
resolve(recovered)
|
||||
},
|
||||
(error: unknown) => {
|
||||
this.settle(key, entry)
|
||||
reject(error instanceof Error ? error : new Error(String(error)))
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
this.running.set(key, entry)
|
||||
if (this.active < this.maxConcurrency) {
|
||||
entry.start()
|
||||
return
|
||||
}
|
||||
this.waiting.push(entry)
|
||||
})
|
||||
}
|
||||
|
||||
// Why queued entries are dropped rather than started: each one is keyed to the provider generation
|
||||
// the teardown just ended, so running it would attach onto a mux that is already gone.
|
||||
clear(): void {
|
||||
this.waiting.length = 0
|
||||
this.running.clear()
|
||||
}
|
||||
|
||||
private settle(key: string, entry: TargetedReattach): void {
|
||||
if (this.running.get(key) === entry) {
|
||||
this.running.delete(key)
|
||||
}
|
||||
this.active--
|
||||
this.waiting.shift()?.start()
|
||||
}
|
||||
}
|
||||
@@ -519,7 +519,7 @@ describe('SshRelaySession data delivery', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('rejects missing negotiated source identity before main admission', async () => {
|
||||
it('quarantines missing negotiated source identity before main admission', async () => {
|
||||
const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps()
|
||||
const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward)
|
||||
await session.establish(mockConn)
|
||||
@@ -536,10 +536,8 @@ describe('SshRelaySession data delivery', () => {
|
||||
})
|
||||
|
||||
expect(acceptOutputDataMock).not.toHaveBeenCalled()
|
||||
expect(closeSshPtyOutputGeneration).toHaveBeenCalledWith(
|
||||
23,
|
||||
'ssh_source_frame_malformed_or_missing'
|
||||
)
|
||||
expect(closeSshPtyOutputGeneration).not.toHaveBeenCalled()
|
||||
expect(muxDisposeMock).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('keeps unoffered source metadata out of legacy intake', async () => {
|
||||
|
||||
@@ -420,6 +420,7 @@ describe('SshRelaySession recovery race fencing', () => {
|
||||
})
|
||||
await session.reconnect(deps.mockConn)
|
||||
const closeCount = vi.mocked(closeSshPtyOutputGeneration).mock.calls.length
|
||||
const muxDisposeCount = muxDisposeMock.mock.calls.length
|
||||
|
||||
emitSourceFrame({
|
||||
targetId,
|
||||
@@ -431,11 +432,9 @@ describe('SshRelaySession recovery race fencing', () => {
|
||||
})
|
||||
|
||||
expect(acceptOutputDataMock).not.toHaveBeenCalled()
|
||||
expect(closeSshPtyOutputGeneration).toHaveBeenCalledTimes(closeCount + 1)
|
||||
expect(closeSshPtyOutputGeneration).toHaveBeenLastCalledWith(
|
||||
23,
|
||||
'ssh_source_frame_stale_or_non_contiguous'
|
||||
)
|
||||
expect(closeSshPtyOutputGeneration).toHaveBeenCalledTimes(closeCount)
|
||||
expect(muxDisposeMock).toHaveBeenCalledTimes(muxDisposeCount)
|
||||
await vi.waitFor(() => expect(attachForReconnectMock).toHaveBeenCalledTimes(2))
|
||||
})
|
||||
|
||||
it('drops late frames from a token after its cancellation proof is validated', async () => {
|
||||
|
||||
@@ -0,0 +1,496 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { SshPtyDataCallback } from '../providers/ssh-pty-provider-contract'
|
||||
import type { SshPtyProvider } from '../providers/ssh-pty-provider'
|
||||
import type { SshChannelMultiplexer } from './ssh-channel-multiplexer'
|
||||
import type { SshPtyConsumerSessionState } from './ssh-pty-consumer-session'
|
||||
import { SshRelaySession } from './ssh-relay-session'
|
||||
import { createMockDeps } from './ssh-relay-session-test-fixtures'
|
||||
|
||||
const { acceptOutputDataMock, getSshPtyProviderMock } = vi.hoisted(() => ({
|
||||
acceptOutputDataMock: vi.fn().mockResolvedValue(undefined),
|
||||
getSshPtyProviderMock: vi.fn()
|
||||
}))
|
||||
|
||||
vi.mock('../ipc/ssh-pty-output-intake-registry', async (importOriginal) => {
|
||||
const original = (await importOriginal()) as object
|
||||
return { ...original, acceptSshPtyOutputData: acceptOutputDataMock }
|
||||
})
|
||||
|
||||
vi.mock('../ipc/pty', async (importOriginal) => {
|
||||
const original = (await importOriginal()) as object
|
||||
return { ...original, getSshPtyProvider: getSshPtyProviderMock }
|
||||
})
|
||||
|
||||
type SshPtyDataPayload = Parameters<SshPtyDataCallback>[0]
|
||||
|
||||
type RejectedDeliverySession = {
|
||||
mux: SshChannelMultiplexer | null
|
||||
activePtyProviderGeneration: number | null
|
||||
ptyConsumerSessionState: SshPtyConsumerSessionState | null
|
||||
acceptPtyData: (payload: SshPtyDataPayload) => Promise<unknown>
|
||||
reattachKnownPty: (args: {
|
||||
ptyId: string
|
||||
activeLeaseByPtyId: Map<string, unknown>
|
||||
expectedIdentityByPtyId: Map<string, unknown>
|
||||
attachedLeaseIds: Set<string>
|
||||
targetedDeliveryRecovery?: 'confirm-existing' | 'fresh-activation'
|
||||
}) => Promise<void>
|
||||
reattachRejectedPty: (
|
||||
relayPtyId: string,
|
||||
mux: SshChannelMultiplexer,
|
||||
providerGeneration: number,
|
||||
targetedDeliveryRecovery: 'confirm-existing' | 'fresh-activation'
|
||||
) => Promise<boolean>
|
||||
sourceRecoveryRequest: (appPtyId: string) => Promise<
|
||||
| {
|
||||
status: 'checkpoint'
|
||||
clientGeneration: number
|
||||
ownerGeneration: number
|
||||
ptyIncarnation: string
|
||||
deliveryToken: string
|
||||
acceptedSourceEndSu: number
|
||||
}
|
||||
| undefined
|
||||
>
|
||||
rejectedPtyRecoveryAttempts: Map<string, unknown>
|
||||
sourceIdentityByRelayPtyId: Map<string, unknown>
|
||||
retireExitedPty: (payload: {
|
||||
id: string
|
||||
code: number
|
||||
providerGeneration: number
|
||||
ptyIncarnation: string
|
||||
}) => void
|
||||
}
|
||||
|
||||
function source(overrides: Partial<NonNullable<SshPtyDataPayload['source']>> = {}) {
|
||||
return {
|
||||
relayPtyId: 'pty-bad',
|
||||
spanId: 'token-bad:0:4',
|
||||
clientGeneration: 1,
|
||||
ownerGeneration: 1,
|
||||
deliveryToken: 'token-bad',
|
||||
sourceStartSu: 0,
|
||||
sourceEndSu: 4,
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
function rejectedPayload(overrides: Partial<SshPtyDataPayload> = {}): SshPtyDataPayload {
|
||||
return {
|
||||
id: 'ssh:target-1@@pty-bad',
|
||||
data: 'rejected',
|
||||
providerGeneration: 23,
|
||||
ptyIncarnation: 'incarnation-bad',
|
||||
source: source(),
|
||||
sourceRejected: true,
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
function prepareSession() {
|
||||
const deps = createMockDeps()
|
||||
const mux = {
|
||||
isDisposed: vi.fn(() => false),
|
||||
dispose: vi.fn()
|
||||
} as unknown as SshChannelMultiplexer
|
||||
const session = new SshRelaySession(
|
||||
'target-1',
|
||||
deps.getMainWindow,
|
||||
deps.mockStore,
|
||||
deps.mockPortForward
|
||||
)
|
||||
const internals = session as unknown as RejectedDeliverySession
|
||||
internals.mux = mux
|
||||
internals.activePtyProviderGeneration = 23
|
||||
internals.ptyConsumerSessionState = {
|
||||
mode: 'negotiated',
|
||||
clientInstanceId: 'client-1',
|
||||
clientGeneration: 1,
|
||||
ownerGeneration: 1,
|
||||
ownerLease: 'owner-lease',
|
||||
outputFlowControl: { version: 1, windowSu: 64 }
|
||||
}
|
||||
return { deps, internals, mux, session }
|
||||
}
|
||||
|
||||
describe('SshRelaySession rejected PTY delivery recovery', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
it('quarantines source-less output queued before a flow-control grant', async () => {
|
||||
const { internals, mux } = prepareSession()
|
||||
const reattach = vi.fn().mockResolvedValue(true)
|
||||
internals.reattachRejectedPty = reattach
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
|
||||
await internals.acceptPtyData(rejectedPayload({ source: undefined, sourceRejected: undefined }))
|
||||
|
||||
expect(warn).toHaveBeenCalledWith(
|
||||
'[ssh-relay-session] Rejected PTY delivery',
|
||||
expect.objectContaining({
|
||||
sourceState: 'source-missing',
|
||||
recoveryAction: 'quarantine-legacy-frame'
|
||||
})
|
||||
)
|
||||
expect(reattach).not.toHaveBeenCalled()
|
||||
expect(acceptOutputDataMock).not.toHaveBeenCalled()
|
||||
expect(mux.dispose).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it.each([
|
||||
['client', 9, 1],
|
||||
['owner', 1, 9]
|
||||
] as const)(
|
||||
'retires a superseded %s generation without disposing the mux',
|
||||
async (_generation, clientGeneration, ownerGeneration) => {
|
||||
const { internals, mux } = prepareSession()
|
||||
const reattach = vi.fn().mockResolvedValue(true)
|
||||
internals.reattachRejectedPty = reattach
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
|
||||
await internals.acceptPtyData(
|
||||
rejectedPayload({ source: source({ clientGeneration, ownerGeneration }) })
|
||||
)
|
||||
|
||||
expect(reattach).toHaveBeenCalledWith('pty-bad', mux, 23, 'confirm-existing')
|
||||
expect(acceptOutputDataMock).not.toHaveBeenCalled()
|
||||
expect(mux.dispose).not.toHaveBeenCalled()
|
||||
}
|
||||
)
|
||||
|
||||
it('logs malformed source metadata and reattaches from the exact outer PTY identity', async () => {
|
||||
const { internals } = prepareSession()
|
||||
const reattach = vi.fn().mockResolvedValue(true)
|
||||
internals.reattachRejectedPty = reattach
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
|
||||
await internals.acceptPtyData(
|
||||
rejectedPayload({ source: undefined, sourceMalformed: true, sourceRejected: undefined })
|
||||
)
|
||||
|
||||
expect(warn).toHaveBeenCalledWith(
|
||||
'[ssh-relay-session] Rejected PTY delivery',
|
||||
expect.objectContaining({
|
||||
ptyId: 'ssh:target-1@@pty-bad',
|
||||
providerGeneration: 23,
|
||||
expectedClientGeneration: 1,
|
||||
expectedOwnerGeneration: 1,
|
||||
sourceState: 'source-malformed',
|
||||
recoveryAction: 'retire-and-reattach-delivery'
|
||||
})
|
||||
)
|
||||
expect(reattach).toHaveBeenCalledWith('pty-bad', expect.anything(), 23, 'confirm-existing')
|
||||
expect(acceptOutputDataMock).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('reattaches only the rejected PTY while a healthy sibling keeps delivering', async () => {
|
||||
const { deps, internals, mux } = prepareSession()
|
||||
const provider = {} as SshPtyProvider
|
||||
getSshPtyProviderMock.mockReturnValue(provider)
|
||||
vi.mocked(deps.mockStore.getSshRemotePtyLeases).mockReturnValue([
|
||||
{
|
||||
targetId: 'target-1',
|
||||
ptyId: 'pty-bad',
|
||||
state: 'detached',
|
||||
createdAt: 1,
|
||||
updatedAt: 1
|
||||
},
|
||||
{
|
||||
targetId: 'target-1',
|
||||
ptyId: 'pty-healthy',
|
||||
state: 'attached',
|
||||
createdAt: 1,
|
||||
updatedAt: 1
|
||||
}
|
||||
])
|
||||
const reattachKnownPty = vi.fn(
|
||||
async (args: Parameters<RejectedDeliverySession['reattachKnownPty']>[0]) => {
|
||||
args.attachedLeaseIds.add(args.ptyId)
|
||||
}
|
||||
)
|
||||
internals.reattachKnownPty = reattachKnownPty
|
||||
|
||||
await internals.acceptPtyData(rejectedPayload({ rejectedSourceRecovery: 'fresh-activation' }))
|
||||
await vi.waitFor(() => expect(reattachKnownPty).toHaveBeenCalledOnce())
|
||||
await internals.acceptPtyData(
|
||||
rejectedPayload({
|
||||
id: 'ssh:target-1@@pty-healthy',
|
||||
data: 'healthy',
|
||||
ptyIncarnation: 'incarnation-healthy',
|
||||
source: source({
|
||||
relayPtyId: 'pty-healthy',
|
||||
spanId: 'token-healthy:0:7',
|
||||
deliveryToken: 'token-healthy',
|
||||
sourceEndSu: 7
|
||||
}),
|
||||
sourceRejected: undefined
|
||||
})
|
||||
)
|
||||
|
||||
const recovery = reattachKnownPty.mock.calls[0]?.[0]
|
||||
expect(recovery?.ptyId).toBe('pty-bad')
|
||||
expect(Array.from(recovery?.activeLeaseByPtyId.keys() ?? [])).toEqual(['pty-bad'])
|
||||
expect(Array.from(recovery?.expectedIdentityByPtyId.keys() ?? [])).toEqual([])
|
||||
expect(recovery?.targetedDeliveryRecovery).toBe('fresh-activation')
|
||||
expect(deps.mockStore.markSshRemotePtyLeasesAttachedAsync).toHaveBeenCalledWith('target-1', [
|
||||
'pty-bad'
|
||||
])
|
||||
expect(acceptOutputDataMock).toHaveBeenCalledOnce()
|
||||
expect(acceptOutputDataMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ id: 'ssh:target-1@@pty-healthy', data: 'healthy' })
|
||||
)
|
||||
expect(mux.dispose).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('reconnects instead of canceling an unprovable malformed delivery', async () => {
|
||||
const { internals, mux } = prepareSession()
|
||||
const reattach = vi.fn().mockResolvedValue(true)
|
||||
internals.reattachRejectedPty = reattach
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
|
||||
await internals.acceptPtyData(
|
||||
rejectedPayload({
|
||||
source: undefined,
|
||||
sourceMalformed: true,
|
||||
sourceRejected: undefined,
|
||||
rejectedSourceRecovery: 'reconnect-channel'
|
||||
})
|
||||
)
|
||||
|
||||
expect(mux.dispose).toHaveBeenCalledWith('connection_lost')
|
||||
expect(reattach).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('opens a fresh activation after exact rejected-delivery cancellation', async () => {
|
||||
const { deps, internals, mux } = prepareSession()
|
||||
internals.sourceIdentityByRelayPtyId.set('pty-bad', {
|
||||
clientGeneration: 1,
|
||||
ownerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-bad',
|
||||
deliveryToken: 'token-old',
|
||||
nextSourceSu: 4
|
||||
})
|
||||
const commit = vi.fn()
|
||||
const attachForReconnect = vi.fn(async () => ({
|
||||
incarnationId: 'incarnation-bad',
|
||||
sourceActivation: {
|
||||
status: 'pending' as const,
|
||||
clientGeneration: 1,
|
||||
ownerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-bad',
|
||||
deliveryToken: 'token-fresh',
|
||||
checkpointSourceEndSu: 0,
|
||||
recoveryEndSu: 0
|
||||
},
|
||||
sourceActivationLease: { commit, rollback: vi.fn(async () => true) }
|
||||
}))
|
||||
getSshPtyProviderMock.mockReturnValue({ attachForReconnect } as unknown as SshPtyProvider)
|
||||
|
||||
await expect(
|
||||
internals.reattachRejectedPty('pty-bad', mux, 23, 'fresh-activation')
|
||||
).resolves.toBe(true)
|
||||
|
||||
expect(attachForReconnect).toHaveBeenCalledWith('pty-bad')
|
||||
expect(commit).toHaveBeenCalledOnce()
|
||||
expect(deps.mockStore.markSshRemotePtyLeasesAttachedAsync).toHaveBeenCalledWith('target-1', [
|
||||
'pty-bad'
|
||||
])
|
||||
await internals.acceptPtyData(
|
||||
rejectedPayload({
|
||||
data: 'fresh',
|
||||
sourceRejected: undefined,
|
||||
source: source({ deliveryToken: 'token-fresh' })
|
||||
})
|
||||
)
|
||||
expect(acceptOutputDataMock).toHaveBeenCalledWith(expect.objectContaining({ data: 'fresh' }))
|
||||
expect(mux.dispose).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('accepts an exact existing activation as stale-frame confirmation', async () => {
|
||||
const { internals, mux } = prepareSession()
|
||||
const checkpoint = {
|
||||
status: 'checkpoint' as const,
|
||||
clientGeneration: 1,
|
||||
ownerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-bad',
|
||||
deliveryToken: 'token-current',
|
||||
acceptedSourceEndSu: 7
|
||||
}
|
||||
internals.sourceRecoveryRequest = vi.fn(async () => checkpoint)
|
||||
const commit = vi.fn()
|
||||
const attachForReconnect = vi.fn(async () => ({
|
||||
incarnationId: 'incarnation-bad',
|
||||
sourceActivation: {
|
||||
status: 'pending' as const,
|
||||
clientGeneration: 1,
|
||||
ownerGeneration: 1,
|
||||
ptyIncarnation: 'incarnation-bad',
|
||||
deliveryToken: 'token-current',
|
||||
checkpointSourceEndSu: 0,
|
||||
recoveryEndSu: 0
|
||||
},
|
||||
sourceActivationLease: { commit, rollback: vi.fn(async () => true) }
|
||||
}))
|
||||
getSshPtyProviderMock.mockReturnValue({ attachForReconnect } as unknown as SshPtyProvider)
|
||||
|
||||
await expect(
|
||||
internals.reattachRejectedPty('pty-bad', mux, 23, 'confirm-existing')
|
||||
).resolves.toBe(true)
|
||||
|
||||
expect(attachForReconnect).toHaveBeenCalledWith('pty-bad', undefined, checkpoint)
|
||||
expect(commit).toHaveBeenCalledOnce()
|
||||
expect(mux.dispose).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('forgets rejected-delivery recovery history when the PTY exits', () => {
|
||||
const { internals } = prepareSession()
|
||||
internals.rejectedPtyRecoveryAttempts.set('ssh:target-1@@pty-bad', {})
|
||||
|
||||
internals.retireExitedPty({
|
||||
id: 'ssh:target-1@@pty-bad',
|
||||
code: 0,
|
||||
providerGeneration: 23,
|
||||
ptyIncarnation: 'incarnation-bad'
|
||||
})
|
||||
|
||||
expect(internals.rejectedPtyRecoveryAttempts).toHaveLength(0)
|
||||
})
|
||||
|
||||
// Why a channel drop rather than a terminal relay error: a terminal error clears the reconnect
|
||||
// backoff and rotates provider authority, aborting every fs and git request on the target, so one
|
||||
// PTY's undeliverable output would strand the whole connection in manual recovery.
|
||||
it('bounds failed targeted recovery and escalates to a recoverable relay reconnect', async () => {
|
||||
const { internals, mux, session } = prepareSession()
|
||||
const reattach = vi.fn().mockResolvedValue(false)
|
||||
internals.reattachRejectedPty = reattach
|
||||
getSshPtyProviderMock.mockReturnValue({ hasPty: () => true } as unknown as SshPtyProvider)
|
||||
const onTerminalError = vi.fn()
|
||||
session.setOnTerminalRelayError(onTerminalError)
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
|
||||
await internals.acceptPtyData(rejectedPayload())
|
||||
await vi.waitFor(() => expect(mux.dispose).toHaveBeenCalledOnce(), { timeout: 2000 })
|
||||
|
||||
expect(reattach).toHaveBeenCalledTimes(2)
|
||||
expect(reattach.mock.calls).toEqual([
|
||||
['pty-bad', mux, 23, 'confirm-existing'],
|
||||
['pty-bad', mux, 23, 'confirm-existing']
|
||||
])
|
||||
expect(mux.dispose).toHaveBeenCalledWith('connection_lost')
|
||||
expect(warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining('PTY pty-bad delivery recovery exhausted')
|
||||
)
|
||||
expect(onTerminalError).not.toHaveBeenCalled()
|
||||
expect(acceptOutputDataMock).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('stops without an error when the rejected PTY exited during recovery', async () => {
|
||||
const { internals, mux, session } = prepareSession()
|
||||
const reattach = vi.fn().mockResolvedValue(false)
|
||||
internals.reattachRejectedPty = reattach
|
||||
// Why: reattachKnownPty resolves without claiming the lease when the PTY exits mid-attach, so a
|
||||
// plain "not recovered" is indistinguishable from a failure until liveness is checked.
|
||||
getSshPtyProviderMock.mockReturnValue({ hasPty: () => false } as unknown as SshPtyProvider)
|
||||
const onTerminalError = vi.fn()
|
||||
session.setOnTerminalRelayError(onTerminalError)
|
||||
|
||||
await internals.acceptPtyData(rejectedPayload())
|
||||
await vi.waitFor(() => expect(reattach).toHaveBeenCalledOnce())
|
||||
await new Promise((resolve) => setTimeout(resolve, 400))
|
||||
|
||||
expect(reattach).toHaveBeenCalledOnce()
|
||||
expect(onTerminalError).not.toHaveBeenCalled()
|
||||
expect(mux.dispose).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
// Why bounded: the bulk reconnect path caps the identical attach work at 8, and a relay that
|
||||
// starts rejecting across every PTY at once would otherwise open one attach round trip per PTY.
|
||||
it('caps concurrent targeted reattaches and coalesces repeats for one PTY', async () => {
|
||||
const { internals } = prepareSession()
|
||||
getSshPtyProviderMock.mockReturnValue({ hasPty: () => true } as unknown as SshPtyProvider)
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
const release: (() => void)[] = []
|
||||
let active = 0
|
||||
let peak = 0
|
||||
const reattach = vi.fn(async () => {
|
||||
active++
|
||||
peak = Math.max(peak, active)
|
||||
await new Promise<void>((resolve) => release.push(resolve))
|
||||
active--
|
||||
return true
|
||||
})
|
||||
internals.reattachRejectedPty = reattach
|
||||
|
||||
for (let index = 0; index < 20; index++) {
|
||||
const relayPtyId = `pty-${index}`
|
||||
await internals.acceptPtyData(
|
||||
rejectedPayload({
|
||||
id: `ssh:target-1@@${relayPtyId}`,
|
||||
source: source({ relayPtyId, deliveryToken: `token-${index}` })
|
||||
})
|
||||
)
|
||||
// Why twice: a repeat for a PTY already recovering must not consume a second slot.
|
||||
await internals.acceptPtyData(
|
||||
rejectedPayload({
|
||||
id: `ssh:target-1@@${relayPtyId}`,
|
||||
source: source({ relayPtyId, deliveryToken: `token-${index}-repeat` })
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
expect(peak).toBe(8)
|
||||
expect(reattach).toHaveBeenCalledTimes(8)
|
||||
for (const resolve of release.splice(0)) {
|
||||
resolve()
|
||||
}
|
||||
await vi.waitFor(() => expect(reattach).toHaveBeenCalledTimes(16))
|
||||
expect(peak).toBe(8)
|
||||
for (const resolve of release.splice(0)) {
|
||||
resolve()
|
||||
}
|
||||
})
|
||||
|
||||
it('does not let accepted frames refill the recovery budget indefinitely', async () => {
|
||||
const { internals, mux, session } = prepareSession()
|
||||
const reattach = vi.fn().mockResolvedValue(true)
|
||||
internals.reattachRejectedPty = reattach
|
||||
getSshPtyProviderMock.mockReturnValue({ hasPty: () => true } as unknown as SshPtyProvider)
|
||||
const onTerminalError = vi.fn()
|
||||
session.setOnTerminalRelayError(onTerminalError)
|
||||
|
||||
// Why alternating, with a fresh bad token each round: every rejection retires its own delivery,
|
||||
// so a flapping PTY only keeps asking for recovery by moving onto new ones, and the accepted
|
||||
// frame in between is what used to clear the budget outright. Each reattach here succeeds, so
|
||||
// the consecutive budget is cleared legitimately too — only the per-generation ceiling can stop
|
||||
// it.
|
||||
for (let round = 0; round < 40; round++) {
|
||||
await internals.acceptPtyData(
|
||||
rejectedPayload({
|
||||
source: source({
|
||||
spanId: `token-bad-${round}:0:4`,
|
||||
deliveryToken: `token-bad-${round}`,
|
||||
clientGeneration: 9
|
||||
})
|
||||
})
|
||||
)
|
||||
await internals.acceptPtyData(
|
||||
rejectedPayload({
|
||||
sourceRejected: undefined,
|
||||
source: source({
|
||||
spanId: `token-good:${round * 4}:${round * 4 + 4}`,
|
||||
deliveryToken: 'token-good',
|
||||
sourceStartSu: round * 4,
|
||||
sourceEndSu: round * 4 + 4
|
||||
})
|
||||
})
|
||||
)
|
||||
await Promise.resolve()
|
||||
}
|
||||
await vi.waitFor(() => expect(mux.dispose).toHaveBeenCalledOnce(), { timeout: 2000 })
|
||||
|
||||
expect(onTerminalError).not.toHaveBeenCalled()
|
||||
expect(acceptOutputDataMock).toHaveBeenCalledTimes(40)
|
||||
expect(reattach.mock.calls.length).toBeLessThanOrEqual(12)
|
||||
})
|
||||
})
|
||||
@@ -1,5 +1,6 @@
|
||||
import { vi, type Mock } from 'vitest'
|
||||
import type { BrowserWindow } from 'electron'
|
||||
import { PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR } from '../../shared/pty-consumer-session'
|
||||
import type { SshConnection } from './ssh-connection'
|
||||
import type { Store } from '../persistence'
|
||||
import type { SshPortForwardManager } from './ssh-port-forward'
|
||||
@@ -53,3 +54,9 @@ export function mockDeploySuccess(): void {
|
||||
platform: 'linux-x64'
|
||||
})
|
||||
}
|
||||
|
||||
export function createMismatchedOwnerRecoveryError(): unknown {
|
||||
return Object.assign(new Error('Owner recovery lease is stale'), {
|
||||
code: PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR
|
||||
})
|
||||
}
|
||||
|
||||
@@ -79,6 +79,7 @@ import {
|
||||
import type { Store } from '../persistence'
|
||||
import type { OrcaRuntimeService } from '../runtime/orca-runtime'
|
||||
import { DEFAULT_PTY_SOURCE_WINDOW_SU } from '../../shared/pty-source-credit-contract'
|
||||
import { PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR } from '../../shared/pty-consumer-session'
|
||||
import {
|
||||
isSshOwnerAdmissionBlocked,
|
||||
retrySshOwnerRecoveryWhileBlocked
|
||||
@@ -121,6 +122,8 @@ import {
|
||||
getSshPtyConsumerRecovery,
|
||||
rememberSshPtyConsumerRecovery
|
||||
} from './ssh-pty-consumer-recovery'
|
||||
import { classifySshPtyFrameRejection, SshPtyFrameRejectionLog } from './ssh-pty-frame-rejection'
|
||||
import { SshPtyTargetedReattachQueue } from './ssh-pty-targeted-reattach-queue'
|
||||
|
||||
export type RelaySessionState = 'idle' | 'deploying' | 'ready' | 'reconnecting' | 'disposed'
|
||||
|
||||
@@ -131,6 +134,12 @@ const SSH_PTY_REATTACH_MAX_CONCURRENCY = 8
|
||||
const SSH_PTY_REATTACH_ATTEMPT_TIMEOUT_MS = 10_000
|
||||
const SSH_PTY_REATTACH_RETRY_MIN_DELAY_MS = 50
|
||||
const SSH_PTY_REATTACH_RETRY_JITTER_MS = 200
|
||||
const SSH_REJECTED_PTY_RECOVERY_MAX_ATTEMPTS = 2
|
||||
// Why a second ceiling: the consecutive budget resets whenever a reattach succeeds, so a PTY that
|
||||
// alternates recovered and rejected frames would otherwise reattach forever — each one costs a
|
||||
// store read, an attach round trip and a store write.
|
||||
const SSH_REJECTED_PTY_RECOVERY_MAX_GENERATION_ATTEMPTS = 12
|
||||
const SSH_REJECTED_PTY_RECOVERY_RETRY_DELAY_MS = 150
|
||||
const SSH_SOURCE_RECOVERY_CANCELLATION_FAILED = 'ssh_source_recovery_cancellation_failed'
|
||||
|
||||
// Why: superseded attempts stop quietly; a dead mux still owned by this attempt must enter recovery.
|
||||
@@ -179,6 +188,7 @@ type RemoteCliBridgeEnv = {
|
||||
}
|
||||
|
||||
type ExpectedPtyIdentity = { paneKey?: string; tabId?: string }
|
||||
type TargetedDeliveryRecovery = 'confirm-existing' | 'fresh-activation'
|
||||
|
||||
function expectedIdentityForLease(lease: {
|
||||
tabId?: string
|
||||
@@ -324,6 +334,20 @@ export class SshRelaySession {
|
||||
}>
|
||||
>()
|
||||
private readonly retiredSourceDeliveries = new SshPtyRetiredSourceDeliveries()
|
||||
private readonly rejectedPtyRecoveryAttempts = new Map<
|
||||
string,
|
||||
{
|
||||
providerGeneration: number
|
||||
attempts: number
|
||||
generationAttempts: number
|
||||
reported: boolean
|
||||
}
|
||||
>()
|
||||
private readonly rejectedPtyRecoveryRetries = new Set<ReturnType<typeof setTimeout>>()
|
||||
private readonly rejectedPtyReattaches = new SshPtyTargetedReattachQueue(
|
||||
SSH_PTY_REATTACH_MAX_CONCURRENCY
|
||||
)
|
||||
private readonly ptyFrameRejectionLog = new SshPtyFrameRejectionLog()
|
||||
private readonly ptyConsumerClientInstanceId: string
|
||||
private ptyConsumerSessionState: SshPtyConsumerSessionState | null = null
|
||||
private activeCompatibilityAttachmentIds = new Set<string>()
|
||||
@@ -1072,13 +1096,28 @@ export class SshRelaySession {
|
||||
ownsAttempt: () => boolean
|
||||
): Promise<SshPtyConsumerSessionState> {
|
||||
const previousOwner = this.recoverablePtyConsumerOwner(serverBuildId)
|
||||
const options = {
|
||||
const options: OpenSshPtyConsumerSessionOptions = {
|
||||
clientInstanceId: this.ptyConsumerClientInstanceId,
|
||||
expectedServerBuildId: serverBuildId,
|
||||
allowSameBuildLegacyFallback: true,
|
||||
outputFlowControl: { requestedWindowSu: DEFAULT_PTY_SOURCE_WINDOW_SU }
|
||||
}
|
||||
const admission = await this.admitPtyConsumerOwner(mux, previousOwner, options, ownsAttempt)
|
||||
let admission: SshPtyConsumerAdmission
|
||||
try {
|
||||
admission = await this.admitPtyConsumerOwner(mux, previousOwner, options, ownsAttempt)
|
||||
} catch (error) {
|
||||
if (
|
||||
!previousOwner ||
|
||||
(error as { code?: unknown }).code !== PTY_CONSUMER_STALE_OWNER_RECOVERY_ERROR
|
||||
) {
|
||||
throw error
|
||||
}
|
||||
this.voidPtyConsumerCheckpoints(previousOwner, ownsAttempt)
|
||||
if (!ownsAttempt()) {
|
||||
throw new Error('Session disposed during owner recovery')
|
||||
}
|
||||
admission = await openSshPtyConsumerSession(mux, options)
|
||||
}
|
||||
if (previousOwner && !admission.resumed) {
|
||||
this.voidPtyConsumerCheckpoints(previousOwner, ownsAttempt)
|
||||
}
|
||||
@@ -1535,6 +1574,13 @@ export class SshRelaySession {
|
||||
unregisterSshGitProvider(this.targetId)
|
||||
this.sourceIdentityByRelayPtyId.clear()
|
||||
this.retiredSourceDeliveries.clear()
|
||||
this.rejectedPtyRecoveryAttempts.clear()
|
||||
for (const timer of this.rejectedPtyRecoveryRetries) {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
this.rejectedPtyRecoveryRetries.clear()
|
||||
this.rejectedPtyReattaches.clear()
|
||||
this.ptyFrameRejectionLog.clear()
|
||||
for (const pending of this.pendingPtyReattaches.values()) {
|
||||
for (const resolve of pending.recoveryWaiters) {
|
||||
resolve()
|
||||
@@ -1637,6 +1683,24 @@ export class SshRelaySession {
|
||||
}
|
||||
void this.acceptPtyData(payload).catch(() => {})
|
||||
})
|
||||
ptyProvider.onRejectedData?.((payload) => {
|
||||
if (
|
||||
this.mux !== mux ||
|
||||
this.activePtyProviderGeneration !== providerGeneration ||
|
||||
payload.providerGeneration !== providerGeneration
|
||||
) {
|
||||
return
|
||||
}
|
||||
const pending = this.pendingPtyReattaches.get(payload.id)
|
||||
if (pending) {
|
||||
pending.restoreRequired = payload.sourceMalformed
|
||||
? 'recoverySourceMalformed'
|
||||
: 'recoverySourceUnadmitted'
|
||||
this.wakeRecovery(pending)
|
||||
return
|
||||
}
|
||||
void this.acceptPtyData(payload).catch(() => {})
|
||||
})
|
||||
ptyProvider.onReplay((payload) => {
|
||||
if (this.mux !== mux || this.activePtyProviderGeneration !== providerGeneration) {
|
||||
return
|
||||
@@ -1677,22 +1741,19 @@ export class SshRelaySession {
|
||||
) {
|
||||
return Promise.resolve()
|
||||
}
|
||||
if (
|
||||
consumerOwner?.outputFlowControl &&
|
||||
(!offeredSource ||
|
||||
payload.sourceMalformed ||
|
||||
offeredSource.clientGeneration !== consumerOwner.clientGeneration ||
|
||||
offeredSource.ownerGeneration !== consumerOwner.ownerGeneration)
|
||||
) {
|
||||
closeSshPtyOutputGeneration(
|
||||
payload.providerGeneration,
|
||||
'ssh_source_frame_malformed_or_missing'
|
||||
)
|
||||
this.mux?.dispose('connection_lost')
|
||||
return Promise.reject(new Error('ssh_source_frame_malformed_or_missing'))
|
||||
const rejection = classifySshPtyFrameRejection(payload, consumerOwner)
|
||||
if (rejection) {
|
||||
if (offeredSource) {
|
||||
this.retiredSourceDeliveries.retire(payload.providerGeneration, offeredSource)
|
||||
}
|
||||
this.ptyFrameRejectionLog.record(payload, consumerOwner, rejection)
|
||||
if (rejection.action === 'retire-and-reattach-delivery') {
|
||||
this.recoverRejectedPtyDelivery(payload, offeredSource)
|
||||
}
|
||||
return Promise.resolve()
|
||||
}
|
||||
const source = consumerOwner?.outputFlowControl ? offeredSource : undefined
|
||||
if (source) {
|
||||
if (source && consumerOwner) {
|
||||
const current = this.sourceIdentityByRelayPtyId.get(source.relayPtyId)
|
||||
if (
|
||||
source.sourceEndSu <= source.sourceStartSu ||
|
||||
@@ -1703,12 +1764,14 @@ export class SshRelaySession {
|
||||
current.ptyIncarnation !== payload.ptyIncarnation ||
|
||||
(current.nextSourceSu !== undefined && current.nextSourceSu !== source.sourceStartSu)))
|
||||
) {
|
||||
closeSshPtyOutputGeneration(
|
||||
payload.providerGeneration,
|
||||
'ssh_source_frame_stale_or_non_contiguous'
|
||||
)
|
||||
this.mux?.dispose('connection_lost')
|
||||
return Promise.reject(new Error('ssh_source_frame_stale_or_non_contiguous'))
|
||||
const rejection = {
|
||||
reason: 'source-range-invalid',
|
||||
action: 'retire-and-reattach-delivery'
|
||||
} as const
|
||||
this.retiredSourceDeliveries.retire(payload.providerGeneration, source)
|
||||
this.ptyFrameRejectionLog.record(payload, consumerOwner, rejection)
|
||||
this.recoverRejectedPtyDelivery(payload, source)
|
||||
return Promise.resolve()
|
||||
}
|
||||
this.sourceIdentityByRelayPtyId.set(source.relayPtyId, {
|
||||
deliveryToken: source.deliveryToken,
|
||||
@@ -1731,6 +1794,163 @@ export class SshRelaySession {
|
||||
})
|
||||
}
|
||||
|
||||
private recoverRejectedPtyDelivery(
|
||||
payload: SshPtyDataPayload,
|
||||
source: SshPtyDataPayload['source']
|
||||
): void {
|
||||
const mux = this.mux
|
||||
const providerGeneration = this.activePtyProviderGeneration
|
||||
let relayPtyId: string
|
||||
try {
|
||||
relayPtyId = source?.relayPtyId ?? toRelaySshPtyId(this.targetId, payload.id)
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
const appPtyId = toAppSshPtyId(this.targetId, relayPtyId)
|
||||
if (
|
||||
payload.id !== appPtyId ||
|
||||
!mux ||
|
||||
mux.isDisposed() ||
|
||||
providerGeneration !== payload.providerGeneration ||
|
||||
this.pendingPtyReattaches.has(appPtyId) ||
|
||||
this.rejectedPtyReattaches.has(appPtyId)
|
||||
) {
|
||||
return
|
||||
}
|
||||
if (payload.rejectedSourceRecovery === 'reconnect-channel') {
|
||||
console.warn(
|
||||
`[ssh-relay-session] PTY ${relayPtyId} delivery identity could not be retired safely for ${this.targetId}; dropping the relay channel to reconnect`
|
||||
)
|
||||
mux.dispose('connection_lost')
|
||||
return
|
||||
}
|
||||
const previous = this.rejectedPtyRecoveryAttempts.get(appPtyId)
|
||||
const attempt =
|
||||
previous?.providerGeneration === providerGeneration
|
||||
? previous
|
||||
: { providerGeneration, attempts: 0, generationAttempts: 0, reported: false }
|
||||
if (
|
||||
attempt.attempts >= SSH_REJECTED_PTY_RECOVERY_MAX_ATTEMPTS ||
|
||||
attempt.generationAttempts >= SSH_REJECTED_PTY_RECOVERY_MAX_GENERATION_ATTEMPTS
|
||||
) {
|
||||
if (!attempt.reported) {
|
||||
attempt.reported = true
|
||||
console.warn(
|
||||
`[ssh-relay-session] PTY ${relayPtyId} delivery recovery exhausted for ${this.targetId}; dropping the relay channel to reconnect`
|
||||
)
|
||||
// Why a channel drop and not a terminal relay error: a terminal error clears the reconnect
|
||||
// backoff, rotates provider authority (aborting every in-flight fs and git request on the
|
||||
// target) and parks the target in a manual-recovery state — over one PTY's delivery. Losing
|
||||
// the channel is the recoverable escalation, and it is what this path did before targeted
|
||||
// recovery existed.
|
||||
mux.dispose('connection_lost')
|
||||
}
|
||||
return
|
||||
}
|
||||
attempt.attempts++
|
||||
attempt.generationAttempts++
|
||||
this.rejectedPtyRecoveryAttempts.set(appPtyId, attempt)
|
||||
void this.rejectedPtyReattaches
|
||||
.run(appPtyId, () =>
|
||||
this.reattachRejectedPty(
|
||||
relayPtyId,
|
||||
mux,
|
||||
providerGeneration,
|
||||
payload.rejectedSourceRecovery === 'fresh-activation'
|
||||
? 'fresh-activation'
|
||||
: 'confirm-existing'
|
||||
)
|
||||
)
|
||||
.then(
|
||||
(recovered) => {
|
||||
if (recovered) {
|
||||
// Why only a completed reattach clears this: an accepted frame proves nothing about the
|
||||
// delivery that was rejected, and resetting on one lets a flapping PTY reattach forever.
|
||||
attempt.attempts = 0
|
||||
return
|
||||
}
|
||||
this.retryRejectedPtyDelivery(payload, source, appPtyId)
|
||||
},
|
||||
(error: unknown) => {
|
||||
console.warn(`[ssh-relay-session] PTY ${relayPtyId} targeted delivery recovery failed`, {
|
||||
providerGeneration,
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
})
|
||||
this.retryRejectedPtyDelivery(payload, source, appPtyId)
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
// Why liveness is checked before retrying: reattachKnownPty resolves without claiming the lease
|
||||
// when the PTY exited mid-attach, which is indistinguishable from a failed reattach at the call
|
||||
// site. Retrying that race twice would drop the relay channel over an ordinary PTY exit.
|
||||
private retryRejectedPtyDelivery(
|
||||
payload: SshPtyDataPayload,
|
||||
source: SshPtyDataPayload['source'],
|
||||
appPtyId: string
|
||||
): void {
|
||||
const ptyProvider = getSshPtyProvider(this.targetId) as SshPtyProvider | undefined
|
||||
if (!ptyProvider || typeof ptyProvider.hasPty !== 'function' || !ptyProvider.hasPty(appPtyId)) {
|
||||
this.rejectedPtyRecoveryAttempts.delete(appPtyId)
|
||||
return
|
||||
}
|
||||
const timer = setTimeout(() => {
|
||||
this.rejectedPtyRecoveryRetries.delete(timer)
|
||||
this.recoverRejectedPtyDelivery(payload, source)
|
||||
}, SSH_REJECTED_PTY_RECOVERY_RETRY_DELAY_MS)
|
||||
timer.unref?.()
|
||||
this.rejectedPtyRecoveryRetries.add(timer)
|
||||
}
|
||||
|
||||
private async reattachRejectedPty(
|
||||
relayPtyId: string,
|
||||
mux: SshChannelMultiplexer,
|
||||
providerGeneration: number,
|
||||
targetedDeliveryRecovery: TargetedDeliveryRecovery
|
||||
): Promise<boolean> {
|
||||
const shouldContinue = () =>
|
||||
this.mux === mux &&
|
||||
!mux.isDisposed() &&
|
||||
this.activePtyProviderGeneration === providerGeneration
|
||||
const ptyProvider = getSshPtyProvider(this.targetId) as SshPtyProvider | undefined
|
||||
// Why re-checked here: this can have waited for a queue slot, and a superseded generation must
|
||||
// not pay for a lease read or an attach round trip.
|
||||
if (!ptyProvider || !shouldContinue()) {
|
||||
return false
|
||||
}
|
||||
const activeLease = this.store
|
||||
.getSshRemotePtyLeases(this.targetId)
|
||||
.find(
|
||||
(lease) =>
|
||||
lease.ptyId === relayPtyId && lease.state !== 'terminated' && lease.state !== 'expired'
|
||||
)
|
||||
const activeLeaseByPtyId = activeLease
|
||||
? new Map<string, SshPtyLease>([[relayPtyId, activeLease]])
|
||||
: new Map<string, SshPtyLease>()
|
||||
const expectedIdentity = activeLease ? expectedIdentityForLease(activeLease) : undefined
|
||||
const attachedLeaseIds = new Set<string>()
|
||||
await this.reattachKnownPty({
|
||||
ptyProvider,
|
||||
ptyId: relayPtyId,
|
||||
activeLeaseByPtyId,
|
||||
expectedIdentityByPtyId: expectedIdentity
|
||||
? new Map([[relayPtyId, expectedIdentity]])
|
||||
: new Map(),
|
||||
attachedLeaseIds,
|
||||
mux,
|
||||
providerGeneration,
|
||||
shouldContinue,
|
||||
targetedDeliveryRecovery
|
||||
})
|
||||
if (attachedLeaseIds.size > 0 && shouldContinue()) {
|
||||
await this.store.markSshRemotePtyLeasesAttachedAsync(
|
||||
this.targetId,
|
||||
Array.from(attachedLeaseIds)
|
||||
)
|
||||
}
|
||||
return attachedLeaseIds.has(relayPtyId)
|
||||
}
|
||||
|
||||
private quarantineReattachData(pending: PendingPtyReattach, payload: SshPtyDataPayload): void {
|
||||
this.observePrivateRecoveryFrame(pending, payload)
|
||||
if (pending.restoreRequired) {
|
||||
@@ -1943,6 +2163,7 @@ export class SshRelaySession {
|
||||
this.retiredSourceDeliveries.activate(relayPtyId)
|
||||
clearProviderPtyState(payload.id)
|
||||
deletePtyOwnership(payload.id)
|
||||
this.rejectedPtyRecoveryAttempts.delete(payload.id)
|
||||
getSshPtyConsumerRecovery(this.targetId)?.checkpointsByAppPtyId.delete(payload.id)
|
||||
getSshPtyConsumerRecovery(this.targetId)?.checkpointsByAppPtyId.delete(
|
||||
toRelaySshPtyId(this.targetId, payload.id)
|
||||
@@ -2051,6 +2272,7 @@ export class SshRelaySession {
|
||||
mux: SshChannelMultiplexer
|
||||
providerGeneration: number
|
||||
shouldContinue: () => boolean
|
||||
targetedDeliveryRecovery?: TargetedDeliveryRecovery
|
||||
}): Promise<void> {
|
||||
const {
|
||||
ptyProvider,
|
||||
@@ -2060,7 +2282,8 @@ export class SshRelaySession {
|
||||
attachedLeaseIds,
|
||||
mux,
|
||||
providerGeneration,
|
||||
shouldContinue
|
||||
shouldContinue,
|
||||
targetedDeliveryRecovery
|
||||
} = args
|
||||
const appPtyId = toAppSshPtyId(this.targetId, ptyId)
|
||||
const pendingReattach: PendingPtyReattach = {
|
||||
@@ -2079,7 +2302,10 @@ export class SshRelaySession {
|
||||
let sourceActivationLease: SshPtyAttachResult['sourceActivationLease']
|
||||
let recoveryActivationLease: SshPtyRecoveryActivationLease | undefined
|
||||
try {
|
||||
const recoveryRequest = await this.sourceRecoveryRequest(appPtyId)
|
||||
const recoveryRequest =
|
||||
targetedDeliveryRecovery === 'fresh-activation'
|
||||
? undefined
|
||||
: await this.sourceRecoveryRequest(appPtyId)
|
||||
const attachResult = await this.attachPtyWithRetry(
|
||||
ptyProvider,
|
||||
ptyId,
|
||||
@@ -2105,7 +2331,27 @@ export class SshRelaySession {
|
||||
await this.acceptPtyExit(exitDuringAttach)
|
||||
return
|
||||
}
|
||||
if (recoveryRequest) {
|
||||
const existingDeliveryConfirmed =
|
||||
targetedDeliveryRecovery === 'confirm-existing' &&
|
||||
recoveryRequest?.status === 'checkpoint' &&
|
||||
!attachResult.sourceRecovery &&
|
||||
Boolean(
|
||||
attachResult.sourceActivation &&
|
||||
this.sameSourceDelivery(attachResult.sourceActivation, recoveryRequest)
|
||||
)
|
||||
if (targetedDeliveryRecovery) {
|
||||
const owner = this.activePtyConsumerOwner()
|
||||
const activation = attachResult.sourceActivation
|
||||
if (
|
||||
!owner?.outputFlowControl ||
|
||||
!activation ||
|
||||
activation.clientGeneration !== owner.clientGeneration ||
|
||||
activation.ownerGeneration !== owner.ownerGeneration
|
||||
) {
|
||||
return
|
||||
}
|
||||
}
|
||||
if (recoveryRequest && !existingDeliveryConfirmed) {
|
||||
const recovered = await this.finishSourceRecovery(
|
||||
ptyId,
|
||||
appPtyId,
|
||||
@@ -2177,6 +2423,18 @@ export class SshRelaySession {
|
||||
pendingReattach.activated = true
|
||||
recoveryActivationLease?.commit()
|
||||
recoveryActivationLease = undefined
|
||||
if (targetedDeliveryRecovery) {
|
||||
if (targetedDeliveryRecovery === 'fresh-activation') {
|
||||
this.retiredSourceDeliveries.activate(ptyId)
|
||||
this.sourceIdentityByRelayPtyId.delete(ptyId)
|
||||
getSshPtyConsumerRecovery(this.targetId)?.checkpointsByAppPtyId.delete(appPtyId)
|
||||
getSshPtyConsumerRecovery(this.targetId)?.checkpointsByAppPtyId.delete(ptyId)
|
||||
}
|
||||
while (pendingReattach.queuedData.length > 0) {
|
||||
await this.acceptPtyData(pendingReattach.queuedData.shift()!)
|
||||
}
|
||||
pendingReattach.livePassthrough = true
|
||||
}
|
||||
const exitAfterActivation = pendingReattach.exits.find(
|
||||
(exit) =>
|
||||
!exit.incarnationId ||
|
||||
@@ -2187,7 +2445,7 @@ export class SshRelaySession {
|
||||
await this.acceptPtyExit(exitAfterActivation)
|
||||
return
|
||||
}
|
||||
if (!recoveryRequest) {
|
||||
if (!recoveryRequest && !targetedDeliveryRecovery) {
|
||||
this.forwardReattachReplay(appPtyId, attachResult.replay ?? '')
|
||||
}
|
||||
sourceActivationLease?.commit()
|
||||
|
||||
Reference in New Issue
Block a user