Files
orca/src/relay/ssh-pty-consumer-session-adapter.test.ts
T
Jinwoo Hong eea0bb64db fix(ssh): make PTY owner admission explicit and non-destructive (#12673)
An owner-capable `pty.openClient` had two failure modes that presented as something else.

If the relay still held an owner record but the request carried no matching resume proof, admission fell through to a SUBSCRIBER grant — a success-shaped response the client cannot use, which it then rejected as "did not grant an authenticated PTY session owner". And if the relay had forgotten the record the client named, admission threw a stale-recovery error, which the client answered by deleting its own recovery row — `clientInstanceId` included — and reopening. Two round trips, and the identity that lets it resume that target at all went with the deletion.

Now every owner grant carries a required `resumed` flag, a forgotten record mints a fresh claim in one round trip, a held claim returns one of three coded refusals, duplicate opens on one connection are rejected even when identical, and an attached-holder refusal becomes a typed error routed through the terminal-relay-error callback instead of feeding redeploy backoff a link that is working fine.

Independent review caught two regressions in the first attempt, both now fixed and both with tests that fail without them:

**A backpressure teardown could take a live owner's session.** The safety argument was that a record only becomes `disconnected` from an observed peer close — but two of the six paths there are capacity paths, where the relay destroys the client's socket itself because its lane queue filled. That is the signature of a client that is ALIVE but not draining fast enough. Demonstrated: the real owner is torn down for backpressure, a rival is granted ownership 270ms into a nominal 30s grace, and the owner's later reconnect with a valid resume proof is refused permanently, backoff cleared, no retry. Closes now carry a cause (`peer-closed` | `local`, defaulting to `local`, which only ever widens a grace), and the floor applies only to closes the transport actually observed on the peer's side. Capacity teardowns, decode faults and sink failures keep the default.

**A client's own zombie connection blocked it permanently.** Only `SshRelaySession` ever requests owner, and every endpoint-credential client shares one principal — so in a normal single-app deployment an `active` incumbent refusing you is almost always your own half-open connection the relay never saw close. That was refused as terminal, where main recovered on bounded backoff once keepalive noticed. The refusal already held both client identities; a match is now a distinct transient refusal that falls through to relay-lost backoff, restoring that recovery. A genuinely different client is still blocked.

Also: each retry deadline now starts when its own phase begins, instead of both being computed at entry where a slow first phase could leave the second with zero attempts.

Fixes STA-3365.
2026-08-05 02:56:07 -07:00

693 lines
21 KiB
TypeScript

import { afterEach, describe, expect, it, vi } from 'vitest'
import {
PTY_CONSUMER_OWNER_GRACE_MS,
PTY_CONSUMER_OWNER_HELD_DISCONNECTED_ERROR,
PTY_CONSUMER_OWNER_HELD_GRACE_FLOOR_MS,
PTY_CONSUMER_OWNER_RECOVERY_PENDING_ERROR
} from '../shared/pty-consumer-session'
import { RelayDispatcher, type RelayClientSessionIdentity } from './dispatcher'
import { encodeJsonRpcFrame, MessageType } from './protocol'
import { SshPtyConsumerSessionAdapter } from './ssh-pty-consumer-session-adapter'
const endpointIdentity: RelayClientSessionIdentity = {
principal: 'endpoint-principal',
authenticated: true,
allowSessionOwner: true,
authenticationKind: 'endpoint-credential'
}
function openFrame(id: number, overrides: Record<string, unknown> = {}): Buffer {
return encodeJsonRpcFrame(
{
jsonrpc: '2.0',
id,
method: 'pty.openClient',
params: {
protocolVersion: 1,
clientInstanceId: `client-${id}`,
requestedRole: 'session-owner',
...overrides
}
},
1,
0
)
}
function responsePayload(buffer: Buffer): Record<string, unknown> {
expect(buffer[0]).toBe(MessageType.Regular)
const length = buffer.readUInt32BE(9)
return JSON.parse(buffer.subarray(13, 13 + length).toString('utf8'))
}
function responseResult(buffer: Buffer): Record<string, unknown> {
return responsePayload(buffer).result as Record<string, unknown>
}
function responseError(buffer: Buffer): Record<string, unknown> {
return responsePayload(buffer).error as Record<string, unknown>
}
async function flushRequests(): Promise<void> {
await new Promise((resolve) => setImmediate(resolve))
}
describe('SshPtyConsumerSessionAdapter', () => {
let dispatcher: RelayDispatcher | null = null
afterEach(() => {
dispatcher?.dispose()
dispatcher = null
vi.useRealTimers()
})
it('does not activate owner authority until the grant write settles', async () => {
const firstWrites: Buffer[] = []
const firstSettlements: ((result: { ok: true } | { ok: false; error: Error }) => void)[] = []
dispatcher = new RelayDispatcher(
(data, onSettled) => {
firstWrites.push(Buffer.from(data))
firstSettlements.push(onSettled)
return true
},
{ supportsWriteCallback: true },
endpointIdentity
)
new SshPtyConsumerSessionAdapter(dispatcher, 'build-a')
dispatcher.feed(openFrame(1))
await flushRequests()
const secondWrites: Buffer[] = []
const secondId = dispatcher.attachClient(
(data, onSettled) => {
secondWrites.push(Buffer.from(data))
onSettled({ ok: true })
return true
},
{ supportsWriteCallback: true },
{ ...endpointIdentity, principal: 'competitor' }
)
dispatcher.feedClient(secondId, openFrame(2))
await flushRequests()
expect(responseResult(firstWrites[0])).toMatchObject({
role: 'session-owner',
ownerGeneration: 1,
resumed: false
})
// Why a coded refusal reaches the wire: the dispatcher transports code and message only, so the
// competitor has to be able to tell "still settling" from "held" without parsing prose.
expect(responseError(secondWrites[0])).toMatchObject({
code: PTY_CONSUMER_OWNER_RECOVERY_PENDING_ERROR
})
firstSettlements[0]({ ok: true })
})
it('rolls back owner election when the grant write fails', async () => {
dispatcher = new RelayDispatcher(
(_data, onSettled) => {
onSettled({ ok: false, error: new Error('send failed') })
return true
},
{ supportsWriteCallback: true },
endpointIdentity
)
new SshPtyConsumerSessionAdapter(dispatcher, 'build-a')
dispatcher.feed(openFrame(1))
await flushRequests()
const retryWrites: Buffer[] = []
const retryId = dispatcher.attachClient(
(data, onSettled) => {
retryWrites.push(Buffer.from(data))
onSettled({ ok: true })
return true
},
{ supportsWriteCallback: true },
endpointIdentity
)
dispatcher.feedClient(retryId, openFrame(2, { clientInstanceId: 'client-1' }))
await flushRequests()
expect(responseResult(retryWrites[0])).toMatchObject({
role: 'session-owner',
ownerGeneration: 2
})
})
it.each([['peer-closed'], ['local']] as const)(
'shortens a refused owner grace only for a %s detach',
async (cause) => {
// Why only Date: flushRequests rides setImmediate, which fake timers would otherwise capture.
vi.useFakeTimers({ toFake: ['Date'] })
dispatcher = new RelayDispatcher(
(_data, onSettled) => {
onSettled({ ok: true })
return true
},
{ supportsWriteCallback: true },
endpointIdentity
)
new SshPtyConsumerSessionAdapter(dispatcher, 'build-a')
const ownerWrites: Buffer[] = []
const ownerId = dispatcher.attachClient(
(data, onSettled) => {
ownerWrites.push(Buffer.from(data))
onSettled({ ok: true })
return true
},
{ supportsWriteCallback: true },
endpointIdentity
)
dispatcher.feedClient(ownerId, openFrame(1))
await flushRequests()
expect(responseResult(ownerWrites[0])).toMatchObject({ role: 'session-owner' })
dispatcher.detachClient(ownerId, cause)
const rivalWrites: Buffer[] = []
const rivalId = dispatcher.attachClient(
(data, onSettled) => {
rivalWrites.push(Buffer.from(data))
onSettled({ ok: true })
return true
},
{ supportsWriteCallback: true },
{ ...endpointIdentity, principal: 'competitor' }
)
// The first refusal is what applies the floor, so it has to happen before the clock moves.
dispatcher.feedClient(rivalId, openFrame(2))
await flushRequests()
expect(responseError(rivalWrites[0])).toMatchObject({
code: PTY_CONSUMER_OWNER_HELD_DISCONNECTED_ERROR
})
vi.setSystemTime(Date.now() + PTY_CONSUMER_OWNER_HELD_GRACE_FLOOR_MS + 1)
const retryId = dispatcher.attachClient(
(data, onSettled) => {
rivalWrites.push(Buffer.from(data))
onSettled({ ok: true })
return true
},
{ supportsWriteCallback: true },
{ ...endpointIdentity, principal: 'competitor' }
)
dispatcher.feedClient(retryId, openFrame(3))
await flushRequests()
// Why this pair is the whole point of the cause: a socket that ended is evidence the owner is
// gone, and a queue the relay overran is not. Only the first may cost the incumbent its claim.
if (cause === 'peer-closed') {
expect(responseResult(rivalWrites[1])).toMatchObject({ role: 'session-owner' })
} else {
expect(responseError(rivalWrites[1])).toMatchObject({
code: PTY_CONSUMER_OWNER_HELD_DISCONNECTED_ERROR
})
}
}
)
it('rejects an unproved constructor stream as an owner principal', async () => {
const writes: Buffer[] = []
dispatcher = new RelayDispatcher((data, onSettled) => {
writes.push(Buffer.from(data))
onSettled({ ok: true })
return true
})
new SshPtyConsumerSessionAdapter(dispatcher, 'build-a')
dispatcher.feed(openFrame(1))
await flushRequests()
const response = JSON.parse(
writes[0].subarray(13, 13 + writes[0].readUInt32BE(9)).toString('utf8')
)
expect(response.error.message).toContain('authentication')
})
it('rejects an invalid requested role instead of promoting it to owner', async () => {
const writes: Buffer[] = []
dispatcher = new RelayDispatcher(
(data, onSettled) => {
writes.push(Buffer.from(data))
onSettled({ ok: true })
return true
},
{ supportsWriteCallback: true },
endpointIdentity
)
new SshPtyConsumerSessionAdapter(dispatcher, 'build-a')
dispatcher.feed(openFrame(1, { requestedRole: 'administrator' }))
await flushRequests()
const response = JSON.parse(
writes[0].subarray(13, 13 + writes[0].readUInt32BE(9)).toString('utf8')
)
expect(response.error.message).toContain('requestedRole')
})
it('generation-fences per-PTY delivery pause notifications', async () => {
const setPaused: { id: string; paused: boolean }[] = []
dispatcher = new RelayDispatcher(
(_data, onSettled) => {
onSettled({ ok: true })
return true
},
{ supportsWriteCallback: true },
endpointIdentity
)
new SshPtyConsumerSessionAdapter(dispatcher, 'build-a', (id, paused) => {
setPaused.push({ id, paused })
})
dispatcher.feed(openFrame(1))
await flushRequests()
dispatcher.feed(
encodeJsonRpcFrame(
{
jsonrpc: '2.0',
method: 'pty.setDeliveryPaused',
params: {
id: 'pty-1',
paused: true,
clientGeneration: 1,
ownerGeneration: 1
}
},
2,
0
)
)
dispatcher.feed(
encodeJsonRpcFrame(
{
jsonrpc: '2.0',
method: 'pty.setDeliveryPaused',
params: {
id: 'pty-1',
paused: false,
clientGeneration: 99,
ownerGeneration: 1
}
},
3,
0
)
)
expect(setPaused).toEqual([{ id: 'pty-1', paused: true }])
})
it('token-fences V1 pause and clears an owned pause before detach retention', async () => {
const setPaused: { id: string; paused: boolean }[] = []
dispatcher = new RelayDispatcher(
(_data, onSettled) => {
onSettled({ ok: true })
return true
},
{ supportsWriteCallback: true },
endpointIdentity
)
const adapter = new SshPtyConsumerSessionAdapter(dispatcher, 'build-a', (id, paused) => {
setPaused.push({ id, paused })
})
dispatcher.feed(
openFrame(1, {
capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 8 } }
})
)
await flushRequests()
const identity = adapter.openDelivery(1, 'pty-1', 'incarnation-1')!
for (const [sequence, deliveryToken] of [
[2, 'stale-token'],
[3, identity.deliveryToken]
] as const) {
dispatcher.feed(
encodeJsonRpcFrame(
{
jsonrpc: '2.0',
method: 'pty.setDeliveryPaused',
params: {
id: identity.id,
paused: true,
clientGeneration: identity.clientGeneration,
ownerGeneration: identity.ownerGeneration,
deliveryToken
}
},
sequence,
0
)
)
}
dispatcher.invalidateClient()
expect(setPaused).toEqual([
{ id: 'pty-1', paused: true },
{ id: 'pty-1', paused: false }
])
})
it('clears the exact token pause before cancellation cleanup', async () => {
const setPaused: { id: string; paused: boolean }[] = []
dispatcher = new RelayDispatcher(
(_data, onSettled) => {
onSettled({ ok: true })
return true
},
{ supportsWriteCallback: true },
endpointIdentity
)
const adapter = new SshPtyConsumerSessionAdapter(dispatcher, 'build-a', (id, paused) => {
setPaused.push({ id, paused })
})
dispatcher.feed(
openFrame(1, {
capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 8 } }
})
)
await flushRequests()
const identity = adapter.openDelivery(1, 'pty-1', 'incarnation-1')!
dispatcher.feed(
encodeJsonRpcFrame(
{
jsonrpc: '2.0',
method: 'pty.setDeliveryPaused',
params: {
id: identity.id,
paused: true,
clientGeneration: identity.clientGeneration,
ownerGeneration: identity.ownerGeneration,
deliveryToken: identity.deliveryToken
}
},
2,
0
)
)
adapter.cancelDelivery(identity, 'restore-required')
expect(setPaused).toEqual([
{ id: 'pty-1', paused: true },
{ id: 'pty-1', paused: false }
])
})
it('intersects the negotiated V1 source window without changing legacy omission', async () => {
const writes: Buffer[] = []
dispatcher = new RelayDispatcher(
(data, onSettled) => {
writes.push(Buffer.from(data))
onSettled({ ok: true })
return true
},
{ supportsWriteCallback: true },
endpointIdentity
)
new SshPtyConsumerSessionAdapter(dispatcher, 'build-a')
dispatcher.feed(
openFrame(1, {
capabilities: {
outputFlowControl: { versions: [1], requestedWindowSu: 512 * 1024 }
}
})
)
await flushRequests()
expect(responseResult(writes[0])).toMatchObject({
capabilities: {
outputFlowControl: { version: 1, windowSu: 256 * 1024 }
}
})
})
it('keeps an old client token-free when the new relay supports V1', async () => {
const writes: Buffer[] = []
dispatcher = new RelayDispatcher(
(data, onSettled) => {
writes.push(Buffer.from(data))
onSettled({ ok: true })
return true
},
{ supportsWriteCallback: true },
endpointIdentity
)
new SshPtyConsumerSessionAdapter(dispatcher, 'build-a')
dispatcher.feed(openFrame(1))
await flushRequests()
const grant = responseResult(writes[0])
expect(grant).not.toHaveProperty('capabilities')
expect(grant).not.toHaveProperty('deliveryToken')
})
it('accepts cumulative source ACKs only from the negotiated token owner', async () => {
dispatcher = new RelayDispatcher(
(_data, onSettled) => {
onSettled({ ok: true })
return true
},
{ supportsWriteCallback: true },
endpointIdentity
)
const adapter = new SshPtyConsumerSessionAdapter(dispatcher, 'build-a')
dispatcher.feed(
openFrame(1, {
capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 4 } }
})
)
await flushRequests()
const identity = adapter.openDelivery(1, 'pty-1', 'incarnation-1')!
adapter.appendSource(identity, {
spanId: 'span-1',
data: 'data',
displayStart: 0,
displayEnd: 4,
splittable: true,
transform: { transformed: false, rawLengthSu: 4, scalarSafe: true }
})
const reservation = adapter.reserveSourceSend(identity)!
adapter.commitSourceSend(reservation)
dispatcher.feed(
encodeJsonRpcFrame(
{
jsonrpc: '2.0',
method: 'pty.ackData',
params: {
acknowledgements: [
{
id: identity.id,
clientGeneration: identity.clientGeneration,
ownerGeneration: identity.ownerGeneration,
deliveryToken: identity.deliveryToken,
creditedEndSu: 4
}
]
}
},
2,
0
)
)
expect(adapter.sourceDeliverySnapshot(identity).creditedEndSu).toBe(4)
})
it('returns exact token-scoped cancellation proof before local cleanup', async () => {
const writes: Buffer[] = []
dispatcher = new RelayDispatcher(
(data, onSettled) => {
writes.push(Buffer.from(data))
onSettled({ ok: true })
return true
},
{ supportsWriteCallback: true },
endpointIdentity
)
const adapter = new SshPtyConsumerSessionAdapter(dispatcher, 'build-a')
dispatcher.feed(
openFrame(1, {
capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 8 } }
})
)
await flushRequests()
const identity = adapter.openDelivery(1, 'pty-1', 'incarnation-1')!
adapter.appendSource(identity, {
spanId: 'span-1',
data: 'data',
displayStart: 0,
displayEnd: 4,
splittable: true,
transform: { transformed: false, rawLengthSu: 4, scalarSafe: true }
})
adapter.commitSourceSend(adapter.reserveSourceSend(identity)!)
dispatcher.feed(
encodeJsonRpcFrame(
{
jsonrpc: '2.0',
id: 2,
method: 'pty.cancelDelivery',
params: {
id: identity.id,
clientGeneration: identity.clientGeneration,
ownerGeneration: identity.ownerGeneration,
deliveryToken: identity.deliveryToken
}
},
2,
0
)
)
await flushRequests()
expect(responseResult(writes.at(-1)!)).toEqual({
canceled: true,
sentEndSu: 4,
creditedEndSu: 0
})
expect(adapter.sourceDeliverySnapshot(identity).state).toBe('closed')
})
it('rotates reconnect ownership and transfers only exact retained recovery', async () => {
const firstWrites: Buffer[] = []
dispatcher = new RelayDispatcher(
(data, onSettled) => {
firstWrites.push(Buffer.from(data))
onSettled({ ok: true })
return true
},
{ supportsWriteCallback: true },
endpointIdentity
)
const adapter = new SshPtyConsumerSessionAdapter(dispatcher, 'build-a')
dispatcher.feed(
openFrame(1, {
clientInstanceId: 'stable-client',
capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 8 } }
})
)
await flushRequests()
const firstGrant = responseResult(firstWrites[0])
const oldIdentity = adapter.openDelivery(1, 'pty-1', 'incarnation-1')!
adapter.appendSource(oldIdentity, {
spanId: 'span-1',
data: 'data',
displayStart: 0,
displayEnd: 4,
splittable: true,
transform: { transformed: false, rawLengthSu: 4, scalarSafe: true }
})
dispatcher.invalidateClient()
const recoveredWrites: Buffer[] = []
const recoveredClientId = dispatcher.attachClient(
(data, onSettled) => {
recoveredWrites.push(Buffer.from(data))
onSettled({ ok: true })
return true
},
{ supportsWriteCallback: true },
endpointIdentity
)
dispatcher.feedClient(
recoveredClientId,
openFrame(2, {
clientInstanceId: 'stable-client',
resume: {
ownerGeneration: firstGrant.ownerGeneration,
ownerLease: firstGrant.ownerLease
},
capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 8 } }
})
)
await flushRequests()
const rotation = adapter.rotateDelivery(oldIdentity, recoveredClientId, 0)
expect(rotation.recovery.map((sourceSpan) => sourceSpan.data)).toEqual(['data'])
expect(rotation.identity).toMatchObject({
providerGeneration: oldIdentity.providerGeneration,
clientGeneration: 2,
ownerGeneration: 2
})
expect(rotation.cancellation.replacementDeliveryToken).toBe(rotation.identity.deliveryToken)
})
it('keeps the old-token deadline after a resume grant until exact rotation', async () => {
vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] })
const firstWrites: Buffer[] = []
dispatcher = new RelayDispatcher(
(data, onSettled) => {
firstWrites.push(Buffer.from(data))
onSettled({ ok: true })
return true
},
{ supportsWriteCallback: true },
endpointIdentity
)
const adapter = new SshPtyConsumerSessionAdapter(dispatcher, 'build-a')
dispatcher.feed(
openFrame(1, {
clientInstanceId: 'stable-client',
capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 8 } }
})
)
await flushRequests()
const firstGrant = responseResult(firstWrites[0])
const oldIdentity = adapter.openDelivery(1, 'pty-1', 'incarnation-1')!
dispatcher.invalidateClient()
const recoveredClientId = dispatcher.attachClient(
(_data, onSettled) => {
onSettled({ ok: true })
return true
},
{ supportsWriteCallback: true },
endpointIdentity
)
dispatcher.feedClient(
recoveredClientId,
openFrame(2, {
clientInstanceId: 'stable-client',
resume: {
ownerGeneration: firstGrant.ownerGeneration,
ownerLease: firstGrant.ownerLease
},
capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 8 } }
})
)
await flushRequests()
vi.advanceTimersByTime(PTY_CONSUMER_OWNER_GRACE_MS)
expect(adapter.sourceDeliverySnapshot(oldIdentity).state).toBe('closed')
})
it('closes retained source tokens when the dispatcher is disposed', async () => {
dispatcher = new RelayDispatcher(
(_data, onSettled) => {
onSettled({ ok: true })
return true
},
{ supportsWriteCallback: true },
endpointIdentity
)
const adapter = new SshPtyConsumerSessionAdapter(dispatcher, 'build-a')
dispatcher.feed(
openFrame(1, {
capabilities: { outputFlowControl: { versions: [1], requestedWindowSu: 8 } }
})
)
await flushRequests()
const identity = adapter.openDelivery(1, 'pty-1', 'incarnation-1')!
dispatcher.dispose()
expect(adapter.sourceDeliverySnapshot(identity).state).toBe('closed')
})
})