mirror of
https://github.com/stablyai/orca.git
synced 2026-10-01 08:01:56 +00:00
An owner-capable `pty.openClient` had two failure modes that presented as something else. If the relay still held an owner record but the request carried no matching resume proof, admission fell through to a SUBSCRIBER grant — a success-shaped response the client cannot use, which it then rejected as "did not grant an authenticated PTY session owner". And if the relay had forgotten the record the client named, admission threw a stale-recovery error, which the client answered by deleting its own recovery row — `clientInstanceId` included — and reopening. Two round trips, and the identity that lets it resume that target at all went with the deletion. Now every owner grant carries a required `resumed` flag, a forgotten record mints a fresh claim in one round trip, a held claim returns one of three coded refusals, duplicate opens on one connection are rejected even when identical, and an attached-holder refusal becomes a typed error routed through the terminal-relay-error callback instead of feeding redeploy backoff a link that is working fine. Independent review caught two regressions in the first attempt, both now fixed and both with tests that fail without them: **A backpressure teardown could take a live owner's session.** The safety argument was that a record only becomes `disconnected` from an observed peer close — but two of the six paths there are capacity paths, where the relay destroys the client's socket itself because its lane queue filled. That is the signature of a client that is ALIVE but not draining fast enough. Demonstrated: the real owner is torn down for backpressure, a rival is granted ownership 270ms into a nominal 30s grace, and the owner's later reconnect with a valid resume proof is refused permanently, backoff cleared, no retry. Closes now carry a cause (`peer-closed` | `local`, defaulting to `local`, which only ever widens a grace), and the floor applies only to closes the transport actually observed on the peer's side. Capacity teardowns, decode faults and sink failures keep the default. **A client's own zombie connection blocked it permanently.** Only `SshRelaySession` ever requests owner, and every endpoint-credential client shares one principal — so in a normal single-app deployment an `active` incumbent refusing you is almost always your own half-open connection the relay never saw close. That was refused as terminal, where main recovered on bounded backoff once keepalive noticed. The refusal already held both client identities; a match is now a distinct transient refusal that falls through to relay-lost backoff, restoring that recovery. A genuinely different client is still blocked. Also: each retry deadline now starts when its own phase begins, instead of both being computed at entry where a slow first phase could leave the second with zero attempts. Fixes STA-3365.
160 lines
5.1 KiB
TypeScript
160 lines
5.1 KiB
TypeScript
import {
|
|
PTY_CONSUMER_SESSION_PROTOCOL_VERSION,
|
|
type PtyConsumerSessionGrant
|
|
} from '../../shared/pty-consumer-session'
|
|
import type { SshChannelMultiplexer } from './ssh-channel-multiplexer'
|
|
|
|
export const SSH_PTY_OPEN_CLIENT_METHOD = 'pty.openClient'
|
|
export const SSH_PTY_OPEN_CLIENT_TIMEOUT_MS = 10_000
|
|
|
|
export type SshPtyConsumerOwnerState = {
|
|
mode: 'negotiated'
|
|
clientInstanceId: string
|
|
clientGeneration: number
|
|
ownerGeneration: number
|
|
ownerLease: string
|
|
outputFlowControl?: {
|
|
version: 1
|
|
windowSu: number
|
|
}
|
|
}
|
|
|
|
export type SshPtyLegacyFallbackState = {
|
|
mode: 'legacy-fallback'
|
|
clientInstanceId: string
|
|
serverBuildId: string
|
|
}
|
|
|
|
export type SshPtyConsumerSessionState = SshPtyConsumerOwnerState | SshPtyLegacyFallbackState
|
|
|
|
export type SshPtyConsumerAdmission = {
|
|
state: SshPtyConsumerSessionState
|
|
// Why not on the owner state itself: this describes one admission's outcome, not the persisted
|
|
// claim, and it must never round-trip through the recovery record.
|
|
resumed: boolean
|
|
}
|
|
|
|
export type OpenSshPtyConsumerSessionOptions = {
|
|
clientInstanceId: string
|
|
expectedServerBuildId: string | undefined
|
|
resume?: Pick<SshPtyConsumerOwnerState, 'ownerGeneration' | 'ownerLease'>
|
|
outputFlowControl?: {
|
|
requestedWindowSu: number
|
|
}
|
|
allowSameBuildLegacyFallback?: boolean
|
|
}
|
|
|
|
function validateGrant(
|
|
value: unknown,
|
|
options: OpenSshPtyConsumerSessionOptions
|
|
): PtyConsumerSessionGrant {
|
|
if (typeof value !== 'object' || value === null) {
|
|
throw new Error('Remote relay returned an invalid pty.openClient grant')
|
|
}
|
|
if (!options.expectedServerBuildId) {
|
|
throw new Error('Local relay build identity is unavailable')
|
|
}
|
|
const grant = value as Partial<PtyConsumerSessionGrant>
|
|
if (
|
|
grant.protocolVersion !== PTY_CONSUMER_SESSION_PROTOCOL_VERSION ||
|
|
grant.serverBuildId !== options.expectedServerBuildId
|
|
) {
|
|
throw new Error(
|
|
`Remote relay session contract mismatch — expected build ${options.expectedServerBuildId}, got ${grant.serverBuildId ?? 'unknown'}`
|
|
)
|
|
}
|
|
if (
|
|
!Number.isSafeInteger(grant.clientGeneration) ||
|
|
grant.clientGeneration! <= 0 ||
|
|
grant.role !== 'session-owner' ||
|
|
!Number.isSafeInteger(grant.ownerGeneration) ||
|
|
grant.ownerGeneration! <= 0 ||
|
|
typeof grant.ownerLease !== 'string' ||
|
|
grant.ownerLease.length === 0 ||
|
|
grant.ownerLease.length > 512
|
|
) {
|
|
throw new Error('Remote relay did not grant an authenticated PTY session owner')
|
|
}
|
|
// Why not treated as a legacy relay: client and relay ship in one build, and the build id was already
|
|
// matched above — a missing `resumed` here is corruption, not an older peer.
|
|
if (typeof grant.resumed !== 'boolean') {
|
|
throw new Error('Remote relay owner grant did not state whether the claim was resumed')
|
|
}
|
|
const requestedFlow = options.outputFlowControl
|
|
const grantedFlow = grant.capabilities?.outputFlowControl
|
|
if (requestedFlow) {
|
|
if (
|
|
grantedFlow?.version !== 1 ||
|
|
!Number.isSafeInteger(grantedFlow.windowSu) ||
|
|
grantedFlow.windowSu <= 0 ||
|
|
grantedFlow.windowSu > requestedFlow.requestedWindowSu
|
|
) {
|
|
throw new Error('Remote relay did not grant the offered PTY output-flow-control capability')
|
|
}
|
|
} else if (grantedFlow) {
|
|
throw new Error('Remote relay granted an unoffered PTY output-flow-control capability')
|
|
}
|
|
return grant as PtyConsumerSessionGrant
|
|
}
|
|
|
|
export async function openSshPtyConsumerSession(
|
|
mux: SshChannelMultiplexer,
|
|
options: OpenSshPtyConsumerSessionOptions
|
|
): Promise<SshPtyConsumerAdmission> {
|
|
let result: unknown
|
|
try {
|
|
result = await mux.request(
|
|
SSH_PTY_OPEN_CLIENT_METHOD,
|
|
{
|
|
protocolVersion: PTY_CONSUMER_SESSION_PROTOCOL_VERSION,
|
|
clientInstanceId: options.clientInstanceId,
|
|
requestedRole: 'session-owner',
|
|
...(options.resume ? { resume: options.resume } : {}),
|
|
...(options.outputFlowControl
|
|
? {
|
|
capabilities: {
|
|
outputFlowControl: {
|
|
versions: [1],
|
|
requestedWindowSu: options.outputFlowControl.requestedWindowSu
|
|
}
|
|
}
|
|
}
|
|
: {})
|
|
},
|
|
{ timeoutMs: SSH_PTY_OPEN_CLIENT_TIMEOUT_MS }
|
|
)
|
|
} catch (error) {
|
|
const code = (error as { code?: unknown })?.code
|
|
if (
|
|
code === -32601 &&
|
|
options.allowSameBuildLegacyFallback === true &&
|
|
typeof options.expectedServerBuildId === 'string' &&
|
|
options.expectedServerBuildId.length > 0
|
|
) {
|
|
return {
|
|
state: Object.freeze({
|
|
mode: 'legacy-fallback',
|
|
clientInstanceId: options.clientInstanceId,
|
|
serverBuildId: options.expectedServerBuildId
|
|
}),
|
|
resumed: false
|
|
}
|
|
}
|
|
throw error
|
|
}
|
|
const grant = validateGrant(result, options)
|
|
return {
|
|
state: {
|
|
mode: 'negotiated',
|
|
clientInstanceId: options.clientInstanceId,
|
|
clientGeneration: grant.clientGeneration,
|
|
ownerGeneration: grant.ownerGeneration!,
|
|
ownerLease: grant.ownerLease!,
|
|
...(grant.capabilities?.outputFlowControl
|
|
? { outputFlowControl: grant.capabilities.outputFlowControl }
|
|
: {})
|
|
},
|
|
resumed: grant.resumed!
|
|
}
|
|
}
|