fix(runtime): bound the remote-runtime connect against an unreachable host

A host that is powered off or firewalled black-holes the TCP SYN, so the
remote-runtime WebSocket neither opens nor errors. The Node-side transports
set no connect bound, leaving the caller's whole-request timeout as the only
one: every `orca <cmd> --environment <unreachable>` sat silent for 60s before
failing with a generic `runtime_timeout`.

Measured on an unreachable paired host (win-lowspec, SYNs dropped): terminal
list / worktree list / repo list / status each took 60.19-60.26s; the same
command against a reachable host answered in 0.24s. So this was the shared
transport, not one command.

Pass `handshakeTimeout` at the three shared remote-runtime WebSocket
construction sites, which `ws` applies across TCP connect and the HTTP
upgrade. The value matches the bound the browser transport already used.

The failure keeps code `remote_runtime_unavailable` so the existing
transport-loss classification in terminal-process-inspection still applies,
and the message names the endpoint and stops at "unverifiable" — per
docs/reference/ssh-execution-boundary.md, loss of contact is never evidence
that the host's work stopped.
This commit is contained in:
Neil
2026-09-10 23:50:03 -07:00
parent b3e0a33fa4
commit 52dc3d20fb
5 changed files with 221 additions and 15 deletions
@@ -0,0 +1,128 @@
import { readFileSync, readdirSync } from 'node:fs'
import { createServer, type Server, type Socket } from 'node:net'
import { join } from 'node:path'
import { afterEach, describe, expect, it, vi } from 'vitest'
import { generateKeyPair, publicKeyToBase64 } from './e2ee-crypto'
import type { RemoteRuntimeClientError } from './remote-runtime-client-error'
import {
REMOTE_RUNTIME_CONNECT_TIMEOUT_MS,
isRemoteRuntimeConnectTimeout,
remoteRuntimeConnectFailureMessage,
remoteRuntimeConnectOptions
} from './remote-runtime-connect-bound'
import { openRemoteRuntimeWebSocket } from './remote-runtime-request-websocket'
const servers = new Set<Server>()
const sockets = new Set<Socket>()
afterEach(async () => {
for (const socket of sockets) {
socket.destroy()
}
sockets.clear()
await Promise.all(
[...servers].map(
(server) =>
new Promise<void>((resolve) => {
server.close(() => resolve())
})
)
)
servers.clear()
})
/**
* Accepts TCP but never answers the HTTP upgrade, which is the same silent
* stall a black-holed host produces and is bounded by the same `ws` timer.
*/
async function listenSilentUpgradeServer(): Promise<string> {
const server = createServer((socket) => {
sockets.add(socket)
socket.once('close', () => sockets.delete(socket))
})
servers.add(server)
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
const address = server.address()
if (address === null || typeof address === 'string') {
throw new Error('expected a TCP address')
}
return `ws://127.0.0.1:${address.port}`
}
describe('remote runtime connect bound', () => {
it('bounds the production connect with a finite handshake timeout', () => {
const options = remoteRuntimeConnectOptions({ maxPayload: 1024 })
expect(Number.isFinite(options.handshakeTimeout)).toBe(true)
expect(options.handshakeTimeout).toBe(REMOTE_RUNTIME_CONNECT_TIMEOUT_MS)
expect(options.maxPayload).toBe(1024)
})
// Why: the bound only helps if every Node-side remote-runtime socket carries
// it; a new transport that calls `new WebSocket` directly reintroduces #18191.
it('routes every shared remote-runtime WebSocket through the bounded options', () => {
const dir = join(__dirname)
const offenders: string[] = []
let scannedConstructions = 0
for (const name of readdirSync(dir)) {
if (!name.startsWith('remote-runtime-') || !name.endsWith('.ts') || name.includes('.test.')) {
continue
}
const source = readFileSync(join(dir, name), 'utf8')
const constructions = source.split('new WebSocket(').length - 1
const bounded = source.split('remoteRuntimeConnectOptions(').length - 1
scannedConstructions += constructions
if (constructions > bounded) {
offenders.push(`${name}: ${constructions} WebSocket(s), ${bounded} bounded`)
}
}
expect(offenders).toEqual([])
// Guards against the scan silently matching nothing and passing vacuously.
expect(scannedConstructions).toBeGreaterThan(0)
})
it('reports an unanswered host as unreachable rather than as an empty result', async () => {
const endpoint = await listenSilentUpgradeServer()
const keyPair = generateKeyPair()
const onError = vi.fn()
const onTextFrame = vi.fn()
const opened = openRemoteRuntimeWebSocket(
{
v: 2,
endpoint,
deviceToken: 'device-token',
publicKeyB64: publicKeyToBase64(keyPair.publicKey)
},
{ onClose: vi.fn(), onError, onTextFrame },
150
)
if (!opened.ok) {
throw opened.error
}
await vi.waitFor(() => expect(onError).toHaveBeenCalledTimes(1), {
timeout: 5_000
})
// The bounded path was taken: a connect failure, not a silent empty answer.
const error = onError.mock.calls[0][1] as RemoteRuntimeClientError
expect(error.code).toBe('remote_runtime_unavailable')
expect(error.message).toContain(endpoint)
expect(error.message).toContain('unverifiable')
expect(onTextFrame).not.toHaveBeenCalled()
// Loss of contact is never evidence the host's work stopped.
expect(error.message).not.toMatch(/\b(exited|gone|stopped|empty|no terminals)\b/i)
opened.socket.cleanup()
opened.socket.ws.terminate()
})
it('only calls an elapsed handshake a connect timeout', () => {
expect(isRemoteRuntimeConnectTimeout(new Error('Opening handshake has timed out'))).toBe(true)
expect(isRemoteRuntimeConnectTimeout(new Error('connect ECONNREFUSED'))).toBe(false)
expect(remoteRuntimeConnectFailureMessage(new Error('connect ECONNREFUSED'), 'ws://h')).toBe(
'Could not connect to the remote Orca runtime.'
)
})
})
@@ -0,0 +1,52 @@
import type { ClientOptions } from 'ws'
/**
* Connect-phase bound for the Node-side remote-runtime WebSocket transports.
*
* Why: a host that is powered off or firewalled black-holes the TCP SYN, so the
* socket neither opens nor errors. Without this the only bound is the caller's
* whole-request timeout (60s in the CLI), which reads to the user as a frozen
* terminal. `ws` applies `handshakeTimeout` across TCP connect *and* the HTTP
* upgrade, so one option covers both silent stalls.
*
* The value matches `CONNECT_TIMEOUT_MS` in
* `src/renderer/src/web/web-runtime-connection-transport.ts`, which already
* bounded the browser transport; this brings the Node transports in line.
*/
export const REMOTE_RUNTIME_CONNECT_TIMEOUT_MS = 12_000
/** The `ws` message for an elapsed `handshakeTimeout`; matched, never thrown by us. */
const WS_HANDSHAKE_TIMEOUT_MESSAGE = 'Opening handshake has timed out'
export function remoteRuntimeConnectOptions<TOptions extends ClientOptions>(
options?: TOptions,
connectTimeoutMs: number = REMOTE_RUNTIME_CONNECT_TIMEOUT_MS
): TOptions & { handshakeTimeout: number } {
return {
...(options ?? ({} as TOptions)),
handshakeTimeout: connectTimeoutMs
}
}
export function isRemoteRuntimeConnectTimeout(error: unknown): boolean {
return error instanceof Error && error.message === WS_HANDSHAKE_TIMEOUT_MESSAGE
}
/**
* Why: per `docs/reference/ssh-execution-boundary.md`, loss of contact is never
* evidence that remote work stopped. This message says the host did not answer
* and stops there — it must not imply the host's terminals are gone.
*/
export function remoteRuntimeConnectFailureMessage(
error: unknown,
endpoint: string,
connectTimeoutMs: number = REMOTE_RUNTIME_CONNECT_TIMEOUT_MS
): string {
if (!isRemoteRuntimeConnectTimeout(error)) {
return 'Could not connect to the remote Orca runtime.'
}
return (
`Could not reach the remote Orca runtime at ${endpoint} within ${connectTimeoutMs}ms. ` +
'The host did not answer, so anything running on it is unverifiable.'
)
}
+10 -3
View File
@@ -16,6 +16,10 @@ import {
ignoreSettledRemoteRuntimeSocketError
} from './remote-runtime-client-handshake'
import { RemoteRuntimeClientError } from './remote-runtime-client-error'
import {
remoteRuntimeConnectFailureMessage,
remoteRuntimeConnectOptions
} from './remote-runtime-connect-bound'
import {
REMOTE_RUNTIME_MAX_WEBSOCKET_FRAME_BYTES,
serializeRemoteRuntimePayload,
@@ -183,7 +187,10 @@ export async function sendRemoteRuntimeRequestOnSocket<TResult>(
}
try {
ws = new WebSocket(pairing.endpoint, { maxPayload: REMOTE_RUNTIME_MAX_WEBSOCKET_FRAME_BYTES })
const connectOptions = remoteRuntimeConnectOptions({
maxPayload: REMOTE_RUNTIME_MAX_WEBSOCKET_FRAME_BYTES
})
ws = new WebSocket(pairing.endpoint, connectOptions)
} catch (error) {
const message = error instanceof Error ? error.message : String(error)
finishError(
@@ -201,11 +208,11 @@ export async function sendRemoteRuntimeRequestOnSocket<TResult>(
)
}
function onError(): void {
function onError(error: Error): void {
finishError(
new RemoteRuntimeClientError(
'remote_runtime_unavailable',
'Could not connect to the remote Orca runtime.',
remoteRuntimeConnectFailureMessage(error, pairing.endpoint),
{ pairingStage: router.pairingStage }
)
)
+20 -6
View File
@@ -7,6 +7,10 @@ import {
publicKeyToBase64
} from './e2ee-crypto'
import { RemoteRuntimeClientError } from './remote-runtime-client'
import {
remoteRuntimeConnectFailureMessage,
remoteRuntimeConnectOptions
} from './remote-runtime-connect-bound'
import {
invalidRemoteRuntimeResponseError,
remoteRuntimeUnavailableError
@@ -30,9 +34,12 @@ export type RemoteRuntimeWebSocketCallbacks = {
export function openRemoteRuntimeWebSocket(
pairing: PairingOffer,
callbacks: RemoteRuntimeWebSocketCallbacks
callbacks: RemoteRuntimeWebSocketCallbacks,
// Why: overridable so the connect-bound regression test can pin the behaviour
// without spending the production budget of wall-clock time.
connectTimeoutMs?: number
): { ok: true; socket: RemoteRuntimeWebSocket } | { ok: false; error: RemoteRuntimeClientError } {
const opened = createSocket(pairing)
const opened = createSocket(pairing, connectTimeoutMs)
if (!opened.ok) {
return opened
}
@@ -49,10 +56,12 @@ export function openRemoteRuntimeWebSocket(
})
)
}
const onError = (): void => {
const onError = (error: Error): void => {
callbacks.onError(
ws,
remoteRuntimeUnavailableError('Could not connect to the remote Orca runtime.')
remoteRuntimeUnavailableError(
remoteRuntimeConnectFailureMessage(error, pairing.endpoint, connectTimeoutMs)
)
)
}
const onClose = (code: number, reason: Buffer): void => callbacks.onClose(ws, code, reason)
@@ -100,7 +109,8 @@ export function openRemoteRuntimeWebSocket(
function ignoreLateSocketError(): void {}
function createSocket(
pairing: PairingOffer
pairing: PairingOffer,
connectTimeoutMs?: number
):
| { ok: true; ws: WebSocket; keyPair: ReturnType<typeof generateKeyPair> }
| { ok: false; error: RemoteRuntimeClientError } {
@@ -119,7 +129,11 @@ function createSocket(
}
}
try {
return { ok: true, ws: new WebSocket(pairing.endpoint), keyPair }
return {
ok: true,
ws: new WebSocket(pairing.endpoint, remoteRuntimeConnectOptions(undefined, connectTimeoutMs)),
keyPair
}
} catch (error) {
const message = error instanceof Error ? error.message : String(error)
return {
@@ -14,6 +14,10 @@ import {
ignoreSettledRemoteRuntimeSocketError
} from './remote-runtime-client-handshake'
import { RemoteRuntimeClientError } from './remote-runtime-client-error'
import {
remoteRuntimeConnectFailureMessage,
remoteRuntimeConnectOptions
} from './remote-runtime-connect-bound'
import {
isRemoteRuntimeBinaryFrameWithinLimit,
REMOTE_RUNTIME_MAX_WEBSOCKET_FRAME_BYTES,
@@ -225,11 +229,12 @@ export async function subscribeRemoteRuntimeTransport<TResult>(
callbacks.onClose?.()
}
const connectOptions = remoteRuntimeConnectOptions({
maxPayload: REMOTE_RUNTIME_MAX_WEBSOCKET_FRAME_BYTES,
...(options?.perMessageDeflate === false ? { perMessageDeflate: false } : {})
})
try {
ws = new WebSocket(pairing.endpoint, {
maxPayload: REMOTE_RUNTIME_MAX_WEBSOCKET_FRAME_BYTES,
...(options?.perMessageDeflate === false ? { perMessageDeflate: false } : {})
})
ws = new WebSocket(pairing.endpoint, connectOptions)
} catch (error) {
const message = error instanceof Error ? error.message : String(error)
fail(new RemoteRuntimeClientError('invalid_argument', `Invalid remote endpoint: ${message}`))
@@ -242,11 +247,11 @@ export async function subscribeRemoteRuntimeTransport<TResult>(
)
}
function onError(): void {
function onError(error: Error): void {
fail(
new RemoteRuntimeClientError(
'remote_runtime_unavailable',
'Could not connect to the remote Orca runtime.'
remoteRuntimeConnectFailureMessage(error, pairing.endpoint)
)
)
}