mirror of
https://github.com/stablyai/orca.git
synced 2026-09-28 08:02:43 +00:00
fix(runtime): bound the remote-runtime connect against an unreachable host
A host that is powered off or firewalled black-holes the TCP SYN, so the remote-runtime WebSocket neither opens nor errors. The Node-side transports set no connect bound, leaving the caller's whole-request timeout as the only one: every `orca <cmd> --environment <unreachable>` sat silent for 60s before failing with a generic `runtime_timeout`. Measured on an unreachable paired host (win-lowspec, SYNs dropped): terminal list / worktree list / repo list / status each took 60.19-60.26s; the same command against a reachable host answered in 0.24s. So this was the shared transport, not one command. Pass `handshakeTimeout` at the three shared remote-runtime WebSocket construction sites, which `ws` applies across TCP connect and the HTTP upgrade. The value matches the bound the browser transport already used. The failure keeps code `remote_runtime_unavailable` so the existing transport-loss classification in terminal-process-inspection still applies, and the message names the endpoint and stops at "unverifiable" — per docs/reference/ssh-execution-boundary.md, loss of contact is never evidence that the host's work stopped.
This commit is contained in:
@@ -0,0 +1,128 @@
|
||||
import { readFileSync, readdirSync } from 'node:fs'
|
||||
import { createServer, type Server, type Socket } from 'node:net'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { generateKeyPair, publicKeyToBase64 } from './e2ee-crypto'
|
||||
import type { RemoteRuntimeClientError } from './remote-runtime-client-error'
|
||||
import {
|
||||
REMOTE_RUNTIME_CONNECT_TIMEOUT_MS,
|
||||
isRemoteRuntimeConnectTimeout,
|
||||
remoteRuntimeConnectFailureMessage,
|
||||
remoteRuntimeConnectOptions
|
||||
} from './remote-runtime-connect-bound'
|
||||
import { openRemoteRuntimeWebSocket } from './remote-runtime-request-websocket'
|
||||
|
||||
const servers = new Set<Server>()
|
||||
const sockets = new Set<Socket>()
|
||||
|
||||
afterEach(async () => {
|
||||
for (const socket of sockets) {
|
||||
socket.destroy()
|
||||
}
|
||||
sockets.clear()
|
||||
await Promise.all(
|
||||
[...servers].map(
|
||||
(server) =>
|
||||
new Promise<void>((resolve) => {
|
||||
server.close(() => resolve())
|
||||
})
|
||||
)
|
||||
)
|
||||
servers.clear()
|
||||
})
|
||||
|
||||
/**
|
||||
* Accepts TCP but never answers the HTTP upgrade, which is the same silent
|
||||
* stall a black-holed host produces and is bounded by the same `ws` timer.
|
||||
*/
|
||||
async function listenSilentUpgradeServer(): Promise<string> {
|
||||
const server = createServer((socket) => {
|
||||
sockets.add(socket)
|
||||
socket.once('close', () => sockets.delete(socket))
|
||||
})
|
||||
servers.add(server)
|
||||
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
|
||||
const address = server.address()
|
||||
if (address === null || typeof address === 'string') {
|
||||
throw new Error('expected a TCP address')
|
||||
}
|
||||
return `ws://127.0.0.1:${address.port}`
|
||||
}
|
||||
|
||||
describe('remote runtime connect bound', () => {
|
||||
it('bounds the production connect with a finite handshake timeout', () => {
|
||||
const options = remoteRuntimeConnectOptions({ maxPayload: 1024 })
|
||||
expect(Number.isFinite(options.handshakeTimeout)).toBe(true)
|
||||
expect(options.handshakeTimeout).toBe(REMOTE_RUNTIME_CONNECT_TIMEOUT_MS)
|
||||
expect(options.maxPayload).toBe(1024)
|
||||
})
|
||||
|
||||
// Why: the bound only helps if every Node-side remote-runtime socket carries
|
||||
// it; a new transport that calls `new WebSocket` directly reintroduces #18191.
|
||||
it('routes every shared remote-runtime WebSocket through the bounded options', () => {
|
||||
const dir = join(__dirname)
|
||||
const offenders: string[] = []
|
||||
let scannedConstructions = 0
|
||||
for (const name of readdirSync(dir)) {
|
||||
if (!name.startsWith('remote-runtime-') || !name.endsWith('.ts') || name.includes('.test.')) {
|
||||
continue
|
||||
}
|
||||
const source = readFileSync(join(dir, name), 'utf8')
|
||||
const constructions = source.split('new WebSocket(').length - 1
|
||||
const bounded = source.split('remoteRuntimeConnectOptions(').length - 1
|
||||
scannedConstructions += constructions
|
||||
if (constructions > bounded) {
|
||||
offenders.push(`${name}: ${constructions} WebSocket(s), ${bounded} bounded`)
|
||||
}
|
||||
}
|
||||
expect(offenders).toEqual([])
|
||||
// Guards against the scan silently matching nothing and passing vacuously.
|
||||
expect(scannedConstructions).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('reports an unanswered host as unreachable rather than as an empty result', async () => {
|
||||
const endpoint = await listenSilentUpgradeServer()
|
||||
const keyPair = generateKeyPair()
|
||||
const onError = vi.fn()
|
||||
const onTextFrame = vi.fn()
|
||||
|
||||
const opened = openRemoteRuntimeWebSocket(
|
||||
{
|
||||
v: 2,
|
||||
endpoint,
|
||||
deviceToken: 'device-token',
|
||||
publicKeyB64: publicKeyToBase64(keyPair.publicKey)
|
||||
},
|
||||
{ onClose: vi.fn(), onError, onTextFrame },
|
||||
150
|
||||
)
|
||||
if (!opened.ok) {
|
||||
throw opened.error
|
||||
}
|
||||
|
||||
await vi.waitFor(() => expect(onError).toHaveBeenCalledTimes(1), {
|
||||
timeout: 5_000
|
||||
})
|
||||
|
||||
// The bounded path was taken: a connect failure, not a silent empty answer.
|
||||
const error = onError.mock.calls[0][1] as RemoteRuntimeClientError
|
||||
expect(error.code).toBe('remote_runtime_unavailable')
|
||||
expect(error.message).toContain(endpoint)
|
||||
expect(error.message).toContain('unverifiable')
|
||||
expect(onTextFrame).not.toHaveBeenCalled()
|
||||
|
||||
// Loss of contact is never evidence the host's work stopped.
|
||||
expect(error.message).not.toMatch(/\b(exited|gone|stopped|empty|no terminals)\b/i)
|
||||
|
||||
opened.socket.cleanup()
|
||||
opened.socket.ws.terminate()
|
||||
})
|
||||
|
||||
it('only calls an elapsed handshake a connect timeout', () => {
|
||||
expect(isRemoteRuntimeConnectTimeout(new Error('Opening handshake has timed out'))).toBe(true)
|
||||
expect(isRemoteRuntimeConnectTimeout(new Error('connect ECONNREFUSED'))).toBe(false)
|
||||
expect(remoteRuntimeConnectFailureMessage(new Error('connect ECONNREFUSED'), 'ws://h')).toBe(
|
||||
'Could not connect to the remote Orca runtime.'
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,52 @@
|
||||
import type { ClientOptions } from 'ws'
|
||||
|
||||
/**
|
||||
* Connect-phase bound for the Node-side remote-runtime WebSocket transports.
|
||||
*
|
||||
* Why: a host that is powered off or firewalled black-holes the TCP SYN, so the
|
||||
* socket neither opens nor errors. Without this the only bound is the caller's
|
||||
* whole-request timeout (60s in the CLI), which reads to the user as a frozen
|
||||
* terminal. `ws` applies `handshakeTimeout` across TCP connect *and* the HTTP
|
||||
* upgrade, so one option covers both silent stalls.
|
||||
*
|
||||
* The value matches `CONNECT_TIMEOUT_MS` in
|
||||
* `src/renderer/src/web/web-runtime-connection-transport.ts`, which already
|
||||
* bounded the browser transport; this brings the Node transports in line.
|
||||
*/
|
||||
export const REMOTE_RUNTIME_CONNECT_TIMEOUT_MS = 12_000
|
||||
|
||||
/** The `ws` message for an elapsed `handshakeTimeout`; matched, never thrown by us. */
|
||||
const WS_HANDSHAKE_TIMEOUT_MESSAGE = 'Opening handshake has timed out'
|
||||
|
||||
export function remoteRuntimeConnectOptions<TOptions extends ClientOptions>(
|
||||
options?: TOptions,
|
||||
connectTimeoutMs: number = REMOTE_RUNTIME_CONNECT_TIMEOUT_MS
|
||||
): TOptions & { handshakeTimeout: number } {
|
||||
return {
|
||||
...(options ?? ({} as TOptions)),
|
||||
handshakeTimeout: connectTimeoutMs
|
||||
}
|
||||
}
|
||||
|
||||
export function isRemoteRuntimeConnectTimeout(error: unknown): boolean {
|
||||
return error instanceof Error && error.message === WS_HANDSHAKE_TIMEOUT_MESSAGE
|
||||
}
|
||||
|
||||
/**
|
||||
* Why: per `docs/reference/ssh-execution-boundary.md`, loss of contact is never
|
||||
* evidence that remote work stopped. This message says the host did not answer
|
||||
* and stops there — it must not imply the host's terminals are gone.
|
||||
*/
|
||||
export function remoteRuntimeConnectFailureMessage(
|
||||
error: unknown,
|
||||
endpoint: string,
|
||||
connectTimeoutMs: number = REMOTE_RUNTIME_CONNECT_TIMEOUT_MS
|
||||
): string {
|
||||
if (!isRemoteRuntimeConnectTimeout(error)) {
|
||||
return 'Could not connect to the remote Orca runtime.'
|
||||
}
|
||||
return (
|
||||
`Could not reach the remote Orca runtime at ${endpoint} within ${connectTimeoutMs}ms. ` +
|
||||
'The host did not answer, so anything running on it is unverifiable.'
|
||||
)
|
||||
}
|
||||
@@ -16,6 +16,10 @@ import {
|
||||
ignoreSettledRemoteRuntimeSocketError
|
||||
} from './remote-runtime-client-handshake'
|
||||
import { RemoteRuntimeClientError } from './remote-runtime-client-error'
|
||||
import {
|
||||
remoteRuntimeConnectFailureMessage,
|
||||
remoteRuntimeConnectOptions
|
||||
} from './remote-runtime-connect-bound'
|
||||
import {
|
||||
REMOTE_RUNTIME_MAX_WEBSOCKET_FRAME_BYTES,
|
||||
serializeRemoteRuntimePayload,
|
||||
@@ -183,7 +187,10 @@ export async function sendRemoteRuntimeRequestOnSocket<TResult>(
|
||||
}
|
||||
|
||||
try {
|
||||
ws = new WebSocket(pairing.endpoint, { maxPayload: REMOTE_RUNTIME_MAX_WEBSOCKET_FRAME_BYTES })
|
||||
const connectOptions = remoteRuntimeConnectOptions({
|
||||
maxPayload: REMOTE_RUNTIME_MAX_WEBSOCKET_FRAME_BYTES
|
||||
})
|
||||
ws = new WebSocket(pairing.endpoint, connectOptions)
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error)
|
||||
finishError(
|
||||
@@ -201,11 +208,11 @@ export async function sendRemoteRuntimeRequestOnSocket<TResult>(
|
||||
)
|
||||
}
|
||||
|
||||
function onError(): void {
|
||||
function onError(error: Error): void {
|
||||
finishError(
|
||||
new RemoteRuntimeClientError(
|
||||
'remote_runtime_unavailable',
|
||||
'Could not connect to the remote Orca runtime.',
|
||||
remoteRuntimeConnectFailureMessage(error, pairing.endpoint),
|
||||
{ pairingStage: router.pairingStage }
|
||||
)
|
||||
)
|
||||
|
||||
@@ -7,6 +7,10 @@ import {
|
||||
publicKeyToBase64
|
||||
} from './e2ee-crypto'
|
||||
import { RemoteRuntimeClientError } from './remote-runtime-client'
|
||||
import {
|
||||
remoteRuntimeConnectFailureMessage,
|
||||
remoteRuntimeConnectOptions
|
||||
} from './remote-runtime-connect-bound'
|
||||
import {
|
||||
invalidRemoteRuntimeResponseError,
|
||||
remoteRuntimeUnavailableError
|
||||
@@ -30,9 +34,12 @@ export type RemoteRuntimeWebSocketCallbacks = {
|
||||
|
||||
export function openRemoteRuntimeWebSocket(
|
||||
pairing: PairingOffer,
|
||||
callbacks: RemoteRuntimeWebSocketCallbacks
|
||||
callbacks: RemoteRuntimeWebSocketCallbacks,
|
||||
// Why: overridable so the connect-bound regression test can pin the behaviour
|
||||
// without spending the production budget of wall-clock time.
|
||||
connectTimeoutMs?: number
|
||||
): { ok: true; socket: RemoteRuntimeWebSocket } | { ok: false; error: RemoteRuntimeClientError } {
|
||||
const opened = createSocket(pairing)
|
||||
const opened = createSocket(pairing, connectTimeoutMs)
|
||||
if (!opened.ok) {
|
||||
return opened
|
||||
}
|
||||
@@ -49,10 +56,12 @@ export function openRemoteRuntimeWebSocket(
|
||||
})
|
||||
)
|
||||
}
|
||||
const onError = (): void => {
|
||||
const onError = (error: Error): void => {
|
||||
callbacks.onError(
|
||||
ws,
|
||||
remoteRuntimeUnavailableError('Could not connect to the remote Orca runtime.')
|
||||
remoteRuntimeUnavailableError(
|
||||
remoteRuntimeConnectFailureMessage(error, pairing.endpoint, connectTimeoutMs)
|
||||
)
|
||||
)
|
||||
}
|
||||
const onClose = (code: number, reason: Buffer): void => callbacks.onClose(ws, code, reason)
|
||||
@@ -100,7 +109,8 @@ export function openRemoteRuntimeWebSocket(
|
||||
function ignoreLateSocketError(): void {}
|
||||
|
||||
function createSocket(
|
||||
pairing: PairingOffer
|
||||
pairing: PairingOffer,
|
||||
connectTimeoutMs?: number
|
||||
):
|
||||
| { ok: true; ws: WebSocket; keyPair: ReturnType<typeof generateKeyPair> }
|
||||
| { ok: false; error: RemoteRuntimeClientError } {
|
||||
@@ -119,7 +129,11 @@ function createSocket(
|
||||
}
|
||||
}
|
||||
try {
|
||||
return { ok: true, ws: new WebSocket(pairing.endpoint), keyPair }
|
||||
return {
|
||||
ok: true,
|
||||
ws: new WebSocket(pairing.endpoint, remoteRuntimeConnectOptions(undefined, connectTimeoutMs)),
|
||||
keyPair
|
||||
}
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error)
|
||||
return {
|
||||
|
||||
@@ -14,6 +14,10 @@ import {
|
||||
ignoreSettledRemoteRuntimeSocketError
|
||||
} from './remote-runtime-client-handshake'
|
||||
import { RemoteRuntimeClientError } from './remote-runtime-client-error'
|
||||
import {
|
||||
remoteRuntimeConnectFailureMessage,
|
||||
remoteRuntimeConnectOptions
|
||||
} from './remote-runtime-connect-bound'
|
||||
import {
|
||||
isRemoteRuntimeBinaryFrameWithinLimit,
|
||||
REMOTE_RUNTIME_MAX_WEBSOCKET_FRAME_BYTES,
|
||||
@@ -225,11 +229,12 @@ export async function subscribeRemoteRuntimeTransport<TResult>(
|
||||
callbacks.onClose?.()
|
||||
}
|
||||
|
||||
const connectOptions = remoteRuntimeConnectOptions({
|
||||
maxPayload: REMOTE_RUNTIME_MAX_WEBSOCKET_FRAME_BYTES,
|
||||
...(options?.perMessageDeflate === false ? { perMessageDeflate: false } : {})
|
||||
})
|
||||
try {
|
||||
ws = new WebSocket(pairing.endpoint, {
|
||||
maxPayload: REMOTE_RUNTIME_MAX_WEBSOCKET_FRAME_BYTES,
|
||||
...(options?.perMessageDeflate === false ? { perMessageDeflate: false } : {})
|
||||
})
|
||||
ws = new WebSocket(pairing.endpoint, connectOptions)
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error)
|
||||
fail(new RemoteRuntimeClientError('invalid_argument', `Invalid remote endpoint: ${message}`))
|
||||
@@ -242,11 +247,11 @@ export async function subscribeRemoteRuntimeTransport<TResult>(
|
||||
)
|
||||
}
|
||||
|
||||
function onError(): void {
|
||||
function onError(error: Error): void {
|
||||
fail(
|
||||
new RemoteRuntimeClientError(
|
||||
'remote_runtime_unavailable',
|
||||
'Could not connect to the remote Orca runtime.'
|
||||
remoteRuntimeConnectFailureMessage(error, pairing.endpoint)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user