diff --git a/src/shared/remote-runtime-client-error-classification.ts b/src/shared/remote-runtime-client-error-classification.ts index 37d55f271d2..d19027938e9 100644 --- a/src/shared/remote-runtime-client-error-classification.ts +++ b/src/shared/remote-runtime-client-error-classification.ts @@ -3,7 +3,9 @@ export type RemoteRuntimeClientErrorLike = { code?: string; message: string } export const RUNTIME_RPC_QUEUE_OVERLOAD_CODE = 'runtime_rpc_queue_overloaded' export const RUNTIME_RPC_QUEUE_OVERLOAD_MESSAGE_FRAGMENT = 'remote runtime call queue is full' -const RECOVERABLE_CODES = new Set([ +// Exported so the transport-error corpus guard can name the offending entry when a +// code and its message disagree; see remote-runtime-transport-error-agreement.test.ts. +export const RECOVERABLE_CODES: ReadonlySet = new Set([ 'remote_runtime_unavailable', RUNTIME_RPC_QUEUE_OVERLOAD_CODE, 'runtime_timeout', @@ -12,7 +14,7 @@ const RECOVERABLE_CODES = new Set([ 'timeout' ]) -const RECOVERABLE_MESSAGE_FRAGMENTS = [ +export const RECOVERABLE_MESSAGE_FRAGMENTS: readonly string[] = [ 'could not connect to the remote orca runtime', 'remote orca runtime closed the connection', 'remote orca runtime connection closed', diff --git a/src/shared/remote-runtime-transport-error-agreement.test.ts b/src/shared/remote-runtime-transport-error-agreement.test.ts new file mode 100644 index 00000000000..4c1324434ca --- /dev/null +++ b/src/shared/remote-runtime-transport-error-agreement.test.ts @@ -0,0 +1,487 @@ +/** + * Differential guard: a transport error's CODE must not classify as fatal when its own + * MESSAGE reads as recoverable. + * + * Since #12667, `isRecoverableRemoteRuntimeConnectionError` and + * `isRuntimeRpcQueueOverloadError` treat a present code as authoritative and consult + * `RECOVERABLE_MESSAGE_FRAGMENTS` only when there is no code. A transient code missing from + * `RECOVERABLE_CODES` therefore classifies as fatal, and a fatal classification is what + * dead-ends a terminal pane (#12650: the transport cancels recovery, unmounting the Reconnect + * banner). This file enumerates the reachable (code, message) pairs and fails if any pair takes + * that shape, plus pins that the code-less fragment fallback still works for untyped producers. + * + * The reverse direction — recoverable code, fatal-reading message — is deliberately allowed: + * that is the code doing its job (e.g. 'Refreshing remote runtime control transport.' carries + * remote_runtime_unavailable and no fragment would have rescued it). + * + * WHAT THIS GUARD CANNOT CATCH: a transient code whose message matches no fragment. The + * message side has nothing to disagree with, so the pair looks consistent. `remote_runtime_busy` + * is exactly that case today — its messages say "retry after…" but match no fragment, so it + * classifies fatal and this guard stays silent. Tracked as STA-3479; do not "fix" it here. + */ +import { Buffer } from 'node:buffer' +import { describe, expect, it } from 'vitest' +import { + RECOVERABLE_CODES, + RECOVERABLE_MESSAGE_FRAGMENTS, + isRecoverableRemoteRuntimeConnectionError, + isRuntimeRpcQueueOverloadError, + toRemoteRuntimeClientErrorLike, + type RemoteRuntimeClientErrorLike +} from './remote-runtime-client-error-classification' +import { + invalidRemoteRuntimeResponseError, + parseAuthenticatedFrame, + parseReadyFrame, + parseRemoteRuntimeRpcFrame, + remoteRuntimeTimeoutError, + remoteRuntimeUnavailableError +} from './remote-runtime-request-frames' +import { formatSharedControlCloseMessage } from './remote-runtime-shared-control-protocol' +import { RuntimeRpcCallQueueOverloadError } from './runtime-rpc-call-queue' +import { withRemoteRuntimeTailscaleHint } from './remote-runtime-tailscale-hint' +import { MAX_TIMER_DELAY_MS } from './timer-delay' + +type TransportErrorPair = RemoteRuntimeClientErrorLike & { producer: string } + +/** Derives the pair from the real producer so a message edit updates the corpus with it. */ +function producedPair(producer: string, error: unknown): TransportErrorPair { + return { producer, ...toRemoteRuntimeClientErrorLike(error) } +} + +function frameError(producer: string, frame: string): TransportErrorPair { + const parsed = parseRemoteRuntimeRpcFrame(frame) + return producedPair(producer, parsed.type === 'error' ? parsed.error : parsed) +} + +// Why: these two live behind socket callbacks / module-private helpers that a unit test cannot +// reach, so their close-code shapes are reproduced here rather than derived. +const CLOSE_REASON = Buffer.from('server restarting') +const EMPTY_CLOSE_REASON = Buffer.from('') + +const REQUEST_TRANSPORT_ERRORS: TransportErrorPair[] = [ + { + producer: 'remote-runtime-client.ts:120', + code: 'invalid_argument', + message: `Runtime request timeout must be an integer between 0 and ${MAX_TIMER_DELAY_MS}ms.` + }, + { + producer: 'remote-runtime-client.ts:192', + code: 'runtime_timeout', + message: 'Timed out waiting for the remote Orca runtime to respond.' + }, + { + producer: 'remote-runtime-client.ts:238 / :639', + code: 'invalid_argument', + message: 'Invalid remote endpoint: Invalid URL' + }, + { + producer: 'remote-runtime-client.ts:258 / :654', + code: 'remote_runtime_unavailable', + message: 'Could not connect to the remote Orca runtime.' + }, + { + producer: 'remote-runtime-client.ts:270 / :667 (formatRemoteRuntimeCloseMessage, 1006)', + code: 'remote_runtime_unavailable', + message: 'Remote Orca runtime closed the connection.' + }, + { + producer: 'remote-runtime-client.ts:270 / :667 (formatRemoteRuntimeCloseMessage, 1011)', + code: 'remote_runtime_unavailable', + message: `Remote Orca runtime closed the connection (1011: ${CLOSE_REASON.toString()}).` + }, + { + producer: 'remote-runtime-client.ts:289', + code: 'invalid_runtime_response', + message: 'Remote Orca runtime returned an unexpected binary frame.' + }, + { + producer: 'remote-runtime-client.ts:310 / :693', + code: 'invalid_runtime_response', + message: 'Remote Orca runtime returned an undecryptable frame.' + }, + { + producer: 'remote-runtime-client.ts:393 / :801 (rejected token)', + code: 'unauthorized', + message: 'Remote Orca runtime rejected the pairing token.' + }, + { + producer: 'remote-runtime-client.ts:393 / :801 (unparseable auth failure)', + code: 'invalid_runtime_response', + message: 'Remote Orca runtime rejected the pairing token.' + }, + { + producer: 'remote-runtime-client.ts:415', + code: 'remote_runtime_unavailable', + message: 'Remote Orca runtime request was released before it could be sent.' + }, + { + producer: 'remote-runtime-client.ts:459 / :829', + code: 'invalid_runtime_response', + message: 'Remote Orca runtime returned a mismatched response id.' + }, + { + producer: 'remote-runtime-client.ts:476 (non-Error status validation failure)', + code: 'runtime_error', + message: 'status preflight rejected' + }, + { + producer: 'remote-runtime-client.ts:556', + code: 'runtime_timeout', + message: 'Timed out waiting for the remote Orca runtime subscription to start.' + }, + { + producer: 'remote-runtime-client.ts:587', + code: 'remote_runtime_unavailable', + message: 'Remote Orca runtime send buffer overflow; reconnecting.' + }, + { + producer: 'remote-runtime-client.ts:735', + code: 'remote_runtime_unavailable', + message: 'Remote Orca runtime stopped responding; the stream connection was reset.' + }, + { + producer: 'remote-runtime-client.ts:842', + code: 'invalid_runtime_response', + message: 'Remote Orca runtime returned binary data before authentication.' + }, + { + producer: 'remote-runtime-client.ts:852', + code: 'invalid_runtime_response', + message: 'Remote Orca runtime returned an undecryptable binary frame.' + }, + { + producer: 'remote-runtime-request-websocket.ts:115', + code: 'invalid_argument', + message: 'Invalid remote pairing key: bad base64' + }, + { + producer: 'remote-runtime-request-websocket.ts:127', + code: 'invalid_argument', + message: 'Invalid remote endpoint: Invalid URL' + }, + { + producer: 'remote-runtime-memory-limits.ts:26', + code: 'invalid_argument', + message: 'Remote runtime JSON payload exceeds 8388608 bytes.' + }, + { + producer: 'remote-runtime-memory-limits.ts:32', + code: 'invalid_argument', + message: 'Remote runtime JSON payload could not be serialized: Converting circular structure' + }, + { + producer: 'remote-runtime-memory-limits.ts:48', + code: 'invalid_argument', + message: 'Remote runtime subscription parameters exceed 1048576 bytes.' + }, + { + producer: 'remote-runtime-memory-limits.ts:54', + code: 'invalid_argument', + message: + 'Remote runtime subscription parameters could not be serialized: Converting circular structure' + }, + producedPair('remote-runtime-request-frames.ts:24', remoteRuntimeUnavailableError()), + producedPair('remote-runtime-request-frames.ts:30', remoteRuntimeTimeoutError()), + producedPair('remote-runtime-request-frames.ts:46', parseReadyFrame('not-json')), + producedPair('remote-runtime-request-frames.ts:55', parseReadyFrame('{"type":"nope"}')), + producedPair('remote-runtime-request-frames.ts:67', parseAuthenticatedFrame('not-json')), + producedPair( + 'remote-runtime-request-frames.ts:81 (rejected token)', + parseAuthenticatedFrame(JSON.stringify({ error: { code: 'unauthorized' } })) + ), + producedPair( + 'remote-runtime-request-frames.ts:81 (unrecognized auth frame)', + parseAuthenticatedFrame(JSON.stringify({ type: 'e2ee_other' })) + ), + frameError('remote-runtime-request-frames.ts:91', 'not-json'), + frameError('remote-runtime-request-frames.ts:103', JSON.stringify({ not: 'an envelope' })), + producedPair( + 'remote-runtime-request-frames.ts:37 (invalid handshake frame)', + invalidRemoteRuntimeResponseError('Remote Orca runtime returned an invalid E2EE auth frame.') + ), + producedPair( + 'runtime-rpc-call-queue.ts:70 / :74 / :81', + new RuntimeRpcCallQueueOverloadError('global') + ) +] + +const SHARED_CONTROL_TRANSPORT_ERRORS: TransportErrorPair[] = [ + producedPair( + 'remote-runtime-shared-control-open.ts:37 (1006)', + remoteRuntimeUnavailableError(formatSharedControlCloseMessage(1006, EMPTY_CLOSE_REASON)) + ), + producedPair( + 'remote-runtime-shared-control-open.ts:37 (1011 with reason)', + remoteRuntimeUnavailableError(formatSharedControlCloseMessage(1011, CLOSE_REASON)) + ), + producedPair( + 'remote-runtime-shared-control-open.ts:37 (1011 without reason)', + remoteRuntimeUnavailableError(formatSharedControlCloseMessage(1011, EMPTY_CLOSE_REASON)) + ), + producedPair( + 'remote-runtime-shared-control-open.ts:86', + remoteRuntimeUnavailableError( + 'Remote Orca runtime stopped responding; resetting the control connection.' + ) + ), + producedPair( + 'remote-runtime-shared-control-connection.ts:138', + remoteRuntimeUnavailableError('Refreshing remote runtime control transport.') + ), + producedPair( + 'remote-runtime-shared-control-subscription-start.ts:48', + remoteRuntimeUnavailableError('Remote runtime subscription closed before it started.') + ), + { + producer: 'remote-runtime-shared-control-protocol.ts:160 / :162 (toRemoteRuntimeClientError)', + code: 'runtime_error', + message: 'Unexpected shared control failure' + }, + { + producer: 'remote-runtime-prepared-request-admission.ts:96', + code: 'runtime_error', + message: 'prepared request admission failed' + } +] + +/** + * `remote_runtime_busy` is semantically transient but absent from `RECOVERABLE_CODES`, so it + * classifies fatal. It is listed here (not omitted) so the corpus stays complete; STA-3479 + * covers the classification itself. + */ +const BUSY_TRANSPORT_ERRORS: TransportErrorPair[] = [ + { + producer: 'remote-runtime-prepared-request-admission.ts:100', + code: 'remote_runtime_busy', + message: 'Remote runtime request limit reached; retry after pending requests finish.' + }, + { + producer: 'remote-runtime-shared-control-admission.ts:17', + code: 'remote_runtime_busy', + message: 'Remote runtime subscription limit reached; close a subscription and retry.' + }, + { + producer: 'remote-runtime-shared-control-admission.ts:27', + code: 'remote_runtime_busy', + message: 'Remote runtime subscription memory limit reached; close a subscription and retry.' + }, + { + producer: 'remote-runtime-shared-control-ready.ts:25', + code: 'remote_runtime_busy', + message: 'Remote runtime connection wait limit reached; retry after pending work finishes.' + } +] + +/** + * Codes the host forwards through `mapRuntimeError`, reconstructed by `RuntimeRpcCallError`. + * `RUNTIME_PASSTHROUGH_CODES` entries arrive with the code as their own message; the + * `STRUCTURED_RUNTIME_PASSTHROUGH_CODES` and `runtime_error` entries carry the host's message, + * which is open-ended — see the limitation note at the bottom of this file. + */ +const HOST_FORWARDED_TRANSPORT_ERRORS: TransportErrorPair[] = [ + { + producer: 'main/runtime/rpc/errors.ts:155 (RUNTIME_PASSTHROUGH_CODES)', + code: 'runtime_unavailable', + message: 'runtime_unavailable' + }, + { + producer: 'main/runtime/rpc/errors.ts:155 (RUNTIME_PASSTHROUGH_CODES)', + code: 'timeout', + message: 'timeout' + }, + { + producer: 'main/runtime/rpc/errors.ts:155 (RUNTIME_PASSTHROUGH_CODES)', + code: 'terminal_gone', + message: 'terminal_gone' + }, + { + producer: 'main/runtime/rpc/errors.ts:145 (STRUCTURED_RUNTIME_PASSTHROUGH_CODES)', + code: 'remote_runtime_unavailable', + message: 'Remote Orca runtime closed the connection.' + }, + { + producer: 'main/runtime/rpc/errors.ts:145 (STRUCTURED_RUNTIME_PASSTHROUGH_CODES)', + code: 'runtime_timeout', + message: 'Timed out waiting for the remote Orca runtime to respond.' + }, + { + producer: 'main/runtime/rpc/errors.ts:145 (STRUCTURED_RUNTIME_PASSTHROUGH_CODES)', + code: 'invalid_runtime_response', + message: 'Remote Orca runtime returned an invalid response frame.' + }, + { + producer: 'main/runtime/rpc/errors.ts:145 (STRUCTURED_RUNTIME_PASSTHROUGH_CODES)', + code: 'capability_unsupported', + message: 'Remote host does not support this capability.' + }, + { + producer: 'main/runtime/rpc/errors.ts:161 (runtime_error fallthrough)', + code: 'runtime_error', + message: 'Worktree is missing on the remote host.' + } +] + +// Why: main rewrites the message of an already-coded error before it crosses IPC, so the +// hinted variants are distinct corpus members. +const TAILSCALE_HINTED_TRANSPORT_ERRORS: TransportErrorPair[] = [ + { + producer: 'main/ipc/runtime-environment-transport-routing.ts:153', + code: 'remote_runtime_unavailable', + message: withRemoteRuntimeTailscaleHint( + 'Could not connect to the remote Orca runtime.', + 'https://desk.example.com' + ) + }, + { + producer: 'main/ipc/runtime-environment-transport-routing.ts:200', + code: 'remote_runtime_unavailable', + message: withRemoteRuntimeTailscaleHint( + 'Remote Orca runtime closed the connection.', + 'https://desk.tail1234.ts.net' + ) + } +] + +const CODED_TRANSPORT_ERRORS: TransportErrorPair[] = [ + ...REQUEST_TRANSPORT_ERRORS, + ...SHARED_CONTROL_TRANSPORT_ERRORS, + ...BUSY_TRANSPORT_ERRORS, + ...HOST_FORWARDED_TRANSPORT_ERRORS, + ...TAILSCALE_HINTED_TRANSPORT_ERRORS +] + +/** Untyped producers that still depend on the message-fragment fallback. */ +const CODELESS_TRANSPORT_ERRORS: (TransportErrorPair & { recoverable: boolean })[] = [ + { + producer: 'web-runtime-client.ts:117 / :328', + message: 'Remote Orca runtime is not connected.', + recoverable: true + }, + { + producer: 'web-runtime-client.ts:359 / :360 / :591', + message: 'Remote Orca runtime connection closed.', + recoverable: true + }, + { + producer: 'web-runtime-client.ts:437 / :601', + message: withRemoteRuntimeTailscaleHint( + 'Could not connect to the remote Orca runtime.', + 'https://desk.example.com' + ), + recoverable: true + }, + { + producer: 'remote-runtime-terminal-multiplexer.ts:455', + message: 'Remote terminal stream is not connected.', + recoverable: true + }, + { + producer: 'remote-runtime-terminal-multiplexer.ts:511', + message: 'Remote Orca runtime closed the connection.', + recoverable: true + }, + { + producer: 'remote-runtime-terminal-multiplexer.ts:1295 / :1308', + message: 'Remote runtime connection closed.', + recoverable: true + }, + { + producer: 'ipcMain.handle rejection that carries no code', + message: + "Error invoking remote method 'runtimeEnvironments:call': RuntimeRpcCallQueueOverloadError: Remote runtime call queue is full; retry after current calls finish.", + recoverable: true + }, + { + producer: 'untyped host rejection with no connection wording', + message: 'Worktree is missing on the remote host.', + recoverable: false + } +] + +function matchingRecoverableFragment(message: string): string | null { + const lowered = message.toLowerCase() + return RECOVERABLE_MESSAGE_FRAGMENTS.find((fragment) => lowered.includes(fragment)) ?? null +} + +function classifyByMessageOnly(pair: TransportErrorPair): boolean { + return isRecoverableRemoteRuntimeConnectionError({ message: pair.message }) +} + +describe('transport error code/message classification agreement', () => { + it('enumerates every reachable coded producer', () => { + // Floor, not an exact count: the corpus should only grow. Lower it deliberately when a + // producer is genuinely deleted. (#12667's review enumerated 34 of these by hand.) + expect(CODED_TRANSPORT_ERRORS.length).toBeGreaterThanOrEqual(57) + expect(CODED_TRANSPORT_ERRORS.every((pair) => typeof pair.code === 'string')).toBe(true) + }) + + it('never classifies a coded error as fatal while its own message reads as recoverable', () => { + const violations = CODED_TRANSPORT_ERRORS.filter( + (pair) => !isRecoverableRemoteRuntimeConnectionError(pair) && classifyByMessageOnly(pair) + ).map( + (pair) => + `${pair.producer}: code "${pair.code}" classifies fatal but its message matches recoverable fragment "${matchingRecoverableFragment(pair.message)}" — either add "${pair.code}" to RECOVERABLE_CODES in remote-runtime-client-error-classification.ts, or change the message so it no longer reads as a transient connection failure.` + ) + expect(violations).toEqual([]) + }) + + it('never classifies a coded error as non-overload while its own message reads as overload', () => { + const violations = CODED_TRANSPORT_ERRORS.filter( + (pair) => + !isRuntimeRpcQueueOverloadError(pair) && + isRuntimeRpcQueueOverloadError({ message: pair.message }) + ).map( + (pair) => + `${pair.producer}: code "${pair.code}" is not the queue-overload code but its message reads as queue overload — either raise it with RUNTIME_RPC_QUEUE_OVERLOAD_CODE, or change the message.` + ) + expect(violations).toEqual([]) + }) + + it('keeps the code-less fragment fallback intact for untyped producers', () => { + const misclassified = CODELESS_TRANSPORT_ERRORS.filter( + (pair) => + isRecoverableRemoteRuntimeConnectionError({ message: pair.message }) !== pair.recoverable + ).map( + (pair) => + `${pair.producer}: expected message-only classification ${pair.recoverable} — untyped producers have no code, so removing a fragment from RECOVERABLE_MESSAGE_FRAGMENTS strands them.` + ) + expect(misclassified).toEqual([]) + }) + + it('backs every recoverable message fragment with a producer in the corpus', () => { + const allMessages = [...CODED_TRANSPORT_ERRORS, ...CODELESS_TRANSPORT_ERRORS].map((pair) => + pair.message.toLowerCase() + ) + const unbacked = RECOVERABLE_MESSAGE_FRAGMENTS.filter( + (fragment) => !allMessages.some((message) => message.includes(fragment)) + ).map( + (fragment) => + `recoverable fragment "${fragment}" matches no producer in this corpus — add the producer that emits it, or drop the fragment.` + ) + expect(unbacked).toEqual([]) + }) + + it('backs every recoverable code with a producer in the corpus', () => { + // 'reconnecting' is vocabulary borrowed from the SSH/runtime status states; no error + // producer raises it. Kept recoverable defensively. + const codesWithoutProducer = new Set(['reconnecting']) + const corpusCodes = new Set(CODED_TRANSPORT_ERRORS.map((pair) => pair.code)) + const unbacked = [...RECOVERABLE_CODES] + .filter((code) => !corpusCodes.has(code) && !codesWithoutProducer.has(code)) + .map( + (code) => + `recoverable code "${code}" matches no producer in this corpus — add the producer that raises it so its message is checked, or declare it producer-less here.` + ) + expect(unbacked).toEqual([]) + }) + + it('pins why the guard cannot see the remote_runtime_busy exception (STA-3479)', () => { + for (const pair of BUSY_TRANSPORT_ERRORS) { + // Fatal by code with a message that matches nothing: the two sides cannot disagree, so the + // differential above stays silent. Fixing the classification is STA-3479, not this file. + expect(isRecoverableRemoteRuntimeConnectionError(pair)).toBe(false) + expect(matchingRecoverableFragment(pair.message)).toBeNull() + } + }) +})