orca/src/shared/remote-runtime-transport-error-agreement.test.ts

/**
 * Differential guard: a transport error's CODE must not classify as fatal when its own
 * MESSAGE reads as recoverable.
 *
 * Since #12667, `isRecoverableRemoteRuntimeConnectionError` and
 * `isRuntimeRpcQueueOverloadError` treat a present code as authoritative and consult
 * `RECOVERABLE_MESSAGE_FRAGMENTS` only when there is no code. A transient code missing from
 * `RECOVERABLE_CODES` therefore classifies as fatal, and a fatal classification is what
 * dead-ends a terminal pane (#12650: the transport cancels recovery, unmounting the Reconnect
 * banner). This file enumerates the reachable (code, message) pairs and fails if any pair takes
 * that shape, plus pins that the code-less fragment fallback still works for untyped producers.
 *
 * The reverse direction — recoverable code, fatal-reading message — is deliberately allowed:
 * that is the code doing its job (e.g. 'Refreshing remote runtime control transport.' carries
 * remote_runtime_unavailable and no fragment would have rescued it).
 *
 * WHAT THIS GUARD CANNOT CATCH: a transient code whose message matches no fragment. The
 * message side has nothing to disagree with, so the pair looks consistent. `remote_runtime_busy`
 * is exactly that case today — its messages say "retry after…" but match no fragment, so it
 * classifies fatal and this guard stays silent. Tracked as STA-3479; do not "fix" it here.
 */
import { Buffer } from 'node:buffer'
import { describe, expect, it } from 'vitest'
import {
  RECOVERABLE_CODES,
  RECOVERABLE_MESSAGE_FRAGMENTS,
  isRecoverableRemoteRuntimeConnectionError,
  isRuntimeRpcQueueOverloadError,
  toRemoteRuntimeClientErrorLike,
  type RemoteRuntimeClientErrorLike
} from './remote-runtime-client-error-classification'
import {
  invalidRemoteRuntimeResponseError,
  parseAuthenticatedFrame,
  parseReadyFrame,
  parseRemoteRuntimeRpcFrame,
  remoteRuntimeTimeoutError,
  remoteRuntimeUnavailableError
} from './remote-runtime-request-frames'
import { formatSharedControlCloseMessage } from './remote-runtime-shared-control-protocol'
import { RuntimeRpcCallQueueOverloadError } from './runtime-rpc-call-queue'
import { withRemoteRuntimeTailscaleHint } from './remote-runtime-tailscale-hint'
import { MAX_TIMER_DELAY_MS } from './timer-delay'

type TransportErrorPair = RemoteRuntimeClientErrorLike & { producer: string }

/** Derives the pair from the real producer so a message edit updates the corpus with it. */
function producedPair(producer: string, error: unknown): TransportErrorPair {
  return { producer, ...toRemoteRuntimeClientErrorLike(error) }
}

function frameError(producer: string, frame: string): TransportErrorPair {
  const parsed = parseRemoteRuntimeRpcFrame(frame)
  return producedPair(producer, parsed.type === 'error' ? parsed.error : parsed)
}

// Why: these two live behind socket callbacks / module-private helpers that a unit test cannot
// reach, so their close-code shapes are reproduced here rather than derived.
const CLOSE_REASON = Buffer.from('server restarting')
const EMPTY_CLOSE_REASON = Buffer.from('')

const REQUEST_TRANSPORT_ERRORS: TransportErrorPair[] = [
  {
    producer: 'remote-runtime-client.ts:120',
    code: 'invalid_argument',
    message: `Runtime request timeout must be an integer between 0 and ${MAX_TIMER_DELAY_MS}ms.`
  },
  {
    producer: 'remote-runtime-client.ts:192',
    code: 'runtime_timeout',
    message: 'Timed out waiting for the remote Orca runtime to respond.'
  },
  {
    producer: 'remote-runtime-client.ts:238 / :639',
    code: 'invalid_argument',
    message: 'Invalid remote endpoint: Invalid URL'
  },
  {
    producer: 'remote-runtime-client.ts:258 / :654',
    code: 'remote_runtime_unavailable',
    message: 'Could not connect to the remote Orca runtime.'
  },
  {
    producer: 'remote-runtime-client.ts:270 / :667 (formatRemoteRuntimeCloseMessage, 1006)',
    code: 'remote_runtime_unavailable',
    message: 'Remote Orca runtime closed the connection.'
  },
  {
    producer: 'remote-runtime-client.ts:270 / :667 (formatRemoteRuntimeCloseMessage, 1011)',
    code: 'remote_runtime_unavailable',
    message: `Remote Orca runtime closed the connection (1011: ${CLOSE_REASON.toString()}).`
  },
  {
    producer: 'remote-runtime-client.ts:289',
    code: 'invalid_runtime_response',
    message: 'Remote Orca runtime returned an unexpected binary frame.'
  },
  {
    producer: 'remote-runtime-client.ts:310 / :693',
    code: 'invalid_runtime_response',
    message: 'Remote Orca runtime returned an undecryptable frame.'
  },
  {
    producer: 'remote-runtime-client.ts:393 / :801 (rejected token)',
    code: 'unauthorized',
    message: 'Remote Orca runtime rejected the pairing token.'
  },
  {
    producer: 'remote-runtime-client.ts:393 / :801 (unparseable auth failure)',
    code: 'invalid_runtime_response',
    message: 'Remote Orca runtime returned an invalid E2EE auth frame.'
  },
  {
    producer: 'remote-runtime-client.ts:415',
    code: 'remote_runtime_unavailable',
    message: 'Remote Orca runtime request was released before it could be sent.'
  },
  {
    producer: 'remote-runtime-client.ts:459 / :829',
    code: 'invalid_runtime_response',
    message: 'Remote Orca runtime returned a mismatched response id.'
  },
  {
    producer: 'remote-runtime-client.ts:476 (non-Error status validation failure)',
    code: 'runtime_error',
    message: 'status preflight rejected'
  },
  {
    producer: 'remote-runtime-client.ts:556',
    code: 'runtime_timeout',
    message: 'Timed out waiting for the remote Orca runtime subscription to start.'
  },
  {
    producer: 'remote-runtime-client.ts:587',
    code: 'remote_runtime_unavailable',
    message: 'Remote Orca runtime send buffer overflow; reconnecting.'
  },
  {
    producer: 'remote-runtime-client.ts:735',
    code: 'remote_runtime_unavailable',
    message: 'Remote Orca runtime stopped responding; the stream connection was reset.'
  },
  {
    producer: 'remote-runtime-client.ts:842',
    code: 'invalid_runtime_response',
    message: 'Remote Orca runtime returned binary data before authentication.'
  },
  {
    producer: 'remote-runtime-client.ts:852',
    code: 'invalid_runtime_response',
    message: 'Remote Orca runtime returned an undecryptable binary frame.'
  },
  {
    producer: 'remote-runtime-request-websocket.ts:115',
    code: 'invalid_argument',
    message: 'Invalid remote pairing key: bad base64'
  },
  {
    producer: 'remote-runtime-request-websocket.ts:127',
    code: 'invalid_argument',
    message: 'Invalid remote endpoint: Invalid URL'
  },
  {
    producer: 'remote-runtime-memory-limits.ts:26',
    code: 'invalid_argument',
    message: 'Remote runtime JSON payload exceeds 8388608 bytes.'
  },
  {
    producer: 'remote-runtime-memory-limits.ts:32',
    code: 'invalid_argument',
    message: 'Remote runtime JSON payload could not be serialized: Converting circular structure'
  },
  {
    producer: 'remote-runtime-memory-limits.ts:48',
    code: 'invalid_argument',
    message: 'Remote runtime subscription parameters exceed 1048576 bytes.'
  },
  {
    producer: 'remote-runtime-memory-limits.ts:54',
    code: 'invalid_argument',
    message:
      'Remote runtime subscription parameters could not be serialized: Converting circular structure'
  },
  producedPair('remote-runtime-request-frames.ts:24', remoteRuntimeUnavailableError()),
  producedPair('remote-runtime-request-frames.ts:30', remoteRuntimeTimeoutError()),
  producedPair('remote-runtime-request-frames.ts:46', parseReadyFrame('not-json')),
  producedPair('remote-runtime-request-frames.ts:55', parseReadyFrame('{"type":"nope"}')),
  producedPair('remote-runtime-request-frames.ts:67', parseAuthenticatedFrame('not-json')),
  producedPair(
    'remote-runtime-request-frames.ts:81 (rejected token)',
    parseAuthenticatedFrame(JSON.stringify({ error: { code: 'unauthorized' } }))
  ),
  producedPair(
    'remote-runtime-request-frames.ts:81 (unrecognized auth frame)',
    parseAuthenticatedFrame(JSON.stringify({ type: 'e2ee_other' }))
  ),
  frameError('remote-runtime-request-frames.ts:91', 'not-json'),
  frameError('remote-runtime-request-frames.ts:103', JSON.stringify({ not: 'an envelope' })),
  producedPair(
    'remote-runtime-request-frames.ts:37 (invalid handshake frame)',
    invalidRemoteRuntimeResponseError('Remote Orca runtime returned an invalid E2EE auth frame.')
  ),
  producedPair(
    'runtime-rpc-call-queue.ts:70 / :74 / :81',
    new RuntimeRpcCallQueueOverloadError('global')
  )
]

const SHARED_CONTROL_TRANSPORT_ERRORS: TransportErrorPair[] = [
  producedPair(
    'remote-runtime-shared-control-open.ts:37 (1006)',
    remoteRuntimeUnavailableError(formatSharedControlCloseMessage(1006, EMPTY_CLOSE_REASON))
  ),
  producedPair(
    'remote-runtime-shared-control-open.ts:37 (1011 with reason)',
    remoteRuntimeUnavailableError(formatSharedControlCloseMessage(1011, CLOSE_REASON))
  ),
  producedPair(
    'remote-runtime-shared-control-open.ts:37 (1011 without reason)',
    remoteRuntimeUnavailableError(formatSharedControlCloseMessage(1011, EMPTY_CLOSE_REASON))
  ),
  producedPair(
    'remote-runtime-shared-control-open.ts:86',
    remoteRuntimeUnavailableError(
      'Remote Orca runtime stopped responding; resetting the control connection.'
    )
  ),
  producedPair(
    'remote-runtime-shared-control-connection.ts:138',
    remoteRuntimeUnavailableError('Refreshing remote runtime control transport.')
  ),
  producedPair(
    'remote-runtime-shared-control-subscription-start.ts:48',
    remoteRuntimeUnavailableError('Remote runtime subscription closed before it started.')
  ),
  {
    producer: 'remote-runtime-shared-control-protocol.ts:160 / :162 (toRemoteRuntimeClientError)',
    code: 'runtime_error',
    message: 'Unexpected shared control failure'
  },
  {
    producer: 'remote-runtime-prepared-request-admission.ts:96',
    code: 'runtime_error',
    message: 'prepared request admission failed'
  }
]

/**
 * `remote_runtime_busy` is semantically transient but absent from `RECOVERABLE_CODES`, so it
 * classifies fatal. It is listed here (not omitted) so the corpus stays complete; STA-3479
 * covers the classification itself.
 */
const BUSY_TRANSPORT_ERRORS: TransportErrorPair[] = [
  {
    producer: 'remote-runtime-prepared-request-admission.ts:100',
    code: 'remote_runtime_busy',
    message: 'Remote runtime request limit reached; retry after pending requests finish.'
  },
  {
    producer: 'remote-runtime-shared-control-admission.ts:17',
    code: 'remote_runtime_busy',
    message: 'Remote runtime subscription limit reached; close a subscription and retry.'
  },
  {
    producer: 'remote-runtime-shared-control-admission.ts:27',
    code: 'remote_runtime_busy',
    message: 'Remote runtime subscription memory limit reached; close a subscription and retry.'
  },
  {
    producer: 'remote-runtime-shared-control-ready.ts:25',
    code: 'remote_runtime_busy',
    message: 'Remote runtime connection wait limit reached; retry after pending work finishes.'
  }
]

/**
 * Codes the host forwards through `mapRuntimeError`, reconstructed by `RuntimeRpcCallError`.
 * `RUNTIME_PASSTHROUGH_CODES` entries arrive with the code as their own message; the
 * `STRUCTURED_RUNTIME_PASSTHROUGH_CODES` and `runtime_error` entries carry the host's message,
 * which is open-ended — see the limitation note at the bottom of this file.
 */
const HOST_FORWARDED_TRANSPORT_ERRORS: TransportErrorPair[] = [
  {
    producer: 'main/runtime/rpc/errors.ts:155 (RUNTIME_PASSTHROUGH_CODES)',
    code: 'runtime_unavailable',
    message: 'runtime_unavailable'
  },
  {
    producer: 'main/runtime/rpc/errors.ts:155 (RUNTIME_PASSTHROUGH_CODES)',
    code: 'timeout',
    message: 'timeout'
  },
  {
    producer: 'main/runtime/rpc/errors.ts:155 (RUNTIME_PASSTHROUGH_CODES)',
    code: 'terminal_gone',
    message: 'terminal_gone'
  },
  {
    producer: 'main/runtime/rpc/errors.ts:145 (STRUCTURED_RUNTIME_PASSTHROUGH_CODES)',
    code: 'remote_runtime_unavailable',
    message: 'Remote Orca runtime closed the connection.'
  },
  {
    producer: 'main/runtime/rpc/errors.ts:145 (STRUCTURED_RUNTIME_PASSTHROUGH_CODES)',
    code: 'runtime_timeout',
    message: 'Timed out waiting for the remote Orca runtime to respond.'
  },
  {
    producer: 'main/runtime/rpc/errors.ts:145 (STRUCTURED_RUNTIME_PASSTHROUGH_CODES)',
    code: 'invalid_runtime_response',
    message: 'Remote Orca runtime returned an invalid response frame.'
  },
  {
    producer: 'main/runtime/rpc/errors.ts:145 (STRUCTURED_RUNTIME_PASSTHROUGH_CODES)',
    code: 'capability_unsupported',
    message: 'Remote host does not support this capability.'
  },
  {
    producer: 'main/runtime/rpc/errors.ts:161 (runtime_error fallthrough)',
    code: 'runtime_error',
    message: 'Worktree is missing on the remote host.'
  }
]

// Why: main rewrites the message of an already-coded error before it crosses IPC, so the
// hinted variants are distinct corpus members.
const TAILSCALE_HINTED_TRANSPORT_ERRORS: TransportErrorPair[] = [
  {
    producer: 'main/ipc/runtime-environment-transport-routing.ts:153',
    code: 'remote_runtime_unavailable',
    message: withRemoteRuntimeTailscaleHint(
      'Could not connect to the remote Orca runtime.',
      'https://desk.example.com'
    )
  },
  {
    producer: 'main/ipc/runtime-environment-transport-routing.ts:200',
    code: 'remote_runtime_unavailable',
    message: withRemoteRuntimeTailscaleHint(
      'Remote Orca runtime closed the connection.',
      'https://desk.tail1234.ts.net'
    )
  }
]

const CODED_TRANSPORT_ERRORS: TransportErrorPair[] = [
  ...REQUEST_TRANSPORT_ERRORS,
  ...SHARED_CONTROL_TRANSPORT_ERRORS,
  ...BUSY_TRANSPORT_ERRORS,
  ...HOST_FORWARDED_TRANSPORT_ERRORS,
  ...TAILSCALE_HINTED_TRANSPORT_ERRORS
]

/** Untyped producers that still depend on the message-fragment fallback. */
const CODELESS_TRANSPORT_ERRORS: (TransportErrorPair & { recoverable: boolean })[] = [
  {
    producer: 'web-runtime-client.ts:117 / :328',
    message: 'Remote Orca runtime is not connected.',
    recoverable: true
  },
  {
    producer: 'web-runtime-client.ts:359 / :360 / :591',
    message: 'Remote Orca runtime connection closed.',
    recoverable: true
  },
  {
    producer: 'web-runtime-client.ts:437 / :601',
    message: withRemoteRuntimeTailscaleHint(
      'Could not connect to the remote Orca runtime.',
      'https://desk.example.com'
    ),
    recoverable: true
  },
  {
    producer: 'remote-runtime-terminal-multiplexer.ts:455',
    message: 'Remote terminal stream is not connected.',
    recoverable: true
  },
  {
    producer: 'remote-runtime-terminal-multiplexer.ts:511',
    message: 'Remote Orca runtime closed the connection.',
    recoverable: true
  },
  {
    producer: 'remote-runtime-terminal-multiplexer.ts:1295 / :1308',
    message: 'Remote runtime connection closed.',
    recoverable: true
  },
  {
    producer: 'ipcMain.handle rejection that carries no code',
    message:
      "Error invoking remote method 'runtimeEnvironments:call': RuntimeRpcCallQueueOverloadError: Remote runtime call queue is full; retry after current calls finish.",
    recoverable: true
  },
  {
    producer: 'untyped host rejection with no connection wording',
    message: 'Worktree is missing on the remote host.',
    recoverable: false
  }
]

function matchingRecoverableFragment(message: string): string | null {
  const lowered = message.toLowerCase()
  return RECOVERABLE_MESSAGE_FRAGMENTS.find((fragment) => lowered.includes(fragment)) ?? null
}

function classifyByMessageOnly(pair: TransportErrorPair): boolean {
  return isRecoverableRemoteRuntimeConnectionError({ message: pair.message })
}

describe('transport error code/message classification agreement', () => {
  it('enumerates every reachable coded producer', () => {
    // Floor, not an exact count: the corpus should only grow. Lower it deliberately when a
    // producer is genuinely deleted. (#12667's review enumerated 34 of these by hand.)
    expect(CODED_TRANSPORT_ERRORS.length).toBeGreaterThanOrEqual(57)
    expect(CODED_TRANSPORT_ERRORS.every((pair) => typeof pair.code === 'string')).toBe(true)
  })

  it('never classifies a coded error as fatal while its own message reads as recoverable', () => {
    const violations = CODED_TRANSPORT_ERRORS.filter(
      (pair) => !isRecoverableRemoteRuntimeConnectionError(pair) && classifyByMessageOnly(pair)
    ).map(
      (pair) =>
        `${pair.producer}: code "${pair.code}" classifies fatal but its message matches recoverable fragment "${matchingRecoverableFragment(pair.message)}" — either add "${pair.code}" to RECOVERABLE_CODES in remote-runtime-client-error-classification.ts, or change the message so it no longer reads as a transient connection failure.`
    )
    expect(violations).toEqual([])
  })

  it('never classifies a coded error as non-overload while its own message reads as overload', () => {
    const violations = CODED_TRANSPORT_ERRORS.filter(
      (pair) =>
        !isRuntimeRpcQueueOverloadError(pair) &&
        isRuntimeRpcQueueOverloadError({ message: pair.message })
    ).map(
      (pair) =>
        `${pair.producer}: code "${pair.code}" is not the queue-overload code but its message reads as queue overload — either raise it with RUNTIME_RPC_QUEUE_OVERLOAD_CODE, or change the message.`
    )
    expect(violations).toEqual([])
  })

  it('keeps the code-less fragment fallback intact for untyped producers', () => {
    const misclassified = CODELESS_TRANSPORT_ERRORS.filter(
      (pair) =>
        isRecoverableRemoteRuntimeConnectionError({ message: pair.message }) !== pair.recoverable
    ).map(
      (pair) =>
        `${pair.producer}: expected message-only classification ${pair.recoverable} — untyped producers have no code, so removing a fragment from RECOVERABLE_MESSAGE_FRAGMENTS strands them.`
    )
    expect(misclassified).toEqual([])
  })

  it('backs every recoverable message fragment with a producer in the corpus', () => {
    const allMessages = [...CODED_TRANSPORT_ERRORS, ...CODELESS_TRANSPORT_ERRORS].map((pair) =>
      pair.message.toLowerCase()
    )
    const unbacked = RECOVERABLE_MESSAGE_FRAGMENTS.filter(
      (fragment) => !allMessages.some((message) => message.includes(fragment))
    ).map(
      (fragment) =>
        `recoverable fragment "${fragment}" matches no producer in this corpus — add the producer that emits it, or drop the fragment.`
    )
    expect(unbacked).toEqual([])
  })

  it('backs every recoverable code with a producer in the corpus', () => {
    // 'reconnecting' is vocabulary borrowed from the SSH/runtime status states; no error
    // producer raises it. Kept recoverable defensively.
    const codesWithoutProducer = new Set(['reconnecting'])
    const corpusCodes = new Set(CODED_TRANSPORT_ERRORS.map((pair) => pair.code))
    const unbacked = [...RECOVERABLE_CODES]
      .filter((code) => !corpusCodes.has(code) && !codesWithoutProducer.has(code))
      .map(
        (code) =>
          `recoverable code "${code}" matches no producer in this corpus — add the producer that raises it so its message is checked, or declare it producer-less here.`
      )
    expect(unbacked).toEqual([])
  })

  it('pins why the guard cannot see the remote_runtime_busy exception (STA-3479)', () => {
    for (const pair of BUSY_TRANSPORT_ERRORS) {
      // Fatal by code with a message that matches nothing: the two sides cannot disagree, so the
      // differential above stays silent. Fixing the classification is STA-3479, not this file.
      expect(isRecoverableRemoteRuntimeConnectionError(pair)).toBe(false)
      expect(matchingRecoverableFragment(pair.message)).toBeNull()
    }
  })
})