From fec52e441e089445733d4985d86baec736324811 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:03:57 -0700 Subject: [PATCH] fix(orchestration): preserve mutation recovery identity --- src/cli/runtime/client-recovery.test.ts | 107 ++++++++++++++++++ src/cli/runtime/client.ts | 32 +++++- .../orchestration-recovery-command.test.ts | 33 ++++++ .../runtime/orchestration-recovery-command.ts | 36 ++++++ .../runtime/websocket-transport-error.test.ts | 101 +++++++++++++++++ src/cli/runtime/websocket-transport.ts | 4 +- .../runtime-rpc-long-poll-transport.test.ts | 60 +++++++++- src/main/runtime/runtime-rpc.ts | 12 +- src/shared/remote-runtime-client-error.ts | 4 + .../remote-runtime-shared-control-protocol.ts | 6 +- .../terminal-send-agent-prompt-submit.spec.ts | 3 +- 11 files changed, 386 insertions(+), 12 deletions(-) create mode 100644 src/cli/runtime/client-recovery.test.ts create mode 100644 src/cli/runtime/orchestration-recovery-command.test.ts create mode 100644 src/cli/runtime/orchestration-recovery-command.ts create mode 100644 src/cli/runtime/websocket-transport-error.test.ts diff --git a/src/cli/runtime/client-recovery.test.ts b/src/cli/runtime/client-recovery.test.ts new file mode 100644 index 00000000000..ae77a009b0b --- /dev/null +++ b/src/cli/runtime/client-recovery.test.ts @@ -0,0 +1,107 @@ +import { createServer, type Server } from 'node:net' +import { mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY } from '../../shared/protocol-version' +import { orchestrationMutationRecoveryError } from '../orchestration-mutation-recovery' +import { RuntimeClient } from './client' + +const servers = new Set() + +afterEach(async () => { + await Promise.all( + [...servers].map( + (server) => + new Promise((resolve) => { + server.close(() => resolve()) + }) + ) + ) + servers.clear() +}) + +describe('RuntimeClient orchestration recovery identity', () => { + it('attaches the request and exact retry identity to a real RPC failure response', async () => { + const userDataPath = mkdtempSync(join(tmpdir(), 'orca-runtime-recovery-')) + const endpoint = join(userDataPath, 'runtime.sock') + const server = createServer((socket) => { + let buffer = '' + socket.setEncoding('utf8') + socket.on('data', (chunk: string) => { + buffer += chunk + const newline = buffer.indexOf('\n') + if (newline === -1) { + return + } + const request = JSON.parse(buffer.slice(0, newline)) as { id: string; method: string } + const response = + request.method === 'status.get' + ? { + id: request.id, + ok: true, + result: { capabilities: [ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY] }, + _meta: { runtimeId: 'runtime-1' } + } + : { + id: request.id, + ok: false, + error: { + code: 'runtime_timeout', + message: 'request timed out', + data: { requestId: 'request_1', dispatchId: 'dispatch_1' } + }, + _meta: { runtimeId: 'runtime-1' } + } + socket.end(`${JSON.stringify(response)}\n`) + }) + }) + servers.add(server) + await new Promise((resolve) => server.listen(endpoint, resolve)) + writeFileSync( + join(userDataPath, 'orca-runtime.json'), + JSON.stringify({ + runtimeId: 'runtime-1', + pid: 1, + transports: [{ kind: 'unix', endpoint }], + authToken: 'token', + startedAt: 1 + }) + ) + + const client = new RuntimeClient(userDataPath, 500) + try { + await client.call('orchestration.workerStart', { task: 'task_1' }) + throw new Error('expected worker-start failure') + } catch (error) { + const recovered = orchestrationMutationRecoveryError(error) as { + data?: Record + } + expect(recovered.data).toMatchObject({ + orchestrationRequestId: expect.any(String), + dispatchId: 'dispatch_1', + originalCommand: ['orca', 'orchestration', 'worker-start', '--task', 'task_1'], + recovery: { + queryCommand: [ + 'orca', + 'orchestration', + 'worker-show', + '--dispatch', + 'dispatch_1', + '--json' + ], + retryCommand: [ + 'orca', + 'orchestration', + 'worker-start', + '--task', + 'task_1', + '--retry-request', + expect.any(String) + ], + recoveryBlocked: false + } + }) + } + }) +}) diff --git a/src/cli/runtime/client.ts b/src/cli/runtime/client.ts index ad52b2868c3..8cd322462dd 100644 --- a/src/cli/runtime/client.ts +++ b/src/cli/runtime/client.ts @@ -20,6 +20,7 @@ import { RemoteRuntimeCompatGate } from './remote-runtime-compat-gate' import { createOrchestrationCompatibilityEnvelope } from './orchestration-compatibility-envelope' import { getTimeoutMsParam, isWaitingCheck } from './runtime-request-timeout' import { resolveWorkerStartClientTimeoutMs } from '../../shared/orchestration-timing-budgets' +import { buildOrchestrationRecoveryCommand } from './orchestration-recovery-command' // Why: for long-poll methods the caller's method-level // `params.timeoutMs` is the inner waiter budget; we extend the client-side @@ -80,6 +81,9 @@ export class RuntimeClient { const orchestrationRequestId = orchestrationMutation ? (options?.orchestrationRequestId ?? randomUUID()) : undefined + const originalCommand = orchestrationMutation + ? buildOrchestrationRecoveryCommand(method, params) + : undefined const compatibilityEnvelope = method.startsWith('orchestration.') ? { ...this.orchestrationCompatibility, @@ -108,10 +112,14 @@ export class RuntimeClient { envelope }) } catch (error) { - throw attachMutationRecovery(error, orchestrationRequestId) + throw attachMutationRecovery(error, orchestrationRequestId, originalCommand) } if (response.ok === false) { - throw new RuntimeRpcFailureError(response) + throw attachMutationRecovery( + new RuntimeRpcFailureError(response), + orchestrationRequestId, + originalCommand + ) } if (this.environmentSelector) { markEnvironmentUsed(this.userDataPath, this.environmentSelector, { @@ -125,10 +133,14 @@ export class RuntimeClient { try { response = await sendRequest(metadata, method, params, effectiveTimeoutMs, envelope) } catch (error) { - throw attachMutationRecovery(error, orchestrationRequestId) + throw attachMutationRecovery(error, orchestrationRequestId, originalCommand) } if (response.ok === false) { - throw new RuntimeRpcFailureError(response) + throw attachMutationRecovery( + new RuntimeRpcFailureError(response), + orchestrationRequestId, + originalCommand + ) } return response } @@ -246,7 +258,11 @@ export class RuntimeClient { } } -function attachMutationRecovery(error: unknown, requestId: string | undefined): unknown { +function attachMutationRecovery( + error: unknown, + requestId: string | undefined, + originalCommand?: string[] +): unknown { if (!requestId || !(error instanceof RuntimeClientError)) { return error } @@ -255,7 +271,11 @@ function attachMutationRecovery(error: unknown, requestId: string | undefined): `${error.message} Orchestration mutation request ID: ${requestId}.`, { ...(error.data && typeof error.data === 'object' ? error.data : {}), - orchestrationRequestId: requestId + orchestrationRequestId: requestId, + ...(originalCommand && + !(error.data && typeof error.data === 'object' && 'originalCommand' in error.data) + ? { originalCommand } + : {}) } ) } diff --git a/src/cli/runtime/orchestration-recovery-command.test.ts b/src/cli/runtime/orchestration-recovery-command.test.ts new file mode 100644 index 00000000000..134ab618bd8 --- /dev/null +++ b/src/cli/runtime/orchestration-recovery-command.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from 'vitest' +import { buildOrchestrationRecoveryCommand } from './orchestration-recovery-command' + +describe('orchestration recovery command identity', () => { + it('reconstructs the keyed worker-start command from its RPC params', () => { + expect( + buildOrchestrationRecoveryCommand('orchestration.workerStart', { + task: 'task_1', + on: 'windows', + worktree: 'new-top-level', + timeoutMs: 90_000, + devMode: false + }) + ).toEqual([ + 'orca', + 'orchestration', + 'worker-start', + '--task', + 'task_1', + '--on', + 'windows', + '--worktree', + 'new-top-level', + '--timeout-ms', + '90000' + ]) + }) + + it('does not fabricate identity for unsupported mutations or malformed params', () => { + expect(buildOrchestrationRecoveryCommand('orchestration.send', {})).toBeUndefined() + expect(buildOrchestrationRecoveryCommand('orchestration.workerStart', null)).toBeUndefined() + }) +}) diff --git a/src/cli/runtime/orchestration-recovery-command.ts b/src/cli/runtime/orchestration-recovery-command.ts new file mode 100644 index 00000000000..5cf6ecb054e --- /dev/null +++ b/src/cli/runtime/orchestration-recovery-command.ts @@ -0,0 +1,36 @@ +export function buildOrchestrationRecoveryCommand( + method: string, + params: unknown +): string[] | undefined { + const command = + method === 'orchestration.workerStart' + ? 'worker-start' + : method === 'orchestration.workerStop' + ? 'worker-stop' + : method === 'orchestration.workerAbandon' + ? 'worker-abandon' + : method === 'orchestration.workerRelease' + ? 'worker-release' + : undefined + if (!command || params === null || typeof params !== 'object') { + return undefined + } + const record = params as Record + const requiredKey = command === 'worker-start' ? 'task' : 'dispatch' + if (typeof record[requiredKey] !== 'string' || record[requiredKey].length === 0) { + return undefined + } + const args = ['orca', 'orchestration', command] + for (const [key, value] of Object.entries(record)) { + if (value === undefined || value === null || value === false) { + continue + } + const flag = `--${key.replace(/[A-Z]/g, (letter) => `-${letter.toLowerCase()}`)}` + if (value === true) { + args.push(flag) + } else if (typeof value === 'string' || typeof value === 'number') { + args.push(flag, String(value)) + } + } + return args +} diff --git a/src/cli/runtime/websocket-transport-error.test.ts b/src/cli/runtime/websocket-transport-error.test.ts new file mode 100644 index 00000000000..ab3df967f03 --- /dev/null +++ b/src/cli/runtime/websocket-transport-error.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, it, vi } from 'vitest' +import { RemoteRuntimeClientError } from '../../shared/remote-runtime-client-error' +import { orchestrationMutationRecoveryError } from '../orchestration-mutation-recovery' + +const { sendRemoteRuntimeRequest, sendRemoteRuntimeRequestWithStatusPreflight } = vi.hoisted( + () => ({ + sendRemoteRuntimeRequest: vi.fn(), + sendRemoteRuntimeRequestWithStatusPreflight: vi.fn() + }) +) + +vi.mock('../../shared/remote-runtime-client', () => ({ + RemoteRuntimeClientError, + sendRemoteRuntimeRequest, + sendRemoteRuntimeRequestWithStatusPreflight +})) + +import { + sendWebSocketRequest, + sendWebSocketRequestWithStatusPreflight +} from './websocket-transport' + +const pairing = { + v: 2, + endpoint: 'ws://127.0.0.1:1', + deviceToken: 'token', + publicKeyB64: 'key' +} as never + +describe('CLI WebSocket transport error identity', () => { + it('preserves recovery data from one-shot transport errors', async () => { + sendRemoteRuntimeRequest.mockRejectedValueOnce( + new RemoteRuntimeClientError('runtime_timeout', 'timed out', { + data: { + orchestrationRequestId: 'request_1', + dispatchId: 'dispatch_1', + originalCommand: ['orca', 'orchestration', 'worker-start', '--task', 'task_1'] + } + }) + ) + + try { + await sendWebSocketRequest(pairing, 'orchestration.workerStart', {}, 100) + throw new Error('expected transport failure') + } catch (error) { + expect(error).toMatchObject({ + code: 'runtime_timeout', + data: { + orchestrationRequestId: 'request_1', + dispatchId: 'dispatch_1', + originalCommand: ['orca', 'orchestration', 'worker-start', '--task', 'task_1'] + } + }) + expect(orchestrationMutationRecoveryError(error)).toMatchObject({ + data: { + recovery: { + dispatchId: 'dispatch_1', + queryCommand: [ + 'orca', + 'orchestration', + 'worker-show', + '--dispatch', + 'dispatch_1', + '--json' + ], + retryCommand: [ + 'orca', + 'orchestration', + 'worker-start', + '--task', + 'task_1', + '--retry-request', + 'request_1' + ] + } + } + }) + } + }) + + it('preserves recovery data through status-preflight transport errors', async () => { + sendRemoteRuntimeRequestWithStatusPreflight.mockRejectedValueOnce( + new RemoteRuntimeClientError('runtime_timeout', 'timed out', { + data: { orchestrationRequestId: 'request_2' } + }) + ) + + await expect( + sendWebSocketRequestWithStatusPreflight( + pairing, + 'orchestration.workerStart', + {}, + 100, + () => {} + ) + ).rejects.toMatchObject({ + code: 'runtime_timeout', + data: { orchestrationRequestId: 'request_2' } + }) + }) +}) diff --git a/src/cli/runtime/websocket-transport.ts b/src/cli/runtime/websocket-transport.ts index d71c1cbd671..8c7696d9f7d 100644 --- a/src/cli/runtime/websocket-transport.ts +++ b/src/cli/runtime/websocket-transport.ts @@ -19,7 +19,7 @@ export async function sendWebSocketRequest( return await sendRemoteRuntimeRequest(pairing, method, params, timeoutMs, envelope) } catch (error) { if (error instanceof RemoteRuntimeClientError) { - throw new RuntimeClientError(error.code, error.message) + throw new RuntimeClientError(error.code, error.message, error.data) } throw error } @@ -44,7 +44,7 @@ export async function sendWebSocketRequestWithStatusPreflight( ) } catch (error) { if (error instanceof RemoteRuntimeClientError) { - throw new RuntimeClientError(error.code, error.message) + throw new RuntimeClientError(error.code, error.message, error.data) } throw error } diff --git a/src/main/runtime/runtime-rpc-long-poll-transport.test.ts b/src/main/runtime/runtime-rpc-long-poll-transport.test.ts index e14b4561dcc..2286827ddeb 100644 --- a/src/main/runtime/runtime-rpc-long-poll-transport.test.ts +++ b/src/main/runtime/runtime-rpc-long-poll-transport.test.ts @@ -6,7 +6,7 @@ import { describe, expect, it, vi } from 'vitest' import { OrcaRuntimeService } from './orca-runtime' import { OrchestrationDb } from './orchestration/db' import { readRuntimeMetadata } from './runtime-metadata' -import { OrcaRuntimeRpcServer } from './runtime-rpc' +import { longPollClassOf, OrcaRuntimeRpcServer } from './runtime-rpc' import { sendRequest, openFramedSession, @@ -32,6 +32,25 @@ vi.mock('../git/worktree', () => { }) describe('OrcaRuntimeRpcServer', () => { + it('keeps agent-prompt submission sockets alive during verification', () => { + expect( + longPollClassOf({ + id: 'req_prompt', + authToken: 'token', + method: 'terminal.send', + params: { agentPrompt: true } + }) + ).toBe('wait') + expect( + longPollClassOf({ + id: 'req_direct', + authToken: 'token', + method: 'terminal.send', + params: { agentPrompt: false } + }) + ).toBeNull() + }) + it('rejects oversized RPC frames instead of buffering them indefinitely', async () => { const userDataPath = mkdtempSync(join(tmpdir(), 'orca-runtime-rpc-')) const runtime = new OrcaRuntimeService() @@ -241,6 +260,45 @@ describe('OrcaRuntimeRpcServer', () => { } }) + it('emits keepalive frames while agent-prompt verification blocks', async () => { + const userDataPath = mkdtempSync(join(tmpdir(), 'orca-runtime-rpc-')) + const runtime = new OrcaRuntimeService() + const server = new OrcaRuntimeRpcServer({ + runtime, + userDataPath, + keepaliveIntervalMs: 30 + }) + const dispatch = server['dispatcher'] + vi.spyOn(dispatch, 'dispatch').mockImplementation(async (request) => { + await sleep(120) + return { + id: request.id, + ok: true, + result: { send: { accepted: true } }, + _meta: { runtimeId: runtime.getRuntimeId() } + } + }) + await server.start() + + try { + const metadata = readRuntimeMetadata(userDataPath) + const session = openFramedSession(metadata!.transports[0]!.endpoint, { + id: 'req_prompt', + authToken: metadata!.authToken, + method: 'terminal.send', + params: { agentPrompt: true } + }) + await session.done + + expect( + session.frames.filter((frame) => frame._keepalive === true).length + ).toBeGreaterThanOrEqual(2) + expect(session.frames.filter((frame) => frame.ok !== undefined)).toHaveLength(1) + } finally { + await server.stop() + } + }) + it('releases terminal.wait long-poll slot when the client closes mid-wait', async () => { const userDataPath = mkdtempSync(join(tmpdir(), 'orca-runtime-rpc-')) const runtime = new OrcaRuntimeService() diff --git a/src/main/runtime/runtime-rpc.ts b/src/main/runtime/runtime-rpc.ts index 3759ac356c1..6d897d19641 100644 --- a/src/main/runtime/runtime-rpc.ts +++ b/src/main/runtime/runtime-rpc.ts @@ -445,10 +445,20 @@ const MOBILE_RPC_METHOD_ALLOWLIST = new Set([ type LongPollClass = 'ask' | 'wait' // Why: single classifier for long-poll requests (handlers that block on an external event), shared by counter/abort/keepalive. See ยง3.1. -function longPollClassOf(request: RpcRequest): LongPollClass | null { +export function longPollClassOf(request: RpcRequest): LongPollClass | null { if (request.method === 'terminal.wait') { return 'wait' } + // Agent-prompt submission waits for the PTY's lifecycle transition (up to + // the verification budget); keep the local socket alive for that wait. + if ( + request.method === 'terminal.send' && + typeof request.params === 'object' && + request.params !== null && + (request.params as { agentPrompt?: unknown }).agentPrompt === true + ) { + return 'wait' + } // Why: orchestration.ask blocks unconditionally (default 600 s) holding the // RPC open until a reply lands or the deadline passes, so it needs the same // keepalive as check --wait or the 30 s socket idle timer tears it down. It diff --git a/src/shared/remote-runtime-client-error.ts b/src/shared/remote-runtime-client-error.ts index 487a1108b87..ec76b55af64 100644 --- a/src/shared/remote-runtime-client-error.ts +++ b/src/shared/remote-runtime-client-error.ts @@ -9,6 +9,8 @@ export type RemoteRuntimePairingStage = 'connect' | 'host-identity' | 'access-gr */ export class RemoteRuntimeClientError extends Error { readonly code: string + /** Structured recovery data must survive the WebSocket adapter unchanged. */ + readonly data?: unknown readonly pairingStage?: RemoteRuntimePairingStage readonly closeCode?: number @@ -16,6 +18,7 @@ export class RemoteRuntimeClientError extends Error { code: string, message: string, details?: { + data?: unknown pairingStage?: RemoteRuntimePairingStage closeCode?: number } @@ -23,6 +26,7 @@ export class RemoteRuntimeClientError extends Error { super(message) this.name = 'RemoteRuntimeClientError' this.code = code + this.data = details?.data this.pairingStage = details?.pairingStage this.closeCode = details?.closeCode } diff --git a/src/shared/remote-runtime-shared-control-protocol.ts b/src/shared/remote-runtime-shared-control-protocol.ts index 4daae215989..e7857a08d69 100644 --- a/src/shared/remote-runtime-shared-control-protocol.ts +++ b/src/shared/remote-runtime-shared-control-protocol.ts @@ -157,7 +157,11 @@ export function toRemoteRuntimeClientError(error: unknown): RemoteRuntimeClientE return error } if (error instanceof Error) { - return new RemoteRuntimeClientError('runtime_error', error.message) + const data = + typeof error === 'object' && error !== null && 'data' in error + ? (error as { data?: unknown }).data + : undefined + return new RemoteRuntimeClientError('runtime_error', error.message, { data }) } return new RemoteRuntimeClientError('runtime_error', String(error)) } diff --git a/tests/e2e/terminal-send-agent-prompt-submit.spec.ts b/tests/e2e/terminal-send-agent-prompt-submit.spec.ts index 747a82a26ea..bec3a976694 100644 --- a/tests/e2e/terminal-send-agent-prompt-submit.spec.ts +++ b/tests/e2e/terminal-send-agent-prompt-submit.spec.ts @@ -5,6 +5,7 @@ import path from 'node:path' import { promisify } from 'node:util' import { expect, test } from './helpers/orca-app' import { waitForSessionReady } from './helpers/store' +import { SWALLOWED_ENTER_FIXTURE_TIMEOUT_MS } from '../../src/shared/orchestration-timing-budgets' const execFileAsync = promisify(execFile) const fixtureRoot = mkdtempSync(path.join(os.tmpdir(), 'orca-terminal-send-agent-prompt-')) @@ -13,7 +14,7 @@ const fixtureReport = path.join(fixtureRoot, 'report.json') const fixtureMarker = `ORCA_TERMINAL_SEND_E2E_${process.pid}` const fixtureScript = path.join(process.cwd(), 'tests', 'tools', 'repro-terminal-send-submit.mjs') const fakeCodex = path.join(fixtureBin, process.platform === 'win32' ? 'codex.cmd' : 'codex') -const swallowedEnterFixtureTimeoutMs = 60_000 +const swallowedEnterFixtureTimeoutMs = SWALLOWED_ENTER_FIXTURE_TIMEOUT_MS mkdirSync(fixtureBin) writeFileSync(