From aec5ffb331f17510f3a0ec8513d6d64a724ec251 Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Tue, 10 Nov 2026 04:48:51 -0500 Subject: [PATCH] fix(orchestration): a remote worker's question is answered only by its Run's coordinator; tests for the forwarder --- .../remote-owned/remote-dispatch-records.ts | 19 ++ .../remote-owned/remote-owned-route.test.ts | 307 ++++++++++++++++++ .../remote-owned/remote-owned-route.ts | 9 +- .../orchestration/messaging/reply-access.ts | 19 ++ 4 files changed, 347 insertions(+), 7 deletions(-) create mode 100644 src/main/runtime/orchestration/federation/remote-owned/remote-owned-route.test.ts diff --git a/src/main/runtime/orchestration/db/remote-owned/remote-dispatch-records.ts b/src/main/runtime/orchestration/db/remote-owned/remote-dispatch-records.ts index 3a045909fcb..71a11d95f00 100644 --- a/src/main/runtime/orchestration/db/remote-owned/remote-dispatch-records.ts +++ b/src/main/runtime/orchestration/db/remote-owned/remote-dispatch-records.ts @@ -265,3 +265,22 @@ export function deferRemoteDispatchStop( record.stop_attempt ) } + +export type RemoteOwnedQuestionRow = { + message_id: string + conversation_id: string + run_id: string + environment_id: string + dispatch_id: string +} + +/** A question a remote worker asked here, which its server answers. */ +export function getRemoteOwnedQuestion( + db: OrchestrationDb, + messageId: string +): RemoteOwnedQuestionRow | undefined { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SELECT * over this table returns exactly its row shape. + return db.db + .prepare('SELECT * FROM remote_owned_questions WHERE message_id = ?') + .get(messageId) as RemoteOwnedQuestionRow | undefined +} diff --git a/src/main/runtime/orchestration/federation/remote-owned/remote-owned-route.test.ts b/src/main/runtime/orchestration/federation/remote-owned/remote-owned-route.test.ts new file mode 100644 index 00000000000..2ad24d9a17f --- /dev/null +++ b/src/main/runtime/orchestration/federation/remote-owned/remote-owned-route.test.ts @@ -0,0 +1,307 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY, + ORCHESTRATION_CONTRACT_VERSION, + ORCHESTRATION_REMOTE_OWNED_WORKERS_RUNTIME_CAPABILITY +} from '../../../../../shared/orchestration/orchestration-runtime-capabilities' +import type { + RuntimeOrchestrationEnvelope, + RuntimeRpcResponse +} from '../../../../../shared/runtime-rpc-envelope' +import { OrcaRuntimeService } from '../../../orca-runtime' +import { RpcDispatcher } from '../../../rpc/dispatcher' +import type { RpcResponse } from '../../../rpc/core' +import { ORCHESTRATION_METHODS } from '../../../rpc/methods/orchestration/orchestration' +import { OrchestrationDb } from '../../db/orchestration-db' +import { getRemoteDispatch } from '../../db/remote-owned/remote-dispatch-records' +import { importRemoteOwnedItem } from '../../db/remote-owned/remote-owned-import' +import { getRemotePlacement } from '../../db/remote-owned/remote-placements' +import { remoteServerPin } from '../../db/remote-owned/remote-run-links' +import type { + OrchestrationEnvironmentTransport, + OrchestrationWorkerServer +} from '../environment-transport' + +const COORDINATOR_PANE = 'tab_coord:11111111-1111-4111-8111-111111111111' +const OTHER_PANE = 'tab_other:22222222-2222-4222-8222-222222222222' +const REQUEST = '6f1c2f1e-3a7b-4d55-9c1e-0a1b2c3d4e5f' + +type Sent = { + method: string + params: Record + envelope?: RuntimeOrchestrationEnvelope +} +type Answer = { result: unknown } | { error: string; data?: unknown } | 'unreachable' + +describe('commands about a worker a paired server owns', () => { + let db: OrchestrationDb + let runtime: OrcaRuntimeService + let dispatcher: RpcDispatcher + let sent: Sent[] + let answers: Record + let capabilities: string[] + let sequence = 0 + const server: OrchestrationWorkerServer = { + environmentId: 'env_b', + name: 'devB', + peerFingerprint: 'host_b', + homePeerFingerprint: 'cred_a', + pairingRevision: 1 + } + + const transport = (): OrchestrationEnvironmentTransport => ({ + resolve: () => server, + hasEnvironment: () => true, + call: async (_selector, method, params, _timeout, envelope, _revision, send) => { + const ok = (result: unknown): RuntimeRpcResponse => ({ + id: method, + ok: true, + result, + _meta: { runtimeId: 'runtime_b' } + }) + if (method === 'status.get') { + return ok({ runtimeId: 'runtime_b', capabilities }) + } + if (method === 'orchestration.peerMailPull') { + return ok({ items: [] }) + } + const prepared = send?.prepareRequest?.({ params, envelope, measure: () => 0 }) + sent.push({ + method, + params: (prepared?.params ?? params) as Record, + envelope: prepared?.envelope ?? envelope + }) + const answer = answers[method]?.shift() ?? 'unreachable' + if (answer === 'unreachable') { + throw Object.assign(new Error('Could not connect to the remote Orca runtime.'), { + code: 'remote_runtime_unavailable' + }) + } + return 'result' in answer + ? ok(answer.result) + : { + id: method, + ok: false, + error: { code: answer.error, message: answer.error, data: answer.data } + } + } + }) + + beforeEach(() => { + sent = [] + answers = {} + capabilities = [ + ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY, + ORCHESTRATION_REMOTE_OWNED_WORKERS_RUNTIME_CAPABILITY + ] + db = new OrchestrationDb(':memory:') + runtime = new OrcaRuntimeService(null, undefined, { + orchestrationEnvironmentTransport: transport() + }) + vi.spyOn(runtime, 'readMachineName').mockReturnValue('Mac A') + const panes: Record = { term_coord: COORDINATOR_PANE, term_other: OTHER_PANE } + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => panes[handle] ?? null) + vi.spyOn(runtime, 'getLiveTerminalPaneKey').mockImplementation( + (handle) => panes[handle] ?? null + ) + runtime.setOrchestrationDb(db) + dispatcher = new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS }) + }) + + afterEach(() => { + runtime.stopOrchestrationFederationRelay() + db.close() + vi.restoreAllMocks() + }) + + function call( + method: string, + params: Record, + { pane = COORDINATOR_PANE, requestId }: { pane?: string; requestId?: string } = {} + ): Promise { + sequence += 1 + return dispatcher.dispatch({ + id: `call_${sequence}`, + authToken: 'token', + method, + params, + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationCompatibilityEvidence: { paneKey: pane }, + ...(requestId ? { orchestrationRequestId: requestId } : {}) + }) + } + + const started = (dispatchId = 'ctx_b1'): Answer => ({ + result: { dispatchId, runId: 'run_on_b', run: { id: 'run_on_b' }, state: 'ready' } + }) + + async function startOnB(requestId = REQUEST) { + answers['orchestration.workerStart'] = [started()] + const response = await call( + 'orchestration.workerStart', + { from: 'term_coord', on: 'devB', spec: 'build it', worktree: 'new-top-level' }, + { requestId } + ) + if (!response.ok) { + throw new Error(response.error.message) + } + return response.result as { taskId: string; runId: string; dispatchId: string } + } + + it("starts the worker on a server that owns its workers, then records it under this Orca's Task", async () => { + const receipt = await startOnB() + + const start = sent.find((s) => s.method === 'orchestration.workerStart')! + expect(start.params).toMatchObject({ spec: 'build it', adoptTaskId: receipt.taskId }) + expect(start.params).not.toHaveProperty('from') + expect(start.envelope?.orchestrationPeerRun?.label).toBe('Mac A') + expect(receipt.runId).not.toBe('run_on_b') + expect(db.getTask(receipt.taskId)).toMatchObject({ + run_id: receipt.runId, + status: 'dispatched' + }) + expect(getRemoteDispatch(db, 'ctx_b1')).toMatchObject({ task_id: receipt.taskId }) + }) + + it("keeps today's remote attachment for a server without the capability", async () => { + capabilities = [ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY] + await call( + 'orchestration.workerStart', + { from: 'term_coord', on: 'devB', spec: 'build it', worktree: 'new-top-level' }, + { requestId: REQUEST } + ) + + expect(sent.some((s) => s.method === 'orchestration.workerStart')).toBe(false) + expect(db.db.prepare('SELECT COUNT(*) AS n FROM remote_placements').get()).toEqual({ n: 0 }) + }) + + it('frees the Task when the server refuses the start, and keeps it fenced when the answer is lost', async () => { + answers['orchestration.workerStart'] = [ + { error: 'invalid_argument', data: { effectsApplied: false } } + ] + await call( + 'orchestration.workerStart', + { from: 'term_coord', on: 'devB', spec: 'refused', worktree: 'current' }, + { requestId: REQUEST } + ) + const refusedTask = String((sent[0].params as { adoptTaskId: string }).adoptTaskId) + expect(getRemotePlacement(db, refusedTask)).toBeUndefined() + + const lost = await call( + 'orchestration.workerStart', + { from: 'term_coord', on: 'devB', spec: 'lost', worktree: 'new-top-level' }, + { requestId: 'a3f1e9d2-5b6c-4d7e-8f90-1a2b3c4d5e6f' } + ) + expect(lost.ok ? undefined : lost.error.code).toBe('remote_runtime_unavailable') + const lostTask = String((sent[1].params as { adoptTaskId: string }).adoptTaskId) + expect(getRemotePlacement(db, lostTask)).toMatchObject({ + state: 'reserved', + outcome_unknown: 1 + }) + }) + + it('scopes the retry key to the caller: two terminals reusing an ID never collide, one retrying gets its own key', async () => { + const receipt = await startOnB() + answers['orchestration.workerRelease'] = [ + { result: { state: 'released' } }, + { result: { state: 'released' } }, + { result: { state: 'released' } } + ] + const release = (pane: string) => + call('orchestration.workerRelease', { dispatch: 'ctx_b1' }, { pane, requestId: REQUEST }) + await release(COORDINATOR_PANE) + await release(COORDINATOR_PANE) + await release(OTHER_PANE) + + const keys = sent + .filter((s) => s.method === 'orchestration.workerRelease') + .map((s) => s.envelope?.orchestrationRequestId) + expect(keys[0]).toBe(keys[1]) + expect(keys[2]).not.toBe(keys[0]) + expect(keys[0]).not.toBe(REQUEST) + expect(receipt.dispatchId).toBe('ctx_b1') + }) + + it("answers in the caller's terms: its own request ID and its own Run, never the server's", async () => { + const receipt = await startOnB() + answers['orchestration.workerShow'] = [] + const key = () => + sent.filter((s) => s.method === 'orchestration.workerRelease').at(-1)?.envelope + ?.orchestrationRequestId + answers['orchestration.workerRelease'] = [{ result: { state: 'released' } }] + await call('orchestration.workerRelease', { dispatch: 'ctx_b1' }, { requestId: REQUEST }) + answers['orchestration.workerRelease'] = [ + { + error: 'operation_unknown', + data: { requestId: key(), runId: 'run_on_b' } + } + ] + const unknown = await call( + 'orchestration.workerRelease', + { dispatch: 'ctx_b1' }, + { requestId: REQUEST } + ) + + expect(unknown.ok ? undefined : unknown.error.data).toMatchObject({ + requestId: REQUEST, + runId: receipt.runId, + server: 'devB' + }) + }) + + it("answers a remote worker's question only from its Run's coordinator, as a local one is", async () => { + const receipt = await startOnB() + const link = db.db.prepare('SELECT * FROM remote_run_links').get() as { + conversation_id: string + } + importRemoteOwnedItem(db, remoteServerPin(server), { + conversationId: link.conversation_id, + sequence: 1, + messageId: 'msg_q1', + kind: 'worker_mail', + payload: JSON.stringify({ + v: 1, + id: 'msg_q1', + homeRunId: receipt.runId, + peerRunId: 'run_on_b', + message: { + from: 'term_w', + type: 'question', + subject: 'Which color?', + body: 'Which color?', + payload: null, + priority: 'normal', + threadId: null + }, + dispatchId: 'ctx_b1', + taskId: receipt.taskId + }) + }) + answers['orchestration.reply'] = [{ result: { messageId: 'msg_a1' } }] + + const stranger = await call( + 'orchestration.reply', + { id: 'msg_q1', body: 'red', from: 'term_other' }, + { pane: OTHER_PANE } + ) + const coordinator = await call('orchestration.reply', { + id: 'msg_q1', + body: 'teal', + from: 'term_coord' + }) + + expect(stranger.ok).toBe(false) + expect(coordinator.ok).toBe(true) + expect( + sent.filter((s) => s.method === 'orchestration.reply').map((s) => s.params.body) + ).toEqual(['teal']) + }) + + it('owes a stop the server did not answer and says so', async () => { + await startOnB() + const stop = await call('orchestration.workerStop', { dispatch: 'ctx_b1' }) + + expect(stop.ok ? stop.result : undefined).toMatchObject({ state: 'stop_requested' }) + expect(getRemoteDispatch(db, 'ctx_b1')).toMatchObject({ stop_owed: 1 }) + }) +}) diff --git a/src/main/runtime/orchestration/federation/remote-owned/remote-owned-route.ts b/src/main/runtime/orchestration/federation/remote-owned/remote-owned-route.ts index 8d21e13f0ca..1d29c3c0a62 100644 --- a/src/main/runtime/orchestration/federation/remote-owned/remote-owned-route.ts +++ b/src/main/runtime/orchestration/federation/remote-owned/remote-owned-route.ts @@ -12,6 +12,7 @@ import type { OrchestrationDb } from '../../db/orchestration-db' import { runLifecycleWriteTransaction } from '../../db/lifecycle-write-transaction-runner' import { getRemoteDispatch, + getRemoteOwnedQuestion, oweRemoteDispatchStop, remoteStopKey } from '../../db/remote-owned/remote-dispatch-records' @@ -179,13 +180,7 @@ function remoteSubject( ? value.slice('dispatch:'.length) : undefined const record = dispatchId ? getRemoteDispatch(db, dispatchId) : undefined - const question = - kind === 'message' && value - ? // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SELECT of one column returns exactly that column. - (db.db - .prepare('SELECT conversation_id FROM remote_owned_questions WHERE message_id = ?') - .get(value) as { conversation_id: string } | undefined) - : undefined + const question = kind === 'message' && value ? getRemoteOwnedQuestion(db, value) : undefined const conversationId = record?.conversation_id ?? question?.conversation_id const link = conversationId ? getRemoteRunLink(db, conversationId) : undefined if (link) { diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/reply-access.ts b/src/main/runtime/rpc/methods/orchestration/messaging/reply-access.ts index 963250d6044..13b96bc1d5f 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/reply-access.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/reply-access.ts @@ -10,6 +10,7 @@ import type { ReplyParams } from '../../../../../../shared/rpc-contract/orchestr import { resolveRunScope } from '../runs/run-scope' import type { OrchestrationAccessRuleInput } from '../access/orchestration-method-access' import { helperQuestionAnswerer } from './helper-question-answerer' +import { getRemoteOwnedQuestion } from '../../../../orchestration/db/remote-owned/remote-dispatch-records' /** Who may answer: the helper a question was asked of, the Run's current consumer, or anyone for plain mail. */ export type ReplyAccess = @@ -21,6 +22,8 @@ export type ReplyAccess = } | { kind: 'run'; original: MessageRow; question: QuestionRow; run: RunRow } | { kind: 'message'; original: MessageRow } + /** A question a worker on a paired server asked; the reply goes to that server. */ + | { kind: 'remote'; original: MessageRow; run: RunRow } export function replyAccess({ params, @@ -56,6 +59,22 @@ export function replyAccess({ ) } const question = db.getQuestion(params.id) + const remote = question ? undefined : getRemoteOwnedQuestion(db, params.id) + if (remote) { + // Answered by its Run's coordinator, as a question from a worker here is. + const run = resolveRunScope(runtime, { + runId: params.run ?? remote.run_id, + callerTerminalHandle: caller, + requireCurrentConsumer: true, + legacyCoordinatorRunId, + callerEvidence: ctx.orchestrationCompatibilityEvidence, + callerSession: ctx.orchestrationCaller + }) + if (run.id !== remote.run_id) { + throw new Error(`Message not found: ${params.id}`) + } + return { kind: 'remote', original, run } + } if (!question) { return { kind: 'message', original } }