From b5282e846bfbc61fa9411edd0330f9d003d6b2dc Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Tue, 10 Nov 2026 04:26:27 -0500 Subject: [PATCH] feat(orchestration): offline show and server-named failures for forwarded commands --- .../remote-owned/remote-owned-route.ts | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/src/main/runtime/orchestration/federation/remote-owned/remote-owned-route.ts b/src/main/runtime/orchestration/federation/remote-owned/remote-owned-route.ts index f1b1c7a06dc..01f33d5e445 100644 --- a/src/main/runtime/orchestration/federation/remote-owned/remote-owned-route.ts +++ b/src/main/runtime/orchestration/federation/remote-owned/remote-owned-route.ts @@ -24,6 +24,8 @@ import { callRemoteOwnedServer } from './remote-owned-calls' import { remoteForwardKey, type RemoteForwardKey } from './remote-forward-caller' import { inServerTerms, serverNeverAnswered } from './remote-owned-answers' import { sendOwedRemoteStop } from './remote-owed-actions' +import { offlineRemoteWorkerShow, remoteServerName } from './remote-owned-reads' +import { explainRemoteServerFailure, isRemoteServerUnreachable } from './remote-server-unreachable' import { serverOwnsPlacedWorkers, startRemoteOwnedWorker } from './remote-owned-start' type Routed = { handled: false } | { handled: true; result: unknown } @@ -79,7 +81,7 @@ export async function routeRemoteOwnedCall( } return { handled: true, - result: await forwardToLink(runtime, subject.link, request.method, access, values, forward) + result: await forwardToLink(runtime, subject, request.method, access, values, forward) } } @@ -127,7 +129,7 @@ async function routeServerCall( return { handled: true, result: link - ? await forwardToLink(runtime, link, request.method, access, values) + ? await forwardToLink(runtime, { link }, request.method, access, values) : await runtime.callOrchestrationWorkerServer( server.environmentId, request.method, @@ -208,25 +210,36 @@ function paramsForServer(access: OrchestrationMethodAccessDeclaration, values: V async function forwardToLink( runtime: OrcaRuntimeService, - link: RemoteRunLinkRow, + { link, dispatchId }: RemoteSubject, method: string, access: OrchestrationMethodAccessDeclaration, values: Values, forward?: RemoteForwardKey ): Promise { - const server = runtime.resolveOrchestrationWorkerServer(link.environment_id).name + const db = runtime.getOrchestrationDb() + const server = remoteServerName(runtime, link.environment_id) + const params = paramsForServer(access, values) try { const answer = await callRemoteOwnedServer(runtime, link, { method, - params: paramsForServer(access, values), + params, timeoutMs: FORWARD_TIMEOUT_MS, ...(forward ? { requestId: forward.key } : {}), whileLink: ['open'] }) - seedNamedDispatches(runtime.getOrchestrationDb(), link, answer) + seedNamedDispatches(db, link, answer) return inServerTerms(answer, forward, server) } catch (error) { - throw inServerTerms(error, forward, server) + const record = dispatchId ? getRemoteDispatch(db, dispatchId) : undefined + if (record && method === 'orchestration.workerShow' && isRemoteServerUnreachable(error)) { + // The server is off: what this Orca last heard about the worker, never its output. + return offlineRemoteWorkerShow(runtime, record) + } + throw inServerTerms( + explainRemoteServerFailure(error, { server, method, params, dispatchId }), + forward, + server + ) } } @@ -238,7 +251,7 @@ async function stopRemoteWorker( ): Promise { const db = runtime.getOrchestrationDb() const dispatchId = subject.dispatchId! - const server = runtime.resolveOrchestrationWorkerServer(subject.link.environment_id).name + const server = remoteServerName(runtime, subject.link.environment_id) const owed = oweRemoteDispatchStop(db, dispatchId, forward?.key ?? `stop:${randomUUID()}`) runtime.ensureOrchestrationFederationRelay(subject.link.run_id) try {