diff --git a/src/main/runtime/orchestration/federation/remote-owned/remote-owned-answers.ts b/src/main/runtime/orchestration/federation/remote-owned/remote-owned-answers.ts index 7482ffda851..7a3b9f2f98d 100644 --- a/src/main/runtime/orchestration/federation/remote-owned/remote-owned-answers.ts +++ b/src/main/runtime/orchestration/federation/remote-owned/remote-owned-answers.ts @@ -15,19 +15,27 @@ export function errorData(error: unknown): Record | undefined { : undefined } +/** What the server's answer says that this Orca says otherwise. */ +export type ServerTerms = { + server: string + forward?: RemoteForwardKey + /** The link the call went under: the server's Run for it is this Orca's Run here. */ + link?: { peer_run_id: string | null; run_id: string } +} + /** * The server's answer, or its error, in the caller's own terms: the caller knows its request ID, - * not the scoped key the server saw, and an error names the server it came from so recovery - * (`request-show --on`) asks there. + * not the scoped key the server saw, and its own Run, not the server's; an error names the server + * it came from so recovery (`request-show --on`) asks there. */ -export function inServerTerms( - value: T, - forward: RemoteForwardKey | undefined, - server: string -): T { +export function inServerTerms(value: T, { server, forward, link }: ServerTerms): T { + const pairs: [string, string][] = [ + ...(forward ? [[forward.key, forward.requestId] as [string, string]] : []), + ...(link?.peer_run_id ? [[link.peer_run_id, link.run_id] as [string, string]] : []) + ] const swap = (item: unknown): unknown => { if (typeof item === 'string') { - return forward ? item.split(forward.key).join(forward.requestId) : item + return pairs.reduce((text, [from, to]) => text.split(from).join(to), item) } if (Array.isArray(item)) { return item.map(swap) diff --git a/src/main/runtime/orchestration/federation/remote-owned/remote-owned-route.ts b/src/main/runtime/orchestration/federation/remote-owned/remote-owned-route.ts index 60a52338698..8d21e13f0ca 100644 --- a/src/main/runtime/orchestration/federation/remote-owned/remote-owned-route.ts +++ b/src/main/runtime/orchestration/federation/remote-owned/remote-owned-route.ts @@ -158,7 +158,7 @@ async function routeServerCall( undefined, { expectedEnvironmentPairingRevision: server.pairingRevision } ) - return { handled: true, result: inServerTerms(answer, forward, server.name) } + return { handled: true, result: inServerTerms(answer, { server: server.name, forward }) } } } @@ -230,18 +230,18 @@ async function forwardToLink( whileLink: ['open'] }) seedNamedDispatches(db, link, answer) - return inServerTerms(answer, forward, server) + return inServerTerms(answer, { server, forward, link }) } catch (error) { const record = dispatchId ? getRemoteDispatch(db, dispatchId) : undefined if (record && method === 'orchestration.workerShow' && isRemoteServerUnreachable(error)) { // The server is off: what this Orca last heard about the worker, never its output. return offlineRemoteWorkerShow(runtime, record) } - throw inServerTerms( - explainRemoteServerFailure(error, { server, method, params, dispatchId }), + throw inServerTerms(explainRemoteServerFailure(error, { server, method, params, dispatchId }), { + server, forward, - server - ) + link + }) } } @@ -265,10 +265,14 @@ async function stopRemoteWorker( // The server knows this attempt by its stop key; the caller knows only its own request ID. const stopKey = forward && { key: remoteStopKey(owed.record), requestId: forward.requestId } try { - return inServerTerms(await sendOwedRemoteStop(runtime, owed.record), stopKey, server) + return inServerTerms(await sendOwedRemoteStop(runtime, owed.record), { + server, + forward: stopKey, + link: subject.link + }) } catch (error) { if (!serverNeverAnswered(error) || !getOpenRemoteRunLink(db, subject.link.conversation_id)) { - throw inServerTerms(error, stopKey, server) + throw inServerTerms(error, { server, forward: stopKey, link: subject.link }) } return { dispatchId, diff --git a/src/main/runtime/orchestration/federation/remote-owned/remote-owned-start.ts b/src/main/runtime/orchestration/federation/remote-owned/remote-owned-start.ts index d41b11d2de2..e62b48baf5d 100644 --- a/src/main/runtime/orchestration/federation/remote-owned/remote-owned-start.ts +++ b/src/main/runtime/orchestration/federation/remote-owned/remote-owned-start.ts @@ -143,7 +143,7 @@ export async function startRemoteOwnedWorker( error, key: forward.key }) - throw inServerTerms(error, forward, server.name) + throw inServerTerms(error, { server: server.name, forward }) } const started = StartedOnServer.safeParse(receipt) if (started.success) { @@ -165,7 +165,7 @@ export async function startRemoteOwnedWorker( agent: null }) } - const answer = inServerTerms(receipt, forward, server.name) + const answer = inServerTerms(receipt, { server: server.name, forward }) return withRun( answer && typeof answer === 'object' ? {