From e7b3908a52cdf142bed477231a5ba1fb79070395 Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Tue, 10 Nov 2026 04:25:21 -0500 Subject: [PATCH] feat(orchestration): run commands about a paired server's workers there, and switch new pairs to it --- .../mutation-request-show-handler.ts | 8 +- .../orchestration/worker-agent-views.ts | 4 + .../orchestration/worker-terminal-handlers.ts | 1 + src/cli/orchestration-mutation-recovery.ts | 11 +- src/cli/specs/orchestration-worker-specs.ts | 15 +- src/cli/specs/orchestration.ts | 7 +- .../remote-owned/remote-forward-caller.ts | 46 +++ .../remote-owned/remote-owed-actions.ts | 3 +- .../remote-owned/remote-owned-answers.ts | 50 +++ .../remote-owned/remote-owned-route.ts | 300 ++++++++++++++++++ .../remote-owned/remote-owned-start.ts | 199 ++++++++++++ src/main/runtime/rpc/dispatcher.ts | 16 + .../access/orchestration-method-access.ts | 10 + .../federation-lifecycle-settlement.test.ts | 3 +- ...deration-peer-capabilities.test-support.ts | 10 + .../federation/federation.test.ts | 5 +- .../messaging/message-methods.ts | 21 +- .../runs/mutation-request-show.ts | 2 +- .../orchestration/worker/worker-control.ts | 3 +- .../worker/worker-list-method.ts | 4 +- .../methods/orchestration/worker/workers.ts | 93 +++--- .../orchestration-runtime-capabilities.ts | 6 +- ...ation-runs-mutation-request-show-params.ts | 6 +- ...estration-worker-release-schemas-params.ts | 4 +- 24 files changed, 762 insertions(+), 65 deletions(-) create mode 100644 src/main/runtime/orchestration/federation/remote-owned/remote-forward-caller.ts create mode 100644 src/main/runtime/orchestration/federation/remote-owned/remote-owned-answers.ts create mode 100644 src/main/runtime/orchestration/federation/remote-owned/remote-owned-route.ts create mode 100644 src/main/runtime/orchestration/federation/remote-owned/remote-owned-start.ts diff --git a/src/cli/handlers/orchestration/mutation-request-show-handler.ts b/src/cli/handlers/orchestration/mutation-request-show-handler.ts index 7418565620e..5438fc70500 100644 --- a/src/cli/handlers/orchestration/mutation-request-show-handler.ts +++ b/src/cli/handlers/orchestration/mutation-request-show-handler.ts @@ -1,14 +1,18 @@ import type { CommandHandler } from '../../dispatch' import { printOrchestrationReceipt } from './agent-view' -import { getRequiredStringFlag } from '../../flags' +import { getOptionalStringFlag, getRequiredStringFlag } from '../../flags' import { RuntimeClientError } from '../../runtime-client' import type { OrchestrationMutationRequestShowResult } from '../../../shared/orchestration/orchestration-mutation-request' export const ORCHESTRATION_REQUEST_SHOW_HANDLER: Record = { 'orchestration request-show': async ({ flags, client, json }) => { const request = getRequiredStringFlag(flags, 'request') + const on = getOptionalStringFlag(flags, 'on') const result = await client - .call('orchestration.requestShow', { request }) + .call('orchestration.requestShow', { + request, + ...(on ? { on } : {}) + }) .catch((error: unknown) => { if (error instanceof RuntimeClientError && error.code === 'method_not_found') { throw new RuntimeClientError( diff --git a/src/cli/handlers/orchestration/worker-agent-views.ts b/src/cli/handlers/orchestration/worker-agent-views.ts index d33a81af3a6..07a883c2ae6 100644 --- a/src/cli/handlers/orchestration/worker-agent-views.ts +++ b/src/cli/handlers/orchestration/worker-agent-views.ts @@ -44,6 +44,8 @@ export type WorkerShowResult = { server?: RemoteWorkerServer & { reachable: false } lastReport?: string | null autoReleaseAt?: string | null + /** The paired Orca whose Run here coordinates this worker. */ + coordinatedFrom?: string | null } /** A worker a paired server runs, as this Orca names that server and last heard from it. */ @@ -70,6 +72,7 @@ type WorkerShowFields = { waitingOnHuman?: string lastError?: string autoReleaseAt?: string + coordinatedFrom?: string next?: string } @@ -108,6 +111,7 @@ export const WORKER_SHOW_VIEW: OrchestrationAgentView] [--terminal-state ] [--include-remote] [--cursor ] [--limit <1-100>] [--json]', - allowedFlags: [...GLOBAL_FLAGS, 'run', 'terminal-state', 'include-remote', 'cursor', 'limit'], + 'orca orchestration worker-list [--run ] [--terminal-state ] [--include-remote] [--on ] [--cursor ] [--limit <1-100>] [--json]', + allowedFlags: [ + ...GLOBAL_FLAGS, + 'run', + 'terminal-state', + 'include-remote', + 'on', + 'cursor', + 'limit' + ], notes: [ 'Terminal state is process accounting and is reported separately from Task status; a completed Task can still own a live terminal.', 'A reclaimable worker with resource.autoReleaseAt is released automatically at that time unless it is reused with worker-start --terminal or kept open (worker-keep-open, worker-start --keep-open, or the user).', 'Context-only Dispatches created by orchestration dispatch are included as unsupervised with terminal state retained.', 'Returns at most 100 local rows, newest first, by default; --include-remote adds connected-server observations when the host supports fleet listing. Continue with the opaque nextCursor value unchanged.', - 'Without --run the list is scoped to the Run bound to the calling terminal, and to every Run when there is no binding; the receipt reports which in scope.source (flag, bound, or all).' + 'Without --run the list is scoped to the Run bound to the calling terminal, and to every Run when there is no binding; the receipt reports which in scope.source (flag, bound, or all).', + "--on lists that paired server's own workers for your Run there (or all of them without a Run), paged with its own cursor." ] } ] diff --git a/src/cli/specs/orchestration.ts b/src/cli/specs/orchestration.ts index 44f8f118418..bbb9e89c261 100644 --- a/src/cli/specs/orchestration.ts +++ b/src/cli/specs/orchestration.ts @@ -209,12 +209,13 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [ { path: ['orchestration', 'request-show'], summary: 'Ask whether one orchestration mutation request already took effect', - usage: 'orca orchestration request-show --request [--json]', - allowedFlags: [...GLOBAL_FLAGS, 'request'], + usage: 'orca orchestration request-show --request [--on ] [--json]', + allowedFlags: [...GLOBAL_FLAGS, 'request', 'on'], notes: [ 'Read-only: it never starts, retries, or settles anything, so it is safe to run after any lost response.', 'completed means the mutation landed and --retry-request replays the recorded outcome instead of starting a second one. pending means the original mutation is still running or Orca restarted before recording its outcome; wait for a live original command, otherwise replay with --retry-request.', - 'absent means this runtime holds no receipt for that request under your caller identity: it never arrived, it failed before recording anything, or the receipt was pruned. Absent is not proof that nothing happened.' + 'absent means this runtime holds no receipt for that request under your caller identity: it never arrived, it failed before recording anything, or the receipt was pruned. Absent is not proof that nothing happened.', + 'A command about a worker on a paired server keeps its receipt on that server; --on asks there.' ] }, { diff --git a/src/main/runtime/orchestration/federation/remote-owned/remote-forward-caller.ts b/src/main/runtime/orchestration/federation/remote-owned/remote-forward-caller.ts new file mode 100644 index 00000000000..f109be72c77 --- /dev/null +++ b/src/main/runtime/orchestration/federation/remote-owned/remote-forward-caller.ts @@ -0,0 +1,46 @@ +import type { OrcaRuntimeService } from '../../../orca-runtime' +import type { RpcContext } from '../../../rpc/core' +import { remoteForwardRequestId } from './remote-owned-calls' + +/** + * Who is asking, as a retry of the same request must find it again: the caller's session, else + * its terminal's pane, which survives a restart, else the person at this Orca outside any terminal. + * Never one shared identity for every terminal, so two callers reusing an ID never collide. + */ +export function remoteForwardCallerNamespace( + runtime: Pick, + ctx: Pick, + callerHandle: string | undefined +): string { + const session = ctx.orchestrationCaller?.orcaSessionId + if (session) { + return `session:${session}` + } + const handle = callerHandle ?? ctx.orchestrationCompatibilityEvidence?.terminalHandle + const pane = + ctx.orchestrationCompatibilityEvidence?.paneKey ?? + (handle ? runtime.getTerminalPaneKey(handle) : null) + return pane + ? `pane:${pane}` + : `local:${runtime.getOrchestrationDb().getOrCreateLocalMutationCallerFingerprint()}` +} + +/** The key a forwarded call goes out under, and how to say it back in the caller's own terms. */ +export type RemoteForwardKey = { key: string; requestId: string } + +export function remoteForwardKey( + runtime: Pick, + ctx: Pick, + callerHandle: string | undefined, + requestId: string | undefined +): RemoteForwardKey | undefined { + return requestId + ? { + key: remoteForwardRequestId( + remoteForwardCallerNamespace(runtime, ctx, callerHandle), + requestId + ), + requestId + } + : undefined +} diff --git a/src/main/runtime/orchestration/federation/remote-owned/remote-owed-actions.ts b/src/main/runtime/orchestration/federation/remote-owned/remote-owed-actions.ts index ecefa7d4d0c..d2912ac9304 100644 --- a/src/main/runtime/orchestration/federation/remote-owned/remote-owed-actions.ts +++ b/src/main/runtime/orchestration/federation/remote-owned/remote-owed-actions.ts @@ -45,7 +45,8 @@ const NO_ANSWER_CODES = new Set([ 'remote_link_closed' ]) -function serverAnswered(error: unknown): boolean { +/** Whether the server got the call and answered it, rather than this Orca never reaching it. */ +export function serverAnswered(error: unknown): boolean { return error instanceof OrchestrationError && !NO_ANSWER_CODES.has(error.code) } diff --git a/src/main/runtime/orchestration/federation/remote-owned/remote-owned-answers.ts b/src/main/runtime/orchestration/federation/remote-owned/remote-owned-answers.ts new file mode 100644 index 00000000000..65f2b8ea381 --- /dev/null +++ b/src/main/runtime/orchestration/federation/remote-owned/remote-owned-answers.ts @@ -0,0 +1,50 @@ +import { OrchestrationError } from '../../orchestration-error' +import { serverAnswered } from './remote-owed-actions' +import type { RemoteForwardKey } from './remote-forward-caller' + +/** The server got nothing, or never answered, so whatever the call did there is unknown here. */ +export function serverNeverAnswered(error: unknown): boolean { + return !serverAnswered(error) +} + +export function errorData(error: unknown): Record | undefined { + const data = error instanceof OrchestrationError ? error.data : undefined + return data && typeof data === 'object' && !Array.isArray(data) + ? // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a non-array object's own fields, read as unknowns. + (data as Record) + : undefined +} + +/** + * The server's answer, or its error, in the caller's own terms: the caller knows its request ID, + * not the scoped key the server saw, and an error names the server it came from so recovery + * (`request-show --on`) asks there. + */ +export function inServerTerms( + value: T, + forward: RemoteForwardKey | undefined, + server: string +): T { + const swap = (item: unknown): unknown => { + if (typeof item === 'string') { + return forward ? item.split(forward.key).join(forward.requestId) : item + } + if (Array.isArray(item)) { + return item.map(swap) + } + if (item && typeof item === 'object') { + return Object.fromEntries(Object.entries(item).map(([field, inner]) => [field, swap(inner)])) + } + return item + } + if (value instanceof OrchestrationError) { + const data = swap(errorData(value) ?? {}) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: an error stays an OrchestrationError with the same code. + return new OrchestrationError(value.code, String(swap(value.message)), { + ...(data as Record), + server + }) as T + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: only string contents change; every shape is kept. + return swap(value) as T +} diff --git a/src/main/runtime/orchestration/federation/remote-owned/remote-owned-route.ts b/src/main/runtime/orchestration/federation/remote-owned/remote-owned-route.ts new file mode 100644 index 00000000000..f1b1c7a06dc --- /dev/null +++ b/src/main/runtime/orchestration/federation/remote-owned/remote-owned-route.ts @@ -0,0 +1,300 @@ +import { randomUUID } from 'node:crypto' +import type { OrcaRuntimeService } from '../../../orca-runtime' +import type { RpcContext, RpcMethod, RpcRequest } from '../../../rpc/core' +import { + checkDeclaredOrchestrationAccess, + type OrchestrationMethodAccessDeclaration +} from '../../../rpc/methods/orchestration/access/orchestration-method-access' +import { emptyWorkerListPage } from '../../../rpc/methods/orchestration/worker/worker-list-method' +import { ORCHESTRATION_FLEET_PAGE_MAX } from '../../../../../shared/orchestration/fleet/orchestration-fleet-projection' +import { WorkerStartParams } from '../../../../../shared/rpc-contract/orchestration-worker-start-params' +import type { OrchestrationDb } from '../../db/orchestration-db' +import { runLifecycleWriteTransaction } from '../../db/lifecycle-write-transaction-runner' +import { + getRemoteDispatch, + oweRemoteDispatchStop +} from '../../db/remote-owned/remote-dispatch-records' +import { adoptRemoteAttempt } from '../../db/remote-owned/remote-placements' +import { + getOpenRemoteRunLink, + getRemoteRunLink, + type RemoteRunLinkRow +} from '../../db/remote-owned/remote-run-links' +import { callRemoteOwnedServer } from './remote-owned-calls' +import { remoteForwardKey, type RemoteForwardKey } from './remote-forward-caller' +import { inServerTerms, serverNeverAnswered } from './remote-owned-answers' +import { sendOwedRemoteStop } from './remote-owed-actions' +import { serverOwnsPlacedWorkers, startRemoteOwnedWorker } from './remote-owned-start' + +type Routed = { handled: false } | { handled: true; result: unknown } + +const LOCAL: Routed = { handled: false } +// Long enough for a send that starts a follow-up worker there and waits for it to be ready. +const FORWARD_TIMEOUT_MS = 180_000 +// What `--on ` sends there; send and reply --on stay peer mail. +const SERVER_CALLS: ReadonlySet = new Set([ + 'orchestration.workerStart', + 'orchestration.workerList', + 'orchestration.requestShow' +]) + +type Values = Record + +/** + * Before this Orca's receipts: a command about a worker a paired server owns runs there, by that + * server's ordinary handler, as its Run's coordinator. The worker is found from the subjects the + * method declares; the caller passes the method's one access check here first, as it would for a + * worker of its own. A paired Orca's calls always run here: pairing is one-way. + */ +export async function routeRemoteOwnedCall( + runtime: OrcaRuntimeService, + method: RpcMethod, + request: RpcRequest, + params: unknown, + ctx: RpcContext +): Promise { + const access = method.orchestrationAccess + if (!access || access.kind === 'localOnly' || ctx.orchestrationCaller?.kind === 'peer') { + return LOCAL + } + const values: Values = params && typeof params === 'object' ? { ...params } : {} + const db = runtime.getOrchestrationDb() + const on = access.server && stringField(values, access.server) + if (on !== undefined) { + return routeServerCall(runtime, request, access, values, ctx, on) + } + const subject = remoteSubject(db, access, values) + if (!subject) { + return LOCAL + } + checkDeclaredOrchestrationAccess(access, params, ctx) + const forward = remoteForwardKey( + runtime, + ctx, + access.caller && stringField(values, access.caller), + request.orchestrationRequestId + ) + if (request.method === 'orchestration.workerStop' && subject.dispatchId) { + return { handled: true, result: await stopRemoteWorker(runtime, subject, forward) } + } + return { + handled: true, + result: await forwardToLink(runtime, subject.link, request.method, access, values, forward) + } +} + +/** `--on `: a start, a list, or a receipt lookup there. */ +async function routeServerCall( + runtime: OrcaRuntimeService, + request: RpcRequest, + access: OrchestrationMethodAccessDeclaration, + values: Values, + ctx: RpcContext, + on: string +): Promise { + if (!SERVER_CALLS.has(request.method)) { + return LOCAL + } + checkDeclaredOrchestrationAccess(access, values, ctx) + const server = runtime.resolveOrchestrationWorkerServer(on) + const db = runtime.getOrchestrationDb() + if (request.method === 'orchestration.workerStart') { + if (!(await serverOwnsPlacedWorkers(runtime, server))) { + // An older server: today's remote attachment, the one legacy branch for new work. + return LOCAL + } + return { + handled: true, + result: await startRemoteOwnedWorker( + runtime, + WorkerStartParams.parse(values), + ctx, + request.orchestrationRequestId + ) + } + } + if (request.method === 'orchestration.workerList') { + const runId = stringField(values, 'run') + // A Run lists what it placed there, as its coordinator; no Run lists the server as a whole. + const link = runId ? openLinkFor(db, runId, server.environmentId) : undefined + if (runId && !link) { + return { + handled: true, + result: emptyWorkerListPage(numberField(values, 'limit') ?? ORCHESTRATION_FLEET_PAGE_MAX) + } + } + const forwarded = paramsForServer(access, values) + return { + handled: true, + result: link + ? await forwardToLink(runtime, link, request.method, access, values) + : await runtime.callOrchestrationWorkerServer( + server.environmentId, + request.method, + forwarded, + FORWARD_TIMEOUT_MS, + undefined, + { expectedEnvironmentPairingRevision: server.pairingRevision } + ) + } + } + { + // The server keyed the request by this caller's scoped key, as the forward that sent it did. + const forward = remoteForwardKey( + runtime, + ctx, + undefined, + stringField(values, 'request') ?? randomUUID() + )! + const answer = await runtime.callOrchestrationWorkerServer( + server.environmentId, + request.method, + { request: forward.key }, + FORWARD_TIMEOUT_MS, + undefined, + { expectedEnvironmentPairingRevision: server.pairingRevision } + ) + return { handled: true, result: inServerTerms(answer, forward, server.name) } + } +} + +type RemoteSubject = { link: RemoteRunLinkRow; dispatchId?: string } + +/** The worker or question a call names, when a paired server owns it. Tasks stay in this Orca's graph. */ +function remoteSubject( + db: OrchestrationDb, + access: OrchestrationMethodAccessDeclaration, + values: Values +): RemoteSubject | undefined { + for (const [field, kind] of Object.entries(access.targets)) { + const value = stringField(values, field) + const dispatchId = + kind === 'dispatch' + ? value + : kind === 'party' && value?.startsWith('dispatch:') + ? value.slice('dispatch:'.length) + : undefined + const record = dispatchId ? getRemoteDispatch(db, dispatchId) : undefined + const question = + kind === 'message' && value + ? // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SELECT of one column returns exactly that column. + (db.db + .prepare('SELECT conversation_id FROM remote_owned_questions WHERE message_id = ?') + .get(value) as { conversation_id: string } | undefined) + : undefined + const conversationId = record?.conversation_id ?? question?.conversation_id + const link = conversationId ? getRemoteRunLink(db, conversationId) : undefined + if (link) { + return { link, dispatchId: record?.dispatch_id } + } + } + return undefined +} + +/** This Orca's caller, Run and server fields mean nothing there: the server names the caller itself. */ +function paramsForServer(access: OrchestrationMethodAccessDeclaration, values: Values): Values { + const local = new Set( + [ + access.caller, + access.callerPane, + access.server, + ...Object.entries(access.targets) + .filter(([, kind]) => kind === 'run') + .map(([field]) => field) + ].filter((field): field is string => field !== undefined) + ) + return Object.fromEntries(Object.entries(values).filter(([field]) => !local.has(field))) +} + +async function forwardToLink( + runtime: OrcaRuntimeService, + link: RemoteRunLinkRow, + method: string, + access: OrchestrationMethodAccessDeclaration, + values: Values, + forward?: RemoteForwardKey +): Promise { + const server = runtime.resolveOrchestrationWorkerServer(link.environment_id).name + try { + const answer = await callRemoteOwnedServer(runtime, link, { + method, + params: paramsForServer(access, values), + timeoutMs: FORWARD_TIMEOUT_MS, + ...(forward ? { requestId: forward.key } : {}), + whileLink: ['open'] + }) + seedNamedDispatches(runtime.getOrchestrationDb(), link, answer) + return inServerTerms(answer, forward, server) + } catch (error) { + throw inServerTerms(error, forward, server) + } +} + +/** A stop is owed until the server proves it; the ordinary stop answers when it can, and the pull loop resends otherwise. */ +async function stopRemoteWorker( + runtime: OrcaRuntimeService, + subject: RemoteSubject, + forward: RemoteForwardKey | undefined +): Promise { + const db = runtime.getOrchestrationDb() + const dispatchId = subject.dispatchId! + const server = runtime.resolveOrchestrationWorkerServer(subject.link.environment_id).name + const owed = oweRemoteDispatchStop(db, dispatchId, forward?.key ?? `stop:${randomUUID()}`) + runtime.ensureOrchestrationFederationRelay(subject.link.run_id) + try { + return inServerTerms( + await sendOwedRemoteStop(runtime, owed?.record ?? getRemoteDispatch(db, dispatchId)!), + forward, + server + ) + } catch (error) { + if (!serverNeverAnswered(error) || !getOpenRemoteRunLink(db, subject.link.conversation_id)) { + throw inServerTerms(error, forward, server) + } + return { + dispatchId, + state: 'stop_requested', + server, + next: `${server} did not answer, so the stop is owed: this Orca sends it again until ${server} confirms the worker stopped.` + } + } +} + +/** A Dispatch the server's answer names (a follow-up it started) is its; record it so commands about it route there. */ +function seedNamedDispatches(db: OrchestrationDb, link: RemoteRunLinkRow, answer: unknown): void { + const named = answer && typeof answer === 'object' ? (answer as Values) : {} + const dispatchId = stringField(named, 'dispatchId') + const taskId = stringField(named, 'taskId') + if (!dispatchId || !taskId || getRemoteDispatch(db, dispatchId)) { + return + } + runLifecycleWriteTransaction(db.db, 'seed_remote_dispatch', () => { + const open = getOpenRemoteRunLink(db, link.conversation_id) + if (open) { + adoptRemoteAttempt(db, open, { taskId, dispatchId }) + } + }) +} + +function openLinkFor( + db: OrchestrationDb, + runId: string, + environmentId: string +): RemoteRunLinkRow | undefined { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SELECT * over this table returns exactly its row shape. + return db.db + .prepare( + `SELECT * FROM remote_run_links WHERE run_id = ? AND environment_id = ? AND state = 'open'` + ) + .get(runId, environmentId) as RemoteRunLinkRow | undefined +} + +function stringField(values: Values, field: string): string | undefined { + const value = values[field] + return typeof value === 'string' && value !== '' ? value : undefined +} + +function numberField(values: Values, field: string): number | undefined { + const value = values[field] + return typeof value === 'number' ? value : undefined +} diff --git a/src/main/runtime/orchestration/federation/remote-owned/remote-owned-start.ts b/src/main/runtime/orchestration/federation/remote-owned/remote-owned-start.ts new file mode 100644 index 00000000000..1368c21a3a3 --- /dev/null +++ b/src/main/runtime/orchestration/federation/remote-owned/remote-owned-start.ts @@ -0,0 +1,199 @@ +import { createHash, randomUUID } from 'node:crypto' +import { z } from 'zod' +import type { WorkerStartParams } from '../../../../../shared/rpc-contract/orchestration-worker-start-params' +import { ORCHESTRATION_REMOTE_OWNED_WORKERS_RUNTIME_CAPABILITY } from '../../../../../shared/orchestration/orchestration-runtime-capabilities' +import { ORCHESTRATION_CONTRACT_PREFLIGHT_TIMEOUT_MS } from '../../../../../shared/orchestration/orchestration-timing-budgets' +import type { RuntimeStatus } from '../../../../../shared/runtime-session-contracts' +import type { OrcaRuntimeService } from '../../../orca-runtime' +import type { RpcContext } from '../../../rpc/core' +import { prepareWorkerStart } from '../../../rpc/methods/orchestration/worker/workers' +import { resolveFederatedWorkerStartBudgets } from '../../../rpc/methods/orchestration/worker/worker-start-budgets' +import { OrchestrationError } from '../../orchestration-error' +import { prepareWorkerStartCard } from '../../workers/launch/worker-start-card' +import { getOrchestrationPeerCapabilityCache } from '../orchestration-peer-capability-cache' +import type { OrchestrationWorkerServer } from '../environment-transport' +import { + markRemoteStartOutcomeUnknown, + recordRemoteStart, + releaseRemotePlacement, + reserveRemotePlacement +} from '../../db/remote-owned/remote-placements' +import { openRemoteRunLink } from '../../db/remote-owned/remote-run-links' +import { callRemoteOwnedServer } from './remote-owned-calls' +import { remoteForwardKey } from './remote-forward-caller' +import { errorData, inServerTerms, serverNeverAnswered } from './remote-owned-answers' + +type StartParams = z.infer +type StartContext = Pick< + RpcContext, + 'runtime' | 'orchestrationCompatibilityEvidence' | 'orchestrationCaller' +> + +/** Whether the server runs the workers placed on it as its own; an older one keeps today's remote attachments. */ +export async function serverOwnsPlacedWorkers( + runtime: OrcaRuntimeService, + server: OrchestrationWorkerServer +): Promise { + const decision = await getOrchestrationPeerCapabilityCache(runtime).resolve({ + peerFingerprint: server.peerFingerprint, + expectedRuntimeEpoch: null, + capability: ORCHESTRATION_REMOTE_OWNED_WORKERS_RUNTIME_CAPABILITY, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: status.get answers RuntimeStatus; the cache reads only its capabilities and runtime ID. + probe: async () => + (await runtime.callOrchestrationWorkerServer( + server.environmentId, + 'status.get', + undefined, + ORCHESTRATION_CONTRACT_PREFLIGHT_TIMEOUT_MS, + undefined, + { expectedEnvironmentPairingRevision: server.pairingRevision } + )) as RuntimeStatus + }) + return decision.supported +} + +/** One ID for the Task here and the start there, from the start's retry key, so a retry finds both. */ +function adoptedTaskId(key: string): string { + return `task_${createHash('sha256').update(`task\n${key}`).digest('hex').slice(0, 12)}` +} + +const StartedOnServer = z.object({ + dispatchId: z.string().min(1), + run: z.object({ id: z.string().min(1) }).optional(), + agent: z.string().nullable().optional() +}) + +/** + * worker-start --on a server that owns its workers: the Task stays in this Orca's graph, and the + * server starts an ordinary worker for it as the Run's coordinator. Synchronous, launch then + * record: the Task is reserved for the server first, and becomes its attempt once it answers. + */ +export async function startRemoteOwnedWorker( + runtime: OrcaRuntimeService, + params: StartParams, + ctx: StartContext, + callerRequestId: string | undefined +): Promise { + if (params.terminal) { + throw new OrchestrationError( + 'invalid_argument', + '--terminal names a terminal on this Orca, so it cannot be used with --on. No effects were applied.', + { effectsApplied: false } + ) + } + const { coordinator, bindRun, taskDeps, existingTask, withRun } = await prepareWorkerStart( + params, + ctx + ) + const db = runtime.getOrchestrationDb() + const server = runtime.resolveOrchestrationWorkerServer(params.on!) + const { run } = bindRun() + // An older CLI sends no request ID; the start still runs, but a retry cannot find it. + const forward = remoteForwardKey(runtime, ctx, params.from, callerRequestId ?? randomUUID())! + const taskId = existingTask?.id ?? adoptedTaskId(forward.key) + const budgets = resolveFederatedWorkerStartBudgets(params.timeoutMs) + const link = openRemoteRunLink(db, run.id, server) + const { task } = reserveRemotePlacement(db, { + link, + taskId, + requestKey: forward.key, + runtimeId: runtime.getRuntimeId(), + retryOf: params.retryOf, + create: existingTask + ? undefined + : { + spec: params.spec!, + taskTitle: params.taskTitle, + deps: taskDeps, + parentId: params.parent, + createdByTerminalHandle: coordinator?.terminalHandle ?? undefined, + createdByPaneKey: coordinator?.paneKey ?? undefined, + createdByRunGeneration: run.consumer_generation + } + }) + // Its queue has a reader from the start, so mail sent before the answer still comes home. + runtime.ensureOrchestrationFederationRelay(run.id) + const { + from: _from, + run: _run, + on: _on, + task: _task, + deps: _deps, + parent: _parent, + ...options + } = params + let receipt: unknown + try { + receipt = await callRemoteOwnedServer(runtime, link, { + method: 'orchestration.workerStart', + params: { + ...options, + spec: task.spec, + ...(task.task_title ? { taskTitle: task.task_title } : {}), + adoptTaskId: task.id + }, + timeoutMs: Math.max(budgets.attachDeadlineMs - Date.now(), 1), + requestId: forward.key, + whileLink: ['open'] + }) + } catch (error) { + settleUnansweredStart(runtime, { + conversationId: link.conversation_id, + taskId, + error, + key: forward.key + }) + throw inServerTerms(error, forward, server.name) + } + const started = StartedOnServer.safeParse(receipt) + if (started.success) { + recordRemoteStart(db, { + conversationId: link.conversation_id, + taskId, + dispatchId: started.data.dispatchId, + requestKey: forward.key, + peerRunId: started.data.run?.id, + agent: started.data.agent ?? null + }) + prepareWorkerStartCard( + runtime, + ctx.orchestrationCaller + )({ + dispatchId: started.data.dispatchId, + workerAddress: `dispatch:${started.data.dispatchId}`, + task: task.spec, + agent: null + }) + } + const answer = inServerTerms(receipt, forward, server.name) + return withRun( + answer && typeof answer === 'object' + ? { ...answer, taskId, ...(started.data?.run ? { serverRun: started.data.run } : {}) } + : answer + ) +} + +/** What a start's failure says about its reservation: adopt the Dispatch it names, keep it for the orphan lookup, or free it. */ +function settleUnansweredStart( + runtime: OrcaRuntimeService, + start: { conversationId: string; taskId: string; key: string; error: unknown } +): void { + const db = runtime.getOrchestrationDb() + const data = errorData(start.error) + const named = typeof data?.dispatchId === 'string' ? data.dispatchId : undefined + if (named) { + recordRemoteStart(db, { + conversationId: start.conversationId, + taskId: start.taskId, + dispatchId: named, + requestKey: start.key + }) + } else if (serverNeverAnswered(start.error) && data?.effectsApplied !== false) { + markRemoteStartOutcomeUnknown(db, start.taskId, start.key) + } else if ( + !(start.error instanceof OrchestrationError && start.error.code === 'operation_unknown') + ) { + // The server's ordinary start refused it, which a start that had launched answers with a receipt. + releaseRemotePlacement(db, start.taskId, start.key) + } +} diff --git a/src/main/runtime/rpc/dispatcher.ts b/src/main/runtime/rpc/dispatcher.ts index 8ad691b1fda..1852913b085 100644 --- a/src/main/runtime/rpc/dispatcher.ts +++ b/src/main/runtime/rpc/dispatcher.ts @@ -33,6 +33,7 @@ import type { RpcCallerScope } from './rpc-caller-scope' import { runWithActiveRpcCaller } from './rpc-active-caller' import { admitRpcRequest } from './rpc-request-admission-prologue' import { resolveOrcadFeatureRequirement, rpcMethodFeatureWait } from './orcad-feature-readiness' +import { routeRemoteOwnedCall } from '../orchestration/federation/remote-owned/remote-owned-route' export type DispatcherOptions = { runtime: OrcaRuntimeService @@ -244,6 +245,21 @@ export class RpcDispatcher { request, compatibility.legacyCoordinatorAuthority ) + // Before this Orca's receipts: the server that owns the worker keeps the receipt, under the caller's scoped key. + const remote = await routeRemoteOwnedCall(runtime, method, request, effectiveParams, { + ...context, + orchestrationCompatibilityEvidence: request.orchestrationCompatibilityEvidence + }) + if (remote.handled) { + recordRuntimeFeatureInteraction( + runtime, + request.method, + remote.result, + undefined, + request.params + ) + return remote.result + } // A legacy coordinator fingerprint outranks the transport identity; mint a fallback only when absent. const fallbackCallerFingerprint = context.authenticatedCallerFingerprint ?? diff --git a/src/main/runtime/rpc/methods/orchestration/access/orchestration-method-access.ts b/src/main/runtime/rpc/methods/orchestration/access/orchestration-method-access.ts index c36f9dd7336..faa50678f51 100644 --- a/src/main/runtime/rpc/methods/orchestration/access/orchestration-method-access.ts +++ b/src/main/runtime/rpc/methods/orchestration/access/orchestration-method-access.ts @@ -101,6 +101,16 @@ export function checkOrchestrationAccess( } } +/** The same check for a call this Orca sends on to a paired server instead of running it here. */ +export function checkDeclaredOrchestrationAccess( + access: OrchestrationMethodAccessDeclaration, + params: unknown, + ctx: RpcContext +): void { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a registered declaration was built for its method's params, which the dispatcher parsed with that method's schema. + checkOrchestrationAccess(access as OrchestrationMethodAccess, params, ctx) +} + function declaredCaller

( field: Extract, OrchestrationCallerField> | undefined, params: P diff --git a/src/main/runtime/rpc/methods/orchestration/federation/federation-lifecycle-settlement.test.ts b/src/main/runtime/rpc/methods/orchestration/federation/federation-lifecycle-settlement.test.ts index 9ffc18d5dd9..cd236302204 100644 --- a/src/main/runtime/rpc/methods/orchestration/federation/federation-lifecycle-settlement.test.ts +++ b/src/main/runtime/rpc/methods/orchestration/federation/federation-lifecycle-settlement.test.ts @@ -13,6 +13,7 @@ import '../../../../orchestration/db/dispatch-row-writer' import '../../../../orchestration/db/attempt-outcome-projection' import type { OrchestrationEnvironmentTransport } from '../../../../orchestration/federation/environment-transport' import '../../../../orchestration/legacy-remote/federation-lifecycle-settlement' +import { attachmentHostCapabilities } from './federation-peer-capabilities.test-support' import { RpcDispatcher } from '../../../dispatcher' import { ORCHESTRATION_METHODS } from '../orchestration' import { createFederationWorkerStartRequest as startRequest } from './federation-request.test-support' @@ -35,7 +36,7 @@ describe('orchestration federation lifecycle settlement', () => { workerRuntime = new OrcaRuntimeService() workerRuntime.setOrchestrationDb(workerDb) workerDispatcher = new RpcDispatcher({ runtime: workerRuntime, methods: ORCHESTRATION_METHODS }) - workerCapabilities = [...(workerRuntime.getStatus().capabilities ?? [])] + workerCapabilities = attachmentHostCapabilities(workerRuntime) failNextAckBeforeDelivery = false ackAttempts = 0 transport = { diff --git a/src/main/runtime/rpc/methods/orchestration/federation/federation-peer-capabilities.test-support.ts b/src/main/runtime/rpc/methods/orchestration/federation/federation-peer-capabilities.test-support.ts index ea946d15380..2180ec199f8 100644 --- a/src/main/runtime/rpc/methods/orchestration/federation/federation-peer-capabilities.test-support.ts +++ b/src/main/runtime/rpc/methods/orchestration/federation/federation-peer-capabilities.test-support.ts @@ -2,6 +2,7 @@ import { ORCHESTRATION_FEDERATION_FLEET_SNAPSHOT_RUNTIME_CAPABILITY, ORCHESTRATION_FEDERATION_RELEASE_ARCHIVE_RUNTIME_CAPABILITY, ORCHESTRATION_FEDERATION_STRUCTURED_READ_RUNTIME_CAPABILITY, + ORCHESTRATION_REMOTE_OWNED_WORKERS_RUNTIME_CAPABILITY, ORCHESTRATION_WORKER_AUTO_RELEASE_RUNTIME_CAPABILITY } from '../../../../../../shared/orchestration/orchestration-runtime-capabilities' import type { OrcaRuntimeService } from '../../../../orca-runtime' @@ -13,6 +14,8 @@ function withheldPeerCapabilities(host: { autoRelease: boolean }): ReadonlySet { return new Set([ + // These suites drive today's remote attachments, which a server that owns its workers never gets. + ORCHESTRATION_REMOTE_OWNED_WORKERS_RUNTIME_CAPABILITY, ...(host.newCapabilities ? [] : [ @@ -42,3 +45,10 @@ export function olderPeerStatusResponse( _meta: { runtimeId: runtime.getRuntimeId() } } } + +/** A host's capabilities as one that keeps today's remote attachments advertises them. */ +export function attachmentHostCapabilities(runtime: OrcaRuntimeService): string[] { + return (runtime.getStatus().capabilities ?? []).filter( + (capability) => capability !== ORCHESTRATION_REMOTE_OWNED_WORKERS_RUNTIME_CAPABILITY + ) +} diff --git a/src/main/runtime/rpc/methods/orchestration/federation/federation.test.ts b/src/main/runtime/rpc/methods/orchestration/federation/federation.test.ts index 909c25152f5..d72c8628d7a 100644 --- a/src/main/runtime/rpc/methods/orchestration/federation/federation.test.ts +++ b/src/main/runtime/rpc/methods/orchestration/federation/federation.test.ts @@ -9,6 +9,7 @@ import '../../../../orchestration/db/messages/mailbox-routing-page' import '../../../../orchestration/db/dispatch-row-writer' import '../../../../orchestration/db/attempt-outcome-projection' import type { OrchestrationEnvironmentTransport } from '../../../../orchestration/federation/environment-transport' +import { attachmentHostCapabilities } from './federation-peer-capabilities.test-support' import { RpcDispatcher } from '../../../dispatcher' import { ORCHESTRATION_METHODS } from '../orchestration' import { createFederationWorkerStartRequest as startRequest } from './federation-request.test-support' @@ -37,7 +38,7 @@ describe('orchestration federation', () => { runtime: workerRuntime, methods: ORCHESTRATION_METHODS }) - workerCapabilities = [...(workerRuntime.getStatus().capabilities ?? [])] + workerCapabilities = attachmentHostCapabilities(workerRuntime) workerPeerFingerprint = 'windows_peer_fingerprint' loseNextAckResponse = false const transport: OrchestrationEnvironmentTransport = { @@ -109,7 +110,7 @@ describe('orchestration federation', () => { runtime: workerRuntime, methods: ORCHESTRATION_METHODS }) - workerCapabilities = [...(workerRuntime.getStatus().capabilities ?? [])] + workerCapabilities = attachmentHostCapabilities(workerRuntime) } it('fails before remote task input when wait-for-setup fails', async () => { diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/message-methods.ts b/src/main/runtime/rpc/methods/orchestration/messaging/message-methods.ts index 1247c8adb73..577f39e9d76 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/message-methods.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/message-methods.ts @@ -12,6 +12,7 @@ import { } from '../../../../../../shared/rpc-contract/orchestration-params' import { TaskUpdateParams } from './task-update-params' import { forwardRemoteTaskUpdate } from '../../../../orchestration/federation/remote-owned/remote-owned-task-update' +import { remoteForwardCallerNamespace } from '../../../../orchestration/federation/remote-owned/remote-forward-caller' export const ORCHESTRATION_MESSAGE_METHODS = [ ORCHESTRATION_REPLY_METHOD, @@ -126,19 +127,21 @@ export const ORCHESTRATION_MESSAGE_METHODS = [ targets: { id: 'task', run: 'run' }, rule: taskUpdateAccess }, - handler: async (params, { orchestrationCaller, orchestrationMutation, runtime }) => { + handler: async (params, ctx) => { + const { orchestrationMutation, runtime } = ctx const db = runtime.getOrchestrationDb() - // The server sees every caller here as one pairing; the caller's own identity scopes its retry key. - const callerNamespace = - orchestrationCaller?.orcaSessionId ?? - (params.callerTerminalHandle - ? runtime.getTerminalPaneKey(params.callerTerminalHandle) - : null) const forwarded = await forwardRemoteTaskUpdate( runtime, params, - callerNamespace && orchestrationMutation - ? { callerNamespace, requestId: orchestrationMutation.requestId } + orchestrationMutation + ? { + callerNamespace: remoteForwardCallerNamespace( + runtime, + ctx, + params.callerTerminalHandle + ), + requestId: orchestrationMutation.requestId + } : undefined ) if (forwarded) { diff --git a/src/main/runtime/rpc/methods/orchestration/runs/mutation-request-show.ts b/src/main/runtime/rpc/methods/orchestration/runs/mutation-request-show.ts index f36db68f630..764d3f89580 100644 --- a/src/main/runtime/rpc/methods/orchestration/runs/mutation-request-show.ts +++ b/src/main/runtime/rpc/methods/orchestration/runs/mutation-request-show.ts @@ -8,7 +8,7 @@ import { RequestShowParams } from '../../../../../../shared/rpc-contract/orchest export const ORCHESTRATION_MUTATION_REQUEST_METHODS = [ defineOrchestrationMethod({ name: 'orchestration.requestShow', - access: { kind: 'reads', targets: { request: 'request' } }, + access: { kind: 'reads', server: 'on', targets: { request: 'request' } }, permission: 'workspace', params: RequestShowParams, // Read durable receipts without writing/mutating again; keep this recovery query outside ORCHESTRATION_MUTATION_METHODS. diff --git a/src/main/runtime/rpc/methods/orchestration/worker/worker-control.ts b/src/main/runtime/rpc/methods/orchestration/worker/worker-control.ts index 91e9c55d3b3..ce553163fc1 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/worker-control.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/worker-control.ts @@ -79,7 +79,8 @@ export const ORCHESTRATION_WORKER_CONTROL_METHODS = [ terminal: observation.exact ? observation.terminal : null, observation: exposeObservation(observation), terminalResource: resource ? exposeWorkerTerminalResource(resource) : null, - autoReleaseAt: liveWorkerTerminalAutoReleaseAt(resource) + autoReleaseAt: liveWorkerTerminalAutoReleaseAt(resource), + coordinatedFrom: db.getPeerRun(dispatch.run_id)?.home_label ?? null } } }), diff --git a/src/main/runtime/rpc/methods/orchestration/worker/worker-list-method.ts b/src/main/runtime/rpc/methods/orchestration/worker/worker-list-method.ts index 66f8bbabca7..958da8fee3c 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/worker-list-method.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/worker-list-method.ts @@ -25,7 +25,7 @@ import { WorkerListParams } from '../../../../../../shared/rpc-contract/orchestration-worker-release-schemas-params' -const emptyWorkerListPage = (limit: number) => ({ +export const emptyWorkerListPage = (limit: number) => ({ workers: [], counts: {}, page: { limit, total: 0, hasMore: false, nextCursor: null } @@ -33,7 +33,7 @@ const emptyWorkerListPage = (limit: number) => ({ export const ORCHESTRATION_WORKER_LIST_METHOD = defineOrchestrationMethod({ name: 'orchestration.workerList', - access: { kind: 'reads', targets: { run: 'run' }, collection: true }, + access: { kind: 'reads', server: 'on', targets: { run: 'run' }, collection: true }, permission: 'workspace', params: WorkerListParams, handler: async (params, { runtime }) => { diff --git a/src/main/runtime/rpc/methods/orchestration/worker/workers.ts b/src/main/runtime/rpc/methods/orchestration/worker/workers.ts index 2538b27521b..d5bcb324514 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/workers.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/workers.ts @@ -60,15 +60,62 @@ export const ORCHESTRATION_WORKER_START_METHODS = [ /** worker-start, and a follow-up sent to a finished worker, which is worker-start run for the sender. */ export async function startWorker( + startParams: z.infer, + ctx: WorkerStartContext, + startFollowUp?: WorkerFollowUp +): Promise { + const { runtime, orchestrationMutation, orchestrationCaller } = ctx + const prepared = await prepareWorkerStart(startParams, ctx, startFollowUp) + const { params, followUp, coordinator, bindRun, helperCreator, taskDeps, existingTask, withRun } = + prepared + const db = runtime.getOrchestrationDb() + const mode = decideWorkerStartMode({ + params, + settings: readWorkerStartModeSettings(runtime) + }) + if (params.on) { + // Remote workers are terminal-only; carry the mode receipt to explain structured-preference fallback. + const receipt = await startFederatedWorker({ + params, + runtime, + db, + bindRunId: () => bindRun().run.id, + task: existingTask, + taskDeps, + orchestrationMutation, + callerSession: orchestrationCaller + }) + await stopHelperAfterStart(runtime, receipt, orchestrationMutation) + return withRun(receipt && typeof receipt === 'object' ? { ...receipt, mode } : receipt) + } + const receipt = await startLocalWorker({ + params: { ...params, timeoutMs: resolveWorkerStartReadinessTimeoutMs(params.timeoutMs) }, + runtime, + db, + bindRun: () => bindRun().run, + taskDeps, + coordinator, + callerSession: orchestrationCaller, + existingTask, + orchestrationMutation, + mode, + helperCreator, + followUp + }) + await stopHelperAfterStart(runtime, receipt, orchestrationMutation) + return withRun(withWorkerStartNext(receipt)) +} + +/** What every start checks and binds before it is placed, here or on a paired server. */ +export async function prepareWorkerStart( params: z.infer, { runtime, - orchestrationMutation, orchestrationCompatibilityEvidence, orchestrationCaller - }: WorkerStartContext, + }: Omit, followUp?: WorkerFollowUp -): Promise { +) { if (!isWorkerStartTimeoutWithinTimerLimit(params.timeoutMs)) { throw new OrchestrationError( 'invalid_argument', @@ -103,7 +150,6 @@ export async function startWorker( { effectsApplied: false } ) } - const readinessTimeoutMs = resolveWorkerStartReadinessTimeoutMs(params.timeoutMs) const db = runtime.getOrchestrationDb() const coordinator = resolveOrchestrationCaller(runtime, { callerTerminalHandle: params.from, @@ -200,39 +246,14 @@ export async function startWorker( followUp = { ...followUp, reopen: placement.reopen, earlierWork: placement.earlierWork } await assertWorkerStartTaskSpecWithinPromptBudget(spec) } - const mode = decideWorkerStartMode({ + return { params, - settings: readWorkerStartModeSettings(runtime) - }) - if (params.on) { - // Remote workers are terminal-only; carry the mode receipt to explain structured-preference fallback. - const receipt = await startFederatedWorker({ - params, - runtime, - db, - bindRunId: () => bindRun().run.id, - task: existingTask, - taskDeps, - orchestrationMutation, - callerSession: orchestrationCaller - }) - await stopHelperAfterStart(runtime, receipt, orchestrationMutation) - return withRun(receipt && typeof receipt === 'object' ? { ...receipt, mode } : receipt) - } - const receipt = await startLocalWorker({ - params: { ...params, timeoutMs: readinessTimeoutMs }, - runtime, - db, - bindRun: () => bindRun().run, - taskDeps, + followUp, coordinator, - callerSession: orchestrationCaller, - existingTask, - orchestrationMutation, - mode, + bindRun, helperCreator, - followUp - }) - await stopHelperAfterStart(runtime, receipt, orchestrationMutation) - return withRun(withWorkerStartNext(receipt)) + taskDeps, + existingTask, + withRun + } } diff --git a/src/shared/orchestration/orchestration-runtime-capabilities.ts b/src/shared/orchestration/orchestration-runtime-capabilities.ts index 9aac5d474e3..43c3e0f836d 100644 --- a/src/shared/orchestration/orchestration-runtime-capabilities.ts +++ b/src/shared/orchestration/orchestration-runtime-capabilities.ts @@ -40,6 +40,9 @@ export const ORCHESTRATION_CONTRACT_VERSION = 1 as const export const ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY = 'orchestration.contract.v1' as const /** Receives agent mail a paired Orca carries for its agents, and queues replies for it to pull. */ export const ORCHESTRATION_PEER_MAIL_RUNTIME_CAPABILITY = 'orchestration.peer-mail.v1' as const +/** Runs the workers a paired Orca places here as its own, coordinated from that Orca (wire v1). */ +export const ORCHESTRATION_REMOTE_OWNED_WORKERS_RUNTIME_CAPABILITY = + 'orchestration.remote-owned-workers.v1' as const export const ORCHESTRATION_RUNTIME_CAPABILITIES = [ ORCHESTRATION_FEDERATION_RUNTIME_CAPABILITY, @@ -58,5 +61,6 @@ export const ORCHESTRATION_RUNTIME_CAPABILITIES = [ ORCHESTRATION_WORKER_TURN_STOP_REPORTS_RUNTIME_CAPABILITY, ORCHESTRATION_ASK_RETURNS_AT_ONCE_RUNTIME_CAPABILITY, ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY, - ORCHESTRATION_PEER_MAIL_RUNTIME_CAPABILITY + ORCHESTRATION_PEER_MAIL_RUNTIME_CAPABILITY, + ORCHESTRATION_REMOTE_OWNED_WORKERS_RUNTIME_CAPABILITY ] as const diff --git a/src/shared/rpc-contract/orchestration-runs-mutation-request-show-params.ts b/src/shared/rpc-contract/orchestration-runs-mutation-request-show-params.ts index 496c1ea3827..063b02614fc 100644 --- a/src/shared/rpc-contract/orchestration-runs-mutation-request-show-params.ts +++ b/src/shared/rpc-contract/orchestration-runs-mutation-request-show-params.ts @@ -1,4 +1,8 @@ import { z } from 'zod' import { requiredString } from './rpc-param-primitives' -export const RequestShowParams = z.object({ request: requiredString('Missing --request') }) +export const RequestShowParams = z.object({ + request: requiredString('Missing --request'), + // The paired server a forwarded request ran on; its receipt is kept there. + on: z.string().min(1).optional() +}) diff --git a/src/shared/rpc-contract/orchestration-worker-release-schemas-params.ts b/src/shared/rpc-contract/orchestration-worker-release-schemas-params.ts index 065e4e871f3..5178184782a 100644 --- a/src/shared/rpc-contract/orchestration-worker-release-schemas-params.ts +++ b/src/shared/rpc-contract/orchestration-worker-release-schemas-params.ts @@ -21,5 +21,7 @@ export const WorkerListParams = z.object({ cursor: z.string().min(1).max(2_048).optional(), limit: z.number().int().min(1).max(ORCHESTRATION_FLEET_PAGE_MAX).optional(), includeRemote: z.boolean().optional(), - paginate: z.boolean().optional() + paginate: z.boolean().optional(), + // A paired server's own list, paged by its cursor. + on: z.string().min(1).optional() })