feat(orchestration): run commands about a paired server's workers there, and switch new pairs to it

This commit is contained in:
Jinwoo-H
2026-11-10 15:45:28 -05:00
parent ed1b753bbf
commit e7b3908a52
24 changed files with 762 additions and 65 deletions
@@ -1,14 +1,18 @@
import type { CommandHandler } from '../../dispatch'
import { printOrchestrationReceipt } from './agent-view'
import { getRequiredStringFlag } from '../../flags'
import { getOptionalStringFlag, getRequiredStringFlag } from '../../flags'
import { RuntimeClientError } from '../../runtime-client'
import type { OrchestrationMutationRequestShowResult } from '../../../shared/orchestration/orchestration-mutation-request'
export const ORCHESTRATION_REQUEST_SHOW_HANDLER: Record<string, CommandHandler> = {
'orchestration request-show': async ({ flags, client, json }) => {
const request = getRequiredStringFlag(flags, 'request')
const on = getOptionalStringFlag(flags, 'on')
const result = await client
.call<OrchestrationMutationRequestShowResult>('orchestration.requestShow', { request })
.call<OrchestrationMutationRequestShowResult>('orchestration.requestShow', {
request,
...(on ? { on } : {})
})
.catch((error: unknown) => {
if (error instanceof RuntimeClientError && error.code === 'method_not_found') {
throw new RuntimeClientError(
@@ -44,6 +44,8 @@ export type WorkerShowResult = {
server?: RemoteWorkerServer & { reachable: false }
lastReport?: string | null
autoReleaseAt?: string | null
/** The paired Orca whose Run here coordinates this worker. */
coordinatedFrom?: string | null
}
/** A worker a paired server runs, as this Orca names that server and last heard from it. */
@@ -70,6 +72,7 @@ type WorkerShowFields = {
waitingOnHuman?: string
lastError?: string
autoReleaseAt?: string
coordinatedFrom?: string
next?: string
}
@@ -108,6 +111,7 @@ export const WORKER_SHOW_VIEW: OrchestrationAgentView<WorkerShowResult, WorkerSh
: undefined,
lastError: value.worker.lastError ?? undefined,
autoReleaseAt: value.autoReleaseAt ?? undefined,
coordinatedFrom: value.coordinatedFrom ?? undefined,
next: value.projection ? renderOrchestrationArgv(value.projection.nextAction.argv) : undefined
}
}
@@ -133,6 +133,7 @@ export const ORCHESTRATION_WORKER_TERMINAL_HANDLERS: Record<string, CommandHandl
run: scope.run,
terminalState,
...(flags.has('include-remote') ? { includeRemote: true } : {}),
on: getOptionalStringFlag(flags, 'on'),
cursor: getOptionalStringFlag(flags, 'cursor'),
limit: getOptionalPositiveIntegerFlag(flags, 'limit')
})
+10 -1
View File
@@ -35,7 +35,16 @@ export function orchestrationMutationRecoveryError(
// Without a dispatch id, query by request id to inspect the outcome before retrying a mutation.
const queryCommand = dispatchId
? [executable, 'orchestration', 'worker-show', '--dispatch', dispatchId, '--json']
: [executable, 'orchestration', 'request-show', '--request', requestId, '--json']
: [
executable,
'orchestration',
'request-show',
'--request',
requestId,
// A command forwarded to a paired server keeps its receipt there.
...(typeof data?.server === 'string' ? ['--on', data.server] : []),
'--json'
]
const recovery = {
orchestrationRequestId: requestId,
...(dispatchId ? { dispatchId } : {}),
+12 -3
View File
@@ -145,14 +145,23 @@ export const ORCHESTRATION_WORKER_COMMAND_SPECS: CommandSpec[] = [
path: ['orchestration', 'worker-list'],
summary: 'List supervised worker terminal resource accounting',
usage:
'orca orchestration worker-list [--run <run_id>] [--terminal-state <active|reclaimable|retained|release_pending|release_unknown|released>] [--include-remote] [--cursor <cursor>] [--limit <1-100>] [--json]',
allowedFlags: [...GLOBAL_FLAGS, 'run', 'terminal-state', 'include-remote', 'cursor', 'limit'],
'orca orchestration worker-list [--run <run_id>] [--terminal-state <active|reclaimable|retained|release_pending|release_unknown|released>] [--include-remote] [--on <server>] [--cursor <cursor>] [--limit <1-100>] [--json]',
allowedFlags: [
...GLOBAL_FLAGS,
'run',
'terminal-state',
'include-remote',
'on',
'cursor',
'limit'
],
notes: [
'Terminal state is process accounting and is reported separately from Task status; a completed Task can still own a live terminal.',
'A reclaimable worker with resource.autoReleaseAt is released automatically at that time unless it is reused with worker-start --terminal or kept open (worker-keep-open, worker-start --keep-open, or the user).',
'Context-only Dispatches created by orchestration dispatch are included as unsupervised with terminal state retained.',
'Returns at most 100 local rows, newest first, by default; --include-remote adds connected-server observations when the host supports fleet listing. Continue with the opaque nextCursor value unchanged.',
'Without --run the list is scoped to the Run bound to the calling terminal, and to every Run when there is no binding; the receipt reports which in scope.source (flag, bound, or all).'
'Without --run the list is scoped to the Run bound to the calling terminal, and to every Run when there is no binding; the receipt reports which in scope.source (flag, bound, or all).',
"--on <server> lists that paired server's own workers for your Run there (or all of them without a Run), paged with its own cursor."
]
}
]
+4 -3
View File
@@ -209,12 +209,13 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [
{
path: ['orchestration', 'request-show'],
summary: 'Ask whether one orchestration mutation request already took effect',
usage: 'orca orchestration request-show --request <request_id> [--json]',
allowedFlags: [...GLOBAL_FLAGS, 'request'],
usage: 'orca orchestration request-show --request <request_id> [--on <server>] [--json]',
allowedFlags: [...GLOBAL_FLAGS, 'request', 'on'],
notes: [
'Read-only: it never starts, retries, or settles anything, so it is safe to run after any lost response.',
'completed means the mutation landed and --retry-request replays the recorded outcome instead of starting a second one. pending means the original mutation is still running or Orca restarted before recording its outcome; wait for a live original command, otherwise replay with --retry-request.',
'absent means this runtime holds no receipt for that request under your caller identity: it never arrived, it failed before recording anything, or the receipt was pruned. Absent is not proof that nothing happened.'
'absent means this runtime holds no receipt for that request under your caller identity: it never arrived, it failed before recording anything, or the receipt was pruned. Absent is not proof that nothing happened.',
'A command about a worker on a paired server keeps its receipt on that server; --on <server> asks there.'
]
},
{
@@ -0,0 +1,46 @@
import type { OrcaRuntimeService } from '../../../orca-runtime'
import type { RpcContext } from '../../../rpc/core'
import { remoteForwardRequestId } from './remote-owned-calls'
/**
* Who is asking, as a retry of the same request must find it again: the caller's session, else
* its terminal's pane, which survives a restart, else the person at this Orca outside any terminal.
* Never one shared identity for every terminal, so two callers reusing an ID never collide.
*/
export function remoteForwardCallerNamespace(
runtime: Pick<OrcaRuntimeService, 'getOrchestrationDb' | 'getTerminalPaneKey'>,
ctx: Pick<RpcContext, 'orchestrationCaller' | 'orchestrationCompatibilityEvidence'>,
callerHandle: string | undefined
): string {
const session = ctx.orchestrationCaller?.orcaSessionId
if (session) {
return `session:${session}`
}
const handle = callerHandle ?? ctx.orchestrationCompatibilityEvidence?.terminalHandle
const pane =
ctx.orchestrationCompatibilityEvidence?.paneKey ??
(handle ? runtime.getTerminalPaneKey(handle) : null)
return pane
? `pane:${pane}`
: `local:${runtime.getOrchestrationDb().getOrCreateLocalMutationCallerFingerprint()}`
}
/** The key a forwarded call goes out under, and how to say it back in the caller's own terms. */
export type RemoteForwardKey = { key: string; requestId: string }
export function remoteForwardKey(
runtime: Pick<OrcaRuntimeService, 'getOrchestrationDb' | 'getTerminalPaneKey'>,
ctx: Pick<RpcContext, 'orchestrationCaller' | 'orchestrationCompatibilityEvidence'>,
callerHandle: string | undefined,
requestId: string | undefined
): RemoteForwardKey | undefined {
return requestId
? {
key: remoteForwardRequestId(
remoteForwardCallerNamespace(runtime, ctx, callerHandle),
requestId
),
requestId
}
: undefined
}
@@ -45,7 +45,8 @@ const NO_ANSWER_CODES = new Set([
'remote_link_closed'
])
function serverAnswered(error: unknown): boolean {
/** Whether the server got the call and answered it, rather than this Orca never reaching it. */
export function serverAnswered(error: unknown): boolean {
return error instanceof OrchestrationError && !NO_ANSWER_CODES.has(error.code)
}
@@ -0,0 +1,50 @@
import { OrchestrationError } from '../../orchestration-error'
import { serverAnswered } from './remote-owed-actions'
import type { RemoteForwardKey } from './remote-forward-caller'
/** The server got nothing, or never answered, so whatever the call did there is unknown here. */
export function serverNeverAnswered(error: unknown): boolean {
return !serverAnswered(error)
}
export function errorData(error: unknown): Record<string, unknown> | undefined {
const data = error instanceof OrchestrationError ? error.data : undefined
return data && typeof data === 'object' && !Array.isArray(data)
? // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a non-array object's own fields, read as unknowns.
(data as Record<string, unknown>)
: undefined
}
/**
* The server's answer, or its error, in the caller's own terms: the caller knows its request ID,
* not the scoped key the server saw, and an error names the server it came from so recovery
* (`request-show --on`) asks there.
*/
export function inServerTerms<T>(
value: T,
forward: RemoteForwardKey | undefined,
server: string
): T {
const swap = (item: unknown): unknown => {
if (typeof item === 'string') {
return forward ? item.split(forward.key).join(forward.requestId) : item
}
if (Array.isArray(item)) {
return item.map(swap)
}
if (item && typeof item === 'object') {
return Object.fromEntries(Object.entries(item).map(([field, inner]) => [field, swap(inner)]))
}
return item
}
if (value instanceof OrchestrationError) {
const data = swap(errorData(value) ?? {})
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: an error stays an OrchestrationError with the same code.
return new OrchestrationError(value.code, String(swap(value.message)), {
...(data as Record<string, unknown>),
server
}) as T
}
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: only string contents change; every shape is kept.
return swap(value) as T
}
@@ -0,0 +1,300 @@
import { randomUUID } from 'node:crypto'
import type { OrcaRuntimeService } from '../../../orca-runtime'
import type { RpcContext, RpcMethod, RpcRequest } from '../../../rpc/core'
import {
checkDeclaredOrchestrationAccess,
type OrchestrationMethodAccessDeclaration
} from '../../../rpc/methods/orchestration/access/orchestration-method-access'
import { emptyWorkerListPage } from '../../../rpc/methods/orchestration/worker/worker-list-method'
import { ORCHESTRATION_FLEET_PAGE_MAX } from '../../../../../shared/orchestration/fleet/orchestration-fleet-projection'
import { WorkerStartParams } from '../../../../../shared/rpc-contract/orchestration-worker-start-params'
import type { OrchestrationDb } from '../../db/orchestration-db'
import { runLifecycleWriteTransaction } from '../../db/lifecycle-write-transaction-runner'
import {
getRemoteDispatch,
oweRemoteDispatchStop
} from '../../db/remote-owned/remote-dispatch-records'
import { adoptRemoteAttempt } from '../../db/remote-owned/remote-placements'
import {
getOpenRemoteRunLink,
getRemoteRunLink,
type RemoteRunLinkRow
} from '../../db/remote-owned/remote-run-links'
import { callRemoteOwnedServer } from './remote-owned-calls'
import { remoteForwardKey, type RemoteForwardKey } from './remote-forward-caller'
import { inServerTerms, serverNeverAnswered } from './remote-owned-answers'
import { sendOwedRemoteStop } from './remote-owed-actions'
import { serverOwnsPlacedWorkers, startRemoteOwnedWorker } from './remote-owned-start'
type Routed = { handled: false } | { handled: true; result: unknown }
const LOCAL: Routed = { handled: false }
// Long enough for a send that starts a follow-up worker there and waits for it to be ready.
const FORWARD_TIMEOUT_MS = 180_000
// What `--on <server>` sends there; send and reply --on stay peer mail.
const SERVER_CALLS: ReadonlySet<string> = new Set([
'orchestration.workerStart',
'orchestration.workerList',
'orchestration.requestShow'
])
type Values = Record<string, unknown>
/**
* Before this Orca's receipts: a command about a worker a paired server owns runs there, by that
* server's ordinary handler, as its Run's coordinator. The worker is found from the subjects the
* method declares; the caller passes the method's one access check here first, as it would for a
* worker of its own. A paired Orca's calls always run here: pairing is one-way.
*/
export async function routeRemoteOwnedCall(
runtime: OrcaRuntimeService,
method: RpcMethod,
request: RpcRequest,
params: unknown,
ctx: RpcContext
): Promise<Routed> {
const access = method.orchestrationAccess
if (!access || access.kind === 'localOnly' || ctx.orchestrationCaller?.kind === 'peer') {
return LOCAL
}
const values: Values = params && typeof params === 'object' ? { ...params } : {}
const db = runtime.getOrchestrationDb()
const on = access.server && stringField(values, access.server)
if (on !== undefined) {
return routeServerCall(runtime, request, access, values, ctx, on)
}
const subject = remoteSubject(db, access, values)
if (!subject) {
return LOCAL
}
checkDeclaredOrchestrationAccess(access, params, ctx)
const forward = remoteForwardKey(
runtime,
ctx,
access.caller && stringField(values, access.caller),
request.orchestrationRequestId
)
if (request.method === 'orchestration.workerStop' && subject.dispatchId) {
return { handled: true, result: await stopRemoteWorker(runtime, subject, forward) }
}
return {
handled: true,
result: await forwardToLink(runtime, subject.link, request.method, access, values, forward)
}
}
/** `--on <server>`: a start, a list, or a receipt lookup there. */
async function routeServerCall(
runtime: OrcaRuntimeService,
request: RpcRequest,
access: OrchestrationMethodAccessDeclaration,
values: Values,
ctx: RpcContext,
on: string
): Promise<Routed> {
if (!SERVER_CALLS.has(request.method)) {
return LOCAL
}
checkDeclaredOrchestrationAccess(access, values, ctx)
const server = runtime.resolveOrchestrationWorkerServer(on)
const db = runtime.getOrchestrationDb()
if (request.method === 'orchestration.workerStart') {
if (!(await serverOwnsPlacedWorkers(runtime, server))) {
// An older server: today's remote attachment, the one legacy branch for new work.
return LOCAL
}
return {
handled: true,
result: await startRemoteOwnedWorker(
runtime,
WorkerStartParams.parse(values),
ctx,
request.orchestrationRequestId
)
}
}
if (request.method === 'orchestration.workerList') {
const runId = stringField(values, 'run')
// A Run lists what it placed there, as its coordinator; no Run lists the server as a whole.
const link = runId ? openLinkFor(db, runId, server.environmentId) : undefined
if (runId && !link) {
return {
handled: true,
result: emptyWorkerListPage(numberField(values, 'limit') ?? ORCHESTRATION_FLEET_PAGE_MAX)
}
}
const forwarded = paramsForServer(access, values)
return {
handled: true,
result: link
? await forwardToLink(runtime, link, request.method, access, values)
: await runtime.callOrchestrationWorkerServer(
server.environmentId,
request.method,
forwarded,
FORWARD_TIMEOUT_MS,
undefined,
{ expectedEnvironmentPairingRevision: server.pairingRevision }
)
}
}
{
// The server keyed the request by this caller's scoped key, as the forward that sent it did.
const forward = remoteForwardKey(
runtime,
ctx,
undefined,
stringField(values, 'request') ?? randomUUID()
)!
const answer = await runtime.callOrchestrationWorkerServer(
server.environmentId,
request.method,
{ request: forward.key },
FORWARD_TIMEOUT_MS,
undefined,
{ expectedEnvironmentPairingRevision: server.pairingRevision }
)
return { handled: true, result: inServerTerms(answer, forward, server.name) }
}
}
type RemoteSubject = { link: RemoteRunLinkRow; dispatchId?: string }
/** The worker or question a call names, when a paired server owns it. Tasks stay in this Orca's graph. */
function remoteSubject(
db: OrchestrationDb,
access: OrchestrationMethodAccessDeclaration,
values: Values
): RemoteSubject | undefined {
for (const [field, kind] of Object.entries(access.targets)) {
const value = stringField(values, field)
const dispatchId =
kind === 'dispatch'
? value
: kind === 'party' && value?.startsWith('dispatch:')
? value.slice('dispatch:'.length)
: undefined
const record = dispatchId ? getRemoteDispatch(db, dispatchId) : undefined
const question =
kind === 'message' && value
? // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SELECT of one column returns exactly that column.
(db.db
.prepare('SELECT conversation_id FROM remote_owned_questions WHERE message_id = ?')
.get(value) as { conversation_id: string } | undefined)
: undefined
const conversationId = record?.conversation_id ?? question?.conversation_id
const link = conversationId ? getRemoteRunLink(db, conversationId) : undefined
if (link) {
return { link, dispatchId: record?.dispatch_id }
}
}
return undefined
}
/** This Orca's caller, Run and server fields mean nothing there: the server names the caller itself. */
function paramsForServer(access: OrchestrationMethodAccessDeclaration, values: Values): Values {
const local = new Set<string>(
[
access.caller,
access.callerPane,
access.server,
...Object.entries(access.targets)
.filter(([, kind]) => kind === 'run')
.map(([field]) => field)
].filter((field): field is string => field !== undefined)
)
return Object.fromEntries(Object.entries(values).filter(([field]) => !local.has(field)))
}
async function forwardToLink(
runtime: OrcaRuntimeService,
link: RemoteRunLinkRow,
method: string,
access: OrchestrationMethodAccessDeclaration,
values: Values,
forward?: RemoteForwardKey
): Promise<unknown> {
const server = runtime.resolveOrchestrationWorkerServer(link.environment_id).name
try {
const answer = await callRemoteOwnedServer(runtime, link, {
method,
params: paramsForServer(access, values),
timeoutMs: FORWARD_TIMEOUT_MS,
...(forward ? { requestId: forward.key } : {}),
whileLink: ['open']
})
seedNamedDispatches(runtime.getOrchestrationDb(), link, answer)
return inServerTerms(answer, forward, server)
} catch (error) {
throw inServerTerms(error, forward, server)
}
}
/** A stop is owed until the server proves it; the ordinary stop answers when it can, and the pull loop resends otherwise. */
async function stopRemoteWorker(
runtime: OrcaRuntimeService,
subject: RemoteSubject,
forward: RemoteForwardKey | undefined
): Promise<unknown> {
const db = runtime.getOrchestrationDb()
const dispatchId = subject.dispatchId!
const server = runtime.resolveOrchestrationWorkerServer(subject.link.environment_id).name
const owed = oweRemoteDispatchStop(db, dispatchId, forward?.key ?? `stop:${randomUUID()}`)
runtime.ensureOrchestrationFederationRelay(subject.link.run_id)
try {
return inServerTerms(
await sendOwedRemoteStop(runtime, owed?.record ?? getRemoteDispatch(db, dispatchId)!),
forward,
server
)
} catch (error) {
if (!serverNeverAnswered(error) || !getOpenRemoteRunLink(db, subject.link.conversation_id)) {
throw inServerTerms(error, forward, server)
}
return {
dispatchId,
state: 'stop_requested',
server,
next: `${server} did not answer, so the stop is owed: this Orca sends it again until ${server} confirms the worker stopped.`
}
}
}
/** A Dispatch the server's answer names (a follow-up it started) is its; record it so commands about it route there. */
function seedNamedDispatches(db: OrchestrationDb, link: RemoteRunLinkRow, answer: unknown): void {
const named = answer && typeof answer === 'object' ? (answer as Values) : {}
const dispatchId = stringField(named, 'dispatchId')
const taskId = stringField(named, 'taskId')
if (!dispatchId || !taskId || getRemoteDispatch(db, dispatchId)) {
return
}
runLifecycleWriteTransaction(db.db, 'seed_remote_dispatch', () => {
const open = getOpenRemoteRunLink(db, link.conversation_id)
if (open) {
adoptRemoteAttempt(db, open, { taskId, dispatchId })
}
})
}
function openLinkFor(
db: OrchestrationDb,
runId: string,
environmentId: string
): RemoteRunLinkRow | undefined {
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SELECT * over this table returns exactly its row shape.
return db.db
.prepare(
`SELECT * FROM remote_run_links WHERE run_id = ? AND environment_id = ? AND state = 'open'`
)
.get(runId, environmentId) as RemoteRunLinkRow | undefined
}
function stringField(values: Values, field: string): string | undefined {
const value = values[field]
return typeof value === 'string' && value !== '' ? value : undefined
}
function numberField(values: Values, field: string): number | undefined {
const value = values[field]
return typeof value === 'number' ? value : undefined
}
@@ -0,0 +1,199 @@
import { createHash, randomUUID } from 'node:crypto'
import { z } from 'zod'
import type { WorkerStartParams } from '../../../../../shared/rpc-contract/orchestration-worker-start-params'
import { ORCHESTRATION_REMOTE_OWNED_WORKERS_RUNTIME_CAPABILITY } from '../../../../../shared/orchestration/orchestration-runtime-capabilities'
import { ORCHESTRATION_CONTRACT_PREFLIGHT_TIMEOUT_MS } from '../../../../../shared/orchestration/orchestration-timing-budgets'
import type { RuntimeStatus } from '../../../../../shared/runtime-session-contracts'
import type { OrcaRuntimeService } from '../../../orca-runtime'
import type { RpcContext } from '../../../rpc/core'
import { prepareWorkerStart } from '../../../rpc/methods/orchestration/worker/workers'
import { resolveFederatedWorkerStartBudgets } from '../../../rpc/methods/orchestration/worker/worker-start-budgets'
import { OrchestrationError } from '../../orchestration-error'
import { prepareWorkerStartCard } from '../../workers/launch/worker-start-card'
import { getOrchestrationPeerCapabilityCache } from '../orchestration-peer-capability-cache'
import type { OrchestrationWorkerServer } from '../environment-transport'
import {
markRemoteStartOutcomeUnknown,
recordRemoteStart,
releaseRemotePlacement,
reserveRemotePlacement
} from '../../db/remote-owned/remote-placements'
import { openRemoteRunLink } from '../../db/remote-owned/remote-run-links'
import { callRemoteOwnedServer } from './remote-owned-calls'
import { remoteForwardKey } from './remote-forward-caller'
import { errorData, inServerTerms, serverNeverAnswered } from './remote-owned-answers'
type StartParams = z.infer<typeof WorkerStartParams>
type StartContext = Pick<
RpcContext,
'runtime' | 'orchestrationCompatibilityEvidence' | 'orchestrationCaller'
>
/** Whether the server runs the workers placed on it as its own; an older one keeps today's remote attachments. */
export async function serverOwnsPlacedWorkers(
runtime: OrcaRuntimeService,
server: OrchestrationWorkerServer
): Promise<boolean> {
const decision = await getOrchestrationPeerCapabilityCache(runtime).resolve({
peerFingerprint: server.peerFingerprint,
expectedRuntimeEpoch: null,
capability: ORCHESTRATION_REMOTE_OWNED_WORKERS_RUNTIME_CAPABILITY,
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: status.get answers RuntimeStatus; the cache reads only its capabilities and runtime ID.
probe: async () =>
(await runtime.callOrchestrationWorkerServer(
server.environmentId,
'status.get',
undefined,
ORCHESTRATION_CONTRACT_PREFLIGHT_TIMEOUT_MS,
undefined,
{ expectedEnvironmentPairingRevision: server.pairingRevision }
)) as RuntimeStatus
})
return decision.supported
}
/** One ID for the Task here and the start there, from the start's retry key, so a retry finds both. */
function adoptedTaskId(key: string): string {
return `task_${createHash('sha256').update(`task\n${key}`).digest('hex').slice(0, 12)}`
}
const StartedOnServer = z.object({
dispatchId: z.string().min(1),
run: z.object({ id: z.string().min(1) }).optional(),
agent: z.string().nullable().optional()
})
/**
* worker-start --on a server that owns its workers: the Task stays in this Orca's graph, and the
* server starts an ordinary worker for it as the Run's coordinator. Synchronous, launch then
* record: the Task is reserved for the server first, and becomes its attempt once it answers.
*/
export async function startRemoteOwnedWorker(
runtime: OrcaRuntimeService,
params: StartParams,
ctx: StartContext,
callerRequestId: string | undefined
): Promise<unknown> {
if (params.terminal) {
throw new OrchestrationError(
'invalid_argument',
'--terminal names a terminal on this Orca, so it cannot be used with --on. No effects were applied.',
{ effectsApplied: false }
)
}
const { coordinator, bindRun, taskDeps, existingTask, withRun } = await prepareWorkerStart(
params,
ctx
)
const db = runtime.getOrchestrationDb()
const server = runtime.resolveOrchestrationWorkerServer(params.on!)
const { run } = bindRun()
// An older CLI sends no request ID; the start still runs, but a retry cannot find it.
const forward = remoteForwardKey(runtime, ctx, params.from, callerRequestId ?? randomUUID())!
const taskId = existingTask?.id ?? adoptedTaskId(forward.key)
const budgets = resolveFederatedWorkerStartBudgets(params.timeoutMs)
const link = openRemoteRunLink(db, run.id, server)
const { task } = reserveRemotePlacement(db, {
link,
taskId,
requestKey: forward.key,
runtimeId: runtime.getRuntimeId(),
retryOf: params.retryOf,
create: existingTask
? undefined
: {
spec: params.spec!,
taskTitle: params.taskTitle,
deps: taskDeps,
parentId: params.parent,
createdByTerminalHandle: coordinator?.terminalHandle ?? undefined,
createdByPaneKey: coordinator?.paneKey ?? undefined,
createdByRunGeneration: run.consumer_generation
}
})
// Its queue has a reader from the start, so mail sent before the answer still comes home.
runtime.ensureOrchestrationFederationRelay(run.id)
const {
from: _from,
run: _run,
on: _on,
task: _task,
deps: _deps,
parent: _parent,
...options
} = params
let receipt: unknown
try {
receipt = await callRemoteOwnedServer(runtime, link, {
method: 'orchestration.workerStart',
params: {
...options,
spec: task.spec,
...(task.task_title ? { taskTitle: task.task_title } : {}),
adoptTaskId: task.id
},
timeoutMs: Math.max(budgets.attachDeadlineMs - Date.now(), 1),
requestId: forward.key,
whileLink: ['open']
})
} catch (error) {
settleUnansweredStart(runtime, {
conversationId: link.conversation_id,
taskId,
error,
key: forward.key
})
throw inServerTerms(error, forward, server.name)
}
const started = StartedOnServer.safeParse(receipt)
if (started.success) {
recordRemoteStart(db, {
conversationId: link.conversation_id,
taskId,
dispatchId: started.data.dispatchId,
requestKey: forward.key,
peerRunId: started.data.run?.id,
agent: started.data.agent ?? null
})
prepareWorkerStartCard(
runtime,
ctx.orchestrationCaller
)({
dispatchId: started.data.dispatchId,
workerAddress: `dispatch:${started.data.dispatchId}`,
task: task.spec,
agent: null
})
}
const answer = inServerTerms(receipt, forward, server.name)
return withRun(
answer && typeof answer === 'object'
? { ...answer, taskId, ...(started.data?.run ? { serverRun: started.data.run } : {}) }
: answer
)
}
/** What a start's failure says about its reservation: adopt the Dispatch it names, keep it for the orphan lookup, or free it. */
function settleUnansweredStart(
runtime: OrcaRuntimeService,
start: { conversationId: string; taskId: string; key: string; error: unknown }
): void {
const db = runtime.getOrchestrationDb()
const data = errorData(start.error)
const named = typeof data?.dispatchId === 'string' ? data.dispatchId : undefined
if (named) {
recordRemoteStart(db, {
conversationId: start.conversationId,
taskId: start.taskId,
dispatchId: named,
requestKey: start.key
})
} else if (serverNeverAnswered(start.error) && data?.effectsApplied !== false) {
markRemoteStartOutcomeUnknown(db, start.taskId, start.key)
} else if (
!(start.error instanceof OrchestrationError && start.error.code === 'operation_unknown')
) {
// The server's ordinary start refused it, which a start that had launched answers with a receipt.
releaseRemotePlacement(db, start.taskId, start.key)
}
}
+16
View File
@@ -33,6 +33,7 @@ import type { RpcCallerScope } from './rpc-caller-scope'
import { runWithActiveRpcCaller } from './rpc-active-caller'
import { admitRpcRequest } from './rpc-request-admission-prologue'
import { resolveOrcadFeatureRequirement, rpcMethodFeatureWait } from './orcad-feature-readiness'
import { routeRemoteOwnedCall } from '../orchestration/federation/remote-owned/remote-owned-route'
export type DispatcherOptions = {
runtime: OrcaRuntimeService
@@ -244,6 +245,21 @@ export class RpcDispatcher {
request,
compatibility.legacyCoordinatorAuthority
)
// Before this Orca's receipts: the server that owns the worker keeps the receipt, under the caller's scoped key.
const remote = await routeRemoteOwnedCall(runtime, method, request, effectiveParams, {
...context,
orchestrationCompatibilityEvidence: request.orchestrationCompatibilityEvidence
})
if (remote.handled) {
recordRuntimeFeatureInteraction(
runtime,
request.method,
remote.result,
undefined,
request.params
)
return remote.result
}
// A legacy coordinator fingerprint outranks the transport identity; mint a fallback only when absent.
const fallbackCallerFingerprint =
context.authenticatedCallerFingerprint ??
@@ -101,6 +101,16 @@ export function checkOrchestrationAccess<P, G>(
}
}
/** The same check for a call this Orca sends on to a paired server instead of running it here. */
export function checkDeclaredOrchestrationAccess(
access: OrchestrationMethodAccessDeclaration,
params: unknown,
ctx: RpcContext
): void {
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a registered declaration was built for its method's params, which the dispatcher parsed with that method's schema.
checkOrchestrationAccess(access as OrchestrationMethodAccess<unknown, unknown>, params, ctx)
}
function declaredCaller<P>(
field: Extract<FieldOf<P>, OrchestrationCallerField> | undefined,
params: P
@@ -13,6 +13,7 @@ import '../../../../orchestration/db/dispatch-row-writer'
import '../../../../orchestration/db/attempt-outcome-projection'
import type { OrchestrationEnvironmentTransport } from '../../../../orchestration/federation/environment-transport'
import '../../../../orchestration/legacy-remote/federation-lifecycle-settlement'
import { attachmentHostCapabilities } from './federation-peer-capabilities.test-support'
import { RpcDispatcher } from '../../../dispatcher'
import { ORCHESTRATION_METHODS } from '../orchestration'
import { createFederationWorkerStartRequest as startRequest } from './federation-request.test-support'
@@ -35,7 +36,7 @@ describe('orchestration federation lifecycle settlement', () => {
workerRuntime = new OrcaRuntimeService()
workerRuntime.setOrchestrationDb(workerDb)
workerDispatcher = new RpcDispatcher({ runtime: workerRuntime, methods: ORCHESTRATION_METHODS })
workerCapabilities = [...(workerRuntime.getStatus().capabilities ?? [])]
workerCapabilities = attachmentHostCapabilities(workerRuntime)
failNextAckBeforeDelivery = false
ackAttempts = 0
transport = {
@@ -2,6 +2,7 @@ import {
ORCHESTRATION_FEDERATION_FLEET_SNAPSHOT_RUNTIME_CAPABILITY,
ORCHESTRATION_FEDERATION_RELEASE_ARCHIVE_RUNTIME_CAPABILITY,
ORCHESTRATION_FEDERATION_STRUCTURED_READ_RUNTIME_CAPABILITY,
ORCHESTRATION_REMOTE_OWNED_WORKERS_RUNTIME_CAPABILITY,
ORCHESTRATION_WORKER_AUTO_RELEASE_RUNTIME_CAPABILITY
} from '../../../../../../shared/orchestration/orchestration-runtime-capabilities'
import type { OrcaRuntimeService } from '../../../../orca-runtime'
@@ -13,6 +14,8 @@ function withheldPeerCapabilities(host: {
autoRelease: boolean
}): ReadonlySet<string> {
return new Set([
// These suites drive today's remote attachments, which a server that owns its workers never gets.
ORCHESTRATION_REMOTE_OWNED_WORKERS_RUNTIME_CAPABILITY,
...(host.newCapabilities
? []
: [
@@ -42,3 +45,10 @@ export function olderPeerStatusResponse(
_meta: { runtimeId: runtime.getRuntimeId() }
}
}
/** A host's capabilities as one that keeps today's remote attachments advertises them. */
export function attachmentHostCapabilities(runtime: OrcaRuntimeService): string[] {
return (runtime.getStatus().capabilities ?? []).filter(
(capability) => capability !== ORCHESTRATION_REMOTE_OWNED_WORKERS_RUNTIME_CAPABILITY
)
}
@@ -9,6 +9,7 @@ import '../../../../orchestration/db/messages/mailbox-routing-page'
import '../../../../orchestration/db/dispatch-row-writer'
import '../../../../orchestration/db/attempt-outcome-projection'
import type { OrchestrationEnvironmentTransport } from '../../../../orchestration/federation/environment-transport'
import { attachmentHostCapabilities } from './federation-peer-capabilities.test-support'
import { RpcDispatcher } from '../../../dispatcher'
import { ORCHESTRATION_METHODS } from '../orchestration'
import { createFederationWorkerStartRequest as startRequest } from './federation-request.test-support'
@@ -37,7 +38,7 @@ describe('orchestration federation', () => {
runtime: workerRuntime,
methods: ORCHESTRATION_METHODS
})
workerCapabilities = [...(workerRuntime.getStatus().capabilities ?? [])]
workerCapabilities = attachmentHostCapabilities(workerRuntime)
workerPeerFingerprint = 'windows_peer_fingerprint'
loseNextAckResponse = false
const transport: OrchestrationEnvironmentTransport = {
@@ -109,7 +110,7 @@ describe('orchestration federation', () => {
runtime: workerRuntime,
methods: ORCHESTRATION_METHODS
})
workerCapabilities = [...(workerRuntime.getStatus().capabilities ?? [])]
workerCapabilities = attachmentHostCapabilities(workerRuntime)
}
it('fails before remote task input when wait-for-setup fails', async () => {
@@ -12,6 +12,7 @@ import {
} from '../../../../../../shared/rpc-contract/orchestration-params'
import { TaskUpdateParams } from './task-update-params'
import { forwardRemoteTaskUpdate } from '../../../../orchestration/federation/remote-owned/remote-owned-task-update'
import { remoteForwardCallerNamespace } from '../../../../orchestration/federation/remote-owned/remote-forward-caller'
export const ORCHESTRATION_MESSAGE_METHODS = [
ORCHESTRATION_REPLY_METHOD,
@@ -126,19 +127,21 @@ export const ORCHESTRATION_MESSAGE_METHODS = [
targets: { id: 'task', run: 'run' },
rule: taskUpdateAccess
},
handler: async (params, { orchestrationCaller, orchestrationMutation, runtime }) => {
handler: async (params, ctx) => {
const { orchestrationMutation, runtime } = ctx
const db = runtime.getOrchestrationDb()
// The server sees every caller here as one pairing; the caller's own identity scopes its retry key.
const callerNamespace =
orchestrationCaller?.orcaSessionId ??
(params.callerTerminalHandle
? runtime.getTerminalPaneKey(params.callerTerminalHandle)
: null)
const forwarded = await forwardRemoteTaskUpdate(
runtime,
params,
callerNamespace && orchestrationMutation
? { callerNamespace, requestId: orchestrationMutation.requestId }
orchestrationMutation
? {
callerNamespace: remoteForwardCallerNamespace(
runtime,
ctx,
params.callerTerminalHandle
),
requestId: orchestrationMutation.requestId
}
: undefined
)
if (forwarded) {
@@ -8,7 +8,7 @@ import { RequestShowParams } from '../../../../../../shared/rpc-contract/orchest
export const ORCHESTRATION_MUTATION_REQUEST_METHODS = [
defineOrchestrationMethod({
name: 'orchestration.requestShow',
access: { kind: 'reads', targets: { request: 'request' } },
access: { kind: 'reads', server: 'on', targets: { request: 'request' } },
permission: 'workspace',
params: RequestShowParams,
// Read durable receipts without writing/mutating again; keep this recovery query outside ORCHESTRATION_MUTATION_METHODS.
@@ -79,7 +79,8 @@ export const ORCHESTRATION_WORKER_CONTROL_METHODS = [
terminal: observation.exact ? observation.terminal : null,
observation: exposeObservation(observation),
terminalResource: resource ? exposeWorkerTerminalResource(resource) : null,
autoReleaseAt: liveWorkerTerminalAutoReleaseAt(resource)
autoReleaseAt: liveWorkerTerminalAutoReleaseAt(resource),
coordinatedFrom: db.getPeerRun(dispatch.run_id)?.home_label ?? null
}
}
}),
@@ -25,7 +25,7 @@ import {
WorkerListParams
} from '../../../../../../shared/rpc-contract/orchestration-worker-release-schemas-params'
const emptyWorkerListPage = (limit: number) => ({
export const emptyWorkerListPage = (limit: number) => ({
workers: [],
counts: {},
page: { limit, total: 0, hasMore: false, nextCursor: null }
@@ -33,7 +33,7 @@ const emptyWorkerListPage = (limit: number) => ({
export const ORCHESTRATION_WORKER_LIST_METHOD = defineOrchestrationMethod({
name: 'orchestration.workerList',
access: { kind: 'reads', targets: { run: 'run' }, collection: true },
access: { kind: 'reads', server: 'on', targets: { run: 'run' }, collection: true },
permission: 'workspace',
params: WorkerListParams,
handler: async (params, { runtime }) => {
@@ -60,15 +60,62 @@ export const ORCHESTRATION_WORKER_START_METHODS = [
/** worker-start, and a follow-up sent to a finished worker, which is worker-start run for the sender. */
export async function startWorker(
startParams: z.infer<typeof WorkerStartParams>,
ctx: WorkerStartContext,
startFollowUp?: WorkerFollowUp
): Promise<unknown> {
const { runtime, orchestrationMutation, orchestrationCaller } = ctx
const prepared = await prepareWorkerStart(startParams, ctx, startFollowUp)
const { params, followUp, coordinator, bindRun, helperCreator, taskDeps, existingTask, withRun } =
prepared
const db = runtime.getOrchestrationDb()
const mode = decideWorkerStartMode({
params,
settings: readWorkerStartModeSettings(runtime)
})
if (params.on) {
// Remote workers are terminal-only; carry the mode receipt to explain structured-preference fallback.
const receipt = await startFederatedWorker({
params,
runtime,
db,
bindRunId: () => bindRun().run.id,
task: existingTask,
taskDeps,
orchestrationMutation,
callerSession: orchestrationCaller
})
await stopHelperAfterStart(runtime, receipt, orchestrationMutation)
return withRun(receipt && typeof receipt === 'object' ? { ...receipt, mode } : receipt)
}
const receipt = await startLocalWorker({
params: { ...params, timeoutMs: resolveWorkerStartReadinessTimeoutMs(params.timeoutMs) },
runtime,
db,
bindRun: () => bindRun().run,
taskDeps,
coordinator,
callerSession: orchestrationCaller,
existingTask,
orchestrationMutation,
mode,
helperCreator,
followUp
})
await stopHelperAfterStart(runtime, receipt, orchestrationMutation)
return withRun(withWorkerStartNext(receipt))
}
/** What every start checks and binds before it is placed, here or on a paired server. */
export async function prepareWorkerStart(
params: z.infer<typeof WorkerStartParams>,
{
runtime,
orchestrationMutation,
orchestrationCompatibilityEvidence,
orchestrationCaller
}: WorkerStartContext,
}: Omit<WorkerStartContext, 'orchestrationMutation'>,
followUp?: WorkerFollowUp
): Promise<unknown> {
) {
if (!isWorkerStartTimeoutWithinTimerLimit(params.timeoutMs)) {
throw new OrchestrationError(
'invalid_argument',
@@ -103,7 +150,6 @@ export async function startWorker(
{ effectsApplied: false }
)
}
const readinessTimeoutMs = resolveWorkerStartReadinessTimeoutMs(params.timeoutMs)
const db = runtime.getOrchestrationDb()
const coordinator = resolveOrchestrationCaller(runtime, {
callerTerminalHandle: params.from,
@@ -200,39 +246,14 @@ export async function startWorker(
followUp = { ...followUp, reopen: placement.reopen, earlierWork: placement.earlierWork }
await assertWorkerStartTaskSpecWithinPromptBudget(spec)
}
const mode = decideWorkerStartMode({
return {
params,
settings: readWorkerStartModeSettings(runtime)
})
if (params.on) {
// Remote workers are terminal-only; carry the mode receipt to explain structured-preference fallback.
const receipt = await startFederatedWorker({
params,
runtime,
db,
bindRunId: () => bindRun().run.id,
task: existingTask,
taskDeps,
orchestrationMutation,
callerSession: orchestrationCaller
})
await stopHelperAfterStart(runtime, receipt, orchestrationMutation)
return withRun(receipt && typeof receipt === 'object' ? { ...receipt, mode } : receipt)
}
const receipt = await startLocalWorker({
params: { ...params, timeoutMs: readinessTimeoutMs },
runtime,
db,
bindRun: () => bindRun().run,
taskDeps,
followUp,
coordinator,
callerSession: orchestrationCaller,
existingTask,
orchestrationMutation,
mode,
bindRun,
helperCreator,
followUp
})
await stopHelperAfterStart(runtime, receipt, orchestrationMutation)
return withRun(withWorkerStartNext(receipt))
taskDeps,
existingTask,
withRun
}
}
@@ -40,6 +40,9 @@ export const ORCHESTRATION_CONTRACT_VERSION = 1 as const
export const ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY = 'orchestration.contract.v1' as const
/** Receives agent mail a paired Orca carries for its agents, and queues replies for it to pull. */
export const ORCHESTRATION_PEER_MAIL_RUNTIME_CAPABILITY = 'orchestration.peer-mail.v1' as const
/** Runs the workers a paired Orca places here as its own, coordinated from that Orca (wire v1). */
export const ORCHESTRATION_REMOTE_OWNED_WORKERS_RUNTIME_CAPABILITY =
'orchestration.remote-owned-workers.v1' as const
export const ORCHESTRATION_RUNTIME_CAPABILITIES = [
ORCHESTRATION_FEDERATION_RUNTIME_CAPABILITY,
@@ -58,5 +61,6 @@ export const ORCHESTRATION_RUNTIME_CAPABILITIES = [
ORCHESTRATION_WORKER_TURN_STOP_REPORTS_RUNTIME_CAPABILITY,
ORCHESTRATION_ASK_RETURNS_AT_ONCE_RUNTIME_CAPABILITY,
ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY,
ORCHESTRATION_PEER_MAIL_RUNTIME_CAPABILITY
ORCHESTRATION_PEER_MAIL_RUNTIME_CAPABILITY,
ORCHESTRATION_REMOTE_OWNED_WORKERS_RUNTIME_CAPABILITY
] as const
@@ -1,4 +1,8 @@
import { z } from 'zod'
import { requiredString } from './rpc-param-primitives'
export const RequestShowParams = z.object({ request: requiredString('Missing --request') })
export const RequestShowParams = z.object({
request: requiredString('Missing --request'),
// The paired server a forwarded request ran on; its receipt is kept there.
on: z.string().min(1).optional()
})
@@ -21,5 +21,7 @@ export const WorkerListParams = z.object({
cursor: z.string().min(1).max(2_048).optional(),
limit: z.number().int().min(1).max(ORCHESTRATION_FLEET_PAGE_MAX).optional(),
includeRemote: z.boolean().optional(),
paginate: z.boolean().optional()
paginate: z.boolean().optional(),
// A paired server's own list, paged by its cursor.
on: z.string().min(1).optional()
})