mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 02:47:34 +00:00
feat(orchestration): run commands about a paired server's workers there, and switch new pairs to it
This commit is contained in:
@@ -1,14 +1,18 @@
|
||||
import type { CommandHandler } from '../../dispatch'
|
||||
import { printOrchestrationReceipt } from './agent-view'
|
||||
import { getRequiredStringFlag } from '../../flags'
|
||||
import { getOptionalStringFlag, getRequiredStringFlag } from '../../flags'
|
||||
import { RuntimeClientError } from '../../runtime-client'
|
||||
import type { OrchestrationMutationRequestShowResult } from '../../../shared/orchestration/orchestration-mutation-request'
|
||||
|
||||
export const ORCHESTRATION_REQUEST_SHOW_HANDLER: Record<string, CommandHandler> = {
|
||||
'orchestration request-show': async ({ flags, client, json }) => {
|
||||
const request = getRequiredStringFlag(flags, 'request')
|
||||
const on = getOptionalStringFlag(flags, 'on')
|
||||
const result = await client
|
||||
.call<OrchestrationMutationRequestShowResult>('orchestration.requestShow', { request })
|
||||
.call<OrchestrationMutationRequestShowResult>('orchestration.requestShow', {
|
||||
request,
|
||||
...(on ? { on } : {})
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
if (error instanceof RuntimeClientError && error.code === 'method_not_found') {
|
||||
throw new RuntimeClientError(
|
||||
|
||||
@@ -44,6 +44,8 @@ export type WorkerShowResult = {
|
||||
server?: RemoteWorkerServer & { reachable: false }
|
||||
lastReport?: string | null
|
||||
autoReleaseAt?: string | null
|
||||
/** The paired Orca whose Run here coordinates this worker. */
|
||||
coordinatedFrom?: string | null
|
||||
}
|
||||
|
||||
/** A worker a paired server runs, as this Orca names that server and last heard from it. */
|
||||
@@ -70,6 +72,7 @@ type WorkerShowFields = {
|
||||
waitingOnHuman?: string
|
||||
lastError?: string
|
||||
autoReleaseAt?: string
|
||||
coordinatedFrom?: string
|
||||
next?: string
|
||||
}
|
||||
|
||||
@@ -108,6 +111,7 @@ export const WORKER_SHOW_VIEW: OrchestrationAgentView<WorkerShowResult, WorkerSh
|
||||
: undefined,
|
||||
lastError: value.worker.lastError ?? undefined,
|
||||
autoReleaseAt: value.autoReleaseAt ?? undefined,
|
||||
coordinatedFrom: value.coordinatedFrom ?? undefined,
|
||||
next: value.projection ? renderOrchestrationArgv(value.projection.nextAction.argv) : undefined
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,6 +133,7 @@ export const ORCHESTRATION_WORKER_TERMINAL_HANDLERS: Record<string, CommandHandl
|
||||
run: scope.run,
|
||||
terminalState,
|
||||
...(flags.has('include-remote') ? { includeRemote: true } : {}),
|
||||
on: getOptionalStringFlag(flags, 'on'),
|
||||
cursor: getOptionalStringFlag(flags, 'cursor'),
|
||||
limit: getOptionalPositiveIntegerFlag(flags, 'limit')
|
||||
})
|
||||
|
||||
@@ -35,7 +35,16 @@ export function orchestrationMutationRecoveryError(
|
||||
// Without a dispatch id, query by request id to inspect the outcome before retrying a mutation.
|
||||
const queryCommand = dispatchId
|
||||
? [executable, 'orchestration', 'worker-show', '--dispatch', dispatchId, '--json']
|
||||
: [executable, 'orchestration', 'request-show', '--request', requestId, '--json']
|
||||
: [
|
||||
executable,
|
||||
'orchestration',
|
||||
'request-show',
|
||||
'--request',
|
||||
requestId,
|
||||
// A command forwarded to a paired server keeps its receipt there.
|
||||
...(typeof data?.server === 'string' ? ['--on', data.server] : []),
|
||||
'--json'
|
||||
]
|
||||
const recovery = {
|
||||
orchestrationRequestId: requestId,
|
||||
...(dispatchId ? { dispatchId } : {}),
|
||||
|
||||
@@ -145,14 +145,23 @@ export const ORCHESTRATION_WORKER_COMMAND_SPECS: CommandSpec[] = [
|
||||
path: ['orchestration', 'worker-list'],
|
||||
summary: 'List supervised worker terminal resource accounting',
|
||||
usage:
|
||||
'orca orchestration worker-list [--run <run_id>] [--terminal-state <active|reclaimable|retained|release_pending|release_unknown|released>] [--include-remote] [--cursor <cursor>] [--limit <1-100>] [--json]',
|
||||
allowedFlags: [...GLOBAL_FLAGS, 'run', 'terminal-state', 'include-remote', 'cursor', 'limit'],
|
||||
'orca orchestration worker-list [--run <run_id>] [--terminal-state <active|reclaimable|retained|release_pending|release_unknown|released>] [--include-remote] [--on <server>] [--cursor <cursor>] [--limit <1-100>] [--json]',
|
||||
allowedFlags: [
|
||||
...GLOBAL_FLAGS,
|
||||
'run',
|
||||
'terminal-state',
|
||||
'include-remote',
|
||||
'on',
|
||||
'cursor',
|
||||
'limit'
|
||||
],
|
||||
notes: [
|
||||
'Terminal state is process accounting and is reported separately from Task status; a completed Task can still own a live terminal.',
|
||||
'A reclaimable worker with resource.autoReleaseAt is released automatically at that time unless it is reused with worker-start --terminal or kept open (worker-keep-open, worker-start --keep-open, or the user).',
|
||||
'Context-only Dispatches created by orchestration dispatch are included as unsupervised with terminal state retained.',
|
||||
'Returns at most 100 local rows, newest first, by default; --include-remote adds connected-server observations when the host supports fleet listing. Continue with the opaque nextCursor value unchanged.',
|
||||
'Without --run the list is scoped to the Run bound to the calling terminal, and to every Run when there is no binding; the receipt reports which in scope.source (flag, bound, or all).'
|
||||
'Without --run the list is scoped to the Run bound to the calling terminal, and to every Run when there is no binding; the receipt reports which in scope.source (flag, bound, or all).',
|
||||
"--on <server> lists that paired server's own workers for your Run there (or all of them without a Run), paged with its own cursor."
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
@@ -209,12 +209,13 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [
|
||||
{
|
||||
path: ['orchestration', 'request-show'],
|
||||
summary: 'Ask whether one orchestration mutation request already took effect',
|
||||
usage: 'orca orchestration request-show --request <request_id> [--json]',
|
||||
allowedFlags: [...GLOBAL_FLAGS, 'request'],
|
||||
usage: 'orca orchestration request-show --request <request_id> [--on <server>] [--json]',
|
||||
allowedFlags: [...GLOBAL_FLAGS, 'request', 'on'],
|
||||
notes: [
|
||||
'Read-only: it never starts, retries, or settles anything, so it is safe to run after any lost response.',
|
||||
'completed means the mutation landed and --retry-request replays the recorded outcome instead of starting a second one. pending means the original mutation is still running or Orca restarted before recording its outcome; wait for a live original command, otherwise replay with --retry-request.',
|
||||
'absent means this runtime holds no receipt for that request under your caller identity: it never arrived, it failed before recording anything, or the receipt was pruned. Absent is not proof that nothing happened.'
|
||||
'absent means this runtime holds no receipt for that request under your caller identity: it never arrived, it failed before recording anything, or the receipt was pruned. Absent is not proof that nothing happened.',
|
||||
'A command about a worker on a paired server keeps its receipt on that server; --on <server> asks there.'
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import type { OrcaRuntimeService } from '../../../orca-runtime'
|
||||
import type { RpcContext } from '../../../rpc/core'
|
||||
import { remoteForwardRequestId } from './remote-owned-calls'
|
||||
|
||||
/**
|
||||
* Who is asking, as a retry of the same request must find it again: the caller's session, else
|
||||
* its terminal's pane, which survives a restart, else the person at this Orca outside any terminal.
|
||||
* Never one shared identity for every terminal, so two callers reusing an ID never collide.
|
||||
*/
|
||||
export function remoteForwardCallerNamespace(
|
||||
runtime: Pick<OrcaRuntimeService, 'getOrchestrationDb' | 'getTerminalPaneKey'>,
|
||||
ctx: Pick<RpcContext, 'orchestrationCaller' | 'orchestrationCompatibilityEvidence'>,
|
||||
callerHandle: string | undefined
|
||||
): string {
|
||||
const session = ctx.orchestrationCaller?.orcaSessionId
|
||||
if (session) {
|
||||
return `session:${session}`
|
||||
}
|
||||
const handle = callerHandle ?? ctx.orchestrationCompatibilityEvidence?.terminalHandle
|
||||
const pane =
|
||||
ctx.orchestrationCompatibilityEvidence?.paneKey ??
|
||||
(handle ? runtime.getTerminalPaneKey(handle) : null)
|
||||
return pane
|
||||
? `pane:${pane}`
|
||||
: `local:${runtime.getOrchestrationDb().getOrCreateLocalMutationCallerFingerprint()}`
|
||||
}
|
||||
|
||||
/** The key a forwarded call goes out under, and how to say it back in the caller's own terms. */
|
||||
export type RemoteForwardKey = { key: string; requestId: string }
|
||||
|
||||
export function remoteForwardKey(
|
||||
runtime: Pick<OrcaRuntimeService, 'getOrchestrationDb' | 'getTerminalPaneKey'>,
|
||||
ctx: Pick<RpcContext, 'orchestrationCaller' | 'orchestrationCompatibilityEvidence'>,
|
||||
callerHandle: string | undefined,
|
||||
requestId: string | undefined
|
||||
): RemoteForwardKey | undefined {
|
||||
return requestId
|
||||
? {
|
||||
key: remoteForwardRequestId(
|
||||
remoteForwardCallerNamespace(runtime, ctx, callerHandle),
|
||||
requestId
|
||||
),
|
||||
requestId
|
||||
}
|
||||
: undefined
|
||||
}
|
||||
@@ -45,7 +45,8 @@ const NO_ANSWER_CODES = new Set([
|
||||
'remote_link_closed'
|
||||
])
|
||||
|
||||
function serverAnswered(error: unknown): boolean {
|
||||
/** Whether the server got the call and answered it, rather than this Orca never reaching it. */
|
||||
export function serverAnswered(error: unknown): boolean {
|
||||
return error instanceof OrchestrationError && !NO_ANSWER_CODES.has(error.code)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import { OrchestrationError } from '../../orchestration-error'
|
||||
import { serverAnswered } from './remote-owed-actions'
|
||||
import type { RemoteForwardKey } from './remote-forward-caller'
|
||||
|
||||
/** The server got nothing, or never answered, so whatever the call did there is unknown here. */
|
||||
export function serverNeverAnswered(error: unknown): boolean {
|
||||
return !serverAnswered(error)
|
||||
}
|
||||
|
||||
export function errorData(error: unknown): Record<string, unknown> | undefined {
|
||||
const data = error instanceof OrchestrationError ? error.data : undefined
|
||||
return data && typeof data === 'object' && !Array.isArray(data)
|
||||
? // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a non-array object's own fields, read as unknowns.
|
||||
(data as Record<string, unknown>)
|
||||
: undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* The server's answer, or its error, in the caller's own terms: the caller knows its request ID,
|
||||
* not the scoped key the server saw, and an error names the server it came from so recovery
|
||||
* (`request-show --on`) asks there.
|
||||
*/
|
||||
export function inServerTerms<T>(
|
||||
value: T,
|
||||
forward: RemoteForwardKey | undefined,
|
||||
server: string
|
||||
): T {
|
||||
const swap = (item: unknown): unknown => {
|
||||
if (typeof item === 'string') {
|
||||
return forward ? item.split(forward.key).join(forward.requestId) : item
|
||||
}
|
||||
if (Array.isArray(item)) {
|
||||
return item.map(swap)
|
||||
}
|
||||
if (item && typeof item === 'object') {
|
||||
return Object.fromEntries(Object.entries(item).map(([field, inner]) => [field, swap(inner)]))
|
||||
}
|
||||
return item
|
||||
}
|
||||
if (value instanceof OrchestrationError) {
|
||||
const data = swap(errorData(value) ?? {})
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: an error stays an OrchestrationError with the same code.
|
||||
return new OrchestrationError(value.code, String(swap(value.message)), {
|
||||
...(data as Record<string, unknown>),
|
||||
server
|
||||
}) as T
|
||||
}
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: only string contents change; every shape is kept.
|
||||
return swap(value) as T
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import type { OrcaRuntimeService } from '../../../orca-runtime'
|
||||
import type { RpcContext, RpcMethod, RpcRequest } from '../../../rpc/core'
|
||||
import {
|
||||
checkDeclaredOrchestrationAccess,
|
||||
type OrchestrationMethodAccessDeclaration
|
||||
} from '../../../rpc/methods/orchestration/access/orchestration-method-access'
|
||||
import { emptyWorkerListPage } from '../../../rpc/methods/orchestration/worker/worker-list-method'
|
||||
import { ORCHESTRATION_FLEET_PAGE_MAX } from '../../../../../shared/orchestration/fleet/orchestration-fleet-projection'
|
||||
import { WorkerStartParams } from '../../../../../shared/rpc-contract/orchestration-worker-start-params'
|
||||
import type { OrchestrationDb } from '../../db/orchestration-db'
|
||||
import { runLifecycleWriteTransaction } from '../../db/lifecycle-write-transaction-runner'
|
||||
import {
|
||||
getRemoteDispatch,
|
||||
oweRemoteDispatchStop
|
||||
} from '../../db/remote-owned/remote-dispatch-records'
|
||||
import { adoptRemoteAttempt } from '../../db/remote-owned/remote-placements'
|
||||
import {
|
||||
getOpenRemoteRunLink,
|
||||
getRemoteRunLink,
|
||||
type RemoteRunLinkRow
|
||||
} from '../../db/remote-owned/remote-run-links'
|
||||
import { callRemoteOwnedServer } from './remote-owned-calls'
|
||||
import { remoteForwardKey, type RemoteForwardKey } from './remote-forward-caller'
|
||||
import { inServerTerms, serverNeverAnswered } from './remote-owned-answers'
|
||||
import { sendOwedRemoteStop } from './remote-owed-actions'
|
||||
import { serverOwnsPlacedWorkers, startRemoteOwnedWorker } from './remote-owned-start'
|
||||
|
||||
type Routed = { handled: false } | { handled: true; result: unknown }
|
||||
|
||||
const LOCAL: Routed = { handled: false }
|
||||
// Long enough for a send that starts a follow-up worker there and waits for it to be ready.
|
||||
const FORWARD_TIMEOUT_MS = 180_000
|
||||
// What `--on <server>` sends there; send and reply --on stay peer mail.
|
||||
const SERVER_CALLS: ReadonlySet<string> = new Set([
|
||||
'orchestration.workerStart',
|
||||
'orchestration.workerList',
|
||||
'orchestration.requestShow'
|
||||
])
|
||||
|
||||
type Values = Record<string, unknown>
|
||||
|
||||
/**
|
||||
* Before this Orca's receipts: a command about a worker a paired server owns runs there, by that
|
||||
* server's ordinary handler, as its Run's coordinator. The worker is found from the subjects the
|
||||
* method declares; the caller passes the method's one access check here first, as it would for a
|
||||
* worker of its own. A paired Orca's calls always run here: pairing is one-way.
|
||||
*/
|
||||
export async function routeRemoteOwnedCall(
|
||||
runtime: OrcaRuntimeService,
|
||||
method: RpcMethod,
|
||||
request: RpcRequest,
|
||||
params: unknown,
|
||||
ctx: RpcContext
|
||||
): Promise<Routed> {
|
||||
const access = method.orchestrationAccess
|
||||
if (!access || access.kind === 'localOnly' || ctx.orchestrationCaller?.kind === 'peer') {
|
||||
return LOCAL
|
||||
}
|
||||
const values: Values = params && typeof params === 'object' ? { ...params } : {}
|
||||
const db = runtime.getOrchestrationDb()
|
||||
const on = access.server && stringField(values, access.server)
|
||||
if (on !== undefined) {
|
||||
return routeServerCall(runtime, request, access, values, ctx, on)
|
||||
}
|
||||
const subject = remoteSubject(db, access, values)
|
||||
if (!subject) {
|
||||
return LOCAL
|
||||
}
|
||||
checkDeclaredOrchestrationAccess(access, params, ctx)
|
||||
const forward = remoteForwardKey(
|
||||
runtime,
|
||||
ctx,
|
||||
access.caller && stringField(values, access.caller),
|
||||
request.orchestrationRequestId
|
||||
)
|
||||
if (request.method === 'orchestration.workerStop' && subject.dispatchId) {
|
||||
return { handled: true, result: await stopRemoteWorker(runtime, subject, forward) }
|
||||
}
|
||||
return {
|
||||
handled: true,
|
||||
result: await forwardToLink(runtime, subject.link, request.method, access, values, forward)
|
||||
}
|
||||
}
|
||||
|
||||
/** `--on <server>`: a start, a list, or a receipt lookup there. */
|
||||
async function routeServerCall(
|
||||
runtime: OrcaRuntimeService,
|
||||
request: RpcRequest,
|
||||
access: OrchestrationMethodAccessDeclaration,
|
||||
values: Values,
|
||||
ctx: RpcContext,
|
||||
on: string
|
||||
): Promise<Routed> {
|
||||
if (!SERVER_CALLS.has(request.method)) {
|
||||
return LOCAL
|
||||
}
|
||||
checkDeclaredOrchestrationAccess(access, values, ctx)
|
||||
const server = runtime.resolveOrchestrationWorkerServer(on)
|
||||
const db = runtime.getOrchestrationDb()
|
||||
if (request.method === 'orchestration.workerStart') {
|
||||
if (!(await serverOwnsPlacedWorkers(runtime, server))) {
|
||||
// An older server: today's remote attachment, the one legacy branch for new work.
|
||||
return LOCAL
|
||||
}
|
||||
return {
|
||||
handled: true,
|
||||
result: await startRemoteOwnedWorker(
|
||||
runtime,
|
||||
WorkerStartParams.parse(values),
|
||||
ctx,
|
||||
request.orchestrationRequestId
|
||||
)
|
||||
}
|
||||
}
|
||||
if (request.method === 'orchestration.workerList') {
|
||||
const runId = stringField(values, 'run')
|
||||
// A Run lists what it placed there, as its coordinator; no Run lists the server as a whole.
|
||||
const link = runId ? openLinkFor(db, runId, server.environmentId) : undefined
|
||||
if (runId && !link) {
|
||||
return {
|
||||
handled: true,
|
||||
result: emptyWorkerListPage(numberField(values, 'limit') ?? ORCHESTRATION_FLEET_PAGE_MAX)
|
||||
}
|
||||
}
|
||||
const forwarded = paramsForServer(access, values)
|
||||
return {
|
||||
handled: true,
|
||||
result: link
|
||||
? await forwardToLink(runtime, link, request.method, access, values)
|
||||
: await runtime.callOrchestrationWorkerServer(
|
||||
server.environmentId,
|
||||
request.method,
|
||||
forwarded,
|
||||
FORWARD_TIMEOUT_MS,
|
||||
undefined,
|
||||
{ expectedEnvironmentPairingRevision: server.pairingRevision }
|
||||
)
|
||||
}
|
||||
}
|
||||
{
|
||||
// The server keyed the request by this caller's scoped key, as the forward that sent it did.
|
||||
const forward = remoteForwardKey(
|
||||
runtime,
|
||||
ctx,
|
||||
undefined,
|
||||
stringField(values, 'request') ?? randomUUID()
|
||||
)!
|
||||
const answer = await runtime.callOrchestrationWorkerServer(
|
||||
server.environmentId,
|
||||
request.method,
|
||||
{ request: forward.key },
|
||||
FORWARD_TIMEOUT_MS,
|
||||
undefined,
|
||||
{ expectedEnvironmentPairingRevision: server.pairingRevision }
|
||||
)
|
||||
return { handled: true, result: inServerTerms(answer, forward, server.name) }
|
||||
}
|
||||
}
|
||||
|
||||
type RemoteSubject = { link: RemoteRunLinkRow; dispatchId?: string }
|
||||
|
||||
/** The worker or question a call names, when a paired server owns it. Tasks stay in this Orca's graph. */
|
||||
function remoteSubject(
|
||||
db: OrchestrationDb,
|
||||
access: OrchestrationMethodAccessDeclaration,
|
||||
values: Values
|
||||
): RemoteSubject | undefined {
|
||||
for (const [field, kind] of Object.entries(access.targets)) {
|
||||
const value = stringField(values, field)
|
||||
const dispatchId =
|
||||
kind === 'dispatch'
|
||||
? value
|
||||
: kind === 'party' && value?.startsWith('dispatch:')
|
||||
? value.slice('dispatch:'.length)
|
||||
: undefined
|
||||
const record = dispatchId ? getRemoteDispatch(db, dispatchId) : undefined
|
||||
const question =
|
||||
kind === 'message' && value
|
||||
? // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SELECT of one column returns exactly that column.
|
||||
(db.db
|
||||
.prepare('SELECT conversation_id FROM remote_owned_questions WHERE message_id = ?')
|
||||
.get(value) as { conversation_id: string } | undefined)
|
||||
: undefined
|
||||
const conversationId = record?.conversation_id ?? question?.conversation_id
|
||||
const link = conversationId ? getRemoteRunLink(db, conversationId) : undefined
|
||||
if (link) {
|
||||
return { link, dispatchId: record?.dispatch_id }
|
||||
}
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
/** This Orca's caller, Run and server fields mean nothing there: the server names the caller itself. */
|
||||
function paramsForServer(access: OrchestrationMethodAccessDeclaration, values: Values): Values {
|
||||
const local = new Set<string>(
|
||||
[
|
||||
access.caller,
|
||||
access.callerPane,
|
||||
access.server,
|
||||
...Object.entries(access.targets)
|
||||
.filter(([, kind]) => kind === 'run')
|
||||
.map(([field]) => field)
|
||||
].filter((field): field is string => field !== undefined)
|
||||
)
|
||||
return Object.fromEntries(Object.entries(values).filter(([field]) => !local.has(field)))
|
||||
}
|
||||
|
||||
async function forwardToLink(
|
||||
runtime: OrcaRuntimeService,
|
||||
link: RemoteRunLinkRow,
|
||||
method: string,
|
||||
access: OrchestrationMethodAccessDeclaration,
|
||||
values: Values,
|
||||
forward?: RemoteForwardKey
|
||||
): Promise<unknown> {
|
||||
const server = runtime.resolveOrchestrationWorkerServer(link.environment_id).name
|
||||
try {
|
||||
const answer = await callRemoteOwnedServer(runtime, link, {
|
||||
method,
|
||||
params: paramsForServer(access, values),
|
||||
timeoutMs: FORWARD_TIMEOUT_MS,
|
||||
...(forward ? { requestId: forward.key } : {}),
|
||||
whileLink: ['open']
|
||||
})
|
||||
seedNamedDispatches(runtime.getOrchestrationDb(), link, answer)
|
||||
return inServerTerms(answer, forward, server)
|
||||
} catch (error) {
|
||||
throw inServerTerms(error, forward, server)
|
||||
}
|
||||
}
|
||||
|
||||
/** A stop is owed until the server proves it; the ordinary stop answers when it can, and the pull loop resends otherwise. */
|
||||
async function stopRemoteWorker(
|
||||
runtime: OrcaRuntimeService,
|
||||
subject: RemoteSubject,
|
||||
forward: RemoteForwardKey | undefined
|
||||
): Promise<unknown> {
|
||||
const db = runtime.getOrchestrationDb()
|
||||
const dispatchId = subject.dispatchId!
|
||||
const server = runtime.resolveOrchestrationWorkerServer(subject.link.environment_id).name
|
||||
const owed = oweRemoteDispatchStop(db, dispatchId, forward?.key ?? `stop:${randomUUID()}`)
|
||||
runtime.ensureOrchestrationFederationRelay(subject.link.run_id)
|
||||
try {
|
||||
return inServerTerms(
|
||||
await sendOwedRemoteStop(runtime, owed?.record ?? getRemoteDispatch(db, dispatchId)!),
|
||||
forward,
|
||||
server
|
||||
)
|
||||
} catch (error) {
|
||||
if (!serverNeverAnswered(error) || !getOpenRemoteRunLink(db, subject.link.conversation_id)) {
|
||||
throw inServerTerms(error, forward, server)
|
||||
}
|
||||
return {
|
||||
dispatchId,
|
||||
state: 'stop_requested',
|
||||
server,
|
||||
next: `${server} did not answer, so the stop is owed: this Orca sends it again until ${server} confirms the worker stopped.`
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** A Dispatch the server's answer names (a follow-up it started) is its; record it so commands about it route there. */
|
||||
function seedNamedDispatches(db: OrchestrationDb, link: RemoteRunLinkRow, answer: unknown): void {
|
||||
const named = answer && typeof answer === 'object' ? (answer as Values) : {}
|
||||
const dispatchId = stringField(named, 'dispatchId')
|
||||
const taskId = stringField(named, 'taskId')
|
||||
if (!dispatchId || !taskId || getRemoteDispatch(db, dispatchId)) {
|
||||
return
|
||||
}
|
||||
runLifecycleWriteTransaction(db.db, 'seed_remote_dispatch', () => {
|
||||
const open = getOpenRemoteRunLink(db, link.conversation_id)
|
||||
if (open) {
|
||||
adoptRemoteAttempt(db, open, { taskId, dispatchId })
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function openLinkFor(
|
||||
db: OrchestrationDb,
|
||||
runId: string,
|
||||
environmentId: string
|
||||
): RemoteRunLinkRow | undefined {
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SELECT * over this table returns exactly its row shape.
|
||||
return db.db
|
||||
.prepare(
|
||||
`SELECT * FROM remote_run_links WHERE run_id = ? AND environment_id = ? AND state = 'open'`
|
||||
)
|
||||
.get(runId, environmentId) as RemoteRunLinkRow | undefined
|
||||
}
|
||||
|
||||
function stringField(values: Values, field: string): string | undefined {
|
||||
const value = values[field]
|
||||
return typeof value === 'string' && value !== '' ? value : undefined
|
||||
}
|
||||
|
||||
function numberField(values: Values, field: string): number | undefined {
|
||||
const value = values[field]
|
||||
return typeof value === 'number' ? value : undefined
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
import { createHash, randomUUID } from 'node:crypto'
|
||||
import { z } from 'zod'
|
||||
import type { WorkerStartParams } from '../../../../../shared/rpc-contract/orchestration-worker-start-params'
|
||||
import { ORCHESTRATION_REMOTE_OWNED_WORKERS_RUNTIME_CAPABILITY } from '../../../../../shared/orchestration/orchestration-runtime-capabilities'
|
||||
import { ORCHESTRATION_CONTRACT_PREFLIGHT_TIMEOUT_MS } from '../../../../../shared/orchestration/orchestration-timing-budgets'
|
||||
import type { RuntimeStatus } from '../../../../../shared/runtime-session-contracts'
|
||||
import type { OrcaRuntimeService } from '../../../orca-runtime'
|
||||
import type { RpcContext } from '../../../rpc/core'
|
||||
import { prepareWorkerStart } from '../../../rpc/methods/orchestration/worker/workers'
|
||||
import { resolveFederatedWorkerStartBudgets } from '../../../rpc/methods/orchestration/worker/worker-start-budgets'
|
||||
import { OrchestrationError } from '../../orchestration-error'
|
||||
import { prepareWorkerStartCard } from '../../workers/launch/worker-start-card'
|
||||
import { getOrchestrationPeerCapabilityCache } from '../orchestration-peer-capability-cache'
|
||||
import type { OrchestrationWorkerServer } from '../environment-transport'
|
||||
import {
|
||||
markRemoteStartOutcomeUnknown,
|
||||
recordRemoteStart,
|
||||
releaseRemotePlacement,
|
||||
reserveRemotePlacement
|
||||
} from '../../db/remote-owned/remote-placements'
|
||||
import { openRemoteRunLink } from '../../db/remote-owned/remote-run-links'
|
||||
import { callRemoteOwnedServer } from './remote-owned-calls'
|
||||
import { remoteForwardKey } from './remote-forward-caller'
|
||||
import { errorData, inServerTerms, serverNeverAnswered } from './remote-owned-answers'
|
||||
|
||||
type StartParams = z.infer<typeof WorkerStartParams>
|
||||
type StartContext = Pick<
|
||||
RpcContext,
|
||||
'runtime' | 'orchestrationCompatibilityEvidence' | 'orchestrationCaller'
|
||||
>
|
||||
|
||||
/** Whether the server runs the workers placed on it as its own; an older one keeps today's remote attachments. */
|
||||
export async function serverOwnsPlacedWorkers(
|
||||
runtime: OrcaRuntimeService,
|
||||
server: OrchestrationWorkerServer
|
||||
): Promise<boolean> {
|
||||
const decision = await getOrchestrationPeerCapabilityCache(runtime).resolve({
|
||||
peerFingerprint: server.peerFingerprint,
|
||||
expectedRuntimeEpoch: null,
|
||||
capability: ORCHESTRATION_REMOTE_OWNED_WORKERS_RUNTIME_CAPABILITY,
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: status.get answers RuntimeStatus; the cache reads only its capabilities and runtime ID.
|
||||
probe: async () =>
|
||||
(await runtime.callOrchestrationWorkerServer(
|
||||
server.environmentId,
|
||||
'status.get',
|
||||
undefined,
|
||||
ORCHESTRATION_CONTRACT_PREFLIGHT_TIMEOUT_MS,
|
||||
undefined,
|
||||
{ expectedEnvironmentPairingRevision: server.pairingRevision }
|
||||
)) as RuntimeStatus
|
||||
})
|
||||
return decision.supported
|
||||
}
|
||||
|
||||
/** One ID for the Task here and the start there, from the start's retry key, so a retry finds both. */
|
||||
function adoptedTaskId(key: string): string {
|
||||
return `task_${createHash('sha256').update(`task\n${key}`).digest('hex').slice(0, 12)}`
|
||||
}
|
||||
|
||||
const StartedOnServer = z.object({
|
||||
dispatchId: z.string().min(1),
|
||||
run: z.object({ id: z.string().min(1) }).optional(),
|
||||
agent: z.string().nullable().optional()
|
||||
})
|
||||
|
||||
/**
|
||||
* worker-start --on a server that owns its workers: the Task stays in this Orca's graph, and the
|
||||
* server starts an ordinary worker for it as the Run's coordinator. Synchronous, launch then
|
||||
* record: the Task is reserved for the server first, and becomes its attempt once it answers.
|
||||
*/
|
||||
export async function startRemoteOwnedWorker(
|
||||
runtime: OrcaRuntimeService,
|
||||
params: StartParams,
|
||||
ctx: StartContext,
|
||||
callerRequestId: string | undefined
|
||||
): Promise<unknown> {
|
||||
if (params.terminal) {
|
||||
throw new OrchestrationError(
|
||||
'invalid_argument',
|
||||
'--terminal names a terminal on this Orca, so it cannot be used with --on. No effects were applied.',
|
||||
{ effectsApplied: false }
|
||||
)
|
||||
}
|
||||
const { coordinator, bindRun, taskDeps, existingTask, withRun } = await prepareWorkerStart(
|
||||
params,
|
||||
ctx
|
||||
)
|
||||
const db = runtime.getOrchestrationDb()
|
||||
const server = runtime.resolveOrchestrationWorkerServer(params.on!)
|
||||
const { run } = bindRun()
|
||||
// An older CLI sends no request ID; the start still runs, but a retry cannot find it.
|
||||
const forward = remoteForwardKey(runtime, ctx, params.from, callerRequestId ?? randomUUID())!
|
||||
const taskId = existingTask?.id ?? adoptedTaskId(forward.key)
|
||||
const budgets = resolveFederatedWorkerStartBudgets(params.timeoutMs)
|
||||
const link = openRemoteRunLink(db, run.id, server)
|
||||
const { task } = reserveRemotePlacement(db, {
|
||||
link,
|
||||
taskId,
|
||||
requestKey: forward.key,
|
||||
runtimeId: runtime.getRuntimeId(),
|
||||
retryOf: params.retryOf,
|
||||
create: existingTask
|
||||
? undefined
|
||||
: {
|
||||
spec: params.spec!,
|
||||
taskTitle: params.taskTitle,
|
||||
deps: taskDeps,
|
||||
parentId: params.parent,
|
||||
createdByTerminalHandle: coordinator?.terminalHandle ?? undefined,
|
||||
createdByPaneKey: coordinator?.paneKey ?? undefined,
|
||||
createdByRunGeneration: run.consumer_generation
|
||||
}
|
||||
})
|
||||
// Its queue has a reader from the start, so mail sent before the answer still comes home.
|
||||
runtime.ensureOrchestrationFederationRelay(run.id)
|
||||
const {
|
||||
from: _from,
|
||||
run: _run,
|
||||
on: _on,
|
||||
task: _task,
|
||||
deps: _deps,
|
||||
parent: _parent,
|
||||
...options
|
||||
} = params
|
||||
let receipt: unknown
|
||||
try {
|
||||
receipt = await callRemoteOwnedServer(runtime, link, {
|
||||
method: 'orchestration.workerStart',
|
||||
params: {
|
||||
...options,
|
||||
spec: task.spec,
|
||||
...(task.task_title ? { taskTitle: task.task_title } : {}),
|
||||
adoptTaskId: task.id
|
||||
},
|
||||
timeoutMs: Math.max(budgets.attachDeadlineMs - Date.now(), 1),
|
||||
requestId: forward.key,
|
||||
whileLink: ['open']
|
||||
})
|
||||
} catch (error) {
|
||||
settleUnansweredStart(runtime, {
|
||||
conversationId: link.conversation_id,
|
||||
taskId,
|
||||
error,
|
||||
key: forward.key
|
||||
})
|
||||
throw inServerTerms(error, forward, server.name)
|
||||
}
|
||||
const started = StartedOnServer.safeParse(receipt)
|
||||
if (started.success) {
|
||||
recordRemoteStart(db, {
|
||||
conversationId: link.conversation_id,
|
||||
taskId,
|
||||
dispatchId: started.data.dispatchId,
|
||||
requestKey: forward.key,
|
||||
peerRunId: started.data.run?.id,
|
||||
agent: started.data.agent ?? null
|
||||
})
|
||||
prepareWorkerStartCard(
|
||||
runtime,
|
||||
ctx.orchestrationCaller
|
||||
)({
|
||||
dispatchId: started.data.dispatchId,
|
||||
workerAddress: `dispatch:${started.data.dispatchId}`,
|
||||
task: task.spec,
|
||||
agent: null
|
||||
})
|
||||
}
|
||||
const answer = inServerTerms(receipt, forward, server.name)
|
||||
return withRun(
|
||||
answer && typeof answer === 'object'
|
||||
? { ...answer, taskId, ...(started.data?.run ? { serverRun: started.data.run } : {}) }
|
||||
: answer
|
||||
)
|
||||
}
|
||||
|
||||
/** What a start's failure says about its reservation: adopt the Dispatch it names, keep it for the orphan lookup, or free it. */
|
||||
function settleUnansweredStart(
|
||||
runtime: OrcaRuntimeService,
|
||||
start: { conversationId: string; taskId: string; key: string; error: unknown }
|
||||
): void {
|
||||
const db = runtime.getOrchestrationDb()
|
||||
const data = errorData(start.error)
|
||||
const named = typeof data?.dispatchId === 'string' ? data.dispatchId : undefined
|
||||
if (named) {
|
||||
recordRemoteStart(db, {
|
||||
conversationId: start.conversationId,
|
||||
taskId: start.taskId,
|
||||
dispatchId: named,
|
||||
requestKey: start.key
|
||||
})
|
||||
} else if (serverNeverAnswered(start.error) && data?.effectsApplied !== false) {
|
||||
markRemoteStartOutcomeUnknown(db, start.taskId, start.key)
|
||||
} else if (
|
||||
!(start.error instanceof OrchestrationError && start.error.code === 'operation_unknown')
|
||||
) {
|
||||
// The server's ordinary start refused it, which a start that had launched answers with a receipt.
|
||||
releaseRemotePlacement(db, start.taskId, start.key)
|
||||
}
|
||||
}
|
||||
@@ -33,6 +33,7 @@ import type { RpcCallerScope } from './rpc-caller-scope'
|
||||
import { runWithActiveRpcCaller } from './rpc-active-caller'
|
||||
import { admitRpcRequest } from './rpc-request-admission-prologue'
|
||||
import { resolveOrcadFeatureRequirement, rpcMethodFeatureWait } from './orcad-feature-readiness'
|
||||
import { routeRemoteOwnedCall } from '../orchestration/federation/remote-owned/remote-owned-route'
|
||||
|
||||
export type DispatcherOptions = {
|
||||
runtime: OrcaRuntimeService
|
||||
@@ -244,6 +245,21 @@ export class RpcDispatcher {
|
||||
request,
|
||||
compatibility.legacyCoordinatorAuthority
|
||||
)
|
||||
// Before this Orca's receipts: the server that owns the worker keeps the receipt, under the caller's scoped key.
|
||||
const remote = await routeRemoteOwnedCall(runtime, method, request, effectiveParams, {
|
||||
...context,
|
||||
orchestrationCompatibilityEvidence: request.orchestrationCompatibilityEvidence
|
||||
})
|
||||
if (remote.handled) {
|
||||
recordRuntimeFeatureInteraction(
|
||||
runtime,
|
||||
request.method,
|
||||
remote.result,
|
||||
undefined,
|
||||
request.params
|
||||
)
|
||||
return remote.result
|
||||
}
|
||||
// A legacy coordinator fingerprint outranks the transport identity; mint a fallback only when absent.
|
||||
const fallbackCallerFingerprint =
|
||||
context.authenticatedCallerFingerprint ??
|
||||
|
||||
@@ -101,6 +101,16 @@ export function checkOrchestrationAccess<P, G>(
|
||||
}
|
||||
}
|
||||
|
||||
/** The same check for a call this Orca sends on to a paired server instead of running it here. */
|
||||
export function checkDeclaredOrchestrationAccess(
|
||||
access: OrchestrationMethodAccessDeclaration,
|
||||
params: unknown,
|
||||
ctx: RpcContext
|
||||
): void {
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a registered declaration was built for its method's params, which the dispatcher parsed with that method's schema.
|
||||
checkOrchestrationAccess(access as OrchestrationMethodAccess<unknown, unknown>, params, ctx)
|
||||
}
|
||||
|
||||
function declaredCaller<P>(
|
||||
field: Extract<FieldOf<P>, OrchestrationCallerField> | undefined,
|
||||
params: P
|
||||
|
||||
+2
-1
@@ -13,6 +13,7 @@ import '../../../../orchestration/db/dispatch-row-writer'
|
||||
import '../../../../orchestration/db/attempt-outcome-projection'
|
||||
import type { OrchestrationEnvironmentTransport } from '../../../../orchestration/federation/environment-transport'
|
||||
import '../../../../orchestration/legacy-remote/federation-lifecycle-settlement'
|
||||
import { attachmentHostCapabilities } from './federation-peer-capabilities.test-support'
|
||||
import { RpcDispatcher } from '../../../dispatcher'
|
||||
import { ORCHESTRATION_METHODS } from '../orchestration'
|
||||
import { createFederationWorkerStartRequest as startRequest } from './federation-request.test-support'
|
||||
@@ -35,7 +36,7 @@ describe('orchestration federation lifecycle settlement', () => {
|
||||
workerRuntime = new OrcaRuntimeService()
|
||||
workerRuntime.setOrchestrationDb(workerDb)
|
||||
workerDispatcher = new RpcDispatcher({ runtime: workerRuntime, methods: ORCHESTRATION_METHODS })
|
||||
workerCapabilities = [...(workerRuntime.getStatus().capabilities ?? [])]
|
||||
workerCapabilities = attachmentHostCapabilities(workerRuntime)
|
||||
failNextAckBeforeDelivery = false
|
||||
ackAttempts = 0
|
||||
transport = {
|
||||
|
||||
+10
@@ -2,6 +2,7 @@ import {
|
||||
ORCHESTRATION_FEDERATION_FLEET_SNAPSHOT_RUNTIME_CAPABILITY,
|
||||
ORCHESTRATION_FEDERATION_RELEASE_ARCHIVE_RUNTIME_CAPABILITY,
|
||||
ORCHESTRATION_FEDERATION_STRUCTURED_READ_RUNTIME_CAPABILITY,
|
||||
ORCHESTRATION_REMOTE_OWNED_WORKERS_RUNTIME_CAPABILITY,
|
||||
ORCHESTRATION_WORKER_AUTO_RELEASE_RUNTIME_CAPABILITY
|
||||
} from '../../../../../../shared/orchestration/orchestration-runtime-capabilities'
|
||||
import type { OrcaRuntimeService } from '../../../../orca-runtime'
|
||||
@@ -13,6 +14,8 @@ function withheldPeerCapabilities(host: {
|
||||
autoRelease: boolean
|
||||
}): ReadonlySet<string> {
|
||||
return new Set([
|
||||
// These suites drive today's remote attachments, which a server that owns its workers never gets.
|
||||
ORCHESTRATION_REMOTE_OWNED_WORKERS_RUNTIME_CAPABILITY,
|
||||
...(host.newCapabilities
|
||||
? []
|
||||
: [
|
||||
@@ -42,3 +45,10 @@ export function olderPeerStatusResponse(
|
||||
_meta: { runtimeId: runtime.getRuntimeId() }
|
||||
}
|
||||
}
|
||||
|
||||
/** A host's capabilities as one that keeps today's remote attachments advertises them. */
|
||||
export function attachmentHostCapabilities(runtime: OrcaRuntimeService): string[] {
|
||||
return (runtime.getStatus().capabilities ?? []).filter(
|
||||
(capability) => capability !== ORCHESTRATION_REMOTE_OWNED_WORKERS_RUNTIME_CAPABILITY
|
||||
)
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import '../../../../orchestration/db/messages/mailbox-routing-page'
|
||||
import '../../../../orchestration/db/dispatch-row-writer'
|
||||
import '../../../../orchestration/db/attempt-outcome-projection'
|
||||
import type { OrchestrationEnvironmentTransport } from '../../../../orchestration/federation/environment-transport'
|
||||
import { attachmentHostCapabilities } from './federation-peer-capabilities.test-support'
|
||||
import { RpcDispatcher } from '../../../dispatcher'
|
||||
import { ORCHESTRATION_METHODS } from '../orchestration'
|
||||
import { createFederationWorkerStartRequest as startRequest } from './federation-request.test-support'
|
||||
@@ -37,7 +38,7 @@ describe('orchestration federation', () => {
|
||||
runtime: workerRuntime,
|
||||
methods: ORCHESTRATION_METHODS
|
||||
})
|
||||
workerCapabilities = [...(workerRuntime.getStatus().capabilities ?? [])]
|
||||
workerCapabilities = attachmentHostCapabilities(workerRuntime)
|
||||
workerPeerFingerprint = 'windows_peer_fingerprint'
|
||||
loseNextAckResponse = false
|
||||
const transport: OrchestrationEnvironmentTransport = {
|
||||
@@ -109,7 +110,7 @@ describe('orchestration federation', () => {
|
||||
runtime: workerRuntime,
|
||||
methods: ORCHESTRATION_METHODS
|
||||
})
|
||||
workerCapabilities = [...(workerRuntime.getStatus().capabilities ?? [])]
|
||||
workerCapabilities = attachmentHostCapabilities(workerRuntime)
|
||||
}
|
||||
|
||||
it('fails before remote task input when wait-for-setup fails', async () => {
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
} from '../../../../../../shared/rpc-contract/orchestration-params'
|
||||
import { TaskUpdateParams } from './task-update-params'
|
||||
import { forwardRemoteTaskUpdate } from '../../../../orchestration/federation/remote-owned/remote-owned-task-update'
|
||||
import { remoteForwardCallerNamespace } from '../../../../orchestration/federation/remote-owned/remote-forward-caller'
|
||||
|
||||
export const ORCHESTRATION_MESSAGE_METHODS = [
|
||||
ORCHESTRATION_REPLY_METHOD,
|
||||
@@ -126,19 +127,21 @@ export const ORCHESTRATION_MESSAGE_METHODS = [
|
||||
targets: { id: 'task', run: 'run' },
|
||||
rule: taskUpdateAccess
|
||||
},
|
||||
handler: async (params, { orchestrationCaller, orchestrationMutation, runtime }) => {
|
||||
handler: async (params, ctx) => {
|
||||
const { orchestrationMutation, runtime } = ctx
|
||||
const db = runtime.getOrchestrationDb()
|
||||
// The server sees every caller here as one pairing; the caller's own identity scopes its retry key.
|
||||
const callerNamespace =
|
||||
orchestrationCaller?.orcaSessionId ??
|
||||
(params.callerTerminalHandle
|
||||
? runtime.getTerminalPaneKey(params.callerTerminalHandle)
|
||||
: null)
|
||||
const forwarded = await forwardRemoteTaskUpdate(
|
||||
runtime,
|
||||
params,
|
||||
callerNamespace && orchestrationMutation
|
||||
? { callerNamespace, requestId: orchestrationMutation.requestId }
|
||||
orchestrationMutation
|
||||
? {
|
||||
callerNamespace: remoteForwardCallerNamespace(
|
||||
runtime,
|
||||
ctx,
|
||||
params.callerTerminalHandle
|
||||
),
|
||||
requestId: orchestrationMutation.requestId
|
||||
}
|
||||
: undefined
|
||||
)
|
||||
if (forwarded) {
|
||||
|
||||
@@ -8,7 +8,7 @@ import { RequestShowParams } from '../../../../../../shared/rpc-contract/orchest
|
||||
export const ORCHESTRATION_MUTATION_REQUEST_METHODS = [
|
||||
defineOrchestrationMethod({
|
||||
name: 'orchestration.requestShow',
|
||||
access: { kind: 'reads', targets: { request: 'request' } },
|
||||
access: { kind: 'reads', server: 'on', targets: { request: 'request' } },
|
||||
permission: 'workspace',
|
||||
params: RequestShowParams,
|
||||
// Read durable receipts without writing/mutating again; keep this recovery query outside ORCHESTRATION_MUTATION_METHODS.
|
||||
|
||||
@@ -79,7 +79,8 @@ export const ORCHESTRATION_WORKER_CONTROL_METHODS = [
|
||||
terminal: observation.exact ? observation.terminal : null,
|
||||
observation: exposeObservation(observation),
|
||||
terminalResource: resource ? exposeWorkerTerminalResource(resource) : null,
|
||||
autoReleaseAt: liveWorkerTerminalAutoReleaseAt(resource)
|
||||
autoReleaseAt: liveWorkerTerminalAutoReleaseAt(resource),
|
||||
coordinatedFrom: db.getPeerRun(dispatch.run_id)?.home_label ?? null
|
||||
}
|
||||
}
|
||||
}),
|
||||
|
||||
@@ -25,7 +25,7 @@ import {
|
||||
WorkerListParams
|
||||
} from '../../../../../../shared/rpc-contract/orchestration-worker-release-schemas-params'
|
||||
|
||||
const emptyWorkerListPage = (limit: number) => ({
|
||||
export const emptyWorkerListPage = (limit: number) => ({
|
||||
workers: [],
|
||||
counts: {},
|
||||
page: { limit, total: 0, hasMore: false, nextCursor: null }
|
||||
@@ -33,7 +33,7 @@ const emptyWorkerListPage = (limit: number) => ({
|
||||
|
||||
export const ORCHESTRATION_WORKER_LIST_METHOD = defineOrchestrationMethod({
|
||||
name: 'orchestration.workerList',
|
||||
access: { kind: 'reads', targets: { run: 'run' }, collection: true },
|
||||
access: { kind: 'reads', server: 'on', targets: { run: 'run' }, collection: true },
|
||||
permission: 'workspace',
|
||||
params: WorkerListParams,
|
||||
handler: async (params, { runtime }) => {
|
||||
|
||||
@@ -60,15 +60,62 @@ export const ORCHESTRATION_WORKER_START_METHODS = [
|
||||
|
||||
/** worker-start, and a follow-up sent to a finished worker, which is worker-start run for the sender. */
|
||||
export async function startWorker(
|
||||
startParams: z.infer<typeof WorkerStartParams>,
|
||||
ctx: WorkerStartContext,
|
||||
startFollowUp?: WorkerFollowUp
|
||||
): Promise<unknown> {
|
||||
const { runtime, orchestrationMutation, orchestrationCaller } = ctx
|
||||
const prepared = await prepareWorkerStart(startParams, ctx, startFollowUp)
|
||||
const { params, followUp, coordinator, bindRun, helperCreator, taskDeps, existingTask, withRun } =
|
||||
prepared
|
||||
const db = runtime.getOrchestrationDb()
|
||||
const mode = decideWorkerStartMode({
|
||||
params,
|
||||
settings: readWorkerStartModeSettings(runtime)
|
||||
})
|
||||
if (params.on) {
|
||||
// Remote workers are terminal-only; carry the mode receipt to explain structured-preference fallback.
|
||||
const receipt = await startFederatedWorker({
|
||||
params,
|
||||
runtime,
|
||||
db,
|
||||
bindRunId: () => bindRun().run.id,
|
||||
task: existingTask,
|
||||
taskDeps,
|
||||
orchestrationMutation,
|
||||
callerSession: orchestrationCaller
|
||||
})
|
||||
await stopHelperAfterStart(runtime, receipt, orchestrationMutation)
|
||||
return withRun(receipt && typeof receipt === 'object' ? { ...receipt, mode } : receipt)
|
||||
}
|
||||
const receipt = await startLocalWorker({
|
||||
params: { ...params, timeoutMs: resolveWorkerStartReadinessTimeoutMs(params.timeoutMs) },
|
||||
runtime,
|
||||
db,
|
||||
bindRun: () => bindRun().run,
|
||||
taskDeps,
|
||||
coordinator,
|
||||
callerSession: orchestrationCaller,
|
||||
existingTask,
|
||||
orchestrationMutation,
|
||||
mode,
|
||||
helperCreator,
|
||||
followUp
|
||||
})
|
||||
await stopHelperAfterStart(runtime, receipt, orchestrationMutation)
|
||||
return withRun(withWorkerStartNext(receipt))
|
||||
}
|
||||
|
||||
/** What every start checks and binds before it is placed, here or on a paired server. */
|
||||
export async function prepareWorkerStart(
|
||||
params: z.infer<typeof WorkerStartParams>,
|
||||
{
|
||||
runtime,
|
||||
orchestrationMutation,
|
||||
orchestrationCompatibilityEvidence,
|
||||
orchestrationCaller
|
||||
}: WorkerStartContext,
|
||||
}: Omit<WorkerStartContext, 'orchestrationMutation'>,
|
||||
followUp?: WorkerFollowUp
|
||||
): Promise<unknown> {
|
||||
) {
|
||||
if (!isWorkerStartTimeoutWithinTimerLimit(params.timeoutMs)) {
|
||||
throw new OrchestrationError(
|
||||
'invalid_argument',
|
||||
@@ -103,7 +150,6 @@ export async function startWorker(
|
||||
{ effectsApplied: false }
|
||||
)
|
||||
}
|
||||
const readinessTimeoutMs = resolveWorkerStartReadinessTimeoutMs(params.timeoutMs)
|
||||
const db = runtime.getOrchestrationDb()
|
||||
const coordinator = resolveOrchestrationCaller(runtime, {
|
||||
callerTerminalHandle: params.from,
|
||||
@@ -200,39 +246,14 @@ export async function startWorker(
|
||||
followUp = { ...followUp, reopen: placement.reopen, earlierWork: placement.earlierWork }
|
||||
await assertWorkerStartTaskSpecWithinPromptBudget(spec)
|
||||
}
|
||||
const mode = decideWorkerStartMode({
|
||||
return {
|
||||
params,
|
||||
settings: readWorkerStartModeSettings(runtime)
|
||||
})
|
||||
if (params.on) {
|
||||
// Remote workers are terminal-only; carry the mode receipt to explain structured-preference fallback.
|
||||
const receipt = await startFederatedWorker({
|
||||
params,
|
||||
runtime,
|
||||
db,
|
||||
bindRunId: () => bindRun().run.id,
|
||||
task: existingTask,
|
||||
taskDeps,
|
||||
orchestrationMutation,
|
||||
callerSession: orchestrationCaller
|
||||
})
|
||||
await stopHelperAfterStart(runtime, receipt, orchestrationMutation)
|
||||
return withRun(receipt && typeof receipt === 'object' ? { ...receipt, mode } : receipt)
|
||||
}
|
||||
const receipt = await startLocalWorker({
|
||||
params: { ...params, timeoutMs: readinessTimeoutMs },
|
||||
runtime,
|
||||
db,
|
||||
bindRun: () => bindRun().run,
|
||||
taskDeps,
|
||||
followUp,
|
||||
coordinator,
|
||||
callerSession: orchestrationCaller,
|
||||
existingTask,
|
||||
orchestrationMutation,
|
||||
mode,
|
||||
bindRun,
|
||||
helperCreator,
|
||||
followUp
|
||||
})
|
||||
await stopHelperAfterStart(runtime, receipt, orchestrationMutation)
|
||||
return withRun(withWorkerStartNext(receipt))
|
||||
taskDeps,
|
||||
existingTask,
|
||||
withRun
|
||||
}
|
||||
}
|
||||
|
||||
@@ -40,6 +40,9 @@ export const ORCHESTRATION_CONTRACT_VERSION = 1 as const
|
||||
export const ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY = 'orchestration.contract.v1' as const
|
||||
/** Receives agent mail a paired Orca carries for its agents, and queues replies for it to pull. */
|
||||
export const ORCHESTRATION_PEER_MAIL_RUNTIME_CAPABILITY = 'orchestration.peer-mail.v1' as const
|
||||
/** Runs the workers a paired Orca places here as its own, coordinated from that Orca (wire v1). */
|
||||
export const ORCHESTRATION_REMOTE_OWNED_WORKERS_RUNTIME_CAPABILITY =
|
||||
'orchestration.remote-owned-workers.v1' as const
|
||||
|
||||
export const ORCHESTRATION_RUNTIME_CAPABILITIES = [
|
||||
ORCHESTRATION_FEDERATION_RUNTIME_CAPABILITY,
|
||||
@@ -58,5 +61,6 @@ export const ORCHESTRATION_RUNTIME_CAPABILITIES = [
|
||||
ORCHESTRATION_WORKER_TURN_STOP_REPORTS_RUNTIME_CAPABILITY,
|
||||
ORCHESTRATION_ASK_RETURNS_AT_ONCE_RUNTIME_CAPABILITY,
|
||||
ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY,
|
||||
ORCHESTRATION_PEER_MAIL_RUNTIME_CAPABILITY
|
||||
ORCHESTRATION_PEER_MAIL_RUNTIME_CAPABILITY,
|
||||
ORCHESTRATION_REMOTE_OWNED_WORKERS_RUNTIME_CAPABILITY
|
||||
] as const
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
import { z } from 'zod'
|
||||
import { requiredString } from './rpc-param-primitives'
|
||||
|
||||
export const RequestShowParams = z.object({ request: requiredString('Missing --request') })
|
||||
export const RequestShowParams = z.object({
|
||||
request: requiredString('Missing --request'),
|
||||
// The paired server a forwarded request ran on; its receipt is kept there.
|
||||
on: z.string().min(1).optional()
|
||||
})
|
||||
|
||||
@@ -21,5 +21,7 @@ export const WorkerListParams = z.object({
|
||||
cursor: z.string().min(1).max(2_048).optional(),
|
||||
limit: z.number().int().min(1).max(ORCHESTRATION_FLEET_PAGE_MAX).optional(),
|
||||
includeRemote: z.boolean().optional(),
|
||||
paginate: z.boolean().optional()
|
||||
paginate: z.boolean().optional(),
|
||||
// A paired server's own list, paged by its cursor.
|
||||
on: z.string().min(1).optional()
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user