Files
archived-orca/src/cli/runtime/client.ts
T
Jinwoo Hong bd56be7146 feat(orchestration): mail an agent on a paired server with send --on; its reply comes back by itself (#40106)
* feat(orchestration): mail agents on a paired server with send --on, replies come back by themselves

* fix(orchestration): gate --on on the carrying Orca, ack retried deliveries, refuse payloads and federated workers

* chore(orchestration): annotate peer-mail row casts and drop a wall-clock wait in its test

* chore(orchestration): name mail-between-servers codes server_mail_unsupported and mail_requires_server_pairing

* docs(cli): list --on in the orchestration reply usage

* fix(orchestration): name --environment, the flag on main, in agent mail refusals and the guide

* chore(orchestration): mark the two deliberate agent-kind branches mail between servers needs

* docs(cli): leave --on out of the reply usage; it is accepted only to refuse it

* fix(orchestration): keep peer conversations to peer ids, survive resets and re-pairs, keep pulling on a registry error

- A server takes only peerconv_ conversation ids, so a paired computer cannot name another Orca's dispatch in the relay.
- A replayed reply after a mail reset is acknowledged, not retried forever; reset tasks keeps replies waiting to be pulled.
- A re-paired server starts a new conversation instead of refusing every later send and reply.
- A failed registry read keeps the pull loop retrying.
- Drop the two ast-grep-ignore lines that only #40193's rule needed.

* fix(orchestration): worker-start --on runs on the caller's own Orca, like send --on

An inherited ORCA_ENVIRONMENT sent worker-start --on to that server, so the
--on command the agent refusal suggests did not work. One rule now keeps
send, reply and worker-start with --on on the caller's own Orca.

* fix(orchestration): refuse reply --on on both sides of a conversation between servers

The side that opened the conversation still took --on when it named the right
server; the reply usage says --on is accepted only to refuse it.
2026-11-09 17:00:11 -05:00

310 lines
12 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { randomUUID } from 'node:crypto'
import type { CliStatusResult, RuntimeStatus } from '../../shared/runtime-types'
import { runtimeHostConnectionState } from '../../shared/remote-runtime/host-status/runtime-host-connection-state'
import {
isDurableMutation,
isOrchestrationMutation,
isTerminalPromptMutation,
orchestrationMigrationData
} from '../../shared/orchestration/orchestration-rpc-contract'
import type { PairedRuntimeConnection } from '../../shared/remote-runtime/pairing/paired-runtime-connection'
import { openLocalOrca } from './open-local-orca'
import { getDefaultUserDataPath, readMetadata } from './metadata'
import {
getCliStatus,
getOfflineRuntimeAdmission,
type OfflineRuntimeAdmission,
projectRemoteAppStatus
} from './status'
import { sendRequest } from './transport'
import { attachMutationRecovery, isRequestUnsent } from './client-error-recovery'
import {
RuntimeClientError,
RuntimeRpcFailureError,
type RuntimeRpcSuccess,
type RuntimeCallOptions
} from './types'
import {
attachDurableMutationRecovery,
attachLegacyTerminalPromptRecovery,
attachUnverifiedTerminalPromptRecovery,
didAnotherRuntimeHandleTerminalPrompt
} from './terminal-prompt-mutation-recovery'
import { recordRuntimeEnvironmentUsage } from '../../shared/remote-runtime/host-store/runtime-environment-usage-record'
import { createRuntimeSourceFence, type RuntimeSourceFence } from './runtime-source-fence'
import { resolveRemoteConnection } from './runtime-remote-pairing'
import {
ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY,
ORCHESTRATION_CONTRACT_VERSION
} from '../../shared/orchestration/orchestration-runtime-capabilities'
import { RemoteRuntimeCompatGate } from './remote-runtime-compat-gate'
import { OrchestrationRequestEvidence } from './orchestration-compatibility-envelope'
import { getTimeoutMsParam, isWaitingCheck } from './runtime-request-timeout'
import {
isWorkerStartTimeoutWithinTimerLimit,
resolveWorkerStartClientTimeoutMs,
resolveWorkerStartReadinessTimeoutMs
} from '../../shared/orchestration/orchestration-timing-budgets'
import { MAX_TIMER_DELAY_MS } from '../../shared/timer-delay'
import {
buildOrchestrationRecoveryCommand,
resolveOrchestrationCliExecutable
} from './orchestration-recovery-command'
const LONG_POLL_CLIENT_GRACE_MS = 10_000
// The host records the launched agent in the caller's chat; older hosts ignore the evidence.
const loadWebSocketTransport = async () => await import('./websocket-transport.js')
export class RuntimeClient {
readonly userDataPath: string
private readonly remote: PairedRuntimeConnection | null
readonly environmentSelector: string | null
private readonly cliExecutable: string
private readonly originalArgs: readonly string[] | undefined
private readonly remoteCompat: RemoteRuntimeCompatGate
private orchestrationContractCheck: Promise<void> | null = null
private readonly requestEvidence = new OrchestrationRequestEvidence(process.env)
private readonly runtimeSourceFence: RuntimeSourceFence
// Browser session cold starts can take 15–30 seconds; allow time for initialization.
constructor(
userDataPath = getDefaultUserDataPath(),
private readonly requestTimeoutMs = 60_000,
remotePairingCode = process.env.ORCA_PAIRING_CODE ?? process.env.ORCA_REMOTE_PAIRING ?? null,
environmentSelector = process.env.ORCA_ENVIRONMENT ?? null,
cliExecutable = resolveOrchestrationCliExecutable(),
originalArgs?: readonly string[]
) {
this.userDataPath = userDataPath
this.environmentSelector = environmentSelector
this.cliExecutable = cliExecutable
this.originalArgs = originalArgs ? [...originalArgs] : undefined
this.remote = resolveRemoteConnection(userDataPath, remotePairingCode, environmentSelector)
this.remoteCompat = new RemoteRuntimeCompatGate(userDataPath, environmentSelector)
this.runtimeSourceFence = createRuntimeSourceFence(process.env, userDataPath, requestTimeoutMs)
}
get isRemote(): boolean {
return this.remote !== null
}
async call<TResult>(
method: string,
params?: unknown,
options?: RuntimeCallOptions
): Promise<RuntimeRpcSuccess<TResult>> {
const effectiveTimeoutMs = options?.timeoutMs ?? this.resolveMethodTimeoutMs(method, params)
const orchestrationMutation = isOrchestrationMutation(method, params)
const terminalPromptMutation = isTerminalPromptMutation(method, params)
const legacyTerminalPrompt = options?.legacyTerminalPrompt === true && terminalPromptMutation
const durableMutation = !legacyTerminalPrompt && isDurableMutation(method, params)
const orchestrationRequestId = durableMutation
? (options?.orchestrationRequestId ?? randomUUID())
: undefined
const originalCommand = durableMutation
? buildOrchestrationRecoveryCommand(method, params, this.cliExecutable, this.originalArgs)
: undefined
const recover = (error: unknown, targetRuntimeId: string | null) => {
if (legacyTerminalPrompt) {
return attachLegacyTerminalPromptRecovery(error)
}
if (
terminalPromptMutation &&
options?.terminalPromptPreflight &&
didAnotherRuntimeHandleTerminalPrompt(
error,
options.terminalPromptPreflight.runtimeId,
targetRuntimeId
)
) {
return attachUnverifiedTerminalPromptRecovery(error)
}
return attachDurableMutationRecovery(error, orchestrationRequestId, originalCommand, method)
}
if (orchestrationMutation) {
try {
await this.ensureOrchestrationContractCompatible(effectiveTimeoutMs)
} catch (error) {
// A caller-chosen request id is the only rerun handle, so keep it when the probe fails first.
throw options?.orchestrationRequestId
? attachMutationRecovery(error, orchestrationRequestId, originalCommand, true)
: error
}
}
const compatibilityEnvelope = this.requestEvidence.envelopeFor(method, {
remote: this.remote !== null,
mutation: orchestrationMutation,
requestId: orchestrationRequestId
})
const envelope = {
orchestrationCapability: options?.orchestrationCapability,
orchestrationContractVersion: method.startsWith('orchestration.')
? ORCHESTRATION_CONTRACT_VERSION
: undefined,
orchestrationRequestId,
...compatibilityEnvelope
}
if (this.remote) {
const transport = await loadWebSocketTransport()
let response
try {
response = await this.remoteCompat.send<TResult>({
transport,
connection: this.remote,
method,
params,
timeoutMs: effectiveTimeoutMs,
envelope,
requestSelection: options?.requestSelection
})
} catch (error) {
throw recover(error, null)
}
if (response.ok === false) {
throw recover(new RuntimeRpcFailureError(response), null)
}
if (this.environmentSelector) {
recordRuntimeEnvironmentUsage(this.userDataPath, this.environmentSelector, {
runtimeId: response._meta.runtimeId
})
}
return response
}
const metadata = readMetadata(this.userDataPath)
// Refusal precedes sending, so there is no request to recover.
const expectedRuntimeSource = await this.runtimeSourceFence.check(metadata, method)
let response
try {
response = await sendRequest<TResult>(
metadata,
method,
params,
effectiveTimeoutMs,
envelope,
expectedRuntimeSource,
options?.requestSelection ? { requestSelection: options.requestSelection } : undefined
)
} catch (error) {
// A pre-send refusal has no outcome to recover; retry advice would resend the same bytes.
throw isRequestUnsent(error) ? error : recover(error, metadata.runtimeId ?? null)
}
if (response.ok === false) {
throw recover(new RuntimeRpcFailureError(response), metadata.runtimeId ?? null)
}
return response
}
// Give server-side waits time to finish before the client transport times out.
private resolveMethodTimeoutMs(method: string, params?: unknown): number {
if (method === 'orchestration.workerStart') {
const requestedValue = getTimeoutMsParam(params)
const requested = typeof requestedValue === 'number' ? requestedValue : Number(requestedValue)
if (!isWorkerStartTimeoutWithinTimerLimit(requested)) {
throw new RuntimeClientError(
'invalid_argument',
`--timeout-ms is too large for worker-start transport grace; the derived timeout must be <= ${MAX_TIMER_DELAY_MS}ms.`
)
}
const readiness = resolveWorkerStartReadinessTimeoutMs(requested)
return Math.max(resolveWorkerStartClientTimeoutMs(readiness), this.requestTimeoutMs)
}
// A send to a finished worker starts its follow-up, which waits as worker-start does.
if (method === 'orchestration.send') {
return Math.max(resolveWorkerStartClientTimeoutMs(), this.requestTimeoutMs)
}
if (
(method === 'orchestration.check' && isWaitingCheck(params)) ||
method === 'terminal.wait'
) {
const inner = Number(getTimeoutMsParam(params))
if (Number.isFinite(inner) && inner > 0) {
return Math.max(inner + LONG_POLL_CLIENT_GRACE_MS, this.requestTimeoutMs)
}
}
return this.requestTimeoutMs
}
async getCliStatus(): Promise<RuntimeRpcSuccess<CliStatusResult>> {
if (this.remote) {
const response = await this.call<RuntimeStatus>('status.get')
this.remoteCompat.noteVerifiedStatus(response.result)
const graphState = response.result.graphStatus
return {
id: response.id,
ok: true,
result: {
target: {
kind: 'environment',
environment: this.environmentSelector ?? 'pairing-code'
},
app: projectRemoteAppStatus(response.result),
runtime: {
state: graphState === 'ready' ? 'ready' : 'graph_not_ready',
reachable: true,
connectionState: runtimeHostConnectionState({
hasStatusEntry: true,
status: response.result
}),
runtimeId: response.result.runtimeId,
...(response.result.appVersion ? { appVersion: response.result.appVersion } : {}),
...(response.result.remoteUpdateSupport
? { remoteUpdateSupport: response.result.remoteUpdateSupport }
: {}),
...(response.result.capabilities ? { capabilities: response.result.capabilities } : {}),
...(response.result.degradations ? { degradations: response.result.degradations } : {})
},
graph: {
state: graphState
}
},
_meta: response._meta
}
}
return getCliStatus(this.userDataPath)
}
async getOfflineRuntimeAdmission(): Promise<OfflineRuntimeAdmission> {
if (this.remote) {
const status = await this.getCliStatus()
return {
reachable: status.result.runtime.reachable,
legacyPidRunning: status.result.app.running,
pid: status.result.app.pid ?? null
}
}
return getOfflineRuntimeAdmission(this.userDataPath)
}
private async ensureOrchestrationContractCompatible(timeoutMs: number): Promise<void> {
if (!this.orchestrationContractCheck) {
this.orchestrationContractCheck = this.checkOrchestrationContractCompatibility(
timeoutMs
).catch((error: unknown) => {
// Failed probes must be retried after transient outages.
this.orchestrationContractCheck = null
throw error
})
}
await this.orchestrationContractCheck
}
private async checkOrchestrationContractCompatibility(timeoutMs: number): Promise<void> {
const response = await this.call<RuntimeStatus>('status.get', undefined, { timeoutMs })
if (this.remote) {
this.remoteCompat.noteVerifiedStatus(response.result)
}
if (!response.result.capabilities?.includes(ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY)) {
throw new RuntimeClientError(
'orchestration_migration_required',
'The connected Orca runtime does not support the current orchestration contract. No effects were applied.',
orchestrationMigrationData('runtime_capability_missing')
)
}
}
async openOrca(timeoutMs = 15_000): Promise<RuntimeRpcSuccess<CliStatusResult>> {
return this.remote ? this.getCliStatus() : openLocalOrca(() => this.getCliStatus(), timeoutMs)
}
}