mirror of
https://github.com/stablyai/orca.git
synced 2026-11-19 04:23:57 +00:00
* feat(orchestration): mail agents on a paired server with send --on, replies come back by themselves * fix(orchestration): gate --on on the carrying Orca, ack retried deliveries, refuse payloads and federated workers * chore(orchestration): annotate peer-mail row casts and drop a wall-clock wait in its test * chore(orchestration): name mail-between-servers codes server_mail_unsupported and mail_requires_server_pairing * docs(cli): list --on in the orchestration reply usage * fix(orchestration): name --environment, the flag on main, in agent mail refusals and the guide * chore(orchestration): mark the two deliberate agent-kind branches mail between servers needs * docs(cli): leave --on out of the reply usage; it is accepted only to refuse it * fix(orchestration): keep peer conversations to peer ids, survive resets and re-pairs, keep pulling on a registry error - A server takes only peerconv_ conversation ids, so a paired computer cannot name another Orca's dispatch in the relay. - A replayed reply after a mail reset is acknowledged, not retried forever; reset tasks keeps replies waiting to be pulled. - A re-paired server starts a new conversation instead of refusing every later send and reply. - A failed registry read keeps the pull loop retrying. - Drop the two ast-grep-ignore lines that only #40193's rule needed. * fix(orchestration): worker-start --on runs on the caller's own Orca, like send --on An inherited ORCA_ENVIRONMENT sent worker-start --on to that server, so the --on command the agent refusal suggests did not work. One rule now keeps send, reply and worker-start with --on on the caller's own Orca. * fix(orchestration): refuse reply --on on both sides of a conversation between servers The side that opened the conversation still took --on when it named the right server; the reply usage says --on is accepted only to refuse it.
310 lines
12 KiB
TypeScript
310 lines
12 KiB
TypeScript
import { randomUUID } from 'node:crypto'
|
||
import type { CliStatusResult, RuntimeStatus } from '../../shared/runtime-types'
|
||
import { runtimeHostConnectionState } from '../../shared/remote-runtime/host-status/runtime-host-connection-state'
|
||
import {
|
||
isDurableMutation,
|
||
isOrchestrationMutation,
|
||
isTerminalPromptMutation,
|
||
orchestrationMigrationData
|
||
} from '../../shared/orchestration/orchestration-rpc-contract'
|
||
import type { PairedRuntimeConnection } from '../../shared/remote-runtime/pairing/paired-runtime-connection'
|
||
import { openLocalOrca } from './open-local-orca'
|
||
import { getDefaultUserDataPath, readMetadata } from './metadata'
|
||
import {
|
||
getCliStatus,
|
||
getOfflineRuntimeAdmission,
|
||
type OfflineRuntimeAdmission,
|
||
projectRemoteAppStatus
|
||
} from './status'
|
||
import { sendRequest } from './transport'
|
||
import { attachMutationRecovery, isRequestUnsent } from './client-error-recovery'
|
||
import {
|
||
RuntimeClientError,
|
||
RuntimeRpcFailureError,
|
||
type RuntimeRpcSuccess,
|
||
type RuntimeCallOptions
|
||
} from './types'
|
||
import {
|
||
attachDurableMutationRecovery,
|
||
attachLegacyTerminalPromptRecovery,
|
||
attachUnverifiedTerminalPromptRecovery,
|
||
didAnotherRuntimeHandleTerminalPrompt
|
||
} from './terminal-prompt-mutation-recovery'
|
||
import { recordRuntimeEnvironmentUsage } from '../../shared/remote-runtime/host-store/runtime-environment-usage-record'
|
||
import { createRuntimeSourceFence, type RuntimeSourceFence } from './runtime-source-fence'
|
||
import { resolveRemoteConnection } from './runtime-remote-pairing'
|
||
import {
|
||
ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY,
|
||
ORCHESTRATION_CONTRACT_VERSION
|
||
} from '../../shared/orchestration/orchestration-runtime-capabilities'
|
||
import { RemoteRuntimeCompatGate } from './remote-runtime-compat-gate'
|
||
import { OrchestrationRequestEvidence } from './orchestration-compatibility-envelope'
|
||
import { getTimeoutMsParam, isWaitingCheck } from './runtime-request-timeout'
|
||
import {
|
||
isWorkerStartTimeoutWithinTimerLimit,
|
||
resolveWorkerStartClientTimeoutMs,
|
||
resolveWorkerStartReadinessTimeoutMs
|
||
} from '../../shared/orchestration/orchestration-timing-budgets'
|
||
import { MAX_TIMER_DELAY_MS } from '../../shared/timer-delay'
|
||
import {
|
||
buildOrchestrationRecoveryCommand,
|
||
resolveOrchestrationCliExecutable
|
||
} from './orchestration-recovery-command'
|
||
|
||
const LONG_POLL_CLIENT_GRACE_MS = 10_000
|
||
// The host records the launched agent in the caller's chat; older hosts ignore the evidence.
|
||
|
||
const loadWebSocketTransport = async () => await import('./websocket-transport.js')
|
||
|
||
export class RuntimeClient {
|
||
readonly userDataPath: string
|
||
private readonly remote: PairedRuntimeConnection | null
|
||
readonly environmentSelector: string | null
|
||
private readonly cliExecutable: string
|
||
private readonly originalArgs: readonly string[] | undefined
|
||
private readonly remoteCompat: RemoteRuntimeCompatGate
|
||
private orchestrationContractCheck: Promise<void> | null = null
|
||
private readonly requestEvidence = new OrchestrationRequestEvidence(process.env)
|
||
private readonly runtimeSourceFence: RuntimeSourceFence
|
||
|
||
// Browser session cold starts can take 15–30 seconds; allow time for initialization.
|
||
constructor(
|
||
userDataPath = getDefaultUserDataPath(),
|
||
private readonly requestTimeoutMs = 60_000,
|
||
remotePairingCode = process.env.ORCA_PAIRING_CODE ?? process.env.ORCA_REMOTE_PAIRING ?? null,
|
||
environmentSelector = process.env.ORCA_ENVIRONMENT ?? null,
|
||
cliExecutable = resolveOrchestrationCliExecutable(),
|
||
originalArgs?: readonly string[]
|
||
) {
|
||
this.userDataPath = userDataPath
|
||
this.environmentSelector = environmentSelector
|
||
this.cliExecutable = cliExecutable
|
||
this.originalArgs = originalArgs ? [...originalArgs] : undefined
|
||
this.remote = resolveRemoteConnection(userDataPath, remotePairingCode, environmentSelector)
|
||
this.remoteCompat = new RemoteRuntimeCompatGate(userDataPath, environmentSelector)
|
||
this.runtimeSourceFence = createRuntimeSourceFence(process.env, userDataPath, requestTimeoutMs)
|
||
}
|
||
|
||
get isRemote(): boolean {
|
||
return this.remote !== null
|
||
}
|
||
|
||
async call<TResult>(
|
||
method: string,
|
||
params?: unknown,
|
||
options?: RuntimeCallOptions
|
||
): Promise<RuntimeRpcSuccess<TResult>> {
|
||
const effectiveTimeoutMs = options?.timeoutMs ?? this.resolveMethodTimeoutMs(method, params)
|
||
const orchestrationMutation = isOrchestrationMutation(method, params)
|
||
const terminalPromptMutation = isTerminalPromptMutation(method, params)
|
||
const legacyTerminalPrompt = options?.legacyTerminalPrompt === true && terminalPromptMutation
|
||
const durableMutation = !legacyTerminalPrompt && isDurableMutation(method, params)
|
||
const orchestrationRequestId = durableMutation
|
||
? (options?.orchestrationRequestId ?? randomUUID())
|
||
: undefined
|
||
const originalCommand = durableMutation
|
||
? buildOrchestrationRecoveryCommand(method, params, this.cliExecutable, this.originalArgs)
|
||
: undefined
|
||
const recover = (error: unknown, targetRuntimeId: string | null) => {
|
||
if (legacyTerminalPrompt) {
|
||
return attachLegacyTerminalPromptRecovery(error)
|
||
}
|
||
if (
|
||
terminalPromptMutation &&
|
||
options?.terminalPromptPreflight &&
|
||
didAnotherRuntimeHandleTerminalPrompt(
|
||
error,
|
||
options.terminalPromptPreflight.runtimeId,
|
||
targetRuntimeId
|
||
)
|
||
) {
|
||
return attachUnverifiedTerminalPromptRecovery(error)
|
||
}
|
||
return attachDurableMutationRecovery(error, orchestrationRequestId, originalCommand, method)
|
||
}
|
||
if (orchestrationMutation) {
|
||
try {
|
||
await this.ensureOrchestrationContractCompatible(effectiveTimeoutMs)
|
||
} catch (error) {
|
||
// A caller-chosen request id is the only rerun handle, so keep it when the probe fails first.
|
||
throw options?.orchestrationRequestId
|
||
? attachMutationRecovery(error, orchestrationRequestId, originalCommand, true)
|
||
: error
|
||
}
|
||
}
|
||
const compatibilityEnvelope = this.requestEvidence.envelopeFor(method, {
|
||
remote: this.remote !== null,
|
||
mutation: orchestrationMutation,
|
||
requestId: orchestrationRequestId
|
||
})
|
||
const envelope = {
|
||
orchestrationCapability: options?.orchestrationCapability,
|
||
orchestrationContractVersion: method.startsWith('orchestration.')
|
||
? ORCHESTRATION_CONTRACT_VERSION
|
||
: undefined,
|
||
orchestrationRequestId,
|
||
...compatibilityEnvelope
|
||
}
|
||
if (this.remote) {
|
||
const transport = await loadWebSocketTransport()
|
||
let response
|
||
try {
|
||
response = await this.remoteCompat.send<TResult>({
|
||
transport,
|
||
connection: this.remote,
|
||
method,
|
||
params,
|
||
timeoutMs: effectiveTimeoutMs,
|
||
envelope,
|
||
requestSelection: options?.requestSelection
|
||
})
|
||
} catch (error) {
|
||
throw recover(error, null)
|
||
}
|
||
if (response.ok === false) {
|
||
throw recover(new RuntimeRpcFailureError(response), null)
|
||
}
|
||
if (this.environmentSelector) {
|
||
recordRuntimeEnvironmentUsage(this.userDataPath, this.environmentSelector, {
|
||
runtimeId: response._meta.runtimeId
|
||
})
|
||
}
|
||
return response
|
||
}
|
||
const metadata = readMetadata(this.userDataPath)
|
||
// Refusal precedes sending, so there is no request to recover.
|
||
const expectedRuntimeSource = await this.runtimeSourceFence.check(metadata, method)
|
||
let response
|
||
try {
|
||
response = await sendRequest<TResult>(
|
||
metadata,
|
||
method,
|
||
params,
|
||
effectiveTimeoutMs,
|
||
envelope,
|
||
expectedRuntimeSource,
|
||
options?.requestSelection ? { requestSelection: options.requestSelection } : undefined
|
||
)
|
||
} catch (error) {
|
||
// A pre-send refusal has no outcome to recover; retry advice would resend the same bytes.
|
||
throw isRequestUnsent(error) ? error : recover(error, metadata.runtimeId ?? null)
|
||
}
|
||
if (response.ok === false) {
|
||
throw recover(new RuntimeRpcFailureError(response), metadata.runtimeId ?? null)
|
||
}
|
||
return response
|
||
}
|
||
|
||
// Give server-side waits time to finish before the client transport times out.
|
||
private resolveMethodTimeoutMs(method: string, params?: unknown): number {
|
||
if (method === 'orchestration.workerStart') {
|
||
const requestedValue = getTimeoutMsParam(params)
|
||
const requested = typeof requestedValue === 'number' ? requestedValue : Number(requestedValue)
|
||
if (!isWorkerStartTimeoutWithinTimerLimit(requested)) {
|
||
throw new RuntimeClientError(
|
||
'invalid_argument',
|
||
`--timeout-ms is too large for worker-start transport grace; the derived timeout must be <= ${MAX_TIMER_DELAY_MS}ms.`
|
||
)
|
||
}
|
||
const readiness = resolveWorkerStartReadinessTimeoutMs(requested)
|
||
return Math.max(resolveWorkerStartClientTimeoutMs(readiness), this.requestTimeoutMs)
|
||
}
|
||
// A send to a finished worker starts its follow-up, which waits as worker-start does.
|
||
if (method === 'orchestration.send') {
|
||
return Math.max(resolveWorkerStartClientTimeoutMs(), this.requestTimeoutMs)
|
||
}
|
||
if (
|
||
(method === 'orchestration.check' && isWaitingCheck(params)) ||
|
||
method === 'terminal.wait'
|
||
) {
|
||
const inner = Number(getTimeoutMsParam(params))
|
||
if (Number.isFinite(inner) && inner > 0) {
|
||
return Math.max(inner + LONG_POLL_CLIENT_GRACE_MS, this.requestTimeoutMs)
|
||
}
|
||
}
|
||
return this.requestTimeoutMs
|
||
}
|
||
|
||
async getCliStatus(): Promise<RuntimeRpcSuccess<CliStatusResult>> {
|
||
if (this.remote) {
|
||
const response = await this.call<RuntimeStatus>('status.get')
|
||
this.remoteCompat.noteVerifiedStatus(response.result)
|
||
const graphState = response.result.graphStatus
|
||
return {
|
||
id: response.id,
|
||
ok: true,
|
||
result: {
|
||
target: {
|
||
kind: 'environment',
|
||
environment: this.environmentSelector ?? 'pairing-code'
|
||
},
|
||
app: projectRemoteAppStatus(response.result),
|
||
runtime: {
|
||
state: graphState === 'ready' ? 'ready' : 'graph_not_ready',
|
||
reachable: true,
|
||
connectionState: runtimeHostConnectionState({
|
||
hasStatusEntry: true,
|
||
status: response.result
|
||
}),
|
||
runtimeId: response.result.runtimeId,
|
||
...(response.result.appVersion ? { appVersion: response.result.appVersion } : {}),
|
||
...(response.result.remoteUpdateSupport
|
||
? { remoteUpdateSupport: response.result.remoteUpdateSupport }
|
||
: {}),
|
||
...(response.result.capabilities ? { capabilities: response.result.capabilities } : {}),
|
||
...(response.result.degradations ? { degradations: response.result.degradations } : {})
|
||
},
|
||
graph: {
|
||
state: graphState
|
||
}
|
||
},
|
||
_meta: response._meta
|
||
}
|
||
}
|
||
return getCliStatus(this.userDataPath)
|
||
}
|
||
|
||
async getOfflineRuntimeAdmission(): Promise<OfflineRuntimeAdmission> {
|
||
if (this.remote) {
|
||
const status = await this.getCliStatus()
|
||
return {
|
||
reachable: status.result.runtime.reachable,
|
||
legacyPidRunning: status.result.app.running,
|
||
pid: status.result.app.pid ?? null
|
||
}
|
||
}
|
||
return getOfflineRuntimeAdmission(this.userDataPath)
|
||
}
|
||
|
||
private async ensureOrchestrationContractCompatible(timeoutMs: number): Promise<void> {
|
||
if (!this.orchestrationContractCheck) {
|
||
this.orchestrationContractCheck = this.checkOrchestrationContractCompatibility(
|
||
timeoutMs
|
||
).catch((error: unknown) => {
|
||
// Failed probes must be retried after transient outages.
|
||
this.orchestrationContractCheck = null
|
||
throw error
|
||
})
|
||
}
|
||
await this.orchestrationContractCheck
|
||
}
|
||
|
||
private async checkOrchestrationContractCompatibility(timeoutMs: number): Promise<void> {
|
||
const response = await this.call<RuntimeStatus>('status.get', undefined, { timeoutMs })
|
||
if (this.remote) {
|
||
this.remoteCompat.noteVerifiedStatus(response.result)
|
||
}
|
||
if (!response.result.capabilities?.includes(ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY)) {
|
||
throw new RuntimeClientError(
|
||
'orchestration_migration_required',
|
||
'The connected Orca runtime does not support the current orchestration contract. No effects were applied.',
|
||
orchestrationMigrationData('runtime_capability_missing')
|
||
)
|
||
}
|
||
}
|
||
|
||
async openOrca(timeoutMs = 15_000): Promise<RuntimeRpcSuccess<CliStatusResult>> {
|
||
return this.remote ? this.getCliStatus() : openLocalOrca(() => this.getCliStatus(), timeoutMs)
|
||
}
|
||
}
|