From cd05f2ff93b52c4b9b4b39e1c65d1a16bb93da99 Mon Sep 17 00:00:00 2001 From: OrcaWin Date: Mon, 27 Jul 2026 12:31:37 -0700 Subject: [PATCH] Implement robust orchestration primitives and connected-server workers (#9925) --- ORCHESTRATION_IMPLEMENTATION_CHECKLIST.md | 1787 ++++++++++ ORCHESTRATION_STRUCTURED_OUTPUT_DESIGN.md | 604 ++++ config/scripts/dev-cli-terminal-wrapper.mjs | 40 + .../scripts/dev-cli-terminal-wrapper.test.mjs | 70 + config/scripts/orca-dev-bin.test.mjs | 2 + config/scripts/orca-dev.mjs | 10 + .../orchestration-skill-guidance.test.mjs | 37 +- config/scripts/run-electron-vite-dev.mjs | 35 +- docs/orchestration-primitives.html | 2937 +++++++++++++++++ skill-guides/orchestration.md | 174 +- src/cli/bundled-skill-guides.ts | 2 +- src/cli/format.test.ts | 35 +- src/cli/handler-group-manifest.ts | 14 +- .../orchestration-lifecycle-rejection.test.ts | 3 +- .../handlers/orchestration-migration.test.ts | 28 + .../handlers/orchestration-run-cli.test.ts | 160 + .../orchestration-timeout-cli.test.ts | 230 ++ .../handlers/orchestration-worker-cli.test.ts | 161 + src/cli/handlers/orchestration.test.ts | 415 ++- src/cli/handlers/orchestration.ts | 514 ++- src/cli/help.ts | 20 +- src/cli/index.test.ts | 89 +- src/cli/runtime-client.test.ts | 82 + src/cli/runtime/client.ts | 88 +- src/cli/runtime/transport.ts | 9 +- src/cli/runtime/websocket-transport.test.ts | 23 + src/cli/runtime/websocket-transport.ts | 6 +- src/cli/specs/orchestration-worker-specs.ts | 71 + src/cli/specs/orchestration.ts | 157 +- src/main/index.ts | 31 +- ...time-environment-shared-control-support.ts | 80 + .../runtime-environment-transport-routing.ts | 99 +- .../native-chat/transcript-tail-reader.ts | 22 +- src/main/runtime/orca-runtime.test.ts | 238 +- src/main/runtime/orca-runtime.ts | 421 ++- .../orchestration-cli-subprocess.test.ts | 38 +- .../__snapshots__/preamble.test.ts.snap | 25 +- .../runtime/orchestration/coordinator.test.ts | 17 +- src/main/runtime/orchestration/coordinator.ts | 5 + src/main/runtime/orchestration/db.test.ts | 4 +- src/main/runtime/orchestration/db.ts | 2857 +++++++++++++++- .../orchestration/environment-transport.ts | 26 + .../federation-control-message.ts | 94 + .../orchestration/federation-sync.test.ts | 18 + .../runtime/orchestration/federation-sync.ts | 253 ++ .../runtime/orchestration/formatter.test.ts | 1 + .../lifecycle-reconciliation.test.ts | 129 +- .../orchestration/lifecycle-reconciliation.ts | 141 +- .../orchestration-db-permissions.test.ts | 27 + .../orchestration/orchestration-error.ts | 11 + ...orchestration-mutation-question-db.test.ts | 167 + .../orchestration-reset-db.test.ts | 101 + .../orchestration-run-delivery-db.test.ts | 279 ++ .../orchestration-worker-dispatch-db.test.ts | 274 ++ .../runtime/orchestration/preamble.test.ts | 29 +- src/main/runtime/orchestration/preamble.ts | 29 +- .../setup-completion-signal.test.ts | 59 + .../orchestration/setup-completion-signal.ts | 80 + src/main/runtime/orchestration/types.ts | 174 +- .../worker-output-cursor.test.ts | 42 + .../orchestration/worker-output-cursor.ts | 122 + .../worker-provider-session.test.ts | 79 + .../orchestration/worker-provider-session.ts | 34 + .../worker-transcript-payload.test.ts | 101 + .../worker-transcript-payload.ts | 226 ++ .../worker-transcript-read.test.ts | 188 ++ .../orchestration/worker-transcript-read.ts | 263 ++ src/main/runtime/rpc/core.ts | 16 + .../dispatcher-feature-interactions.test.ts | 11 +- src/main/runtime/rpc/dispatcher.ts | 114 +- src/main/runtime/rpc/errors.test.ts | 12 + src/main/runtime/rpc/errors.ts | 36 +- ...ration-federated-message-targeting.test.ts | 104 + .../orchestration-federated-worker-start.ts | 292 ++ ...hestration-federation-control-mail.test.ts | 345 ++ .../orchestration-federation-control.ts | 207 ++ .../orchestration-federation-effects.test.ts | 39 + .../orchestration-federation-effects.ts | 79 + ...ration-federation-folder-placement.test.ts | 57 + .../orchestration-federation-methods.ts | 10 + .../orchestration-federation-output.test.ts | 312 ++ .../methods/orchestration-federation-relay.ts | 179 + .../orchestration-federation-setup.test.ts | 196 ++ .../methods/orchestration-federation-setup.ts | 99 + .../orchestration-federation-start-receipt.ts | 32 + .../orchestration-federation-start-schema.ts | 21 + .../methods/orchestration-federation.test.ts | 862 +++++ .../rpc/methods/orchestration-federation.ts | 297 ++ ...orchestration-folder-worktree-placement.ts | 17 + .../orchestration-migration-behavior.test.ts | 181 + .../runtime/rpc/methods/orchestration-runs.ts | 100 + .../methods/orchestration-worker-control.ts | 294 ++ .../methods/orchestration-worker-methods.ts | 10 + .../orchestration-worker-observation.ts | 82 + .../orchestration-worker-output.test.ts | 205 ++ .../methods/orchestration-worker-output.ts | 208 ++ .../orchestration-worker-setup-gate.ts | 67 + .../orchestration-worker-start-receipt.ts | 41 + .../orchestration-worker-start-schema.ts | 23 + .../rpc/methods/orchestration-worker-stop.ts | 148 + .../methods/orchestration-worker-topology.ts | 228 ++ ...orchestration-workers-new-worktree.test.ts | 614 ++++ .../orchestration-workers-recovery.test.ts | 246 ++ .../rpc/methods/orchestration-workers.ts | 305 ++ .../runtime/rpc/methods/orchestration.test.ts | 1093 +++++- src/main/runtime/rpc/methods/orchestration.ts | 1038 +++++- .../rpc/orchestration-contract-fence.test.ts | 138 + .../rpc/orchestration-contract-fence.ts | 36 + .../rpc/orchestration-mutation-executor.ts | 152 + .../rpc/orchestration-mutation-ledger.test.ts | 309 ++ .../rpc/runtime-feature-interaction.ts | 44 + src/main/runtime/runtime-rpc.test.ts | 70 +- src/main/ssh/ssh-remote-cli-error-response.ts | 10 + src/main/ssh/ssh-remote-orca-cli.test.ts | 95 +- src/main/ssh/ssh-remote-orca-cli.ts | 60 +- .../ssh/ssh-remote-orchestration-send.test.ts | 2 + src/main/ssh/ssh-remote-orchestration-send.ts | 12 +- src/shared/agent-prompt-injection.test.ts | 7 + src/shared/agent-prompt-injection.ts | 14 +- src/shared/orchestration-rpc-contract.test.ts | 75 + src/shared/orchestration-rpc-contract.ts | 119 + src/shared/orchestration-worker-output.ts | 65 + src/shared/protocol-version.ts | 9 + src/shared/remote-runtime-client.test.ts | 33 +- src/shared/remote-runtime-client.ts | 13 +- src/shared/runtime-rpc-envelope.ts | 6 + src/shared/types.ts | 7 + 127 files changed, 23196 insertions(+), 887 deletions(-) create mode 100644 ORCHESTRATION_IMPLEMENTATION_CHECKLIST.md create mode 100644 ORCHESTRATION_STRUCTURED_OUTPUT_DESIGN.md create mode 100644 config/scripts/dev-cli-terminal-wrapper.mjs create mode 100644 config/scripts/dev-cli-terminal-wrapper.test.mjs create mode 100644 docs/orchestration-primitives.html create mode 100644 src/cli/handlers/orchestration-migration.test.ts create mode 100644 src/cli/handlers/orchestration-run-cli.test.ts create mode 100644 src/cli/handlers/orchestration-timeout-cli.test.ts create mode 100644 src/cli/handlers/orchestration-worker-cli.test.ts create mode 100644 src/cli/specs/orchestration-worker-specs.ts create mode 100644 src/main/ipc/runtime-environment-shared-control-support.ts create mode 100644 src/main/runtime/orchestration/environment-transport.ts create mode 100644 src/main/runtime/orchestration/federation-control-message.ts create mode 100644 src/main/runtime/orchestration/federation-sync.test.ts create mode 100644 src/main/runtime/orchestration/federation-sync.ts create mode 100644 src/main/runtime/orchestration/orchestration-db-permissions.test.ts create mode 100644 src/main/runtime/orchestration/orchestration-error.ts create mode 100644 src/main/runtime/orchestration/orchestration-mutation-question-db.test.ts create mode 100644 src/main/runtime/orchestration/orchestration-reset-db.test.ts create mode 100644 src/main/runtime/orchestration/orchestration-run-delivery-db.test.ts create mode 100644 src/main/runtime/orchestration/orchestration-worker-dispatch-db.test.ts create mode 100644 src/main/runtime/orchestration/setup-completion-signal.test.ts create mode 100644 src/main/runtime/orchestration/setup-completion-signal.ts create mode 100644 src/main/runtime/orchestration/worker-output-cursor.test.ts create mode 100644 src/main/runtime/orchestration/worker-output-cursor.ts create mode 100644 src/main/runtime/orchestration/worker-provider-session.test.ts create mode 100644 src/main/runtime/orchestration/worker-provider-session.ts create mode 100644 src/main/runtime/orchestration/worker-transcript-payload.test.ts create mode 100644 src/main/runtime/orchestration/worker-transcript-payload.ts create mode 100644 src/main/runtime/orchestration/worker-transcript-read.test.ts create mode 100644 src/main/runtime/orchestration/worker-transcript-read.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-federated-message-targeting.test.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-federated-worker-start.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-federation-control-mail.test.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-federation-control.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-federation-effects.test.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-federation-effects.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-federation-folder-placement.test.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-federation-methods.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-federation-output.test.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-federation-relay.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-federation-setup.test.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-federation-setup.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-federation-start-receipt.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-federation-start-schema.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-federation.test.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-federation.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-folder-worktree-placement.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-migration-behavior.test.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-runs.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-worker-control.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-worker-methods.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-worker-observation.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-worker-output.test.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-worker-output.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-worker-setup-gate.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-worker-start-receipt.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-worker-start-schema.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-worker-stop.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-worker-topology.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-workers-new-worktree.test.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-workers-recovery.test.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-workers.ts create mode 100644 src/main/runtime/rpc/orchestration-contract-fence.test.ts create mode 100644 src/main/runtime/rpc/orchestration-contract-fence.ts create mode 100644 src/main/runtime/rpc/orchestration-mutation-executor.ts create mode 100644 src/main/runtime/rpc/orchestration-mutation-ledger.test.ts create mode 100644 src/main/runtime/rpc/runtime-feature-interaction.ts create mode 100644 src/main/ssh/ssh-remote-cli-error-response.ts create mode 100644 src/shared/orchestration-rpc-contract.test.ts create mode 100644 src/shared/orchestration-rpc-contract.ts create mode 100644 src/shared/orchestration-worker-output.ts diff --git a/ORCHESTRATION_IMPLEMENTATION_CHECKLIST.md b/ORCHESTRATION_IMPLEMENTATION_CHECKLIST.md new file mode 100644 index 00000000000..57c00ba3a8b --- /dev/null +++ b/ORCHESTRATION_IMPLEMENTATION_CHECKLIST.md @@ -0,0 +1,1787 @@ +# Orca Orchestration Implementation Checklist + +This is the durable implementation ledger for the orchestration primitives proposal. Update it in +the same change that implements, removes, or materially revises an item. The design source is +`docs/orchestration-primitives.html`; keep it synchronized with this checklist. + +## How to use this file + +- Check an item only after its implementation and proportionate tests are complete. +- If an item changes meaning, edit the checklist and add a dated decision-log entry explaining why. +- After every implementation session, append a progress-log entry with files, tests, findings, and + the next concrete step. +- Do not mark a phase complete while any acceptance test in that phase remains open. +- Preserve the non-goals. A new subsystem requires separate evidence and a separate proposal. + +Status meanings: + +- `[ ]` not started or not proven +- `[x]` implemented and verified +- `DEFERRED` deliberately outside the current implementation sequence + +## Current summary + +- [x] Fresh primitive-oriented design written and repeatedly reviewed. +- [x] Product UI changes explicitly excluded. +- [x] Current orchestration skill now teaches setup-run for new worktrees, batch processing, + `agentTerminalHandle` preference, and the custom-argv setup-policy limitation. +- [x] Phase 0 command/skill compatibility work complete. +- [x] Phase 1 local Run, mailbox, lifecycle, and idempotency primitives complete. +- [x] Phase 2 same-server composed worker lifecycle complete. +- [x] Phase 3 connected-server federation complete. +- [x] Revalidate Phase 3 after the 2026-07-24 post-rebase dogfood exposed a renderer-adoption + process-identity regression. +- [x] Phase 4 structured worker output is implemented with passing automated coverage and physical + local, mixed-version, restart, disconnect, and Windows-home to Mac-worker evidence; optional + symmetric acceptance checks remain tracked below. +- [x] Hard-cutover migration fence implemented and locally verified; branch-head CI remains the + final remote check. + +## Scope invariants + +- [x] Agents choose decomposition, topology, placement, parallelism, and recovery strategy. +- [x] Low-level worktree, terminal, setup, and handoff commands remain independently usable. +- [x] Every composed mutation with external effects returns explicit effects and an honest outcome; + control-plane-only mutations return their exact resource receipt. +- [x] Worker assertions remain labeled as worker reports, not Orca-verified correctness. +- [x] Silence alone never proves worker death or triggers replacement. +- [x] Multi-server Runs use one authoritative Run home and connected worker servers. +- [x] Runtime/server/host identity mechanics remain hidden from ordinary agent commands. + +Explicit non-goals: + +- [x] No product dashboard, Run UI, badges, or coordinator chat UI. +- [x] No scheduler, automatic placement, capacity allocator, fairness, or priority aging. +- [x] No automatic retry or replacement based on silence. +- [x] No commit, branch, merge, integration, or target-ref tracking. +- [x] No filesystem read-only/writer enforcement. +- [x] No generalized ACL, organization, role, or worker-profile system. +- [x] No replicated Run database, leader election, or automatic Run-home failover. +- [x] No dead-letter/poison-message workflow. +- [x] No universal provider-session or transcript framework. + +## Phase 0 — Vocabulary and current-agent ergonomics + +### Command compatibility + +- [x] Remove or rename the existing scheduler-like `orca orchestration run --spec` before shipping + lightweight `run-*` commands. +- [x] Publish the canonical command map in CLI help. +- [x] Define aliases or explicit deprecations for any renamed current command. +- [x] Keep current flat task commands unless a separately justified CLI migration changes them. +- [x] Require explicit destructive reset scope; bare reset must not imply reset-all. + +### Skill and recipes + +- [x] Teach that `check --wait` returns a message batch, not one message. +- [x] Teach processing every returned message and waiting until expected Dispatches settle. +- [x] Document crash-safe explicit acknowledgment and redelivery of an unacknowledged batch. +- [x] Prefer `agentTerminalHandle`, then legacy `startupTerminal.handle`, then exact terminal-list + resolution. +- [x] Pass `--setup run` for new worktrees by default. +- [x] State a concrete reason before using `--setup skip` or `--setup inherit`. +- [x] Preserve `start-immediately` as the normal setup/agent startup policy. +- [x] Warn that the two-step custom-argv launch cannot preserve explicit `wait-for-setup`. +- [x] Add executable current-version recipes for local fan-out, new worktrees, completion/failure, + questions, timeout recovery, and restart limitations. +- [x] Ensure installed, bundled, repository, and generated orchestration skill copies stay in sync. + +### Phase 0 acceptance + +- [x] An agent reading only CLI help and the skill can select current versus new worktree correctly. +- [x] The agent starts all independent workers before blocking. +- [x] The agent cannot mistake the old scheduler Run for a lightweight namespace Run. +- [x] Recipes contain only commands supported by the matching shipped CLI version. + +## Phase 1 — Run, mailbox, lifecycle, and durable mutation receipts + +### Lightweight Run + +- [x] Add a Run table with stable Run ID, objective, created/updated timestamps, and home database. +- [x] Add mandatory Run association to new Tasks, Dispatches, Messages, deliveries, and questions. +- [x] Migrate pre-Run orchestration rows into one unbound, inspect-only legacy Run. +- [x] Keep old scheduler-run storage distinct from lightweight Runs. +- [x] Implement `run-create`, `run-use`, `run-current`, `run-list`, and `run-show`. +- [x] Bind a coordinator pane explicitly; never infer a Run from a worktree or sole candidate. +- [x] Store one active mailbox consumer generation per Run. +- [x] Rebinding fences the old consumer and cancels its active waiter. +- [x] Do not implement Run archive/delete in V1. + +### Logical routing and prompt safety + +- [x] Add stable `run:` and exact `dispatch:` message recipients. +- [x] Do not add a task-recipient retargeting rule in V1. +- [x] Make send, ask, reply, completion, heartbeat, and runtime notices inbox-only. +- [x] Ensure only explicit dispatch injection and terminal-send operations can modify terminal input. +- [x] Define send success as durable acceptance, not observation or action. +- [x] Rename or remove `check --inject` if its name implies remote delivery. +- [x] Reject explicit Run/recipient targets from federated workers when the only valid destination is + their authenticated Run home. + +### Crash-safe inbox consumption + +- [x] Add one FIFO mailbox sequence. +- [x] Bound each actionable delivery to 50 messages. +- [x] Allow one outstanding Delivery and one active actionable waiter per Run mailbox. +- [x] Return the identical Delivery ID and batch until acknowledgment. +- [x] Implement whole-batch idempotent acknowledgment. +- [x] Bind Delivery acknowledgment to the current consumer generation. +- [x] Implement atomic `ack -> check -> register waiter`. +- [x] Keep peek/all/history modes read-only. +- [x] Treat type filters as wake predicates only; return the oldest full actionable batch. +- [x] Return typed timeout, cancelled, connection-lost, waiter-exists, stale-delivery, and + consumer-fenced outcomes. +- [x] Preserve unacknowledged mail across client and Orca process restart. + +### Truthful lifecycle + +- [x] Require `outcome=succeeded|failed` on terminal worker reports. +- [x] Map authenticated succeeded reports to Dispatch succeeded and Task completed. +- [x] Map authenticated failed reports to Dispatch failed and Task failed. +- [x] Persist stale/foreign reports as history without lifecycle mutation. +- [x] Reject malformed lifecycle transitions with typed missing/invalid fields. +- [x] Label result provenance as `worker_report`. +- [x] Apply every terminal transition as one transactional compare-and-set. +- [x] First committed completion, stop fence, or abandon wins. +- [x] Make duplicate identical completion idempotent. + +### Questions + +- [x] Model a question as durable message/thread state, not a task gate. +- [x] Default ask from an active Dispatch to its owning Run mailbox. +- [x] Record one idempotent first answer from the current Run consumer generation. +- [x] Reject conflicting later answers. +- [x] Resume by original message ID after timeout or disconnect. +- [x] Recover a lost ask-acceptance response through the same mutation retry receipt. +- [x] Close pending questions when their Dispatch stops or is abandoned. +- [x] Wake closed question waits with `dispatch_inactive`. + +### Narrow pane authority + +- [x] Mint an unforgeable per-Dispatch capability at lifecycle injection. +- [x] Carry the capability outside user-controlled request parameters on native, WSL, and SSH CLI + bridges. +- [x] Persist only its verifier or secure-store reference. +- [x] Verify capability, exact managed pane, Dispatch ID, and process incarnation for lifecycle calls. +- [x] Revoke/fence the capability on stop, abandon, or replacement. +- [x] Do not generalize this into user/role access control. + +### Durable mutation ledger + +- [x] Let clients retain/reuse one opaque retry request ID after unknown acceptance. +- [x] Before effects, persist authenticated caller/peer, request ID, canonical payload hash, + operation state, and receipt. +- [x] Join concurrent identical mutations or return the recorded result. +- [x] Return `request_mismatch` for the same request ID with a changed payload. +- [x] Cover Run/Task creation, send, ask, reply, acknowledgment, start, stop, and abandon. +- [x] Persist dedupe receipts across Orca restart. + +### Phase 1 acceptance + +- [x] Two Runs on the same runtime never mix tasks or mail. +- [x] An old coordinator cannot acknowledge or reply after `run-use` fences it. +- [x] A returned but unacknowledged batch is replayed after client/runtime restart. +- [x] Success, failure, stale completion, malformed completion, and duplicate completion tests pass. +- [x] Ask timeout/resume, same reply replay, conflicting reply, and stopped-Dispatch tests pass. +- [x] A forged pane/handle field cannot mutate lifecycle state. + +## Phase 2 — Same-server composed worker lifecycle + +### Command grammar and topology + +- [x] Implement `worker-start`, `worker-show`, `worker-read`, `worker-stop`, and + `worker-abandon` for workers owned by the Run home. +- [x] Current worktree creates one fresh agent terminal unless `--terminal` is explicit. +- [x] Named existing worktree creates one fresh agent terminal unless reuse is explicit. +- [x] Current/existing worktrees do not rerun creation-time setup or configured tabs. +- [x] New child worktree uses agent-first creation and reuses its returned agent terminal. +- [x] New top-level worktree uses agent-first creation with independent Orca lineage. +- [x] Reject child/top-level creation for folder projects before effects; use current/existing folder + workspaces instead. +- [x] Pass the supported exact repo, base, lineage, display/comment metadata, and setup options to + the existing worktree primitive rather than duplicating policy. `--on` owns connected-server + placement; project/host convenience selection remains on low-level `worktree create`. +- [x] Require a configured agent launcher before any mutation. +- [x] Reject selector/option conflicts before effects for current/existing worktrees. + +### Setup and startup + +- [x] Omitted setup on a new worktree resolves to `run` for orchestration starts. +- [x] No configured setup hook resolves to `not_configured`, not failure. +- [x] Preserve repository `setupAgentStartupPolicy`. +- [x] Preserve whether setup came from an explicit request or Orca's orchestration default across + connected-server starts. +- [x] Default `start-immediately` launches setup and agent side by side. +- [x] Under `start-immediately`, setup outcome never gates Dispatch readiness regardless of when + it is observed. +- [x] Track the setup command's exit code without waiting for its interactive terminal shell to + exit or closing the setup tab. +- [x] Register the completion observer before replaying bounded recent output so fast local setup + commands cannot finish in an observation gap. +- [x] Carry the exact created setup terminal handle; never infer setup identity from a display + title shared with configured tabs or split panes. +- [x] Scope completion signals to a private per-invocation token and preserve uncertain terminal + outcomes as running rather than converting a disconnect into setup failure. +- [x] The return receipt contains the latest setup state. +- [x] Only post-return setup state changes emit a typed setup notice. +- [x] Setup failure never automatically stops or fails an already-ready worker. +- [x] Explicit `wait-for-setup` completes setup successfully before agent launch and task injection. +- [x] Under `wait-for-setup`, setup failure produces start failure before task delivery. +- [x] Custom-argv two-step launch is rejected or clearly unsupported under `wait-for-setup`. + +### Start operation and receipts + +- [x] In one transaction, create a starting Dispatch, move the Task, and record the mutation request. +- [x] Persist the accepted Dispatch ID in a pending start receipt so restart recovery returns an + exact `worker-show` command. +- [x] Persist before/after stage receipts around irreversible effects. +- [x] Return only `ready`, `failed`, or `outcome_unknown`. +- [x] Define ready as TUI idle, durable Dispatch attachment, and accepted lifecycle/task input. +- [x] Echo effective timeout, setup startup policy, defaults, and resolution sources. +- [x] Enumerate every effect: worktree, setup, agent terminal, setup terminal, each configured + terminal pane, and dispatch input; include exact tab/leaf identity when available. +- [x] Persist accepted dispatch input atomically with the ready transition so later setup refreshes + cannot erase it. +- [x] Tag every terminal effect with role and created/reused action. +- [x] Report setup as running only after its exact PTY spawn receipt is durable. +- [x] List every residual resource on failure or unknown outcome. +- [x] Never claim an effect was created before it exists. +- [x] Do not add a background provisioning executor; intentionally launched setup may continue as + its receipt states. + +### Dispatch and Task state machine + +- [x] Implement starting, ready, start-unknown, failed, succeeded, stopping, stop-unknown, stopped, + and abandoned Dispatch states. +- [x] Block the Task while start/stop outcome remains unknown. +- [x] Allow semantic `--retry-of` only from explicit failed, stopped, abandoned, or proven no-effect + states. +- [x] Require the replacement to repeat its intended placement and agent/terminal choice; do not + silently inherit a prior attempt's topology. +- [x] Reject unsafe retry without mutation. +- [x] Completed Tasks require a follow-up Task rather than retry. + +### Show, read, stop, and abandon + +- [x] Route operations by Dispatch ID after start; do not require resource IDs again. +- [x] Implement V1 `worker-read` as a thin route to bounded terminal-read. +- [x] Preserve cursor, limit, terminal status, and limited/truncated fields. +- [x] Stop fences lifecycle and blocks the Task in one home-side compare-and-set. +- [x] Stop affects only the supervised agent terminal/process. +- [x] Never delete the worktree, setup terminal, configured tabs, or unrelated processes. +- [x] Return stopped, already-settled, failed, and stop-unknown receipts truthfully. +- [x] Abandon performs no remote/process action, retains possibly-live resources, and enables a + warned replacement. + +### Same-server recovery tests + +- [x] Current, existing, child, top-level, explicit-terminal, and configured-tab starts pass. +- [x] Setup run/skip/inherit and start-immediately/wait-for-setup combinations pass. +- [x] Trust/update prompt, setup failure, terminal failure, and task-input failure receipts pass. +- [x] Crash before effect, after possible effect/before receipt, and after durable receipt are + distinguishable as failed/no-residual, outcome-unknown, and failed-with-residual respectively. +- [x] Stop/completion races preserve the first committed terminal transition. +- [x] Restart never adopts a same-looking pane or process incarnation. + +## Phase 3 — Connected Orca server federation + +### Placement and identity + +- [x] Add worker-only `--on ` without changing global `--environment` meaning. +- [x] Default worker placement to the Run home. +- [x] Require `--on` for remote existing worktree/terminal selectors in V1. +- [x] Resolve remote resources through explicit read-only discovery; never guess by name/path. +- [x] Return `server_required`, `worktree_not_found_on_server`, and + `terminal_worktree_mismatch` before related worker effects; treat a mismatched remote + Dispatch/home receipt as `resource_server_mismatch` and never adopt it. +- [x] Pin each remote Dispatch to the authenticated worker-server public-key fingerprint. +- [x] Store runtime ID only as a process epoch, never durable server identity. +- [x] Return `peer_changed` with no effect if a saved environment is re-paired to a different server. +- [x] Preserve a routing tombstone when an environment with nonterminal Dispatches is removed. + +### Remote Dispatch attachment + +- [x] Persist a narrow attachment on the worker server before task input. +- [x] Store home peer identity, Dispatch capability verifier, stable pane/process incarnation, + resource receipts, protocol version, and relay cursors. +- [x] Do not copy the Run DAG/database to the worker server. +- [x] Protect attachment credentials/database/WAL/SHM as current-user-only on macOS, Linux, and + Windows. + +### Bidirectional relay + +- [x] Use the existing authenticated saved-environment connection; require no public callback or + reciprocal pairing. +- [x] Run a Run-home subscription/pull service for active remote Dispatches. +- [x] Persist worker-to-home lifecycle/questions until home import acknowledgment. +- [x] Persist home-to-worker replies/control mail until worker import acknowledgment. +- [x] Route coordinator `send --to dispatch:` through that same durable relay and wake the + exact remote worker's local `check --wait`. +- [x] Key relay items by pinned peer, Dispatch ID, direction, monotonic sequence, and a + 128-bit-or-stronger message ID. +- [x] Import only contiguous source sequences; buffer/reject gaps. +- [x] Acknowledge only the highest contiguous committed sequence. +- [x] Assign ordinary Run-mailbox order only at home import. +- [x] Apply lifecycle transition and message import in one transaction before acknowledgment. +- [x] Enforce per-message and per-Dispatch count/byte quotas. +- [x] Coalesce heartbeats and reserve room for one terminal lifecycle report. +- [x] Return `relay_quota_exceeded`; do not add a dead-letter system. + +### Federated control and recovery + +- [x] Route show/read/stop/retry by Dispatch receipt; agents do not repeat `--on`. +- [x] Forward the same application retry request ID across home and worker server. +- [x] Return typed unknown outcome with last durable stage and exact next commands. +- [x] Reconcile a lost federated stop response from a later authoritative stopped receipt. +- [x] Treat abandonment of a superseded Dispatch as a no-op for the replacement Task. +- [x] Preserve and relay post-return federated setup evidence without changing worker lifecycle. +- [x] Recreate active relay subscriptions after Run-home restart. +- [x] Preserve worker attachment and relay state after worker-server restart. +- [x] Report running only when pane and process incarnation match after restart. +- [x] One disconnected worker server must not block local or other-server inbox delivery. +- [x] No automatic worker replacement on disconnect or silence. + +### Capability negotiation + +- [x] Advertise one aggregate `orchestrationFederationV1` control-plane capability. +- [x] Pin peer fingerprint and protocol version in the durable operation record. +- [x] Revalidate them inside the worker-side mutation, not only in a preflight probe. +- [x] Return `capability_unsupported` before Dispatch/resource/prompt effects. +- [x] Keep host/Git/setup validation inside existing primitives rather than a generalized capability + matrix. + +### Federation scenario matrix + +- [x] Post-rebase physical Mac Run home -> Windows worker preserves exact process identity through + renderer adoption, routed read, heartbeat, question/reply, completion, and stop. +- [x] Mac Run home -> Windows worker: start, completion, failure, question/reply, read, and stop. +- [x] Windows Run home -> Mac worker: the same flows through a saved Mac pairing. +- [x] Native, WSL, SSH, and relay-backed execution-host paths preserve ownership and CLI capability. +- [x] Run home restarts alone; worker server restarts alone; both restart. +- [x] Disconnect before send proves no effect. +- [x] Disconnect after possible acceptance returns unknown and deduplicates exact retry. +- [x] Duplicate and reordered relay frames/acknowledgments converge without loss or duplication. +- [x] Re-pair/key change cannot retarget an active Dispatch. +- [x] Same-looking handles/resources on two servers never cross-route. +- [x] Mixed server versions fail before effects. +- [x] Windows PowerShell quoting, Windows paths, WSL environment propagation, and SSH bridge + allowlists pass. + +## Phase 4 — Structured worker output + +- [x] Reuse Orca's exact pane/process-to-provider-session association; do not create a second status + system. +- [x] Keep bounded terminal-read as the universal fallback. +- [x] Read only Codex, Claude/OpenClaude, and Grok transcripts supported by the existing + Native Chat decoders. +- [x] Never guess the latest session by current working directory, terminal title, logo, or agent + type. +- [x] Pin Dispatch, process, source, and provider session for the full opaque cursor chain. +- [x] Preserve the existing structured native-chat message/block representation and emit bounded + parsing/clipping warnings. +- [x] Label terminal fallback and its reason explicitly. +- [x] Read transcripts on the worker-owning server and never serialize their filesystem paths. +- [x] Fall back to the legacy federated terminal-read RPC when a connected server lacks the additive + structured-read method. +- [x] Cover exact selection, sibling-session isolation, source changes, malformed input, limits, + path privacy, CLI rendering, and mixed-version fallback with automated tests. +- [x] Physically verify local Codex, two same-worktree Codex sessions, cursor continuation, + provider-session replacement, explicit terminal selection, and safe Run-home restart behavior. +- [x] Physically verify Mac Run home -> older Windows worker terminal fallback, including an opaque + continuation cursor and explicit transcript-required failure. +- [ ] Physically verify hooks-disabled automatic fallback and disconnect/reconnect. +- [ ] Physically verify exact structured Mac-to-Windows and Windows-to-Mac reads after both worker + servers run the new additive method. +- [x] Do not add resume, live-stream control, session exclusivity, or a universal transcript ontology. + +## Migration — Hard cutover from pre-Run orchestration + +### Contract and effect fence + +- [x] Add one orchestration contract version and one advertised runtime capability without changing + the global runtime protocol. +- [x] Keep one shared mutation/read classifier for CLI, runtime dispatch, durable receipts, and + connected-server calls. +- [x] Require the contract before parameter parsing, mutation receipts, database writes, prompt + injection, process actions, or connected-server mutations. +- [x] Preflight local and paired runtime capabilities before a new CLI sends a mutation. +- [x] Carry the contract through native Unix/named-pipe, WebSocket, and connected-server envelopes. +- [x] Retire `coordinator-start`, `coordinator-stop`, `run`, and `run-stop` before RPC effects. +- [x] Do not add a compatibility executor, automatic rewrite, legacy scheduler, or in-flight drain. + +### Agent recovery and legacy inspection + +- [x] Return `effectsApplied=false`, structured guide metadata, and executable argument-only + `skills get orchestration --full` recovery. +- [x] Attach the same guide recovery to no-bound-Run and missing worker outcome errors. +- [x] Preserve explicit read-only Run, task, inbox, Dispatch, gate, and terminal inspection. +- [x] Allow `task-list --run run_legacy_local` without binding the legacy Run. +- [x] Keep default/actionable check and acknowledgment fenced; only explicit peek/all history reads + may inspect legacy mail. +- [x] Document that active pre-upgrade agents keep running as processes but are unsupervised and + must be inspected before replacement. +- [x] Remove the legacy scheduler recipe from the version-matched full orchestration guide. + +### Migration acceptance + +- [x] Missing/wrong contract rejects every classified mutation before parsing, receipt, and effect. +- [x] Current-contract mutations still execute and retain durable retry receipts. +- [x] Read-only inspection works without a contract and does not consume legacy data. +- [x] Local and remote clients reject a runtime missing the contract capability before mutation. +- [x] Native and encrypted WebSocket transports preserve the contract field. +- [x] Old `worker_done` leaves message, Task, and Dispatch state unchanged. +- [x] Human and JSON errors preserve no-effects and guide-reload recovery. +- [x] `skills get orchestration --full` remains runtime-independent and generated guides stay in + sync. +- [ ] Branch-head CI passes after the verified migration commit is pushed. Local orchestration, + repository tests, typechecks, reliability gates, and production builds pass. + +## Cross-cutting quality gates + +### Persistence and transactions + +- [x] Define process-crash durability separately from sudden-power-loss durability. +- [x] Keep SQLite WAL + `synchronous=NORMAL` for the documented process-crash guarantee; require a + separate policy change before promising sudden-power-loss durability. +- [x] Keep lifecycle import, terminal transitions, and acknowledgment transaction boundaries explicit. +- [x] Exercise migrations from existing task/message/dispatch/scheduler-run data. + +### Cross-platform + +- [x] Native macOS, Linux, and Windows tests cover each new CLI/RPC contract. +- [x] WSL and SSH host identity/capability state is scoped to the actual execution host. +- [x] Paths use platform utilities; examples are PowerShell/cmd/POSIX safe. +- [x] Named-pipe, Unix-socket, WebSocket, WSL, and SSH bridges carry Dispatch capabilities safely. + +### Documentation + +- [x] CLI help owns exact flags, selectors, defaults, outcome fields, and exit-code behavior; typed + RPC errors remain the machine-readable error contract. +- [x] The skill owns short decision recipes and common misuses, not protocol internals. +- [x] Every shipped phase updates this checklist and adds a progress-log entry. +- [x] The ignored HTML proposal and this tracked checklist remain semantically synchronized. + +## Findings and decision log + +### 2026-07-22 — Phase 2 closure without option-surface creep + +- `worker-start` passes exact repository, base, child/top-level lineage, display/comment metadata, + and setup choices into the existing worktree primitive. It does not duplicate `worktree create`'s + project/host convenience resolver: `--on` already names the connected Orca server and `--repo` + names the repository on that server. +- A gated setup receipt becomes `succeeded` only after the agent wrapper proves setup completed. A + confirmed spawn/script failure fails before task input; timeout keeps `running` because silence is + not failure. +- The existing single durable worker row is the operation stage journal. A pre-effect failure has no + residuals, possible acceptance before receipt is unknown, and later failure after a durable effect + lists exact residual resources. No background saga executor or general effect engine was added. +- This earlier Phase 4 deferral was based on an incomplete audit. Orca already retained an exact + pane-scoped provider-session association from agent hooks; the narrow implementation now exposes + it to the worker-owning runtime and still falls back when that evidence is absent. + +### 2026-07-22 — Final setup/startup review + +- Keep setup-run as the new-worktree orchestration default. +- Preserve Orca's existing `start-immediately` default: setup and agent run side by side. +- Only explicit `wait-for-setup` gates agent launch/task delivery. +- A custom-argv two-step terminal launch cannot preserve wait-for-setup and must not silently bypass + it. +- Receipts must enumerate role-tagged agent, setup, and configured terminals; a boolean + `setupSpawned` is insufficient. +- Setup outcome never gates readiness under start-immediately, regardless of observation timing. + +### 2026-07-22 — Federation robustness review + +- Multi-server operation is a core requirement, not a later optional product feature. +- Use a single authoritative Run home with narrow remote Dispatch attachments and bidirectional + relay; do not replicate the Run database. +- Pin active Dispatches to authenticated peer identity so re-pairing cannot redirect work. +- Use contiguous, scoped relay sequences and bounded storage. +- Hide peer fingerprints, relay cursors, process incarnations, and capabilities from normal agents. + +### 2026-07-22 — Validation and scope boundary + +- Phase 0 and Phase 1 are complete: their command, recipe, Run, inbox, lifecycle, question, + authority, and mutation-ledger acceptance rows now have focused passing tests. +- Phase 2 remains open until the full existing-worktree/explicit-terminal/setup-policy/failure-stage + matrix is covered. Passing current/new-worktree and recovery slices are not enough to claim it. +- Phase 3 remains open until the named Mac/Windows, WSL, SSH, relay, restart, disconnect, and quoting + matrix runs on those actual paths. The in-process federation harness proves protocol behavior but + is not a substitute for cross-platform acceptance. + - At this point Phase 4 stayed deferred pending proof of an exact association. The later + structured-output audit found the existing pane-scoped hook association and superseded this + decision without adding a universal provider framework. + +### 2026-07-21 — Simplification decision + +- Replace the original broad orchestration redesign with four public concepts: Run, Task, Dispatch, + and Message. +- Keep agent-owned strategy and strong control-plane primitives. +- Remove UI, scheduler, capacity allocation, access enforcement, commit/integration tracking, + generalized provider/session abstractions, and other speculative product machinery. +- Retain Run because it provides a durable namespace and home mailbox across connected servers, not + because it schedules work. + +## Progress log + +Append new entries chronologically. Do not rewrite older entries except to correct factual errors. + +### 2026-07-22 — Checklist initialized + +- Changes: + - Created this tracked implementation ledger from the reviewed orchestration proposal. + - Recorded all phases, acceptance tests, non-goals, and review-derived invariants. + - Updated the current orchestration skill to prefer setup-run, preserve start-immediately, process + message batches, prefer `agentTerminalHandle`, and reject custom-argv wait-policy bypass. +- Files: + - `ORCHESTRATION_IMPLEMENTATION_CHECKLIST.md` + - `skills/orchestration/SKILL.md` + - `docs/orchestration-primitives.html` (ignored design source) +- Verification: + - HTML parsed successfully with balanced tags and unique IDs. + - Final review Task completed without an architectural blocker. +- Findings: + - Implementation may begin with Phase 0. + - Phase 2 readiness/receipt work depends on the explicit setup and role-tagged-effect contracts + recorded above. +- Next: + - Finish Phase 0 command compatibility and version-matched recipes. + +### 2026-07-22 — Phase 0 command safety and vocabulary + +- Changes: + - Renamed the scheduler-like command surface to `coordinator-start` and `coordinator-stop` while + retaining `run` and `run-stop` as documented deprecated aliases. + - Updated root help and both orchestration skill sources to distinguish the legacy automatic loop + from the proposed lightweight Run namespace and to prefer the explicit task/dispatch/wait loop. + - Made `orchestration reset` require exactly one of `--all`, `--tasks`, or `--messages` before it + contacts the runtime. + - Synchronized setup, worker-terminal selection, and message-batch guidance into the canonical + guide and regenerated the bundled CLI guide. +- Files: + - `src/cli/specs/orchestration.ts` + - `src/cli/handlers/orchestration.ts` + - `src/cli/help.ts` + - `src/cli/handlers/orchestration.test.ts` + - `src/cli/index.test.ts` + - `src/main/runtime/orchestration-cli-subprocess.test.ts` + - `skill-guides/orchestration.md` + - `skills/orchestration/SKILL.md` + - `src/cli/bundled-skill-guides.ts` +- Verification: + - `pnpm vitest run --config config/vitest.config.ts src/cli/index.test.ts src/cli/handlers/orchestration.test.ts` — 199 tests passed. + - `pnpm verify:bundled-skill-guides` — passed. + - `pnpm typecheck:cli` — passed. + - `git diff --check` — passed. +- Findings: + - Keeping hidden compatibility aliases preserves existing scripts without advertising the old + scheduler noun as the normal agent path. + - Truthful success/failure recipes depend on the Phase 1 explicit lifecycle outcome; do not + document the current behavior as if a failed `worker_done` failed the Task. +- Next: + - Implement explicit succeeded/failed worker-report semantics and then finish the version-matched + Phase 0 recipes without lying about failure behavior. + +### 2026-07-22 — Truthful worker terminal outcomes + +- Changes: + - Added the structured `--outcome succeeded|failed` worker-report field to local and SSH fallback + CLI payload construction and injected preambles. + - Added one transactional compare-and-set that settles the Dispatch and Task together, promotes + dependents only on success, and replays an identical terminal outcome idempotently. + - Persisted worker result provenance, message identity, summary, files, and report path as a + labeled `worker_report` rather than an Orca-verified result. + - Converted missing, invalid, unknown, stale, mismatched, inactive, and foreign reports into + typed, high-priority audit rows without mutating lifecycle state. + - Updated the legacy automatic coordinator loop to record failed worker reports as failed tasks. +- Files: + - `src/cli/specs/orchestration.ts` + - `src/cli/handlers/orchestration.ts` + - `src/main/ssh/ssh-remote-orchestration-send.ts` + - `src/main/runtime/orchestration/types.ts` + - `src/main/runtime/orchestration/db.ts` + - `src/main/runtime/orchestration/lifecycle-reconciliation.ts` + - `src/main/runtime/orchestration/preamble.ts` + - `src/main/runtime/orchestration/coordinator.ts` + - Corresponding CLI, SSH, preamble, DB lifecycle, coordinator, and RPC tests/snapshots + - Both orchestration skill sources and the generated bundled guide +- Verification: + - Five focused CLI/lifecycle/preamble/coordinator/SSH test files — 124 tests passed. + - `src/main/runtime/rpc/methods/orchestration.test.ts` — 114 tests passed. + - `pnpm typecheck:node` — passed. + - `pnpm typecheck:cli` — passed. + - `pnpm verify:bundled-skill-guides` and `git diff --check` — passed. +- Findings: + - The old subject-based failure convention was not merely confusing: it irreversibly completed a + failed Task. Requiring a tiny enum is a robust primitive, not workflow policy. + - Stop and abandon still need to share this terminal-transition fence before the broader + first-writer-wins checklist item can be marked complete. +- Next: + - Add the lightweight Run schema and explicit coordinator binding, then key new Task/Dispatch/ + Message state to that Run without changing agent placement policy. + +### 2026-07-22 — Lightweight Run foundation and inbox-only mail + +- Changes: + - Added schema v7 with lightweight Runs, stable explicit pane binding, consumer generations, and + an inspect-only legacy Run for all migrated pre-Run rows. + - Kept legacy automatic coordinator-loop storage in its existing `coordinator_runs` table. + - Added `run-create`, `run-use`, `run-current`, `run-list`, and `run-show` across CLI/RPC, with + explicit binding and no worktree or sole-candidate inference. + - Scoped new Task creation/list/update/dispatch operations to an explicit or currently bound Run; + Dispatches and decision gates inherit their Task's Run. + - Removed structured-mail prompt injection from send and ask. Mail now persists and wakes waiters + only; deliberate `dispatch --inject` and `terminal send` remain the input-writing paths. + - Renamed the local message renderer from `check --inject` to `check --format`, retaining only a + one-release RPC compatibility field. +- Files: + - `src/main/runtime/orchestration/db.ts` + - `src/main/runtime/orchestration/types.ts` + - `src/main/runtime/rpc/methods/orchestration-runs.ts` + - `src/main/runtime/rpc/methods/orchestration.ts` + - `src/main/runtime/orchestration/orchestration-error.ts` + - `src/main/runtime/rpc/errors.ts` + - `src/cli/specs/orchestration.ts` + - `src/cli/handlers/orchestration.ts` + - Related DB, RPC, CLI, and help tests; both skill sources and the bundled guide +- Verification: + - Focused DB/RPC/CLI command suite — 384 tests passed. + - `pnpm typecheck:node` and `pnpm typecheck:cli` — passed. + - `pnpm generate:bundled-skill-guides` and `pnpm verify:bundled-skill-guides` — passed. + - `git diff --check` — passed. +- Findings: + - Run identity is now real without changing scheduling or placement policy, but Messages and + questions still need stable logical recipients before mandatory Run association is complete. + - Consumer generation exists, but waiter cancellation and acknowledgment fencing belong to the + crash-safe Delivery implementation and remain intentionally unchecked. +- Next: + - Implement Run-owned logical mailboxes and stable `run:` / `dispatch:` routing, then + replace consume-on-read with one crash-safe outstanding Delivery per Run. + +### 2026-07-22 — Crash-safe Run inbox and durable questions + +- Changes: + - Added stable `run:` and `dispatch:` recipients; lifecycle sends from an active Dispatch + now default to its Run and no longer require agents to carry a coordinator terminal handle. + - Added schema v8 Deliveries: one FIFO batch of at most 50 rows, one outstanding batch per Run, + exact replay until whole-batch acknowledgment, and consumer-generation fencing. + - Made `check --ack --wait` perform ack, check, and waiter registration without an + intervening async gap; type filters are wake predicates and never split the FIFO batch. + - Added typed timeout, cancellation/connection-loss, second-waiter, stale-Delivery, and fenced- + consumer outcomes, plus persisted Delivery replay after an Orca database reopen. + - Added schema v9 question threads keyed by the original message ID, with active-Dispatch Run + defaulting, timeout-safe resume, current-consumer first-answer authority, idempotent replay, and + conflicting-answer rejection. + - Updated injected worker guidance and the orchestration skill so normal lifecycle and question + commands omit internal Run/server identity and coordinators explicitly process/ack each batch. +- Files: + - `src/main/runtime/orchestration/types.ts` + - `src/main/runtime/orchestration/db.ts` + - `src/main/runtime/orca-runtime.ts` + - `src/main/runtime/rpc/methods/orchestration.ts` + - `src/main/runtime/rpc/methods/orchestration-runs.ts` + - `src/main/runtime/orchestration/preamble.ts` + - CLI specs/handlers, RPC error mapping, skill sources, snapshots, and focused tests +- Verification: + - DB/RPC/runtime/CLI/preamble focused suite — 1,190 tests passed after the expected preamble + snapshot update. + - Delivery tests cover 50-row bounds, FIFO replay, idempotent ack, filter wake semantics, + consumer fencing, and reopen recovery. + - Question tests cover create/answer, same-answer replay, answer conflict, timeout persistence, + resume, unrelated wakes, and Dispatch closure storage behavior. +- Findings: + - A Run inbox needs only one durable Delivery row plus immutable message IDs; no dead-letter, + selective NACK, or second Event subsystem is necessary. + - Question answers belong in thread state rather than inbox-read state, so replying never + accidentally acknowledges the coordinator's whole Delivery. +- Next: + - Replace caller-supplied pane claims with a narrow Dispatch capability and add the durable + mutation ledger needed to recover unknown acceptance without replaying effects. + +### 2026-07-22 — Narrow Dispatch lifecycle capability + +- Changes: + - Minted a 256-bit per-Dispatch secret for injected workers while persisting only its SHA-256 + verifier. + - Bound lifecycle/question authority to the exact runtime-observed pane and PTY process + incarnation in addition to the Dispatch ID. + - Carried the secret in the authenticated RPC envelope across local socket, WebSocket, + shared remote-runtime, and SSH fallback transports rather than orchestration payload fields. + - Revoked the capability when worker completion/failure settles the Dispatch and stopped trusting + caller-supplied pane metadata in the SSH fallback. +- Files: + - `src/main/runtime/orchestration/db.ts` + - `src/main/runtime/orchestration/types.ts` + - `src/main/runtime/orchestration/preamble.ts` + - `src/main/runtime/rpc/core.ts` + - `src/main/runtime/rpc/dispatcher.ts` + - `src/main/runtime/rpc/methods/orchestration.ts` + - CLI and remote transport implementations plus focused tests +- Verification: + - Ten focused DB/RPC/runtime/CLI/SSH files — 1,238 tests passed. + - Capability cases cover missing token, wrong token, wrong pane, changed process incarnation, + success, and post-settlement revocation. + - `pnpm typecheck:node` and `pnpm typecheck:cli` — passed. +- Findings: + - Terminal handles and environment-provided pane strings are useful routing metadata but are not + lifecycle authority. + - Stop, abandon, and replacement must use the same revocation fence before that remaining + checklist item can be completed. +- Next: + - Add the durable mutation ledger and use it to recover unknown acceptance without repeating + external effects. + +### 2026-07-22 — Durable mutation receipts + +- Changes: + - Added schema v11 mutation receipts keyed by an authenticated-caller fingerprint and opaque + request ID, with canonical payload hashing and pending/completed state. + - Replayed completed results across retries and restart, joined concurrent identical mutations, + rejected changed input as `request_mismatch`, and surfaced orphaned pending work as + `operation_unknown`. + - Added request IDs to local socket, named-pipe, WebSocket, saved-environment, and SSH fallback + envelopes; successful receipts echo the ID while transport failures retain it as recovery data. + - Persisted blocking-question acceptance before waiting, so retry after a lost response returns + the original question instead of creating another. +- Files: + - `src/main/runtime/orchestration/db.ts` + - `src/main/runtime/orchestration/types.ts` + - `src/main/runtime/rpc/core.ts` + - `src/main/runtime/rpc/dispatcher.ts` + - Runtime-client and transport files, CLI orchestration handlers/specs, and focused tests +- Verification: + - Twelve focused DB/RPC/runtime/CLI/SSH files — 1,260 tests passed. + - `pnpm typecheck:node`, `pnpm typecheck:cli`, and `git diff --check` — passed. +- Findings: + - Generic control-plane mutations can safely discard a pending receipt when their handler returns + a known failure; future worker start/stop must instead return and preserve typed unknown outcomes + around external effects. + - The ledger infrastructure covers every existing V1 mutation; worker start/stop/abandon will be + added to the same policy when those commands exist. +- Next: + - Implement the same-server worker lifecycle and its first-writer-wins stop/abandon fence. + +### 2026-07-22 — Same-server worker lifecycle foundation + +- Changes: + - Added schema v12 composed-worker state and created the starting Dispatch plus Task transition + before terminal effects. + - Added synchronous `worker-start` for the current or an exact existing worktree, with fresh-agent + default, explicit-terminal reuse, TUI readiness, capability attachment, lifecycle injection, and + honest failed receipts with residual resources. + - Added Dispatch-routed `worker-show` and bounded `worker-read`. + - Added first-writer-wins `worker-stop` and `worker-abandon`; stop closes only the supervised agent + terminal, while abandon performs no process/filesystem action and retains residual receipts. + - Closed and woke pending question waits when a worker is settled, stopped, or abandoned. +- Files: + - `src/main/runtime/orchestration/db.ts` + - `src/main/runtime/orchestration/types.ts` + - `src/main/runtime/rpc/methods/orchestration-workers.ts` + - `src/main/runtime/rpc/methods/orchestration.ts` + - `src/main/runtime/orca-runtime.ts` + - CLI orchestration specs/handlers and focused DB/RPC/CLI tests +- Verification: + - Focused DB/RPC/CLI command suite — 412 tests passed. + - `pnpm typecheck:node` and `pnpm typecheck:cli` — passed. +- Findings: + - Keeping composed state in a narrow extension table preserves legacy Dispatch compatibility while + giving worker operations the richer start/stop states they need. + - New-worktree setup, stage journaling around worktree creation, and true unknown-start recovery + remain open; current/existing workers are the verified slice. +- Next: + - Add agent-first child/top-level creation with setup-run default and startup-policy receipts, then + extend the same Dispatch routing across connected Orca servers. + +### 2026-07-22 — Agent-first new-worktree workers and setup receipts + +- Changes: + - Added child and top-level agent-first worktree creation to `worker-start`, reusing the exact + startup agent terminal and listing setup/configured terminals as role-tagged effects. + - Made omitted setup resolve to `run`, preserved explicit run/skip/inherit and repository startup + policy, and returned `not_configured` when no setup hook exists. + - Kept `start-immediately` setup non-gating, persisted later setup success/failure, and emitted a + typed Run notice without changing a ready worker's lifecycle state. + - Narrowed setup receipts to callers that explicitly await terminal provisioning so ordinary + worktree creation does not report renderer-delegated setup as a false spawn failure. + - Added typed unknown-start recovery when worktree creation may have been accepted before a + connection failure. +- Files: + - `src/main/runtime/orca-runtime.ts` + - `src/main/runtime/rpc/methods/orchestration-workers.ts` + - `src/main/runtime/rpc/methods/orchestration-workers-new-worktree.test.ts` + - `src/main/runtime/rpc/methods/orchestration.test.ts` + - `src/shared/types.ts` +- Verification: + - New-worktree worker scenarios — 8 tests passed. + - Worker RPC plus runtime worktree suites — 911 tests passed. + - `pnpm typecheck:node` — passed. +- Findings: + - A setup status is only truthful after the supervised caller awaits the runtime's terminal + provisioning result; normal renderer-delegated creation should retain its existing launch + payload instead. + - The generic mutation receipt and starting Dispatch are still separate commits, so atomic start + acceptance and crash-boundary reconciliation remain open. +- Next: + - Make worker-start acceptance one transaction, then add restart reconciliation that never adopts + a same-looking pane or process. + +### 2026-07-22 — Atomic worker acceptance and conservative restart recovery + +- Changes: + - Moved the worker-start retry request insertion into the same SQLite transaction that creates the + starting Dispatch and moves its Task to dispatched. + - Added a process runtime epoch to composed Dispatches so interrupted starts/stops become explicit + unknown outcomes after restart instead of remaining indefinitely in transitional states. + - Made worker show/read/stop verify the persisted stable pane plus exact PTY process incarnation; + a same-looking replacement is reported as changed and is never read or closed. + - Restricted semantic retry to the Task's latest failed, stopped, or abandoned Dispatch. +- Files: + - `src/main/runtime/orchestration/db.ts` + - `src/main/runtime/orchestration/types.ts` + - `src/main/runtime/rpc/core.ts` + - `src/main/runtime/rpc/dispatcher.ts` + - `src/main/runtime/rpc/errors.ts` + - `src/main/runtime/rpc/methods/orchestration-workers.ts` + - Focused DB, mutation-ledger, new-worktree, and recovery tests +- Verification: + - Atomic acceptance and recovery suite — 231 tests passed. + - Follow-up DB/recovery/new-worktree suite — 93 tests passed. + - `pnpm typecheck:node` — passed. +- Findings: + - Runtime ID is useful only as a process epoch. It is not a durable server identity and must not be + used to route federated Dispatches. + - After restart, preserving uncertainty is safer than adopting a restored pane: the prior worker + may still exist, but only explicit stop/abandon/retry recovery may replace it. +- Next: + - Audit and close the remaining same-server acceptance rows, then implement saved-environment + placement and the narrow federated Dispatch attachment. + +### 2026-07-22 — Connected-server federation and crash-safe relay + +- Changes: + - Added worker-only `--on ` placement while keeping Run/Task authority on the + current server and routing later show/read/stop operations solely by Dispatch ID. + - Pinned remote Dispatches to the saved server public-key fingerprint and stored runtime identity + only as a replaceable process epoch; re-pairing returns `peer_changed` before effects. + - Added a narrow worker-server attachment with protocol version, capability verifier, exact + pane/process identity, effects, setup state, and bidirectional relay cursors—without copying the + Run DAG. + - Added durable worker-to-home lifecycle/question relay and home-to-worker reply relay with + contiguous sequence checks, source acknowledgment, quotas, heartbeat coalescing, and reserved + terminal-report capacity. + - Made home import commit the message, question/lifecycle transition, and source cursor in one + transaction before acknowledgment. + - Added federated show/read/stop, stop/completion ordering, timeout/resume after worker restart, + relay restart, exact-process checks, ack-loss replay, gap rejection, peer-change fencing, and + POSIX database/WAL/SHM permission coverage. Windows uses Orca's existing current-user-only + userData DACL boundary. + - Added an agent-facing cookbook for local fan-out, setup-default new worktrees, Mac/Windows + placement, completion/failure, ask/resume/reply, and conditional recovery. +- Files: + - `src/main/runtime/orchestration/environment-transport.ts` + - `src/main/runtime/orchestration/federation-sync.ts` + - `src/main/runtime/orchestration/db.ts` + - `src/main/runtime/rpc/methods/orchestration-federation.ts` + - `src/main/runtime/rpc/methods/orchestration-workers.ts` + - Federation, permissions, CLI, transport, skill-guide, and protocol capability tests/sources +- Verification: + - Federation scenarios — 14 tests passed. + - Federation plus database-permission scenarios — 15 tests passed. + - Focused database/lifecycle/federation regression suite — 111 tests passed. + - `pnpm typecheck:node` and bundled skill-guide verification — passed. +- Findings: + - The stable saved-environment public-key fingerprint is server identity; a runtime UUID is only an + epoch and must never retarget a Dispatch after restart. + - The worker server needs only an authenticated attachment and relay outbox, not a replicated Run + database, scheduler, callback listener, or general ACL system. + - Setup remains explicit in receipts but non-gating by default; only repository + `wait-for-setup` policy delays agent launch. +- Next: + - Run the complete orchestration/CLI regression matrix, close any remaining checklist gaps, then + run repository validation and prepare the commit/PR. + +### 2026-07-22 — Final modularization and verification pass + +- Changes: + - Split federation, worker control/observation/topology, mutation execution, CLI specs, runtime + transport support, SSH error formatting, and database/CLI tests into domain-named modules so no + new max-lines bypass was needed. + - Regenerated the bundled skill manifests after the orchestration guide changed. + - Updated skill-guidance and lifecycle-rejection tests for the renamed legacy coordinator command, + setup-run default, agent-terminal fallback, and mandatory worker outcome. + - Kept unrelated daemon/UI/localization baseline failures out of the orchestration diff. +- Files: + - `src/main/runtime/rpc/methods/orchestration-federation-*.ts` + - `src/main/runtime/rpc/methods/orchestration-worker-*.ts` + - `src/main/runtime/orchestration/orchestration-*-db.test.ts` + - `src/main/runtime/rpc/orchestration-mutation-executor.ts` + - `src/main/runtime/rpc/runtime-feature-interaction.ts` + - `src/main/ipc/runtime-environment-shared-control-support.ts` + - `src/cli/specs/orchestration-worker-specs.ts` + - `src/cli/handlers/orchestration-run-cli.test.ts` + - `src/main/ssh/ssh-remote-cli-error-response.ts` +- Verification: + - Focused orchestration/CLI/SSH suite: 20 files, 565 tests passed. + - Runtime/subprocess/transport suite: 780 passed, 2 skipped. + - Updated skill-guidance/lifecycle-rejection tests: 12 passed. + - Full TypeScript check (Node, CLI, web), ordinary `oxlint`, max-lines ratchet, reliability gates, + bundled guide verification, manifest verification, and localization catalog verification passed. + - `pnpm test` could not start because the patched `node-pty` artifact does not load under local + Node 24.18.0. Direct full Vitest ran 33,058 passing tests; its 24 remaining failures and 3 worker + errors were native-PTY or unrelated timeout/baseline failures after the three stale + orchestration assertions were fixed and rerun. + - Full `pnpm lint` remains blocked only by unrelated existing switch-exhaustiveness and + localization-coverage failures outside this change. +- Findings: + - The implemented protocol has a clean authority split: the Run home owns orchestration truth; + connected worker servers own only exact resources, an authenticated Dispatch attachment, and + durable relay state. + - The remaining unchecked Phase 2/3 rows are real acceptance work, not reasons to add a scheduler, + UI, generalized capability matrix, provider-session layer, or automatic recovery. +- Next: + - Add the missing focused Phase 2 scenarios and run the Phase 3 matrix on real Mac/Windows and + WSL/SSH paths before marking those phases complete. + +### 2026-07-22 — Phase 2 setup and receipt acceptance complete + +- Changes: + - Made wait-for-setup receipts settle to `succeeded` only after gated agent readiness and fail at + `setup_start` or `setup_wait` before lifecycle/task input when setup is confirmed failed. + - Preserved `running` on a gated timeout, avoiding a false setup-failure claim. + - Added stage, role-tagged dispatch input, rich setup effect data, and exact terminal tab/leaf + coordinates to composed-worker receipts. + - Kept worker-start's option surface narrow: exact server via `--on`, exact repo via `--repo`, and + pass-through base/lineage/display/comment/setup choices. + - Updated CLI help, both skill sources, the generated guide/manifests, and the ignored HTML design. +- Files: + - `src/main/runtime/rpc/methods/orchestration-workers.ts` + - `src/main/runtime/rpc/methods/orchestration-worker-topology.ts` + - `src/main/runtime/rpc/methods/orchestration-federation.ts` + - `src/main/runtime/rpc/methods/orchestration-federated-worker-start.ts` + - `src/main/runtime/rpc/methods/orchestration-federation-effects.ts` + - Focused worker, federation, and CLI tests plus help/skill/checklist/design sources +- Verification: + - Local/new-worktree/federation worker suites: 166 tests passed. + - CLI handler/help suite: 197 tests passed. + - Full Node/CLI/web TypeScript check passed. +- Findings: + - The existing worktree startup wrapper already enforces setup-before-agent ordering; the missing + work was truthful orchestration state and acceptance coverage, not a second setup runner. + - Phase 2 is complete. Remaining unchecked rows are the real connected-platform Phase 3 matrix + and cross-platform transport evidence. +- Next: + - Exercise federation disconnect/reorder/restart semantics, then run branch-head Mac/Windows + acceptance without replacing either production Orca runtime. + +### 2026-07-22 — Native Mac-to-Windows acceptance gaps + +- Changes: + - Built and launched branch-head Mac and Windows servers on isolated profiles and paired them over + the existing authenticated WebSocket transport. + - Used a temporary, exact Tailscale TCP proxy because Windows Firewall correctly blocked the new + test-binary port; production Orca and firewall policy were left unchanged. + - Added explicit dev-CLI provenance so custom profile paths still generate `orca-dev` worker + commands. + - Increased only the Windows ConPTY bracketed-paste render gap before Enter from 500 ms to 1.5 s. +- Files: + - `config/scripts/orca-dev.mjs` + - `src/cli/handlers/orchestration.ts` + - `src/shared/agent-prompt-injection.ts` + - Focused wrapper, CLI, and prompt-injection tests +- Verification: + - Native Windows discovery, exact worktree routing, pre-effect failure, retry, ready receipt, + worker-read, and remote worker-stop all returned truthful branch-head receipts. + - Focused dev-provenance and prompt-injection suite: 160 tests passed. +- Findings: + - A fresh Windows agent profile surfaces trust/login/update prompts as typed `agent_readiness` + failures with residual terminals, as designed. + - The first authenticated Windows worker proved that 500 ms could leave a long preamble in the + Codex input buffer, and that a custom dev profile could incorrectly call the production CLI. + - Phase 3 remains open until the fixes are rebuilt on Windows and the full completion/question/ + failure/restart matrix succeeds without a manual Enter or CLI substitution. +- Next: + - Rebuild both branch servers with these fixes and repeat the real Mac-home to Windows-worker flow. + +### 2026-07-22 — Headless Windows dev-worker CLI routing + +- Changes: + - Made every `orca-dev` entry path install profile-scoped `orca-dev` and `orca` terminal wrappers, + including Windows `.cmd` wrappers and headless `orca-dev serve`. + - Kept the wrapper generation shared with the Electron dev runner so interactive and headless dev + servers expose the same exact CLI and user-data profile to worker terminals. +- Files: + - `config/scripts/dev-cli-terminal-wrapper.mjs` + - `config/scripts/orca-dev.mjs` + - `config/scripts/run-electron-vite-dev.mjs` + - Focused cross-platform wrapper tests +- Verification: + - Wrapper, CLI provenance, dev-runner, preamble, and orchestration handler suites: 55 tests passed. + - Focused oxlint and formatting checks passed. +- Findings: + - The rebuilt prompt submitted automatically on Windows, but the worker then found no + profile-scoped `orca-dev` command because headless serve bypassed the Electron dev runner and the + runner itself had never written Windows wrappers into the PATH directory used by Orca terminals. + - This is a dev/acceptance launcher defect, not a new federation primitive or production routing + requirement. +- Next: + - Rebuild and restart the Windows branch server, then repeat the same Dispatch and require an + automatically relayed `worker_done` before checking any federation acceptance row. + +### 2026-07-22 — Mac-home to Windows-worker federation accepted + +- Changes: + - Fast-forwarded and restarted the isolated Windows branch server with the profile-scoped wrapper + fix while preserving its authenticated server key and saved-environment binding. + - Exercised separate success, intentional failure, and blocking question/reply Dispatches from the + isolated Mac Run home to native Windows Codex workers. +- Files: + - `ORCHESTRATION_IMPLEMENTATION_CHECKLIST.md` +- Verification: + - Success `ctx_f982ddb1bdf9` submitted without manual input, relayed `worker_done`, and atomically + settled its Task/Dispatch as completed/succeeded. + - Failure `ctx_5f28665cf04a` relayed `outcome=failed` and atomically settled its Task/Dispatch as + failed without treating failure prose as success. + - Question `ctx_4aec24c47e30` relayed a typed question to the Mac Delivery, carried the `blue` reply + back to the blocked Windows ask, then relayed a successful terminal report. + - Routed bounded read and exact-agent stop were also exercised against Windows Dispatch receipts; + stop closed only the accepted agent terminal. +- Findings: + - Windows ConPTY prompt submission, profile-specific CLI selection, authenticated lifecycle + acceptance, contiguous bidirectional relay, whole-batch acknowledgment, and terminal-state + reconciliation now pass together in the real Mac-to-Windows path. + - This completes only the named Mac-home to Windows-worker row; reverse direction, restart with an + active Dispatch, disconnect/unknown-outcome, WSL/SSH/relay-host, and transport coverage remain + open. +- Next: + - Pair the isolated Mac server into the Windows test profile and run the same acceptance flow with + Windows as the Run home. + +### 2026-07-22 — Windows-home to Mac-worker federation accepted + +- Changes: + - Added a reciprocal saved Mac environment to the isolated Windows profile without exposing its + pairing credential in terminal history. + - Exercised separate success, intentional failure, blocking question/reply, bounded read, and exact + stop Dispatches with Windows as Run home and native macOS as worker server. + - Made worker observations report `exited` for the exact disconnected terminal instead of + misleadingly projecting `running`; stop now refuses to close an exact-but-exited process again. +- Files: + - `src/main/runtime/rpc/methods/orchestration-worker-observation.ts` + - `src/main/runtime/rpc/methods/orchestration-federation-control.ts` + - `src/main/runtime/rpc/methods/orchestration-worker-stop.ts` + - Focused local/federated observation and stop tests +- Verification: + - Success `ctx_5188e1f8417e` relayed from macOS and settled at the Windows Run home. + - Failure `ctx_a3418dc84ed6` relayed `outcome=failed` and settled failed at the Windows home. + - Question `ctx_a2521b88b6c9` carried `square` from Windows to the blocked macOS ask, followed by a + successful terminal report. + - Stop `ctx_bec120349d3f` first proved routed read against the exact Mac worker, then closed only that + terminal and settled stopped/failed; the observation regression suites pass 36 tests. +- Findings: + - One reciprocal pairing is sufficient for a Windows-owned Run to route Mac worker control while + preserving a single Run database on Windows; no replicated scheduler or failover layer is needed. + - Stable process identity and live process status are separate facts. A disconnected terminal can + still be the exact historical worker, but it must not be labeled running or closed again. +- Next: + - Validate active-Dispatch restart/disconnect and exact-retry behavior, then exercise WSL/SSH/relay + execution-host propagation without broadening the federation protocol. + +### 2026-07-22 — Federated restart and pre-acceptance disconnect accepted + +- Changes: + - Restarted the Windows Run home alone, the macOS worker server alone, and both servers while each + had an active federated Dispatch. + - Stopped the macOS worker server before a Windows-home start request could be accepted. +- Files: + - `ORCHESTRATION_IMPLEMENTATION_CHECKLIST.md` +- Verification: + - Home-only restart preserved `ctx_f0693ff30c27`; `worker-show` found the same exact running macOS + worker under the unchanged worker epoch, and routed stop succeeded. + - Worker-only restart preserved `ctx_3bbb0142f9aa` at its Run home while the new macOS epoch + truthfully reported the missing terminal as non-exact; routed stop returned `stop_unknown` + without adopting or closing another process. + - Restarting both sides preserved `ctx_d46f68fa1400` and its attachment; inspection used the new + worker epoch, reported `missing` with `exactWorker=false`, and stop again returned + `stop_unknown` safely. + - With macOS already unreachable, retry request `disconnect-before-send-01` created neither a + Dispatch nor an attachment for `task_04fd0dc61065`; the Task remained ready. +- Findings: + - Durable home state and worker identity fencing survive independent epochs without requiring Run + replication or authority failover. + - A failed connection before remote acceptance is a clean no-effect result; it must not be + promoted to an ambiguous outcome or consume the Task. +- Next: + - Disconnect the worker route after possible acceptance, then follow the returned exact recovery + command and prove that retry deduplicates to one remote effect. + +### 2026-07-22 — Post-acceptance disconnect deduplicated + +- Changes: + - Cut the Windows Tailscale proxy while a Mac-home `worker-start` was provisioning remotely, then + restored the same route and replayed the exact application request ID. + - Allowed an explicit `worker-stop` to fence `start_unknown` locally and on the worker server; + exact pane/process observation still decides whether a terminal may actually be closed. +- Files: + - `src/main/runtime/orchestration/db.ts` + - `src/main/runtime/orchestration/orchestration-worker-dispatch-db.test.ts` + - `ORCHESTRATION_IMPLEMENTATION_CHECKLIST.md` +- Verification: + - The lost response left `task_bb86c87bdc60` attached to one pending Dispatch, + `ctx_6580d07d9006`, rather than creating a replacement. + - Replaying request `disconnect-after-start-01` returned the same Dispatch with + `state=outcome_unknown` and `mutation.replayed=true`. + - Remote inspection found one `disconnect-after-acceptance-01` worktree, one exact agent + terminal, and one expected setup terminal; no duplicate topology was created. + - Worker Dispatch DB, federated control, and recovery suites passed 29 tests; node typecheck and + focused oxlint passed. Branch-head CLI and Electron builds succeeded on macOS and Windows. + - Relay tests reject an out-of-order sequence without advancing the cursor, later accept the + missing and retried frames contiguously, treat the repeated frame as a duplicate, and preserve + exactly two messages. Lost-ack retry and mailbox delivery suites passed with it (24 tests). +- Findings: + - The acceptance run exposed one narrow recovery inconsistency: `worker-show` could prove an exact + worker existed while `worker-stop` rejected the durable `start_unknown` state. Explicit stop now + enters the same fenced stopping path from `ready` or `start_unknown`; unattached, missing, or + identity-changed workers still become `stop_unknown` without a process action. + - No scheduler, automatic retry, adoption, cleanup, or general distributed-operation framework is + needed for this recovery path. +- Next: + - Exercise duplicate/reordered relay convergence and the native/WSL/SSH transport matrix, then + synchronize the HTML proposal with the implemented contract. + +### 2026-07-22 — Cross-platform capability transport verified + +- Changes: + - Added an SSH compatibility-bridge test that carries the opaque Dispatch capability in the RPC + envelope and settles only the matching pane/process Dispatch. + - Added a composed worker-start test that binds the host-resolved `orca-ide` command and the + Dispatch capability into one WSL worker preamble. + - Updated the built-CLI reset fixture to recreate its required coordinator Run after a task reset. + - Updated the HTML recovery contract so explicit stop from `start_unknown` remains fenced and + process-identity checked. +- Files: + - `src/main/ssh/ssh-remote-orca-cli.test.ts` + - `src/main/runtime/rpc/methods/orchestration-workers-new-worktree.test.ts` + - `src/main/runtime/orchestration-cli-subprocess.test.ts` + - `docs/orchestration-primitives.html` (ignored design source) + - `ORCHESTRATION_IMPLEMENTATION_CHECKLIST.md` +- Verification: + - Native macOS Unix-socket worker control, native Windows named-pipe lifecycle reporting, and + bidirectional authenticated WebSocket federation passed in the real Mac/Windows matrix. + - On the real Windows host, the PowerShell/SSH launcher, SSH command allowlist, host passthrough, + WSL/native command selection, and multiline Windows quoting suites passed 31 tests; 13 + Unix-socket-only runtime-client tests correctly skipped on Windows. + - Platform-neutral SSH capability, WSL command selection, preamble, worker-start, and CLI envelope + suites passed 94 focused tests on macOS. + - The rebuilt CLI plus every orchestration, composed-worker, federation, and SSH regression file + passed together: 25 files and 456 tests. Full Node/CLI/web typecheck, focused oxlint/format, + generated-skill verification, and `git diff --check` passed. +- Findings: + - The available Windows acceptance host has no WSL distribution installed and prompts to install + the feature. Acceptance therefore uses explicit WSL host/path tests rather than mutating the + machine. SSH is likewise exercised at the bridge and lifecycle boundary rather than requiring a + new external host. + - Native and relay-backed behavior is real end-to-end evidence; WSL and SSH evidence is bounded to + the host-selection, prompt, envelope, quoting, allowlist, and lifecycle contracts Orca owns. +- Next: + - Run the remaining full validation and Linux CI, then finish the ignored HTML synchronization and + PR evidence without adding new orchestration concepts. + +### 2026-07-22 — Phase 3 and cross-platform gates closed + +- Changes: + - Marked connected-server federation and the native cross-platform quality gate complete after + branch-head Linux CI joined the real macOS/Windows acceptance evidence. +- Files: + - `ORCHESTRATION_IMPLEMENTATION_CHECKLIST.md` +- Verification: + - PR #9925 completed with 17 successful checks and no failures: full Linux verify, Ubuntu and + Windows native smoke, packaged Windows crash survival, and macOS/Ubuntu/Windows skill round trips. + - Linux verify passed lint, generated-skill checks, max-lines enforcement, typecheck, repository- + wide tests, unpacked-app build, and packaged CLI smoke. +- Findings: + - Every implementation phase is now either complete or explicitly deferred with its prerequisite; + no additional orchestration subsystem is required for V1. +- Next: + - Review and merge PR #9925; keep Phase 4 deferred until exact provider-session association exists. + +### 2026-07-22 — Final agent-contract audit + +- Changes: + - Synchronized the ignored HTML examples with the shipped mutation, Delivery, worker-start, and + terminal-read result shapes; removed speculative Phase 4 output fields from the V1 path. + - Made semantic retry explicitly repeat placement and agent/terminal choices, while transport + recovery reuses only the exact `mutation.requestId` after a lost response. + - Corrected worker state/error terminology, the cross-platform structured completion recipe, and + the HTML implementation-status footer. + - Updated the installed/versioned orchestration guidance to prefer `worker-start`, use + `question` mail, and reserve low-level `dispatch --inject` for custom topology. + - Fixed `worker-read --cursor 0`; the runtime supported the initial retained-output cursor but the + orchestration CLI incorrectly required a positive value. +- Files: + - `src/cli/handlers/orchestration.ts` + - `src/cli/handlers/orchestration-worker-cli.test.ts` + - `src/cli/specs/orchestration-worker-specs.ts` + - `skill-guides/orchestration.md` + - `skills/orchestration/SKILL.md` + - generated bundled skill guide and skill-bundle manifests + - `docs/orchestration-primitives.html` (ignored design source) + - `ORCHESTRATION_IMPLEMENTATION_CHECKLIST.md` +- Verification: + - Focused worker CLI suite passed 3 tests, including cursor zero. + - Final orchestration runtime/CLI/renderer/skill regression set passed 24 files and 450 tests. + - Node, CLI, and web typechecks, bundled-guide and skill-manifest verification, focused + formatting, and `git diff --check` passed. + - A repository-wide run reached 34,190 passing tests and exposed the one intentionally changed + skill assertion; after updating that assertion, the focused configured rerun above passed. The + broad run also exhausted several Vitest fork-start deadlines under full local concurrency, while + the previously green PR checks remain the authoritative clean full-suite baseline. +- Findings: + - The implementation already returned durable request IDs as `mutation.requestId`; the remaining + problem was stale naming and invented provenance fields in the ignored design example. + - Stored start options are recovery evidence, not implicit replacement policy. Requiring explicit + replacement placement keeps agents in control and avoids recreating a possibly-existing remote + worktree by accident. + - No new scheduler, retry engine, output adapter, projection layer, or federation subsystem is + needed. Phase 4 remains deferred. +- Next: + - Push the audit corrections and confirm PR #9925 is green at the new head. + +### 2026-07-22 — CodeRabbit and internal review until clean + +- Changes: + - Validated every CodeRabbit finding; fixed relay type validation, stale runtime-owned terminal + identity, setup receipt classification, Windows batch percent escaping, worker reconciliation, + root-help discovery, canonical question schema, and deterministic waiter timing. + - Coalesced overlapping per-Dispatch federation polls and added one warning per outage window. + - Centralized the orchestration RPC envelope type without adding a new runtime abstraction. + - Made reset scopes atomic and cleared matching worker/federation state while preserving relay + cursors for message-only resets and the mutation ledger needed for lost-response deduplication. +- Files: + - orchestration runtime, federation, worker-control, database, CLI help/client, wrapper, skill, and + focused regression tests listed in the current working diff. +- Verification: + - Focused runtime, federation, database, CLI, SSH, wrapper, and skill suites passed 1,103 tests. + - The repository-wide configured suite passed 34,253 tests with 58 intentional skips. + - Node, CLI, and web typechecks passed; focused oxlint, formatting, generated-skill checks, and + `git diff --check` passed. +- Findings: + - The type-only pairing import is valid TypeScript and remains type-only; Dispatch capability + flags stay intentionally hidden because the authenticated worker preamble supplies them. + - Persistent envelope-bearing WebSocket reuse remains a measured-later optimization; the V1 + single-flight relay removes overlapping connection churn without growing transport scope. + - The final re-review found no remaining in-scope correctness, ergonomics, elegance, or + performance defect after fixing reset scope and relay-cursor preservation in round 2. + - Full lint reaches unrelated existing failures in the unchanged skill-freshness switch and + localization catalog; changed-file lint and every in-scope quality gate pass. +- Next: + - Resolve CodeRabbit threads and confirm the PR checks at the final head. + +### 2026-07-24 — Post-rebase physical federation dogfood + +- Changes: + - Rebased the branch onto current main and launched isolated branch-head desktop runtimes on the + Mac Run home and the physical Windows worker server. + - Started a real Windows Codex worker in a new top-level worktree with explicit setup-run. + - Replaced the orchestration process fence's renderer generation with the controller-issued PTY + incarnation when available, retaining the prior value only for legacy providers. + - Added a runtime regression covering a visible terminal surface detaching and reattaching around + the same process, followed by a replacement incarnation. +- Files: + - `src/main/runtime/orca-runtime.ts` + - `src/main/runtime/orca-runtime.test.ts` + - `ORCHESTRATION_IMPLEMENTATION_CHECKLIST.md` +- Verification: + - The remote start returned ready in about nine seconds with setup `running`, + `startupPolicy=start-immediately`, one agent terminal, one setup terminal, and accepted task + input. + - The original branch head then reproduced immediate `identity_changed`: routed read failed and + the exact injected capability could not send a heartbeat or question. + - The new focused process-identity regression passed, the 18-test federation suite passed, and + Node typecheck passed. +- Findings: + - A renderer pane generation is presentation state, not process identity. A healthy PTY can move + between a runtime-owned background surface and the renderer without losing Dispatch authority. + - The PTY controller incarnation is already available on native Windows/macOS and SSH/relay paths; + using it fixes the race without relaxing replacement-process fencing or adding a new identity + subsystem. +- Next: + - Rebuild both branch runtimes, repeat the full physical Mac-to-Windows lifecycle, then run the + remaining local error/recovery matrix before closing the revalidation rows. + +### 2026-07-24 — Post-fix physical federation revalidation + +- Changes: + - Rebuilt and restarted the physical Windows branch runtime from `b09c635ec`, then repeated the + Mac Run-home to Windows-worker flow on a fresh top-level worktree. + - Abandoned the controlled stale-build attempt, linked the replacement with `--retry-of`, and + exercised exact start-request replay plus exact remote worker stop. + - Exercised missing-agent and invalid-remote-repo rejection without adding recovery automation. +- Verification: + - The replacement returned ready while setup was running under `start-immediately`; + `worker-show` reported the exact running process, bounded `worker-read` succeeded, and the + worker's heartbeat reached the Mac Run home. + - An inferred-Run blocking question carried the `blue` reply back to Windows, and the authenticated + success report atomically settled the Task and Dispatch. + - A transient connection close before acceptance left the second Task ready with no Dispatch. + Retrying its exact request ID started one worker, and repeating that request returned the same + Dispatch with `replayed=true` and no duplicate worktree or terminal. + - `worker-stop` killed only the exact agent PTY. The setup terminal remained present, the Dispatch + became stopped/failed, and the Task became blocked for explicit recovery. + - An unconfigured agent failed locally. A missing remote repo produced a typed failed Dispatch + with `effects=[]` and no residual resources. +- Findings: + - The first repeated failure was a stale Windows build artifact, not a failed fix: the source + checkout was at `b09c635ec` while `out/main/index.js` still had the old generation fence. + Relaunching the branch dev process produced a new runtime epoch and the fixed behavior. + - Omitted setup correctly resolved to Orca's `run` default. Its independent Windows install later + failed in `windows-native-registry`, but that did not delay task delivery and remained isolated + from exact agent stop. + - From an unmanaged shell, a coordinator mailbox check must name `--terminal`; a CLI running + inside the bound coordinator terminal continues to infer that identity normally. +- Next: + - Resolve only concrete findings from the independent ergonomics/federation re-review, run the + final local verification set, and reconcile PR review threads and CI. + +### 2026-07-24 — Post-dogfood recovery and ergonomics hardening + +- Changes: + - Made lost remote stop responses reconcilable and prevented stale Dispatch abandonment from + blocking an active replacement. + - Persisted setup completion as evidence only, preserving settled lifecycle state locally and + relaying the same outcome from a connected worker server. + - Rejected misleading explicit targets from federated workers and consumed `ask` answers exactly + once while retaining their durable thread record. + - Stored the accepted Dispatch in pending worker-start receipts so a post-restart retry returns the + exact inspection command. + - Rejected new-worktree placement for folder projects before effects. + - Restored the generated skill-history ledgers that the branch had accidentally truncated. +- Verification: + - The full orchestration DB/RPC/CLI/SSH regression selection passed 516 tests; its focused + recovery, setup, messaging, and mutation slice passed 239 tests. + - The earlier physical Mac-home to Windows-worker lifecycle covered ready/read/heartbeat, + ask/reply, completion, exact request replay, and exact stop. +- Findings: + - These were narrow truthfulness and recovery gaps; none required a scheduler, automatic retry, + access-control framework, replicated Run database, UI, or provider-session abstraction. + - The release-contract test failure is already present on `main`; it is separate from this + orchestration change. The skill round-trip failures were branch-caused and are fixed by + restoring their committed history. +- Next: + - Run the complete changed-file quality gates, rebuild the physical Windows dev runtime with this + final patch, repeat the setup-status slice, then push and recheck PR CI/review state. + +### 2026-07-24 — Physical receipt follow-up + +- Changes: + - Made local and connected-server ready transitions persist the accepted `dispatch_input` effect + atomically, so later setup evidence cannot replace it with an older effect snapshot. + - Carried the already-resolved setup source through the internal federation attach request, keeping + omitted setup labeled `orchestration_default` and explicit setup labeled `explicit_request`. +- Verification: + - The focused new-worktree, federation, and setup-evidence slice passed 37 tests. + - The broader orchestration DB/RPC/CLI/SSH execution-host selection passed 630 tests. + - Node and CLI typechecks, changed-file lint/format, and `git diff --check` passed. +- Findings: + - Both defects were receipt-provenance bugs found by the physical Mac-home to Windows-worker run; + neither changes worker placement, setup timing, lifecycle authority, or agent-facing commands. +- Next: + - Rebuild both dev runtimes from this patch and repeat the physical setup-status slice before final + PR reconciliation. + +### 2026-07-24 — Truthful setup command completion + +- Changes: + - Wrapped only orchestration-created non-gating setup commands with a private per-invocation + completion signal that preserves the command exit code. + - Added one runtime observer that subscribes to raw PTY output, replays the bounded recent-output + buffer, scans across chunk boundaries, and treats terminal exit as a fallback. + - Updated local and connected-server setup evidence monitors to observe command completion while + leaving the interactive setup terminal open. + - Propagated the exact setup terminal handle through worktree receipts and hardened native Windows + launch with an encoded PowerShell command plus an environment-carried runner path. +- Files: + - `src/main/runtime/orchestration/setup-completion-signal.ts` + - `src/main/runtime/orca-runtime.ts` + - local and federation setup monitors and focused tests + - `ORCHESTRATION_IMPLEMENTATION_CHECKLIST.md` +- Verification: + - The completion helper, exact-effect, local worker, federation, and setup-evidence suites passed + 43 tests. + - Focused runtime tests proved live completion, replay-before-observer recovery, and opt-in setup + wrapping while the shell remains running. + - The complete runtime service suite passed 884 tests. + - The broader orchestration/CLI/SSH/execution-host selection passed 529 tests. + - Node, CLI, and web typechecks, the CLI/Electron build, focused lint/format, max-lines ratchet, + and `git diff --check` passed. + - The independent no-scope-creep recheck found no remaining correctness blocker. +- Findings: + - Terminal exit is not setup-command completion because Orca intentionally runs setup in an + interactive terminal that returns to a shell prompt. + - Display titles are not terminal identity, and a disconnected terminal is not proof that its + setup command failed. + - The correction is runtime evidence only: it adds no public flag, setup job, scheduler, retry + policy, process heuristic, or automatic tab closure. +- Next: + - Rebuild both dev runtimes and repeat the physical Mac Run-home to Windows-worker setup-status + slice, including a failing setup command that returns to a PowerShell prompt. + +### 2026-07-24 — Physical Windows setup-completion proof + +- Changes: + - Rebuilt and restarted the Mac Run-home and physical Windows worker runtimes from `4aba390af`. + - Started a fresh Windows Codex worker from the Mac with omitted `--setup`, a new top-level + worktree, and the exact Windows repo selector. +- Verification: + - Mac runtime epoch `4a2a1cba-fd8b-41c7-b50b-caeac7415d9c` and Windows runtime epoch + `04cecad3-65a9-49a5-90ff-cdf04b09050f` both became ready after restart. + - Run `run_0f6b471005af`, Task `task_11e981c692b5`, and Dispatch `ctx_f009a65c6d9d` returned + ready with setup `running`, source `orchestration_default`, policy `start-immediately`, the exact + setup terminal `term_bdcb2a0b-89c5-429c-8ae6-0fdc2457db15`, and accepted dispatch input. + - The real Windows setup command later exited 1 in `windows-native-registry`; `worker-show` + changed setup to `failed` while preserving the succeeded worker, settled Dispatch, accepted + input effect, and exact setup-terminal effect. + - The setup terminal remained running and accepted a follow-up PowerShell command after failure. + The Run mailbox contained exactly one high-priority setup-failed notice for the Dispatch. +- Findings: + - The private per-invocation marker carried exit code 1 in raw setup-terminal output and did not + appear in orchestration receipts or lifecycle messages. + - Windows terminal reads still flatten PowerShell line-editor redraws into noisy repeated input + text. The command executed once and orchestration state remained correct, so this pre-existing + rendering artifact stays outside this PR. +- Next: + - Re-run the final local quality gates, push this evidence-only checklist update, and reconcile PR + CI and review state. + +### 2026-07-24 — Structured worker-output implementation + +- Changes: + - Extended `worker-read` with `auto|transcript|terminal` source selection while preserving one + Dispatch-only agent command. + - Added exact pane/process/session selection from existing hook evidence, bounded Codex/Claude + transcript reading, path-free source identities, and opaque source-pinned cursors. + - Added a worker-local federated output RPC; mixed-version servers fall back through the existing + terminal-read method and still receive an opaque Run-home cursor. + - Added readable non-JSON transcript rendering, CLI help, skill guidance, typed errors, and + malformed/oversized/clipping warnings. +- Files: + - `src/shared/orchestration-worker-output.ts` + - `src/main/runtime/orchestration/worker-output-cursor.ts` + - `src/main/runtime/orchestration/worker-provider-session.ts` + - `src/main/runtime/orchestration/worker-transcript-payload.ts` + - `src/main/runtime/orchestration/worker-transcript-read.ts` + - `src/main/runtime/rpc/methods/orchestration-worker-output.ts` + - Worker control/federation, runtime status lookup, CLI, skill, design, and focused tests +- Verification: + - Node and CLI typechecks passed. + - Nineteen native-chat/structured-output suites passed 142 tests. + - Sixteen orchestration CLI/RPC/federation suites passed 289 tests. + - Five CLI registry/help/runtime-error suites passed 213 tests. + - Mixed-version fallback continuation was additionally verified with an opaque cursor. +- Findings: + - The prior Phase 4 deferral was factually wrong: hook snapshots already bind provider sessions to + exact panes. Reusing that evidence avoids directory/title/logo guessing and avoids a second + status subsystem. + - Method probing is enough for mixed-version compatibility; a generalized provider capability + matrix is unnecessary. +- Next: + - Run the remaining full lint/test gates and the physical local plus Mac/Windows dogfood matrix + before marking Phase 4 complete. + +### 2026-07-24 — Structured-output local and mixed-version dogfood + +- Changes: + - Ran two simultaneous same-worktree Codex Dispatches with unique markers and verified exact + transcript isolation plus opaque continuation. + - Made transcript-position fallback IDs opaque after the physical response exposed the local + Codex JSONL path. + - Redacted pane-bound Dispatch capability tokens from structured prose, tool input, tool output, + metadata, and image URLs after continuation exposed the lifecycle send command. + - Corrected `worker-read --help` so `--cursor` is described as opaque rather than numeric. + - Made forward paging advance safely across a transcript record larger than the bounded scan + window, while continuing to discard its unfinished fragments. + - Extended Dispatch-capability redaction to tool-input object keys as well as values. +- Verification: + - Both simultaneous local reads selected different exact Codex source identities and contained + only their own marker. + - Continuation returned newly appended tool/assistant messages, `limited=true`, and the expected + completion marker. + - A fresh local response contained stable `worker-message-*` IDs, no `.codex/sessions` path, no + `dcap_` token, and explicit privacy/redaction warnings. + - Starting a new Codex chat in the same pane caused the old cursor to return `source_changed`; a + fresh read selected only the new chat marker. + - Restarting the Run-home runtime preserved settled state and rejected a read when the exact + worker process was no longer present. + - Mac Run home -> older Windows worker returned `source=terminal`, + `fallbackReason=remote_capability_unavailable`, an opaque cursor that continued successfully, + and `transcript_required` when structured output was explicitly required. +- Findings: + - Synthetic path-leak tests need fallback-ID records, not only provider records with explicit IDs. + - Structured output must treat lifecycle capability text as secret even though the capability is + also pane-bound; redaction is a narrow output boundary, not a generalized secret scanner. + - Additive RPC probing works against the physical older Windows server without a capability + matrix or server upgrade gate. + - A bounded scan must still guarantee cursor progress; otherwise one pathological provider record + can trap an agent in a valid-looking continuation loop. +- Next: + - Commit/push the tested implementation, update the physical Windows dev runtime, then run exact + Mac-to-Windows and Windows-to-Mac structured reads plus disconnect/reconnect. + +### 2026-07-24 — Reverse dogfood found missing coordinator control mail + +- Finding: + - A Windows Run home successfully started and read an exact Mac Codex worker, and worker-to-home + status relayed correctly. However, coordinator mail addressed to that remote worker remained + queued at the Run home because only question replies used the home-to-worker relay. + - The injected worker's local `check --wait` also looked only for a same-server Dispatch, so even + an imported generic message could not wake it. +- Changes: + - Route stable `dispatch:` coordinator guidance through the existing per-Dispatch durable + relay; terminal-handle targeting remains a legacy/local path. + - Import control mail idempotently on the worker server, tolerate a replay after a lost import + acknowledgment, and wake only the exact attached worker process. + - Return a direction-aware relay receipt and teach CLI help, the versioned skill, and the + cross-server cookbook to use the Dispatch ID for follow-ups. +- Verification: + - Focused Node/CLI typechecks and 184 orchestration/federation/CLI tests passed before physical + revalidation. +- Next: + - Regenerate the bundled skill guide, re-review the narrow change, then repeat Windows-home to + Mac-worker follow-up, completion, exact transcript continuation, and disconnect/reconnect. + +### 2026-07-24 — Federated control-mail race hardening + +- Changes: + - Fence already-imported relay sequences before parsing or applying message side effects. + - Require the remote attachment to remain ready before accepting each new coordinator message. + - Recheck the Run-home worker state after importing worker lifecycle mail and do not push queued + guidance after the worker settles. + - Wake filtered worker waiters with the imported message's real type instead of always using + `status`. + - Negotiate a narrow control-mail capability and reject the send before queueing when an older + worker server supports base federation but not the new relay kind. +- Verification: + - Regression tests prove a replayed sequence with a different message ID creates no duplicate. + - A waiter registered before `worker_done` receives no stale control mail after completion, and a + direct late import is rejected as inactive. + - An imported escalation wakes an escalation-filtered waiter while a status-filtered waiter times + out normally. + - A new Run home connected to a prior worker build can still start the worker, but control mail + returns `capability_unsupported` and leaves no undeliverable relay row. + - The focused federation suites passed 24 tests; the broader orchestration/CLI selection passed + 475 tests. + - Full typecheck, lint, bundled-skill verification, CLI build, Electron/Vite build, + `git diff --check`, and the design-document reference-name audit passed. +- Findings: + - Relay ordering and worker settlement are control-plane guardrails, not agent policy: the + coordinator still chooses what to send and when. + - Terminal worker state is authoritative for delivery; queued guidance is retained at the Run + home but never injected into a completed worker. +- Next: + - Complete the final read-only review, then repeat the physical Windows-home to Mac-worker + follow-up, completion, exact transcript continuation, and disconnect/reconnect proof. + +### 2026-07-24 — Physical control-mail acceptance and dogfood fixes + +- Changes: + - Restarted the physical Windows runtime from branch head and confirmed both federation + capabilities before creating a fresh Windows-home Run. + - Fixed `check --ack --peek` so the exact Delivery is acknowledged before the + read-only history projection; the previous early return silently ignored `--ack`. + - Preserved the existing structured remote transport codes through the Run-home RPC boundary + instead of collapsing disconnect, timeout, and malformed-response failures to `runtime_error`. +- Verification: + - Run `run_074503e3edc6`, Task `task_8a07840e7aad`, and Dispatch `ctx_d6bac1ee6409` started a fresh + Codex worker in the existing Mac worktree with setup `not_applicable`. + - The worker relayed `ORCA_MAC_CONTROL_INITIAL_7C31`, blocked on its Dispatch inbox, received + coordinator guidance addressed to the stable Dispatch as `ORCA_MAC_CONTROL_FOLLOWUP_A842`, and + returned one authenticated successful `worker_done` containing both markers. + - The Windows home settled the Task and Dispatch once. `worker-read --source auto` returned + `source=transcript`, `provider=codex`, an opaque cursor, both markers, and no capability token or + transcript path. + - Removing the Mac listener left the Windows Task completed. Reconnecting with the wrong leftover + profile was rejected as unauthorized; reconnecting with the original profile preserved the + settled Dispatch and correctly returned `worker_identity_changed` for transcript reads after the + exact worker process was gone. + - The focused RPC suites passed 163 tests; formatting, diff checks, full typecheck, CLI build, and + desktop/web builds passed. + - After the Windows generated main bundle was verified at `8dd0d1b16`, Delivery + `delivery_888f972841d6` was acknowledged by `check --ack ... --peek`; the response echoed the + exact acknowledged ID and returned zero unread rows. + - With that build running, removing the Mac listener returned + `remote_runtime_unavailable` while the Task stayed completed. Reconnect preserved the succeeded + Dispatch, produced no duplicate Run mail, and returned `worker_identity_changed` rather than + attributing the old transcript to a replacement process. +- Findings: + - Delivery acknowledgment must compose with inspection modes explicitly; a successful command may + not silently ignore the acknowledgment effect. + - Remote transport already had narrow error codes. Preserving them is enough; no federation error + hierarchy or retry engine is needed. + - Saved peer identity fencing prevented accidental adoption of a server started from a different + profile. Exact process fencing also prevented stale transcript attribution after restart. + - On the Windows dogfood shell, the `pnpm` wrapper returned before its spawned Vite build + completed. Verifying the generated bundle before restart exposed the race; invoking the Node + build script directly produced the expected branch-head bundle. +- Next: + - Run the final quality gates and review, push this evidence update, inspect PR CI, and remove only + the temporary dogfood profile and listener after verification is complete. + +### 2026-07-24 — Current-main rebase integration + +- Changes: + - Rebased the full implementation onto current `origin/main`. + - Combined main's bounded one-shot remote-request admission with the orchestration authentication + envelope in the same pre-serialized encrypted request. + - Added a direct WebSocket regression proving the admitted request retains the orchestration + capability and mutation ID. +- Verification: + - The repository-configured orchestration, federation, remote-client, and skill selection passed + 35 files and 465 tests. + - Full Node/CLI/web typecheck, lint/reliability/manifest/localization gates, focused formatting, + generated-skill verification, and `git diff --check` passed after the rebase. + - Relay, CLI, Electron/Vite, and web production builds passed. The local CLI installer reported + only the expected non-fatal lack of permission to replace `/usr/local/bin/orca-dev`. +- Findings: + - The request must be serialized with its authentication envelope before it reserves bounded + admission; rebuilding the frame after authentication would bypass the retained-byte contract. +- Next: + - Push the rebased branch, inspect branch-head CI, then remove only the exact temporary dogfood + resources. + +### 2026-07-24 — Structured-output proposal synchronization + +- Changes: + - Updated the newer HTML proposal from its obsolete terminal-only Phase 4 deferral to the shipped + `auto|transcript|terminal` contract. + - Documented exact pane/process/session selection, opaque source-pinned cursors, labeled fallback, + mixed-version behavior, and the implemented Phase 4 status. +- Verification: + - Confirmed the proposal contains no named references to other orchestration products. +- Findings: + - Implementation status and optional remaining physical acceptance are separate: the narrow output + primitive is complete, while symmetric cross-machine dogfood remains visible in this ledger. +- Next: + - Run document/skill checks, push the synchronization fix, resolve the review thread, and continue + branch-head CI monitoring. + +### 2026-07-25 — Physical Grok and OpenCode provider dogfood + +- Changes: + - Reused Native Chat's existing Grok session resolver and transcript decoder in `worker-read`. + - Kept OpenCode on the generic terminal fallback because Native Chat has no OpenCode transcript + decoder. + - Applied Dispatch-capability redaction to terminal fallback lines as well as structured + transcript blocks. + - Updated the agent-facing skill and proposal to name the current structured provider set. +- Verification: + - An isolated branch-head server started fresh same-worktree Grok and OpenCode workers through + `worker-start`; both accepted their injected tasks and returned authenticated successful + `worker_done` reports. + - Grok returned `source=transcript`, `provider=grok`, the exact marker, opaque message IDs and + cursor, and no capability token or transcript path. + - OpenCode returned `source=terminal`, `fallbackReason=provider_unsupported`, the exact marker, + and a source-pinned opaque cursor; explicitly requiring a transcript returned + `transcript_required`. + - The first OpenCode read exposed its pane-bound Dispatch capability in terminal text. After the + fix and a clean runtime rebuild, the repeated physical read replaced it with + `[dispatch capability redacted]`, emitted an explicit warning, and contained no raw token. + - Focused Native Chat/orchestration output tests passed 49 tests and Node typecheck passed. +- Findings: + - Provider support and structured-output support remain separate: OpenCode orchestration is fully + usable without inventing an OpenCode transcript adapter. + - Terminal fallback is an orchestration output boundary and needs the same narrow secret + redaction as structured output; this does not change direct terminal-read behavior. +- Next: + - Run the complete orchestration regression selection and repository quality gates, then commit + and push the provider dogfood fixes. + +### 2026-07-25 — CI ask-admission fixture correction + +- Changes: + - Updated the WebSocket long-poll admission tests to place each simulated asking worker in a real + Run with an active supervised Dispatch. + - Kept the production rule that unsupervised workers cannot create blocking questions. +- Verification: + - The complete runtime RPC test file passed 59 tests. + - The broader orchestration/RPC/CLI selection passed 23 files and 391 tests. + - Node typecheck and `git diff --check` passed. +- Findings: + - The red CI assertions were stale test setup: old arbitrary terminal handles now fail + `orchestration.ask` before holding an admission slot, exactly as the new contract requires. +- Next: + - Push the test-only correction and confirm the replacement PR check is green. + +### 2026-07-26 — Hard orchestration contract cutover + +- Changes: + - Added one shared orchestration contract version, runtime capability, and mutation classifier. + - Fenced old, missing, and wrong-contract mutations before parsing, durable receipts, database + writes, process actions, prompt injection, and connected-server effects. + - Propagated the contract through Unix/named-pipe, WebSocket, connected-server, and SSH CLI + transports, with capability preflight before local or federated mutations. + - Retired the legacy scheduler commands locally and at RPC dispatch, preserving only explicit + read-only legacy inspection. + - Returned no-effects plus argument-only full-skill recovery and documented that pre-upgrade + worker processes continue unsupervised until inspected. +- Verification: + - Focused orchestration/federation selection: 56 files and 721 tests passed. + - Repository suite excluding the independently reproducible system-SSH native-installer timeout: + 3,479 files and 37,181 tests passed. + - Node, CLI, and web typechecks passed. + - Relay, CLI, Electron/Vite, and web production builds passed. + - Bundled-skill verification, reliability gates, max-lines ratchet, and `git diff --check` passed. + - The newer tracked HTML contains no named references to the audited orchestration projects; the + older redesign HTML remains ignored and untracked. +- Findings: + - A hard version fence plus executable skill recovery is simpler and safer than maintaining a + legacy executor or draining in-flight legacy state. + - Existing pre-upgrade processes are deliberately left alive, but rejected lifecycle calls + cannot mutate current Task, Dispatch, or inbox state. + - Full lint still reports the pre-existing localization audit for six unchanged `Ghostty` + keyword strings; the excluded system-SSH test independently times out while installing native + dependencies. Neither baseline issue is changed by this migration. +- Next: + - Commit and push as `OrcaWin`, then inspect branch-head CI and mark the final remote acceptance + item only after those checks settle. + +### 2026-07-27 — Current-main rebase and CLI registry integration + +- Changes: + - Rebased the 26 orchestration commits onto current `origin/main`. + - Preserved both main's active-worktree plugin context and orchestration's terminal process + incarnation and launcher validation in their one overlapping runtime conflict. + - Registered the current Run, worker, and retired-coordinator handler keys in main's new lazy CLI + handler-group manifest. + - Removed one trailing-whitespace artifact from the structured-output design header. +- Verification: + - Conflict-focused runtime, federation, migration, and transport selection: 6 files and 995 tests + passed. + - Handler manifest, registry parity, and CLI integration: 3 files and 169 tests passed. + - Repository suite excluding the independently reproducible system-SSH native-installer timeout: + 3,586 files and 37,864 tests passed. + - Node, CLI, and web typechecks, bundled-skill verification, reliability gates, max-lines + ratchet, and conflict-marker audit passed. +- Findings: + - The rebase itself had one additive method-placement conflict; neither behavior needed redesign. + - Main's lazy handler manifest is an additional command-registration source of truth, so every new + exported orchestration handler must be listed there. + - Tests added on main depend on newly patched packages; refreshing from the rebased lockfile was + required before their results were meaningful. +- Next: + - Push the rebased branch as `OrcaWin` and inspect replacement branch-head CI. + +### Entry template + +```text +### YYYY-MM-DD — Short implementation milestone + +- Changes: + - ... +- Files: + - `path` +- Verification: + - command/test and result +- Findings: + - decision, surprise, or risk +- Next: + - one concrete next step +``` diff --git a/ORCHESTRATION_STRUCTURED_OUTPUT_DESIGN.md b/ORCHESTRATION_STRUCTURED_OUTPUT_DESIGN.md new file mode 100644 index 00000000000..e7d9443cd95 --- /dev/null +++ b/ORCHESTRATION_STRUCTURED_OUTPUT_DESIGN.md @@ -0,0 +1,604 @@ +# Orchestration Structured Worker Output + +Status: implemented; physical local, mixed-version, and Windows-home to Mac-worker validation complete +Scope: orchestration `worker-read` only +Last updated: 2026-07-24 + +## Summary + +`orca orchestration worker-read` currently reads bounded terminal output. That is always available, +but full-screen agent TUIs can make it noisy or incomplete. + +Orca already knows more than the terminal logo suggests. Agent hooks associate an exact pane with: + +- the agent type, such as Codex or Claude; +- the provider session or conversation ID; and +- when available, the provider-reported transcript path. + +The sidebar, session resume, sleeping-agent recovery, and native chat already use this information. +The missing piece is a narrow orchestration path from an exact Dispatch to that exact session on the +server where the worker runs. + +The proposed behavior is: + +```text +worker-read + exact supported transcript is available -> structured transcript page + otherwise -> bounded terminal page +``` + +This does not add orchestration strategy, a dashboard, a scheduler, or a universal provider layer. +It makes one existing observation command return the best source Orca can prove. + +## User-facing goal + +A coordinator should be able to inspect a worker with one predictable command: + +```bash +orca orchestration worker-read --dispatch --json +``` + +The coordinator should not need to know: + +- which Orca server owns the worker; +- the worker's terminal or pane handle; +- the provider session ID; +- where a transcript lives on disk; or +- whether structured reading is supported by that provider/server version. + +The response must always say which source was used. It must never silently read a different agent +session. + +## Why the existing sidebar is relevant + +The agent logo identifies the detected agent type. By itself, that is not enough to choose a +transcript. + +The richer sidebar status also carries pane-scoped provider-session metadata reported by hooks. +That is the useful foundation: + +```text +Dispatch + -> exact worker process and terminal + -> exact tab/leaf pane + -> hook-reported provider session + -> provider transcript locator +``` + +Some live status ownership is currently renderer-centric, while headless and mobile graph paths +also retain compatible status snapshots. Implementation therefore needs one runtime-owned lookup +that exposes the current exact pane association to `worker-read`. This is a small bridge over +existing status data, not a second agent-status system. + +## Design principles + +### Exactness over convenience + +- Never select the "latest session in this directory." +- Never select a transcript from a terminal title or logo alone. +- Never switch sources or sessions in the middle of a cursor chain. +- If Orca cannot prove the association, return a labeled terminal fallback. + +### Resource-local reads + +The server running the worker resolves and reads its transcript. A Run home on macOS must not try +to interpret a Windows path, and a Windows Run home must not try to interpret a macOS path. + +Only bounded output data crosses the federation connection. Transcript paths do not. + +### One simple agent command + +Agents should not choose provider adapters or supply session metadata. `worker-read` defaults to +automatic source selection. Source selection flags exist for debugging and explicit policy, not +because they are required in the normal loop. + +### Narrow provider support + +Initial support should cover only providers for which Orca already has: + +1. an exact pane-scoped session association; and +2. an existing bounded transcript reader with test coverage. + +Codex is the required first provider. Claude may ship in the same change only if it uses the same +proven reader path without adding a second architecture. Other agents receive terminal fallback. + +### Honest compatibility + +Connected servers can run different Orca versions. A server without structured-read support must +continue to return bounded terminal output rather than failing the whole Run. + +## Public command contract + +### Request + +```bash +orca orchestration worker-read \ + --dispatch \ + [--source auto|transcript|terminal] \ + [--cursor ] \ + [--limit ] \ + [--json] +``` + +`--source` behavior: + +| Value | Behavior | +| ------------ | -------------------------------------------------------------------------------------- | +| `auto` | Use an exact supported transcript; otherwise use terminal output. This is the default. | +| `transcript` | Require an exact supported transcript. Return a typed error instead of falling back. | +| `terminal` | Use the current bounded terminal reader. | + +Existing numeric terminal cursors remain accepted. New responses return an opaque cursor that can +pin either source without exposing provider paths. + +### Response + +The implemented structured response has this shape: + +```json +{ + "dispatchId": "dispatch_123", + "source": "transcript", + "sourceIdentity": "opaque-source-fingerprint", + "provider": "codex", + "transcript": { + "messages": [], + "nextCursor": "opaque-next-cursor", + "limited": false, + "returnedMessageCount": 0 + }, + "cursor": "opaque-next-cursor", + "status": { + "worker": "running", + "terminal": "running" + }, + "fallbackReason": null, + "warnings": [] +} +``` + +Terminal fallback uses the same envelope and keeps the existing terminal data: + +```json +{ + "dispatchId": "dispatch_123", + "source": "terminal", + "sourceIdentity": "opaque-terminal-incarnation", + "terminal": { + "tail": ["..."], + "status": "running", + "nextCursor": "..." + }, + "cursor": "opaque-next-cursor", + "fallbackReason": "session_not_reported", + "warnings": [] +} +``` + +For backward compatibility: + +- the terminal branch retains the existing `terminal.tail`, `terminal.status`, and + `terminal.nextCursor` fields; +- non-JSON output prints readable transcript entries or terminal lines without requiring an agent + to branch on JSON manually; +- a mixed-version federated read can return the legacy terminal shape, which the Run home wraps as + a labeled terminal response. + +### Fallback reasons + +Fallback reasons are bounded typed values, not arbitrary policy: + +- `provider_unsupported` +- `session_not_reported` +- `transcript_missing` +- `transcript_unreadable` +- `transcript_parse_failed` +- `remote_capability_unavailable` + +Warnings may provide safe diagnostic context, but must not contain a remote filesystem path. + +### Typed errors + +Errors are used when returning any source would be misleading: + +- `dispatch_not_found` +- `worker_identity_changed` +- `source_changed` +- `cursor_invalid` +- `cursor_dispatch_mismatch` +- `transcript_required` +- the existing connected-server unavailable/unknown result + +`source_changed` means the exact pane is now associated with a different provider session than the +one pinned by the cursor. The caller starts a new read without the old cursor. Orca never silently +jumps to the replacement session. + +## Source identity and cursor rules + +The cursor is opaque to clients and contains only non-sensitive routing data: + +- cursor version; +- Dispatch ID; +- source kind; +- an opaque digest of the exact source identity; +- provider-specific or terminal paging position. + +It must not contain a transcript path. + +On every continued read, the worker server: + +1. revalidates the Dispatch's exact process attachment; +2. resolves the current pane/session association; +3. compares its source digest with the cursor; +4. reads only when they still match; and +5. otherwise returns `source_changed` or `worker_identity_changed`. + +The cursor is source-pinned even when the request uses `--source auto`. `auto` chooses only on the +first page. + +The implementation uses a versioned stateless token. The token is not an authority credential: +every read revalidates the Dispatch, process, pane, source digest, and provider session before +returning data. This lets paging survive an Orca restart without adding cursor-secret persistence. + +## Runtime architecture + +### 1. Resolve the Dispatch at its Run home + +The Run home remains authoritative for Task and Dispatch state. It looks up the Dispatch and its +pinned worker server exactly as `worker-show` and the current `worker-read` do. + +No automatic placement or server selection is added. + +Current implementation anchors: + +- `src/main/runtime/rpc/methods/orchestration-worker-control.ts` owns local and federated + `worker-read` routing. +- `src/main/runtime/rpc/methods/orchestration-worker-observation.ts` validates the exact attached + worker. +- `src/cli/handlers/orchestration.ts` owns the current CLI request and terminal rendering. + +### 2. Route the read to the worker server + +For a local worker, the Run home and worker server are the same runtime. + +For a federated worker, the Run home calls the existing federation read route on the server pinned +to the Dispatch. The request contains the Dispatch ID, source preference, cursor, and limit—not a +terminal handle, session ID, or transcript path chosen by the coordinator. + +### 3. Revalidate the exact worker + +The worker server uses the existing Dispatch attachment to verify: + +- the exact managed pane; +- the exact terminal/process incarnation; and +- that the Dispatch has not been replaced, stopped, or detached. + +This preserves the same no-cross-worker rule already used by worker lifecycle and terminal reads. + +### 4. Resolve the pane's provider session + +Add one runtime-owned resolver that returns a snapshot similar to: + +```ts +type ExactWorkerProviderSession = { + paneKey: string + agent: TuiAgent + providerSession: AgentProviderSessionMetadata + observedAt: number +} +``` + +The resolver may use the current runtime graph/headless retained status, but it must accept the +Dispatch's exact pane identity as input. It must not search all sessions by working directory or +agent type. + +The source association is considered usable only when: + +- it belongs to the Dispatch's exact current pane/process; +- the hook metadata is fresh enough to belong to that process incarnation; +- the provider session metadata passes existing normalization/canonicalization; and +- a supported adapter can resolve an exact transcript. + +If these checks fail under `auto`, the read falls back to terminal output. + +Current session/status anchors: + +- `src/shared/agent-session-resume.ts` defines normalized provider-session metadata. +- `src/renderer/src/store/slices/agent-status.ts` maintains pane-scoped live agent status. +- `src/shared/runtime-types.ts` carries compatible agent status in runtime/mobile graph snapshots. +- `src/main/runtime/orca-runtime.ts` preserves provider-session metadata when it publishes those + snapshots. + +### 5. Read through a narrow adapter + +Reuse the existing bounded native-chat transcript parser rather than adding another parser stack. +Extract or wrap its pure reader behind a small orchestration adapter: + +```ts +type WorkerTranscriptReader = { + provider: 'codex' | 'claude' | 'openclaude' | 'grok' + readPage(input: ExactTranscriptRead): Promise +} +``` + +This is deliberately not a registry for every possible agent capability. Add an adapter only when +an exact locator and a tested reader already exist. + +Current reader anchors: + +- `src/main/native-chat/transcript-watch.ts` provides bounded transcript reads/subscriptions. +- `src/main/runtime/rpc/methods/native-chat.ts` exposes the existing reader over runtime RPC. +- `src/main/ipc/native-chat.ts` exposes the same reader to the desktop renderer. + +The transcript response should preserve the existing structured message/block representation and +the supported fields that the proven reader already understands. Unknown or skipped input should +produce parsing warnings rather than being silently presented as a complete transcript. + +### 6. Return bounded data + +Every path enforces: + +- at most 50 transcript messages per page (40 by default); +- a maximum serialized response size; +- existing clipping/redaction rules for large tool input and output; +- opaque projection of transcript-position fallback IDs and redaction of Dispatch capability tokens; +- deterministic pagination; and +- no transcript path leakage. + +Transcript observation is read-only. A failure or unknown network result must never trigger worker +restart, retry, stop, or Task mutation. + +## Federation and cross-platform behavior + +The core topology is: + +```text +Mac Run home + -> authenticated connected-server RPC + -> Windows worker server + -> exact Windows pane/session + -> Windows-local transcript reader + -> bounded structured page back to Mac +``` + +The reverse direction must work identically. + +Platform rules: + +- Use Node path operations only on the server that owns the path. +- Do not normalize Windows paths on macOS or macOS/Linux paths on Windows. +- SSH and WSL execution remain behind their owning Orca server. +- If the exact transcript is accessible only on an SSH/WSL execution host, the worker server must + use an existing host-aware read mechanism or fall back to terminal. Do not copy the path to the + Run home. +- Mixed-version capability negotiation applies only at the Orca server protocol boundary. +- A server that does not advertise structured worker read receives the existing terminal-read RPC. + +Federation adds one narrow additive RPC, `orchestration.federationReadOutput`. The Run home probes +it by calling it. If the worker server returns `method_not_found`, the Run home calls the existing +`orchestration.federationRead` terminal method and wraps that result as a labeled +`remote_capability_unavailable` fallback. No generalized capability matrix is added. + +## Lifecycle behavior + +### Provider session appears after worker start + +Hooks may report a session after the TUI becomes ready. An initial `auto` read may therefore return +terminal output. A later first-page `auto` read may select transcript output. + +Once a cursor is returned, that cursor remains pinned to its selected source. + +### Provider session changes + +Compaction, resume, or process replacement may produce a new provider session: + +- a fresh read without a cursor may select the new exact session; +- a cursor for the old session returns `source_changed`; +- Orca does not merge the old and new transcripts implicitly. + +### Orca restart + +After restart, the worker server re-establishes the exact pane/process association using the same +runtime graph and retained-hook mechanisms used by sidebar/session recovery. + +- If exact identity and session still match, paging continues. +- If process identity is uncertain, return `worker_identity_changed`. +- If only transcript identity is unavailable, `auto` may start a new terminal page but must not + reinterpret an old transcript cursor as a terminal cursor. + +### Disconnect + +A disconnected federated read is a read-only unknown result. Reissuing the same read is safe. +No mutation request ledger, durable outbox, automatic failover, or worker replacement is needed. + +## Implementation plan + +### Work package 1 — Correct the contract and types + +- Update the orchestration checklist to replace the inaccurate claim that exact pane-to-session + association does not exist. +- Add the source preference, response envelope, fallback enums, and opaque cursor types. +- Keep the existing terminal response fields compatible. +- Add the CLI `--source` option and accept both legacy numeric and new opaque cursors. + +Exit gate: contract tests cover legacy terminal JSON and the new labeled envelopes. + +### Work package 2 — Runtime exact-session resolver + +- Add a runtime method that resolves agent status for an exact Dispatch pane/process. +- Reuse the existing graph/headless retained status and provider-session normalization. +- Reject stale pane or process-incarnation associations. +- Test multiple panes and multiple sessions in the same worktree and directory. + +Exit gate: the resolver can never return a sibling pane's session. + +### Work package 3 — Codex transcript adapter + +- Reuse the current bounded native-chat Codex reader. +- Add deterministic page conversion and parsing warnings. +- Enforce entry and byte limits. +- Produce a path-free source identity digest and cursor. +- Add Claude only if it follows this same path without new infrastructure. + +Exit gate: exact Codex transcript pages are stable, bounded, and contain no local path. + +### Work package 4 — Local `worker-read` + +- Resolve and validate the exact worker. +- Implement `auto`, `transcript`, and `terminal`. +- Pin the source across cursor pages. +- Preserve the existing terminal fallback and non-JSON rendering. + +Exit gate: local dogfood proves correct selection with several simultaneous same-directory Codex +sessions. + +### Work package 5 — Federated `worker-read` + +- Add the narrow connected-server capability. +- Route transcript resolution and reading to the worker server. +- Wrap legacy remote terminal responses as labeled fallbacks. +- Reject mismatched Dispatch/server/session cursors. +- Ensure paths and internal server identity remain out of ordinary output. + +Exit gate: physical Mac-to-Windows and Windows-to-Mac reads both pass. + +### Work package 6 — Restart, fallback, and documentation + +- Cover runtime restart, renderer restart, disconnect, stale status, missing hooks, unreadable + transcript, unsupported providers, and mixed server versions. +- Update CLI help, the orchestration skill, and the implementation checklist. +- Dogfood the common coordinator loop using only the documented commands. + +Exit gate: every fallback is truthful and no fallback changes worker lifecycle state. + +## Validation plan + +### Unit and contract tests + +| Area | Required proof | +| --------------------- | ------------------------------------------------------------------------------------ | +| Exact resolution | A Dispatch resolves only its attached pane and process incarnation. | +| No directory guessing | Two Codex sessions in the same worktree cannot cross-read. | +| Source choice | `auto` prefers an exact supported transcript and otherwise labels terminal fallback. | +| Explicit source | `transcript` fails truthfully when unavailable; `terminal` never probes transcript. | +| Cursor pinning | Continued pages stay on the same source and provider session. | +| Session replacement | An old cursor returns `source_changed`. | +| Cursor custody | A cursor for another Dispatch is rejected. | +| Parsing | Malformed/skipped transcript records produce bounded warnings. | +| Limits | Entry count, block size, and total serialized response are bounded. | +| Privacy | Responses and cursors contain no transcript path. | +| Compatibility | Existing terminal fields and numeric cursors continue to work. | + +Likely focused test locations: + +- provider-session normalization and pane association tests; +- native-chat transcript reader tests; +- orchestration worker-control RPC tests; +- orchestration worker CLI tests; and +- federation protocol and physical harness tests. + +### Local integration scenarios + +1. Start a Codex worker and confirm the sidebar reports its provider session. +2. Read the Dispatch and verify `source=transcript`. +3. Start two Codex workers in the same worktree. +4. Give them distinct prompts and verify neither read contains the other's content. +5. Page both transcripts and verify stable source identities. +6. replace or resume one session and verify its old cursor returns `source_changed`. +7. Disable hooks and verify a labeled terminal fallback. +8. Remove or make the transcript unreadable and verify a safe fallback or + `transcript_required`, depending on the requested source. + +### Physical federation matrix + +| Run home | Worker server | Worker location | Required outcome | +| -------- | ------------- | --------------- | --------------------------------------------------------------- | +| macOS | Windows | native Windows | Exact structured page or labeled supported fallback | +| Windows | macOS | native macOS | Exact structured page or labeled supported fallback | +| macOS | macOS/Linux | SSH host | Exact host-aware page or terminal fallback without path leakage | +| Windows | Windows | WSL | Exact host-aware page or terminal fallback without path leakage | + +For both Mac/Windows directions: + +- run multiple workers at once; +- page beyond the first response; +- restart the Run home; +- restart the worker server; +- disconnect and reconnect the server; +- verify mixed-version fallback with one server lacking the new capability; and +- compare the selected provider session with the sidebar/native-chat session for the same pane. + +### Dogfood procedure + +The dogfood is successful only if a coordinator can follow this loop without internal IDs: + +1. Create/use a Run. +2. Start one local worker and one connected-server worker. +3. Wait for both starts to settle. +4. Call `worker-read --source auto` for each Dispatch. +5. Continue each cursor through at least two pages. +6. Confirm output belongs to the correct prompt and machine. +7. Trigger one fallback case. +8. Complete both workers and confirm reads never altered lifecycle state. + +Record: + +- command and response; +- chosen source and fallback reason; +- worker server/platform; +- provider and session match; +- cursor behavior; +- path-leak check; +- restart/disconnect outcome; and +- any agent confusion using only CLI help and the orchestration skill. + +## Acceptance criteria + +Implementation is complete only when: + +- `worker-read` selects an exact transcript or returns a clearly labeled terminal fallback. +- No test or dogfood scenario reads a sibling or previous provider session. +- A cursor never switches source or provider session silently. +- Mac-to-Windows and Windows-to-Mac physical reads pass. +- Runtime restart and disconnect behavior are safe and understandable. +- Unsupported agents and mixed versions retain useful terminal output. +- Transcript paths never leave the server that owns them. +- Existing terminal-read clients remain compatible. +- The common agent path remains one command with no server/session/path inputs. +- No UI, scheduling, retry, integration tracking, or generalized provider framework is added. + +## Explicit non-goals + +- No dashboard or sidebar changes. +- No coordinator chat changes. +- No automatic worker placement, retry, replacement, or recovery. +- No commit, test, branch, merge, or integration tracking. +- No provider-session locking or resume orchestration. +- No live transcript subscription in the orchestration API. +- No universal transcript/event ontology. +- No cross-server filesystem access from the Run home. +- No replicated Run database or automatic Run-home failover. +- No generalized access-control or capability framework. + +## Main risks and mitigations + +| Risk | Mitigation | +| ---------------------------------------------- | ------------------------------------------------------------------------------------------------- | +| Live session status is stale or renderer-owned | Resolve through a runtime-owned exact-pane snapshot and bind it to process incarnation. | +| Two sessions share a directory | Never use directory/latest-session lookup; require exact pane/session metadata. | +| A session changes between pages | Pin source identity in the cursor and return `source_changed`. | +| Remote path is meaningless or sensitive | Read only on the worker server and never serialize the path. | +| Transcript parser drops data | Preserve supported structured blocks and return parsing warnings. | +| Transcript metadata exposes a path/credential | Make file-position IDs opaque and redact Dispatch capabilities from all structured text/payloads. | +| Mixed server versions | Negotiate one narrow capability and fall back to existing terminal read. | +| Full-screen terminal output remains noisy | Prefer structured output only when exact; retain terminal as the universal safety path. | +| Scope expands into a provider platform | Ship Codex first and require proven exact association plus an existing reader for every addition. | + +## Decision + +Implement structured worker output as a narrow extension of `worker-read`. + +The prerequisite is not a new sidebar or status system: Orca already tracks exact pane-scoped +provider sessions. The work is to make that existing association available to the worker-owning +runtime, read the transcript locally through proven readers, pin pagination to that source, and +federate only the bounded result. diff --git a/config/scripts/dev-cli-terminal-wrapper.mjs b/config/scripts/dev-cli-terminal-wrapper.mjs new file mode 100644 index 00000000000..6563eb991b7 --- /dev/null +++ b/config/scripts/dev-cli-terminal-wrapper.mjs @@ -0,0 +1,40 @@ +import { chmodSync, mkdirSync, writeFileSync } from 'node:fs' +import path from 'node:path' + +function escapeWindowsBatchValue(value) { + // Why: cmd.exe expands %NAME% even inside quotes, so literal path percent signs must be doubled. + return value.replaceAll('%', '%%') +} + +export function prepareDevCliTerminalWrappers({ + repoRoot, + userDataPath, + electronExecutable, + platform = process.platform +}) { + const binDir = path.join(repoRoot, 'out', 'bin') + const userDataBinDir = path.join(userDataPath, 'cli', 'bin') + const cliPath = path.join(repoRoot, 'out', 'cli', 'index.js') + mkdirSync(binDir, { recursive: true }) + mkdirSync(userDataBinDir, { recursive: true }) + + if (platform === 'win32') { + const wrapperContent = `@echo off\r\nset "ORCA_USER_DATA_PATH=${escapeWindowsBatchValue(userDataPath)}"\r\nset "ORCA_DEV_CLI_INVOCATION=1"\r\nset "ORCA_APP_EXECUTABLE=${escapeWindowsBatchValue(electronExecutable)}"\r\nset "ORCA_APP_EXECUTABLE_NEEDS_APP_ROOT=1"\r\nnode "${escapeWindowsBatchValue(cliPath)}" %*\r\n` + for (const targetDir of [binDir, userDataBinDir]) { + for (const commandName of ['orca-dev.cmd', 'orca.cmd']) { + writeFileSync(path.join(targetDir, commandName), wrapperContent, 'utf8') + } + } + } else { + const wrapperContent = `#!/usr/bin/env bash\nexport ORCA_USER_DATA_PATH=${JSON.stringify(userDataPath)}\nexport ORCA_DEV_CLI_INVOCATION=1\nexport ORCA_APP_EXECUTABLE=${JSON.stringify(electronExecutable)}\nexport ORCA_APP_EXECUTABLE_NEEDS_APP_ROOT=1\nexec node ${JSON.stringify(cliPath)} "$@"\n` + for (const targetDir of [binDir, userDataBinDir]) { + for (const commandName of ['orca-dev', 'orca']) { + const wrapperPath = path.join(targetDir, commandName) + writeFileSync(wrapperPath, wrapperContent, 'utf8') + chmodSync(wrapperPath, 0o755) + } + } + } + + return { binDir, userDataBinDir } +} diff --git a/config/scripts/dev-cli-terminal-wrapper.test.mjs b/config/scripts/dev-cli-terminal-wrapper.test.mjs new file mode 100644 index 00000000000..c5bc5f4cdd1 --- /dev/null +++ b/config/scripts/dev-cli-terminal-wrapper.test.mjs @@ -0,0 +1,70 @@ +import { mkdtempSync, readFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { describe, expect, it } from 'vitest' +import { prepareDevCliTerminalWrappers } from './dev-cli-terminal-wrapper.mjs' + +describe('dev CLI terminal wrappers', () => { + it('writes profile-scoped Windows wrappers for worker terminals', () => { + const root = mkdtempSync(path.join(tmpdir(), 'orca-dev-terminal-wrapper-')) + const userDataPath = path.join(root, 'profile') + prepareDevCliTerminalWrappers({ + repoRoot: root, + userDataPath, + electronExecutable: path.join(root, 'electron.exe'), + platform: 'win32' + }) + + const wrapper = readFileSync(path.join(userDataPath, 'cli', 'bin', 'orca-dev.cmd'), 'utf8') + expect(wrapper).toContain(`set "ORCA_USER_DATA_PATH=${userDataPath}"`) + expect(wrapper).toContain('set "ORCA_DEV_CLI_INVOCATION=1"') + expect(wrapper).toContain(`node "${path.join(root, 'out', 'cli', 'index.js')}" %*`) + expect(readFileSync(path.join(userDataPath, 'cli', 'bin', 'orca.cmd'), 'utf8')).toBe(wrapper) + expect(readFileSync(path.join(root, 'out', 'bin', 'orca-dev.cmd'), 'utf8')).toBe(wrapper) + expect(readFileSync(path.join(root, 'out', 'bin', 'orca.cmd'), 'utf8')).toBe(wrapper) + }) + + it('escapes literal percent signs in every Windows batch path', () => { + const root = path.join(mkdtempSync(path.join(tmpdir(), 'orca-dev-terminal-wrapper-')), '%repo%') + const userDataPath = path.join(root, '%profile%') + const electronExecutable = path.join(root, '%electron%', 'electron.exe') + prepareDevCliTerminalWrappers({ + repoRoot: root, + userDataPath, + electronExecutable, + platform: 'win32' + }) + + const wrapper = readFileSync(path.join(userDataPath, 'cli', 'bin', 'orca-dev.cmd'), 'utf8') + expect(wrapper).toContain(`set "ORCA_USER_DATA_PATH=${userDataPath.replaceAll('%', '%%')}"`) + expect(wrapper).toContain( + `set "ORCA_APP_EXECUTABLE=${electronExecutable.replaceAll('%', '%%')}"` + ) + expect(wrapper).toContain( + `node "${path.join(root, 'out', 'cli', 'index.js').replaceAll('%', '%%')}" %*` + ) + expect(readFileSync(path.join(root, 'out', 'bin', 'orca-dev.cmd'), 'utf8')).toBe(wrapper) + expect(readFileSync(path.join(root, 'out', 'bin', 'orca.cmd'), 'utf8')).toBe(wrapper) + }) + + it('writes executable-style POSIX wrappers with the same profile identity', () => { + const root = mkdtempSync(path.join(tmpdir(), 'orca-dev-terminal-wrapper-')) + const userDataPath = path.join(root, 'profile') + prepareDevCliTerminalWrappers({ + repoRoot: root, + userDataPath, + electronExecutable: path.join(root, 'electron'), + platform: 'linux' + }) + + const wrapper = readFileSync(path.join(userDataPath, 'cli', 'bin', 'orca-dev'), 'utf8') + expect(wrapper).toContain(`export ORCA_USER_DATA_PATH=${JSON.stringify(userDataPath)}`) + expect(wrapper).toContain('export ORCA_DEV_CLI_INVOCATION=1') + expect(wrapper).toContain( + `exec node ${JSON.stringify(path.join(root, 'out', 'cli', 'index.js'))}` + ) + expect(readFileSync(path.join(userDataPath, 'cli', 'bin', 'orca'), 'utf8')).toBe(wrapper) + expect(readFileSync(path.join(root, 'out', 'bin', 'orca-dev'), 'utf8')).toBe(wrapper) + expect(readFileSync(path.join(root, 'out', 'bin', 'orca'), 'utf8')).toBe(wrapper) + }) +}) diff --git a/config/scripts/orca-dev-bin.test.mjs b/config/scripts/orca-dev-bin.test.mjs index 2b0e4f23f0f..afcaf0fb797 100644 --- a/config/scripts/orca-dev-bin.test.mjs +++ b/config/scripts/orca-dev-bin.test.mjs @@ -25,6 +25,7 @@ describe('orca-dev package bin', () => { `fs.writeFileSync(${JSON.stringify(outputPath)}, JSON.stringify({`, ' argv: process.argv.slice(2),', ' userDataPath: process.env.ORCA_USER_DATA_PATH,', + ' devCliInvocation: process.env.ORCA_DEV_CLI_INVOCATION,', ' appExecutable: process.env.ORCA_APP_EXECUTABLE', '}));' ].join('\n'), @@ -47,6 +48,7 @@ describe('orca-dev package bin', () => { expect(JSON.parse(readFileSync(outputPath, 'utf8'))).toEqual({ argv: ['--help'], userDataPath: path.join(root, 'user-data'), + devCliInvocation: '1', appExecutable: path.join(root, 'Electron') }) }) diff --git a/config/scripts/orca-dev.mjs b/config/scripts/orca-dev.mjs index ce5531da6d9..bb4dca441fa 100755 --- a/config/scripts/orca-dev.mjs +++ b/config/scripts/orca-dev.mjs @@ -3,6 +3,7 @@ import { spawnSync } from 'node:child_process' import { accessSync, constants, existsSync, realpathSync, statSync } from 'node:fs' import path from 'node:path' +import { prepareDevCliTerminalWrappers } from './dev-cli-terminal-wrapper.mjs' const scriptPath = realpathSync(import.meta.filename) const scriptDir = path.dirname(scriptPath) @@ -16,6 +17,8 @@ if (!existsSync(cliEntry)) { } process.env.ORCA_USER_DATA_PATH = process.env.ORCA_DEV_USER_DATA_PATH ?? getDefaultDevUserDataPath() +// Why: custom dev profiles do not necessarily contain "orca-dev" in their path; carry explicit provenance into the CLI. +process.env.ORCA_DEV_CLI_INVOCATION = '1' const electronExecutable = getElectronExecutable() if (!process.env.ORCA_APP_EXECUTABLE && isRunnableFile(electronExecutable)) { @@ -23,6 +26,13 @@ if (!process.env.ORCA_APP_EXECUTABLE && isRunnableFile(electronExecutable)) { process.env.ORCA_APP_EXECUTABLE_NEEDS_APP_ROOT = '1' } +// Why: headless `orca-dev serve` skips the Electron dev runner that normally installs terminal CLI shims. +prepareDevCliTerminalWrappers({ + repoRoot, + userDataPath: process.env.ORCA_USER_DATA_PATH, + electronExecutable: process.env.ORCA_APP_EXECUTABLE ?? electronExecutable +}) + const result = spawnSync(process.execPath, [cliEntry, ...process.argv.slice(2)], { stdio: 'inherit', env: process.env diff --git a/config/scripts/orchestration-skill-guidance.test.mjs b/config/scripts/orchestration-skill-guidance.test.mjs index bf9e34dc7ab..74fe975048a 100644 --- a/config/scripts/orchestration-skill-guidance.test.mjs +++ b/config/scripts/orchestration-skill-guidance.test.mjs @@ -29,10 +29,14 @@ describe('orchestration skill guidance', () => { const skill = readSkill() const toolBoundary = getSection(skill, 'Tool Boundary') - expect(toolBoundary).toContain( - 'must create Orca runtime state with `orca orchestration task-create` and `orca orchestration dispatch --inject`' + expect(toolBoundary).toContain('must create or bind a Run') + expect(toolBoundary).toContain('create the Task with `orca orchestration task-create`') + expect(toolBoundary).toContain('preferred `orca orchestration worker-start` composition') + expect(toolBoundary).toContain('low-level `orca orchestration dispatch --inject` path') + expect(toolBoundary).not.toContain('or `orca orchestration run`') + expect(skill).toContain( + '`coordinator-start`, `coordinator-stop`, `run`, and `run-stop` are retired scheduler commands' ) - expect(toolBoundary).toContain('or `orca orchestration run`') expect(toolBoundary).toContain( 'Do not substitute non-Orca subagent tools, generic agent-spawn APIs, or chat-only parallel worker features' ) @@ -47,6 +51,20 @@ describe('orchestration skill guidance', () => { ) }) + it('teaches the hard cutover without reviving a legacy executor', () => { + const skill = readSkill() + const migration = getSection(skill, 'Contract Migration') + + expect(migration).toContain('hard cutover') + expect(migration).toContain('effectsApplied') + expect(migration).toContain('skills get orchestration --full') + expect(migration).toContain('Do not retry the rejected command unchanged') + expect(migration).toContain('no longer supervised') + expect(migration).toContain('task-list --run run_legacy_local') + expect(migration).toContain('Read-only inspection never consumes legacy mail') + expect(migration).toContain('does not run a legacy scheduler, translate old writes, or drain') + }) + it('treats long-running worker waits as liveness checkpoints, not failures', () => { const skill = readSkill() @@ -213,13 +231,13 @@ describe('orchestration skill guidance', () => { const messaging = getSection(skill, 'Messaging') const workerTerminals = getSection(skill, 'Worker Terminals') const agentFirstExample = workerTerminals.match( - /```bash\norca worktree create --name --agent codex --json\n[\s\S]*?```/ + /```bash\norca worktree create --name --agent codex --setup run --json\n[\s\S]*?```/ )?.[0] expect(workerTerminals).toContain('For an allowed new worktree, use agent-first:') expect(workerTerminals).toContain('fallback shell + agent pair') expect(workerTerminals).toContain( - 'Repo setup or default-terminal settings may still add tabs or splits' + 'repo setup and default-terminal settings may add intentional tabs or splits' ) expect(workerTerminals).toContain('without configured default tabs') expect(workerTerminals).toContain( @@ -229,12 +247,11 @@ describe('orchestration skill guidance', () => { expect(workerTerminals).not.toContain('ends with **one** agent tab') expect(agentFirstExample).toBeDefined() expect(agentFirstExample).not.toContain('orca terminal list') + expect(agentFirstExample).toContain('agentTerminalHandle') expect(agentFirstExample).toContain('startupTerminal.handle') - expect(messaging).toContain( - 'Use `startupTerminal.handle` from the create response when present' - ) - expect(messaging).toContain('continue with the replacement only') - expect(messaging).toContain('it does not remotely wake another terminal') + expect(messaging).toContain('Prefer `agentTerminalHandle` from the create response') + expect(messaging).toContain('Continue with the replacement handle only') + expect(messaging).toContain('never writes to terminal input or remotely wakes another terminal') expect(messaging).toContain('Use `orchestration dispatch --inject` to deliver a tracked task') }) }) diff --git a/config/scripts/run-electron-vite-dev.mjs b/config/scripts/run-electron-vite-dev.mjs index 9b8aad89557..0a578140593 100644 --- a/config/scripts/run-electron-vite-dev.mjs +++ b/config/scripts/run-electron-vite-dev.mjs @@ -1,7 +1,6 @@ import { execFileSync, spawn } from 'node:child_process' import { createHash } from 'node:crypto' import { - chmodSync, cpSync, existsSync, lstatSync, @@ -17,6 +16,7 @@ import { import net from 'node:net' import { createRequire } from 'node:module' import path from 'node:path' +import { prepareDevCliTerminalWrappers } from './dev-cli-terminal-wrapper.mjs' // Why: Electron-based hosts (e.g. Claude Code, VS Code) set // ELECTRON_RUN_AS_NODE=1 in their terminal environment. If this leaks into @@ -314,35 +314,12 @@ function getDevUserDataPath() { } function prepareDevCliWrapper() { - const binDir = path.join(repoRoot, 'out', 'bin') - mkdirSync(binDir, { recursive: true }) const userDataPath = getDevUserDataPath() - const userDataBinDir = path.join(userDataPath, 'cli', 'bin') - const cliPath = path.join(repoRoot, 'out', 'cli', 'index.js') - const electronBin = getElectronExecutable() - - if (process.platform === 'win32') { - writeFileSync( - path.join(binDir, 'orca-dev.cmd'), - `@echo off\r\nset "ORCA_USER_DATA_PATH=${userDataPath}"\r\nset "ORCA_APP_EXECUTABLE=${electronBin}"\r\nset "ORCA_APP_EXECUTABLE_NEEDS_APP_ROOT=1"\r\nnode "${cliPath}" %*\r\n`, - 'utf8' - ) - } else { - const wrapperContent = `#!/usr/bin/env bash\nexport ORCA_USER_DATA_PATH=${JSON.stringify(userDataPath)}\nexport ORCA_APP_EXECUTABLE=${JSON.stringify(electronBin)}\nexport ORCA_APP_EXECUTABLE_NEEDS_APP_ROOT=1\nexec node ${JSON.stringify(cliPath)} "$@"\n` - const wrapperPath = path.join(binDir, 'orca-dev') - writeFileSync(wrapperPath, wrapperContent, 'utf8') - chmodSync(wrapperPath, 0o755) - - mkdirSync(userDataBinDir, { recursive: true }) - for (const commandName of ['orca-dev', 'orca']) { - const userDataWrapperPath = path.join(userDataBinDir, commandName) - // Why: dev Orca terminals prepend this directory to PATH; refreshing the - // `orca` alias prevents stale global/userData wrappers from hijacking - // Orca-owned commands such as `orca claude-teams`. - writeFileSync(userDataWrapperPath, wrapperContent, 'utf8') - chmodSync(userDataWrapperPath, 0o755) - } - } + const { binDir } = prepareDevCliTerminalWrappers({ + repoRoot, + userDataPath, + electronExecutable: getElectronExecutable() + }) process.env.PATH = `${binDir}${path.delimiter}${process.env.PATH ?? ''}` console.log(`[orca-dev] Prepared wrapper in ${binDir}`) diff --git a/docs/orchestration-primitives.html b/docs/orchestration-primitives.html new file mode 100644 index 00000000000..00884666035 --- /dev/null +++ b/docs/orchestration-primitives.html @@ -0,0 +1,2937 @@ + + + + + + + Orca Orchestration: Strong Primitives, Little Magic + + + + +
+ + +
+
+
+

Orca orchestration proposal

+

Strong primitives. Little magic. No orchestration product inside the product.

+

+ Orca should make it intuitive for a coordinating agent to start workers, communicate, + wait, observe output, and recover safely. Orca supplies dependable building blocks; + the agent decides the orchestration strategy. +

+ +
+ CLI and runtime only + agent-directed + multi-server capable + explicit effects + no commit tracking +
+ +
+ +
+ The common agent loop +

+ Create and bind one Run for the coordination effort; after that its ID is carried + automatically. Create tasks, start workers on this or another connected Orca + server, send messages, wait for inbox mail, and read or stop workers. Options + refine those operations without adding policy. +

+
+
+ +

+ The test for the design is simple: after reading a few examples, an agent should be + able to predict what every command creates, reuses, blocks on, and returns. +

+
+ +
+
+

What we learned

+

The current problems are coordination problems, not missing product surfaces.

+

+ The research found real reliability gaps, but the earlier proposal responded by + adding a scheduler, integration subsystem, dashboard, and large control-plane model. + Those additions would make the common agent workflow harder to understand. +

+
+ +
+
+

Current

+

Lifecycle mail arrives through prompt injection

+

+ Messages persist, but coordinator delivery may wait for the agent to pause. If it + keeps polling or working, messages can collect and flood the editable input after + a manual interruption. +

+
+ +
+

Needed

+

A structured, blocking inbox call

+

+ Typed worker lifecycle messages return from a pending tool request or the next explicit + read. Routine lifecycle delivery never writes into the coordinator's prompt. +

+
+
+ +
+
+

Current

+

Starting a worker is assembled from low-level pieces

+

+ Creating a worktree may already create a terminal, but an agent can miss that and + create another terminal before launching the worker. +

+
+ +
+

Needed

+

One composed start operation with full topology and setup choices

+

+ The operation composes existing worktree, setup, terminal, and agent creation and + returns exactly which resources it created or reused. +

+
+
+ +
+
+

Current

+

Terminal scrollback is treated as agent history

+

+ Full-screen TUIs can redraw or discard the useful conversation, making terminal + reads incomplete or misleading. +

+
+ +
+

Implemented

+

Exact structured output with a truthful fallback

+

+ Orca reuses its pane-scoped hook association to read a supported Codex, Claude, + OpenClaude, or Grok transcript. When it cannot prove that source, it returns + labeled bounded terminal output instead of guessing a session. +

+
+
+ +
+
+

Current

+

Results are mostly worker assertions

+

+ Orca verifies that the active dispatch reported worker_done, but the + summary, changed files, tests, and report path come from the worker. +

+
+ +
+

Needed

+

Be explicit about what Orca observed

+

+ Keep lifecycle authority separate from worker-reported content. Do not add commit, + test, merge, or integration tracking merely to make the report appear stronger. +

+
+
+
+ +
+
+

Design rules

+

Reliability should live underneath a small interface.

+

+ Strong primitives have narrow, testable contracts. They do not need to expose every + mechanism used to make the contract safe. +

+
+ +
+
+ 01 +

The coordinator owns strategy

+

+ The agent chooses decomposition, ordering, parallelism, placement, review, and when + to wait. Orca does not schedule ready tasks automatically. +

+
+
+ 02 +

Every effect is visible

+

+ Responses say which connected server, worktree, setup, terminal, execution host, + and agent were created or reused. Defaults are reported, not hidden. +

+
+
+ 03 +

Simple default, explicit escape hatch

+

+ The common path needs few arguments and preserves worktree/setup choices. Uncommon + custom launches stay on the existing low-level commands instead of bloating start. +

+
+
+ 04 +

Observation is honest

+

+ Orca distinguishes observed process state from worker-reported claims and labels + the source of transcript or terminal output. +

+
+
+ 05 +

Remote ambiguity stays ambiguous

+

+ A disconnect after a remote mutation returns outcome_unknown. Orca + does not silently repeat a command that may have succeeded. +

+
+
+ 06 +

Safety is not orchestration policy

+

+ Stable identity, stale-worker fencing, and owner routing prevent corruption. They + do not choose what work should happen next. +

+
+
+ +

+ A feature belongs in the core only if it makes an existing primitive safer or clearer. + Tracking extra domain facts—commits, merges, budgets, priorities, or organizational + roles—is not automatically a stronger primitive. +

+
+ +
+
+

Primitive 1 · Scope and identity

+

Four public concepts, each with one job.

+

+ A Run prevents unrelated coordination efforts from mixing. Tasks describe work, + Dispatches authorize workers, and Messages communicate. Existing worktree and + terminal resources remain independently usable. +

+
+ +
+
+

Run

+

A lightweight namespace and stable coordinator mailbox. It never schedules work.

+
+
+

Task

+

A durable description, status, and optional dependencies. Creating it starts nothing.

+
+
+

Dispatch

+

One supervised worker assignment and its current lifecycle authority.

+
+
+

Message

+

Durable communication or a typed lifecycle report returned through the inbox.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Persisted worker stateTask state and allowed next action
starting / readyTask is dispatched; only show, read, message, or stop may act on it.
start_unknownTask is blocked; recover the same request receipt, inspect, stop, or abandon.
failed + Confirmed start failure and authenticated worker failure both leave Task + failed. Either permits an explicit --retry-of replacement. +
succeededTask is completed; create a follow-up Task rather than retrying it.
stopping / stop_unknown + Task is blocked and lifecycle authority is fenced; inspect termination or + explicitly abandon before replacement. +
stopped / abandonedTask is blocked and permits an explicit --retry-of replacement.
+
+ +
+ + + + + + + + + + + + + + + + + + + + + +
Run operationExact effect
run-create + Creates a run and stable coordinator mailbox on the coordinator's selected + Orca server—its Run home—then binds the current coordinator terminal as its + active consumer. +
run-use --id run_123 + Explicitly binds or rebinds the current terminal to that run. Rebinding fences + the prior consumer generation and cancels its waiter. A terminal has at most + one active run binding. +
run-current / run-list / run-showReports bindings and state without changing tasks, workers, or mail.
+
+ +
+
+

Run IDs stay out of the common path

+

+ A command resolves the run only from explicit --run or the current + terminal binding—never from a worktree or an “exactly one candidate” guess. + Commands return the resolved Run or runId where it is needed for later + control; agents do not carry a separate resolution-mode field. +

+
+
+

Low-level commands remain run-free

+

+ Ordinary worktree, terminal, and full-handoff commands do not create or require a + Run. Runs exist only for supervised coordination that needs durable grouping. +

+
+
+

One home, workers anywhere

+

+ A Run has one home server that owns its tasks and inbox. A Dispatch may point to a + worker on any connected Orca server; Orca relays that worker's messages back to the + home automatically. +

+
+
+ +

+ ELI5: a Run is a folder label plus a return address. It keeps one coordination + effort's tasks and mail together and gives workers on your Mac or Windows server the + same stable place to reply. + Create or select it once, then ordinary orchestration commands inherit it. It does + not create resources, choose workers, schedule tasks, or group projects. +

+ +

+ If a supervised command has neither --run nor a terminal binding, return + run_required with exact run-create and run-use + examples. Never silently create or infer a Run. +

+ +

+ V1 does not archive or delete Runs. run-list and run-show keep + old coordination state inspectable; retention administration waits for a demonstrated + clutter or storage problem instead of complicating the common lifecycle now. +

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Canonical commandCompatibility decision
run-create/list/show/use/currentNew lightweight scope commands. They never decompose or schedule tasks.
task-create/list/updateKeep the current flat command names and add Run association.
dispatchKeep as the low-level binding to an exact existing terminal.
worker-start/show/read/stop/abandonNew composed supervised-worker operations.
send/ask/reply/checkKeep and strengthen the current message operations.
Current scheduler-like orchestration run --spec + Must be removed or renamed in Phase 0 before run-* can ship. It is + not an alias for a lightweight Run and is not part of this scheduler-free + design. +
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IDWho normally carries it
runIdThe terminal binding carries it; agents pass it only to override context.
taskIdThe coordinator uses it for dependencies and worker start.
dispatchId + Receipts and worker preambles carry it; show, read, stop, abandon, retry, and + lifecycle reports use it. +
deliveryIdThe current mailbox consumer carries only its last unacknowledged delivery.
messageId / threadId + reply takes a message ID and infers its thread. Agents do not + manage a separate question identifier. +
Resource IDsReceipt data until an agent explicitly reads, stops, or reuses that resource.
RPC request ID + The client transport creates it automatically. Agents only echo the returned + retry token after an unknown outcome; they never invent one. +
+
+ + 
orca orchestration run-create --objective "Improve message delivery"
+# → run_123, bound to this coordinator terminal
+orca orchestration task-create --spec "Audit message delivery"
+# → task_a, runId run_123 (bound)
+orca orchestration worker-start --task task_a --worktree current --agent codex
+orca orchestration check --wait --timeout-ms 60000
+
+ +
+
+

Primitive 2 · Supervised worker start

+

One request, predictable behavior for every topology.

+

+ worker-start is one synchronous composition of existing worktree, setup, + terminal, and dispatch operations. It is not a background executor, external + transaction, or placement engine. The coordinator chooses the topology; Orca returns + only after the composition is ready, failed, or honestly unknown. +

+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
TopologyDefault terminal behaviorSetup behavior
Current worktree + Create one fresh agent terminal. Never reuse the coordinator terminal or an + arbitrary idle terminal. Reuse requires explicit --terminal. + not_applicable; do not rerun setup or configured tabs.
Named existing worktreeCreate one fresh agent terminal; reuse only an explicitly selected terminal.not_applicable; creation-time setup is not replayed.
New child worktree + Use agent-first worktree creation and reuse its returned agent terminal. Never + create a second shell/agent terminal. + + Default to run. Setup and agent launch start side by side unless the + repository explicitly uses wait-for-setup. skip or + inherit must be explicit. +
New top-level worktreeSame agent-first behavior, with top-level Orca lineage. + Same setup default: run a configured hook; use an explicit escape hatch only + for a concrete reason. +
+
+ +
+
+

Worktree option parity means pass-through

+

+ The exact repository selector, name, base branch, child/top-level lineage, setup, + and display/comment metadata are validated and passed to the existing worktree + primitive. --on already chooses the connected Orca server, so V1 does + not add a second project/host placement vocabulary to worker-start; + agents may use low-level worktree create when that convenience selector + is important. +

+

+ Folder projects have no distinct Git worktree to create. New-child and + new-top-level therefore fail before effects; use current or an exact existing + folder workspace. +

+
+
+

Choose a connected server only when needed

+

+ The default is the Run home. Use --on windows (a saved Orca environment + name or ID) only to place a worker on another connected server. V1 resource IDs + are server-scoped, so every remote existing worktree or terminal also requires + --on. Orca never guesses an owner from a same-looking ID and echoes the + resolved server name in the receipt. +

+
+
+

Agent selection is honest

+

+ When creating a terminal, V1 requires an explicit --agent that resolves + through Orca's configured launcher before any effect. Composed start does not + promise custom model, environment, or arbitrary command arguments that agent-first + worktree creation cannot actually pass through. +

+
+
+

Setup is the safe default

+

+ For every new worktree, omitted --setup resolves to + run. If a setup hook exists, Orca launches it; if none exists, the + receipt says not_configured. Preserve the repository's existing + setupAgentStartupPolicy: its default is start-immediately, + so setup does not delay agent launch or task delivery. Only an explicit + wait-for-setup policy gates the agent. An agent may choose + skip or inherit only for a specific reason it states in its + work log. Orca trusts that judgment and adds no approval gate. +

+
+
+

Supervised means lifecycle injection

+

+ Task and dispatch input is delivered only after agent readiness. Ordinary + worktree/terminal commands remain the full-handoff path without lifecycle duties. +

+
+
+ +

+ worker-start --on windows is deliberately different from Orca's existing + global --environment windows. --on keeps the command and Task on + the bound Run home while selecting only the worker server. The global flag still means + “send this whole CLI command to that environment” and must not silently change meaning. +

+ +

+ Validate conflicts before effects: --terminal requires a current or named + existing worktree and cannot combine with --agent, setup, or new-worktree + creation options. New worktrees require a name and accept only options supported by + the selected underlying worktree command. Custom agent argv stays on the explicit + low-level worktree create → terminal create --command → + dispatch path, but that two-step path is legal only with + start-immediately; it cannot preserve an explicit + wait-for-setup policy because the later terminal is not the creation-time + startup. Use an agent-first configured launcher or stop and ask rather than bypassing + that policy. current always means the coordinator's current + worktree on the Run home and cannot combine with remote --on. A remote + existing worker needs an exact worktree/terminal selector or an explicit server so + same-looking resources are never guessed. Every resolved default includes its source. + Discover a remote selector with an explicit read-only command such as + orca --environment windows worktree list --json, then copy the returned + opaque worktree ID into worker-start --on windows. +

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Placement/federation errorMeaning
server_requiredThe selected runtime cannot provide connected-server orchestration.
worktree_not_found_on_serverThe exact worktree does not exist on the selected worker server.
terminal_worktree_mismatchThe exact terminal is not owned by the selected worktree.
resource_server_mismatch + A worker-server receipt names a different Dispatch or authenticated Run home; + Orca never adopts that attachment. +
agent_unconfiguredThe requested launcher is unavailable; no worktree or terminal was created.
+
+ +
+ + + + + + + + + + + + + + + + + + + + + +
Task acceptanceWorker-start effect
First start + Requires a ready Task with no current Dispatch. The runtime creates the + Dispatch and moves the Task to dispatched in one local transaction before + running the composed effects. +
Replacement attempt + Requires --retry-of naming the Task's current failed, stopped, or + abandoned Dispatch while the Task is failed or blocked. Start creates the next + Dispatch generation and moves the Task back to dispatched atomically; the agent + never performs a preparatory task-update. +
Anything else + Return task_not_startable naming the rejected Task/Dispatch and + perform no worker effects. Completed work gets a new follow-up Task; an unknown + or nonterminal Dispatch must first be inspected, stopped, or abandoned. +
+
+ +

+ ready has one testable meaning: the selected agent terminal reached + tui-idle, the local or remote Dispatch attachment is durable, and the + lifecycle preamble plus task input was accepted. Under the default + start-immediately policy, setup may still be running and its exact state is + returned in the receipt; its outcome never gates readiness, even if failure is observed + before the agent reaches tui-idle. Under an explicit + wait-for-setup repository policy, setup must complete successfully before + agent launch and task injection. Agent-first worktree creation launches without the + task prompt so Orca can establish authority before injection. The effective timeout + and startup policy are echoed in every receipt. + A successful gated receipt reports setup succeeded. A confirmed setup + spawn/script failure reports failed before task input, while a timeout may + honestly retain running rather than inventing a failure. +

+ +

+ Underneath the call, one transaction creates a starting Dispatch, moves the + Task, and records the retry request before effects. Orca journals each irreversible + stage and resource receipt before advancing. Restart recovery can therefore prove + no-effect, return a durable remote result, or remain honestly + outcome_unknown. A pending accepted start retains its Dispatch ID, so + repeating the exact request after restart returns that ID and its + worker-show command instead of inviting a second start. Orca does not + resume unfinished orchestration stages in the background; a setup process + intentionally launched under + start-immediately may continue exactly as its receipt states. +

+ +

+ With start-immediately, the start receipt carries the latest observed setup + state. Only a setup state change after that receipt updates worker-show and + emits a typed notice; setup never rewrites Dispatch readiness or decides whether the + worker should stop. With explicit wait-for-setup, setup failure is a start + failure before task input is sent. +

+ +
+ 
orca orchestration worker-start --task task_a --worktree new-child --name message-audit --agent codex --setup run
+
+# Returns when the worker is ready, start failed, or the outcome is unknown.
+# Coordinators can issue independent start calls in parallel.
+ 
# "result" excerpt from --json
+{
+  "runId": "run_123",
+  "taskId": "task_a",
+  "dispatchId": "dispatch_7",
+  "state": "ready",
+  "stage": "input_accepted",
+  "setup": {
+    "requested": "run",
+    "effective": "run",
+    "source": "explicit_request",
+    "hookFound": true,
+    "startupPolicy": "start-immediately",
+    "state": "running"
+  },
+  "timeoutMs": 60000,
+  "effects": [
+    { "kind": "worktree", "action": "created_child", "id": "worktree_9" },
+    { "kind": "terminal", "role": "setup", "action": "created", "id": "term_setup_11", "tabId": "tab_2", "leafId": "leaf_1" },
+    { "kind": "setup", "action": "run", "requested": "run", "effective": "run", "source": "explicit_request", "hookFound": true, "startupPolicy": "start-immediately", "state": "running", "terminalId": "term_setup_11" },
+    { "kind": "terminal", "role": "agent", "action": "reused_agent_terminal", "id": "term_12" },
+    { "kind": "terminal", "role": "configured_tab", "action": "created", "id": "term_13", "tabId": "tab_3", "leafId": "leaf_1" },
+    { "kind": "dispatch_input", "role": "agent", "id": "term_12", "state": "accepted" }
+  ],
+  "residualResources": [],
+  "mutation": { "requestId": "req_7", "replayed": false }
+}
+
+ +

+ Receipts enumerate every created or reused terminal with a role—agent, setup, or each + configured terminal (including each tab/split pane), with exact tab and leaf identity + when available. Setup may be reported running only after its exact + PTY spawn receipt is durable. These role-tagged effects are also the authoritative + residual-resource list used by restart inspection and failure recovery. +

+ +
+ + + + + + + + + + + + + + + + + + + + + +
Start stateMeaning and next action
ready + Agent is ready and lifecycle input was accepted. The return receipt contains + every created or reused effect; no separate startup notice is required. +
failed + Return the failed stage, last error, residual resources, Task/Dispatch state, + and the durable mutation receipt. The composition itself has returned, but any + surviving setup, terminal, or agent process is listed honestly as a residual + resource; Orca never implies that failure cleaned it up. The coordinator + chooses the matching recovery branch below. +
outcome_unknown + The connection failed after an effect may have happened. Return the operation + stage, mutation.requestId, durable effects/residuals, and exact + worker-show/worker-abandon commands. A replacement is + rejected until inspection proves it safe or the coordinator explicitly + abandons the old Dispatch. +
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + +
Unknown-start recoveryExact contract
Query and reconcile + worker-show --dispatch dispatch_7 routes from the Run home to the + owning worker server and execution host. If its durable receipt proves the + original worker became ready, failed, or stopped, the home reconciles that same + Dispatch; no separate adopt operation exists. +
Safe retry + --retry-of dispatch_7 links a new attempt but does not silently + reuse prior placement. The coordinator repeats an explicit valid topology and + agent/terminal choice, which may deliberately differ from the old attempt. A + new Dispatch is allowed only after the prior one is failed, stopped, abandoned, + or reconciled as no-effect. While it remains unknown, return + task_not_startable without mutation. +
Explicit stop + worker-stop --dispatch dispatch_7 may fence a + ready or start_unknown Dispatch. The owning server + closes a terminal only when its durable attachment still matches the exact + pane and process incarnation. Unattached, missing, exited, or identity-changed + workers become stop_unknown with no process action. If the remote + server durably stopped the exact worker but its response was lost, a later + worker-show reconciles that authoritative stopped receipt. +
Explicit abandon + worker-abandon --dispatch dispatch_7 fences future lifecycle + mutations from that Dispatch and records every possibly-live resource. It sends + no remote command, claims no process stopped, deletes nothing, and warns that a + concurrent worker may remain. The coordinator may then start a replacement. + Abandoning an older superseded Dispatch is a no-op and cannot block or rewrite + the replacement Task. +
+
+ +

+ The Dispatch ID is the worker identity. Do not add another agent-facing start ID. + Every mutating CLI result includes one opaque mutation.requestId for + recovering that exact request after a lost response; agents never invent it. A + transport failure exposes the same value as orchestrationRequestId in its + error recovery data. Semantic retry + explicitly names the prior Dispatch ID and never replays an unknown effect. After a + remote start, show, read, stop, abandon, and message routing use the Dispatch receipt; + the agent does not repeat --on for those controls or carry server IDs. A + replacement worker-start is a new placement decision and names its target + again. +

+ +

+ Transport retry and semantic retry are different. If the response itself was lost, + reissue the identical mutation with --retry-request req_7 to recover its + durable receipt; a changed canonical payload returns request_mismatch. Once + that receipt says outcome_unknown, inspect by Dispatch ID instead of + replaying it repeatedly. --retry-of dispatch_7 creates a new worker attempt + only after the old attempt is safe to replace. +

+
+ +
+
+

Primitives 3 and 4 · Message and wait

+

Structured mail is inbox-only, durable, and explicitly consumed.

+

+ The inbox is an agent API, not product UI and not terminal input. It strengthens + Orca's existing message store and check --wait; it does not add an Event + subsystem or a second orchestration engine. +

+
+ +

+ send, ask, reply, completion, heartbeat, and runtime + notices only persist structured state and wake a pending tool waiter. Only explicit + dispatch --inject and terminal send may modify terminal input. + From an active Dispatch, lifecycle send and ask default to that + Dispatch's owning Run mailbox. “Sent” means durably accepted—not pasted, observed, or + acted upon. +

+ +
+
+

run:run_123

+

The stable coordinator mailbox stored with that Run in the Orca runtime.

+
+
+

dispatch:dispatch_7

+

The exact supervised worker generation, independent of terminal handle changes.

+
+
+ +

+ V1 deliberately has no task: message recipient. Use the Run for guidance + that survives reassignment and the exact Dispatch for attempt-specific control. This + avoids a hidden acceptance-time retargeting rule. +

+ +

+ Orca authenticates lifecycle reports automatically. The worker supplies only the Task + and Dispatch IDs injected in its preamble; it never supplies Run, runtime, host, + terminal, capability, or attestation IDs. At injection, Orca gives the managed pane a + narrow unforgeable Dispatch capability through its CLI bridge. The worker server + verifies that capability and its local pane; the Run home accepts the relayed report + only from the pinned paired peer for that Dispatch. Stop, abandon, or replacement + revokes it. Other reports remain stale history and cannot change Task state. This is + lifecycle integrity, not a general permission system. +

+ +

+ A federated worker omits --to and --run; its authenticated + Dispatch already identifies the only valid Run home. Explicit targets are rejected + instead of being silently ignored or redirected. +

+ +
+
+

One Run home

+

+ The coordinator's server stores the authoritative Run, tasks, inbox ordering, and + acknowledgments. Worker servers do not replicate the Run database or elect a new + home. +

+
+
+

Remote mail waits safely

+

+ A worker server durably retains messages for its remote Dispatch until the Run + home imports and acknowledges them. Replies wait at the home until the worker + server reconnects. This is a narrow relay queue, not a replicated global inbox. +

+
+
+ +
+
+
Coordinator
+
Run home ↔ worker server
+
Worker
+
+
+
Starts every independent worker
+
Persists home record and remote attachment
+
Works concurrently
+
+
+
Calls check --wait only when no local work remains
+
Returns the outstanding batch or registers one waiter
+
Continues independently
+
+
+
Pending tool call is blocked
+
Durably relays question, failure, or completion
+
Reports one typed lifecycle message
+
+
+
Receives a structured batch
+
Persists and returns one opaque Delivery ID
+
No coordinator prompt injection
+
+
+
Processes every message, then acknowledges and waits
+
Atomically ack → check → register
+
May continue, stop, or receive a reply
+
+
+ +

+ check --wait always targets the Run home and fans in mail from every active + remote Dispatch. One disconnected worker server does not block local or other-server + messages. Imported relay items are idempotent by their authenticated Dispatch and + source sequence, then follow the same FIFO delivery and acknowledgment rules as local + mail. +

+ +
+ 
orca orchestration check --wait --timeout-ms 60000
+# → returns delivery_81 with an ordered message batch
+
+# Process every message and start newly-ready work.
+orca orchestration check --ack delivery_81 --wait --timeout-ms 60000
+ 
# "result" excerpt from --json
+{
+  "runId": "run_123",
+  "deliveryId": "delivery_81",
+  "messages": [
+    {
+      "id": "msg_481",
+      "run_id": "run_123",
+      "from_handle": "dispatch:dispatch_7",
+      "to_handle": "run:run_123",
+      "subject": "Review complete",
+      "type": "worker_done",
+      "payload": "{\"taskId\":\"task_a\",\"dispatchId\":\"dispatch_7\",\"outcome\":\"succeeded\"}",
+      "read": 0
+    }
+  ],
+  "count": 1,
+  "replayed": false,
+  "acknowledged": null,
+  "timedOut": false,
+  "cancelled": false,
+  "connectionLost": false
+}
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Inbox ruleV1 contract
Ordering and sizeFIFO by mailbox sequence, bounded to 50 messages per delivery.
Outstanding batch + One per mailbox. Until acknowledged, return the same Delivery ID and batch; + newer mail waits behind it. +
Acknowledgment + Whole-batch and idempotent. Repeating an acknowledged Delivery ID returns the + recorded result and does not consume newer mail. A Delivery is bound to the + consumer generation that received it; a fenced coordinator gets + consumer_fenced and cannot consume the replacement's mail. The same + current-consumer check applies to coordinator replies and other mailbox + mutations. +
Waiters + One active actionable waiter per mailbox. A second returns + waiter_exists; it never races to consume the batch. +
Crash safety + Receiving a batch never marks it consumed. Unacknowledged mail survives client, + mailbox-consumer, and runtime restart. +
Atomic continuation + check --ack delivery_81 --wait commits ack, checks queued mail, + then registers the waiter as one runtime operation. +
+
+ +
+
+

History modes do not consume

+

+ check --peek, --all, and type-filtered history reads are + read-only debugging surfaces. Legacy consume-on-check behavior is deprecated. + Rename local formatting flag check --inject so it cannot imply delivery. +

+
+
+

Timeouts are typed checkpoints

+

+ Wait returns timedOut, cancelled, or + connectionLost distinctly. None means worker failure, none consumes + mail, and transport keepalive output is not a worker heartbeat. +

+
+
+ +

+ Actionable check, acknowledgment, and wait never create separate filtered + deliveries. A type filter may decide when a waiter wakes, but the returned Delivery is + always the oldest full FIFO batch, including earlier nonmatching mail. History modes + may filter freely because they do not consume. +

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Lifecycle inputAtomic state effect at message acceptance
Authenticated active Dispatch reports worker_done outcome=succeededSet Dispatch settled/succeeded and Task completed.
Authenticated active Dispatch reports worker_done outcome=failedSet Dispatch settled/failed and Task failed. Coordinator chooses recovery.
Stale or foreign Dispatch reportPersist as stale history; do not change current Task or Dispatch state.
Malformed lifecycle reportReject the transition and return the missing or invalid field.
worker-start returns failedSet Dispatch and Task failed with an explicit recovery reason.
Confirmed worker-stopSet Dispatch stopped and Task blocked; never claim task completion.
worker-stop accepted, termination unknown + Fence lifecycle authority, set Dispatch stop_unknown and Task + blocked, and retain the process/terminal in residualResources as + possibly live. A new start remains unsafe until inspection confirms termination + or the coordinator explicitly abandons with the concurrent-worker warning. +
Confirmed worker-abandon + Set Dispatch abandoned and Task blocked, fence its later reports, and retain + possibly-live resource IDs; never claim process termination. +
+
+ +

+ Inbox acknowledgment confirms recipient consumption only. Lifecycle reconciliation + happens once, atomically, when the Run home imports an authenticated message. Every + terminal Dispatch transition is a home-side transactional compare-and-set: the first + committed completion, stop fence, or abandon wins; later conflicting input is retained + only as stale history. +

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Question stateAsk/reply behavior
pending + ask from an active Dispatch defaults to its owning Run mailbox. The + first reply from the current authenticated Run consumer generation records the + answer at the Run home, relays it to the exact worker server, and wakes the + local asking call. +
answered + The pending ask or an explicit resume by the original Dispatch + returns the recorded answer. Repeating the same reply is harmless; a later + different answer conflicts. +
Wait timed out or client disconnected + Return the original question message ID and an exact + ask --resume msg_question_7 command without closing or duplicating + the question. Resume is allowed only for the original Dispatch. There is no + separate Question ID or durable expiry/deadline state machine. +
Acceptance response was lost + Retry the identical ask with its returned transport retry receipt + to recover the original message ID. A changed question conflicts; a blind new + ask is never the recovery path. +
Dispatch stopped or abandoned + The Run home atomically closes its pending questions, wakes local or resumed + waits with dispatch_inactive, and rejects later replies. +
+
+ +

+ Question threads are message primitives, not task gates. Use message type + question; its target-side delivery follows normal inbox acknowledgment, + while the answer lives only on durable thread state. Keep separately managed task + gates outside this proposal. +

+ +
+ 
# Worker: target defaults to the owning Run mailbox.
+orca orchestration ask --question "Should I preserve the legacy format?" --json
+# → question message msg_question_7
+# After a disconnect/timeout: orca orchestration ask --resume msg_question_7 --json
+ 
# Coordinator: reply to the returned message ID.
+orca orchestration reply --id msg_question_7 --body "Yes; preserve it." --json
+
+ +

+ Blocking is a coordinator decision. The runtime must make waiting race-free, but it + must not decide that the agent has exhausted all parallelizable work. +

+
+ +
+
+

Worker observation

+

Read a supervised worker by Dispatch, regardless of server.

+

+ worker-read --dispatch resolves the worker server and exact process from + the Dispatch receipt. Its default auto source returns the exact + hook-reported Codex, Claude, OpenClaude, or Grok transcript when Orca can prove that + association; + otherwise it returns bounded, explicitly labeled terminal output. Agents never + choose a server, provider session ID, or transcript path, and task authority never + comes from transcript prose. +

+
+ +
+
+

Default path

+

Automatic exact selection

+

+ --source auto uses a proven supported transcript and falls back to + the existing bounded terminal reader with a typed reason such as + session_not_reported or + remote_capability_unavailable. +

+
+
+

Explicit policy

+

Transcript or terminal

+

+ --source transcript requires exact structured output and returns a + typed error rather than falling back. --source terminal always uses + the retained terminal snapshot. +

+
+
+ + 
orca orchestration worker-read --dispatch dispatch_7 --source auto --limit 100 --json
+
+# Exact transcript result for a worker on the saved environment named windows
+{
+  "dispatchId": "dispatch_7",
+  "source": "transcript",
+  "sourceIdentity": "opaque-source-fingerprint",
+  "provider": "codex",
+  "server": { "environmentId": "env_windows", "name": "windows" },
+  "remoteRuntimeEpoch": "runtime_epoch_2",
+  "transcript": {
+    "messages": [ ... ],
+    "nextCursor": "opaque-next-cursor",
+    "limited": false,
+    "returnedMessageCount": 12
+  },
+  "cursor": "opaque-next-cursor",
+  "status": { "worker": "ready", "terminal": "running" },
+  "fallbackReason": null,
+  "warnings": []
+}
+
+# Continue from the returned top-level opaque cursor.
+orca orchestration worker-read --dispatch dispatch_7 \
+  --cursor opaque-next-cursor --limit 100 --json
+ +
+
+

Source-pinned continuation

+

+ The returned cursor pins the Dispatch, process, source kind, and opaque source + identity. auto selects only on the first page. If the process or + provider session changes, Orca returns + worker_identity_changed or source_changed. +

+
+
+

Narrow provider readers

+

+ Structured reading reuses the existing bounded native transcript decoders only + for exact Codex, Claude, OpenClaude, and Grok associations. Other providers and + mixed-version peers retain terminal fallback; no resume, live-stream control, or + universal transcript framework is added. +

+
+
+ +

+ Every response labels source, sourceIdentity, + cursor, status, fallback reason, and bounded warnings. Terminal fallback + preserves the existing terminal fields and accepts legacy numeric cursors; new cursors + are opaque and never expose a transcript path. The worker-owning server performs the + read, and neither local nor federated selection may guess “latest session in this + directory.” +

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + +
Orca can sayOrca cannot infer
The active dispatch sent a completion report.The implementation is correct.
The managed process exited with a particular code.The reported tests actually passed unless Orca ran them itself.
A later exact-session adapter resolved these structured entries.The transcript describes the complete repository state.
The worker reported files, summary, and report path.Those files are exhaustive or the work has been integrated.
+
+
+ +
+
+

Implementation appendix · agents may skip this section

+

Hidden guarantees for one Run home and many worker servers.

+

+ A coordinator on one Orca server must be able to supervise workers on another—for + example, a Run on a Mac with Dispatches on both that Mac and a connected Windows + server. The minimum federation contract routes commands and durably relays messages; + it does not replicate the Run or add scheduling policy. +

+
+ +

+ Agent rule: choose a remote worker once with --on. After that, keep the + Dispatch ID and follow the returned inspection commands. Reuse + mutation.requestId only to recover the same request after a lost response. + Peer identity, sequencing, capabilities, and relay acknowledgments below are Orca + implementation details—not fields agents choose or copy. +

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Internal factV1 contractWhat agents see
Run home + The server where run-create executes owns the only authoritative + Run database: tasks, Dispatch state, inbox order, dedupe receipts, and consumer + generation. + The terminal binding routes ordinary commands home; no Run-home ID is typed.
Connected environment + The Run home stores its existing saved environment ID/name and authenticated + pairing for each worker server. Each Dispatch pins the authenticated peer + fingerprint captured at attachment, so re-pairing the saved environment to a + different server cannot retarget existing work. That relationship survives the + same remote server process restarting; the observed runtimeId + remains only an epoch. + --on windows when placement is explicit; receipts echo the name.
Remote Dispatch attachment + Before prompt injection, the worker server persists a verifier for the opaque + Dispatch capability, pinned Run-home peer identity, stable local pane and + process incarnation, effect receipts, and relay cursors. Credential material + is stored through current-user protected storage, not plaintext in a general + Run row. The worker server does not receive a copy of the Run DAG. + Nothing extra; the worker receives only Task and Dispatch IDs.
Caller pane + Each server's native, WSL, or SSH CLI bridge attaches the minted Dispatch + capability outside user parameters. The worker server verifies capability, + pane, and process incarnation; the Run home verifies the pinned authenticated + peer and Dispatch on relay import. + Orca authenticates lifecycle reports automatically.
+
+ +
+
+

Home-bound coordination

+

+ run-use/current, Task changes, check/ack, and replies go to the Run + home. Workers never choose or mutate the home, and there is no automatic home + failover. +

+
+
+

Server-owned resources

+

+ The selected worker server owns its worktree, terminal, process, and any nested + native/WSL/SSH/relay host. The Run home stores opaque receipts and routes show, + read, stop, and retry back to that owner. +

+
+
+

Durable relay, not replication

+

+ Worker-to-home lifecycle mail and home-to-worker replies remain queued at their + source until the destination imports and acknowledges them. Each item has a stable + ID, so reconnects are at-least-once on the wire and once in each inbox. +

+
+
+

Home-initiated connection

+

+ The Run home uses the same saved, authenticated environment connection already + used for remote RPC. It subscribes or pulls by cursor; the Windows server does not + need a separate pairing back to the Mac or a publicly reachable callback. +

+
+
+

Dispatch generations

+

+ Replacing a worker creates a new Dispatch ID; that ID is the generation. A report + from an older Dispatch or different pane remains history and cannot change current + Task state. There is no second agent-visible generation number. +

+
+
+

Idempotent control operations

+

+ Before effects, both home and worker server durably record authenticated peer, + request ID, canonical payload hash, operation state, and receipt. Identical + concurrent or later attempts join or return that record; a changed payload returns + request_mismatch. Receipts expose the opaque retry ID needed after an + unknown outcome. +

+
+
+

Typed unknown outcome

+

+ If a remote effect may have happened but Orca cannot prove it, return one + outcome_unknown shape with stage, mutation request ID, durable + effects/residuals, and exact inspection commands. +

+
+
+

Stop and replace

+

+ Stop first commits one home-side compare-and-set that fences new lifecycle changes, + blocks the Task, and closes pending questions, then best-effort stops only the + supervised agent process/terminal. If completion already won, stop returns + already_settled; if stop won, later completion is stale history. Orca + never deletes the worktree, setup output, or unrelated configured tabs. +

+
+
+ +

+ The durable request receipt applies to every mutating control-plane call—Run and Task + creation, send, ask, reply, acknowledgment, start, stop, and abandon—not only remote + worker start. Read-only show, list, and output paging do not need mutation dedupe. +

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Federated relay ruleExact contract
Worker send acceptance + Success means the worker server durably stored the authenticated message in + that Dispatch's outbound relay. The worker may finish even while the Run home + is offline. +
Home import and lifecycle + The Run home stores the message and applies any valid lifecycle transition in + one transaction before acknowledging it to the worker server. Until import, + the authoritative Task honestly remains dispatched. +
Replies and control mail + The Run home durably queues them for the exact remote Dispatch; the worker + server stores them before acknowledging the home and waking a local waiter. +
Ordering and duplicates + Each direction uses a scoped key of pinned peer, Dispatch ID, direction, and a + monotonic source sequence, plus a 128-bit-or-stronger message ID. A receiver + imports only the next contiguous sequence, buffers gaps, and acknowledges only + the highest contiguous commit. The home assigns normal inbox order at import, + without pretending simultaneous servers have a global clock. +
Bounded storage + Enforce per-message byte limits and per-Dispatch pending item/byte quotas, + coalesce heartbeats, and reserve space for one terminal lifecycle report. A + full relay returns relay_quota_exceeded; V1 adds no dead-letter or + retention workflow. +
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Remote edgeV1 behavior
Worker server unavailable before send + Return remote_runtime_unavailable before creating a Dispatch or any + remote effect. No remote Dispatch attachment exists, so the caller may retry + normally after reconnecting. +
Connection lost after send + Return outcome_unknown, mutation.requestId, last durable + stage, and exact inspection commands. Repeating the request ID returns the + worker server's original receipt or accepts it once; it never duplicates an + effect. +
Run-home restart + Run state, dedupe receipts, and unacknowledged mail survive. The process runtime + ID is only an epoch. Reconnect resumes relay cursors for every active remote + Dispatch; if the coordinator pane cannot be safely reminted, + run-use explicitly rebinds it. +
Saved environment re-paired or removed + If its authenticated peer fingerprint differs from the Dispatch attachment, + return peer_changed with no effect; never adopt the replacement + server. Removing an environment with a nonterminal Dispatch retains a routing + tombstone for inspection and abandon rather than erasing ownership evidence. +
Servers disconnected after start + The worker may continue. Its server retains lifecycle mail and questions while + replies remain queued at the home. Silence is not failure, no worker is + automatically replaced, and other servers continue delivering normally. +
Worker-server restart + Remote Dispatch attachments, effect receipts, and unacknowledged relay items + survive. The process runtime ID may change; the Run home routes by its saved + environment relationship and pinned peer. worker-show reports + running only when the stable pane and process incarnation match; it never + adopts a same-looking pane or newly launched process. +
Mixed server versions + Before effects, both servers must advertise one aggregate + orchestrationFederationV1 contract. Missing support returns + capability_unsupported; + it never silently degrades to prompt injection, terminal scraping, or + consume-on-read mail. The worker-side mutation revalidates the pinned peer and + advertised protocol recorded for the operation, closing the probe-to-effect + race. +
Structured worker output + The worker-owning server reads an exact hook-reported Codex, Claude, + OpenClaude, or Grok transcript when supported. If the additive federated read + method is absent, auto returns bounded terminal output labeled with + remote_capability_unavailable; + transcript returns transcript_required. +
+
+ +

+ Capability checks cover only the new federation contracts, not every platform or host + feature. Existing worktree, setup, terminal, Git, WSL, SSH, and relay primitives keep + their proven compatibility behavior; optional, truthfully labeled observation may + degrade. +

+ +

+ In this proposal, “durable” means committed state survives a client or Orca process + crash and ordinary restart. Power-loss durability depends on the receiving database's + documented SQLite synchronization policy; lifecycle import and acknowledgment must use + the stronger policy if Orca promises survival across sudden host power loss. +

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + +
MechanismWhy it staysWhy it is not a scheduler
Server ownershipPrevents commands from acting on a same-looking resource on the wrong host.The agent chooses --on; the owner only determines routing.
Dispatch identityPrevents stale workers from overwriting current task state.It does not retry, replace, or start anything automatically.
Unknown outcomePrevents duplicate remote effects after a disconnect. + The receipt gives last durable stage and inspection command; the coordinator + chooses inspect, reconcile, retry, or abandon. +
+
+ +

+ Safety may reject stale, wrong-pane, or wrong-server control-plane mutations. It does + not police worker filesystem access, invent new work, choose a worker, or decide that + waiting is the coordinator's best next action. +

+ +

+ This federation is deliberately hub-and-spoke per Run: one home, explicit worker + servers, and per-Dispatch relay queues. It does not create a global cluster, replicate + Run state, elect leaders, fail over the home, schedule placement, or require agents to + carry server IDs after start. +

+
+ +
+
+

Agent ergonomics

+

The skill should be a cookbook, not a second help page.

+

+ orca --help already owns syntax and exhaustive flags. The skill should + teach judgment: which topology to choose, how to preserve parallelism, what the + command returns, and what not to create afterward. +

+
+ +
+
+

Parallel fan-out

+

Start every independent task first. Only then call check --wait.

+
+
+

Shared-worktree review

+

Use the current worktree when sharing its exact state is useful.

+
+
+

Independent writers

+

Create separate worktrees when a concrete checkout conflict calls for isolation.

+
+
+

Ask and reply

+

Use a threaded question; continue other work while only that worker is blocked.

+
+
+

Read a full-screen agent

+

Use worker-read --dispatch; Orca routes to the owning server.

+
+
+

Replace safely

+

Inspect the stop result before deciding where to start a replacement.

+
+
+ +
+
+

Move out of the skill

+
    +
  • Complete command and flag catalogs
    • +
  • Internal database and delivery terminology
    • +
  • Provider-specific internals before the common path
    • +
  • Large decision trees for features Orca does not implement
    • +
+
+
+

Every recipe must say

+
    +
  • When the pattern is appropriate
    • +
  • What the command creates or reuses
    • +
  • Whether setup runs; if not, the concrete reason
    • +
  • How and when results return
    • +
  • The common misuse to avoid
    • +
+
+
+ +
+ Current Orca · parallel workers in the current worktree +
+

+ Creates two fresh agent terminals; setup does not run. Create both tasks and both + terminals before waiting for readiness, then dispatch both before blocking. The + coordinator keeps checking until both expected Dispatches settle; one batch is not + assumed to contain both completions. Misuse: waiting for worker A before starting B. +

+ 
orca orchestration task-create --spec "Audit message semantics" --json
+orca orchestration task-create --spec "Audit transcript adapters" --json
+orca terminal create --worktree active --title message-audit --command codex --json
+orca terminal create --worktree active --title transcript-audit --command codex --json
+orca terminal wait --terminal term_a --for tui-idle --timeout-ms 60000 --json
+orca terminal wait --terminal term_b --for tui-idle --timeout-ms 60000 --json
+orca orchestration dispatch --task task_a --to term_a --inject --json
+orca orchestration dispatch --task task_b --to term_b --inject --json
+
+# Repeat until task_a and task_b are both settled; process every returned message.
+orca orchestration check --wait --timeout-ms 60000 --json
+
+
+ +
+ Proposed Orca · same fan-out with composed worker start +
+

+ Creates one bound Run and two fresh terminals in the existing worktree. Setup is + not_applicable. Each start returns ready, failed, or + outcome_unknown; independent calls may be issued in parallel. Worker + results arrive through the inbox. Misuse: creating another terminal after start. +

+ 
orca orchestration run-create --objective "Improve orchestration ergonomics" --json
+orca orchestration task-create --spec "Audit message semantics" --json
+orca orchestration task-create --spec "Audit transcript adapters" --json
+ +
+ 
# Concurrent tool call A
+orca orchestration worker-start --task task_a --worktree current --agent codex --json
+ 
# Concurrent tool call B
+orca orchestration worker-start --task task_b --worktree current --agent codex --json
+
+ + 
# After both start calls return, loop; do not assume two waits or one batch is enough.
+# Repeat process → ack → wait until both Dispatches are settled.
+orca orchestration check --wait --timeout-ms 60000 --json
+orca orchestration check --ack delivery_81 --wait --timeout-ms 60000 --json
+
+
+ +
+ Mac coordinator + Windows worker · one Run across connected servers +
+

+ The Run and inbox stay on the Mac. The first worker uses the Mac's current + worktree; the second creates a top-level worktree on the saved + windows Orca environment. Both report into the same inbox, including + after a temporary disconnect. Misuse: creating a second Run on Windows or asking + the worker to carry server/Run-home IDs. +

+ 
# On the Mac coordinator
+orca orchestration run-create --objective "Audit both platforms" --json
+orca orchestration task-create --spec "Audit macOS behavior" --json
+orca orchestration task-create --spec "Audit Windows behavior" --json
+
+# Read-only discovery happens against Windows; copy the opaque repo/worktree IDs returned.
+orca --environment windows worktree list --json
+
+# Issue these as independent concurrent tool calls, not as one sequential shell script.
+orca orchestration worker-start --task task_mac --worktree current --agent codex --json
+orca orchestration worker-start --task task_windows --on windows --worktree new-top-level --repo id:<windows-repo-id> --name windows-audit --agent codex --setup run --json
+
+# One home wait fans in local and Windows messages. Repeat until both Dispatches settle.
+orca orchestration check --wait --timeout-ms 60000 --json
+orca orchestration check --ack delivery_81 --wait --timeout-ms 60000 --json
+
+# Attempt-specific guidance uses the stable Dispatch, never the remote terminal handle.
+orca orchestration send --to dispatch:dispatch_windows --subject "Follow-up" \
+  --body "Run the additional Windows-only check." --json
+
+
+ +
+ Worker completion · success and failure are explicit +
+

+ The injected preamble supplies the only Task and Dispatch IDs a worker copies. + Orca supplies its Dispatch capability automatically; the worker never supplies Run, + runtime, host, terminal, capability, or attestation IDs. Misuse: reporting failure as a + successful completion or inventing IDs from terminal history. +

+ 
# Success
+orca orchestration send --type worker_done --subject "Review complete" \
+  --body "Audited the requested behavior. Found two issues and changed no files. Nothing remains." \
+  --task-id "<taskId from current preamble>" \
+  --dispatch-id "<dispatchId from current preamble>" \
+  --outcome succeeded --json
+
+# Failure
+orca orchestration send --type worker_done --subject "Review failed" \
+  --body "Could not read the required fixture. No findings are reliable. The fixture must be restored before retrying." \
+  --task-id "<taskId from current preamble>" \
+  --dispatch-id "<dispatchId from current preamble>" \
+  --outcome failed --json
+
+
+ +
+ Worker question · ask defaults to its Run +
+

+ An active worker asks its owning coordinator without carrying a Run ID. The + coordinator replies to the returned message ID while other work continues. Misuse: + creating a task gate or a second Question ID for a simple reply. +

+ 
# Worker
+orca orchestration ask --question "Should I preserve the legacy format?" --json
+
+# If acceptance may have happened but the response was lost, repeat the identical command with
+# the orchestrationRequestId reported by the CLI as --retry-request.
+orca orchestration ask --question "Should I preserve the legacy format?" --retry-request req_ask7 --json
+
+# If a server disconnect timed out the wait, resume with the returned message ID.
+orca orchestration ask --resume msg_question_7 --json
+
+# Coordinator, after receiving msg_question_7
+orca orchestration reply --id msg_question_7 --body "Yes; preserve it." --json
+
+
+ +
+ New child or top-level worktree · reuse the returned agent terminal +
+

+ New worktrees run configured setup by default. Agent-first creation returns the + only worker terminal; configured extra tabs remain intentional. Use + skip or inherit only for a concrete stated reason. Misuse: + adding a second agent terminal because the startup receipt was not inspected. +

+ 
orca orchestration worker-start --task task_a --worktree new-child --name message-audit --agent codex --setup run --json
+orca orchestration worker-show --dispatch dispatch_7 --json
+
+orca orchestration worker-start --task task_b --worktree new-top-level --name transcript-audit --agent claude --setup run --json
+
+# Escape hatch: this task audits the pristine fixture, and setup would mutate that fixture.
+orca orchestration worker-start --task task_fixture --worktree new-top-level --name fixture-only --agent codex --setup skip --json
+
+
+ +
+ Intentional existing-terminal reuse +
+

+ Reuses exactly the selected terminal and runs no setup. The terminal must belong to + the chosen worktree. Misuse: treating “an idle terminal somewhere” as equivalent. +

+ 
# Local existing worktree; copy the exact opaque IDs from worktree/terminal list.
+orca orchestration worker-start --task task_a --worktree 'id:<full-worktree-id>' --terminal term_12 --json
+
+# Remote existing worktree; --on is mandatory because V1 IDs are server-scoped.
+orca --environment windows worktree list --json
+orca orchestration worker-start --task task_b --on windows --worktree 'id:<full-windows-worktree-id>' --agent codex --json
+
+
+ +
+ Remote uncertainty · branch on what inspection proves +
+

+ An unknown start or stop is inspected by Dispatch ID; nothing is replayed merely + because a connection failed. Misuse: treating show → stop → abandon → retry as an + unconditional recovery sequence. +

+ 
# First recover the receipt for the exact request; this cannot create a second effect.
+orca orchestration worker-start --task task_a --worktree new-child --name message-audit --agent codex --setup run --retry-request req_7 --json
+
+# Then inspect the Dispatch if the outcome is still unknown.
+orca orchestration worker-show --dispatch dispatch_7 --json
+
+# If ready: keep the worker and wait for its result.
+# If failed or stopped: start an explicit replacement, repeating the intended placement.
+orca orchestration worker-start --task task_a --retry-of dispatch_7 --worktree current --agent codex --json
+
+# If still unknown: stop and inspect again, or explicitly accept the warning and abandon.
+orca orchestration worker-stop --dispatch dispatch_7 --json
+orca orchestration worker-show --dispatch dispatch_7 --json
+orca orchestration worker-abandon --dispatch dispatch_7 --json
+
+
+ +
+ Restart recovery · home and worker server are independent +
+

+ A Run-home restart preserves the Run, retry receipts, and the same unacknowledged + Delivery; run-use is needed only when Orca cannot safely remint the + previous coordinator pane. A worker-server restart preserves its Dispatch + attachment and relay queue but does not pretend the agent process survived. + Misuse: starting a replacement merely because a runtime ID changed. +

+ 
# Home restart: rebind only if run-current says this pane is unbound.
+orca orchestration run-current --json
+orca orchestration run-use --id run_123 --json
+orca orchestration check --wait --timeout-ms 60000 --json
+# → returns the same outstanding delivery_81 until it is acknowledged
+
+# Worker-server restart: inspect the persisted attachment and exact process incarnation.
+orca orchestration worker-show --dispatch dispatch_7 --json
+# running/ready → keep waiting; gone/failed → explicit retry-of; unknown → inspect or stop/abandon.
+
+
+ +

+ The skill must say that check --wait returns a batch. Process every message + before acknowledging it. +

+ +
+

Scenario tests matter more than keyword checks

+
+
    +
  • Reuse the terminal returned by worktree creation.
    • +
  • Start three independent workers before waiting.
    • +
  • Omit runId only when the coordinator terminal is explicitly bound.
    • +
  • Use current-worktree collaborators without unnecessary worktrees.
    • +
  • Fence an old consumer's acknowledgment after run-use rebinds.
    • +
  • Import duplicated and out-of-order relay frames only in contiguous order.
    • +
+
    +
  • Never treat a worker report as verified integration.
    • +
  • Never replay terminal input after unknown acceptance.
    • +
  • Page terminal output by Dispatch; use a session source only when exact.
    • +
  • Inspect an ambiguous remote stop before choosing the next action.
    • +
  • Let the first committed stop/completion transition win transactionally.
    • +
  • Reject a re-paired peer and never adopt a same-looking restarted process.
    • +
+
+
+
+ +
+
+

Complexity audit

+

Keep only the machinery required by Orca's concrete failure modes.

+

+ Every retained primitive below addresses a failure Orca can reproduce today. Broader + policy and product layers remain out of scope until a simpler primitive proves + insufficient in real use. +

+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Observed Orca needKeepDeliberately defer
Lifecycle messages can be lost, injected into prompts, or consumed before a caller receives them.Durable typed mail, explicit acknowledgment, and race-free blocking waits.Dead-letter workflows, priority schedulers, watchdog policy, or a second queue product.
Terminal scrollback is not always the best available source, but session identity can be ambiguous.Terminal output as the baseline and an optional exact, source-pinned adapter.A universal session ontology, resume layer, or global provider exclusivity.
Remote acceptance, process exit, and worker claims provide different levels of evidence.Explicit lifecycle, fenced replacement, and typed unknown-outcome handling.Automatic retry, inferred success, rollback, or generalized recovery policy.
A Run home must coordinate workers across restarts and connected Orca servers.Stable Run, Task, and Dispatch identity with authenticated server relay.Role simulation, worker scoring, organization models, or integration queues.
+
+ +
+
+

Tests prove the contract

+

+ Scenario tests must demonstrate durable messages, authenticated relay, lifecycle + fencing, and truthful recovery. Design analogy is never a substitute for an + Orca-local executable contract. +

+
+
+

Complexity requires local evidence

+

+ A generalized scheduler, fairness policy, dead-letter workflow, or integration + system should be proposed only after Orca users demonstrate that the simpler + primitives cannot solve a recurring problem. +

+
+
+ +

+ Blocking wait, crash-safe acknowledgment, and connected-server relay require + Orca-local implementation and tests. Structured output reuses Orca's exact + pane/process-to-provider-session association and existing bounded transcript decoders; + it does not introduce provider-session control or a universal transcript model. +

+
+ +
+
+

Implementation order

+

Each phase should remove one concrete source of agent confusion.

+

+ There is no UI phase. Each runtime change ships with a version-matched example and a + misuse test so the skill and behavior cannot drift apart. +

+
+ +
+
+
Phase 0
+
+

Rewrite the orchestration skill as recipes

+

+ Teach correct fan-out, current versus new worktree selection, startup terminal + reuse, --setup run for new worktrees unless the agent states a + concrete reason to skip or inherit, while preserving the existing + start-immediately default, current batch behavior, and blocking only + after useful parallel work is exhausted. Correct the one-message claim and + document today's delivery limits. + Rename or remove the existing scheduler-like orchestration run --spec + command before the lightweight run-* vocabulary can ship. +

+
+
+
+
Phase 1
+
+

Run mailbox, truthful completion, and crash-safe consumption

+

+ Add explicit Run-home binding, stable logical recipients, strict + inbox-only structured mail, succeeded/failed worker outcomes, one outstanding + FIFO batch, explicit acknowledgment, typed timeout results, ask/reply state, + a narrow runtime-minted Dispatch capability carried by the CLI bridge, and + resume-by-message-ID after a disconnected ask. Migrate existing global rows to + one unbound inspect-only legacy Run; do not infer bindings. Do not add separate + Question IDs or expiry policy. +

+
+
+
+
Phase 2
+
+

Local synchronous worker start and control

+

+ On the Run home, compose existing worktree, setup, terminal, and dispatch + primitives with setup-run as the new-worktree default, a durable request/stage + receipt, side-by-side setup/agent startup by default, startup-terminal reuse, + exact readiness, and Dispatch-routed + show/read/stop/abandon. Prove current, existing, + child, top-level, failure, restart, and unknown-outcome behavior before adding a + network boundary. Do not add a background provisioning executor. +

+
+
+
+
Phase 3
+
+

Connected-server Dispatch and relay

+

+ Extend the same primitives with saved-environment placement, pinned peer identity, + remote Dispatch attachments, paired-server calls, and bounded bidirectional relay + with contiguous cursors and idempotent acknowledgment. Validate Mac-home/Windows- + worker and Windows-home/Mac-worker completion, question/reply, read, stop, + either-side restart, re-pairing, disconnect, and mixed versions—without Run + replication, failover, or scheduling. +

+
+
+
+
Phase 4
+
+

Exact structured worker output

+

+ Reuse Orca's existing pane-scoped hook association and bounded + Codex/Claude/OpenClaude/Grok transcript decoders. Add + auto|transcript|terminal selection, + path-free source identity, opaque source-pinned paging, and labeled terminal + fallback without adding provider control or a universal transcript layer. +

+
+
+
+ +
+
+

Success looks like

+
    +
  • Agents start all independent work before waiting.
    • +
  • Lifecycle messages never flood editable coordinator input.
    • +
  • One Run can supervise Mac and Windows workers through one home inbox.
    • +
  • Remote completion and replies survive either server temporarily disconnecting.
    • +
  • No delivery is consumed before explicit acknowledgment.
    • +
  • Failed worker reports set failed—not completed—and remain worker assertions.
    • +
  • Worker start never reports a resource created before it exists.
    • +
  • Full-screen agent output remains readable when an adapter supports it.
    • +
  • Stale, wrong-pane, or wrong-server workers cannot mutate current task state.
    • +
+
+
+

Complexity budget

+
    +
  • No new concept without a common recipe that needs it.
    • +
  • No automatic action whose trigger an agent cannot explain.
    • +
  • No provider field that an adapter cannot actually observe.
    • +
  • No hidden default omitted from the operation receipt.
    • +
  • No control-plane fact derived from untrusted transcript prose.
    • +
  • No future feature included merely to keep the architecture open-ended.
    • +
+
+
+
+ +
+
+

Explicit boundaries

+

What this proposal intentionally does not build.

+

+ These are not hidden later phases. They require separate evidence and a separate + proposal if Orca eventually needs them. +

+
+ +
+
+

No product UI

+
    +
  • No dashboard or run/task view
    • +
  • No global inbox, badges, or queue screen
    • +
  • No coordinator chat surface
    • +
  • No task DAG visualization
    • +
  • No changes to existing Orca UI behavior
    • +
+
+
+

No scheduler

+
    +
  • No automatic task dispatch or placement
    • +
  • No capacity vectors or resource classes
    • +
  • No fairness, priority aging, or global queue
    • +
  • No pause, resume, or drain controls
    • +
  • No automatic retry based on silence
    • +
+
+
+

No integration subsystem

+
    +
  • No commit or branch tracking
    • +
  • No automatic merge or landing
    • +
  • No target-ref locking
    • +
  • No independent verification of worker claims
    • +
  • No cross-run work lineage model
    • +
+
+
+

No speculative framework

+
    +
  • No organization charts, roles, or worker profiles
    • +
  • No universal provider transcript schema
    • +
  • No dead-letter or poison-message workflow
    • +
  • No generalized continuation/checkpoint protocol
    • +
  • No project hierarchy above lightweight runs
    • +
  • No replicated Run database, leader election, or automatic home failover
    • +
+
+
+ +
+ Could Orca add these things later? +
+ Yes, but strong primitives do not need speculative abstractions for them today. A + future feature should compose the same start, message, wait, read, stop, identity, + and ownership contracts. It should justify its own concepts from observed Orca use. +
+
+ +
+ Does removing commit tracking make worker results less trustworthy? +
+ It makes the contract more honest. Today Orca verifies who is authorized to report a + result, not that every claim inside the result is true. A coordinator may explicitly + ask another worker to review or run validation. That is agent-directed orchestration, + not an implicit integration subsystem. +
+
+ +
+ What about connecting multiple Orca runtime servers? +
+ It is a core requirement. A Run stays authoritative on one home server while remote + Dispatches execute on saved connected environments such as a Windows machine. + Authenticated routing plus a small durable relay carries lifecycle mail and replies + across disconnects. This intentionally stops short of a global cluster: no Run + replication, leader election, automatic failover, distributed scheduler, or lease + manager is required. +
+
+ +

+ Phases 0–4 of this contract are implemented. Structured output is verified locally, + through a mixed-version terminal fallback, and across a Windows-home to Mac-worker + Dispatch; the remaining physical symmetry checks stay in the acceptance ledger. The + proposal makes no Orca product UI changes. +

+
+
+
+
+ + + + diff --git a/skill-guides/orchestration.md b/skill-guides/orchestration.md index c7d15250e00..c1ac7d5d8f7 100644 --- a/skill-guides/orchestration.md +++ b/skill-guides/orchestration.md @@ -22,7 +22,7 @@ Use this skill when coordination state matters. For lightweight terminal prompts ## Tool Boundary -If a task says to use Orca orchestration, the coordinator must create Orca runtime state with `orca orchestration task-create` and `orca orchestration dispatch --inject` or `orca orchestration run`. +If a task says to use Orca orchestration, the coordinator must create or bind a Run, create the Task with `orca orchestration task-create`, then attach the worker with either the preferred `orca orchestration worker-start` composition or the low-level `orca orchestration dispatch --inject` path. Do not substitute non-Orca subagent tools, generic agent-spawn APIs, or chat-only parallel worker features. Those may create useful workers, but they do not create Orca task/dispatch provenance, injected lifecycle preambles, `worker_done` authority, or decision gates. @@ -51,9 +51,35 @@ Do not use orchestration merely because the user says "hand off", "handoff", "ha - The orchestration experimental feature must be enabled in Settings > Experimental. - `orca orchestration` commands are RPC calls to the running Orca runtime. +## Contract Migration + +Orca uses a hard cutover for orchestration mutations. It does not run a legacy scheduler, translate old writes, or drain pre-upgrade orchestration work. + +If a command returns `orchestration_migration_required`, `run_required`, or a lifecycle validation error with `nextCommandArgs`: + +1. Confirm `effectsApplied` is `false`. +2. Using the same CLI executable that returned the error, run the returned arguments: `skills get orchestration --full`. +3. Read the guide completely. Do not retry the rejected command unchanged. +4. Create or bind a lightweight Run, then restart the work using Run -> Task -> `worker-start`. +5. Inspect any pre-upgrade terminal before creating replacement work. + +The arguments intentionally omit an executable name so this works with `orca`, `orca-ide`, `orca-dev`, or another configured Orca CLI command. + +Pre-upgrade terminals and agents are not killed during upgrade, but they are no longer supervised: old heartbeat, question, completion, scheduler, and mutation calls are rejected before effects. Legacy database rows remain available only for explicit inspection: + +```bash +orca orchestration run-list --json +orca orchestration run-show --id run_legacy_local --json +orca orchestration task-list --run run_legacy_local --json +orca orchestration inbox --full --json +orca orchestration check --terminal --peek --json +``` + +Read-only inspection never consumes legacy mail. Do not use actionable `check`, acknowledgment, send, retry, or task updates against the legacy Run. + ## Ownership -Orchestration messages and tasks are runtime-global. Lifecycle authority comes from the payload `taskId` + `dispatchId` of the active dispatch, verified against the dispatched pane. Terminal handles are routing metadata — a pane can receive a new handle after restart — so never accept or reject lifecycle provenance by comparing handles. Send `worker_done` and `heartbeat` from the worker's own terminal; the runtime ignores them when sent from a different pane. +New orchestration messages and tasks belong to one explicitly bound Run. A Run is only a durable namespace and coordinator inbox; it never schedules or places workers. Lifecycle authority comes from the active Dispatch, and terminal handles remain routing metadata rather than durable identity. Send `worker_done` and `heartbeat` from the worker's own terminal; Orca routes them to that Dispatch's Run. Classify inherited context before sending lifecycle messages: @@ -78,36 +104,39 @@ orca orchestration dispatch-show --task --json ## Messaging ```bash -orca orchestration send --to --subject [--from ] [--body ] [--type ] [--priority ] [--thread-id ] [--payload ] [--json] -orca orchestration check [--terminal ] [--unread|--peek|--all] [--types ] [--inject] [--wait] [--timeout-ms ] [--json] +orca orchestration send --subject [--to ] [--from ] [--body ] [--type ] [--priority ] [--thread-id ] [--payload ] [--json] +orca orchestration check [--terminal ] [--ack ] [--peek|--all] [--types ] [--format] [--wait] [--timeout-ms ] [--json] orca orchestration reply --id --body [--from ] [--json] -orca orchestration ask --to --question [--options ] [--timeout-ms ] [--from ] [--json] +orca orchestration ask (--question |--resume ) [--options ] [--timeout-ms ] [--from ] [--json] orca orchestration inbox [--limit ] [--json] ``` Rules: - Omit `--from` unless impersonating another terminal; Orca auto-resolves it from the current terminal. -- `check` and `check --unread` return unread matches and mark them read. Use `--peek` for unread matches without consuming them; use `--all` for read and unread history without consuming anything. If an older CLI rejects `--peek` as an unknown flag, use `--all` and filter unread rows yourself. -- Message **one** live agent handle per worker. Use `startupTerminal.handle` from the create response when present; if it is missing or later returns `terminal_handle_stale`, re-resolve with `orca terminal list --worktree ... --json` and continue with the replacement only. -- `orca orchestration check --unread --inject --json` renders unread mail for the agent terminal that runs it; it does not remotely wake another terminal. Use `orchestration dispatch --inject` to deliver a tracked task, or `terminal send` when an existing agent needs a free-form prompt. -- While supervising workers manually, use `check --wait --types worker_done,escalation,decision_gate --timeout-ms ` instead of sleep/poll loops. Reply to `decision_gate` messages with `orca orchestration reply --id --body --json`, then keep waiting. +- A coordinator `check` returns the bound Run's oldest FIFO Delivery (up to 50 messages) and replays that exact batch until `--ack `. Process every message before acknowledging; `check --ack --wait` acknowledges, checks, and waits in one operation. +- Use `--peek` and `--all` only for read-only history/debugging. Type filters decide when a waiter wakes; the returned actionable Delivery is still the oldest full batch. +- Use `dispatch:` for coordinator guidance to one supervised worker. Orca routes that stable address locally or through the connected-server relay; do not substitute a remote terminal handle. +- Terminal handles remain appropriate for low-level pre-Dispatch messaging. Prefer `agentTerminalHandle` from the create response, fall back to `startupTerminal.handle` for older runtimes, then re-resolve with `orca terminal list --worktree ... --json` if missing or stale. Continue with the replacement handle only; never dual-send to old and new handles. +- `orca orchestration check --peek --format --json` returns locally formatted unread mail without consuming it; it never writes to terminal input or remotely wakes another terminal. Use `orchestration dispatch --inject` to deliver a tracked task, or `terminal send` when an existing agent needs a free-form prompt. +- While supervising workers manually, use `check --wait --types worker_done,escalation,question --timeout-ms ` instead of sleep/poll loops. Process the whole Delivery, reply to `question` messages with `orca orchestration reply --id --body --json`, then acknowledge and keep waiting. - Treat a `check --wait` timeout or `{count:0}` as a checkpoint, not a worker failure. Long coding tasks routinely run 15-60 minutes; keep using rolling waits unless you receive `worker_done`/`escalation`, the terminal exits or disappears, or the user explicitly asks you to stop. - Heartbeats and visible terminal activity mean the worker is alive, not done. Do not stop, close, kill, or restart a worker just because it has not produced a completion message yet. -- Use `ask` when a worker needs a blocking answer from the coordinator; it waits for the reply and returns the answer directly. -- `check --wait` returns one message at a time. If N workers may finish together, loop N times and dispatch newly ready tasks after each completion. +- Use `ask` when a worker needs a blocking answer from the coordinator; it defaults to the active Dispatch's Run. Timeout or disconnect leaves the question pending, so resume by its original message ID instead of asking again. +- `check --wait` returns one bounded Delivery, not every future completion. Process every message, acknowledge it, then keep waiting until every expected Dispatch settles. - Group addresses include `@all`, `@idle`, `@claude`, `@codex`, `@opencode`, `@gemini`, `@droid`, `@grok`, `@cursor`, and `@worktree:`. -- Message types include `status`, `dispatch`, `worker_done`, `merge_ready`, `escalation`, `handoff`, `decision_gate`, and `heartbeat`. +- Message types include `status`, `dispatch`, `worker_done`, `merge_ready`, `escalation`, `handoff`, `question`, `decision_gate` (legacy/gates), and `heartbeat`. - Use group addresses only for messages that are genuinely useful to many terminals, such as `status` broadcasts or intentional fan-out questions. Do not send dispatch lifecycle messages to groups. -- `worker_done` must target the concrete coordinator handle from the live preamble. It is completion authority for one dispatch; group fanout would create false lifecycle mail in unrelated terminals. +- `worker_done` belongs to the active Dispatch and defaults to its Run mailbox; never target a group. - A valid `worker_done` for the active `taskId` + `dispatchId` marks the task and dispatch completed automatically. Do not follow it with `task-update --status completed`; reserve manual updates for explicit recovery or overrides. -- `heartbeat` is also dispatch-scoped. Send it only to the concrete coordinator handle with both `taskId` and `dispatchId`; use `status` for broad progress updates. +- `heartbeat` is also Dispatch-scoped. Include both IDs and omit `--to` so Orca uses the owning Run; use `status` for broad progress updates. ## Tasks And Dispatch -A task is the work item, a dispatch assigns it to a terminal, and a gate blocks progress until a coordinator or user decision is recorded. +A Run is the namespace/inbox, a Task is the work item, and a Dispatch assigns one Task attempt to a terminal. Create or bind a Run once before the common loop. ```bash +orca orchestration run-create --objective --json orca orchestration task-create --spec [--deps ] [--parent ] [--json] orca orchestration task-list [--status ] [--ready] [--brief] [--json] orca orchestration task-update --id --status [--result ] [--json] @@ -124,19 +153,97 @@ Dispatch rules: - After 3 consecutive failures on one task, the dispatch context circuit-breaks and the task is marked failed. - Use `task-list --brief --json` for coordinator sweeps; it collapses whitespace and caps each echoed spec at 160 characters (`spec_truncated` marks shortened rows). Omit `--brief` when the full spec is required, or when an older CLI rejects it as an unknown flag. -## Gates And Coordinator +## Preferred Supervised Worker Loop + +Use `worker-start` for the normal supervised path. It composes the existing worktree, terminal, readiness, and dispatch primitives while returning exact created/reused effects. Agents still choose placement and concurrency; Orca does not schedule workers or infer conflicts. + +Create the Run and every independent Task first, then start all independent workers before waiting: + +```bash +orca orchestration run-create --objective "" --json +orca orchestration task-create --spec "" --json +orca orchestration task-create --spec "" --json +orca orchestration worker-start --task --worktree current --agent codex --json +orca orchestration worker-start --task --worktree current --agent claude --json +``` + +`current` and exact existing worktrees create a fresh agent terminal and do not rerun setup. Reuse an existing agent only with `--terminal `. + +For a new worktree, setup runs by default and agent-first creation reuses the returned startup agent terminal: + +```bash +orca orchestration worker-start --task --worktree new-child --name --agent codex --setup run --json +# Independent/top-level: +orca orchestration worker-start --task --worktree new-top-level --name --agent codex --setup run --json +``` + +Setup normally starts alongside the agent. Only a repository explicitly configured with `wait-for-setup` delays agent launch until setup succeeds. Use `--setup skip` or `--setup inherit` only for a concrete reason. + +Read the returned receipt before continuing: `ready` plus setup `running` is normal for start-immediately, while wait-for-setup returns setup `succeeded` before accepting task input. A failed or unknown start exits nonzero; inspect its `stage`, `effects`, and `residualResources` instead of guessing or automatically retrying. A wait-for-setup timeout can honestly leave setup `running`, which is not proof of failure. + +To run the worker on another connected Orca server, add `--on `. The Run and Tasks remain authoritative on the current server; later commands route by Dispatch ID, so never repeat `--on`: + +```bash +# Mac Run home -> Windows worker (the reverse is identical from a Windows Run home) +orca orchestration worker-start --task --on windows --worktree new-top-level --repo --name --agent codex --setup run --json +orca orchestration worker-show --dispatch --json +orca orchestration worker-read --dispatch --limit 50 --json +orca orchestration send --to dispatch: --subject "Follow-up" --body "" --json +``` + +Remote `current` and `new-child` are intentionally invalid because those words are ambiguous across servers. Use an exact discovered remote worktree selector or `new-top-level` with an explicit remote repo selector. + +The follow-up is structured inbox mail, not prompt injection. The worker's next +`orchestration check` receives it even when the Dispatch is on another connected Orca server. + +`worker-read` defaults to `--source auto`: Orca returns the exact hook-reported Codex, Claude, OpenClaude, or Grok transcript when it can prove the worker session, otherwise it returns bounded terminal output with `source: "terminal"` and a typed `fallbackReason`. Continue with the returned top-level `cursor`; it stays pinned to that exact source. If Orca reports `source_changed`, start a fresh read without the old cursor. Never supply or guess a provider session ID or transcript path. + +Wait until every expected Dispatch settles, not for a fixed number of batches: + +```bash +orca orchestration check --wait --types worker_done,escalation,question --timeout-ms 900000 --json +# Process every message in the returned Delivery, then atomically ack and continue: +orca orchestration check --ack --wait --types worker_done,escalation,question --timeout-ms 900000 --json +``` + +Workers report exactly once using the IDs and capability injected by Orca; they do not supply Run/server/terminal identity: + +```bash +orca orchestration send --type worker_done --subject "" --body "" --task-id --dispatch-id --outcome succeeded --files-modified "path/a,path/b" --json +# On failure, use --outcome failed; never encode failure only in prose. +``` + +A worker question defaults to its owning Run. Timeout leaves it pending: + +```bash +orca orchestration ask --question "" --options "yes,no" --timeout-ms 600000 --json +orca orchestration ask --resume --timeout-ms 600000 --json +# Coordinator: +orca orchestration reply --id --body "" --json +``` + +Recovery is conditional, never a fixed destructive sequence: + +- `worker-show --dispatch ` says `ready`: keep waiting or read bounded output. +- It proves `failed` or `stopped`: start a replacement with `worker-start --task --retry-of ` plus an explicit `--on`/`--worktree` and `--agent`/`--terminal` choice. Retry does not silently inherit placement. +- It remains `outcome_unknown`: either `worker-stop --dispatch ` and inspect again, or explicitly `worker-abandon --dispatch ` while accepting that resources may still be live. Abandon performs no remote, process, or filesystem action. +- `worker-stop` closes only the exact supervised agent terminal. It never deletes the worktree, setup terminal, configured tabs, or unrelated processes. + +Low-level `worktree create`, `terminal create`, and `dispatch --inject` remain valid recipes for custom argv or topology that `worker-start` does not express. + +## Gates And Legacy Inspection ```bash orca orchestration gate-create --task --question [--options ] [--json] orca orchestration gate-resolve --id --resolution [--json] orca orchestration gate-list [--task ] [--status ] [--json] -orca orchestration run --spec [--from ] [--poll-interval-ms ] [--max-concurrent ] [--worktree ] [--json] -orca orchestration run-stop [--json] ``` -`run` returns immediately with a run ID. Query progress with `task-list`. Use `ask` for worker-to-coordinator questions; it creates a `decision_gate` message that the coordinator answers with `reply`. Use `gate-create` only for coordinator-managed task DAG decisions, not for answering a worker's `ask`. +Use `ask` for worker-to-coordinator questions; it creates a `question` message that the coordinator answers with `reply`. Use `gate-create` only for coordinator-managed task DAG decisions, not for answering a worker's `ask`. -Recovery only: `orca orchestration reset --tasks|--messages|--all --json` clears runtime-global orchestration state. Do not run it during active coordination unless explicitly abandoning that state. +`coordinator-start`, `coordinator-stop`, `run`, and `run-stop` are retired scheduler commands. They perform no effects and return the current-skill recovery action. They are not aliases for lightweight Run creation or binding. + +Recovery only: `orca orchestration reset --tasks|--messages|--all --json` clears the selected local orchestration database state. Do not run it during active coordination unless explicitly abandoning that state. ## Full Handoffs @@ -151,7 +258,7 @@ Do not run `orca orchestration task-create`, `orca orchestration dispatch --inje New top-level worktree handoff: ```bash -orca worktree create --name --no-parent --agent codex --prompt "" --json +orca worktree create --name --no-parent --agent codex --prompt "" --setup run --json ``` Before creating a new worktree from an active feature branch, decide and state whether the desired Orca lineage is child or top-level. Use child worktree lineage only when the new work is conceptually stacked under or dependent on the active worktree. For independent repo-wide fixes, standalone feature work, or unrelated follow-up tasks, create a top-level worktree with `--no-parent`. @@ -166,12 +273,14 @@ Custom Codex model/effort handoff: `orca worktree create --agent codex --prompt ...` launches the known Codex agent but does not accept Codex-specific `--model` or `-c model_reasoning_effort=...` arguments. When the user asks for a specific Codex model or effort, create the independent worktree first, launch Codex with the requested command in that worktree, wait only for TUI readiness if prompt delivery would otherwise race startup, send the prompt, and stop. +The two-step custom-argv path cannot enforce a repository's explicit `wait-for-setup` startup policy because the later `terminal create` is not the startup owned by `worktree create`. Use it only when the repository starts agents immediately. If the repository requires `wait-for-setup`, use an agent-first configured launcher that can preserve sequencing, or stop and ask rather than silently bypassing the policy. + Note: when no repo default-terminal configuration supplies a primary terminal, bare create opens a fallback shell before `terminal create` adds the agent. Configured default tabs are materialized instead and may run real commands. Prefer `--agent` whenever custom argv is not required. With the two-step path, target only the agent handle; close a prior terminal only after `terminal list` or `terminal show` confirms it is an unused shell. Use the exact full `::` worktree id returned by `orca worktree create --json`; a bare repo id cannot target the new worktree. ```bash -orca worktree create --name --no-parent --json +orca worktree create --name --no-parent --setup run --json orca terminal create --worktree id: --title --command 'codex --model gpt-5.5 -c model_reasoning_effort="xhigh"' --json orca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json orca terminal send --terminal --text "" --enter --json @@ -195,17 +304,19 @@ Reuse an idle agent in the required worktree only if the prompt allows reuse; ot When a new worktree is allowed, use child lineage for isolated work that is stacked under or dependent on the active worktree, and use `--no-parent` when it is not stacked. Decide the Git base separately: `--no-parent` makes the worktree top-level in Orca, while omitted `--base-branch` uses the repo default base. +For every new worktree, pass `--setup run` so any configured repository setup hook runs. This does not mean waiting for setup before agent launch: preserve the repository's startup policy, whose default starts setup and the agent side by side. Use `--setup skip` or `--setup inherit` only when there is a concrete task-specific reason, and state that reason before creating the worktree. This rule does not rerun setup for current or existing worktrees. + ```bash -orca worktree create --name --agent codex --json +orca worktree create --name --agent codex --setup run --json # or: --agent claude | omp | pi | grok | ... -# Read from startupTerminal.handle in the create response. +# Read from agentTerminalHandle, falling back to startupTerminal.handle. orca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json orca orchestration dispatch --task --to --inject --json ``` -For new-worktree workers, read the id and `startupTerminal.handle` from `worktree create`. Use that as the sole worker handle when present; otherwise use `terminal list` to resolve the agent handle. Omit `--repo` only inside an Orca-managed worktree; otherwise pass `--repo `. +For new-worktree workers, read the id and `agentTerminalHandle` from `worktree create`, falling back to `startupTerminal.handle` for older runtimes. Use that as the sole worker handle when present; otherwise use `terminal list` to resolve the agent handle. Omit `--repo` only inside an Orca-managed worktree; otherwise pass `--repo `. -**For an allowed new worktree, use agent-first:** `--agent` reveals the new worktree and launches the selected agent **in its first terminal**, without adding a separate fallback shell for that worker. Repo setup or default-terminal settings may still add tabs or splits. Do **not** run bare `worktree create` and then `terminal create --command ` for the same worker when agent-first create is available: without configured default tabs, that two-step path leaves a fallback shell + agent pair. Only use it when custom agent argv is required (for example Codex model/effort flags) or when an older CLI rejects `--agent`; if you must, message only the agent handle. Configured default tabs are intentional surfaces, so close a prior terminal only after `terminal list` or `terminal show` confirms it is an unused shell. Do not run `worktree create` when the task must stay in the current worktree. +**For an allowed new worktree, use agent-first:** `--agent` reveals the new worktree and launches the selected agent **in its first terminal**, without adding a separate fallback shell for that worker. Pass `--setup run`; repo setup and default-terminal settings may add intentional tabs or splits. Do **not** run bare `worktree create` and then `terminal create --command ` for the same worker when agent-first create is available: without configured default tabs, that two-step path leaves a fallback shell + agent pair. Only use it when custom agent argv is required (for example Codex model/effort flags) or when an older CLI rejects `--agent`; if you must, message only the agent handle. Configured default tabs are intentional surfaces, so close a prior terminal only after `terminal list` or `terminal show` confirms it is an unused shell. Do not run `worktree create` when the task must stay in the current worktree. Use `orca worktree create --prompt ...` or `orca terminal send ...` for full handoffs or untracked/lightweight prompts. Those paths do not attach `taskId`/`dispatchId`; the worker should not send lifecycle messages unless the prompt supplies a live orchestration preamble. @@ -228,11 +339,12 @@ Wait for `tui-idle` before dispatching. Always pass `--timeout-ms`; real coding ## Agent Guidance -- Workers with a valid live preamble must send `worker_done` exactly once from their own terminal, even on failure: - `orca orchestration send --to --type worker_done --subject "" --body "<3-sentence summary: what you did, what you found, what's left>" --payload '{"taskId":"","dispatchId":"","filesModified":["path/a"],"reportPath":""}' --json` +- Workers with a valid live preamble must send `worker_done` exactly once from their own terminal with an explicit `--outcome succeeded` or `--outcome failed`: + `orca orchestration send --type worker_done --subject "" --body "<3-sentence summary: what you did, what you found, what's left>" --task-id --dispatch-id --outcome succeeded --files-modified "path/a" --report-path "" --json` +- A failed outcome is still a terminal report, but Orca records both the Dispatch and Task as failed. Never encode failure only in the subject/body. - After sending `worker_done`, end your turn and idle at the agent prompt. Do not poll or keep calling `orca orchestration check`; the coordinator re-engages you with a fresh preamble + TASK block delivered as new terminal input. - For long tasks, send heartbeat/status only when the preamble asks for it, including both IDs: - `orca orchestration send --to --type heartbeat --subject "alive" --payload '{"taskId":"","dispatchId":"","phase":"implementing"}' --json` + `orca orchestration send --type heartbeat --subject "alive" --payload '{"taskId":"","dispatchId":"","phase":"implementing"}' --json` - If blocked before completion, use `ask`; use `escalation` only when ownership is valid and the coordinator must intervene. - Treat preambles inherited through terminal history or full handoffs as stale unless the current prompt explicitly keeps that coordinator in the loop. - Coordinators should use `task-list --ready` as external memory, dispatch parallel waves, and avoid dependency chains deeper than 3-4 steps. @@ -244,11 +356,11 @@ orca terminal create --worktree active --title login-css-worker --command "claud orca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json orca orchestration task-create --spec "Fix the login button CSS" --json orca orchestration dispatch --task --to --inject --json -orca orchestration check --wait --types worker_done,escalation,decision_gate --timeout-ms 900000 --json +orca orchestration check --wait --types worker_done,escalation,question --timeout-ms 900000 --json ``` ## Next Action -Coordinator: confirm `orca status --json`, inspect `task-list`/`dispatch-show` if inheriting state, then choose either a manual loop (`task-create` -> worker -> `dispatch --inject` -> `check --wait`) or `orchestration run`. +Coordinator: confirm `orca status --json`, create or bind a Run, inspect `task-list`/`dispatch-show` if inheriting state, then use the explicit supervised loop (`task-create` -> `worker-start` -> `check --wait`). Use low-level terminal creation plus `dispatch --inject` only when the composed start does not express the needed topology. Worker: if the current prompt contains a live dispatch preamble, do the task, use `ask` for blocking questions, and send `worker_done` once with the required payload. If the preamble is stale or absent, do not send lifecycle messages; inspect state or treat the prompt as an ordinary handoff. diff --git a/src/cli/bundled-skill-guides.ts b/src/cli/bundled-skill-guides.ts index 2400c7d3bcf..8b6e0d41847 100644 --- a/src/cli/bundled-skill-guides.ts +++ b/src/cli/bundled-skill-guides.ts @@ -30,7 +30,7 @@ const ORCA_LINEAR_MARKDOWN = "---\nname: orca-linear\ndescription: >-\n Use Orc const ORCA_PER_WORKSPACE_ENV_MARKDOWN = "---\nname: orca-per-workspace-env\ndescription: >-\n Set up, review, debug, or validate Orca per-workspace environment recipes —\n on-demand, disposable runtimes (cloud sandboxes, VMs, or local) created fresh\n for each workspace. Covers first-time setup (provider prerequisites, the\n reusable base snapshot, the coding-agent auth snapshot, credentials, and\n state), not just the per-workspace lifecycle scripts. Use to stand up\n per-workspace environments, fix an `environmentRecipes` entry in `orca.yaml`, scaffold\n provider lifecycle scripts, or resolve an `orca vm recipe doctor` failure.\n---\n\n# Per-Workspace Environments\n\nHelp a user stand up and maintain a repo-owned per-workspace environment recipe end to end. Each\nworkspace gets its own on-demand, disposable runtime (a cloud sandbox, a VM, or a local one),\ncreated fresh and torn down after.\n\nOrca is a **thin wrapper**: you guide, detect, and scaffold; you never own the user's cloud account,\nbilling, images, or credentials.\n\n- **You DO:** sequence the setup, detect what's detectable (provider CLI present/logged-in? recipe\n present? `doctor` passing?), scaffold provider-templated scripts the user fills in, drive the slow\n snapshot/auth phases with the user, and always show the next action.\n- **You DO NOT:** create accounts, choose plans/regions, invent org/project/scope ids, store or print\n secrets, or run anything that spends money without an explicit user OK.\n\nFirst-time setup has **four phases before the per-workspace recipe runs** — easy to miss, so walk\nthem in order:\n\n1. **Prerequisites** — cloud account, provider CLI, scope/project, plan limits, git token (§2).\n2. **Base snapshot** — reusable image: tools + repo + headless build, snapshotted once (§3).\n3. **Agent-auth snapshot** — boot the base, run interactive device-auth, re-snapshot (§4).\n4. **State** — thread snapshot id / scope / project / port between phases via a state file (§6).\n\nThen the **per-workspace contract** (create/suspend/resume/destroy) runs fast (§8).\n\n**The one branch that shapes everything — connection mode:** **Orca-server** (`create` runs `orca serve`\nin the env and emits a `pairingCode`; §7c/§7f) vs **SSH** (`create` runs no server and emits a\n`connection.type:\"ssh\"` block Orca dials into; §7g/§7h). Settle this first — it changes the `create`\noutput shape and half the templates.\n\n**Quick-start (happy path):** interview the user (connection mode Orca-server vs SSH, provider, agent CLI,\ngit auth — §1.2) + read the provider's CLI docs → scaffold `scripts/orca-vm/` from §7 → run the\nbase-snapshot script, then the auth script (you invoke these by hand; not via `orca.yaml`) → wire\n`environmentRecipes` in `orca.yaml` → `orca vm recipe doctor --json` (free) → then the `--provision`\nself-test loop (§9) until it passes.\n\n---\n\n## 1. Setup workflow\n\nDrive these with the user. **[CHECKPOINT]** steps need explicit confirmation — they spend money, take\na long time, or need the user at the keyboard. Never create an Orca workspace or commit unless asked.\n\n1. **Inspect the repo** for an existing `environmentRecipes` entry, `scripts/orca-vm/`, a state file, or setup\n notes. If a working recipe exists, jump to Doctor (§9) instead of rebuilding.\n2. **Interview the user up front** — gather these choices and confirm them back before scaffolding\n anything. Don't pick for them (§11); don't guess.\n - **Connection mode:** how Orca attaches to the environment — an **Orca server** (the VM runs\n `orca serve` and Orca pairs over its pairing URL; worked example §7f) or **SSH** (Orca connects to\n the host over SSH; §7g). This decides the recipe's connection shape, so settle it first.\n - **Provider:** Vercel Sandbox, Fly, Modal, an existing SSH host, … For non-obvious providers, also\n ask scope/project/region and plan limits (§2). Then **read that provider's CLI/SDK docs** (or\n ` --help`) before scaffolding — you need its exact create/exec/snapshot/remove verbs.\n If a provider advertises `ssh`, verify whether it exposes a real dialable SSH target\n (host/port/user/key or proxy command) or only a provider-mediated interactive shell; Orca SSH mode\n needs the former.\n - **Coding-agent CLI + account:** which agent runs in the VM (`codex`, `claude`, …) and that the user\n has an account for it — it gets logged in during the Phase-3 auth snapshot (§4).\n - **Git auth:** the token source for cloning a private repo (`GH_TOKEN`/`GITHUB_TOKEN` or `gh auth\n token`; §5).\n3. **Check prerequisites (§2)** — detect the provider CLI + auth and confirm the items above are in\n place before any paid step.\n4. **Scaffold scripts + state file** from §7 (worked Vercel example: §7f; SSH host: §7g; Docker SSH:\n §7h; Windows: §7i), filling in the provider's real commands. Make them executable.\n5. **[CHECKPOINT] Build the base snapshot (§3)** — paid, slow.\n6. **[CHECKPOINT] Authenticate the agent (§4)** — interactive; the user follows a URL/code. **You cannot\n drive this step** — you run commands non-interactively, so there's no TTY for `docker exec -it` /\n `ssh -t` to prompt against. The **user** runs the Phase-3 login in their own terminal (or via the\n Claude Code harness bang-prefix — `! `, with the required space after `!`); you scaffold and drive\n the non-interactive phases around it. After kicking it off, **ask the user to report back once the login\n finishes** — you can't observe it completing, and you need that confirmation before resuming the\n non-interactive steps (base/auth commit, doctor, provision).\n7. **Wire the recipe** so `orca.yaml` points create/suspend/resume/destroy at the scripts (§8). The\n workspace composer reads `environmentRecipes` from the project's primary checkout of `orca.yaml`, **not** from\n a feature branch or worktree. So a recipe added only on a branch won't appear as a \"Run on\" option\n until that `orca.yaml` change is committed and merged to the project's primary branch. Tell the user\n this up front: `doctor`/`--provision` validate the scripts from the working copy on any branch, but\n creating a workspace from the recipe in the picker needs it on primary.\n8. **Dry-run doctor** — `orca vm recipe doctor --repo-path --json` (free, static; §9).\n Fix every failure before going live.\n9. **[CHECKPOINT] Live self-test** — get the user's OK once, then run\n `orca vm recipe doctor --provision --json` as a loop: it runs create → validates →\n destroys, and on failure returns a full transcript. Read it, fix the scripts, and re-run yourself until\n it passes (§9). Spends cloud money; the one approval covers the loop.\n10. **[CHECKPOINT] Optional workspace test** — only if asked: create a workspace via the picker, then\n verify sleep/wake/delete.\n\n---\n\n## 2. Phase 1 — Prerequisites\n\nThe user's responsibility; verify what's verifiable, ask for the rest, invent nothing. State which\nitems you verified vs. which the user asserted.\n\n- **Connection mode** (Orca server vs SSH) confirmed with the user — see §1 step 2; it shapes the recipe.\n- **Cloud account + plan** that allows sandboxes/VMs. Ask.\n- **Provider CLI installed + authenticated** — detect (`command -v `), check auth (e.g.\n `vercel whoami`). If missing, point at the provider's docs; don't log them in.\n- **Scope / project / region** the sandboxes live under. Ask; flows into every script via state.\n- **Plan / timeout / RAM caps.** Record them — e.g. Vercel Hobby caps sandbox timeout at **45m**,\n which limits both the base build and per-workspace runtime (see §10).\n- **Git token for private repos** (`GH_TOKEN`/`GITHUB_TOKEN`, or the provider's git auth; can fall back\n to `gh auth token`). See §5.\n- **Coding-agent CLI choice** (`codex`, `claude`…) and that the user has an account — it gets\n authenticated into the VM in Phase 3.\n\n---\n\n## 3. Phase 2 — Base snapshot (the reusable image)\n\nBuild **once**, snapshot, and every workspace boots from it in seconds instead of rebuilding.\nProvisioning + building takes a while (often ~20–30 min), so it runs behind a checkpoint. The script\nshape is §7a; key points:\n\n- Build the **headless Electron main only** (not the renderer) so it fits in plan RAM.\n- Use the VM image's package manager (`apt`/`dnf`/`apk`, per the base distro — not the provider brand).\n- Clone with the git token via `GIT_ASKPASS` (§5).\n- **Trap errors and remove the half-built sandbox** so a crash doesn't leave a paid resource running.\n- Snapshot the stopped sandbox, parse the snapshot id, and write it + scope/project/port/repo to state.\n\n---\n\n## 4. Phase 3 — Agent-auth snapshot (interactive)\n\nThe base snapshot has the agent CLI installed but **not logged in**, and per-workspace VMs are\nephemeral — so authenticate once and bake it into a second snapshot layer. Script shape is §7b:\n\n1. Boot a sandbox from the base `snapshotId` (from state).\n2. Run the agent's login **interactively** (`--interactive --tty`); the user completes the URL/code in\n their browser. On a **headless VM this must be the device-auth flow** (e.g. `codex login --device-auth`),\n **not** plain `codex login`: the default OAuth login starts a loopback callback server on a container\n port the host browser can't reach, so it hangs. Device-auth instead prints a URL + code the user opens\n on the **host**.\n3. Verify login; **refuse to snapshot an unauthenticated VM.** Prefer the status command's **exit code**\n (most agent CLIs exit non-zero when unauthenticated). If you grep instead, agent status often goes to\n **stderr** (e.g. `codex login status` prints \"Logged in using ChatGPT\" there), so **fold stderr first**\n (`... 2>&1 | grep …`) and match the agent's **exact success line** — never `grep -qi 'logged in'`, which\n also matches \"**not** logged in\" and would commit an unauthenticated image.\n4. Re-snapshot, parse the new id, and overwrite `snapshotId` in state to the authenticated image\n (recording `authSourceSnapshotId`). Remove the auth sandbox.\n\n**You can't drive step 2 yourself** (you run commands non-interactively — no TTY). The **user** runs it in\ntheir own terminal, or via the Claude Code harness bang-prefix (`! `, with the required space after\n`!`). You scaffold/boot the sandbox and run steps 3–4, but **you cannot observe the interactive login\nfinishing** — so **ask the user to tell you when it's done** before you verify and re-snapshot.\n\nIf the agent's credentials are short-lived, warn that the snapshot may need periodic re-auth (§10).\n\nFor disposable runtimes, do **not** treat a host agent config directory (for example `~/.codex`) as the\nauth snapshot by bind-mounting or copying it wholesale. Agent homes often contain sqlite state, hook\napproval state, caches, logs, and host-specific env/config. Instead, authenticate/configure the agent\ninside the disposable runtime and snapshot/commit that runtime layer.\n\n---\n\n## 5. Credentials\n\n- **Never** commit secrets or put them in `userData`, recipe JSON, comments, docs, or the state file.\n- **Git token:** read from env (`GH_TOKEN`/`GITHUB_TOKEN`), falling back to `gh auth token`. Pass to the\n VM only via the provider's ephemeral `--env`. Inside the VM, use a `GIT_ASKPASS` helper with\n `x-access-token` (not the token in the clone URL) and `GIT_TERMINAL_PROMPT=0` so a missing token fails\n fast instead of hanging. When you write the helper from inside `bash -lc` under `set -u`, escape the\n positional arg and the token (`\\$1`, `\\$GH_TOKEN`) so they land **literally** and resolve at git-runtime\n — an unescaped `$1` aborts with \"unbound variable\", and a literal `$GH_TOKEN` keeps the real token out of\n the written file. `rm -f` the helper after the clone/fetch.\n- **Provider auth:** rely on the provider CLI's logged-in session, not checked-in keys.\n- **Agent auth:** lives in the authenticated snapshot (Phase 3) — never a file you write or commit.\n- State holds only **non-secret** wiring (snapshot ids, scope, project, port, repo url/ref).\n\n---\n\n## 6. State file\n\nA repo-local JSON file (e.g. `scripts/orca-vm/-state.json`) threads non-secret values between\nphases. Each script resolves values as **env var → state → built-in fallback**, and merges its outputs\nback. Phase 2 writes the base `snapshotId`; Phase 3 overwrites it with the authenticated snapshot;\nper-workspace `create` boots from `snapshotId`.\n\n```json\n{\n \"baseName\": \"orca-base\",\n \"snapshotId\": \"snap_authenticated_image_id\",\n \"authSourceSnapshotId\": \"snap_base_image_id\",\n \"scope\": \"\",\n \"project\": \"\",\n \"port\": 7331,\n \"repoUrl\": \"https://host/org/repo.git\",\n \"repoRef\": \"main\",\n \"projectRoot\": \"/abs/path/on/remote/repo\"\n}\n```\n\n---\n\n## 7. Script templates (provider-agnostic shapes)\n\nScaffold under `scripts/orca-vm/`. These are **shapes** — fill in the provider's real commands. All\nreserve stdout for the final JSON and log progress to stderr. Include a shared `json_value ` /\n`env_value ` reader (env → state → fallback) in each.\n\n**Where each script runs:**\n\n- **Local-side** (`create`/`suspend`/`resume`/`destroy` + the base-snapshot/auth scripts the user\n invokes) runs **on the user's desktop**, so it must run on their OS. macOS/Linux: `#!/usr/bin/env\n bash`, `set -euo pipefail`, quoted paths. **Windows:** a bare `.sh` won't run — scaffold `.ps1`/`.cmd`\n or require WSL/Git-Bash and point `orca.yaml` at the right launcher.\n- **Remote-side** (commands you `exec` *inside* the Linux VM) always runs in the VM's Linux shell, so\n bash is fine there regardless of the user's OS.\n\n### 7a. Base-snapshot (`-base-snapshot.sh`) — Phase 2\n\n```bash\n#!/usr/bin/env bash\nset -euo pipefail\n# resolve base_name/repo_url/repo_ref/project_root/port/scope/project/timeout (env→state→fallback)\n# resolve gh token: GH_TOKEN | GITHUB_TOKEN | `gh auth token`\n# 1. provision a sandbox (timeout/vcpus/published port/snapshot retention); trap: remove on error\n# 2. remote exec (long timeout): install pkgs + gh + corepack/pnpm + agent CLI;\n# clone with GIT_ASKPASS(token); write headless main-only build config;\n# dev setup; pnpm install; build CLI; build headless electron main; smoke-check tools\n# 3. snapshot stopped sandbox; parse snapshot id (fail if unparseable)\n# 4. merge { baseName, snapshotId, projectRoot, repoUrl, repoRef, port, scope, project } into state\n# print only the state JSON to stdout\n```\n\nWorked Vercel commands for this phase are in §7f. You run this script by hand (not via `orca.yaml`),\nafter exporting the first-run inputs the state file doesn't have yet — e.g. provider scope/project, the\nrepo URL/ref, and a git token (`GH_TOKEN`); later runs read them back from state.\n\n### 7b. Auth (`-base-auth.sh`) — Phase 3\n\n```bash\n#!/usr/bin/env bash\nset -euo pipefail\n# read source snapshot from state.snapshotId (fail if absent); auth_name=\"${base_name}-auth\"\n# 1. boot sandbox from source snapshot; trap: remove on error\n# 2. INTERACTIVE/TTY remote exec: agent login — user completes URL/code. Headless VM: MUST use the\n# device-auth flow (e.g. `codex login --device-auth`) — plain OAuth login binds a loopback callback\n# port the host can't reach and hangs. User runs this themselves (you have no interactive TTY); ask\n# them to report back when it's done before continuing.\n# 3. verify login, then refuse to snapshot if not logged in. Prefer the status command's EXIT CODE (most\n# agent CLIs exit non-zero when unauthenticated) over string-matching. If you must grep, fold stderr\n# first (`status 2>&1 | grep …` — many agents print the success line there) and match the agent's exact\n# success line; never `grep -qi 'logged in'`, which also matches \"not logged in\". Codex example: §7f.\n# 4. snapshot; parse new id\n# 5. merge { snapshotId:, authSourceSnapshotId: } into state; remove auth sandbox\n# print only the state JSON to stdout\n```\n\n### 7c. Create (`-create.sh`) — per workspace\n\n```bash\n#!/usr/bin/env bash\nset -euo pipefail\n# read authenticated snapshotId/scope/project/port/repo*/project_root (env→state→fallback)\n# fail clearly if snapshotId is missing (point back to Phases 2–3)\n# name = orca-${ORCA_VM_RECIPE_ID}-${ORCA_VM_INSTANCE_ID} (sanitized, length-capped)\n# 1. boot sandbox from snapshotId with a published port; capture the public URL → pairing address\n# (an externally reachable wss:// URL); trap: remove sandbox on error\n# 2. remote exec: ensure repo at desired commit; rebuild only if commit changed (cache marker)\n# 3. remote exec: start orca serve in the background and read the recipe JSON it writes (see below)\n# 4. print serve's JSON to stdout, optionally enriched with userData:\n# { schemaVersion:1, pairingCode, projectRoot, userData:{ provider, resourceId:name, snapshotId } }\n```\n\n**The exact `orca serve` invocation and its output (verified — do not improvise the flags).** Inside the\nVM, run:\n\n```bash\norca serve \\\n --port \"$PORT\" \\\n --project-root \"$ABS_REPO_PATH_ON_REMOTE\" \\\n --pairing-address \"$EXTERNAL_WSS_URL\" \\\n --recipe-json\n```\n\n**Binary name:** in a VM built from source (the Phase-2 flow), run it as `pnpm exec orca-dev serve …`\nfrom the repo root — `orca-dev` is the in-repo entrypoint and is what the §7f example uses. Plain\n`orca serve …` is the same command when the built CLI is installed on the VM's PATH. The flags/output\nare identical either way.\n\nThere is **no `--host` flag**. `--project-root` must be an absolute directory on the remote. With\n`--recipe-json` the server **stays running** and prints exactly this single object to **stdout**, then\nkeeps serving:\n\n```json\n{ \"schemaVersion\": 1, \"pairingCode\": \"\", \"projectRoot\": \"\" }\n```\n\n`pairingCode` is the pairing URL, already pointing at whatever you passed as `--pairing-address` — so set\n`--pairing-address` to the externally reachable address and **pass `pairingCode` through unchanged; never\nhand-rewrite it**. Because serve runs in the foreground and doesn't exit, redirect its stdout to a file\nand poll until that file parses as JSON (and bail if the process dies — dump its stderr log). Your\n`create` script then prints that JSON (optionally merging `userData`). Concrete pattern: §7f.\n\n### 7d. Suspend / resume / destroy — per workspace\n\n```bash\n#!/usr/bin/env bash\nset -euo pipefail\npayload=\"$(cat)\" # Orca passes lifecycle JSON on stdin\nresource_id=\"$(node -e 'const d=JSON.parse(process.argv[1]); process.stdout.write(d.recipeResult?.userData?.resourceId ?? \"\")' \"$payload\")\"\n[ -n \"$resource_id\" ] || { echo \"No resource id in lifecycle payload\" >&2; exit 1; }\n# suspend: provider suspend \"$resource_id\"\n# resume: provider resume \"$resource_id\"; then RE-EMIT fresh recipe JSON (pairing may change)\n# destroy: provider remove \"$resource_id\" (or set destroy: none in orca.yaml)\n```\n\n### 7e. State file — scaffold with scope/project/repo filled in and snapshot ids empty (§6).\n\n### 7f. Worked example — Vercel Sandbox (all three phases)\n\nA real, working shape (the Vercel surface is a CLI: `vercel sandbox create|exec|snapshot|remove`). Adapt\nnames; verify flags against `vercel sandbox --help` for the user's CLI version before relying on them.\nThese ground §7a (base snapshot) and §7b (auth), which are otherwise generic skeletons.\n\n**Phase 2 — base snapshot (§7a):** provision → install tools + clone + headless build → snapshot.\n\n```bash\n# provision a fresh build sandbox (retain a couple of snapshots); trap-remove on error\nvercel sandbox create --name \"$base\" --runtime node24 --timeout 30m --vcpus 4 --publish-port \"$port\" \\\n --snapshot-expiration 30d --keep-last-snapshots 2 \"${vercel_args[@]}\" >&2\n# remote build (long timeout): install pkgs+gh+pnpm+agent CLI, clone with GIT_ASKPASS (write the helper\n# with LITERAL \\$1/\\$GH_TOKEN so they resolve at git-runtime, not write-time — see §5/§7f create — then\n# `rm -f /tmp/askpass.sh`), write the headless main-only build config (drop the renderer), dev setup,\n# build CLI + headless main, smoke-check\nvercel sandbox exec \"$base\" \"${vercel_args[@]}\" --timeout 25m --env \"GH_TOKEN=$gh_token\" … -- bash -lc '…build…' >&2\n# snapshot the STOPPED sandbox and parse the id from CLI output (fail if unparseable)\nout=\"$(vercel sandbox snapshot \"$base\" --stop --expiration 30d \"${vercel_args[@]}\" 2>&1)\"; printf '%s\\n' \"$out\" >&2\nsnapshot_id=\"$(printf '%s\\n' \"$out\" | sed -nE 's/.*(snap_[A-Za-z0-9]+).*/\\1/p' | tail -1)\"\n# merge { baseName, snapshotId, scope, project, port, repoUrl, repoRef, projectRoot } into state; print state JSON\n```\n\n**Phase 3 — agent-auth snapshot (§7b):** boot the base, log the agent in interactively, re-snapshot.\n(`codex` below is an example — substitute the user's chosen agent's login/status verbs, e.g. `claude`.)\n\n```bash\nvercel sandbox create --name \"$auth\" --snapshot \"$snapshot_id\" --timeout 30m --publish-port \"$port\" \"${vercel_args[@]}\" >&2\n# INTERACTIVE — the USER runs this in their own terminal (you have no interactive TTY) and completes the\n# URL/code on the HOST. --device-auth is MANDATORY on a headless VM: plain `codex login` binds a loopback\n# callback port the host browser can't reach and hangs. Ask the user to report back when login finishes.\nvercel sandbox exec --interactive --tty \"$auth\" \"${vercel_args[@]}\" -- bash -lc 'codex login --device-auth'\n# refuse to snapshot an unauthenticated VM — fold stderr, match codex's exact success line (§4)\nvercel sandbox exec \"$auth\" \"${vercel_args[@]}\" --timeout 30s -- bash -lc 'codex login status 2>&1' | grep -Eqi 'Logged in using ChatGPT|Logged in via device' \\\n || { echo \"agent not logged in; not snapshotting\" >&2; exit 1; }\nout=\"$(vercel sandbox snapshot \"$auth\" --stop --expiration 30d \"${vercel_args[@]}\" 2>&1)\"; printf '%s\\n' \"$out\" >&2\nnew_id=\"$(printf '%s\\n' \"$out\" | sed -nE 's/.*(snap_[A-Za-z0-9]+).*/\\1/p' | tail -1)\"\n# overwrite state.snapshotId = new_id, record authSourceSnapshotId = snapshot_id; remove the auth sandbox\n```\n\n**Per-workspace `create`** (the fast path):\n\n```bash\n#!/usr/bin/env bash\nset -euo pipefail\n# resolve from env→state→fallback: snapshot_id, scope, project, port, repo_url, repo_ref, project_root\nvercel_args=(); [ -n \"$scope\" ] && vercel_args+=(--scope \"$scope\"); [ -n \"$project\" ] && vercel_args+=(--project \"$project\")\n[ -n \"$snapshot_id\" ] || { echo \"snapshotId missing — run Phases 2–3 first\" >&2; exit 1; }\ngh_token=\"${GH_TOKEN:-${GITHUB_TOKEN:-$(command -v gh >/dev/null 2>&1 && gh auth token 2>/dev/null || true)}}\"\nname=\"orca-${ORCA_VM_RECIPE_ID:-vercel-sandbox}-${ORCA_VM_INSTANCE_ID:-$(date +%s)}\" # sanitize+cap to 63 chars\n\n# Arm cleanup BEFORE create so a failing create can't leak a half-built paid sandbox.\ncleanup_on_error() { [ \"$?\" -ne 0 ] && vercel sandbox remove \"$name\" \"${vercel_args[@]}\" >/dev/null 2>&1 || true; }\ntrap cleanup_on_error EXIT\n\n# 1. boot from the authenticated snapshot, publish the serve port\ncreate_output=\"$(vercel sandbox create --name \"$name\" --snapshot \"$snapshot_id\" \\\n --timeout 30m --publish-port \"$port\" \"${vercel_args[@]}\" 2>&1)\"; printf '%s\\n' \"$create_output\" >&2\n# Vercel prints the published https URL; derive the external wss:// pairing address from it\npublic_url=\"$(printf '%s\\n' \"$create_output\" | sed -nE 's#.*(https://[^[:space:]]+\\.vercel\\.run).*#\\1#p' | head -1)\"\n[ -n \"$public_url\" ] || { echo \"no published URL in create output\" >&2; exit 1; }\npairing_ws=\"${public_url/https:\\/\\//wss://}\"\n\n# 2. (remote) ensure the repo is at the right commit; rebuild only if the commit changed (cache marker)\nvercel sandbox exec \"$name\" \"${vercel_args[@]}\" --timeout 20m \\\n --env \"GH_TOKEN=$gh_token\" --env \"ORCA_PROJECT_ROOT=$project_root\" \\\n --env \"ORCA_REPO_URL=$repo_url\" --env \"ORCA_REPO_REF=$repo_ref\" \\\n -- bash -lc 'set -euo pipefail; cd \"$ORCA_PROJECT_ROOT\"; \\\n # Re-establish git auth for the private-repo fetch (why + full rationale: §5); else it hangs on a prompt.\n # Load-bearing escaping: \\$1 and \\$GH_TOKEN must land LITERALLY and resolve at git-runtime. Test after\n # any edit here — reformatting the nested printf/node quoting silently breaks the fetch or leaks the token.\n if [ -n \"${GH_TOKEN:-}\" ]; then \\\n printf \"%s\\n\" \"#!/usr/bin/env bash\" \"case \\\"\\$1\\\" in *Username*) echo x-access-token;; *Password*) echo \\\"\\$GH_TOKEN\\\";; esac\" > /tmp/askpass.sh; \\\n chmod 700 /tmp/askpass.sh; export GIT_ASKPASS=/tmp/askpass.sh GIT_TERMINAL_PROMPT=0; fi; \\\n git fetch origin \"$ORCA_REPO_REF\"; \\\n git checkout -B \"$ORCA_REPO_REF\" FETCH_HEAD; \\\n rm -f /tmp/askpass.sh; \\\n c=\"$(git rev-parse HEAD)\"; [ -f .orca-built ] && [ \"$(cat .orca-built)\" = \"$c\" ] || { \\\n pnpm install --prefer-offline && pnpm run build:cli && \\\n node config/scripts/run-electron-vite-build.mjs --config config/electron-vite.vm-serve.config.ts && \\\n printf \"%s\" \"$c\" > .orca-built; }' >&2\n\n# 3. (remote) start orca serve in the background, writing recipe JSON to a file; poll until it parses\nrecipe_json=\"$(vercel sandbox exec \"$name\" \"${vercel_args[@]}\" --timeout 60s \\\n --env \"ORCA_PORT=$port\" --env \"ORCA_PROJECT_ROOT=$project_root\" --env \"ORCA_PAIRING_ADDRESS=$pairing_ws\" \\\n -- bash -lc 'set -euo pipefail; cd \"$ORCA_PROJECT_ROOT\"; rm -f /tmp/orca-recipe.json /tmp/orca-serve.log; \\\n nohup pnpm exec orca-dev serve --port \"$ORCA_PORT\" --project-root \"$ORCA_PROJECT_ROOT\" \\\n --pairing-address \"$ORCA_PAIRING_ADDRESS\" --recipe-json >/tmp/orca-recipe.json 2>/tmp/orca-serve.log /dev/null 2>&1 && { cat /tmp/orca-recipe.json; exit 0; }; \\\n kill -0 \"$pid\" 2>/dev/null || { cat /tmp/orca-serve.log >&2; exit 1; }; sleep 0.25; \\\n done; cat /tmp/orca-serve.log >&2; echo \"serve recipe JSON timed out\" >&2; exit 1')\"\n\n# 4. print serve's JSON enriched with userData (single object on stdout)\nnode -e 'const p=JSON.parse(process.argv[1]); console.log(JSON.stringify({...p, schemaVersion:1,\n userData:{...p.userData, provider:\"vercel-sandbox\", resourceId:process.argv[2], snapshotId:process.argv[3]}}))' \\\n \"$recipe_json\" \"$name\" \"$snapshot_id\"\ntrap - EXIT\n```\n\n`suspend`/`resume`/`destroy` use `vercel sandbox stop|...|remove \"$resource_id\"` reading\n`userData.resourceId` from stdin (§7d). This is the **Orca-server** connection mode (the recipe emits a\npairing URL). If the user chose **SSH** in the §1 interview, use §7g instead.\n\n### 7g. Worked example — existing SSH host (SSH connection mode)\n\nSSH mode is **fundamentally different from §7c/§7f**, not a relabeling of them:\n\n- **`create` does NOT run `orca serve` and does NOT emit a `pairingCode`.** Orca itself connects to the\n host over its SSH relay, brings up the git + filesystem providers, and imports the repo. The script's\n only job is to make the host ready and **print SSH connection details** Orca will dial.\n- The result uses a `connection` block with `type: \"ssh\"` and a `target`, **not** the flat\n `pairingCode`/`projectRoot` shape. Exact shape (Orca rejects anything else):\n\n```json\n{\n \"schemaVersion\": 1,\n \"connection\": {\n \"type\": \"ssh\",\n \"projectRoot\": \"/abs/path/to/repo/on/host\",\n \"target\": {\n \"label\": \"my-box\",\n \"host\": \"192.0.2.10\",\n \"port\": 22,\n \"username\": \"ubuntu\",\n \"identityFile\": \"~/.ssh/id_ed25519\",\n \"jumpHost\": \"bastion.example.com\",\n \"proxyCommand\": \"cloudflared access ssh --hostname %h\",\n \"relayGracePeriodSeconds\": 0,\n \"portForwards\": []\n }\n }\n}\n```\n\n`label`, `host`, `port`, `username` are required; the rest are optional — omit any you don't need.\n\n**Networking → which `target` fields to set** (how *your desktop* reaches the box — there is no\n`orca serve` URL in SSH mode):\n\n- Public IP / DNS, or a Tailscale/VPN address → `host`; SSH port → `port` (usually 22).\n- Key auth → `identityFile` (add `identitiesOnly: true` if the agent has many keys).\n- Through a bastion → `jumpHost` (a `user@host` ProxyJump) **or** a full `proxyCommand` (e.g. an access\n proxy). Use one, not both.\n- A service port the workspace needs → add entries to `portForwards`.\n- `relayGracePeriodSeconds` (optional): how long Orca keeps the SSH relay alive after the workspace\n detaches before tearing it down; `0` = tear down immediately. Leave it off unless the user wants a\n reconnect grace window.\n\n**Toolchain & agent auth on a persistent (no-snapshot) host — do this ONCE, by hand, before wiring the\nrecipe** (there's no base image to bake; the host *is* the base). Run the §7f Phase-2 install steps and\nthe §7f Phase-3 ` login --device-auth` **directly over SSH on the host** (interactive, e.g.\n`ssh -t user@host ' login --device-auth'`). After that the host stays ready across workspaces.\n\n```bash\n#!/usr/bin/env bash\nset -euo pipefail\n# resolve from env→state→fallback (default unset optionals to \"\"): ssh_username, host,\n# ssh_port (default 22), identity_file, jump_host, proxy_command, project_root, repo_url, repo_ref\n: \"${identity_file:=}\"; : \"${jump_host:=}\"; : \"${proxy_command:=}\" # avoid set -u aborts on optionals\ngh_token=\"${GH_TOKEN:-${GITHUB_TOKEN:-$(command -v gh >/dev/null 2>&1 && gh auth token 2>/dev/null || true)}}\"\nssh_target=\"${ssh_username}@${host}\"\nssh_opts=(-p \"$ssh_port\"); [ -n \"$identity_file\" ] && ssh_opts+=(-i \"$identity_file\")\n# Why: a fresh host's key isn't in known_hosts; a StrictHostKeyChecking prompt would HANG a\n# non-interactive create. Pre-add the key (or set the option) so it can't block.\nssh-keyscan -p \"$ssh_port\" \"$host\" >> \"$HOME/.ssh/known_hosts\" 2>/dev/null || true\n\n# 1. ensure the repo is present and at the right commit on the host (NO orca serve here)\nssh \"${ssh_opts[@]}\" \"$ssh_target\" \\\n \"GH_TOKEN='$gh_token' GIT_TERMINAL_PROMPT=0 bash -lc '\n set -euo pipefail\n [ -d \\\"$project_root/.git\\\" ] || git clone \\\"$repo_url\\\" \\\"$project_root\\\"\n cd \\\"$project_root\\\" && git fetch origin \\\"$repo_ref\\\" && git checkout -B \\\"$repo_ref\\\" FETCH_HEAD\n '\" >&2\n\n# 2. print the SSH connection block (NO pairingCode, NO orca serve). host/port/username tell Orca's\n# relay how to dial in; identityFile/jumpHost/proxyCommand/portForwards are emitted when set.\nnode -e 'const [host,port,user,idf,jh,pc,root]=process.argv.slice(1);\n const target={ label:\"per-workspace-host\", host, port:Number(port), username:user };\n if(idf) target.identityFile=idf; if(jh) target.jumpHost=jh; if(pc) target.proxyCommand=pc;\n // add target.portForwards=[...] here if the workspace needs forwarded service ports\n console.log(JSON.stringify({ schemaVersion:1, connection:{ type:\"ssh\", projectRoot:root, target } }))' \\\n \"$host\" \"$ssh_port\" \"$ssh_username\" \"$identity_file\" \"$jump_host\" \"$proxy_command\" \"$project_root\"\n```\n\n`suspend`/`resume`/`destroy`: on a persistent host there's usually nothing to tear down — set\n`destroy: none` and omit suspend/resume. (Orca still disconnects/reconnects its own SSH relay on\nsleep/wake/delete — that's separate from these scripts.)\n\nIf the SSH host is instead an **ephemeral/snapshot-capable VM** (your hypervisor, or a cloud VM with\nimage support), keep the §7f Phase-2/3 base-image model for provisioning, but still emit the\n`connection.type:\"ssh\"` block above instead of starting `orca serve`.\n\n### 7h. Worked example — local Docker SSH (SSH connection mode)\n\nLocal Docker can model an ephemeral SSH VM without cloud cost: build a base image with `sshd`, tools,\nrepo prerequisites, and the agent CLI; run an **interactive auth container** once; then `docker commit`\nthat container as the authenticated image used by per-workspace `create`.\n\nKey points:\n\n- Publish container SSH to a random localhost port (`-p 127.0.0.1::22`) and emit\n `connection.type:\"ssh\"` with `host:\"127.0.0.1\"`, that port, `username`, `identityFile`, and\n `identitiesOnly:true`.\n- Generate a repo-local SSH key if needed, but gitignore the private/public key files.\n- **Bake SSH host keys into the base image** (`ssh-keygen -A` at **build** time; at runtime only generate\n if absent). Ephemeral containers all present the **same** host key, so `known_hosts` on `127.0.0.1`\n doesn't churn as the published port rotates across workspaces (otherwise every container's freshly\n generated key collides on `localhost` and trips host-key-changed warnings).\n- The auth image is the Docker equivalent of Phase 3: the **user** runs the agent login **inside** the\n container (you can't drive it — you have no interactive TTY), configures proxy env/config, approves\n hooks, and you commit once they report it's done. On a headless container use the **device-auth** flow\n (§4). Verify login before committing — exit code, or fold stderr and match the exact success line (§4).\n- Do not bind-mount or copy the host's full agent home into the image. Let each container have writable\n agent state; only the committed auth image should carry reusable authenticated state.\n- If committing from an interactive shell, force the runtime entrypoint back to `sshd`:\n `docker commit --change='ENTRYPOINT [\"/usr/local/bin/orca-docker-ssh-entrypoint\"]' …`.\n- `destroy` should read `recipeResult.userData.resourceId` and run `docker rm -f \"$resource_id\"`.\n\nValidation before wiring/live use:\n\n```bash\ndocker image inspect \"$auth_image\" --format '{{json .Config.Entrypoint}}'\ndocker run -d --name \"$name\" -p 127.0.0.1::22 -e \"ORCA_SSH_PUBLIC_KEY=$pubkey\" \"$auth_image\"\ndocker ps -a --filter \"name=$name\"\ndocker logs \"$name\"\nssh -i \"$key\" -p \"$port\" -o IdentitiesOnly=yes user@127.0.0.1 'codex --version'\n```\n\nIf the container exits immediately, inspect logs before the cleanup trap removes it; a committed\ninteractive image with `ENTRYPOINT [\"bash\"]` is a common cause.\n\nAlso confirm the **host key is stable** across containers: the SSH `ssh -i … 127.0.0.1` dial should not\ntrigger a host-key-changed warning when a second container reuses the port. If it does, the host keys\nweren't baked into the base image (see the `ssh-keygen -A` point above).\n\n### 7i. Windows local-side scripts\n\nThe local-side scripts run on the user's desktop. On **Windows**, a bare `.sh` won't execute. Either\nrequire WSL/Git-Bash (and point `orca.yaml` at e.g. `bash ./scripts/orca-vm/.sh` via a `.cmd`\nlauncher), or scaffold PowerShell equivalents. Minimal PowerShell shape:\n\n```powershell\n#requires -Version 5\n$ErrorActionPreference = 'Stop'\n# resolve env→state→fallback; run the provider CLI / ssh the same way;\n# capture provider output; build the result object for the chosen mode and write ONE line of JSON to stdout.\n# Orca-server mode: @{ schemaVersion=1; pairingCode=$pairingCode; projectRoot=$projectRoot; userData=@{...} }\n# SSH mode: @{ schemaVersion=1; connection=@{ type=\"ssh\"; projectRoot=$projectRoot;\n# target=@{ label=$label; host=$host; port=$port; username=$user } } } (see §7g/§7h)\n($result | ConvertTo-Json -Compress -Depth 6)\n# progress/errors → Write-Error / the error stream, never stdout.\n```\n\nThe remote-side commands you run *inside* the Linux VM stay bash regardless of the desktop OS.\n\n---\n\n## 8. Per-workspace recipe contract (the fast path)\n\nOnce the authenticated snapshot exists, this runs on every workspace create. Define recipes in\n`orca.yaml`:\n\n```yaml\nenvironmentRecipes:\n - id: cloud-sandbox\n name: Cloud Sandbox\n create: ./scripts/orca-vm/cloud-sandbox-create.sh\n suspend: ./scripts/orca-vm/cloud-sandbox-suspend.sh\n resume: ./scripts/orca-vm/cloud-sandbox-resume.sh\n destroy: ./scripts/orca-vm/cloud-sandbox-destroy.sh\n```\n\n`create` runs **locally from the repo root** and prints **one** JSON object to stdout. Its shape depends\non the connection mode chosen in §1:\n\n**Orca-server mode** — boot the env, start `orca serve` in it, and print serve's result:\n\n```json\n{\n \"schemaVersion\": 1,\n \"pairingCode\": \"orca-pairing-code-or-url\",\n \"projectRoot\": \"/absolute/path/to/repo/on/remote\",\n \"userData\": { \"provider\": \"example\", \"resourceId\": \"provider-resource-id\" }\n}\n```\n\nHere `pairingCode` (from `orca serve --recipe-json`) and `projectRoot` are required; `schemaVersion` (`1`)\nand `userData` are optional.\n\n**SSH mode** — do **not** run `orca serve`; print the `connection.type:\"ssh\"` block instead (full shape +\nworked script in §7g). `pairingCode` is **not** used in SSH mode.\n\nLifecycle hooks (all run locally):\n\n- `create`: required. Prints recipe result JSON.\n- `suspend`: optional. Sleep; reads lifecycle payload on stdin.\n- `resume`: optional. Wake; reads payload on stdin and **prints fresh recipe JSON** (pairing may change).\n- `destroy`: optional unless `destroy: none`. Delete/cleanup; reads payload on stdin.\n\nStart Orca remotely with `orca serve --port \"$PORT\" --project-root \"$ABS_ROOT\" --pairing-address\n\"$EXTERNAL_WSS_URL\" --recipe-json` (exact flags + output in §7c). Set `--pairing-address` to the\nexternally reachable address so the emitted `pairingCode` is reachable; tunneling/port mapping is the\nscript's job.\n\nBackward compatibility: `command`→`create`, `cleanup`→`destroy`, `cleanup: none`→`destroy: none`.\nPrefer the lifecycle names.\n\n---\n\n## 9. Doctor and validation\n\nValidate in two stages — the cheap dry run first, then the live self-test.\n\n### Dry run (free, non-destructive) — always do this first\n\n`orca vm recipe doctor --repo-path --json` validates **static wiring only** — it does\n**not** boot anything. It checks: local-host execution (v1), repo path, recipe id exists,\ncreate/destroy/suspend/resume command paths resolve, suspend/resume are paired, and each script is\nexecutable (POSIX exec bit; skipped on Windows). Fix every failure here before spending any cloud money.\n\n### Live self-test (`--provision`) — diagnose and iterate yourself\n\n`orca vm recipe doctor --repo-path --provision --json` actually runs the recipe end\nto end: it executes `create`, validates the returned recipe JSON, then runs `destroy` to **tear the\nenvironment back down** (so the test leaves nothing running, as long as `destroy` works). It spends real\ncloud money, so get the user's OK **once** before starting — that one approval covers the whole loop\nbelow; do not re-ask before each run.\n\nOn failure, the JSON result includes a `provisionTranscript` with the **complete** captured output of\neach stage so you can self-diagnose without asking the user to relay logs:\n\n```json\n{\n \"ok\": false,\n \"checks\": [ { \"id\": \"recipe.provision\", \"status\": \"fail\", \"message\": \"…\" } ],\n \"provisionTranscript\": {\n \"provision\": { \"exitCode\": 0, \"signal\": null, \"stdout\": \"…\", \"stderr\": \"…\", \"parseError\": \"…\" },\n \"destroy\": { \"exitCode\": 0, \"signal\": null, \"stdout\": \"…\", \"stderr\": \"…\" }\n }\n}\n```\n\n**Run it as a loop:** read `provisionTranscript.provision.stderr` / `.stdout` / `.parseError` (and\n`destroy.*`), fix the script, and re-run `--provision` until `ok` is `true` — iterating on your own\nrather than waiting for the user to paste errors. Common reads: a non-empty `stderr` with `exitCode 0`\nplus a `parseError` means `create` ran but printed something other than the single recipe-result JSON on\nstdout (often a stray `echo` — route it to stderr, see §10); a non-zero `exitCode` is a provider/script\nfailure described in `stderr`. Each stream is redacted and capped (head+tail) — large logs keep both the\nsetup context and the failure.\n\nThe self-test cannot see provider-side truth beyond what the scripts print, so still confirm: state has a\npopulated **authenticated** `snapshotId` (Phases 2–3 done), and `destroy` is implemented/tested (or\nexplicitly `none` — in which case the self-test won't tear down, so clean up manually).\n\nFor SSH recipes, also smoke-test the exact emitted target before declaring success: dial the host/port\nwith the identity/proxy settings, run `pwd`, verify the repo path, check the agent binary, and confirm\n`destroy` removes the provider resource/container. For Docker, inspect the auth image entrypoint and do a\nstartup-only `docker run` before the full clone/install path.\n\n---\n\n## 10. Failure modes\n\n- **Build exceeds plan timeout (e.g. Hobby 45m).** Use enough vCPUs and a timeout covering the build;\n else split work or use a higher plan. The cap also limits per-workspace runtime — surface it.\n- **Build exceeds plan RAM.** Build the **headless main only** (drop the renderer) — the biggest fitter.\n- **Private-repo clone hangs/fails.** Wrong/missing token. Use `GIT_ASKPASS` + `GIT_TERMINAL_PROMPT=0`\n so it fails fast instead of prompting.\n- **`GIT_ASKPASS` helper aborts the clone with \"`$1: unbound variable`\".** The `printf`/heredoc that writes\n the helper inside `bash -lc` under `set -u` expanded `$1`/`$GH_TOKEN` at **write** time. Escape them\n (`\\$1`, `\\$GH_TOKEN`) so they land literally and resolve at git-runtime; this also keeps the real token\n out of the file. `rm -f` the helper afterward (§5, §7f).\n- **Agent verified as \"not logged in\" despite a good login.** `codex login status` (and similar) print\n \"Logged in …\" to **stderr**; an stdout-only `grep` misses it. Prefer the status **exit code**; if you\n grep, fold stderr first (`status 2>&1 | grep …`) and match the exact success line — not `grep -qi\n 'logged in'`, which also matches \"not logged in\".\n- **Headless agent login hangs.** Plain OAuth `login` starts a loopback callback server on a VM/container\n port the host browser can't reach. Use the **device-auth** flow (`login --device-auth`) — it prints a\n URL + code the user opens on the host.\n- **`known_hosts` host-key churn on local Docker.** Each ephemeral container regenerating its SSH host key\n collides on `127.0.0.1` as the published port rotates. Bake host keys into the base image at build time\n (`ssh-keygen -A`; runtime generates only if absent) so all containers share one stable key (§7h).\n- **Snapshot expired/evicted.** If `create` hits an unknown snapshot id, rerun Phases 2–3 and update\n `snapshotId`.\n- **Agent auth didn't persist.** Confirm `snapshotId` points at the **authenticated** snapshot; re-run\n Phase 3. Warn that short-lived tokens may need periodic re-auth.\n- **Agent auth copied from the host breaks.** Do not bind-mount/copy a full host agent home; sqlite\n files can be unwritable or host-specific, hooks may need approval again, and config may reference\n local-only env vars. Authenticate inside the runtime and snapshot/commit that layer.\n- **Docker auth image exits immediately.** Inspect `docker image inspect … .Config.Entrypoint` and\n `docker logs`. If the image was committed from an interactive shell, reset the entrypoint to the SSH\n entrypoint during `docker commit`.\n- **Leaked paid resource.** Every long script must trap errors and remove the sandbox it created.\n- **`create` emits non-JSON on stdout.** A stray `echo` corrupts the result — stdout is for the final\n JSON only; everything else to stderr. The `--provision` self-test surfaces this as `exitCode 0` + a\n `parseError` with the offending stdout in `provisionTranscript` (§9).\n\n---\n\n## 11. Boundaries\n\n- Don't create accounts, choose plans/regions, or invent scope/project/org/image/billing ids.\n- Don't invent or store credentials; no secrets in `userData`, state, comments, docs, or commits.\n- Don't run paid/long phases (base snapshot, auth, live test) without an explicit OK.\n- Don't hide provider errors behind generic messages — preserve actionable stderr.\n- Don't make Orca own provider lifecycle beyond invoking the configured scripts.\n- Don't commit or create an Orca workspace unless asked.\n" // oxfmt-ignore -const ORCHESTRATION_MARKDOWN = "---\nname: orchestration\ndescription: >-\n Use Orca orchestration for structured multi-agent coordination: threaded\n messages, blocking ask/reply flows, task dispatch, worker_done/escalation\n waits, task DAGs, decision gates, coordinator loops, or decomposing work\n across agents. Use `orca-cli` instead for full ownership handoffs, including\n requests phrased as \"hand off\", \"handoff\", \"handover\", \"give this to another\n agent\", or \"another worktree\" when the user did not explicitly ask to\n supervise, monitor, wait for results, or coordinate a DAG. Use `orca-cli` for\n ordinary terminal control, lightweight terminal prompts, shell commands, Orca\n worktree management, reading or waiting on terminals, and automation of the\n browser embedded inside Orca. Use Computer Use for browser windows, webviews,\n Orca app UI, or desktop UI outside Orca's embedded browser.\n---\n\n# Orca Inter-Agent Orchestration\n\nOrchestration is Orca's structured coordination layer for agent messages, task ownership, dispatch state, and worker completion tracking.\n\nUse this skill when coordination state matters. For lightweight terminal prompts or basic worktree/terminal/built-in-browser control, use `orca-cli`.\n\n## Tool Boundary\n\nIf a task says to use Orca orchestration, the coordinator must create Orca runtime state with `orca orchestration task-create` and `orca orchestration dispatch --inject` or `orca orchestration run`.\n\nDo not substitute non-Orca subagent tools, generic agent-spawn APIs, or chat-only parallel worker features. Those may create useful workers, but they do not create Orca task/dispatch provenance, injected lifecycle preambles, `worker_done` authority, or decision gates.\n\nBefore claiming a worker was orchestrated, verify the task/dispatch exists:\n\n```bash\norca orchestration task-list --json\norca orchestration dispatch-show --task --json\n```\n\nIf the work was accidentally run outside Orca orchestration, say so plainly. To repair provenance, rerun or revalidate the needed work through a fresh Orca terminal plus injected dispatch; do not retroactively describe the external worker as orchestrated.\n\n## When To Use\n\n- Send/reply/ask between agent terminals with persistent messages.\n- Dispatch structured tasks to workers and wait for `worker_done` or `escalation`.\n- Track task DAGs with dependencies.\n- Run coordinator loops or decision gates.\n\nDo not use orchestration merely because the user says \"hand off\", \"handoff\", \"handover\", \"give this to another agent\", or asks for another worktree/agent/model/effort. Those are full ownership transfers unless the user explicitly asks to supervise, monitor, wait for worker completion/results, coordinate a DAG, use decision gates, or keep a blocking ask/reply loop.\n\n## Preconditions\n\n- `orca status --json` should show a running runtime.\n- `orca` must be on PATH (`orca-ide` on Linux).\n- The orchestration experimental feature must be enabled in Settings > Experimental.\n- `orca orchestration` commands are RPC calls to the running Orca runtime.\n\n## Ownership\n\nOrchestration messages and tasks are runtime-global. Lifecycle authority comes from the payload `taskId` + `dispatchId` of the active dispatch, verified against the dispatched pane. Terminal handles are routing metadata — a pane can receive a new handle after restart — so never accept or reject lifecycle provenance by comparing handles. Send `worker_done` and `heartbeat` from the worker's own terminal; the runtime ignores them when sent from a different pane.\n\nClassify inherited context before sending lifecycle messages:\n\n- Coordinated subtask: a live coordinator owns the DAG and waits on this dispatch. Follow the preamble exactly, including `worker_done`, heartbeat/status, `ask`, and `escalation`.\n- Full handoff means ownership transfer, not supervised dispatch. The original actor is not monitoring a DAG, so do not create lifecycle obligations unless the user explicitly asks you to supervise.\n- Classify requests containing \"hand off\", \"handoff\", \"handover\", \"give this to another agent\", \"give this to another worktree\", \"another agent\", or \"another worktree\" as full handoffs by default, even when the user names a custom model or reasoning effort.\n- Use supervised orchestration only when the user explicitly asks you to \"supervise\", \"monitor\", \"wait\", \"track completion\", \"wait for worker_done\", return results, coordinate a DAG, use a decision gate, or manage ask/reply flow.\n- Do not use `orca orchestration dispatch --inject` for full handoffs. It injects a coordinator preamble that tells the worker to send `worker_done`, heartbeat, and `ask` messages, then end its turn under the original terminal's dispatch lifecycle.\n- Do not run `orca orchestration task-create`, `orca orchestration dispatch --inject`, or `orca orchestration check --wait` for full handoffs. Do not peek at terminal output after prompt delivery to monitor progress.\n- A review-only `worker_done` reports findings; it does not authorize coordinator file edits. After a review-only completion, synthesize findings, ask a decision gate if ownership is unclear, and dispatch or hand off fixes unless the user explicitly asked the coordinator to own fixes.\n- If the user's plan names a next owner agent (for example, \"then use opencode to create a PR\"), post-review corrections and PR prep belong to that named owner. The coordinator routes, synthesizes, asks decision gates when needed, and supervises; the named owner edits files and creates the PR.\n\nIf unclear, inspect orchestration state before sending lifecycle messages:\n\n```bash\norca orchestration task-list --json\norca terminal list --json\n# If inherited context includes a task id:\norca orchestration dispatch-show --task --json\n```\n\n## Messaging\n\n```bash\norca orchestration send --to --subject [--from ] [--body ] [--type ] [--priority ] [--thread-id ] [--payload ] [--json]\norca orchestration check [--terminal ] [--unread|--peek|--all] [--types ] [--inject] [--wait] [--timeout-ms ] [--json]\norca orchestration reply --id --body [--from ] [--json]\norca orchestration ask --to --question [--options ] [--timeout-ms ] [--from ] [--json]\norca orchestration inbox [--limit ] [--json]\n```\n\nRules:\n\n- Omit `--from` unless impersonating another terminal; Orca auto-resolves it from the current terminal.\n- `check` and `check --unread` return unread matches and mark them read. Use `--peek` for unread matches without consuming them; use `--all` for read and unread history without consuming anything. If an older CLI rejects `--peek` as an unknown flag, use `--all` and filter unread rows yourself.\n- Message **one** live agent handle per worker. Use `startupTerminal.handle` from the create response when present; if it is missing or later returns `terminal_handle_stale`, re-resolve with `orca terminal list --worktree ... --json` and continue with the replacement only.\n- `orca orchestration check --unread --inject --json` renders unread mail for the agent terminal that runs it; it does not remotely wake another terminal. Use `orchestration dispatch --inject` to deliver a tracked task, or `terminal send` when an existing agent needs a free-form prompt.\n- While supervising workers manually, use `check --wait --types worker_done,escalation,decision_gate --timeout-ms ` instead of sleep/poll loops. Reply to `decision_gate` messages with `orca orchestration reply --id --body --json`, then keep waiting.\n- Treat a `check --wait` timeout or `{count:0}` as a checkpoint, not a worker failure. Long coding tasks routinely run 15-60 minutes; keep using rolling waits unless you receive `worker_done`/`escalation`, the terminal exits or disappears, or the user explicitly asks you to stop.\n- Heartbeats and visible terminal activity mean the worker is alive, not done. Do not stop, close, kill, or restart a worker just because it has not produced a completion message yet.\n- Use `ask` when a worker needs a blocking answer from the coordinator; it waits for the reply and returns the answer directly.\n- `check --wait` returns one message at a time. If N workers may finish together, loop N times and dispatch newly ready tasks after each completion.\n- Group addresses include `@all`, `@idle`, `@claude`, `@codex`, `@opencode`, `@gemini`, `@droid`, `@grok`, `@cursor`, and `@worktree:`.\n- Message types include `status`, `dispatch`, `worker_done`, `merge_ready`, `escalation`, `handoff`, `decision_gate`, and `heartbeat`.\n- Use group addresses only for messages that are genuinely useful to many terminals, such as `status` broadcasts or intentional fan-out questions. Do not send dispatch lifecycle messages to groups.\n- `worker_done` must target the concrete coordinator handle from the live preamble. It is completion authority for one dispatch; group fanout would create false lifecycle mail in unrelated terminals.\n- A valid `worker_done` for the active `taskId` + `dispatchId` marks the task and dispatch completed automatically. Do not follow it with `task-update --status completed`; reserve manual updates for explicit recovery or overrides.\n- `heartbeat` is also dispatch-scoped. Send it only to the concrete coordinator handle with both `taskId` and `dispatchId`; use `status` for broad progress updates.\n\n## Tasks And Dispatch\n\nA task is the work item, a dispatch assigns it to a terminal, and a gate blocks progress until a coordinator or user decision is recorded.\n\n```bash\norca orchestration task-create --spec [--deps ] [--parent ] [--json]\norca orchestration task-list [--status ] [--ready] [--brief] [--json]\norca orchestration task-update --id --status [--result ] [--json]\norca orchestration dispatch --task --to [--from ] [--inject] [--json]\norca orchestration dispatch-show --task [--json]\n```\n\nTask statuses: `pending`, `ready`, `dispatched`, `completed`, `failed`, `blocked`.\n\nDispatch rules:\n\n- `--inject` sends the task spec plus preamble into a recognized agent CLI so it can report `worker_done`.\n- If the target is a bare shell, omit `--inject`, dispatch for tracking if needed, then send the prompt manually with `orca terminal send --terminal --text --enter --json`.\n- After 3 consecutive failures on one task, the dispatch context circuit-breaks and the task is marked failed.\n- Use `task-list --brief --json` for coordinator sweeps; it collapses whitespace and caps each echoed spec at 160 characters (`spec_truncated` marks shortened rows). Omit `--brief` when the full spec is required, or when an older CLI rejects it as an unknown flag.\n\n## Gates And Coordinator\n\n```bash\norca orchestration gate-create --task --question [--options ] [--json]\norca orchestration gate-resolve --id --resolution [--json]\norca orchestration gate-list [--task ] [--status ] [--json]\norca orchestration run --spec [--from ] [--poll-interval-ms ] [--max-concurrent ] [--worktree ] [--json]\norca orchestration run-stop [--json]\n```\n\n`run` returns immediately with a run ID. Query progress with `task-list`. Use `ask` for worker-to-coordinator questions; it creates a `decision_gate` message that the coordinator answers with `reply`. Use `gate-create` only for coordinator-managed task DAG decisions, not for answering a worker's `ask`.\n\nRecovery only: `orca orchestration reset --tasks|--messages|--all --json` clears runtime-global orchestration state. Do not run it during active coordination unless explicitly abandoning that state.\n\n## Full Handoffs\n\nFor full ownership transfer, use non-lifecycle terminal/worktree commands and then stop monitoring unless the user asks for supervision.\n\nTreat these as full handoff requests by default: \"hand off\", \"handoff\", \"handover\", \"give this to another agent\", \"give this to another worktree\", \"send this to another agent\", \"another agent\", \"another worktree\", or \"launch another agent to own this.\" Custom model or reasoning effort words such as `gpt-5.5`, `high`, or `xhigh` do not make the handoff supervised.\n\nSupervised orchestration remains available only when the user explicitly asks for supervision or coordination: \"supervise\", \"monitor\", \"wait for worker_done\", \"wait for results\", \"track completion\", \"DAG\", \"decision gate\", \"ask/reply\", or \"coordinate workers.\"\n\nDo not run `orca orchestration task-create`, `orca orchestration dispatch --inject`, or `orca orchestration check --wait` for full handoffs. `task-create` is also forbidden because it records coordinator-owned tracking state; if a task row is needed, the user asked for supervised orchestration. Do not create a `taskId`/`dispatchId`, inject a lifecycle preamble, wait for completion, or read the worker terminal after prompt delivery except to avoid losing the initial prompt.\n\nNew top-level worktree handoff:\n\n```bash\norca worktree create --name --no-parent --agent codex --prompt \"\" --json\n```\n\nBefore creating a new worktree from an active feature branch, decide and state whether the desired Orca lineage is child or top-level. Use child worktree lineage only when the new work is conceptually stacked under or dependent on the active worktree. For independent repo-wide fixes, standalone feature work, or unrelated follow-up tasks, create a top-level worktree with `--no-parent`.\n\nExisting terminal handoff:\n\n```bash\norca terminal send --terminal --text \"\" --enter --json\n```\n\nCustom Codex model/effort handoff:\n\n`orca worktree create --agent codex --prompt ...` launches the known Codex agent but does not accept Codex-specific `--model` or `-c model_reasoning_effort=...` arguments. When the user asks for a specific Codex model or effort, create the independent worktree first, launch Codex with the requested command in that worktree, wait only for TUI readiness if prompt delivery would otherwise race startup, send the prompt, and stop.\n\nNote: when no repo default-terminal configuration supplies a primary terminal, bare create opens a fallback shell before `terminal create` adds the agent. Configured default tabs are materialized instead and may run real commands. Prefer `--agent` whenever custom argv is not required. With the two-step path, target only the agent handle; close a prior terminal only after `terminal list` or `terminal show` confirms it is an unused shell.\n\nUse the exact full `::` worktree id returned by `orca worktree create --json`; a bare repo id cannot target the new worktree.\n\n```bash\norca worktree create --name --no-parent --json\norca terminal create --worktree id: --title --command 'codex --model gpt-5.5 -c model_reasoning_effort=\"xhigh\"' --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca terminal send --terminal --text \"\" --enter --json\n```\n\nWait only for `tui-idle` when needed to avoid losing the prompt. Do not monitor task completion.\n\n`--no-parent` only controls Orca lineage; it does not choose the Git base. If the work should start from the repo default base, omit `--base-branch` so Orca uses that default, or explicitly pass the repo default base (`origin/main`, `origin/master`, or the `orca repo show --repo --json` value); never base it on the current feature branch unless the user explicitly asks for stacked work or \"branch from current\". Put current-branch context in the prompt instead.\n\n## Worker Terminals\n\nChoose the worker location before creating a terminal. `Fresh worker` means a fresh agent session, not a new git worktree. For parallel work, create one fresh agent terminal per worker in the same required worktree, falling back to the active worktree when none is named. If the task says current worktree only, depends on uncommitted files/artifacts, or must validate/PR the current branch, keep every worker in the active worktree:\n\n```bash\norca terminal create --worktree active --title --command \"codex\" --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca orchestration dispatch --task --to --inject --json\n```\n\nReuse an idle agent in the required worktree only if the prompt allows reuse; otherwise create a fresh terminal there. Create a new worktree only when the user explicitly requests one or a concrete checkout or filesystem conflict makes sharing unsafe or impossible; if the user did not request it, state that conflict before running `worktree create`. Independent tasks, parallel execution, convenience, or a preference for separate checkouts are not isolation requirements.\n\nWhen a new worktree is allowed, use child lineage for isolated work that is stacked under or dependent on the active worktree, and use `--no-parent` when it is not stacked. Decide the Git base separately: `--no-parent` makes the worktree top-level in Orca, while omitted `--base-branch` uses the repo default base.\n\n```bash\norca worktree create --name --agent codex --json\n# or: --agent claude | omp | pi | grok | ...\n# Read from startupTerminal.handle in the create response.\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca orchestration dispatch --task --to --inject --json\n```\n\nFor new-worktree workers, read the id and `startupTerminal.handle` from `worktree create`. Use that as the sole worker handle when present; otherwise use `terminal list` to resolve the agent handle. Omit `--repo` only inside an Orca-managed worktree; otherwise pass `--repo `.\n\n**For an allowed new worktree, use agent-first:** `--agent` reveals the new worktree and launches the selected agent **in its first terminal**, without adding a separate fallback shell for that worker. Repo setup or default-terminal settings may still add tabs or splits. Do **not** run bare `worktree create` and then `terminal create --command ` for the same worker when agent-first create is available: without configured default tabs, that two-step path leaves a fallback shell + agent pair. Only use it when custom agent argv is required (for example Codex model/effort flags) or when an older CLI rejects `--agent`; if you must, message only the agent handle. Configured default tabs are intentional surfaces, so close a prior terminal only after `terminal list` or `terminal show` confirms it is an unused shell. Do not run `worktree create` when the task must stay in the current worktree.\n\nUse `orca worktree create --prompt ...` or `orca terminal send ...` for full handoffs or untracked/lightweight prompts. Those paths do not attach `taskId`/`dispatchId`; the worker should not send lifecycle messages unless the prompt supplies a live orchestration preamble.\n\nSidebar lineage and orchestration lifecycle are related but not identical. A same-worktree worker may appear as a peer under that worktree in the sidebar while remaining a child dispatch in orchestration state; only an actual child worktree creates visible parent/child worktree lineage.\n\nOther terminal commands coordinators often need:\n\n```bash\norca terminal list [--worktree ] [--json]\norca terminal create [--worktree ] [--title ] [--command ] [--json]\norca terminal split --terminal [--direction horizontal|vertical] [--command ] [--json]\norca terminal wait --terminal --for tui-idle --timeout-ms --json\norca terminal read --terminal --json\norca terminal send --terminal --text --enter --json\n```\n\nIf an older CLI rejects `worktree create --agent`, create the worktree normally, then run `orca terminal create --worktree --command \"codex\" --json` or `--command \"claude\"`.\n\nWait for `tui-idle` before dispatching. Always pass `--timeout-ms`; real coding tasks can take 15-60 minutes. During supervision, use rolling `check --wait` windows. If a window returns no matching message, inspect `task-list`, `terminal read`, or `terminal wait --for tui-idle` as a liveness checkpoint; if the terminal is still working or producing activity, keep waiting instead of retrying the task.\n\n## Agent Guidance\n\n- Workers with a valid live preamble must send `worker_done` exactly once from their own terminal, even on failure:\n `orca orchestration send --to --type worker_done --subject \"\" --body \"<3-sentence summary: what you did, what you found, what's left>\" --payload '{\"taskId\":\"\",\"dispatchId\":\"\",\"filesModified\":[\"path/a\"],\"reportPath\":\"\"}' --json`\n- After sending `worker_done`, end your turn and idle at the agent prompt. Do not poll or keep calling `orca orchestration check`; the coordinator re-engages you with a fresh preamble + TASK block delivered as new terminal input.\n- For long tasks, send heartbeat/status only when the preamble asks for it, including both IDs:\n `orca orchestration send --to --type heartbeat --subject \"alive\" --payload '{\"taskId\":\"\",\"dispatchId\":\"\",\"phase\":\"implementing\"}' --json`\n- If blocked before completion, use `ask`; use `escalation` only when ownership is valid and the coordinator must intervene.\n- Treat preambles inherited through terminal history or full handoffs as stale unless the current prompt explicitly keeps that coordinator in the loop.\n- Coordinators should use `task-list --ready` as external memory, dispatch parallel waves, and avoid dependency chains deeper than 3-4 steps.\n\n## Example\n\n```bash\norca terminal create --worktree active --title login-css-worker --command \"claude\" --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca orchestration task-create --spec \"Fix the login button CSS\" --json\norca orchestration dispatch --task --to --inject --json\norca orchestration check --wait --types worker_done,escalation,decision_gate --timeout-ms 900000 --json\n```\n\n## Next Action\n\nCoordinator: confirm `orca status --json`, inspect `task-list`/`dispatch-show` if inheriting state, then choose either a manual loop (`task-create` -> worker -> `dispatch --inject` -> `check --wait`) or `orchestration run`.\n\nWorker: if the current prompt contains a live dispatch preamble, do the task, use `ask` for blocking questions, and send `worker_done` once with the required payload. If the preamble is stale or absent, do not send lifecycle messages; inspect state or treat the prompt as an ordinary handoff.\n" +const ORCHESTRATION_MARKDOWN = "---\nname: orchestration\ndescription: >-\n Use Orca orchestration for structured multi-agent coordination: threaded\n messages, blocking ask/reply flows, task dispatch, worker_done/escalation\n waits, task DAGs, decision gates, coordinator loops, or decomposing work\n across agents. Use `orca-cli` instead for full ownership handoffs, including\n requests phrased as \"hand off\", \"handoff\", \"handover\", \"give this to another\n agent\", or \"another worktree\" when the user did not explicitly ask to\n supervise, monitor, wait for results, or coordinate a DAG. Use `orca-cli` for\n ordinary terminal control, lightweight terminal prompts, shell commands, Orca\n worktree management, reading or waiting on terminals, and automation of the\n browser embedded inside Orca. Use Computer Use for browser windows, webviews,\n Orca app UI, or desktop UI outside Orca's embedded browser.\n---\n\n# Orca Inter-Agent Orchestration\n\nOrchestration is Orca's structured coordination layer for agent messages, task ownership, dispatch state, and worker completion tracking.\n\nUse this skill when coordination state matters. For lightweight terminal prompts or basic worktree/terminal/built-in-browser control, use `orca-cli`.\n\n## Tool Boundary\n\nIf a task says to use Orca orchestration, the coordinator must create or bind a Run, create the Task with `orca orchestration task-create`, then attach the worker with either the preferred `orca orchestration worker-start` composition or the low-level `orca orchestration dispatch --inject` path.\n\nDo not substitute non-Orca subagent tools, generic agent-spawn APIs, or chat-only parallel worker features. Those may create useful workers, but they do not create Orca task/dispatch provenance, injected lifecycle preambles, `worker_done` authority, or decision gates.\n\nBefore claiming a worker was orchestrated, verify the task/dispatch exists:\n\n```bash\norca orchestration task-list --json\norca orchestration dispatch-show --task --json\n```\n\nIf the work was accidentally run outside Orca orchestration, say so plainly. To repair provenance, rerun or revalidate the needed work through a fresh Orca terminal plus injected dispatch; do not retroactively describe the external worker as orchestrated.\n\n## When To Use\n\n- Send/reply/ask between agent terminals with persistent messages.\n- Dispatch structured tasks to workers and wait for `worker_done` or `escalation`.\n- Track task DAGs with dependencies.\n- Run coordinator loops or decision gates.\n\nDo not use orchestration merely because the user says \"hand off\", \"handoff\", \"handover\", \"give this to another agent\", or asks for another worktree/agent/model/effort. Those are full ownership transfers unless the user explicitly asks to supervise, monitor, wait for worker completion/results, coordinate a DAG, use decision gates, or keep a blocking ask/reply loop.\n\n## Preconditions\n\n- `orca status --json` should show a running runtime.\n- `orca` must be on PATH (`orca-ide` on Linux).\n- The orchestration experimental feature must be enabled in Settings > Experimental.\n- `orca orchestration` commands are RPC calls to the running Orca runtime.\n\n## Contract Migration\n\nOrca uses a hard cutover for orchestration mutations. It does not run a legacy scheduler, translate old writes, or drain pre-upgrade orchestration work.\n\nIf a command returns `orchestration_migration_required`, `run_required`, or a lifecycle validation error with `nextCommandArgs`:\n\n1. Confirm `effectsApplied` is `false`.\n2. Using the same CLI executable that returned the error, run the returned arguments: `skills get orchestration --full`.\n3. Read the guide completely. Do not retry the rejected command unchanged.\n4. Create or bind a lightweight Run, then restart the work using Run -> Task -> `worker-start`.\n5. Inspect any pre-upgrade terminal before creating replacement work.\n\nThe arguments intentionally omit an executable name so this works with `orca`, `orca-ide`, `orca-dev`, or another configured Orca CLI command.\n\nPre-upgrade terminals and agents are not killed during upgrade, but they are no longer supervised: old heartbeat, question, completion, scheduler, and mutation calls are rejected before effects. Legacy database rows remain available only for explicit inspection:\n\n```bash\norca orchestration run-list --json\norca orchestration run-show --id run_legacy_local --json\norca orchestration task-list --run run_legacy_local --json\norca orchestration inbox --full --json\norca orchestration check --terminal --peek --json\n```\n\nRead-only inspection never consumes legacy mail. Do not use actionable `check`, acknowledgment, send, retry, or task updates against the legacy Run.\n\n## Ownership\n\nNew orchestration messages and tasks belong to one explicitly bound Run. A Run is only a durable namespace and coordinator inbox; it never schedules or places workers. Lifecycle authority comes from the active Dispatch, and terminal handles remain routing metadata rather than durable identity. Send `worker_done` and `heartbeat` from the worker's own terminal; Orca routes them to that Dispatch's Run.\n\nClassify inherited context before sending lifecycle messages:\n\n- Coordinated subtask: a live coordinator owns the DAG and waits on this dispatch. Follow the preamble exactly, including `worker_done`, heartbeat/status, `ask`, and `escalation`.\n- Full handoff means ownership transfer, not supervised dispatch. The original actor is not monitoring a DAG, so do not create lifecycle obligations unless the user explicitly asks you to supervise.\n- Classify requests containing \"hand off\", \"handoff\", \"handover\", \"give this to another agent\", \"give this to another worktree\", \"another agent\", or \"another worktree\" as full handoffs by default, even when the user names a custom model or reasoning effort.\n- Use supervised orchestration only when the user explicitly asks you to \"supervise\", \"monitor\", \"wait\", \"track completion\", \"wait for worker_done\", return results, coordinate a DAG, use a decision gate, or manage ask/reply flow.\n- Do not use `orca orchestration dispatch --inject` for full handoffs. It injects a coordinator preamble that tells the worker to send `worker_done`, heartbeat, and `ask` messages, then end its turn under the original terminal's dispatch lifecycle.\n- Do not run `orca orchestration task-create`, `orca orchestration dispatch --inject`, or `orca orchestration check --wait` for full handoffs. Do not peek at terminal output after prompt delivery to monitor progress.\n- A review-only `worker_done` reports findings; it does not authorize coordinator file edits. After a review-only completion, synthesize findings, ask a decision gate if ownership is unclear, and dispatch or hand off fixes unless the user explicitly asked the coordinator to own fixes.\n- If the user's plan names a next owner agent (for example, \"then use opencode to create a PR\"), post-review corrections and PR prep belong to that named owner. The coordinator routes, synthesizes, asks decision gates when needed, and supervises; the named owner edits files and creates the PR.\n\nIf unclear, inspect orchestration state before sending lifecycle messages:\n\n```bash\norca orchestration task-list --json\norca terminal list --json\n# If inherited context includes a task id:\norca orchestration dispatch-show --task --json\n```\n\n## Messaging\n\n```bash\norca orchestration send --subject [--to ] [--from ] [--body ] [--type ] [--priority ] [--thread-id ] [--payload ] [--json]\norca orchestration check [--terminal ] [--ack ] [--peek|--all] [--types ] [--format] [--wait] [--timeout-ms ] [--json]\norca orchestration reply --id --body [--from ] [--json]\norca orchestration ask (--question |--resume ) [--options ] [--timeout-ms ] [--from ] [--json]\norca orchestration inbox [--limit ] [--json]\n```\n\nRules:\n\n- Omit `--from` unless impersonating another terminal; Orca auto-resolves it from the current terminal.\n- A coordinator `check` returns the bound Run's oldest FIFO Delivery (up to 50 messages) and replays that exact batch until `--ack `. Process every message before acknowledging; `check --ack --wait` acknowledges, checks, and waits in one operation.\n- Use `--peek` and `--all` only for read-only history/debugging. Type filters decide when a waiter wakes; the returned actionable Delivery is still the oldest full batch.\n- Use `dispatch:` for coordinator guidance to one supervised worker. Orca routes that stable address locally or through the connected-server relay; do not substitute a remote terminal handle.\n- Terminal handles remain appropriate for low-level pre-Dispatch messaging. Prefer `agentTerminalHandle` from the create response, fall back to `startupTerminal.handle` for older runtimes, then re-resolve with `orca terminal list --worktree ... --json` if missing or stale. Continue with the replacement handle only; never dual-send to old and new handles.\n- `orca orchestration check --peek --format --json` returns locally formatted unread mail without consuming it; it never writes to terminal input or remotely wakes another terminal. Use `orchestration dispatch --inject` to deliver a tracked task, or `terminal send` when an existing agent needs a free-form prompt.\n- While supervising workers manually, use `check --wait --types worker_done,escalation,question --timeout-ms ` instead of sleep/poll loops. Process the whole Delivery, reply to `question` messages with `orca orchestration reply --id --body --json`, then acknowledge and keep waiting.\n- Treat a `check --wait` timeout or `{count:0}` as a checkpoint, not a worker failure. Long coding tasks routinely run 15-60 minutes; keep using rolling waits unless you receive `worker_done`/`escalation`, the terminal exits or disappears, or the user explicitly asks you to stop.\n- Heartbeats and visible terminal activity mean the worker is alive, not done. Do not stop, close, kill, or restart a worker just because it has not produced a completion message yet.\n- Use `ask` when a worker needs a blocking answer from the coordinator; it defaults to the active Dispatch's Run. Timeout or disconnect leaves the question pending, so resume by its original message ID instead of asking again.\n- `check --wait` returns one bounded Delivery, not every future completion. Process every message, acknowledge it, then keep waiting until every expected Dispatch settles.\n- Group addresses include `@all`, `@idle`, `@claude`, `@codex`, `@opencode`, `@gemini`, `@droid`, `@grok`, `@cursor`, and `@worktree:`.\n- Message types include `status`, `dispatch`, `worker_done`, `merge_ready`, `escalation`, `handoff`, `question`, `decision_gate` (legacy/gates), and `heartbeat`.\n- Use group addresses only for messages that are genuinely useful to many terminals, such as `status` broadcasts or intentional fan-out questions. Do not send dispatch lifecycle messages to groups.\n- `worker_done` belongs to the active Dispatch and defaults to its Run mailbox; never target a group.\n- A valid `worker_done` for the active `taskId` + `dispatchId` marks the task and dispatch completed automatically. Do not follow it with `task-update --status completed`; reserve manual updates for explicit recovery or overrides.\n- `heartbeat` is also Dispatch-scoped. Include both IDs and omit `--to` so Orca uses the owning Run; use `status` for broad progress updates.\n\n## Tasks And Dispatch\n\nA Run is the namespace/inbox, a Task is the work item, and a Dispatch assigns one Task attempt to a terminal. Create or bind a Run once before the common loop.\n\n```bash\norca orchestration run-create --objective --json\norca orchestration task-create --spec [--deps ] [--parent ] [--json]\norca orchestration task-list [--status ] [--ready] [--brief] [--json]\norca orchestration task-update --id --status [--result ] [--json]\norca orchestration dispatch --task --to [--from ] [--inject] [--json]\norca orchestration dispatch-show --task [--json]\n```\n\nTask statuses: `pending`, `ready`, `dispatched`, `completed`, `failed`, `blocked`.\n\nDispatch rules:\n\n- `--inject` sends the task spec plus preamble into a recognized agent CLI so it can report `worker_done`.\n- If the target is a bare shell, omit `--inject`, dispatch for tracking if needed, then send the prompt manually with `orca terminal send --terminal --text --enter --json`.\n- After 3 consecutive failures on one task, the dispatch context circuit-breaks and the task is marked failed.\n- Use `task-list --brief --json` for coordinator sweeps; it collapses whitespace and caps each echoed spec at 160 characters (`spec_truncated` marks shortened rows). Omit `--brief` when the full spec is required, or when an older CLI rejects it as an unknown flag.\n\n## Preferred Supervised Worker Loop\n\nUse `worker-start` for the normal supervised path. It composes the existing worktree, terminal, readiness, and dispatch primitives while returning exact created/reused effects. Agents still choose placement and concurrency; Orca does not schedule workers or infer conflicts.\n\nCreate the Run and every independent Task first, then start all independent workers before waiting:\n\n```bash\norca orchestration run-create --objective \"\" --json\norca orchestration task-create --spec \"\" --json\norca orchestration task-create --spec \"\" --json\norca orchestration worker-start --task --worktree current --agent codex --json\norca orchestration worker-start --task --worktree current --agent claude --json\n```\n\n`current` and exact existing worktrees create a fresh agent terminal and do not rerun setup. Reuse an existing agent only with `--terminal `.\n\nFor a new worktree, setup runs by default and agent-first creation reuses the returned startup agent terminal:\n\n```bash\norca orchestration worker-start --task --worktree new-child --name --agent codex --setup run --json\n# Independent/top-level:\norca orchestration worker-start --task --worktree new-top-level --name --agent codex --setup run --json\n```\n\nSetup normally starts alongside the agent. Only a repository explicitly configured with `wait-for-setup` delays agent launch until setup succeeds. Use `--setup skip` or `--setup inherit` only for a concrete reason.\n\nRead the returned receipt before continuing: `ready` plus setup `running` is normal for start-immediately, while wait-for-setup returns setup `succeeded` before accepting task input. A failed or unknown start exits nonzero; inspect its `stage`, `effects`, and `residualResources` instead of guessing or automatically retrying. A wait-for-setup timeout can honestly leave setup `running`, which is not proof of failure.\n\nTo run the worker on another connected Orca server, add `--on `. The Run and Tasks remain authoritative on the current server; later commands route by Dispatch ID, so never repeat `--on`:\n\n```bash\n# Mac Run home -> Windows worker (the reverse is identical from a Windows Run home)\norca orchestration worker-start --task --on windows --worktree new-top-level --repo --name --agent codex --setup run --json\norca orchestration worker-show --dispatch --json\norca orchestration worker-read --dispatch --limit 50 --json\norca orchestration send --to dispatch: --subject \"Follow-up\" --body \"\" --json\n```\n\nRemote `current` and `new-child` are intentionally invalid because those words are ambiguous across servers. Use an exact discovered remote worktree selector or `new-top-level` with an explicit remote repo selector.\n\nThe follow-up is structured inbox mail, not prompt injection. The worker's next\n`orchestration check` receives it even when the Dispatch is on another connected Orca server.\n\n`worker-read` defaults to `--source auto`: Orca returns the exact hook-reported Codex, Claude, OpenClaude, or Grok transcript when it can prove the worker session, otherwise it returns bounded terminal output with `source: \"terminal\"` and a typed `fallbackReason`. Continue with the returned top-level `cursor`; it stays pinned to that exact source. If Orca reports `source_changed`, start a fresh read without the old cursor. Never supply or guess a provider session ID or transcript path.\n\nWait until every expected Dispatch settles, not for a fixed number of batches:\n\n```bash\norca orchestration check --wait --types worker_done,escalation,question --timeout-ms 900000 --json\n# Process every message in the returned Delivery, then atomically ack and continue:\norca orchestration check --ack --wait --types worker_done,escalation,question --timeout-ms 900000 --json\n```\n\nWorkers report exactly once using the IDs and capability injected by Orca; they do not supply Run/server/terminal identity:\n\n```bash\norca orchestration send --type worker_done --subject \"\" --body \"\" --task-id --dispatch-id --outcome succeeded --files-modified \"path/a,path/b\" --json\n# On failure, use --outcome failed; never encode failure only in prose.\n```\n\nA worker question defaults to its owning Run. Timeout leaves it pending:\n\n```bash\norca orchestration ask --question \"\" --options \"yes,no\" --timeout-ms 600000 --json\norca orchestration ask --resume --timeout-ms 600000 --json\n# Coordinator:\norca orchestration reply --id --body \"\" --json\n```\n\nRecovery is conditional, never a fixed destructive sequence:\n\n- `worker-show --dispatch ` says `ready`: keep waiting or read bounded output.\n- It proves `failed` or `stopped`: start a replacement with `worker-start --task --retry-of ` plus an explicit `--on`/`--worktree` and `--agent`/`--terminal` choice. Retry does not silently inherit placement.\n- It remains `outcome_unknown`: either `worker-stop --dispatch ` and inspect again, or explicitly `worker-abandon --dispatch ` while accepting that resources may still be live. Abandon performs no remote, process, or filesystem action.\n- `worker-stop` closes only the exact supervised agent terminal. It never deletes the worktree, setup terminal, configured tabs, or unrelated processes.\n\nLow-level `worktree create`, `terminal create`, and `dispatch --inject` remain valid recipes for custom argv or topology that `worker-start` does not express.\n\n## Gates And Legacy Inspection\n\n```bash\norca orchestration gate-create --task --question [--options ] [--json]\norca orchestration gate-resolve --id --resolution [--json]\norca orchestration gate-list [--task ] [--status ] [--json]\n```\n\nUse `ask` for worker-to-coordinator questions; it creates a `question` message that the coordinator answers with `reply`. Use `gate-create` only for coordinator-managed task DAG decisions, not for answering a worker's `ask`.\n\n`coordinator-start`, `coordinator-stop`, `run`, and `run-stop` are retired scheduler commands. They perform no effects and return the current-skill recovery action. They are not aliases for lightweight Run creation or binding.\n\nRecovery only: `orca orchestration reset --tasks|--messages|--all --json` clears the selected local orchestration database state. Do not run it during active coordination unless explicitly abandoning that state.\n\n## Full Handoffs\n\nFor full ownership transfer, use non-lifecycle terminal/worktree commands and then stop monitoring unless the user asks for supervision.\n\nTreat these as full handoff requests by default: \"hand off\", \"handoff\", \"handover\", \"give this to another agent\", \"give this to another worktree\", \"send this to another agent\", \"another agent\", \"another worktree\", or \"launch another agent to own this.\" Custom model or reasoning effort words such as `gpt-5.5`, `high`, or `xhigh` do not make the handoff supervised.\n\nSupervised orchestration remains available only when the user explicitly asks for supervision or coordination: \"supervise\", \"monitor\", \"wait for worker_done\", \"wait for results\", \"track completion\", \"DAG\", \"decision gate\", \"ask/reply\", or \"coordinate workers.\"\n\nDo not run `orca orchestration task-create`, `orca orchestration dispatch --inject`, or `orca orchestration check --wait` for full handoffs. `task-create` is also forbidden because it records coordinator-owned tracking state; if a task row is needed, the user asked for supervised orchestration. Do not create a `taskId`/`dispatchId`, inject a lifecycle preamble, wait for completion, or read the worker terminal after prompt delivery except to avoid losing the initial prompt.\n\nNew top-level worktree handoff:\n\n```bash\norca worktree create --name --no-parent --agent codex --prompt \"\" --setup run --json\n```\n\nBefore creating a new worktree from an active feature branch, decide and state whether the desired Orca lineage is child or top-level. Use child worktree lineage only when the new work is conceptually stacked under or dependent on the active worktree. For independent repo-wide fixes, standalone feature work, or unrelated follow-up tasks, create a top-level worktree with `--no-parent`.\n\nExisting terminal handoff:\n\n```bash\norca terminal send --terminal --text \"\" --enter --json\n```\n\nCustom Codex model/effort handoff:\n\n`orca worktree create --agent codex --prompt ...` launches the known Codex agent but does not accept Codex-specific `--model` or `-c model_reasoning_effort=...` arguments. When the user asks for a specific Codex model or effort, create the independent worktree first, launch Codex with the requested command in that worktree, wait only for TUI readiness if prompt delivery would otherwise race startup, send the prompt, and stop.\n\nThe two-step custom-argv path cannot enforce a repository's explicit `wait-for-setup` startup policy because the later `terminal create` is not the startup owned by `worktree create`. Use it only when the repository starts agents immediately. If the repository requires `wait-for-setup`, use an agent-first configured launcher that can preserve sequencing, or stop and ask rather than silently bypassing the policy.\n\nNote: when no repo default-terminal configuration supplies a primary terminal, bare create opens a fallback shell before `terminal create` adds the agent. Configured default tabs are materialized instead and may run real commands. Prefer `--agent` whenever custom argv is not required. With the two-step path, target only the agent handle; close a prior terminal only after `terminal list` or `terminal show` confirms it is an unused shell.\n\nUse the exact full `::` worktree id returned by `orca worktree create --json`; a bare repo id cannot target the new worktree.\n\n```bash\norca worktree create --name --no-parent --setup run --json\norca terminal create --worktree id: --title --command 'codex --model gpt-5.5 -c model_reasoning_effort=\"xhigh\"' --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca terminal send --terminal --text \"\" --enter --json\n```\n\nWait only for `tui-idle` when needed to avoid losing the prompt. Do not monitor task completion.\n\n`--no-parent` only controls Orca lineage; it does not choose the Git base. If the work should start from the repo default base, omit `--base-branch` so Orca uses that default, or explicitly pass the repo default base (`origin/main`, `origin/master`, or the `orca repo show --repo --json` value); never base it on the current feature branch unless the user explicitly asks for stacked work or \"branch from current\". Put current-branch context in the prompt instead.\n\n## Worker Terminals\n\nChoose the worker location before creating a terminal. `Fresh worker` means a fresh agent session, not a new git worktree. For parallel work, create one fresh agent terminal per worker in the same required worktree, falling back to the active worktree when none is named. If the task says current worktree only, depends on uncommitted files/artifacts, or must validate/PR the current branch, keep every worker in the active worktree:\n\n```bash\norca terminal create --worktree active --title --command \"codex\" --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca orchestration dispatch --task --to --inject --json\n```\n\nReuse an idle agent in the required worktree only if the prompt allows reuse; otherwise create a fresh terminal there. Create a new worktree only when the user explicitly requests one or a concrete checkout or filesystem conflict makes sharing unsafe or impossible; if the user did not request it, state that conflict before running `worktree create`. Independent tasks, parallel execution, convenience, or a preference for separate checkouts are not isolation requirements.\n\nWhen a new worktree is allowed, use child lineage for isolated work that is stacked under or dependent on the active worktree, and use `--no-parent` when it is not stacked. Decide the Git base separately: `--no-parent` makes the worktree top-level in Orca, while omitted `--base-branch` uses the repo default base.\n\nFor every new worktree, pass `--setup run` so any configured repository setup hook runs. This does not mean waiting for setup before agent launch: preserve the repository's startup policy, whose default starts setup and the agent side by side. Use `--setup skip` or `--setup inherit` only when there is a concrete task-specific reason, and state that reason before creating the worktree. This rule does not rerun setup for current or existing worktrees.\n\n```bash\norca worktree create --name --agent codex --setup run --json\n# or: --agent claude | omp | pi | grok | ...\n# Read from agentTerminalHandle, falling back to startupTerminal.handle.\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca orchestration dispatch --task --to --inject --json\n```\n\nFor new-worktree workers, read the id and `agentTerminalHandle` from `worktree create`, falling back to `startupTerminal.handle` for older runtimes. Use that as the sole worker handle when present; otherwise use `terminal list` to resolve the agent handle. Omit `--repo` only inside an Orca-managed worktree; otherwise pass `--repo `.\n\n**For an allowed new worktree, use agent-first:** `--agent` reveals the new worktree and launches the selected agent **in its first terminal**, without adding a separate fallback shell for that worker. Pass `--setup run`; repo setup and default-terminal settings may add intentional tabs or splits. Do **not** run bare `worktree create` and then `terminal create --command ` for the same worker when agent-first create is available: without configured default tabs, that two-step path leaves a fallback shell + agent pair. Only use it when custom agent argv is required (for example Codex model/effort flags) or when an older CLI rejects `--agent`; if you must, message only the agent handle. Configured default tabs are intentional surfaces, so close a prior terminal only after `terminal list` or `terminal show` confirms it is an unused shell. Do not run `worktree create` when the task must stay in the current worktree.\n\nUse `orca worktree create --prompt ...` or `orca terminal send ...` for full handoffs or untracked/lightweight prompts. Those paths do not attach `taskId`/`dispatchId`; the worker should not send lifecycle messages unless the prompt supplies a live orchestration preamble.\n\nSidebar lineage and orchestration lifecycle are related but not identical. A same-worktree worker may appear as a peer under that worktree in the sidebar while remaining a child dispatch in orchestration state; only an actual child worktree creates visible parent/child worktree lineage.\n\nOther terminal commands coordinators often need:\n\n```bash\norca terminal list [--worktree ] [--json]\norca terminal create [--worktree ] [--title ] [--command ] [--json]\norca terminal split --terminal [--direction horizontal|vertical] [--command ] [--json]\norca terminal wait --terminal --for tui-idle --timeout-ms --json\norca terminal read --terminal --json\norca terminal send --terminal --text --enter --json\n```\n\nIf an older CLI rejects `worktree create --agent`, create the worktree normally, then run `orca terminal create --worktree --command \"codex\" --json` or `--command \"claude\"`.\n\nWait for `tui-idle` before dispatching. Always pass `--timeout-ms`; real coding tasks can take 15-60 minutes. During supervision, use rolling `check --wait` windows. If a window returns no matching message, inspect `task-list`, `terminal read`, or `terminal wait --for tui-idle` as a liveness checkpoint; if the terminal is still working or producing activity, keep waiting instead of retrying the task.\n\n## Agent Guidance\n\n- Workers with a valid live preamble must send `worker_done` exactly once from their own terminal with an explicit `--outcome succeeded` or `--outcome failed`:\n `orca orchestration send --type worker_done --subject \"\" --body \"<3-sentence summary: what you did, what you found, what's left>\" --task-id --dispatch-id --outcome succeeded --files-modified \"path/a\" --report-path \"\" --json`\n- A failed outcome is still a terminal report, but Orca records both the Dispatch and Task as failed. Never encode failure only in the subject/body.\n- After sending `worker_done`, end your turn and idle at the agent prompt. Do not poll or keep calling `orca orchestration check`; the coordinator re-engages you with a fresh preamble + TASK block delivered as new terminal input.\n- For long tasks, send heartbeat/status only when the preamble asks for it, including both IDs:\n `orca orchestration send --type heartbeat --subject \"alive\" --payload '{\"taskId\":\"\",\"dispatchId\":\"\",\"phase\":\"implementing\"}' --json`\n- If blocked before completion, use `ask`; use `escalation` only when ownership is valid and the coordinator must intervene.\n- Treat preambles inherited through terminal history or full handoffs as stale unless the current prompt explicitly keeps that coordinator in the loop.\n- Coordinators should use `task-list --ready` as external memory, dispatch parallel waves, and avoid dependency chains deeper than 3-4 steps.\n\n## Example\n\n```bash\norca terminal create --worktree active --title login-css-worker --command \"claude\" --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca orchestration task-create --spec \"Fix the login button CSS\" --json\norca orchestration dispatch --task --to --inject --json\norca orchestration check --wait --types worker_done,escalation,question --timeout-ms 900000 --json\n```\n\n## Next Action\n\nCoordinator: confirm `orca status --json`, create or bind a Run, inspect `task-list`/`dispatch-show` if inheriting state, then use the explicit supervised loop (`task-create` -> `worker-start` -> `check --wait`). Use low-level terminal creation plus `dispatch --inject` only when the composed start does not express the needed topology.\n\nWorker: if the current prompt contains a live dispatch preamble, do the task, use `ask` for blocking questions, and send `worker_done` once with the required payload. If the preamble is stale or absent, do not send lifecycle messages; inspect state or treat the prompt as an ordinary handoff.\n" // Why: no current guide has bundled reference documents, so --full is byte-identical for now. // oxfmt-ignore diff --git a/src/cli/format.test.ts b/src/cli/format.test.ts index 7117b169512..ce26baeda1a 100644 --- a/src/cli/format.test.ts +++ b/src/cli/format.test.ts @@ -12,7 +12,8 @@ import { formatTerminalList, formatTerminalRead, formatWorktreeList, - printResult + printResult, + reportCliError } from './format' import type { ComputerActionResult, RuntimeWorktreeRecord } from '../shared/runtime-types' import type { Automation } from '../shared/automations-types' @@ -116,6 +117,38 @@ describe('formatCliError', () => { ].join('\n') ) }) + + it('preserves orchestration migration recovery in human and JSON errors', () => { + const error = new RuntimeRpcFailureError({ + id: 'req_migration', + ok: false, + error: { + code: 'orchestration_migration_required', + message: 'No effects were applied.', + data: { + effectsApplied: false, + nextCommandArgs: ['skills', 'get', 'orchestration', '--full'], + nextSteps: ['Using this same Orca CLI executable, run: skills get orchestration --full'] + } + }, + _meta: { runtimeId: 'runtime-1' } + }) + + expect(formatCliError(error)).toContain( + 'Next step: Using this same Orca CLI executable, run: skills get orchestration --full' + ) + const log = vi.spyOn(console, 'log').mockImplementation(() => {}) + reportCliError(error, true) + expect(JSON.parse(String(log.mock.calls[0]?.[0]))).toMatchObject({ + error: { + code: 'orchestration_migration_required', + data: { + effectsApplied: false, + nextCommandArgs: ['skills', 'get', 'orchestration', '--full'] + } + } + }) + }) }) describe('formatWorktreeList', () => { diff --git a/src/cli/handler-group-manifest.ts b/src/cli/handler-group-manifest.ts index 4d54c3aba21..ffe608ccde7 100644 --- a/src/cli/handler-group-manifest.ts +++ b/src/cli/handler-group-manifest.ts @@ -87,6 +87,11 @@ export const HANDLER_GROUPS: readonly HandlerGroup[] = [ { name: 'orchestration', keys: [ + 'orchestration run-create', + 'orchestration run-use', + 'orchestration run-current', + 'orchestration run-list', + 'orchestration run-show', 'orchestration send', 'orchestration check', 'orchestration reply', @@ -94,11 +99,16 @@ export const HANDLER_GROUPS: readonly HandlerGroup[] = [ 'orchestration task-create', 'orchestration task-list', 'orchestration task-update', + 'orchestration worker-start', + 'orchestration worker-show', + 'orchestration worker-read', + 'orchestration worker-stop', + 'orchestration worker-abandon', 'orchestration dispatch', 'orchestration ask', 'orchestration dispatch-show', - 'orchestration run', - 'orchestration run-stop', + 'orchestration coordinator-start', + 'orchestration coordinator-stop', 'orchestration gate-create', 'orchestration gate-resolve', 'orchestration gate-list', diff --git a/src/cli/handlers/orchestration-lifecycle-rejection.test.ts b/src/cli/handlers/orchestration-lifecycle-rejection.test.ts index 269dfd7a47a..c55eed4391d 100644 --- a/src/cli/handlers/orchestration-lifecycle-rejection.test.ts +++ b/src/cli/handlers/orchestration-lifecycle-rejection.test.ts @@ -30,7 +30,8 @@ it('prints a lifecycle rejection and exits unsuccessfully', async () => { ['from', 'term_foreign'], ['to', 'term_coord'], ['subject', 'done'], - ['type', 'worker_done'] + ['type', 'worker_done'], + ['outcome', 'succeeded'] ]), client: { call: callMock }, cwd: '/tmp/repo', diff --git a/src/cli/handlers/orchestration-migration.test.ts b/src/cli/handlers/orchestration-migration.test.ts new file mode 100644 index 00000000000..8c092dbef83 --- /dev/null +++ b/src/cli/handlers/orchestration-migration.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it, vi } from 'vitest' +import { ORCHESTRATION_HANDLERS } from './orchestration' + +describe('orchestration CLI migration recovery', () => { + it('redirects worker_done without an outcome before resolving or calling the runtime', async () => { + const call = vi.fn() + + await expect( + ORCHESTRATION_HANDLERS['orchestration send']({ + flags: new Map([ + ['from', 'term_worker'], + ['subject', 'Done'], + ['type', 'worker_done'] + ]), + client: { call }, + cwd: '/tmp/repo', + json: true + } as never) + ).rejects.toMatchObject({ + code: 'invalid_argument', + data: { + effectsApplied: false, + nextCommandArgs: ['skills', 'get', 'orchestration', '--full'] + } + }) + expect(call).not.toHaveBeenCalled() + }) +}) diff --git a/src/cli/handlers/orchestration-run-cli.test.ts b/src/cli/handlers/orchestration-run-cli.test.ts new file mode 100644 index 00000000000..84f4dc4c659 --- /dev/null +++ b/src/cli/handlers/orchestration-run-cli.test.ts @@ -0,0 +1,160 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const callMock = vi.fn() +const getTerminalHandleMock = vi.hoisted(() => vi.fn()) + +vi.mock('../format', () => ({ printResult: vi.fn() })) +vi.mock('../selectors', () => ({ getTerminalHandle: getTerminalHandleMock })) + +import { printResult } from '../format' +import { ORCHESTRATION_HANDLERS } from './orchestration' + +describe('lightweight Run CLI handlers', () => { + beforeEach(() => { + callMock.mockReset() + getTerminalHandleMock.mockReset() + process.env.ORCA_TERMINAL_HANDLE = 'term_coord' + }) + + it('creates a Run with the resolved coordinator terminal', async () => { + callMock.mockResolvedValue({ + result: { run: { id: 'run_1', objective: 'Coordinate work', consumer_generation: 1 } } + }) + await ORCHESTRATION_HANDLERS['orchestration run-create']({ + flags: new Map([ + ['objective', 'Coordinate work'], + ['json', true] + ]), + client: { call: callMock }, + cwd: '/tmp/repo', + json: true + } as never) + expect(callMock).toHaveBeenCalledWith('orchestration.runCreate', { + objective: 'Coordinate work', + from: 'term_coord' + }) + }) + + it('reuses the same explicit binding path for run-use and run-current', async () => { + callMock + .mockResolvedValueOnce({ result: { run: { id: 'run_1', objective: 'Work' } } }) + .mockResolvedValueOnce({ result: { run: { id: 'run_1', objective: 'Work' } } }) + await ORCHESTRATION_HANDLERS['orchestration run-use']({ + flags: new Map([ + ['id', 'run_1'], + ['from', 'term_coord'] + ]), + client: { call: callMock }, + cwd: '/tmp/repo', + json: true + } as never) + await ORCHESTRATION_HANDLERS['orchestration run-current']({ + flags: new Map([['from', 'term_coord']]), + client: { call: callMock }, + cwd: '/tmp/repo', + json: true + } as never) + expect(callMock).toHaveBeenNthCalledWith(1, 'orchestration.runUse', { + id: 'run_1', + from: 'term_coord' + }) + expect(callMock).toHaveBeenNthCalledWith(2, 'orchestration.runCurrent', { + from: 'term_coord' + }) + }) +}) + +describe('orchestration reset CLI handler', () => { + beforeEach(() => { + callMock.mockReset().mockResolvedValue({ result: { reset: 'all' } }) + }) + const invoke = (flags: Map) => + ORCHESTRATION_HANDLERS['orchestration reset']({ + flags, + client: { call: callMock }, + json: true + } as never) + + it('rejects a bare reset before calling the runtime', async () => { + await expect(invoke(new Map())).rejects.toMatchObject({ + code: 'invalid_argument', + message: 'Choose exactly one reset scope: --all, --tasks, or --messages.' + }) + expect(callMock).not.toHaveBeenCalled() + }) + + it('sends only the tasks scope for --tasks', async () => { + await invoke(new Map([['tasks', true]])) + expect(callMock).toHaveBeenCalledWith('orchestration.reset', { + all: undefined, + tasks: true, + messages: undefined + }) + }) + + it('sends only the all scope for --all', async () => { + await invoke(new Map([['all', true]])) + expect(callMock).toHaveBeenCalledWith('orchestration.reset', { + all: true, + tasks: undefined, + messages: undefined + }) + }) + + it.each([ + new Map([ + ['tasks', true], + ['messages', true] + ]), + new Map([ + ['all', true], + ['tasks', true] + ]) + ])('rejects multiple reset scopes before calling the runtime', async (flags) => { + await expect(invoke(flags)).rejects.toMatchObject({ code: 'invalid_argument' }) + expect(callMock).not.toHaveBeenCalled() + }) +}) + +describe('orchestration task-list brief output', () => { + it('requests server-side brief and falls back client-side for older runtimes', async () => { + callMock.mockReset().mockResolvedValue({ + result: { + tasks: [{ id: 'task_1', spec: `First line\n${'detail '.repeat(40)}`, status: 'ready' }], + count: 1 + } + }) + vi.mocked(printResult).mockClear() + await ORCHESTRATION_HANDLERS['orchestration task-list']({ + flags: new Map([['brief', true]]), + client: { call: callMock }, + json: true + } as never) + expect(callMock).toHaveBeenCalledWith( + 'orchestration.taskList', + expect.objectContaining({ brief: true }) + ) + const response = vi.mocked(printResult).mock.calls[0]?.[0] as { + result: { tasks: { spec: string; spec_truncated: boolean }[] } + } + expect(response.result.tasks[0].spec).toHaveLength(160) + expect(response.result.tasks[0].spec_truncated).toBe(true) + }) + + it('passes server-abbreviated rows through untouched', async () => { + const serverTasks = [ + { id: 'task_1', spec: 'already brief…', status: 'ready', spec_truncated: true } + ] + callMock.mockReset().mockResolvedValue({ result: { tasks: serverTasks, count: 1 } }) + vi.mocked(printResult).mockClear() + await ORCHESTRATION_HANDLERS['orchestration task-list']({ + flags: new Map([['brief', true]]), + client: { call: callMock }, + json: true + } as never) + const response = vi.mocked(printResult).mock.calls[0]?.[0] as { + result: { tasks: { spec: string; spec_truncated: boolean }[] } + } + expect(response.result.tasks).toBe(serverTasks) + }) +}) diff --git a/src/cli/handlers/orchestration-timeout-cli.test.ts b/src/cli/handlers/orchestration-timeout-cli.test.ts new file mode 100644 index 00000000000..54282345a6c --- /dev/null +++ b/src/cli/handlers/orchestration-timeout-cli.test.ts @@ -0,0 +1,230 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const callMock = vi.fn() + +vi.mock('../format', () => ({ printResult: vi.fn() })) +vi.mock('../selectors', () => ({ getTerminalHandle: vi.fn() })) + +import { printResult } from '../format' +import { ORCHESTRATION_HANDLERS } from './orchestration' + +describe('orchestration timeout flag validation', () => { + const invalidTimeoutValues: [string, string | boolean][] = [ + ['missing', true], + ['empty', ''], + ['non-numeric', 'not-a-number'], + ['zero', '0'], + ['negative', '-1'] + ] + + beforeEach(() => { + callMock.mockReset() + delete process.env.ORCA_TERMINAL_HANDLE + delete process.env.ORCA_PANE_KEY + }) + + const invokeCheck = (flags: Map) => + ORCHESTRATION_HANDLERS['orchestration check']({ + flags, + client: { call: callMock }, + cwd: '/tmp/repo', + json: true + } as never) + + const invokeAsk = (flags: Map) => + ORCHESTRATION_HANDLERS['orchestration ask']({ + flags, + client: { call: callMock }, + cwd: '/tmp/repo', + json: true + } as never) + + it.each(invalidTimeoutValues)('rejects invalid check --timeout-ms: %s', async (_label, value) => { + await expect( + invokeCheck( + new Map([ + ['wait', true], + ['timeout-ms', value] + ]) + ) + ).rejects.toThrow(/--timeout-ms/) + expect(callMock).not.toHaveBeenCalled() + }) + + it('passes a parsed check timeout and peek mode into the RPC payload', async () => { + process.env.ORCA_TERMINAL_HANDLE = 'term_worker' + callMock.mockResolvedValue({ result: { messages: [], count: 0 } }) + await invokeCheck( + new Map([ + ['wait', true], + ['peek', true], + ['timeout-ms', '250'] + ]) + ) + // Why: unread:false makes pre-peek runtimes fall back to non-consuming all mode. + expect(callMock).toHaveBeenCalledWith('orchestration.check', { + terminal: 'term_worker', + unread: false, + peek: true, + all: undefined, + types: undefined, + format: undefined, + run: undefined, + ack: undefined, + wait: true, + timeoutMs: 250 + }) + }) + + it('filters already-read rows from a peek response for pre-peek runtimes', async () => { + process.env.ORCA_TERMINAL_HANDLE = 'term_worker' + callMock.mockResolvedValue({ + result: { + messages: [ + { id: 'msg_old', from_handle: 'a', subject: 'seen', read: 1 }, + { id: 'msg_new', from_handle: 'a', subject: 'fresh', read: 0 } + ], + count: 2, + formatted: 'banners built from all rows' + } + }) + vi.mocked(printResult).mockClear() + await invokeCheck(new Map([['peek', true]])) + const response = vi.mocked(printResult).mock.calls[0]?.[0] as { + result: { messages: { id: string }[]; count: number; formatted?: string } + } + expect(response.result.messages.map((message) => message.id)).toEqual(['msg_new']) + expect(response.result.count).toBe(1) + expect(response.result.formatted).toBeUndefined() + }) + + it('rejects combined read modes before calling the runtime', async () => { + process.env.ORCA_TERMINAL_HANDLE = 'term_worker' + await expect( + invokeCheck( + new Map([ + ['unread', true], + ['peek', true] + ]) + ) + ).rejects.toMatchObject({ + code: 'invalid_argument', + message: expect.stringContaining('read mode') + }) + expect(callMock).not.toHaveBeenCalled() + }) + + it('warns when a pre-peek runtime returned a full 100-row page', async () => { + process.env.ORCA_TERMINAL_HANDLE = 'term_worker' + const rows = Array.from({ length: 100 }, (_, index) => ({ + id: `msg_${index}`, + from_handle: 'a', + subject: `s${index}`, + read: index === 0 ? 0 : 1 + })) + callMock.mockResolvedValue({ result: { messages: rows, count: 100 } }) + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}) + await invokeCheck(new Map([['peek', true]])) + expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('newest 100 messages')) + errorSpy.mockRestore() + }) + + it('fails --peek --wait against a runtime that returned only read rows', async () => { + process.env.ORCA_TERMINAL_HANDLE = 'term_worker' + callMock.mockResolvedValue({ + result: { + messages: [{ id: 'msg_old', from_handle: 'a', subject: 'seen', read: 1 }], + count: 1 + } + }) + await expect( + invokeCheck( + new Map([ + ['peek', true], + ['wait', true] + ]) + ) + ).rejects.toMatchObject({ code: 'peek_wait_unsupported' }) + }) + + it.each(invalidTimeoutValues)('rejects invalid ask --timeout-ms: %s', async (_label, value) => { + await expect( + invokeAsk( + new Map([ + ['to', 'term_coord'], + ['question', 'Proceed?'], + ['timeout-ms', value] + ]) + ) + ).rejects.toThrow(/--timeout-ms/) + expect(callMock).not.toHaveBeenCalled() + }) + + it('uses the parsed ask timeout for both runtime wait and client timeout', async () => { + process.env.ORCA_TERMINAL_HANDLE = 'term_worker' + callMock.mockResolvedValue({ + result: { answer: 'yes', messageId: 'msg_1', threadId: 'thread_1', timedOut: false } + }) + vi.spyOn(console, 'log').mockImplementation(() => {}) + await invokeAsk( + new Map([ + ['to', 'term_coord'], + ['question', 'Proceed?'], + ['timeout-ms', '123'] + ]) + ) + expect(callMock).toHaveBeenCalledWith( + 'orchestration.ask', + { + to: 'term_coord', + run: undefined, + question: 'Proceed?', + resume: undefined, + options: undefined, + timeoutMs: 123, + from: 'term_worker' + }, + { timeoutMs: 5_123, orchestrationCapability: undefined } + ) + }) + + it('passes an ask resume without creating a new question payload', async () => { + process.env.ORCA_TERMINAL_HANDLE = 'term_worker' + callMock.mockResolvedValue({ + result: { + answer: 'yes', + messageId: 'msg_question', + threadId: 'msg_question', + timedOut: false + } + }) + vi.spyOn(console, 'log').mockImplementation(() => {}) + await invokeAsk(new Map([['resume', 'msg_question']])) + expect(callMock).toHaveBeenCalledWith( + 'orchestration.ask', + { + to: undefined, + run: undefined, + question: undefined, + resume: 'msg_question', + options: undefined, + timeoutMs: undefined, + from: 'term_worker' + }, + { timeoutMs: 605_000, orchestrationCapability: undefined } + ) + }) + + it('rejects ambiguous ask create/resume input before RPC', async () => { + process.env.ORCA_TERMINAL_HANDLE = 'term_worker' + await expect( + invokeAsk( + new Map([ + ['question', 'new'], + ['resume', 'msg_old'] + ]) + ) + ).rejects.toMatchObject({ code: 'invalid_argument' }) + expect(callMock).not.toHaveBeenCalled() + }) +}) diff --git a/src/cli/handlers/orchestration-worker-cli.test.ts b/src/cli/handlers/orchestration-worker-cli.test.ts new file mode 100644 index 00000000000..09b4a7a34b9 --- /dev/null +++ b/src/cli/handlers/orchestration-worker-cli.test.ts @@ -0,0 +1,161 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const callMock = vi.fn() +const originalExitCode = process.exitCode + +vi.mock('../format', () => ({ printResult: vi.fn() })) +vi.mock('../selectors', () => ({ getTerminalHandle: vi.fn() })) + +import { ORCHESTRATION_HANDLERS } from './orchestration' + +describe('orchestration worker-start CLI contract', () => { + beforeEach(() => { + callMock.mockReset() + process.exitCode = undefined + }) + + afterEach(() => { + process.exitCode = originalExitCode + }) + + const invokeWorkerStart = (flags: Map) => + ORCHESTRATION_HANDLERS['orchestration worker-start']({ + flags, + client: { call: callMock }, + cwd: '/tmp/repo', + json: true + } as never) + + it('passes the complete supported creation contract and retry receipt', async () => { + callMock.mockResolvedValue({ + result: { + runId: 'run_1', + taskId: 'task_1', + dispatchId: 'ctx_1', + state: 'ready', + effects: [], + residualResources: [] + } + }) + + await invokeWorkerStart( + new Map([ + ['task', 'task_1'], + ['on', 'windows'], + ['worktree', 'new-top-level'], + ['name', 'release-audit'], + ['repo', 'id:windows-repo'], + ['base-branch', 'origin/release'], + ['display-name', 'Release audit'], + ['comment', 'Supervised from the Mac Run home'], + ['setup', 'run'], + ['agent', 'codex'], + ['timeout-ms', '90000'], + ['run', 'run_1'], + ['from', 'term_coord'], + ['retry-request', 'request_1'] + ]) + ) + + expect(callMock).toHaveBeenCalledWith( + 'orchestration.workerStart', + { + task: 'task_1', + on: 'windows', + worktree: 'new-top-level', + name: 'release-audit', + repo: 'id:windows-repo', + baseBranch: 'origin/release', + displayName: 'Release audit', + comment: 'Supervised from the Mac Run home', + setup: 'run', + agent: 'codex', + terminal: undefined, + retryOf: undefined, + timeoutMs: 90_000, + run: 'run_1', + from: 'term_coord', + devMode: false + }, + { orchestrationRequestId: 'request_1' } + ) + expect(process.exitCode).toBeUndefined() + }) + + it('sets an unsuccessful exit code for failed and unknown receipts', async () => { + callMock.mockResolvedValue({ + result: { + taskId: 'task_1', + dispatchId: 'ctx_1', + state: 'outcome_unknown', + effects: [], + residualResources: [] + } + }) + + await invokeWorkerStart( + new Map([ + ['task', 'task_1'], + ['agent', 'codex'], + ['from', 'term_coord'] + ]) + ) + + expect(process.exitCode).toBe(1) + }) + + it('allows the initial zero cursor when paging worker output', async () => { + callMock.mockResolvedValue({ + result: { + dispatchId: 'ctx_1', + terminal: { tail: [], status: 'running', nextCursor: '0' } + } + }) + + await ORCHESTRATION_HANDLERS['orchestration worker-read']({ + flags: new Map([ + ['dispatch', 'ctx_1'], + ['cursor', '0'], + ['limit', '100'] + ]), + client: { call: callMock }, + cwd: '/tmp/repo', + json: true + } as never) + + expect(callMock).toHaveBeenCalledWith('orchestration.workerRead', { + dispatch: 'ctx_1', + cursor: 0, + limit: 100, + source: undefined + }) + }) + + it('passes opaque source-pinned cursors and explicit source selection', async () => { + callMock.mockResolvedValue({ + result: { + dispatchId: 'ctx_1', + source: 'transcript', + transcript: { messages: [], nextCursor: 'owr1_next' } + } + }) + + await ORCHESTRATION_HANDLERS['orchestration worker-read']({ + flags: new Map([ + ['dispatch', 'ctx_1'], + ['cursor', 'owr1_previous'], + ['source', 'transcript'] + ]), + client: { call: callMock }, + cwd: '/tmp/repo', + json: true + } as never) + + expect(callMock).toHaveBeenCalledWith('orchestration.workerRead', { + dispatch: 'ctx_1', + cursor: 'owr1_previous', + limit: undefined, + source: 'transcript' + }) + }) +}) diff --git a/src/cli/handlers/orchestration.test.ts b/src/cli/handlers/orchestration.test.ts index 55f2482f5cc..d5399ece4aa 100644 --- a/src/cli/handlers/orchestration.test.ts +++ b/src/cli/handlers/orchestration.test.ts @@ -5,7 +5,7 @@ const getTerminalHandleMock = vi.hoisted(() => vi.fn()) const originalTerminalHandle = process.env.ORCA_TERMINAL_HANDLE const originalPaneKey = process.env.ORCA_PANE_KEY function lifecycleGroupRecipientError(type: 'worker_done' | 'heartbeat'): string { - return `${type} messages must be sent to a concrete coordinator terminal handle, not a group address.` + return `${type} messages belong to one exact Dispatch and cannot target a group address.` } // Why: isolate the handler's flag-to-param mapping; printResult only writes output. @@ -48,46 +48,6 @@ afterEach(() => { } }) -describe('orchestration reset CLI handler', () => { - beforeEach(() => { - callMock.mockReset().mockResolvedValue({ result: { reset: 'all' } }) - }) - - const invoke = (flags: Map) => - ORCHESTRATION_HANDLERS['orchestration reset']({ - flags, - client: { call: callMock }, - json: true - } as never) - - it('sends all: true for a bare `reset` (no scope flag)', async () => { - await invoke(new Map()) - expect(callMock).toHaveBeenCalledWith('orchestration.reset', { - all: true, - tasks: undefined, - messages: undefined - }) - }) - - it('sends only the tasks scope for --tasks', async () => { - await invoke(new Map([['tasks', true]])) - expect(callMock).toHaveBeenCalledWith('orchestration.reset', { - all: undefined, - tasks: true, - messages: undefined - }) - }) - - it('sends only the all scope for --all (no implicit extra scopes)', async () => { - await invoke(new Map([['all', true]])) - expect(callMock).toHaveBeenCalledWith('orchestration.reset', { - all: true, - tasks: undefined, - messages: undefined - }) - }) -}) - describe('orchestration send structured payload flags', () => { beforeEach(() => { callMock.mockReset().mockResolvedValue({ result: { message: { id: 'msg_1' } } }) @@ -113,6 +73,7 @@ describe('orchestration send structured payload flags', () => { ['type', 'worker_done'], ['task-id', 'task_1'], ['dispatch-id', 'ctx_1'], + ['outcome', 'succeeded'], ['files-modified', 'src/a.ts, src/b.ts'], ['report-path', 'reports/done.md'] ]) @@ -129,6 +90,7 @@ describe('orchestration send structured payload flags', () => { payload: JSON.stringify({ taskId: 'task_1', dispatchId: 'ctx_1', + outcome: 'succeeded', filesModified: ['src/a.ts', 'src/b.ts'], reportPath: 'reports/done.md' }), @@ -151,6 +113,25 @@ describe('orchestration send structured payload flags', () => { expect(callMock).toHaveBeenCalledWith('orchestration.send', expect.objectContaining({ body })) }) + it('carries Dispatch authority in the RPC envelope instead of message params', async () => { + await invokeSend( + new Map([ + ['from', 'term_worker'], + ['subject', 'alive'], + ['type', 'heartbeat'], + ['dispatch-id', 'ctx_1'], + ['dispatch-capability', 'dcap_secret'], + ['retry-request', 'mutation_1'] + ]) + ) + + expect(callMock).toHaveBeenCalledWith( + 'orchestration.send', + expect.not.objectContaining({ dispatchCapability: expect.anything() }), + { orchestrationCapability: 'dcap_secret', orchestrationRequestId: 'mutation_1' } + ) + }) + it('rejects mixing raw payload with structured payload flags', async () => { await expect( invokeSend( @@ -212,7 +193,8 @@ describe('orchestration send structured payload flags', () => { ['from', 'term_worker'], ['to', 'term_coord'], ['subject', 'done'], - ['type', 'worker_done'] + ['type', 'worker_done'], + ['outcome', 'succeeded'] ]) ) @@ -224,7 +206,7 @@ describe('orchestration send structured payload flags', () => { type: 'worker_done', priority: undefined, threadId: undefined, - payload: undefined, + payload: JSON.stringify({ outcome: 'succeeded' }), devMode: false }) }) @@ -236,7 +218,8 @@ describe('orchestration send structured payload flags', () => { new Map([ ['to', 'term_coord'], ['subject', 'done'], - ['type', 'worker_done'] + ['type', 'worker_done'], + ['outcome', 'succeeded'] ]) ) @@ -249,7 +232,7 @@ describe('orchestration send structured payload flags', () => { type: 'worker_done', priority: undefined, threadId: undefined, - payload: undefined, + payload: JSON.stringify({ outcome: 'succeeded' }), devMode: false }) }) @@ -264,7 +247,8 @@ describe('orchestration send structured payload flags', () => { new Map([ ['to', 'term_coord'], ['subject', 'update'], - ['type', type] + ['type', type], + ...(type === 'worker_done' ? ([['outcome', 'succeeded']] as const) : []) ]) ) @@ -288,7 +272,8 @@ describe('orchestration send structured payload flags', () => { new Map([ ['to', 'term_coord'], ['subject', 'done'], - ['type', 'worker_done'] + ['type', 'worker_done'], + ['outcome', 'succeeded'] ]) ) @@ -308,7 +293,8 @@ describe('orchestration send structured payload flags', () => { new Map([ ['to', 'term_coord'], ['subject', 'done'], - ['type', 'worker_done'] + ['type', 'worker_done'], + ['outcome', 'succeeded'] ]) ) ).rejects.toMatchObject({ @@ -328,7 +314,8 @@ describe('orchestration send structured payload flags', () => { new Map([ ['to', 'term_coord'], ['subject', 'update'], - ['type', type] + ['type', type], + ...(type === 'worker_done' ? ([['outcome', 'succeeded']] as const) : []) ]) ) ).rejects.toMatchObject({ code: 'no_active_sender_terminal' }) @@ -364,7 +351,7 @@ describe('orchestration dispatch coordinator handle', () => { } as never) const invokeRun = (flags: Map) => - ORCHESTRATION_HANDLERS['orchestration run']({ + ORCHESTRATION_HANDLERS['orchestration coordinator-start']({ flags, client: { call: callMock }, cwd: '/tmp/repo', @@ -483,29 +470,18 @@ describe('orchestration dispatch coordinator handle', () => { }) }) - it('uses a live coordinator handle for orchestration runs', async () => { - process.env.ORCA_TERMINAL_HANDLE = 'term_stale_coord' - process.env.ORCA_PANE_KEY = 'tab_coord:leaf_coord' - stubStaleHandleRemint('term_live_coord', { - result: { runId: 'run_1', status: 'running' } - }) - getTerminalHandleMock.mockRejectedValue(new Error('active terminal fallback is unsafe')) - - await invokeRun(new Map([['spec', 'run the plan']])) - - expect(callMock).toHaveBeenNthCalledWith(1, 'terminal.show', { - terminal: 'term_stale_coord' - }) - expect(callMock).toHaveBeenNthCalledWith(2, 'terminal.resolvePane', { - paneKey: 'tab_coord:leaf_coord' - }) - expect(callMock).toHaveBeenNthCalledWith(3, 'orchestration.run', { - spec: 'run the plan', - from: 'term_live_coord', - pollIntervalMs: undefined, - maxConcurrent: undefined, - worktree: undefined + it('retires the legacy coordinator command without runtime effects', async () => { + await expect( + invokeRun(new Map([['spec', 'run the plan']])) + ).rejects.toMatchObject({ + code: 'orchestration_migration_required', + data: { + reason: 'command_retired', + effectsApplied: false, + nextCommandArgs: ['skills', 'get', 'orchestration', '--full'] + } }) + expect(callMock).not.toHaveBeenCalled() }) }) @@ -540,75 +516,58 @@ describe('orchestration task-create caller handle', () => { displayName: undefined, deps: undefined, parent: undefined, + run: undefined, callerTerminalHandle: 'term_creator' }) }) - it('does not persist a stale env terminal handle as task creator', async () => { + it('fails closed when a stale task creator handle cannot be reminted', async () => { process.env.ORCA_TERMINAL_HANDLE = 'term_stale' - callMock - .mockRejectedValueOnce(staleHandleError()) - .mockResolvedValueOnce({ result: { task: { id: 'task_1', status: 'ready' } } }) + callMock.mockRejectedValueOnce(staleHandleError()) getTerminalHandleMock.mockResolvedValue('term_wrong_active') - await invokeTaskCreate(new Map([['spec', 'do work']])) + await expect( + invokeTaskCreate(new Map([['spec', 'do work']])) + ).rejects.toMatchObject({ code: 'no_active_sender_terminal' }) expect(callMock).toHaveBeenNthCalledWith(1, 'terminal.show', { terminal: 'term_stale' }) expect(getTerminalHandleMock).not.toHaveBeenCalled() - expect(callMock).toHaveBeenNthCalledWith(2, 'orchestration.taskCreate', { - spec: 'do work', - taskTitle: undefined, - displayName: undefined, - deps: undefined, - parent: undefined, - callerTerminalHandle: undefined - }) + expect(callMock).toHaveBeenCalledTimes(1) }) - it('does not fail task creation when env handle validation cannot inspect the graph', async () => { + it('propagates runtime unavailability while proving the bound coordinator', async () => { process.env.ORCA_TERMINAL_HANDLE = 'term_creator' - callMock - .mockRejectedValueOnce(new RuntimeClientError('runtime_unavailable', 'runtime_unavailable')) - .mockResolvedValueOnce({ result: { task: { id: 'task_1', status: 'ready' } } }) + callMock.mockRejectedValueOnce( + new RuntimeClientError('runtime_unavailable', 'runtime_unavailable') + ) - await invokeTaskCreate(new Map([['spec', 'do work']])) + await expect( + invokeTaskCreate(new Map([['spec', 'do work']])) + ).rejects.toMatchObject({ code: 'runtime_unavailable' }) expect(callMock).toHaveBeenNthCalledWith(1, 'terminal.show', { terminal: 'term_creator' }) expect(getTerminalHandleMock).not.toHaveBeenCalled() - expect(callMock).toHaveBeenNthCalledWith(2, 'orchestration.taskCreate', { - spec: 'do work', - taskTitle: undefined, - displayName: undefined, - deps: undefined, - parent: undefined, - callerTerminalHandle: undefined - }) + expect(callMock).toHaveBeenCalledTimes(1) }) - it('omits caller handle when pane reminting cannot inspect the graph', async () => { + it('propagates runtime unavailability while reminting the bound coordinator', async () => { process.env.ORCA_TERMINAL_HANDLE = 'term_stale' process.env.ORCA_PANE_KEY = 'tab_creator:leaf_creator' stubStaleHandleRemintFailure( new RuntimeClientError('runtime_unavailable', 'runtime_unavailable') ) - callMock.mockResolvedValueOnce({ result: { task: { id: 'task_1', status: 'ready' } } }) getTerminalHandleMock.mockResolvedValue('term_wrong_active') - await invokeTaskCreate(new Map([['spec', 'do work']])) + await expect( + invokeTaskCreate(new Map([['spec', 'do work']])) + ).rejects.toMatchObject({ code: 'runtime_unavailable' }) expect(callMock).toHaveBeenNthCalledWith(1, 'terminal.show', { terminal: 'term_stale' }) expect(callMock).toHaveBeenNthCalledWith(2, 'terminal.resolvePane', { paneKey: 'tab_creator:leaf_creator' }) expect(getTerminalHandleMock).not.toHaveBeenCalled() - expect(callMock).toHaveBeenNthCalledWith(3, 'orchestration.taskCreate', { - spec: 'do work', - taskTitle: undefined, - displayName: undefined, - deps: undefined, - parent: undefined, - callerTerminalHandle: undefined - }) + expect(callMock).toHaveBeenCalledTimes(2) }) it('propagates unexpected caller pane remint failures for task creation', async () => { @@ -665,10 +624,248 @@ describe('orchestration task-create caller handle', () => { displayName: undefined, deps: undefined, parent: undefined, + run: undefined, callerTerminalHandle: 'term_live' }) }) }) +describe('orchestration timeout flag validation', () => { + const invalidTimeoutValues: [string, string | boolean][] = [ + ['missing', true], + ['empty', ''], + ['non-numeric', 'not-a-number'], + ['zero', '0'], + ['negative', '-1'] + ] + + beforeEach(() => { + callMock.mockReset() + delete process.env.ORCA_TERMINAL_HANDLE + delete process.env.ORCA_PANE_KEY + }) + + const invokeCheck = (flags: Map) => + ORCHESTRATION_HANDLERS['orchestration check']({ + flags, + client: { call: callMock }, + cwd: '/tmp/repo', + json: true + } as never) + + const invokeAsk = (flags: Map) => + ORCHESTRATION_HANDLERS['orchestration ask']({ + flags, + client: { call: callMock }, + cwd: '/tmp/repo', + json: true + } as never) + + it.each(invalidTimeoutValues)('rejects invalid check --timeout-ms: %s', async (_label, value) => { + const flags = new Map([ + ['wait', true], + ['timeout-ms', value] + ]) + + await expect(invokeCheck(flags)).rejects.toThrow(/--timeout-ms/) + expect(callMock).not.toHaveBeenCalled() + }) + + it('passes a parsed check timeout and peek mode into the RPC payload', async () => { + process.env.ORCA_TERMINAL_HANDLE = 'term_worker' + callMock.mockResolvedValue({ result: { messages: [], count: 0 } }) + + await invokeCheck( + new Map([ + ['wait', true], + ['peek', true], + ['timeout-ms', '250'] + ]) + ) + + // Why: --peek rides with unread:false so pre-peek runtimes fall back to + // the non-consuming all mode instead of the destructive mark-read default. + expect(callMock).toHaveBeenCalledWith('orchestration.check', { + terminal: 'term_worker', + unread: false, + peek: true, + all: undefined, + types: undefined, + format: undefined, + run: undefined, + ack: undefined, + wait: true, + timeoutMs: 250 + }) + }) + + it('filters already-read rows from a peek response for pre-peek runtimes', async () => { + process.env.ORCA_TERMINAL_HANDLE = 'term_worker' + callMock.mockResolvedValue({ + result: { + messages: [ + { id: 'msg_old', from_handle: 'a', subject: 'seen', read: 1 }, + { id: 'msg_new', from_handle: 'a', subject: 'fresh', read: 0 } + ], + count: 2, + formatted: 'banners built from all rows' + } + }) + vi.mocked(printResult).mockClear() + + await invokeCheck(new Map([['peek', true]])) + + const response = vi.mocked(printResult).mock.calls[0]?.[0] as { + result: { messages: { id: string }[]; count: number; formatted?: string } + } + expect(response.result.messages.map((m) => m.id)).toEqual(['msg_new']) + expect(response.result.count).toBe(1) + // Why: the pre-peek runtime built `formatted` from all rows, including + // the read one the filter just removed. + expect(response.result.formatted).toBeUndefined() + }) + + it('rejects combined read modes before calling the runtime', async () => { + process.env.ORCA_TERMINAL_HANDLE = 'term_worker' + callMock.mockClear() + + await expect( + invokeCheck( + new Map([ + ['unread', true], + ['peek', true] + ]) + ) + ).rejects.toMatchObject({ + code: 'invalid_argument', + message: expect.stringContaining('read mode') + }) + expect(callMock).not.toHaveBeenCalled() + }) + + it('warns when a pre-peek runtime returned a full 100-row page', async () => { + process.env.ORCA_TERMINAL_HANDLE = 'term_worker' + const rows = Array.from({ length: 100 }, (_, i) => ({ + id: `msg_${i}`, + from_handle: 'a', + subject: `s${i}`, + read: i === 0 ? 0 : 1 + })) + callMock.mockResolvedValue({ result: { messages: rows, count: 100 } }) + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}) + + await invokeCheck(new Map([['peek', true]])) + + expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('newest 100 messages')) + errorSpy.mockRestore() + }) + + it('fails --peek --wait against a runtime that returned only read rows', async () => { + process.env.ORCA_TERMINAL_HANDLE = 'term_worker' + callMock.mockResolvedValue({ + result: { + messages: [{ id: 'msg_old', from_handle: 'a', subject: 'seen', read: 1 }], + count: 1 + } + }) + + await expect( + invokeCheck( + new Map([ + ['peek', true], + ['wait', true] + ]) + ) + ).rejects.toMatchObject({ code: 'peek_wait_unsupported' }) + }) + + it.each(invalidTimeoutValues)('rejects invalid ask --timeout-ms: %s', async (_label, value) => { + const flags = new Map([ + ['to', 'term_coord'], + ['question', 'Proceed?'], + ['timeout-ms', value] + ]) + + await expect(invokeAsk(flags)).rejects.toThrow(/--timeout-ms/) + expect(callMock).not.toHaveBeenCalled() + }) + + it('uses the parsed ask timeout for both runtime wait and client timeout', async () => { + process.env.ORCA_TERMINAL_HANDLE = 'term_worker' + callMock.mockResolvedValue({ + result: { + answer: 'yes', + messageId: 'msg_1', + threadId: 'thread_1', + timedOut: false + } + }) + vi.spyOn(console, 'log').mockImplementation(() => {}) + + await invokeAsk( + new Map([ + ['to', 'term_coord'], + ['question', 'Proceed?'], + ['timeout-ms', '123'] + ]) + ) + + expect(callMock).toHaveBeenCalledWith( + 'orchestration.ask', + { + to: 'term_coord', + run: undefined, + question: 'Proceed?', + resume: undefined, + options: undefined, + timeoutMs: 123, + from: 'term_worker' + }, + { timeoutMs: 5_123, orchestrationCapability: undefined } + ) + }) + + it('passes an ask resume without creating a new question payload', async () => { + process.env.ORCA_TERMINAL_HANDLE = 'term_worker' + callMock.mockResolvedValue({ + result: { + answer: 'yes', + messageId: 'msg_question', + threadId: 'msg_question', + timedOut: false + } + }) + vi.spyOn(console, 'log').mockImplementation(() => {}) + + await invokeAsk(new Map([['resume', 'msg_question']])) + + expect(callMock).toHaveBeenCalledWith( + 'orchestration.ask', + { + to: undefined, + run: undefined, + question: undefined, + resume: 'msg_question', + options: undefined, + timeoutMs: undefined, + from: 'term_worker' + }, + { timeoutMs: 605_000, orchestrationCapability: undefined } + ) + }) + + it('rejects ambiguous ask create/resume input before RPC', async () => { + process.env.ORCA_TERMINAL_HANDLE = 'term_worker' + await expect( + invokeAsk( + new Map([ + ['question', 'new'], + ['resume', 'msg_old'] + ]) + ) + ).rejects.toMatchObject({ code: 'invalid_argument' }) + expect(callMock).not.toHaveBeenCalled() + }) +}) describe('orchestration task-list brief output', () => { it('requests server-side brief and falls back client-side for older runtimes', async () => { diff --git a/src/cli/handlers/orchestration.ts b/src/cli/handlers/orchestration.ts index ec02ded26d2..220f0964251 100644 --- a/src/cli/handlers/orchestration.ts +++ b/src/cli/handlers/orchestration.ts @@ -1,5 +1,6 @@ /* eslint-disable max-lines -- Why: orchestration CLI handlers share flag-parsing helpers and dispatch/preamble logic; splitting by verb would fragment the RuntimeClient call shape without reducing complexity. */ import type { CommandHandler } from '../dispatch' +import type { RuntimeClient } from '../runtime-client' import { printResult } from '../format' import { getOptionalPositiveIntegerFlag, @@ -14,11 +15,21 @@ import { } from '../../shared/orchestration-ask-timeout' import { abbreviateOrchestrationTasks } from '../../shared/orchestration-task-summary' import { parsePositiveSafeIntegerText } from '../../shared/timer-delay' +import type { + OrchestrationWorkerReadResult, + OrchestrationWorkerReadSource +} from '../../shared/orchestration-worker-output' +import type { NativeChatMessage } from '../../shared/native-chat-types' +import type { RuntimeTerminalRead } from '../../shared/runtime-types' +import { + orchestrationMigrationData, + orchestrationSkillRecoveryData +} from '../../shared/orchestration-rpc-contract' // Why: 15 s is well under Claude Code's ~2 min Bash-tool silence budget while keeping log volume low. See design doc §3.4. const DEFAULT_KEEPALIVE_INTERVAL_MS = 15_000 function getLifecycleGroupRecipientError(type: 'worker_done' | 'heartbeat'): string { - return `${type} messages must be sent to a concrete coordinator terminal handle, not a group address.` + return `${type} messages belong to one exact Dispatch and cannot target a group address.` } // Why: test-only escape hatch so subprocess tests avoid the full 15 s window; bogus values fall back to the default. @@ -83,6 +94,16 @@ type LifecycleSendRejection = { type OrchestrationSendResult = | { message: { id: string }; lifecycle?: LifecycleSendRejection } | { messages: { id: string }[]; recipients: number } + | { + relay: { + messageId: string + sequence: number + dispatchId: string + destination?: 'run_home' | 'worker' + accepted: true + } + lifecycle?: { action: 'completed' | 'failed' } + } function getOptionalStructuredMessagePayload( flags: Map @@ -90,12 +111,14 @@ function getOptionalStructuredMessagePayload( const rawPayload = getOptionalStringFlag(flags, 'payload') const taskId = getOptionalStringFlag(flags, 'task-id') const dispatchId = getOptionalStringFlag(flags, 'dispatch-id') + const outcome = getOptionalStringFlag(flags, 'outcome') const filesModified = getOptionalStringFlag(flags, 'files-modified') const reportPath = getOptionalStringFlag(flags, 'report-path') const phase = getOptionalStringFlag(flags, 'phase') const hasStructuredPayload = taskId !== undefined || dispatchId !== undefined || + outcome !== undefined || filesModified !== undefined || reportPath !== undefined || phase !== undefined @@ -116,6 +139,15 @@ function getOptionalStructuredMessagePayload( if (dispatchId) { payload.dispatchId = dispatchId } + if (outcome) { + if (outcome !== 'succeeded' && outcome !== 'failed') { + throw new RuntimeClientError( + 'invalid_argument', + 'Invalid --outcome. Expected succeeded or failed.' + ) + } + payload.outcome = outcome + } if (filesModified) { payload.filesModified = filesModified .split(',') @@ -163,29 +195,6 @@ async function resolveOrchestrationTerminalHandle( return await getTerminalHandle(flags, cwd, client) } -async function resolveTaskCreatorTerminalHandle( - client: Parameters[0]['client'] -): Promise { - const envHandle = process.env.ORCA_TERMINAL_HANDLE - if (!envHandle || envHandle.length === 0) { - return undefined - } - let live: boolean - try { - live = await isLiveTerminalHandle(envHandle, client) - } catch (err) { - if (isOptionalTaskCreatorHandleError(err)) { - // Why: creator handles are best-effort lineage metadata; graph unavailability must not block task creation. - return undefined - } - throw err - } - if (live) { - return envHandle - } - return await resolveOrchestrationPaneTerminalHandle(client, { optional: true }) -} - async function isLiveTerminalHandle( handle: string, client: Parameters[0]['client'] @@ -218,11 +227,6 @@ function isNoActiveTerminalError(err: unknown): boolean { return getClientErrorCode(err) === 'no_active_terminal' } -function isOptionalTaskCreatorHandleError(err: unknown): boolean { - const code = getClientErrorCode(err) - return code === 'no_active_sender_terminal' || code === 'runtime_unavailable' -} - async function resolveOrchestrationPaneTerminalHandle( client: Parameters[0]['client'], options: { optional?: boolean } = {} @@ -310,7 +314,10 @@ function throwNoActiveSenderTerminal(): never { } function isDevCliInvocation(): boolean { - return process.env.ORCA_USER_DATA_PATH?.includes('orca-dev') ?? false + return ( + process.env.ORCA_DEV_CLI_INVOCATION === '1' || + (process.env.ORCA_USER_DATA_PATH?.includes('orca-dev') ?? false) + ) } function getOptionalPositiveIntegerValueFlag( @@ -340,11 +347,148 @@ function rejectLifecycleGroupRecipient(type: string | undefined, to: string): vo } } +function callMutation( + client: RuntimeClient, + flags: Map, + method: string, + params: unknown, + options?: { timeoutMs?: number; orchestrationCapability?: string } +) { + const requestId = getOptionalStringFlag(flags, 'retry-request') + if (!requestId) { + return options + ? client.call(method, params, options) + : client.call(method, params) + } + return client.call(method, params, { + ...options, + orchestrationRequestId: requestId + }) +} + +type LegacyWorkerReadResult = { + dispatchId: string + terminal: RuntimeTerminalRead +} + +function formatWorkerRead(value: OrchestrationWorkerReadResult | LegacyWorkerReadResult): string { + if (!('source' in value) || value.source === 'terminal') { + return value.terminal.tail.join('\n') + } + return value.transcript.messages.map(formatWorkerTranscriptMessage).join('\n\n') +} + +function formatWorkerTranscriptMessage(message: NativeChatMessage): string { + const blocks = message.blocks.map((block) => { + if (block.type === 'text') { + return block.text + } + if (block.type === 'tool-call') { + return `[tool ${block.name}] ${safeJson(block.input)}` + } + if (block.type === 'tool-result') { + return `[tool result${block.isError ? ' error' : ''}] ${block.output}` + } + return block.url ? `[image] ${block.url}` : `[image omitted]` + }) + return `[${message.role}] ${blocks.join('\n')}`.trimEnd() +} + +function safeJson(value: unknown): string { + try { + return JSON.stringify(value) + } catch { + return '[unserializable input]' + } +} + export const ORCHESTRATION_HANDLERS: Record = { + 'orchestration run-create': async ({ flags, client, cwd, json }) => { + const from = await resolveCoordinatorTerminalHandle(flags, cwd, client) + const result = await callMutation<{ + run: { id: string; objective: string; consumer_generation: number } + }>(client, flags, 'orchestration.runCreate', { + objective: getRequiredStringFlag(flags, 'objective'), + from + }) + printResult(result, json, (r) => `Run ${r.run.id} created and bound: ${r.run.objective}`) + }, + + 'orchestration run-use': async ({ flags, client, cwd, json }) => { + const from = await resolveCoordinatorTerminalHandle(flags, cwd, client) + const result = await callMutation<{ + run: { id: string; objective: string; consumer_generation: number } + }>(client, flags, 'orchestration.runUse', { + id: getRequiredStringFlag(flags, 'id'), + from + }) + printResult(result, json, (r) => `Using Run ${r.run.id}: ${r.run.objective}`) + }, + + 'orchestration run-current': async ({ flags, client, cwd, json }) => { + const from = await resolveCoordinatorTerminalHandle(flags, cwd, client) + const result = await client.call<{ + run: { id: string; objective: string } | null + }>('orchestration.runCurrent', { from }) + printResult(result, json, (r) => + r.run ? `${r.run.id} ${r.run.objective}` : 'No Run is bound to this terminal.' + ) + }, + + 'orchestration run-list': async ({ client, json }) => { + const result = await client.call<{ + runs: { id: string; objective: string; legacy: number }[] + }>('orchestration.runList', {}) + printResult(result, json, (r) => + r.runs.length === 0 + ? 'No Runs found.' + : r.runs + .map( + (run) => `${run.id}${run.legacy ? ' [legacy, inspect only]' : ''} ${run.objective}` + ) + .join('\n') + ) + }, + + 'orchestration run-show': async ({ flags, client, json }) => { + const result = await client.call<{ + run: { + id: string + objective: string + consumer_generation: number + legacy: number + created_at: string + } + }>('orchestration.runShow', { id: getRequiredStringFlag(flags, 'id') }) + printResult( + result, + json, + (r) => + `${r.run.id}${r.run.legacy ? ' [legacy, inspect only]' : ''} ${r.run.objective}\n` + + `consumer generation ${r.run.consumer_generation}; created ${r.run.created_at}` + ) + }, + 'orchestration send': async ({ flags, client, cwd, json }) => { - const to = getRequiredStringFlag(flags, 'to') + const to = getOptionalStringFlag(flags, 'to') const type = getOptionalStringFlag(flags, 'type') - rejectLifecycleGroupRecipient(type, to) + if (to) { + rejectLifecycleGroupRecipient(type, to) + } + const outcome = getOptionalStringFlag(flags, 'outcome') + if (type === 'worker_done' && outcome === undefined && !flags.has('payload')) { + throw new RuntimeClientError( + 'invalid_argument', + 'worker_done requires --outcome succeeded or --outcome failed. No effects were applied.', + orchestrationSkillRecoveryData() + ) + } + if (type !== 'worker_done' && outcome !== undefined) { + throw new RuntimeClientError( + 'invalid_argument', + '--outcome is only valid with --type worker_done.' + ) + } if ( (type === 'worker_done' || type === 'heartbeat') && @@ -357,9 +501,10 @@ export const ORCHESTRATION_HANDLERS: Record = { // Why: lifecycle senders keep ORCA_TERMINAL_HANDLE verbatim — no liveness probe (worker_done must survive the mid-restart window) and no remint (older runtimes require from === the stale assignee_handle). const from = await resolveOrchestrationTerminalHandle(flags, cwd, client, 'from') - const result = await client.call('orchestration.send', { + const sendParams = { from, to, + run: getOptionalStringFlag(flags, 'run'), subject: getRequiredStringFlag(flags, 'subject'), body: getOptionalStringFlag(flags, 'body'), type, @@ -369,7 +514,15 @@ export const ORCHESTRATION_HANDLERS: Record = { // Why: pane key is the remint-stable sender identity the runtime verifies lifecycle ownership against; older runtimes strip it. senderPaneKey: process.env.ORCA_PANE_KEY || undefined, devMode: isDevCliInvocation() - }) + } + const dispatchCapability = getOptionalStringFlag(flags, 'dispatch-capability') + const result = await callMutation( + client, + flags, + 'orchestration.send', + sendParams, + dispatchCapability ? { orchestrationCapability: dispatchCapability } : undefined + ) if ('message' in result.result && result.result.lifecycle?.action === 'rejected') { // Why: a rejected lifecycle signal isn't completion; non-zero exit stops workers from treating it as such. process.exitCode = 1 @@ -381,6 +534,12 @@ export const ORCHESTRATION_HANDLERS: Record = { } return `Sent ${r.message.id}` } + if ('relay' in r) { + if (r.relay.destination === 'worker') { + return `Queued ${r.relay.messageId} for worker Dispatch ${r.relay.dispatchId}` + } + return `Queued ${r.relay.messageId} for Run home (Dispatch ${r.relay.dispatchId})` + } return `Sent ${r.messages.length} messages to ${r.recipients} recipients` }) }, @@ -404,17 +563,24 @@ export const ORCHESTRATION_HANDLERS: Record = { messages: MessageSummary[] count: number formatted?: string + deliveryId?: string | null + runId?: string + timedOut?: boolean + cancelled?: boolean + connectionLost?: boolean } let result: Awaited>> try { - result = await client.call('orchestration.check', { + result = await callMutation(client, flags, 'orchestration.check', { terminal, // Why: peek also sends unread:false so pre-peek runtimes degrade to non-consuming all mode instead of destructive mark-read. unread: flags.has('unread') ? true : peek ? false : undefined, peek: peek ? true : undefined, all: flags.has('all') ? true : undefined, types: getOptionalStringFlag(flags, 'types'), - inject: flags.has('inject') ? true : undefined, + format: flags.has('format') ? true : undefined, + run: getOptionalStringFlag(flags, 'run'), + ack: getOptionalStringFlag(flags, 'ack'), wait: wait ? true : undefined, timeoutMs }) @@ -454,21 +620,36 @@ export const ORCHESTRATION_HANDLERS: Record = { return r.formatted } if (r.count === 0) { + if (r.timedOut) { + return 'Wait timed out; no messages were consumed.' + } + if (r.cancelled) { + return r.connectionLost + ? 'Wait cancelled because the connection closed; no messages were consumed.' + : 'Wait cancelled; no messages were consumed.' + } return 'No messages.' } - return r.messages + const rendered = r.messages .map((m) => `${m.id} [${m.type ?? 'status'}] from=${m.from_handle} "${m.subject}"`) .join('\n') + return r.deliveryId ? `Delivery ${r.deliveryId}\n${rendered}` : rendered }) }, 'orchestration reply': async ({ flags, client, cwd, json }) => { const from = await resolveOrchestrationTerminalHandle(flags, cwd, client, 'from') - const result = await client.call<{ message: { id: string } }>('orchestration.reply', { - id: getRequiredStringFlag(flags, 'id'), - body: getRequiredStringFlag(flags, 'body'), - from - }) + const result = await callMutation<{ message: { id: string } }>( + client, + flags, + 'orchestration.reply', + { + id: getRequiredStringFlag(flags, 'id'), + body: getRequiredStringFlag(flags, 'body'), + run: getOptionalStringFlag(flags, 'run'), + from + } + ) printResult(result, json, (r) => `Replied ${r.message.id}`) }, @@ -505,9 +686,11 @@ export const ORCHESTRATION_HANDLERS: Record = { }) }, - 'orchestration task-create': async ({ flags, client, json }) => { - const callerTerminalHandle = await resolveTaskCreatorTerminalHandle(client) - const result = await client.call<{ task: { id: string; status: string } }>( + 'orchestration task-create': async ({ flags, client, cwd, json }) => { + const callerTerminalHandle = await resolveCoordinatorTerminalHandle(flags, cwd, client) + const result = await callMutation<{ task: { id: string; status: string } }>( + client, + flags, 'orchestration.taskCreate', { spec: getRequiredStringFlag(flags, 'spec'), @@ -515,14 +698,19 @@ export const ORCHESTRATION_HANDLERS: Record = { displayName: getOptionalStringFlag(flags, 'display-name'), deps: getOptionalStringFlag(flags, 'deps'), parent: getOptionalStringFlag(flags, 'parent'), + run: getOptionalStringFlag(flags, 'run'), callerTerminalHandle } ) printResult(result, json, (r) => `Created ${r.task.id} [${r.task.status}]`) }, - 'orchestration task-list': async ({ flags, client, json }) => { + 'orchestration task-list': async ({ flags, client, cwd, json }) => { const brief = flags.has('brief') + const run = getOptionalStringFlag(flags, 'run') + const callerTerminalHandle = run + ? undefined + : await resolveCoordinatorTerminalHandle(flags, cwd, client) const result = await client.call<{ tasks: { id: string @@ -535,10 +723,14 @@ export const ORCHESTRATION_HANDLERS: Record = { spec_truncated?: boolean }[] count: number + runId?: string + legacyReadOnly?: boolean }>('orchestration.taskList', { status: getOptionalStringFlag(flags, 'status'), ready: flags.has('ready') ? true : undefined, - brief: brief ? true : undefined + brief: brief ? true : undefined, + run, + callerTerminalHandle }) // Why: only older runtimes (no spec_truncated) skip server-side abbreviation and need this client-side fallback. const needsClientAbbreviation = @@ -551,9 +743,9 @@ export const ORCHESTRATION_HANDLERS: Record = { : result printResult(output, json, (r) => { if (r.count === 0) { - return 'No tasks.' + return r.legacyReadOnly ? 'No legacy tasks (read-only).' : 'No tasks.' } - return r.tasks + const tasks = r.tasks .map((t) => { const label = t.display_name ?? t.task_title ?? t.spec const head = `${t.id} [${t.status}] ${label.slice(0, 60)}` @@ -563,10 +755,11 @@ export const ORCHESTRATION_HANDLERS: Record = { return head }) .join('\n') + return r.legacyReadOnly ? `Legacy Run ${r.runId} (read-only)\n${tasks}` : tasks }) }, - 'orchestration task-update': async ({ flags, client, json }) => { + 'orchestration task-update': async ({ flags, client, cwd, json }) => { const status = getRequiredStringFlag(flags, 'status') if (!TASK_STATUS_VALUES.includes(status as (typeof TASK_STATUS_VALUES)[number])) { throw new RuntimeClientError( @@ -574,30 +767,149 @@ export const ORCHESTRATION_HANDLERS: Record = { `invalid status '${status}', expected one of: ${TASK_STATUS_VALUES.join(', ')}` ) } - const result = await client.call<{ task: { id: string; status: string } }>( + const result = await callMutation<{ task: { id: string; status: string } }>( + client, + flags, 'orchestration.taskUpdate', { id: getRequiredStringFlag(flags, 'id'), status, - result: getOptionalStringFlag(flags, 'result') + result: getOptionalStringFlag(flags, 'result'), + run: getOptionalStringFlag(flags, 'run'), + callerTerminalHandle: await resolveCoordinatorTerminalHandle(flags, cwd, client) } ) printResult(result, json, (r) => `Updated ${r.task.id} -> ${r.task.status}`) }, + 'orchestration worker-start': async ({ flags, client, cwd, json }) => { + const result = await callMutation<{ + runId: string + taskId: string + dispatchId: string + state: string + failedStage?: string + lastError?: string + effects: unknown[] + residualResources: unknown[] + }>(client, flags, 'orchestration.workerStart', { + task: getRequiredStringFlag(flags, 'task'), + on: getOptionalStringFlag(flags, 'on'), + worktree: getOptionalStringFlag(flags, 'worktree'), + name: getOptionalStringFlag(flags, 'name'), + repo: getOptionalStringFlag(flags, 'repo'), + baseBranch: getOptionalStringFlag(flags, 'base-branch'), + displayName: getOptionalStringFlag(flags, 'display-name'), + comment: getOptionalStringFlag(flags, 'comment'), + setup: getOptionalStringFlag(flags, 'setup'), + agent: getOptionalStringFlag(flags, 'agent'), + terminal: getOptionalStringFlag(flags, 'terminal'), + retryOf: getOptionalStringFlag(flags, 'retry-of'), + timeoutMs: getOptionalPositiveIntegerValueFlag(flags, 'timeout-ms'), + run: getOptionalStringFlag(flags, 'run'), + from: await resolveCoordinatorTerminalHandle(flags, cwd, client), + devMode: isDevCliInvocation() + }) + if (result.result.state !== 'ready') { + process.exitCode = 1 + } + printResult(result, json, (worker) => { + const base = `Worker ${worker.dispatchId} [${worker.state}] for ${worker.taskId}` + return worker.lastError + ? `${base}\n${worker.failedStage ?? 'start'}: ${worker.lastError}` + : base + }) + }, + + 'orchestration worker-show': async ({ flags, client, json }) => { + const result = await client.call<{ + dispatch: { id: string; task_id: string; status: string } + worker: { state: string; stage: string; agent_terminal_handle: string | null } + }>('orchestration.workerShow', { + dispatch: getRequiredStringFlag(flags, 'dispatch') + }) + printResult( + result, + json, + (value) => + `${value.dispatch.id} task=${value.dispatch.task_id} [${value.worker.state}] stage=${value.worker.stage}` + ) + }, + + 'orchestration worker-read': async ({ flags, client, json }) => { + const cursorFlag = getOptionalStringFlag(flags, 'cursor') + const cursor = + cursorFlag !== undefined && /^\d+$/.test(cursorFlag) + ? Number.parseInt(cursorFlag, 10) + : cursorFlag + const source = getOptionalStringFlag(flags, 'source') + if (source && !['auto', 'transcript', 'terminal'].includes(source)) { + throw new RuntimeClientError( + 'invalid_argument', + '--source must be auto, transcript, or terminal' + ) + } + const result = await client.call( + 'orchestration.workerRead', + { + dispatch: getRequiredStringFlag(flags, 'dispatch'), + cursor, + limit: getOptionalPositiveIntegerFlag(flags, 'limit'), + source: source as OrchestrationWorkerReadSource | undefined + } + ) + printResult(result, json, formatWorkerRead) + }, + + 'orchestration worker-stop': async ({ flags, client, json }) => { + const result = await callMutation<{ + dispatchId: string + state: string + processAction: string + lastError?: string + }>(client, flags, 'orchestration.workerStop', { + dispatch: getRequiredStringFlag(flags, 'dispatch') + }) + if (result.result.state === 'stop_unknown') { + process.exitCode = 1 + } + printResult( + result, + json, + (value) => + `Worker ${value.dispatchId} [${value.state}] process=${value.processAction}${value.lastError ? `\n${value.lastError}` : ''}` + ) + }, + + 'orchestration worker-abandon': async ({ flags, client, json }) => { + const result = await callMutation<{ + dispatchId: string + state: string + warning: string + }>(client, flags, 'orchestration.workerAbandon', { + dispatch: getRequiredStringFlag(flags, 'dispatch') + }) + printResult( + result, + json, + (value) => `Worker ${value.dispatchId} [${value.state}]\nWarning: ${value.warning}` + ) + }, + 'orchestration dispatch': async ({ flags, client, cwd, json }) => { const from = await resolveCoordinatorTerminalHandle(flags, cwd, client) const dryRun = flags.has('dry-run') ? true : undefined const returnPreamble = flags.has('return-preamble') ? true : undefined // Why: --to is only required for non-dry-run; the RPC handler re-enforces. const to = dryRun ? getOptionalStringFlag(flags, 'to') : getRequiredStringFlag(flags, 'to') - const result = await client.call<{ + const result = await callMutation<{ dispatch: { id: string; task_id: string; status: string } | null injected?: boolean dryRun?: boolean preamble?: string - }>('orchestration.dispatch', { + }>(client, flags, 'orchestration.dispatch', { task: getRequiredStringFlag(flags, 'task'), + run: getOptionalStringFlag(flags, 'run'), to, from, inject: flags.has('inject') ? true : undefined, @@ -618,23 +930,46 @@ export const ORCHESTRATION_HANDLERS: Record = { const parsedTimeoutMs = getOptionalPositiveIntegerValueFlag(flags, 'timeout-ms') const timeoutMs = clampOrchestrationAskTimeoutMs(parsedTimeoutMs) const from = await resolveOrchestrationTerminalHandle(flags, cwd, client, 'from') - const result = await client.call<{ + const question = getOptionalStringFlag(flags, 'question') + const resume = getOptionalStringFlag(flags, 'resume') + if ((question ? 1 : 0) + (resume ? 1 : 0) !== 1) { + throw new RuntimeClientError( + 'invalid_argument', + 'Choose exactly one of --question or --resume.' + ) + } + if (resume && flags.has('options')) { + throw new RuntimeClientError( + 'invalid_argument', + '--options is only valid when creating a new question.' + ) + } + const result = await callMutation<{ answer: string | null messageId: string | null threadId: string timedOut: boolean timeoutMs?: number + cancelled?: boolean + connectionLost?: boolean }>( + client, + flags, 'orchestration.ask', { - to: getRequiredStringFlag(flags, 'to'), - question: getRequiredStringFlag(flags, 'question'), + to: getOptionalStringFlag(flags, 'to'), + run: getOptionalStringFlag(flags, 'run'), + question, + resume, options: getOptionalStringFlag(flags, 'options'), timeoutMs: parsedTimeoutMs === undefined ? undefined : timeoutMs, from }, // Why: extend past timeoutMs so the RPC transport's 60s default doesn't abort before the runtime's own timeout resolves. - { timeoutMs: resolveOrchestrationAskClientTimeoutMs(parsedTimeoutMs) } + { + timeoutMs: resolveOrchestrationAskClientTimeoutMs(parsedTimeoutMs), + orchestrationCapability: getOptionalStringFlag(flags, 'dispatch-capability') + } ) // Why: bypass printResult so --json emits a bare JSON object (no envelope) pipeable via `jq -r .answer`, unlike other verbs. if (json) { @@ -650,6 +985,16 @@ export const ORCHESTRATION_HANDLERS: Record = { } process.exitCode = 1 } + if (result.result.cancelled) { + if (!json) { + console.error( + result.result.connectionLost + ? `ask connection closed (question ${result.result.messageId})` + : `ask cancelled (question ${result.result.messageId})` + ) + } + process.exitCode = 1 + } }, 'orchestration dispatch-show': async ({ flags, client, cwd, json }) => { @@ -678,33 +1023,26 @@ export const ORCHESTRATION_HANDLERS: Record = { }) }, - 'orchestration run': async ({ flags, client, cwd, json }) => { - const from = await resolveCoordinatorTerminalHandle(flags, cwd, client) - const result = await client.call<{ - runId: string - status: string - }>('orchestration.run', { - spec: getRequiredStringFlag(flags, 'spec'), - from, - pollIntervalMs: getOptionalPositiveIntegerFlag(flags, 'poll-interval-ms'), - maxConcurrent: getOptionalPositiveIntegerFlag(flags, 'max-concurrent'), - worktree: getOptionalStringFlag(flags, 'worktree') - }) - printResult(result, json, (r) => `Run ${r.runId} started (${r.status})`) + 'orchestration coordinator-start': async () => { + throw new RuntimeClientError( + 'orchestration_migration_required', + 'The legacy automatic coordinator command is retired. No effects were applied.', + orchestrationMigrationData('command_retired') + ) }, - 'orchestration run-stop': async ({ client, json }) => { - const result = await client.call<{ - runId: string - stopped: boolean - }>('orchestration.runStop', {}) - printResult(result, json, (r) => `Run ${r.runId} stopped`) + 'orchestration coordinator-stop': async () => { + throw new RuntimeClientError( + 'orchestration_migration_required', + 'The legacy automatic coordinator command is retired. No effects were applied.', + orchestrationMigrationData('command_retired') + ) }, 'orchestration gate-create': async ({ flags, client, json }) => { - const result = await client.call<{ + const result = await callMutation<{ gate: { id: string; task_id: string; status: string } - }>('orchestration.gateCreate', { + }>(client, flags, 'orchestration.gateCreate', { task: getRequiredStringFlag(flags, 'task'), question: getRequiredStringFlag(flags, 'question'), options: getOptionalStringFlag(flags, 'options') @@ -717,9 +1055,9 @@ export const ORCHESTRATION_HANDLERS: Record = { }, 'orchestration gate-resolve': async ({ flags, client, json }) => { - const result = await client.call<{ + const result = await callMutation<{ gate: { id: string; task_id: string; status: string; resolution: string } - }>('orchestration.gateResolve', { + }>(client, flags, 'orchestration.gateResolve', { id: getRequiredStringFlag(flags, 'id'), resolution: getRequiredStringFlag(flags, 'resolution') }) @@ -745,9 +1083,17 @@ export const ORCHESTRATION_HANDLERS: Record = { }, 'orchestration reset': async ({ flags, client, json }) => { - const hasScopeFlag = flags.has('all') || flags.has('tasks') || flags.has('messages') - const result = await client.call<{ reset: string }>('orchestration.reset', { - all: flags.has('all') || !hasScopeFlag ? true : undefined, + const scopeCount = [flags.has('all'), flags.has('tasks'), flags.has('messages')].filter( + Boolean + ).length + if (scopeCount !== 1) { + throw new RuntimeClientError( + 'invalid_argument', + 'Choose exactly one reset scope: --all, --tasks, or --messages.' + ) + } + const result = await callMutation<{ reset: string }>(client, flags, 'orchestration.reset', { + all: flags.has('all') ? true : undefined, tasks: flags.has('tasks') ? true : undefined, messages: flags.has('messages') ? true : undefined }) diff --git a/src/cli/help.ts b/src/cli/help.ts index f133c95d2af..e7d5c3c24ac 100644 --- a/src/cli/help.ts +++ b/src/cli/help.ts @@ -85,8 +85,14 @@ Terminals: terminal close Close a terminal pane/session, or its whole tab with --tab Orchestration: + orchestration run-create Create and bind a lightweight orchestration Run + orchestration run-use Bind this coordinator terminal to an existing Run + orchestration run-current Show this terminal's bound Run + orchestration run-list List lightweight orchestration Runs + orchestration run-show Show one lightweight orchestration Run orchestration send Send an inter-agent message - orchestration check Check messages for a terminal + orchestration check Check the bound Run mailbox + orchestration ask Ask the coordinator a blocking question orchestration reply Reply to a message orchestration inbox Show all messages across recipients orchestration task-create Create an orchestration task @@ -94,8 +100,13 @@ Orchestration: orchestration task-update Update a task status orchestration dispatch Dispatch a task to a terminal orchestration dispatch-show Show dispatch context for a task - orchestration run Start the coordinator loop - orchestration run-stop Stop the active coordinator run + orchestration worker-start Start a supervised worker locally or on a connected Orca server + orchestration worker-show Inspect one supervised worker + orchestration worker-read Read bounded output from one supervised worker + orchestration worker-stop Stop one supervised worker + orchestration worker-abandon Fence an uncertain worker without claiming it stopped + orchestration coordinator-start Start the legacy automatic coordinator loop + orchestration coordinator-stop Stop the legacy automatic coordinator loop orchestration gate-create Create a decision gate blocking a task orchestration gate-resolve Resolve a pending decision gate orchestration gate-list List decision gates @@ -419,6 +430,9 @@ function formatCommandFlagHelp(flag: string, commandPath: string[]): string { if (command === 'linear list-issues' && flag === 'cursor') { return '--cursor Opaque cursor returned by a previous list-issues page' } + if (command === 'orchestration worker-read' && flag === 'cursor') { + return '--cursor Opaque cursor returned by a previous worker-read page' + } if (command === 'linear list-issues' && flag === 'workspace') { return '--workspace Connected Linear workspace id, or all' } diff --git a/src/cli/index.test.ts b/src/cli/index.test.ts index 525eea46edf..0725fcff419 100644 --- a/src/cli/index.test.ts +++ b/src/cli/index.test.ts @@ -353,6 +353,15 @@ describe('orca root help', () => { expect(logSpy.mock.calls[0][0]).toContain( 'orca terminal create --worktree active --command "codex"' ) + expect(logSpy.mock.calls[0][0]).toContain( + 'orchestration worker-start Start a supervised worker locally or on a connected Orca server' + ) + expect(logSpy.mock.calls[0][0]).toContain( + 'orchestration ask Ask the coordinator a blocking question' + ) + expect(logSpy.mock.calls[0][0]).toContain( + 'orchestration worker-abandon Fence an uncertain worker without claiming it stopped' + ) expect(callMock).not.toHaveBeenCalled() }) @@ -408,6 +417,20 @@ describe('orca root help', () => { expect(callMock).not.toHaveBeenCalled() }) + it('describes worker-read cursors as opaque', async () => { + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}) + logSpy.mockClear() + + await main(['orchestration', 'worker-read', '--help'], '/tmp/repo') + + const help = String(logSpy.mock.calls[0][0]) + expect(help).toContain( + '--cursor Opaque cursor returned by a previous worker-read page' + ) + expect(help).not.toContain('Line cursor from a previous read') + expect(callMock).not.toHaveBeenCalled() + }) + it('advertises Linear issue linking on worktree create and set help', async () => { const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}) logSpy.mockClear() @@ -501,6 +524,7 @@ describe('orca root help', () => { describe('orca cli worktree awareness', () => { const originalTerminalHandle = process.env.ORCA_TERMINAL_HANDLE const originalUserDataPath = process.env.ORCA_USER_DATA_PATH + const originalDevCliInvocation = process.env.ORCA_DEV_CLI_INVOCATION const originalPairingCode = process.env.ORCA_PAIRING_CODE const originalRemotePairing = process.env.ORCA_REMOTE_PAIRING const originalEnvironment = process.env.ORCA_ENVIRONMENT @@ -511,6 +535,7 @@ describe('orca cli worktree awareness', () => { callMock.mockReset() delete process.env.ORCA_TERMINAL_HANDLE delete process.env.ORCA_USER_DATA_PATH + delete process.env.ORCA_DEV_CLI_INVOCATION delete process.env.ORCA_WORKSPACE_ID delete process.env.ORCA_WORKTREE_ID // Isolate the pane key so claude-teams tests that set it don't leak a @@ -555,6 +580,11 @@ describe('orca cli worktree awareness', () => { } else { process.env.ORCA_USER_DATA_PATH = originalUserDataPath } + if (originalDevCliInvocation === undefined) { + delete process.env.ORCA_DEV_CLI_INVOCATION + } else { + process.env.ORCA_DEV_CLI_INVOCATION = originalDevCliInvocation + } if (originalPairingCode === undefined) { delete process.env.ORCA_PAIRING_CODE } else { @@ -3489,17 +3519,11 @@ describe('orca cli worktree awareness', () => { expect(logSpy).toHaveBeenCalledWith('Sent 2 messages to 2 recipients') }) - it('passes all reset scope explicitly for no-flag orchestration reset', async () => { - callMock.mockResolvedValueOnce(okFixture('req_reset', { reset: 'all' })) - vi.spyOn(console, 'log').mockImplementation(() => {}) - + it('rejects no-flag orchestration reset before calling the runtime', async () => { await main(['orchestration', 'reset'], '/tmp/repo') - expect(callMock).toHaveBeenCalledWith('orchestration.reset', { - all: true, - tasks: undefined, - messages: undefined - }) + expect(callMock).not.toHaveBeenCalled() + expect(process.exitCode).toBe(1) }) it.each([ @@ -3517,16 +3541,6 @@ describe('orca cli worktree awareness', () => { args: ['orchestration', 'reset', '--messages'], params: { all: undefined, tasks: undefined, messages: true }, reset: 'messages' - }, - { - args: ['orchestration', 'reset', '--tasks', '--messages'], - params: { all: undefined, tasks: true, messages: true }, - reset: 'tasks' - }, - { - args: ['orchestration', 'reset', '--all', '--tasks'], - params: { all: true, tasks: true, messages: undefined }, - reset: 'all' } ])('passes explicit reset flags through for $args', async ({ args, params, reset }) => { callMock.mockResolvedValueOnce(okFixture('req_reset', { reset })) @@ -3537,6 +3551,16 @@ describe('orca cli worktree awareness', () => { expect(callMock).toHaveBeenCalledWith('orchestration.reset', params) }) + it.each([ + ['orchestration', 'reset', '--tasks', '--messages'], + ['orchestration', 'reset', '--all', '--tasks'] + ])('rejects conflicting reset scopes for $args', async (...args) => { + await main(args, '/tmp/repo') + + expect(callMock).not.toHaveBeenCalled() + expect(process.exitCode).toBe(1) + }) + it('rejects unknown task-update status with an enum-aware error', async () => { process.env.ORCA_TERMINAL_HANDLE = 'term_coord' const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}) @@ -3629,6 +3653,33 @@ describe('orca cli worktree awareness', () => { }) }) + it('passes dev mode from an explicit dev CLI marker with a custom profile path', async () => { + process.env.ORCA_TERMINAL_HANDLE = 'term_sender' + process.env.ORCA_USER_DATA_PATH = '/tmp/federation-acceptance-profile' + process.env.ORCA_DEV_CLI_INVOCATION = '1' + callMock.mockResolvedValueOnce({ + id: 'req_dispatch', + ok: true, + result: { + dispatch: { id: 'ctx_1', task_id: 'task_1', status: 'dispatched' } + }, + _meta: { + runtimeId: 'runtime-1' + } + }) + vi.spyOn(console, 'log').mockImplementation(() => {}) + + await main( + ['orchestration', 'dispatch', '--task', 'task_1', '--to', 'term_worker', '--inject'], + '/tmp/repo' + ) + + expect(callMock).toHaveBeenCalledWith( + 'orchestration.dispatch', + expect.objectContaining({ devMode: true }) + ) + }) + it('uses the resolved enclosing worktree for terminal consumers', async () => { queueFixtures( callMock, diff --git a/src/cli/runtime-client.test.ts b/src/cli/runtime-client.test.ts index c11292453c1..c15e9e1e29c 100644 --- a/src/cli/runtime-client.test.ts +++ b/src/cli/runtime-client.test.ts @@ -3,6 +3,7 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { createServer, type Socket } from 'node:net' import { afterEach, describe, expect, it, vi } from 'vitest' +import { ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY } from '../shared/protocol-version' import { RuntimeClient, RuntimeRpcFailureError } from './runtime-client' import { launchOrcaApp } from './runtime/launch' @@ -74,6 +75,87 @@ function findUnusedPid(seed = 200_000): number { // Windows does not support Unix domain sockets in the same way, causing // EACCES errors on listen(), so the suite is skipped on that platform. describe.skipIf(process.platform === 'win32')('RuntimeClient', () => { + it('adds an opaque durable request ID only to orchestration mutations', async () => { + const userDataPath = mkdtempSync(join(tmpdir(), 'orca-runtime-client-')) + const endpoint = join(userDataPath, 'runtime.sock') + const requests: Record[] = [] + const server = createServer((socket) => { + sockets.add(socket) + socket.once('close', () => sockets.delete(socket)) + socket.once('data', (data) => { + const request = JSON.parse(String(data).trim()) as Record + requests.push(request) + const result = + request.method === 'status.get' + ? { capabilities: [ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY] } + : {} + socket.write( + `${JSON.stringify({ + id: request.id, + ok: true, + result, + _meta: { runtimeId: 'runtime-1' } + })}\n` + ) + }) + }) + servers.add(server) + await new Promise((resolve) => server.listen(endpoint, resolve)) + writeMetadata(userDataPath, endpoint) + + const client = new RuntimeClient(userDataPath, 500) + await client.call( + 'orchestration.send', + { subject: 'hello' }, + { + orchestrationRequestId: 'mutation_explicit' + } + ) + await client.call('orchestration.taskList', {}) + + expect(requests[0]?.method).toBe('status.get') + expect(requests[1]?.orchestrationRequestId).toBe('mutation_explicit') + expect(requests[1]?.orchestrationContractVersion).toBe(1) + expect(requests[2]?.orchestrationRequestId).toBeUndefined() + }) + + it('rejects an old local runtime before sending an orchestration mutation', async () => { + const userDataPath = mkdtempSync(join(tmpdir(), 'orca-runtime-client-')) + const endpoint = join(userDataPath, 'runtime.sock') + const requests: Record[] = [] + const server = createServer((socket) => { + sockets.add(socket) + socket.once('close', () => sockets.delete(socket)) + socket.once('data', (data) => { + const request = JSON.parse(String(data).trim()) as Record + requests.push(request) + socket.write( + `${JSON.stringify({ + id: request.id, + ok: true, + result: { capabilities: [] }, + _meta: { runtimeId: 'runtime-1' } + })}\n` + ) + }) + }) + servers.add(server) + await new Promise((resolve) => server.listen(endpoint, resolve)) + writeMetadata(userDataPath, endpoint) + + const client = new RuntimeClient(userDataPath, 500) + await expect(client.call('orchestration.send', { subject: 'hello' })).rejects.toMatchObject({ + code: 'orchestration_migration_required', + data: { + reason: 'runtime_capability_missing', + effectsApplied: false, + nextCommandArgs: ['skills', 'get', 'orchestration', '--full'] + } + }) + expect(requests).toHaveLength(1) + expect(requests[0]?.method).toBe('status.get') + }) + it('returns the full RPC envelope for successful calls', async () => { const userDataPath = mkdtempSync(join(tmpdir(), 'orca-runtime-client-')) const endpoint = join(userDataPath, 'runtime.sock') diff --git a/src/cli/runtime/client.ts b/src/cli/runtime/client.ts index 097ec3534bc..bb3cb905701 100644 --- a/src/cli/runtime/client.ts +++ b/src/cli/runtime/client.ts @@ -1,4 +1,10 @@ +import { randomUUID } from 'node:crypto' import type { CliStatusResult, RuntimeStatus } from '../../shared/runtime-types' +import type { RuntimeOrchestrationEnvelope } from '../../shared/runtime-rpc-envelope' +import { + isOrchestrationMutation, + orchestrationMigrationData +} from '../../shared/orchestration-rpc-contract' import { parsePairingCode, type PairingOffer } from '../../shared/pairing' import { launchOrcaApp } from './launch' import { getDefaultUserDataPath, readMetadata } from './metadata' @@ -10,6 +16,8 @@ import { markEnvironmentUsed, resolveEnvironmentPairingOffer } from './environme import { describeRuntimeCompatBlock, evaluateRuntimeCompat } from '../../shared/protocol-compat' import { MIN_COMPATIBLE_RUNTIME_SERVER_VERSION, + ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY, + ORCHESTRATION_CONTRACT_VERSION, RUNTIME_PROTOCOL_VERSION } from '../../shared/protocol-version' @@ -27,6 +35,7 @@ export class RuntimeClient { private readonly remotePairing: PairingOffer | null private readonly environmentSelector: string | null private remoteCompatChecked = false + private orchestrationContractCheck: Promise | null = null // Why: browser commands trigger first-time session init (agent-browser connect + // CDP proxy setup) which can take 15-30s. 60s accommodates cold start without @@ -50,21 +59,39 @@ export class RuntimeClient { async call( method: string, params?: unknown, - options?: { - timeoutMs?: number - } + options?: { timeoutMs?: number } & RuntimeOrchestrationEnvelope ): Promise> { const effectiveTimeoutMs = options?.timeoutMs ?? this.resolveMethodTimeoutMs(method, params) + const orchestrationMutation = isOrchestrationMutation(method, params) + if (orchestrationMutation) { + await this.ensureOrchestrationContractCompatible(effectiveTimeoutMs) + } + const orchestrationRequestId = orchestrationMutation + ? (options?.orchestrationRequestId ?? randomUUID()) + : undefined + const envelope = { + orchestrationCapability: options?.orchestrationCapability, + orchestrationContractVersion: method.startsWith('orchestration.') + ? ORCHESTRATION_CONTRACT_VERSION + : undefined, + orchestrationRequestId + } if (this.remotePairing) { if (method !== 'status.get') { await this.ensureRemoteRuntimeCompatible(effectiveTimeoutMs) } - const response = await sendWebSocketRequest( - this.remotePairing, - method, - params, - effectiveTimeoutMs - ) + let response + try { + response = await sendWebSocketRequest( + this.remotePairing, + method, + params, + effectiveTimeoutMs, + envelope + ) + } catch (error) { + throw attachMutationRecovery(error, orchestrationRequestId) + } if (response.ok === false) { throw new RuntimeRpcFailureError(response) } @@ -76,7 +103,12 @@ export class RuntimeClient { return response } const metadata = readMetadata(this.userDataPath) - const response = await sendRequest(metadata, method, params, effectiveTimeoutMs) + let response + try { + response = await sendRequest(metadata, method, params, effectiveTimeoutMs, envelope) + } catch (error) { + throw attachMutationRecovery(error, orchestrationRequestId) + } if (response.ok === false) { throw new RuntimeRpcFailureError(response) } @@ -165,6 +197,28 @@ export class RuntimeClient { } } + private async ensureOrchestrationContractCompatible(timeoutMs: number): Promise { + if (!this.orchestrationContractCheck) { + this.orchestrationContractCheck = this.checkOrchestrationContractCompatibility(timeoutMs) + } + await this.orchestrationContractCheck + } + + private async checkOrchestrationContractCompatibility(timeoutMs: number): Promise { + const response = await this.call('status.get', undefined, { timeoutMs }) + if (this.remotePairing) { + this.assertRemoteRuntimeStatusCompatible(response.result) + this.remoteCompatChecked = true + } + if (!response.result.capabilities?.includes(ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY)) { + throw new RuntimeClientError( + 'orchestration_migration_required', + 'The connected Orca runtime does not support the current orchestration contract. No effects were applied.', + orchestrationMigrationData('runtime_capability_missing') + ) + } + } + private assertRemoteRuntimeStatusCompatible(status: RuntimeStatus): void { const verdict = evaluateRuntimeCompat({ clientProtocolVersion: RUNTIME_PROTOCOL_VERSION, @@ -213,6 +267,20 @@ export class RuntimeClient { } } +function attachMutationRecovery(error: unknown, requestId: string | undefined): unknown { + if (!requestId || !(error instanceof RuntimeClientError)) { + return error + } + return new RuntimeClientError( + error.code, + `${error.message} Orchestration mutation request ID: ${requestId}.`, + { + ...(error.data && typeof error.data === 'object' ? error.data : {}), + orchestrationRequestId: requestId + } + ) +} + function throwDesktopActivationBlocked(): never { throw new RuntimeClientError( 'desktop_activation_blocked', diff --git a/src/cli/runtime/transport.ts b/src/cli/runtime/transport.ts index 18385cb1ae7..fc6c3339ded 100644 --- a/src/cli/runtime/transport.ts +++ b/src/cli/runtime/transport.ts @@ -1,6 +1,7 @@ import { createConnection } from 'node:net' import { randomUUID } from 'node:crypto' import { findTransport, type RuntimeMetadata } from '../../shared/runtime-bootstrap' +import type { RuntimeOrchestrationEnvelope } from '../../shared/runtime-rpc-envelope' import { isKeepaliveFrame, RuntimeRpcEnvelopeSchema } from './envelope-schema' import { RuntimeClientError, type RuntimeRpcResponse } from './types' import { MAX_TIMER_DELAY_MS, isSafeTimerDelayMs } from '../../shared/timer-delay' @@ -9,7 +10,8 @@ export async function sendRequest( metadata: RuntimeMetadata, method: string, params: unknown, - timeoutMs: number + timeoutMs: number, + envelope?: RuntimeOrchestrationEnvelope ): Promise> { if (!isSafeTimerDelayMs(timeoutMs)) { throw new RuntimeClientError( @@ -183,7 +185,10 @@ export async function sendRequest( id: requestId, authToken: metadata.authToken, method, - params + params, + orchestrationCapability: envelope?.orchestrationCapability, + orchestrationContractVersion: envelope?.orchestrationContractVersion, + orchestrationRequestId: envelope?.orchestrationRequestId })}\n` ) }) diff --git a/src/cli/runtime/websocket-transport.test.ts b/src/cli/runtime/websocket-transport.test.ts index 530b20f1363..d69ce71baea 100644 --- a/src/cli/runtime/websocket-transport.test.ts +++ b/src/cli/runtime/websocket-transport.test.ts @@ -162,6 +162,29 @@ describe('CLI remote WebSocket transport', () => { message: expect.stringContaining('server is too old') }) }) + + it('blocks orchestration mutations when a remote runtime lacks the contract capability', async () => { + const runtime = await startTestRuntime('runtime-old-orchestration', { capabilities: [] }) + servers.push(runtime) + const client = new RuntimeClient( + '/tmp/unused', + 5_000, + encodePairingOffer({ + v: 2, + endpoint: runtime.endpoint, + deviceToken: runtime.deviceToken, + publicKeyB64: runtime.publicKeyB64 + }) + ) + + await expect(client.call('orchestration.send', { subject: 'hello' })).rejects.toMatchObject({ + code: 'orchestration_migration_required', + data: { + reason: 'runtime_capability_missing', + effectsApplied: false + } + }) + }) }) async function startTestRuntime( diff --git a/src/cli/runtime/websocket-transport.ts b/src/cli/runtime/websocket-transport.ts index ac5fc677919..e53fbd670bd 100644 --- a/src/cli/runtime/websocket-transport.ts +++ b/src/cli/runtime/websocket-transport.ts @@ -1,4 +1,5 @@ import type { PairingOffer } from '../../shared/pairing' +import type { RuntimeOrchestrationEnvelope } from '../../shared/runtime-rpc-envelope' import { RemoteRuntimeClientError, sendRemoteRuntimeRequest @@ -9,10 +10,11 @@ export async function sendWebSocketRequest( pairing: PairingOffer, method: string, params: unknown, - timeoutMs: number + timeoutMs: number, + envelope?: RuntimeOrchestrationEnvelope ): Promise> { try { - return await sendRemoteRuntimeRequest(pairing, method, params, timeoutMs) + return await sendRemoteRuntimeRequest(pairing, method, params, timeoutMs, envelope) } catch (error) { if (error instanceof RemoteRuntimeClientError) { throw new RuntimeClientError(error.code, error.message) diff --git a/src/cli/specs/orchestration-worker-specs.ts b/src/cli/specs/orchestration-worker-specs.ts new file mode 100644 index 00000000000..2b05853e058 --- /dev/null +++ b/src/cli/specs/orchestration-worker-specs.ts @@ -0,0 +1,71 @@ +import { GLOBAL_FLAGS, type CommandSpec } from '../args' + +export const ORCHESTRATION_WORKER_COMMAND_SPECS: CommandSpec[] = [ + { + path: ['orchestration', 'worker-start'], + summary: 'Start one supervised worker on the Run home or a connected Orca server', + usage: + 'orca orchestration worker-start --task [--on ] [--worktree ] (--agent | --terminal ) [--name ] [--repo ] [--base-branch ] [--display-name ] [--comment ] [--setup ] [--retry-of ] [--timeout-ms ] [--run ] [--from ] [--retry-request ] [--json]', + allowedFlags: [ + ...GLOBAL_FLAGS, + 'task', + 'on', + 'worktree', + 'name', + 'repo', + 'base-branch', + 'display-name', + 'comment', + 'setup', + 'agent', + 'terminal', + 'retry-of', + 'timeout-ms', + 'run', + 'from', + 'retry-request' + ], + notes: [ + 'Current and existing worktrees never rerun setup; a fresh agent terminal is created unless --terminal is explicit.', + 'New worktrees use agent-first creation and default --setup to run. Repository start-immediately runs setup beside the agent; wait-for-setup gates agent readiness and task input.', + 'Creation flags (--name, --repo, --base-branch, --display-name, --comment, --setup) are rejected for current/existing worktrees. Use exact --repo on the selected server; project/host convenience routing remains on worktree create.', + '--on selects only the worker server; the Run and this command remain on the current Orca server.', + 'Remote current and new-child are invalid; discover an exact remote selector or use new-top-level.', + '--retry-of links the replacement attempt but does not inherit placement; repeat the intended --on/worktree and --agent/terminal choices.', + 'The call exits 0 only for ready. Failed or outcome_unknown exits 1 and JSON includes stage/failedStage, setup, effects, residualResources, and recovery commands when needed.' + ] + }, + { + path: ['orchestration', 'worker-show'], + summary: 'Inspect one supervised worker Dispatch', + usage: 'orca orchestration worker-show --dispatch [--json]', + allowedFlags: [...GLOBAL_FLAGS, 'dispatch'] + }, + { + path: ['orchestration', 'worker-read'], + summary: 'Read bounded output from one supervised worker', + usage: + 'orca orchestration worker-read --dispatch [--source ] [--cursor ] [--limit ] [--json]', + allowedFlags: [...GLOBAL_FLAGS, 'dispatch', 'source', 'cursor', 'limit'], + notes: [ + 'The default auto source uses an exact hook-reported transcript when available and otherwise returns labeled terminal output.', + 'A returned cursor is pinned to the exact source; start a fresh read if Orca reports source_changed.' + ] + }, + { + path: ['orchestration', 'worker-stop'], + summary: 'Fence and stop only one supervised agent terminal', + usage: + 'orca orchestration worker-stop --dispatch [--retry-request ] [--json]', + allowedFlags: [...GLOBAL_FLAGS, 'dispatch', 'retry-request'], + notes: ['Never deletes the worktree, setup terminal, configured tabs, or unrelated processes.'] + }, + { + path: ['orchestration', 'worker-abandon'], + summary: 'Fence a worker without claiming its process stopped', + usage: + 'orca orchestration worker-abandon --dispatch [--retry-request ] [--json]', + allowedFlags: [...GLOBAL_FLAGS, 'dispatch', 'retry-request'], + notes: ['Retains all possibly-live resources and performs no process or filesystem action.'] + } +] diff --git a/src/cli/specs/orchestration.ts b/src/cli/specs/orchestration.ts index bb63f261d00..226f9dcb7f4 100644 --- a/src/cli/specs/orchestration.ts +++ b/src/cli/specs/orchestration.ts @@ -1,15 +1,51 @@ import type { CommandSpec } from '../args' import { GLOBAL_FLAGS } from '../args' +import { ORCHESTRATION_WORKER_COMMAND_SPECS } from './orchestration-worker-specs' export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [ + { + path: ['orchestration', 'run-create'], + summary: 'Create and bind a lightweight orchestration Run', + usage: 'orca orchestration run-create --objective [--from ] [--json]', + allowedFlags: [...GLOBAL_FLAGS, 'objective', 'from', 'retry-request'], + notes: [ + 'A Run is a namespace and home inbox. It never schedules or places workers.', + '--retry-request is only for exact recovery after an unknown mutation result.' + ] + }, + { + path: ['orchestration', 'run-use'], + summary: 'Bind this coordinator terminal to an existing Run', + usage: 'orca orchestration run-use --id [--from ] [--json]', + allowedFlags: [...GLOBAL_FLAGS, 'id', 'from', 'retry-request'] + }, + { + path: ['orchestration', 'run-current'], + summary: 'Show the Run bound to this coordinator terminal', + usage: 'orca orchestration run-current [--from ] [--json]', + allowedFlags: [...GLOBAL_FLAGS, 'from'] + }, + { + path: ['orchestration', 'run-list'], + summary: 'List lightweight orchestration Runs', + usage: 'orca orchestration run-list [--json]', + allowedFlags: [...GLOBAL_FLAGS] + }, + { + path: ['orchestration', 'run-show'], + summary: 'Show one lightweight orchestration Run', + usage: 'orca orchestration run-show --id [--json]', + allowedFlags: [...GLOBAL_FLAGS, 'id'] + }, { path: ['orchestration', 'send'], summary: 'Send an inter-agent message', usage: - 'orca orchestration send --to --subject [--from ] [--body ] [--type ] [--priority ] [--thread-id ] [--payload ] [--task-id ] [--dispatch-id ] [--files-modified ] [--report-path ] [--phase ] [--json]', + 'orca orchestration send --subject [--to ] [--run ] [--from ] [--body ] [--type ] [--priority ] [--thread-id ] [--payload ] [--task-id ] [--dispatch-id ] [--outcome ] [--files-modified ] [--report-path ] [--phase ] [--json]', allowedFlags: [ ...GLOBAL_FLAGS, 'to', + 'run', 'from', 'subject', 'body', @@ -19,13 +55,19 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [ 'payload', 'task-id', 'dispatch-id', + 'dispatch-capability', + 'retry-request', + 'outcome', 'files-modified', 'report-path', 'phase' ], notes: [ 'On Windows PowerShell, quote group addresses such as --to "@all" or --to "@worktree:".', - 'worker_done and heartbeat must target a concrete coordinator terminal handle; use status for broadcast updates.', + "worker_done and heartbeat are exact-Dispatch signals and cannot target groups; omit --to to use the Dispatch's Run mailbox.", + 'worker_done requires --outcome succeeded or --outcome failed.', + 'From an active Dispatch, an omitted recipient defaults to its owning Run mailbox.', + 'Use --to dispatch: for attempt-specific coordinator guidance; Orca durably relays it to a connected worker server.', 'A worker_done with the active task/dispatch IDs completes that task only from the dispatched pane. When stable pane identity is unavailable, the sender handle must exactly match the dispatch assignee; injected preambles include the correct --from value.', 'Prefer --task-id/--dispatch-id/etc. over raw --payload JSON in worker commands; PowerShell strips JSON quotes easily.' ] @@ -34,8 +76,9 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [ path: ['orchestration', 'check'], summary: 'Check messages for a terminal', usage: - 'orca orchestration check [--terminal ] [--unread | --peek | --all] [--types ] [--inject] [--wait] [--timeout-ms ] [--json]\n' + - ' --unread (default): return only unread messages and mark them read.\n' + + 'orca orchestration check [--terminal ] [--run ] [--ack ] [--unread | --peek | --all] [--types ] [--format] [--wait] [--timeout-ms ] [--json]\n' + + " default: return the bound Run's oldest unacknowledged FIFO batch.\n" + + ' --ack: acknowledge the prior whole batch before checking/waiting.\n' + ' --peek: return only unread messages without marking them read.\n' + ' --all: return every message for the handle; does not mark read.\n' + ' --wait: block until a matching message arrives or --timeout-ms expires.\n' + @@ -46,23 +89,29 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [ allowedFlags: [ ...GLOBAL_FLAGS, 'terminal', + 'run', + 'ack', 'unread', 'peek', 'all', 'types', - 'inject', + 'format', 'wait', - 'timeout-ms' + 'timeout-ms', + 'retry-request' ], notes: [ - 'On Windows PowerShell, quote comma-separated type filters, e.g. --types "worker_done,escalation".' + 'On Windows PowerShell, quote comma-separated type filters, e.g. --types "worker_done,escalation".', + '--format renders the returned rows as local text only; it never writes to another terminal.', + 'A bound Run replays the same Delivery until --ack; process every message before acknowledging.' ] }, { path: ['orchestration', 'reply'], summary: 'Reply to a message', - usage: 'orca orchestration reply --id --body [--from ] [--json]', - allowedFlags: [...GLOBAL_FLAGS, 'id', 'body', 'from'] + usage: + 'orca orchestration reply --id --body [--run ] [--from ] [--json]', + allowedFlags: [...GLOBAL_FLAGS, 'id', 'body', 'run', 'from', 'retry-request'] }, { path: ['orchestration', 'inbox'], @@ -74,30 +123,52 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [ path: ['orchestration', 'task-create'], summary: 'Create an orchestration task', usage: - 'orca orchestration task-create --spec [--task-title ] [--display-name ] [--deps ] [--parent ] [--json]', - allowedFlags: [...GLOBAL_FLAGS, 'spec', 'task-title', 'display-name', 'deps', 'parent'] + 'orca orchestration task-create --spec [--task-title ] [--display-name ] [--deps ] [--parent ] [--run ] [--from ] [--json]', + allowedFlags: [ + ...GLOBAL_FLAGS, + 'spec', + 'task-title', + 'display-name', + 'deps', + 'parent', + 'run', + 'from', + 'retry-request' + ] }, { path: ['orchestration', 'task-list'], summary: 'List orchestration tasks', - usage: 'orca orchestration task-list [--status ] [--ready] [--brief] [--json]', - allowedFlags: [...GLOBAL_FLAGS, 'status', 'ready', 'brief'], + usage: + 'orca orchestration task-list [--status ] [--ready] [--brief] [--run ] [--from ] [--json]', + allowedFlags: [...GLOBAL_FLAGS, 'status', 'ready', 'brief', 'run', 'from'], notes: ['--brief collapses whitespace and caps each spec at 160 characters.'] }, { path: ['orchestration', 'task-update'], summary: 'Update a task status', usage: - 'orca orchestration task-update --id --status [--result ] [--json]', - allowedFlags: [...GLOBAL_FLAGS, 'id', 'status', 'result'], + 'orca orchestration task-update --id --status [--result ] [--run ] [--from ] [--json]', + allowedFlags: [...GLOBAL_FLAGS, 'id', 'status', 'result', 'run', 'from', 'retry-request'], notes: ['Valid --status values: pending, ready, dispatched, completed, failed, blocked.'] }, + ...ORCHESTRATION_WORKER_COMMAND_SPECS, { path: ['orchestration', 'dispatch'], summary: 'Dispatch a task to a terminal', usage: - 'orca orchestration dispatch --task --to [--from ] [--inject] [--dry-run] [--return-preamble] [--json]', - allowedFlags: [...GLOBAL_FLAGS, 'task', 'to', 'from', 'inject', 'dry-run', 'return-preamble'] + 'orca orchestration dispatch --task --to [--from ] [--run ] [--inject] [--dry-run] [--return-preamble] [--json]', + allowedFlags: [ + ...GLOBAL_FLAGS, + 'task', + 'to', + 'from', + 'run', + 'inject', + 'dry-run', + 'return-preamble', + 'retry-request' + ] }, { path: ['orchestration', 'dispatch-show'], @@ -110,14 +181,30 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [ path: ['orchestration', 'ask'], summary: 'Ask the coordinator a question and block until answered', usage: - 'orca orchestration ask --to --question [--options ] [--timeout-ms ] [--from ] [--json]', - allowedFlags: [...GLOBAL_FLAGS, 'to', 'question', 'options', 'timeout-ms', 'from'] + 'orca orchestration ask (--question | --resume ) [--to ] [--run ] [--options ] [--timeout-ms ] [--from ] [--json]', + allowedFlags: [ + ...GLOBAL_FLAGS, + 'to', + 'run', + 'question', + 'resume', + 'dispatch-capability', + 'options', + 'timeout-ms', + 'from', + 'retry-request' + ], + notes: [ + 'From an active Dispatch, a new question defaults to its owning Run mailbox.', + 'Timeout leaves the question pending; resume with the original message ID.' + ] }, { - path: ['orchestration', 'run'], - summary: 'Start the coordinator loop', + path: ['orchestration', 'coordinator-start'], + aliases: [['orchestration', 'run']], + summary: 'Retired: load the current orchestration skill', usage: - 'orca orchestration run --spec [--from ] [--poll-interval-ms ] [--max-concurrent ] [--worktree ] [--json]', + 'orca orchestration coordinator-start --spec [--from ] [--poll-interval-ms ] [--max-concurrent ] [--worktree ] [--json]', allowedFlags: [ ...GLOBAL_FLAGS, 'spec', @@ -125,26 +212,34 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [ 'poll-interval-ms', 'max-concurrent', 'worktree' + ], + notes: [ + 'This command performs no effects and returns the exact `skills get orchestration --full` recovery action.', + 'Use the lightweight Run, Task, and worker-start primitives described by the current skill.' ] }, { - path: ['orchestration', 'run-stop'], - summary: 'Stop the active coordinator run', - usage: 'orca orchestration run-stop [--json]', - allowedFlags: [...GLOBAL_FLAGS] + path: ['orchestration', 'coordinator-stop'], + aliases: [['orchestration', 'run-stop']], + summary: 'Retired: load the current orchestration skill', + usage: 'orca orchestration coordinator-stop [--json]', + allowedFlags: [...GLOBAL_FLAGS], + notes: [ + 'This command performs no effects and returns the exact `skills get orchestration --full` recovery action.' + ] }, { path: ['orchestration', 'gate-create'], summary: 'Create a decision gate blocking a task', usage: 'orca orchestration gate-create --task --question [--options ] [--json]', - allowedFlags: [...GLOBAL_FLAGS, 'task', 'question', 'options'] + allowedFlags: [...GLOBAL_FLAGS, 'task', 'question', 'options', 'retry-request'] }, { path: ['orchestration', 'gate-resolve'], summary: 'Resolve a pending decision gate', usage: 'orca orchestration gate-resolve --id --resolution [--json]', - allowedFlags: [...GLOBAL_FLAGS, 'id', 'resolution'] + allowedFlags: [...GLOBAL_FLAGS, 'id', 'resolution', 'retry-request'] }, { path: ['orchestration', 'gate-list'], @@ -154,8 +249,8 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [ }, { path: ['orchestration', 'reset'], - summary: 'Reset orchestration state (one scope; bare command resets all)', - usage: 'orca orchestration reset [--all | --tasks | --messages] [--json]', - allowedFlags: [...GLOBAL_FLAGS, 'all', 'tasks', 'messages'] + summary: 'Reset one explicit orchestration state scope', + usage: 'orca orchestration reset (--all | --tasks | --messages) [--json]', + allowedFlags: [...GLOBAL_FLAGS, 'all', 'tasks', 'messages', 'retry-request'] } ] diff --git a/src/main/index.ts b/src/main/index.ts index 3c78c396077..5a4a687999a 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -41,6 +41,13 @@ import { resolveConsent } from './telemetry/consent' import { triggerStartupNotificationRegistration } from './ipc/notifications' import { OrcaRuntimeService, type RuntimeWorktreeLifecycleEvent } from './runtime/orca-runtime' import { loadAgentSessionClaimSigner } from './runtime/agent-session-claim-identity' +import { + fingerprintOrchestrationPeer, + type OrchestrationEnvironmentTransport +} from './runtime/orchestration/environment-transport' +import { callRuntimeEnvironment } from './ipc/runtime-environment-transport-routing' +import { resolveEnvironment } from '../shared/runtime-environment-store' +import { getPreferredPairingOffer } from '../shared/runtime-environments' import { OrcaRuntimeRpcServer } from './runtime/runtime-rpc' import { resolveAdvertisedPairingEndpoint } from './runtime/pairing-endpoint' import { ServeReadinessPublisher } from './server/serve-readiness' @@ -2117,6 +2124,27 @@ app.whenReady().then(async () => { .filter((account) => !activeIds.has(account.id)) .map((account) => ({ id: account.id, managedHomePath: account.managedHomePath })) }) + const orchestrationEnvironmentTransport: OrchestrationEnvironmentTransport = { + resolve: (selector) => { + const environment = resolveEnvironment(app.getPath('userData'), selector) + const pairing = getPreferredPairingOffer(environment) + return { + environmentId: environment.id, + name: environment.name, + peerFingerprint: fingerprintOrchestrationPeer(pairing.publicKeyB64) + } + }, + call: (selector, method, params, timeoutMs, envelope) => + callRuntimeEnvironment( + app.getPath('userData'), + selector, + method, + params, + timeoutMs, + undefined, + envelope + ) + } const runtimeService = new OrcaRuntimeService(store, stats, { agentSessionClaimSigner: loadAgentSessionClaimSigner( getProfileUserDataPath(), @@ -2155,7 +2183,8 @@ app.whenReady().then(async () => { systemCodexHomePath: resolveHostCodexSessionSourceHome(store!.getSettings()) }), buildAgentHookPtyEnv: () => - isAgentStatusHooksEnabled(store?.getSettings()) ? agentHookServer.buildPtyEnv() : {} + isAgentStatusHooksEnabled(store?.getSettings()) ? agentHookServer.buildPtyEnv() : {}, + orchestrationEnvironmentTransport }) runtime = runtimeService publishProviderSessionChanges(agentHookServer.getProviderSessionIdentities()) diff --git a/src/main/ipc/runtime-environment-shared-control-support.ts b/src/main/ipc/runtime-environment-shared-control-support.ts new file mode 100644 index 00000000000..4603ff69d70 --- /dev/null +++ b/src/main/ipc/runtime-environment-shared-control-support.ts @@ -0,0 +1,80 @@ +import { REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY } from '../../shared/protocol-version' +import { sendRemoteRuntimeRequest } from '../../shared/remote-runtime-client' +import { markEnvironmentUsed } from '../../shared/runtime-environment-store' +import type { + getPreferredPairingOffer, + KnownRuntimeEnvironment +} from '../../shared/runtime-environments' +import type { RuntimeStatus } from '../../shared/runtime-types' + +const sharedControlSupport = new Map }>() + +export function resetSharedControlSupport(): void { + sharedControlSupport.clear() +} + +export function clearSharedControlSupport(environmentId: string): void { + sharedControlSupport.delete(environmentId) +} + +export async function supportsSharedControl( + userDataPath: string, + environment: KnownRuntimeEnvironment, + pairing: ReturnType, + timeoutMs: number +): Promise { + const cacheKey = getSharedControlSupportCacheKey(environment, pairing) + const cached = sharedControlSupport.get(environment.id) + if (cached?.cacheKey === cacheKey) { + return cached.check + } + let resolvedCacheKey = cacheKey + const check = (async () => { + const response = await sendRemoteRuntimeRequest( + pairing, + 'status.get', + undefined, + timeoutMs + ) + if (response.ok === true) { + markEnvironmentUsed(userDataPath, environment.id, { runtimeId: response._meta.runtimeId }) + resolvedCacheKey = getSharedControlSupportCacheKey( + environment, + pairing, + response._meta.runtimeId + ) + return ( + response.result.capabilities?.includes(REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY) === true + ) + } + return false + })() + // Why: support belongs to the saved pairing/runtime identity, not its mutable display name. + sharedControlSupport.set(environment.id, { cacheKey, check }) + try { + const supported = await check + const cachedAfterCheck = sharedControlSupport.get(environment.id) + if (cachedAfterCheck?.check === check && cachedAfterCheck.cacheKey !== resolvedCacheKey) { + sharedControlSupport.set(environment.id, { cacheKey: resolvedCacheKey, check }) + } + return supported + } catch (error) { + if (sharedControlSupport.get(environment.id)?.check === check) { + sharedControlSupport.delete(environment.id) + } + throw error + } +} + +function getSharedControlSupportCacheKey( + environment: KnownRuntimeEnvironment, + pairing: ReturnType, + runtimeId = environment.runtimeId +): string { + return [ + runtimeId ?? 'unknown-runtime', + pairing.endpoint, + pairing.deviceToken, + pairing.publicKeyB64 + ].join('\0') +} diff --git a/src/main/ipc/runtime-environment-transport-routing.ts b/src/main/ipc/runtime-environment-transport-routing.ts index 46241a82521..852b97273c5 100644 --- a/src/main/ipc/runtime-environment-transport-routing.ts +++ b/src/main/ipc/runtime-environment-transport-routing.ts @@ -1,11 +1,10 @@ -import { - getPreferredPairingOffer, - type KnownRuntimeEnvironment -} from '../../shared/runtime-environments' +import { getPreferredPairingOffer } from '../../shared/runtime-environments' import { resolveEnvironment, markEnvironmentUsed } from '../../shared/runtime-environment-store' -import type { RuntimeRpcResponse } from '../../shared/runtime-rpc-envelope' +import type { + RuntimeOrchestrationEnvelope, + RuntimeRpcResponse +} from '../../shared/runtime-rpc-envelope' import type { RuntimeStatus } from '../../shared/runtime-types' -import { REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY } from '../../shared/protocol-version' import { sendRemoteRuntimeRequest, subscribeRemoteRuntimeRequest, @@ -22,14 +21,15 @@ import { import { attachRemoteControlDiagnostics } from './runtime-environment-status-diagnostics' import { runtimeEnvironmentRevisionFailure } from './runtime-environment-revision-guard' import { withTailscaleHintForResponse } from './runtime-environment-tailscale-response' +import { + clearSharedControlSupport, + resetSharedControlSupport, + supportsSharedControl +} from './runtime-environment-shared-control-support' const DEFAULT_REMOTE_RUNTIME_TIMEOUT_MS = 15_000 -const sharedControlSupport = new Map }>() -export const resetSharedControlSupport = (): void => sharedControlSupport.clear() - -export const clearSharedControlSupport = (environmentId: string): void => - void sharedControlSupport.delete(environmentId) +export { clearSharedControlSupport, resetSharedControlSupport } export async function getRuntimeEnvironmentStatus( userDataPath: string, @@ -81,7 +81,8 @@ export async function callRuntimeEnvironment( method: string, params: unknown, timeoutMs?: number, - expectedEnvironmentPairingRevision?: number + expectedEnvironmentPairingRevision?: number, + envelope?: RuntimeOrchestrationEnvelope ): Promise> { const environment = resolveEnvironment(userDataPath, selector) // Why: connection failures reject (they don't resolve as ok:false), so the @@ -104,6 +105,17 @@ export async function callRuntimeEnvironment( const pairing = getPreferredPairingOffer(currentEnvironment) endpoint = pairing.endpoint const effectiveTimeoutMs = timeoutMs ?? DEFAULT_REMOTE_RUNTIME_TIMEOUT_MS + if (envelope) { + const response = await sendRemoteRuntimeRequest( + pairing, + method, + params, + effectiveTimeoutMs, + envelope + ) + markEnvironmentUsedFromResponse(userDataPath, currentEnvironment.id, response) + return response + } if (shouldUseCachedRequestConnection(method)) { const response = await sendRemoteRuntimeConnectionRequest( currentEnvironment.id, @@ -257,66 +269,3 @@ function shouldUseSharedControlSubscription(method: string): boolean { method === 'files.watch' ) } - -async function supportsSharedControl( - userDataPath: string, - environment: KnownRuntimeEnvironment, - pairing: ReturnType, - timeoutMs: number -): Promise { - const cacheKey = getSharedControlSupportCacheKey(environment, pairing) - const cached = sharedControlSupport.get(environment.id) - if (cached?.cacheKey === cacheKey) { - return cached.check - } - let resolvedCacheKey = cacheKey - const check = (async () => { - const response = await sendRemoteRuntimeRequest( - pairing, - 'status.get', - undefined, - timeoutMs - ) - if (response.ok === true) { - markEnvironmentUsed(userDataPath, environment.id, { runtimeId: response._meta.runtimeId }) - resolvedCacheKey = getSharedControlSupportCacheKey( - environment, - pairing, - response._meta.runtimeId - ) - return ( - response.result.capabilities?.includes(REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY) === true - ) - } - return false - })() - // Why: the same saved host can be re-paired or point at a different runtime - // binary over time; capability support belongs to that pairing/runtime identity. - sharedControlSupport.set(environment.id, { cacheKey, check }) - try { - const supported = await check - const cachedAfterCheck = sharedControlSupport.get(environment.id) - if (cachedAfterCheck?.check === check && cachedAfterCheck.cacheKey !== resolvedCacheKey) { - sharedControlSupport.set(environment.id, { cacheKey: resolvedCacheKey, check }) - } - return supported - } catch (error) { - if (sharedControlSupport.get(environment.id)?.check === check) { - sharedControlSupport.delete(environment.id) - } - throw error - } -} - -function getSharedControlSupportCacheKey( - environment: KnownRuntimeEnvironment, - pairing: ReturnType, - runtimeId = environment.runtimeId -): string { - return [ - runtimeId ?? 'unknown-runtime', - pairing.endpoint, - pairing.deviceToken, - pairing.publicKeyB64 - ].join('\0') -} diff --git a/src/main/native-chat/transcript-tail-reader.ts b/src/main/native-chat/transcript-tail-reader.ts index 2f874e95746..9fb66705103 100644 --- a/src/main/native-chat/transcript-tail-reader.ts +++ b/src/main/native-chat/transcript-tail-reader.ts @@ -49,6 +49,8 @@ export async function readNativeChatTranscriptTailFile( consumedTo: number hasMore: boolean beforeOffset: number + malformedRecordCount?: number + oversizedRecordCount?: number }> { const end = Math.min((await stat(filePath)).size, endOffset ?? Number.MAX_SAFE_INTEGER) if (end === 0) { @@ -59,6 +61,9 @@ export async function readNativeChatTranscriptTailFile( let lineBytes = 0 let lineOversized = false let lifecycle: NativeChatTurnLifecycle | undefined + let malformedRecordCount = 0 + let oversizedRecordCount = 0 + let ignoreNextMalformedRecord = false try { const consumedTo = includeTrailingLine ? end : await findLastCompleteLineEnd(handle, end) if (consumedTo === 0) { @@ -67,6 +72,7 @@ export async function readNativeChatTranscriptTailFile( const newestFirst: { message: NativeChatMessage; offset: number }[] = [] const finalByte = Buffer.allocUnsafe(1) await handle.read(finalByte, 0, 1, consumedTo - 1) + ignoreNextMalformedRecord = finalByte[0] !== 0x0a let cursor = consumedTo - (finalByte[0] === 0x0a ? 1 : 0) while (cursor > 0 && newestFirst.length <= limit) { const start = Math.max(0, cursor - TAIL_CHUNK_BYTES) @@ -101,7 +107,9 @@ export async function readNativeChatTranscriptTailFile( ...(lifecycle ? { lifecycle } : {}), consumedTo, hasMore: limit > 0 && chronological.length > limit, - beforeOffset: selected[0]?.offset ?? end + beforeOffset: selected[0]?.offset ?? end, + ...(malformedRecordCount > 0 ? { malformedRecordCount } : {}), + ...(oversizedRecordCount > 0 ? { oversizedRecordCount } : {}) } } finally { await handle.close() @@ -115,6 +123,7 @@ export async function readNativeChatTranscriptTailFile( if (lineBytes > MAX_NATIVE_CHAT_TRANSCRIPT_RECORD_BYTES) { lineParts.length = 0 lineOversized = true + oversizedRecordCount++ return } lineParts.push(part) @@ -137,6 +146,17 @@ export async function readNativeChatTranscriptTailFile( if (!line) { return } + try { + JSON.parse(line) + } catch { + if (ignoreNextMalformedRecord) { + ignoreNextMalformedRecord = false + return + } + malformedRecordCount++ + return + } + ignoreNextMalformedRecord = false const fallbackId = transcriptFallbackId(filePath, lineOffset) // Why: scan the same bounded JSONL window for provider-authored lifecycle // records so reconnect snapshots can replay completion without guessing diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index 1486207771f..007b23f3fef 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -1012,6 +1012,7 @@ class InMemoryOrchestrationMessages { this.sequence += 1 const row: MessageRow = { id: `msg_${this.sequence}`, + run_id: 'run_test', from_handle: msg.from, to_handle: msg.to, subject: msg.subject, @@ -12468,6 +12469,102 @@ describe('OrcaRuntimeService', () => { }) }) + it('observes setup command completion without waiting for its interactive shell to exit', async () => { + const runtime = new OrcaRuntimeService(store) + runtime.setPtyController({ + spawn: vi.fn().mockResolvedValue({ id: 'pty-setup' }), + write: () => true, + kill: () => true, + getForegroundProcess: async () => null + }) + runtime.attachWindow(1) + runtime.syncWindowGraph(1, { tabs: [], leaves: [] }) + const { handle } = await runtime.createTerminal(`path:${TEST_WORKTREE_PATH}`) + ;( + runtime as unknown as { setupCompletionTokenByPtyId: Map } + ).setupCompletionTokenByPtyId.set('pty-setup', 'token-live') + + const waiting = runtime.waitForSetupTerminalCompletion(handle) + runtime.onPtyData( + 'pty-setup', + 'setup failed\r\n__ORCA_SETUP_COMPLETE__:token-live:17\r\nPS>', + 100 + ) + + await expect(waiting).resolves.toEqual({ exitCode: 17 }) + await expect(runtime.readTerminal(handle)).resolves.toMatchObject({ status: 'running' }) + }) + + it('replays fast setup completion emitted before its observer is registered', async () => { + const runtime = new OrcaRuntimeService(store) + runtime.setPtyController({ + spawn: vi.fn().mockResolvedValue({ id: 'pty-fast-setup' }), + write: () => true, + kill: () => true, + getForegroundProcess: async () => null + }) + runtime.attachWindow(1) + runtime.syncWindowGraph(1, { tabs: [], leaves: [] }) + const { handle } = await runtime.createTerminal(`path:${TEST_WORKTREE_PATH}`) + ;( + runtime as unknown as { setupCompletionTokenByPtyId: Map } + ).setupCompletionTokenByPtyId.set('pty-fast-setup', 'token-fast') + runtime.onPtyData( + 'pty-fast-setup', + '__ORCA_SETUP_COMPLETE__:wrong:9\r\n__ORCA_SETUP_COMPLETE__:token-fast:0\r\n$', + 100 + ) + + await expect(runtime.waitForSetupTerminalCompletion(handle)).resolves.toEqual({ exitCode: 0 }) + }) + + it('falls back to setup terminal exit when no completion signal is available', async () => { + const runtime = new OrcaRuntimeService(store) + runtime.setPtyController({ + spawn: vi.fn().mockResolvedValue({ id: 'pty-legacy-setup' }), + write: () => true, + kill: () => true, + getForegroundProcess: async () => null + }) + runtime.attachWindow(1) + runtime.syncWindowGraph(1, { tabs: [], leaves: [] }) + const { handle } = await runtime.createTerminal(`path:${TEST_WORKTREE_PATH}`) + + const waiting = runtime.waitForSetupTerminalCompletion(handle) + runtime.onPtyExit('pty-legacy-setup', 9) + + await expect(waiting).resolves.toEqual({ exitCode: 9 }) + }) + + it('keeps observing after an uncertain setup terminal status', async () => { + const runtime = new OrcaRuntimeService(store) + runtime.setPtyController({ + spawn: vi.fn().mockResolvedValue({ id: 'pty-uncertain-setup' }), + write: () => true, + kill: () => true, + getForegroundProcess: async () => null + }) + runtime.attachWindow(1) + runtime.syncWindowGraph(1, { tabs: [], leaves: [] }) + const { handle } = await runtime.createTerminal(`path:${TEST_WORKTREE_PATH}`) + ;( + runtime as unknown as { setupCompletionTokenByPtyId: Map } + ).setupCompletionTokenByPtyId.set('pty-uncertain-setup', 'token-uncertain') + vi.spyOn(runtime, 'waitForTerminal').mockResolvedValue({ + handle, + condition: 'exit', + satisfied: false, + status: 'unknown', + exitCode: null + }) + + const waiting = runtime.waitForSetupTerminalCompletion(handle) + await Promise.resolve() + runtime.onPtyData('pty-uncertain-setup', '__ORCA_SETUP_COMPLETE__:token-uncertain:0\r\n', 100) + + await expect(waiting).resolves.toEqual({ exitCode: 0 }) + }) + it('drops retained PTY transcript memory when a background terminal exits', async () => { const runtime = new OrcaRuntimeService(store) runtime.setPtyController({ @@ -17290,6 +17387,83 @@ describe('OrcaRuntimeService', () => { expect(writes).toEqual(['still writable']) }) + it('preserves runtime-created PTY process identity after graph unavailable', async () => { + const runtime = new OrcaRuntimeService(store) + runtime.setPtyController({ + spawn: vi.fn().mockResolvedValue({ id: 'pty-bg' }), + write: () => true, + kill: () => true, + getForegroundProcess: async () => null + }) + runtime.attachWindow(1) + runtime.syncWindowGraph(1, { tabs: [], leaves: [] }) + const { handle } = await runtime.createTerminal(`path:${TEST_WORKTREE_PATH}`) + const incarnation = runtime.getTerminalProcessIncarnation(handle) + + runtime.markGraphUnavailable(1) + + expect(runtime.getTerminalProcessIncarnation(handle)).toBe(incarnation) + }) + + it('preserves PTY process identity while a renderer surface detaches and reattaches', async () => { + const runtime = new OrcaRuntimeService(store) + runtime.setPtyController({ + spawn: vi.fn().mockResolvedValue({ + id: 'pty-bg', + incarnationId: 'incarnation-bg' + }), + write: () => true, + kill: () => true, + getForegroundProcess: async () => null + }) + runtime.attachWindow(1) + runtime.syncWindowGraph(1, { tabs: [], leaves: [] }) + const created = await runtime.createTerminal(`path:${TEST_WORKTREE_PATH}`) + const [tabId, leafId] = created.paneKey?.split(':') ?? [] + if (!tabId || !leafId) { + throw new Error('expected stable pane identity') + } + const syncSurface = (ptyId: string | null): void => { + runtime.syncWindowGraph(1, { + tabs: [ + { + tabId, + worktreeId: TEST_WORKTREE_ID, + title: 'Codex', + activeLeafId: leafId, + layout: null + } + ], + leaves: [ + { + tabId, + worktreeId: TEST_WORKTREE_ID, + leafId, + paneRuntimeId: 1, + ptyId, + paneTitle: 'Codex' + } + ] + }) + } + + syncSurface('pty-bg') + await runtime.listTerminals() + const before = runtime.getTerminalProcessIncarnation(created.handle) + syncSurface(null) + syncSurface('pty-bg') + await runtime.listTerminals() + + expect(runtime.getTerminalProcessIncarnation(created.handle)).toBe(before) + + runtime.registerPty('pty-bg', TEST_WORKTREE_ID, null, { + tabId, + leafId, + incarnationId: 'incarnation-replacement' + }) + expect(runtime.getTerminalProcessIncarnation(created.handle)).not.toBe(before) + }) + it('recognizes runtime-created PTY handles with agent launch titles', async () => { const runtime = new OrcaRuntimeService(store) runtime.setPtyController({ @@ -27108,7 +27282,7 @@ describe('OrcaRuntimeService', () => { const waitPromise = runtime.waitForMessage('term_abc', { timeoutMs: 5000 }) runtime.notifyMessageArrived('term_abc') - await waitPromise + await expect(waitPromise).resolves.toBe('notified') }) it('does not resolve type-filtered message waiters for unrelated message types', async () => { @@ -27149,13 +27323,38 @@ describe('OrcaRuntimeService', () => { }) it('resolves message waiters on timeout when no message arrives', async () => { - const runtime = new OrcaRuntimeService(store) + vi.useFakeTimers() + try { + const runtime = new OrcaRuntimeService(store) + const wait = runtime.waitForMessage('term_abc', { timeoutMs: 100 }) - const start = Date.now() - await runtime.waitForMessage('term_abc', { timeoutMs: 100 }) - const elapsed = Date.now() - start - expect(elapsed).toBeGreaterThanOrEqual(90) - expect(elapsed).toBeLessThan(500) + await vi.advanceTimersByTimeAsync(99) + let settled = false + void wait.then(() => { + settled = true + }) + await Promise.resolve() + expect(settled).toBe(false) + + await vi.advanceTimersByTimeAsync(1) + await expect(wait).resolves.toBe('timed_out') + } finally { + vi.useRealTimers() + } + }) + + it('allows only one exclusive mailbox waiter and supports explicit cancellation', async () => { + const runtime = new OrcaRuntimeService(store) + const first = runtime.waitForMessage('run:run_1', { + timeoutMs: 5000, + exclusive: true + }) + + await expect( + runtime.waitForMessage('run:run_1', { timeoutMs: 5000, exclusive: true }) + ).resolves.toBe('waiter_exists') + runtime.cancelMessageWaiters('run:run_1') + await expect(first).resolves.toBe('cancelled') }) it('rejects leaf PTY waits when the request signal aborts', async () => { @@ -32908,11 +33107,13 @@ describe('OrcaRuntimeService', () => { } ]) - await runtime.createManagedWorktree({ + const result = await runtime.createManagedWorktree({ repoSelector: 'id:repo-1', name: 'runtime-headless-parallel', setupDecision: 'run', - startup: { command: 'claude' } + startup: { command: 'claude' }, + observeSetupCompletion: true, + awaitTerminalProvisioning: true }) // Why: setup now spawns fire-and-forget on a later tick; wait for both PTYs. @@ -32920,8 +33121,14 @@ describe('OrcaRuntimeService', () => { expect(spawn).toHaveBeenNthCalledWith(1, expect.objectContaining({ command: 'claude' })) expect(spawn).toHaveBeenNthCalledWith( 2, - expect.objectContaining({ command: 'bash /tmp/repo/.git/orca/setup-runner.sh' }) + expect.objectContaining({ + command: expect.stringContaining('__ORCA_SETUP_COMPLETE__:') + }) ) + expect(result.setupReceipt).toMatchObject({ + state: 'running', + terminalHandle: expect.stringMatching(/^term_/) + }) }) it('creates the first terminal for CLI-created worktrees without activating them', async () => { @@ -33133,10 +33340,12 @@ describe('OrcaRuntimeService', () => { const result = await runtime.createManagedWorktree({ repoSelector: 'id:repo-1', name: 'runtime-cli-setup-skip', - setupDecision: 'skip' + setupDecision: 'skip', + awaitTerminalProvisioning: true }) expect(result.warning).toBeUndefined() + expect(result.setupReceipt).toMatchObject({ requested: 'skip', state: 'skipped' }) expect(createSetupRunnerScript).not.toHaveBeenCalled() expect(spawn).toHaveBeenCalledTimes(1) }) @@ -33694,7 +33903,8 @@ describe('OrcaRuntimeService', () => { name: 'runtime-startup-setup-split', startupDraft: 'https://github.com/stablyai/orca/issues/123', setupDecision: 'run', - activate: true + activate: true, + awaitTerminalProvisioning: true }) await vi.waitFor(() => expect(spawn).toHaveBeenCalledTimes(2)) @@ -33732,6 +33942,10 @@ describe('OrcaRuntimeService', () => { const mainEnv = (spawn.mock.calls[0]![0] as { env?: Record }).env ?? {} const setupEnv = (spawn.mock.calls[1]![0] as { env?: Record }).env ?? {} expect(result.setup).toBeUndefined() + expect(result.setupReceipt).toMatchObject({ + state: 'running', + terminalHandle: expect.stringMatching(/^term_/) + }) expect(mainEnv.ORCA_TAB_ID).toBeDefined() expect(mainEnv.ORCA_PANE_KEY).toBeDefined() expect(setupEnv.ORCA_TAB_ID).toBe(mainEnv.ORCA_TAB_ID) diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index 6097f28105a..34004203d41 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -96,7 +96,23 @@ import { mkdir, readFile, readdir, rm, stat } from 'node:fs/promises' import { resolveWorktreeCreateBase } from '../worktree-create-base' import { resolveWorktreeAddBaseRef } from '../../shared/worktree-base-ref' import { OrchestrationDb } from './orchestration/db' +import { OrchestrationError } from './orchestration/orchestration-error' +import { + buildObservedSetupCommand, + createSetupCompletionScanner +} from './orchestration/setup-completion-signal' +import type { RuntimeOrchestrationEnvelope } from '../../shared/runtime-rpc-envelope' +import { + isOrchestrationMutation, + orchestrationMigrationData +} from '../../shared/orchestration-rpc-contract' +import type { + OrchestrationEnvironmentTransport, + OrchestrationWorkerServer +} from './orchestration/environment-transport' +import { syncFederatedDispatch } from './orchestration/federation-sync' import { formatMessagesForInjection } from './orchestration/formatter' +import { selectExactWorkerProviderSession } from './orchestration/worker-provider-session' import type { Automation, AutomationCreateInput, @@ -183,6 +199,7 @@ import type { AgentProviderSessionMetadata, SleepingAgentLaunchConfig } from '../../shared/agent-session-resume' +import type { ExactWorkerProviderSession } from '../../shared/orchestration-worker-output' import type { RuntimeClientEvent } from '../../shared/runtime-client-events' import { toRuntimeActivateWorktreeEvent } from '../../shared/runtime-client-events' import { @@ -332,6 +349,8 @@ import { BROWSER_HEADLESS_RUNTIME_CAPABILITY, BROWSER_CERTIFICATE_TRUST_RUNTIME_CAPABILITY, MIN_COMPATIBLE_RUNTIME_CLIENT_VERSION, + ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY, + ORCHESTRATION_CONTRACT_VERSION, REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY, RUNTIME_CAPABILITIES, RUNTIME_PROTOCOL_VERSION, @@ -1698,11 +1717,13 @@ type TerminalWaiter = { type MessageWaiter = { handle: string typeFilter: string[] | undefined - resolve: (result: void) => void + resolve: (result: MessageWaitResult) => void timeout: NodeJS.Timeout | null abortCleanup: (() => void) | null } +export type MessageWaitResult = 'notified' | 'timed_out' | 'cancelled' | 'waiter_exists' + function omitUndefinedProperties>(value: T): Partial { return Object.fromEntries( Object.entries(value).filter(([, entry]) => entry !== undefined) @@ -2438,6 +2459,10 @@ export class OrcaRuntimeService { private readonly runtimeId = randomUUID() private readonly startedAt = Date.now() private readonly store: RuntimeStore | null + private readonly orchestrationEnvironmentTransport: OrchestrationEnvironmentTransport | null + private readonly orchestrationFederationTimers = new Map>() + private readonly orchestrationFederationSyncs = new Map>() + private readonly orchestrationFederationWarnings = new Set() private rendererGraphEpoch = 0 private graphStatus: RuntimeGraphStatus = 'unavailable' private authoritativeWindowId: number | null = null @@ -2563,6 +2588,7 @@ export class OrcaRuntimeService { // Why: startup draft paste can subscribe after the agent already emitted its // ready marker. Keep a bounded raw buffer so fast startup output is replayed. private recentPtyOutputById = new Map() + private setupCompletionTokenByPtyId = new Map() // Why: mobile clients need to know when the desktop restores a terminal // from mobile-fit so they can update their UI. These listeners are // invoked from resizeForClient and onClientDisconnected/onPtyExit. @@ -2969,6 +2995,7 @@ export class OrcaRuntimeService { buildAgentHookPtyEnv?: () => Record getDesktopWindowStatus?: () => RuntimeDesktopWindowStatus agentSessionClaimSigner?: AgentSessionClaimSigner + orchestrationEnvironmentTransport?: OrchestrationEnvironmentTransport } ) { this.store = store @@ -2980,6 +3007,7 @@ export class OrcaRuntimeService { this.clientSessionTabSelections.setPersistListener((state) => { this.store?.setMobileClientTabSelections?.(state) }) + this.orchestrationEnvironmentTransport = deps?.orchestrationEnvironmentTransport ?? null if (stats) { this.stats = stats this.agentDetector = new AgentDetector(stats) @@ -3440,6 +3468,136 @@ export class OrcaRuntimeService { return this.runtimeId } + resolveOrchestrationWorkerServer(selector: string): OrchestrationWorkerServer { + if (!this.orchestrationEnvironmentTransport) { + throw new OrchestrationError( + 'server_required', + 'Connected-server orchestration is unavailable in this runtime.' + ) + } + return this.orchestrationEnvironmentTransport.resolve(selector) + } + + async callOrchestrationWorkerServer( + selector: string, + method: string, + params: unknown, + timeoutMs?: number, + envelope?: RuntimeOrchestrationEnvelope + ): Promise { + if (!this.orchestrationEnvironmentTransport) { + throw new OrchestrationError( + 'server_required', + 'Connected-server orchestration is unavailable in this runtime.' + ) + } + if (isOrchestrationMutation(method, params)) { + const statusResponse = await this.orchestrationEnvironmentTransport.call( + selector, + 'status.get', + undefined, + timeoutMs + ) + if (statusResponse.ok === false) { + throw new OrchestrationError( + statusResponse.error.code, + statusResponse.error.message, + statusResponse.error.data + ) + } + const status = statusResponse.result as RuntimeStatus + if (!status.capabilities?.includes(ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY)) { + throw new OrchestrationError( + 'orchestration_migration_required', + 'The connected worker server does not support the current orchestration contract. No effects were applied.', + orchestrationMigrationData('runtime_capability_missing') + ) + } + } + const response = await this.orchestrationEnvironmentTransport.call( + selector, + method, + params, + timeoutMs, + method.startsWith('orchestration.') + ? { ...envelope, orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION } + : envelope + ) + if (response.ok === false) { + throw new OrchestrationError(response.error.code, response.error.message, response.error.data) + } + return response.result + } + + async syncOrchestrationFederation(runId?: string): Promise { + if (!this.orchestrationEnvironmentTransport) { + return + } + const dispatches = this.getOrchestrationDb().listActiveFederatedDispatches(runId) + await Promise.allSettled( + dispatches.map((dispatch) => this.syncOrchestrationFederatedDispatch(dispatch.dispatch_id)) + ) + } + + private syncOrchestrationFederatedDispatch(dispatchId: string): Promise { + const current = this.orchestrationFederationSyncs.get(dispatchId) + if (current) { + return current + } + const sync = syncFederatedDispatch(this, dispatchId) + .then(() => { + this.orchestrationFederationWarnings.delete(dispatchId) + }) + .catch((error: unknown) => { + if (!this.orchestrationFederationWarnings.has(dispatchId)) { + console.warn(`[orchestration] Federation sync failed for ${dispatchId}:`, error) + this.orchestrationFederationWarnings.add(dispatchId) + } + throw error + }) + .finally(() => { + this.orchestrationFederationSyncs.delete(dispatchId) + }) + this.orchestrationFederationSyncs.set(dispatchId, sync) + return sync + } + + ensureOrchestrationFederationRelay(runId?: string): void { + if (!this.orchestrationEnvironmentTransport) { + return + } + for (const dispatch of this.getOrchestrationDb().listActiveFederatedDispatches(runId)) { + if (this.orchestrationFederationTimers.has(dispatch.dispatch_id)) { + continue + } + const tick = () => { + const worker = this.getOrchestrationDb().getWorkerDispatch(dispatch.dispatch_id) + if (!worker || !['starting', 'ready', 'stopping'].includes(worker.state)) { + const activeTimer = this.orchestrationFederationTimers.get(dispatch.dispatch_id) + if (activeTimer) { + clearInterval(activeTimer) + } + this.orchestrationFederationTimers.delete(dispatch.dispatch_id) + this.orchestrationFederationWarnings.delete(dispatch.dispatch_id) + return + } + void this.syncOrchestrationFederatedDispatch(dispatch.dispatch_id).catch(() => undefined) + } + const timer = setInterval(tick, 1_000) + timer.unref?.() + this.orchestrationFederationTimers.set(dispatch.dispatch_id, timer) + tick() + } + } + + stopOrchestrationFederationRelay(): void { + for (const timer of this.orchestrationFederationTimers.values()) { + clearInterval(timer) + } + this.orchestrationFederationTimers.clear() + this.orchestrationFederationWarnings.clear() + } + getStartedAt(): number { return this.startedAt } @@ -10901,6 +11059,7 @@ export class OrcaRuntimeService { this.resizeListeners.delete(ptyId) this.lastRendererSizes.delete(ptyId) this.recentPtyOutputById.delete(ptyId) + this.setupCompletionTokenByPtyId.delete(ptyId) this.clearWaitBlockedCheckState(ptyId) this.recentPtyPathCandidatesById.delete(ptyId) this.ptyOutputSequenceById.delete(ptyId) @@ -13568,6 +13727,64 @@ export class OrcaRuntimeService { } } + getTerminalProcessIncarnation(handle: string): string | null { + const live = this.getLivePtyForHandle(handle) + const record = live?.record ?? this.handles.get(handle) + if (!record?.ptyId) { + return null + } + const incarnationId = live?.pty.incarnationId ?? this.ptysById.get(record.ptyId)?.incarnationId + if (incarnationId) { + return `${record.ptyId}:${incarnationId}` + } + // Why: legacy providers may omit process incarnation; retain the prior restart-degraded fence. + return `${this.runtimeId}:${record.ptyId}:${record.ptyGeneration}` + } + + getExactWorkerProviderSession( + handle: string, + observedAfter: number + ): ExactWorkerProviderSession | null { + const paneKey = this.getTerminalPaneKey(handle) + const processIncarnation = this.getTerminalProcessIncarnation(handle) + if (!paneKey || !processIncarnation) { + return null + } + let connectionId: string | null | undefined + let launchToken: string | null | undefined + try { + const ptyId = this.getTerminalAgentStatusPtyId(handle) + const pty = this.ptysById.get(ptyId) + connectionId = pty?.connectionId ?? null + launchToken = pty?.launchToken ?? null + } catch { + // Exact worker validation rejects this in production; test/legacy providers may not expose PTY metadata. + connectionId = undefined + launchToken = undefined + } + return selectExactWorkerProviderSession({ + paneKey, + processIncarnation, + connectionId, + launchToken, + observedAfter, + statuses: this.getAgentStatusSnapshotFn?.() ?? [] + }) + } + + validateOrchestrationAgentLauncher(agent: TuiAgent): void { + const settings = this.store?.getSettings() + if (!settings) { + throw new Error('runtime_unavailable') + } + if (!isTuiAgentEnabled(agent, settings.disabledTuiAgents)) { + throw new OrchestrationError( + 'agent_unconfigured', + `Agent launcher ${agent} is disabled or unavailable.` + ) + } + } + resolveTerminalPane(paneKey: string, expectedWorktreeId?: string): RuntimeTerminalResolvePane { // Why: the renderer context menu only knows the stable pane key; main owns // the runtime terminal handle that agents and CLI commands can address. @@ -14485,6 +14702,62 @@ export class OrcaRuntimeService { }) } + async waitForSetupTerminalCompletion(handle: string): Promise<{ exitCode: number | null }> { + const ptyId = this.getLivePtyForHandle(handle)?.pty.ptyId + if (!ptyId) { + throw new Error('terminal_handle_stale') + } + const completionToken = this.setupCompletionTokenByPtyId.get(ptyId) + const exitAbort = new AbortController() + return await new Promise<{ exitCode: number | null }>((resolve, reject) => { + let settled = false + let unsubscribe: (() => void) | null = null + const cleanup = (): void => { + unsubscribe?.() + exitAbort.abort() + } + const finish = (exitCode: number | null): void => { + if (settled) { + return + } + settled = true + cleanup() + this.setupCompletionTokenByPtyId.delete(ptyId) + resolve({ exitCode }) + } + const fail = (error: unknown): void => { + if (settled) { + return + } + settled = true + cleanup() + reject(error) + } + const scanner = completionToken ? createSetupCompletionScanner(completionToken, finish) : null + + if (scanner) { + unsubscribe = this.subscribeToTerminalData(ptyId, scanner.scan) + } + // Why: setup can finish before the observer is registered on fast local worktrees. + const replay = this.recentPtyOutputById.get(ptyId)?.read() + if (scanner && replay) { + scanner.scan(replay) + } + if (!settled) { + void this.waitForTerminal(handle, { + condition: 'exit', + signal: exitAbort.signal + }) + .then((wait) => { + if (wait.satisfied && wait.condition === 'exit' && wait.status === 'exited') { + finish(wait.exitCode) + } + }) + .catch(fail) + } + }) + } + async getWorktreePs(limit = DEFAULT_WORKTREE_PS_LIMIT): Promise<{ worktrees: RuntimeWorktreePsSummary[] totalCount: number @@ -18172,16 +18445,18 @@ export class OrcaRuntimeService { primaryTerminalHandle?: string | null hasStartupTerminal: boolean setupCommandPlatform: 'windows' | 'posix' + observeSetupCompletion?: boolean // Why: when the agent startup is sequenced to wait for setup // (waitForAgentStartup), the startup PTY runs a wrapper that already embeds // the setup command. Pass that wrapped command through so the Setup tab runs // the same script the agent is waiting on instead of a bare runner. wrappedSetupCommand?: string - }): Promise<{ setupSpawned: boolean }> { + }): Promise<{ setupSpawned: boolean; setupTerminalHandle: string | null }> { if (!this.ptyController?.spawn) { - return { setupSpawned: false } + return { setupSpawned: false, setupTerminalHandle: null } } let setupSpawned = false + let setupTerminalHandle: string | null = null try { const defaultTabHandles = await this.createDefaultTabTerminals( args.worktreeSelector, @@ -18200,25 +18475,41 @@ export class OrcaRuntimeService { primaryTerminalHandle = terminal.handle } if (args.setup) { + const completionToken = + args.observeSetupCompletion && !args.wrappedSetupCommand ? randomUUID() : null + const observedCommand = completionToken + ? buildObservedSetupCommand( + args.setup.runnerScriptPath, + args.setupCommandPlatform, + completionToken + ) + : null const setupCommand = args.wrappedSetupCommand ?? + observedCommand?.command ?? buildSetupRunnerCommand(args.setup.runnerScriptPath, args.setupCommandPlatform) + const setupEnv = { ...args.setup.envVars, ...observedCommand?.env } const shouldSplitSetup = primaryTerminalHandle && (setupLaunchMode === 'split-vertical' || setupLaunchMode === 'split-horizontal') - await (shouldSplitSetup + const setupTerminal = await (shouldSplitSetup ? this.splitTerminal(primaryTerminalHandle!, { direction: setupLaunchMode === 'split-horizontal' ? 'horizontal' : 'vertical', command: setupCommand, - env: args.setup.envVars, + env: setupEnv, activate: false }) : this.createTerminal(args.worktreeSelector, { title: 'Setup', command: setupCommand, - env: args.setup.envVars + env: setupEnv })) + setupTerminalHandle = setupTerminal.handle setupSpawned = true + const ptyId = this.getLivePtyForHandle(setupTerminal.handle)?.pty.ptyId + if (completionToken && ptyId) { + this.setupCompletionTokenByPtyId.set(ptyId, completionToken) + } } } catch (err) { const message = err instanceof Error ? err.message : String(err) @@ -18226,7 +18517,7 @@ export class OrcaRuntimeService { `[worktree-create] Failed to create setup/default terminals for ${args.worktreePath}: ${message}` ) } - return { setupSpawned } + return { setupSpawned, setupTerminalHandle } } private async waitForStartupFollowupReady( @@ -18360,6 +18651,8 @@ export class OrcaRuntimeService { runHooks?: boolean activate?: boolean setupDecision?: 'run' | 'skip' | 'inherit' + awaitTerminalProvisioning?: boolean + observeSetupCompletion?: boolean createdWithAgent?: TuiAgent startupAgent?: TuiAgent startupPrompt?: string @@ -19160,6 +19453,7 @@ export class OrcaRuntimeService { // RPC return value must omit setup so the client does not spawn it a second // time. Mirrors the wait-for-agent setup contract from #6298. let didSpawnSetup = false + let setupTerminalHandle: string | null = null let startupTerminalHandle: string | null = null let startupTerminalTabId: string | null = null let startupTerminalPaneKey: string | null = null @@ -19250,11 +19544,13 @@ export class OrcaRuntimeService { ? 'windows' : 'posix' : 'posix', + observeSetupCompletion: args.observeSetupCompletion, // Why: carry the wait-for-agent wrapped setup command (#6298) so the // Setup tab runs the same script the sequenced agent waits on. ...(wrappedSetupCommandStr ? { wrappedSetupCommand: wrappedSetupCommandStr } : {}) }) didSpawnSetup = provisioned.setupSpawned + setupTerminalHandle = provisioned.setupTerminalHandle } // Why: when runtime spawned setup, omit it from activation. When setup // spawn failed, fall through with the wrapped command so renderer @@ -19290,7 +19586,7 @@ export class OrcaRuntimeService { } else if (this.ptyController?.spawn && (setup || defaultTabs || didSpawnStartup)) { // Why: inactive terminal materialization matches normal worktree creation, // but setup/default tab failures must not gate automation dispatch. - void this.provisionManagedWorktreeTerminals({ + const provisioning = this.provisionManagedWorktreeTerminals({ worktreeSelector: `id:${worktree.id}`, worktreeId: worktree.id, worktreePath, @@ -19303,12 +19599,20 @@ export class OrcaRuntimeService { ? 'windows' : 'posix' : 'posix', + observeSetupCompletion: args.observeSetupCompletion, ...(wrappedSetupCommandStr ? { wrappedSetupCommand: wrappedSetupCommandStr } : {}) }) // Why: runtime owns setup spawning here, so the RPC result must omit setup // to keep the headless/mobile caller from launching it a second time. - if (setup) { - didSpawnSetup = true + if (args.awaitTerminalProvisioning) { + const provisioned = await provisioning + didSpawnSetup = provisioned.setupSpawned + setupTerminalHandle = provisioned.setupTerminalHandle + } else { + void provisioning + if (setup) { + didSpawnSetup = true + } } } else if (this.ptyController?.spawn) { try { @@ -19348,6 +19652,25 @@ export class OrcaRuntimeService { }, ...(lineageInput ? { lineage, workspaceLineage, warnings: lineageWarnings } : {}), ...(returnedSetup ? { setup: returnedSetup } : {}), + ...(args.awaitTerminalProvisioning + ? { + setupReceipt: { + requested: effectiveDecision, + hookFound: Boolean(hooks?.scripts.setup), + startupPolicy: setup?.waitForAgentStartup + ? ('wait-for-setup' as const) + : ('start-immediately' as const), + state: !hooks?.scripts.setup + ? ('not_configured' as const) + : effectiveDecision === 'skip' || !shouldRunSetup + ? ('skipped' as const) + : didSpawnSetup + ? ('running' as const) + : ('spawn_failed' as const), + ...(setupTerminalHandle ? { terminalHandle: setupTerminalHandle } : {}) + } + } + : {}), ...(defaultTabs ? { defaultTabs } : {}), ...(warning ? { warning } : {}), ...(addResult.localBaseRefRefresh @@ -19397,6 +19720,8 @@ export class OrcaRuntimeService { runHooks?: boolean activate?: boolean setupDecision?: 'run' | 'skip' | 'inherit' + awaitTerminalProvisioning?: boolean + observeSetupCompletion?: boolean createdWithAgent?: TuiAgent pendingFirstAgentMessageRename?: boolean automationProvenance?: AutomationWorkspaceProvenance @@ -19474,6 +19799,7 @@ export class OrcaRuntimeService { // Why: same no-double-spawn contract as the local path — once runtime // provisions setup, omit it from activation and the RPC result. let didSpawnSetup = false + let setupTerminalHandle: string | null = null let startupTerminalHandle: string | null = null let startupTerminalTabId: string | null = null let startupTerminalPaneKey: string | null = null @@ -19557,11 +19883,13 @@ export class OrcaRuntimeService { ? 'windows' : 'posix' : 'posix', + observeSetupCompletion: args.observeSetupCompletion, // Why: carry the wait-for-agent wrapped setup command (#6298) so the // remote Setup tab runs the same script the sequenced agent waits on. ...(wrappedSetupCommandStr ? { wrappedSetupCommand: wrappedSetupCommandStr } : {}) }) didSpawnSetup = provisioned.setupSpawned + setupTerminalHandle = provisioned.setupTerminalHandle } // Why: omit setup from activation when runtime spawned it; on spawn // failure fall through with the wrapped command so renderer retries. @@ -19602,7 +19930,7 @@ export class OrcaRuntimeService { ) { // Why: inactive terminal materialization matches normal worktree creation, // but setup/default tab failures must not gate automation dispatch. - void this.provisionManagedWorktreeTerminals({ + const provisioning = this.provisionManagedWorktreeTerminals({ worktreeSelector: `path:${result.worktree.path}`, worktreeId: result.worktree.id, worktreePath: result.worktree.path, @@ -19615,12 +19943,20 @@ export class OrcaRuntimeService { ? 'windows' : 'posix' : 'posix', + observeSetupCompletion: args.observeSetupCompletion, ...(wrappedSetupCommandStr ? { wrappedSetupCommand: wrappedSetupCommandStr } : {}) }) // Why: runtime owns setup spawning here, so omit setup from the RPC result // to keep the headless/mobile caller from launching it a second time. - if (result.setup) { - didSpawnSetup = true + if (args.awaitTerminalProvisioning) { + const provisioned = await provisioning + didSpawnSetup = provisioned.setupSpawned + setupTerminalHandle = provisioned.setupTerminalHandle + } else { + void provisioning + if (result.setup) { + didSpawnSetup = true + } } } else if (!shouldActivate && this.ptyController?.spawn) { try { @@ -19665,7 +20001,27 @@ export class OrcaRuntimeService { } : resultForRenderer - return warning ? { ...resultWithStartupTerminal, warning } : resultWithStartupTerminal + const requestedSetupDecision = args.runHooks ? 'run' : (args.setupDecision ?? 'inherit') + const setupReceipt = { + requested: requestedSetupDecision, + hookFound: Boolean(result.setup), + startupPolicy: result.setup?.waitForAgentStartup + ? ('wait-for-setup' as const) + : ('start-immediately' as const), + state: + requestedSetupDecision === 'skip' + ? ('skipped' as const) + : !result.setup + ? ('not_configured' as const) + : didSpawnSetup + ? ('running' as const) + : ('spawn_failed' as const), + ...(setupTerminalHandle ? { terminalHandle: setupTerminalHandle } : {}) + } + const resultWithSetupReceipt = args.awaitTerminalProvisioning + ? { ...resultWithStartupTerminal, setupReceipt } + : resultWithStartupTerminal + return warning ? { ...resultWithSetupReceipt, warning } : resultWithSetupReceipt } /** @@ -25598,6 +25954,7 @@ export class OrcaRuntimeService { this.advancePtyLifecycleGeneration(ptyId) this.ptysById.delete(ptyId) this.recentPtyOutputById.delete(ptyId) + this.setupCompletionTokenByPtyId.delete(ptyId) this.clearWaitBlockedCheckState(ptyId) this.recentPtyPathCandidatesById.delete(ptyId) this.ptyOutputSequenceById.delete(ptyId) @@ -27057,15 +27414,25 @@ export class OrcaRuntimeService { if (messageType && waiter.typeFilter && !waiter.typeFilter.includes(messageType)) { continue } - this.resolveMessageWaiter(waiter) + this.resolveMessageWaiter(waiter, 'notified') } } waitForMessage( handle: string, - options?: { typeFilter?: string[]; timeoutMs?: number; signal?: AbortSignal } - ): Promise { + options?: { + typeFilter?: string[] + timeoutMs?: number + signal?: AbortSignal + exclusive?: boolean + } + ): Promise { return new Promise((resolve) => { + const currentWaiters = this.messageWaitersByHandle.get(handle) + if (options?.exclusive && currentWaiters && currentWaiters.size > 0) { + resolve('waiter_exists') + return + } const timeoutMs = options?.timeoutMs ?? MESSAGE_WAIT_DEFAULT_TIMEOUT_MS const waiter: MessageWaiter = { @@ -27080,11 +27447,11 @@ export class OrcaRuntimeService { const signal = options?.signal const onAbort = (): void => { this.removeMessageWaiter(waiter) - resolve() + resolve('cancelled') } if (signal) { if (signal.aborted) { - resolve() + resolve('cancelled') return } waiter.abortCleanup = () => signal.removeEventListener('abort', onAbort) @@ -27093,7 +27460,7 @@ export class OrcaRuntimeService { waiter.timeout = setTimeout(() => { this.removeMessageWaiter(waiter) - resolve() + resolve('timed_out') }, timeoutMs) let waiters = this.messageWaitersByHandle.get(handle) @@ -27105,9 +27472,19 @@ export class OrcaRuntimeService { }) } - private resolveMessageWaiter(waiter: MessageWaiter): void { + cancelMessageWaiters(handle: string): void { + const waiters = this.messageWaitersByHandle.get(handle) + if (!waiters) { + return + } + for (const waiter of [...waiters]) { + this.resolveMessageWaiter(waiter, 'cancelled') + } + } + + private resolveMessageWaiter(waiter: MessageWaiter, result: MessageWaitResult): void { this.removeMessageWaiter(waiter) - waiter.resolve() + waiter.resolve(result) } private removeMessageWaiter(waiter: MessageWaiter): void { diff --git a/src/main/runtime/orchestration-cli-subprocess.test.ts b/src/main/runtime/orchestration-cli-subprocess.test.ts index 6ec938a7e28..778ee9d8545 100644 --- a/src/main/runtime/orchestration-cli-subprocess.test.ts +++ b/src/main/runtime/orchestration-cli-subprocess.test.ts @@ -19,7 +19,7 @@ import { spawn } from 'node:child_process' import { existsSync, mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { OrcaRuntimeService } from './orca-runtime' import { OrchestrationDb } from './orchestration/db' import { OrcaRuntimeRpcServer } from './runtime-rpc' @@ -198,6 +198,15 @@ describeIfBuilt('orca orchestration reset subprocess', () => { const runtime = new OrcaRuntimeService() const db = new OrchestrationDb(':memory:') runtime.setOrchestrationDb(db) + const coordinatorPaneKey = 'tab_cli:11111111-1111-4111-8111-111111111111' + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === 'term_cli' ? coordinatorPaneKey : null + ) + db.createRun({ + objective: 'CLI reset subprocess fixture', + coordinatorHandle: 'term_cli', + coordinatorPaneKey + }) const server = new OrcaRuntimeRpcServer({ runtime, userDataPath }) await server.start() @@ -218,6 +227,8 @@ describeIfBuilt('orca orchestration reset subprocess', () => { 'task-create', '--spec', 'throwaway task', + '--from', + 'term_cli', '--json' ]) expect(create.exitCode, create.stderr).toBe(0) @@ -253,18 +264,41 @@ describeIfBuilt('orca orchestration reset subprocess', () => { expect(db.getInbox()).toHaveLength(1) expect(db.listTasks()).toHaveLength(0) + // Why: task resets intentionally remove Runs, so recreate the caller's + // normal binding before exercising task creation again. + db.createRun({ + objective: 'CLI reset subprocess fixture after reset', + coordinatorHandle: 'term_cli', + coordinatorPaneKey + }) const recreate = await runBuiltCli(userDataPath, [ 'orchestration', 'task-create', '--spec', 'throwaway task after partial reset', + '--from', + 'term_cli', '--json' ]) expect(recreate.exitCode, recreate.stderr).toBe(0) expect(db.getInbox()).toHaveLength(1) expect(db.listTasks()).toHaveLength(1) - const resetAll = await runBuiltCli(userDataPath, ['orchestration', 'reset', '--json']) + const bareReset = await runBuiltCli(userDataPath, ['orchestration', 'reset', '--json']) + expect(bareReset.exitCode).toBe(1) + expect(JSON.parse(bareReset.stdout)).toMatchObject({ + ok: false, + error: { code: 'invalid_argument' } + }) + expect(db.getInbox()).toHaveLength(1) + expect(db.listTasks()).toHaveLength(1) + + const resetAll = await runBuiltCli(userDataPath, [ + 'orchestration', + 'reset', + '--all', + '--json' + ]) expect(resetAll.exitCode, resetAll.stderr).toBe(0) expect(JSON.parse(resetAll.stdout)).toMatchObject({ ok: true, result: { reset: 'all' } }) expect(db.getInbox()).toHaveLength(0) diff --git a/src/main/runtime/orchestration/__snapshots__/preamble.test.ts.snap b/src/main/runtime/orchestration/__snapshots__/preamble.test.ts.snap index c55426c4216..39593677687 100644 --- a/src/main/runtime/orchestration/__snapshots__/preamble.test.ts.snap +++ b/src/main/runtime/orchestration/__snapshots__/preamble.test.ts.snap @@ -10,7 +10,7 @@ Slack, GitHub comments, or any other channel to reach a human during the run. === CLI COMMANDS === - # Report task completion (REQUIRED when done — even on failure). + # Report the terminal task outcome (REQUIRED exactly once). # # RULE: --body must be a 3-sentence executive summary (what you did, # what you found, what's left). Never send an empty body; the coordinator @@ -18,14 +18,15 @@ Slack, GitHub comments, or any other channel to reach a human during the run. # If you produced a long-form artifact, include its path as # payload.reportPath so the coordinator can find it without a file search. # - # RULE: send worker_done exactly once. Failure is still a worker_done - # with subject like "Failed: " — never silently exit. + # RULE: send worker_done exactly once. Use --outcome succeeded when the + # requested work is done, or replace it with --outcome failed when it is not. + # Never encode failure only in prose and never silently exit. # Include BOTH taskId and dispatchId in the payload so a late completion # from a failed retry cannot complete the current dispatch. - orca orchestration send --to term_COORD --from term_WORKER \\ + orca orchestration send --from term_WORKER \\ --type worker_done --subject "" \\ --body "<3-sentence summary: what you did, what you found, what's left>" \\ - --task-id task_SNAP --dispatch-id ctx_SNAP \\ + --task-id task_SNAP --dispatch-id ctx_SNAP --outcome succeeded \\ --files-modified "path/a,path/b" \\ --report-path "" @@ -39,7 +40,7 @@ Slack, GitHub comments, or any other channel to reach a human during the run. # attributes the heartbeat to the specific dispatch context, not just # the task, so a straggler heartbeat from a previously-failed dispatch # cannot mask a hung retry. - orca orchestration send --to term_COORD --from term_WORKER \\ + orca orchestration send --from term_WORKER \\ --type heartbeat --subject "alive" \\ --task-id task_SNAP --dispatch-id ctx_SNAP \\ --phase "" @@ -52,18 +53,18 @@ Slack, GitHub comments, or any other channel to reach a human during the run. # coordinator cannot see and cannot answer — your session will hang forever # waiting on a human. Every interactive question goes through \`ask\` below. # - # The \`ask\` verb is a thin wrapper: it sends a decision_gate message and - # blocks on \`check --wait\` until the coordinator replies, then prints the - # reply body. Use it anywhere you would otherwise have reached for - # AskUserQuestion. - orca orchestration ask --to term_COORD --from term_WORKER \\ + # The \`ask\` verb durably records a question in this Dispatch's Run and + # blocks until the coordinator replies, then prints the reply body. If the + # call times out or disconnects, resume with the returned message ID instead + # of creating a duplicate question. + orca orchestration ask --from term_WORKER \\ --question "" \\ --options "" \\ --timeout-ms 600000 # Escalate a blocker or failure (pre-completion, when you need the # coordinator to do something before you can continue): - orca orchestration send --to term_COORD --from term_WORKER \\ + orca orchestration send --from term_WORKER \\ --type escalation --subject "Blocked: " \\ --body "
" \\ --task-id task_SNAP diff --git a/src/main/runtime/orchestration/coordinator.test.ts b/src/main/runtime/orchestration/coordinator.test.ts index cb532b71e36..31e9180c93e 100644 --- a/src/main/runtime/orchestration/coordinator.test.ts +++ b/src/main/runtime/orchestration/coordinator.test.ts @@ -98,6 +98,7 @@ function insertWorkerDone( payload: JSON.stringify({ taskId: params.taskId, dispatchId, + outcome: 'succeeded', ...(params.filesModified ? { filesModified: params.filesModified } : {}) }), senderPaneKey: @@ -192,7 +193,7 @@ describe('Coordinator', () => { to: 'coord', subject: 'Done', type: 'worker_done', - payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id }) + payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id, outcome: 'succeeded' }) }) reconcileLifecycleMessage(db, msg) @@ -214,7 +215,11 @@ describe('Coordinator', () => { const task = db.createTask({ spec: 'duplicate completion' }) const dispatch = db.createDispatchContext(task.id, 'term_a') - const payload = JSON.stringify({ taskId: task.id, dispatchId: dispatch.id }) + const payload = JSON.stringify({ + taskId: task.id, + dispatchId: dispatch.id, + outcome: 'succeeded' + }) const first = db.insertMessage({ from: 'term_a', to: 'coord', @@ -569,7 +574,11 @@ describe('Coordinator', () => { to: 'coord', subject: 'Late done', type: 'worker_done', - payload: JSON.stringify({ taskId: task.id, dispatchId: staleCtx.id }) + payload: JSON.stringify({ + taskId: task.id, + dispatchId: staleCtx.id, + outcome: 'succeeded' + }) }) const staleCoordinator = new Coordinator(db, runtime, { @@ -621,7 +630,7 @@ describe('Coordinator', () => { to: 'coord', subject: 'Done after restart', type: 'worker_done', - payload: JSON.stringify({ taskId: task.id, dispatchId: ctx.id }), + payload: JSON.stringify({ taskId: task.id, dispatchId: ctx.id, outcome: 'succeeded' }), senderPaneKey: `tab_after:${leafId}` }) diff --git a/src/main/runtime/orchestration/coordinator.ts b/src/main/runtime/orchestration/coordinator.ts index d0e56b7e40a..2af865a0df5 100644 --- a/src/main/runtime/orchestration/coordinator.ts +++ b/src/main/runtime/orchestration/coordinator.ts @@ -242,6 +242,7 @@ export class Coordinator { case 'dispatch': case 'handoff': case 'merge_ready': + case 'question': break } } @@ -255,6 +256,10 @@ export class Coordinator { if (!this.state.completedTasks.includes(result.taskId)) { this.state.completedTasks.push(result.taskId) } + return + } + if (result.action === 'failed' && !this.state.failedTasks.includes(result.taskId)) { + this.state.failedTasks.push(result.taskId) } } diff --git a/src/main/runtime/orchestration/db.test.ts b/src/main/runtime/orchestration/db.test.ts index 15fb0a06a00..73e0468d011 100644 --- a/src/main/runtime/orchestration/db.test.ts +++ b/src/main/runtime/orchestration/db.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, describe, expect, it } from 'vitest' import Database from '../../sqlite/sync-database' -import { OrchestrationDb } from './db' +import { LEGACY_RUN_ID, OrchestrationDb } from './db' import type { MessageType } from './db' // Overwrites the datetime('now')-seeded timestamps with explicit fixture values @@ -942,6 +942,8 @@ describe('OrchestrationDb', () => { // v1 data preserved expect(d.getMessageById('msg_v1')?.subject).toBe('pre-migration') + expect(d.getMessageById('msg_v1')?.run_id).toBe(LEGACY_RUN_ID) + expect(d.getRun(LEGACY_RUN_ID)).toMatchObject({ legacy: 1 }) }) it('adds pane-identity columns (v6) and persists them', () => { diff --git a/src/main/runtime/orchestration/db.ts b/src/main/runtime/orchestration/db.ts index b287cb0182c..04fcb29fb55 100644 --- a/src/main/runtime/orchestration/db.ts +++ b/src/main/runtime/orchestration/db.ts @@ -1,5 +1,6 @@ /* eslint-disable max-lines -- Why: the orchestration DB keeps schema creation, message CRUD, task DAG resolution, and dispatch context management in one class so transactional invariants (e.g. promoteReadyTasks running inside the same writer as updateTaskStatus) are enforced by locality. */ -import { randomBytes } from 'node:crypto' +import { createHash, randomBytes, timingSafeEqual } from 'node:crypto' +import { chmodSync, existsSync } from 'node:fs' import Database from '../../sqlite/sync-database' import type { MessageType, @@ -12,10 +13,26 @@ import type { TaskRow, DispatchContextRow, DecisionGateRow, - CoordinatorRun + CoordinatorRun, + WorkerReportOutcome, + WorkerReportSettlement, + RunRow, + DeliveryRow, + DeliveryStatus, + QuestionRow, + QuestionStatus, + MutationReceiptRow, + MutationState, + WorkerDispatchRow, + WorkerDispatchState, + FederatedDispatchRow, + RemoteDispatchAttachmentRow, + FederationRelayDirection, + FederationRelayItemRow } from './types' import { buildOrchestrationTaskDisplayMetadata } from '../../../shared/orchestration-task-display' import { parsePaneKey } from '../../../shared/stable-pane-id' +import { OrchestrationError } from './orchestration-error' // Why: leaf UUID is the remint-stable pane identity (tab half changes on break-out); exact match covers legacy/unparseable keys. function isEquivalentPaneKey(a: string, b: string): boolean { @@ -38,14 +55,29 @@ export type { TaskRow, DispatchContextRow, DecisionGateRow, - CoordinatorRun + CoordinatorRun, + WorkerReportOutcome, + WorkerReportSettlement, + RunRow, + DeliveryRow, + DeliveryStatus, + QuestionRow, + QuestionStatus, + MutationReceiptRow, + MutationState, + WorkerDispatchRow, + WorkerDispatchState } function generateId(prefix: string): string { return `${prefix}_${randomBytes(6).toString('hex')}` } -function addLifecycleRejectionMarker(payload: string | null, reason: string): string { +function hashDispatchCapability(capability: string): string { + return createHash('sha256').update(capability).digest('hex') +} + +function addLifecycleRejectionMarker(payload: string | null, code: string, reason: string): string { let parsed: Record = {} try { const value: unknown = payload ? JSON.parse(payload) : {} @@ -57,7 +89,7 @@ function addLifecycleRejectionMarker(payload: string | null, reason: string): st } return JSON.stringify({ ...parsed, - _orcaLifecycleRejection: { code: 'sender_not_assignee', reason } + _orcaLifecycleRejection: { code, reason } }) } @@ -83,8 +115,47 @@ function exposeMessageListTimestamps(messages: MessageRow[]): MessageRow[] { return messages.map(exposeMessageTimestamps) } -// Schema versions: v2 'heartbeat'+last_heartbeat_at, v3 delivered_at, v4 task-creator terminal, v5 task_title/display_name, v6 pane-identity columns. -const SCHEMA_VERSION = 6 +function exposeRunTimestamps(run: RunRow): RunRow { + return { + ...run, + created_at: exposeUtcTimestamp(run.created_at) ?? run.created_at, + updated_at: exposeUtcTimestamp(run.updated_at) ?? run.updated_at + } +} + +function exposeDeliveryTimestamps(delivery: DeliveryRow): DeliveryRow { + return { + ...delivery, + created_at: exposeUtcTimestamp(delivery.created_at) ?? delivery.created_at, + acknowledged_at: exposeUtcTimestamp(delivery.acknowledged_at) + } +} + +function exposeQuestionTimestamps(question: QuestionRow): QuestionRow { + return { + ...question, + created_at: exposeUtcTimestamp(question.created_at) ?? question.created_at, + answered_at: exposeUtcTimestamp(question.answered_at), + closed_at: exposeUtcTimestamp(question.closed_at) + } +} + +export const LEGACY_RUN_ID = 'run_legacy_local' + +// Schema versions: v2 'heartbeat'+last_heartbeat_at, v3 delivered_at, v4 task-creator terminal, v5 task_title/display_name, v6 pane identity, v7 lightweight Runs, v8 crash-safe Run deliveries, v9 durable question threads, v10 Dispatch capabilities, v11 durable mutation receipts, v12 composed worker state. +const SCHEMA_VERSION = 17 + +function hardenOrchestrationDatabaseFiles(dbPath: string | ':memory:'): void { + if (dbPath === ':memory:' || process.platform === 'win32') { + // Why: Windows protects these files through Orca's current-user-only userData DACL; POSIX mode bits are inert there. + return + } + for (const path of [dbPath, `${dbPath}-wal`, `${dbPath}-shm`]) { + if (existsSync(path)) { + chmodSync(path, 0o600) + } + } +} export class OrchestrationDb { private db: Database.Database @@ -104,12 +175,26 @@ export class OrchestrationDb { this.db.pragma('busy_timeout = 5000') this.createTables() this.migrate() + hardenOrchestrationDatabaseFiles(dbPath) } private createTables(): void { this.db.exec(` + CREATE TABLE IF NOT EXISTS runs ( + id TEXT PRIMARY KEY, + objective TEXT NOT NULL, + home_database TEXT NOT NULL DEFAULT 'this_database', + coordinator_handle TEXT, + coordinator_pane_key TEXT, + consumer_generation INTEGER NOT NULL DEFAULT 0, + legacy INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + CREATE TABLE IF NOT EXISTS messages ( id TEXT NOT NULL, + run_id TEXT NOT NULL DEFAULT '${LEGACY_RUN_ID}', from_handle TEXT NOT NULL, to_handle TEXT NOT NULL, subject TEXT NOT NULL, @@ -117,7 +202,7 @@ export class OrchestrationDb { type TEXT NOT NULL DEFAULT 'status' CHECK(type IN ( 'status', 'dispatch', 'worker_done', 'merge_ready', - 'escalation', 'handoff', 'decision_gate', 'heartbeat' + 'escalation', 'handoff', 'decision_gate', 'question', 'heartbeat' )), priority TEXT NOT NULL DEFAULT 'normal' CHECK(priority IN ('normal', 'high', 'urgent')), @@ -134,8 +219,129 @@ export class OrchestrationDb { CREATE INDEX IF NOT EXISTS idx_inbox ON messages(to_handle, read); CREATE INDEX IF NOT EXISTS idx_thread ON messages(thread_id); + CREATE TABLE IF NOT EXISTS deliveries ( + id TEXT PRIMARY KEY, + run_id TEXT NOT NULL, + consumer_generation INTEGER NOT NULL, + message_ids TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'outstanding' + CHECK(status IN ('outstanding', 'acknowledged', 'fenced')), + created_at TEXT NOT NULL DEFAULT (datetime('now')), + acknowledged_at TEXT + ); + + CREATE UNIQUE INDEX IF NOT EXISTS idx_deliveries_one_outstanding + ON deliveries(run_id) WHERE status = 'outstanding'; + CREATE INDEX IF NOT EXISTS idx_deliveries_run_created + ON deliveries(run_id, created_at); + + CREATE TABLE IF NOT EXISTS mutation_receipts ( + caller_fingerprint TEXT NOT NULL, + request_id TEXT NOT NULL, + method TEXT NOT NULL, + payload_hash TEXT NOT NULL, + state TEXT NOT NULL DEFAULT 'pending' + CHECK(state IN ('pending', 'completed')), + receipt TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY (caller_fingerprint, request_id) + ); + + CREATE TABLE IF NOT EXISTS worker_dispatches ( + dispatch_id TEXT PRIMARY KEY, + runtime_epoch TEXT, + state TEXT NOT NULL DEFAULT 'starting' + CHECK(state IN ( + 'starting', 'ready', 'start_unknown', 'failed', 'succeeded', + 'stopping', 'stop_unknown', 'stopped', 'abandoned' + )), + stage TEXT NOT NULL DEFAULT 'accepted', + worktree_id TEXT, + agent_terminal_handle TEXT, + setup_state TEXT NOT NULL DEFAULT 'not_applicable', + effects TEXT NOT NULL DEFAULT '[]', + residual_resources TEXT NOT NULL DEFAULT '[]', + start_options TEXT NOT NULL DEFAULT '{}', + last_error TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + + CREATE TABLE IF NOT EXISTS federated_dispatches ( + dispatch_id TEXT PRIMARY KEY, + environment_id TEXT NOT NULL, + environment_name TEXT NOT NULL, + peer_fingerprint TEXT NOT NULL, + remote_runtime_epoch TEXT, + protocol_version INTEGER NOT NULL DEFAULT 1, + remote_worktree_id TEXT, + remote_terminal_handle TEXT, + to_home_imported_sequence INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + + CREATE TABLE IF NOT EXISTS remote_dispatch_attachments ( + dispatch_id TEXT PRIMARY KEY, + task_id TEXT NOT NULL, + home_peer_fingerprint TEXT NOT NULL, + protocol_version INTEGER NOT NULL DEFAULT 1, + runtime_epoch TEXT NOT NULL, + capability_hash TEXT, + pane_key TEXT, + process_incarnation TEXT, + state TEXT NOT NULL DEFAULT 'starting' + CHECK(state IN ( + 'starting', 'ready', 'start_unknown', 'failed', 'succeeded', + 'stopping', 'stop_unknown', 'stopped', 'abandoned' + )), + stage TEXT NOT NULL DEFAULT 'accepted', + worktree_id TEXT, + terminal_handle TEXT, + setup_state TEXT NOT NULL DEFAULT 'not_applicable', + effects TEXT NOT NULL DEFAULT '[]', + residual_resources TEXT NOT NULL DEFAULT '[]', + to_worker_imported_sequence INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + + CREATE TABLE IF NOT EXISTS federation_relay_items ( + dispatch_id TEXT NOT NULL, + direction TEXT NOT NULL CHECK(direction IN ('to_home', 'to_worker')), + sequence INTEGER NOT NULL, + message_id TEXT NOT NULL, + kind TEXT NOT NULL, + payload TEXT NOT NULL, + byte_count INTEGER NOT NULL, + acked_at TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY (dispatch_id, direction, sequence), + UNIQUE (dispatch_id, direction, message_id) + ); + + CREATE INDEX IF NOT EXISTS idx_federation_relay_pending + ON federation_relay_items(dispatch_id, direction, acked_at, sequence); + + CREATE TABLE IF NOT EXISTS remote_questions ( + message_id TEXT PRIMARY KEY, + dispatch_id TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending' + CHECK(status IN ('pending', 'answered', 'closed')), + answer_message_id TEXT, + answer_body TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + answered_at TEXT + ); + + CREATE INDEX IF NOT EXISTS idx_remote_questions_dispatch_status + ON remote_questions(dispatch_id, status); + CREATE TABLE IF NOT EXISTS tasks ( id TEXT PRIMARY KEY, + run_id TEXT NOT NULL DEFAULT '${LEGACY_RUN_ID}', parent_id TEXT, created_by_terminal_handle TEXT, task_title TEXT, @@ -157,9 +363,13 @@ export class OrchestrationDb { CREATE TABLE IF NOT EXISTS dispatch_contexts ( id TEXT PRIMARY KEY, + run_id TEXT NOT NULL DEFAULT '${LEGACY_RUN_ID}', task_id TEXT NOT NULL, assignee_handle TEXT, assignee_pane_key TEXT, + capability_hash TEXT, + process_incarnation TEXT, + capability_revoked_at TEXT, status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending', 'dispatched', 'completed', 'failed', 'circuit_broken')), failure_count INTEGER NOT NULL DEFAULT 0, @@ -175,6 +385,7 @@ export class OrchestrationDb { CREATE TABLE IF NOT EXISTS decision_gates ( id TEXT PRIMARY KEY, + run_id TEXT NOT NULL DEFAULT '${LEGACY_RUN_ID}', task_id TEXT NOT NULL, question TEXT NOT NULL, options TEXT NOT NULL DEFAULT '[]', @@ -229,7 +440,7 @@ export class OrchestrationDb { type TEXT NOT NULL DEFAULT 'status' CHECK(type IN ( 'status', 'dispatch', 'worker_done', 'merge_ready', - 'escalation', 'handoff', 'decision_gate', 'heartbeat' + 'escalation', 'handoff', 'decision_gate', 'question', 'heartbeat' )), priority TEXT NOT NULL DEFAULT 'normal' CHECK(priority IN ('normal', 'high', 'urgent')), @@ -287,6 +498,253 @@ export class OrchestrationDb { this.db.exec(`ALTER TABLE messages ADD COLUMN sender_pane_key TEXT`) } } + if (current < 7) { + this.db + .prepare( + `INSERT OR IGNORE INTO runs ( + id, objective, home_database, consumer_generation, legacy + ) VALUES (?, ?, 'this_database', 0, 1)` + ) + .run(LEGACY_RUN_ID, 'Legacy orchestration state (inspect only)') + for (const table of ['messages', 'tasks', 'dispatch_contexts', 'decision_gates']) { + if (!this.hasColumn(table, 'run_id')) { + this.db.exec( + `ALTER TABLE ${table} ADD COLUMN run_id TEXT NOT NULL DEFAULT '${LEGACY_RUN_ID}'` + ) + } + } + this.db.exec(` + CREATE INDEX IF NOT EXISTS idx_messages_run_sequence ON messages(run_id, sequence); + CREATE INDEX IF NOT EXISTS idx_tasks_run_status ON tasks(run_id, status); + CREATE INDEX IF NOT EXISTS idx_dispatch_run_status ON dispatch_contexts(run_id, status); + CREATE INDEX IF NOT EXISTS idx_gates_run_status ON decision_gates(run_id, status); + CREATE INDEX IF NOT EXISTS idx_runs_coordinator_pane ON runs(coordinator_pane_key); + `) + } + if (current < 8) { + this.db.exec(` + CREATE TABLE IF NOT EXISTS deliveries ( + id TEXT PRIMARY KEY, + run_id TEXT NOT NULL, + consumer_generation INTEGER NOT NULL, + message_ids TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'outstanding' + CHECK(status IN ('outstanding', 'acknowledged', 'fenced')), + created_at TEXT NOT NULL DEFAULT (datetime('now')), + acknowledged_at TEXT + ); + CREATE UNIQUE INDEX IF NOT EXISTS idx_deliveries_one_outstanding + ON deliveries(run_id) WHERE status = 'outstanding'; + CREATE INDEX IF NOT EXISTS idx_deliveries_run_created + ON deliveries(run_id, created_at); + + CREATE TABLE IF NOT EXISTS question_threads ( + message_id TEXT PRIMARY KEY, + run_id TEXT NOT NULL, + dispatch_id TEXT NOT NULL, + asker_handle TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending' + CHECK(status IN ('pending', 'answered', 'closed')), + answer_message_id TEXT, + answer_body TEXT, + answered_by_generation INTEGER, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + answered_at TEXT, + closed_at TEXT + ); + + CREATE INDEX IF NOT EXISTS idx_questions_dispatch_status + ON question_threads(dispatch_id, status); + `) + } + if (current < 9 && !this.messagesTypeCheckAllowsQuestion()) { + this.db.exec(` + CREATE TABLE messages_new ( + id TEXT NOT NULL, + run_id TEXT NOT NULL DEFAULT '${LEGACY_RUN_ID}', + from_handle TEXT NOT NULL, + to_handle TEXT NOT NULL, + subject TEXT NOT NULL, + body TEXT NOT NULL DEFAULT '', + type TEXT NOT NULL DEFAULT 'status' + CHECK(type IN ( + 'status', 'dispatch', 'worker_done', 'merge_ready', + 'escalation', 'handoff', 'decision_gate', 'question', 'heartbeat' + )), + priority TEXT NOT NULL DEFAULT 'normal' + CHECK(priority IN ('normal', 'high', 'urgent')), + thread_id TEXT, + payload TEXT, + read INTEGER NOT NULL DEFAULT 0, + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + delivered_at TEXT, + sender_pane_key TEXT + ); + INSERT INTO messages_new ( + id, run_id, from_handle, to_handle, subject, body, type, priority, + thread_id, payload, read, sequence, created_at, delivered_at, sender_pane_key + ) + SELECT + id, run_id, from_handle, to_handle, subject, body, type, priority, + thread_id, payload, read, sequence, created_at, delivered_at, sender_pane_key + FROM messages; + DROP TABLE messages; + ALTER TABLE messages_new RENAME TO messages; + + CREATE UNIQUE INDEX idx_messages_id ON messages(id); + CREATE INDEX idx_inbox ON messages(to_handle, read); + CREATE INDEX idx_thread ON messages(thread_id); + CREATE INDEX idx_messages_run_sequence ON messages(run_id, sequence); + CREATE INDEX idx_messages_undelivered_inbox + ON messages(to_handle, read, delivered_at, sequence); + `) + } + if (current < 10) { + if (!this.hasColumn('dispatch_contexts', 'capability_hash')) { + this.db.exec('ALTER TABLE dispatch_contexts ADD COLUMN capability_hash TEXT') + } + if (!this.hasColumn('dispatch_contexts', 'process_incarnation')) { + this.db.exec('ALTER TABLE dispatch_contexts ADD COLUMN process_incarnation TEXT') + } + if (!this.hasColumn('dispatch_contexts', 'capability_revoked_at')) { + this.db.exec('ALTER TABLE dispatch_contexts ADD COLUMN capability_revoked_at TEXT') + } + } + if (current < 11) { + this.db.exec(` + CREATE TABLE IF NOT EXISTS mutation_receipts ( + caller_fingerprint TEXT NOT NULL, + request_id TEXT NOT NULL, + method TEXT NOT NULL, + payload_hash TEXT NOT NULL, + state TEXT NOT NULL DEFAULT 'pending' + CHECK(state IN ('pending', 'completed')), + receipt TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY (caller_fingerprint, request_id) + ); + `) + } + if (current < 12) { + this.db.exec(` + CREATE TABLE IF NOT EXISTS worker_dispatches ( + dispatch_id TEXT PRIMARY KEY, + runtime_epoch TEXT, + state TEXT NOT NULL DEFAULT 'starting' + CHECK(state IN ( + 'starting', 'ready', 'start_unknown', 'failed', 'succeeded', + 'stopping', 'stop_unknown', 'stopped', 'abandoned' + )), + stage TEXT NOT NULL DEFAULT 'accepted', + worktree_id TEXT, + agent_terminal_handle TEXT, + setup_state TEXT NOT NULL DEFAULT 'not_applicable', + effects TEXT NOT NULL DEFAULT '[]', + residual_resources TEXT NOT NULL DEFAULT '[]', + start_options TEXT NOT NULL DEFAULT '{}', + last_error TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + `) + } + if (current < 13 && !this.hasColumn('worker_dispatches', 'runtime_epoch')) { + this.db.exec('ALTER TABLE worker_dispatches ADD COLUMN runtime_epoch TEXT') + } + if (current < 14) { + this.db.exec(` + CREATE TABLE IF NOT EXISTS federated_dispatches ( + dispatch_id TEXT PRIMARY KEY, + environment_id TEXT NOT NULL, + environment_name TEXT NOT NULL, + peer_fingerprint TEXT NOT NULL, + remote_runtime_epoch TEXT, + protocol_version INTEGER NOT NULL DEFAULT 1, + remote_worktree_id TEXT, + remote_terminal_handle TEXT, + to_home_imported_sequence INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + CREATE TABLE IF NOT EXISTS remote_dispatch_attachments ( + dispatch_id TEXT PRIMARY KEY, + task_id TEXT NOT NULL, + home_peer_fingerprint TEXT NOT NULL, + protocol_version INTEGER NOT NULL DEFAULT 1, + runtime_epoch TEXT NOT NULL, + capability_hash TEXT, + pane_key TEXT, + process_incarnation TEXT, + state TEXT NOT NULL DEFAULT 'starting' + CHECK(state IN ( + 'starting', 'ready', 'start_unknown', 'failed', 'succeeded', + 'stopping', 'stop_unknown', 'stopped', 'abandoned' + )), + stage TEXT NOT NULL DEFAULT 'accepted', + worktree_id TEXT, + terminal_handle TEXT, + setup_state TEXT NOT NULL DEFAULT 'not_applicable', + effects TEXT NOT NULL DEFAULT '[]', + residual_resources TEXT NOT NULL DEFAULT '[]', + to_worker_imported_sequence INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + `) + } + if (current < 15) { + if (!this.hasColumn('federated_dispatches', 'to_home_imported_sequence')) { + this.db.exec( + 'ALTER TABLE federated_dispatches ADD COLUMN to_home_imported_sequence INTEGER NOT NULL DEFAULT 0' + ) + } + if (!this.hasColumn('remote_dispatch_attachments', 'to_worker_imported_sequence')) { + this.db.exec( + 'ALTER TABLE remote_dispatch_attachments ADD COLUMN to_worker_imported_sequence INTEGER NOT NULL DEFAULT 0' + ) + } + this.db.exec(` + CREATE TABLE IF NOT EXISTS federation_relay_items ( + dispatch_id TEXT NOT NULL, + direction TEXT NOT NULL CHECK(direction IN ('to_home', 'to_worker')), + sequence INTEGER NOT NULL, + message_id TEXT NOT NULL, + kind TEXT NOT NULL, + payload TEXT NOT NULL, + byte_count INTEGER NOT NULL, + acked_at TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY (dispatch_id, direction, sequence), + UNIQUE (dispatch_id, direction, message_id) + ); + CREATE INDEX IF NOT EXISTS idx_federation_relay_pending + ON federation_relay_items(dispatch_id, direction, acked_at, sequence); + `) + } + if (current < 16) { + this.db.exec(` + CREATE TABLE IF NOT EXISTS remote_questions ( + message_id TEXT PRIMARY KEY, + dispatch_id TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending' + CHECK(status IN ('pending', 'answered', 'closed')), + answer_message_id TEXT, + answer_body TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + answered_at TEXT + ); + CREATE INDEX IF NOT EXISTS idx_remote_questions_dispatch_status + ON remote_questions(dispatch_id, status); + `) + } + if (current < 17 && !this.hasColumn('remote_dispatch_attachments', 'protocol_version')) { + this.db.exec( + 'ALTER TABLE remote_dispatch_attachments ADD COLUMN protocol_version INTEGER NOT NULL DEFAULT 1' + ) + } this.createUndeliveredInboxIndexIfPossible() this.db.pragma(`user_version = ${SCHEMA_VERSION}`) @@ -320,9 +778,414 @@ export class OrchestrationDb { return !!row && row.sql.includes("'heartbeat'") } + private messagesTypeCheckAllowsQuestion(): boolean { + const row = this.db + .prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'messages'") + .get() as { sql: string } | undefined + return !!row && row.sql.includes("'question'") + } + + // ── Durable mutation receipts ── + + beginMutationReceipt(params: { + callerFingerprint: string + requestId: string + method: string + payloadHash: string + }): + | { disposition: 'started'; row: MutationReceiptRow } + | { disposition: 'pending'; row: MutationReceiptRow } + | { disposition: 'completed'; row: MutationReceiptRow } { + this.db.exec('BEGIN IMMEDIATE') + try { + const existing = this.getMutationReceipt(params.callerFingerprint, params.requestId) + if (existing) { + if (existing.method !== params.method || existing.payload_hash !== params.payloadHash) { + throw new OrchestrationError( + 'request_mismatch', + `Mutation request ${params.requestId} was already used with different input.` + ) + } + this.db.exec('COMMIT') + return { disposition: existing.state, row: existing } + } + this.db + .prepare( + `INSERT INTO mutation_receipts ( + caller_fingerprint, request_id, method, payload_hash, state + ) VALUES (?, ?, ?, ?, 'pending')` + ) + .run(params.callerFingerprint, params.requestId, params.method, params.payloadHash) + const row = this.getMutationReceipt(params.callerFingerprint, params.requestId) + this.db.exec('COMMIT') + return { disposition: 'started', row: row as MutationReceiptRow } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + } + + completeMutationReceipt(params: { + callerFingerprint: string + requestId: string + method: string + payloadHash: string + receipt: string + }): MutationReceiptRow { + const result = this.db + .prepare( + `UPDATE mutation_receipts + SET state = 'completed', receipt = ?, updated_at = datetime('now') + WHERE caller_fingerprint = ? AND request_id = ? AND method = ? + AND payload_hash = ?` + ) + .run( + params.receipt, + params.callerFingerprint, + params.requestId, + params.method, + params.payloadHash + ) + const row = this.getMutationReceipt(params.callerFingerprint, params.requestId) + if (result.changes !== 1 || !row) { + throw new OrchestrationError( + 'request_mismatch', + `Mutation request ${params.requestId} no longer matches its pending operation.` + ) + } + return row + } + + discardPendingMutationReceipt(callerFingerprint: string, requestId: string): void { + this.db + .prepare( + `DELETE FROM mutation_receipts + WHERE caller_fingerprint = ? AND request_id = ? AND state = 'pending'` + ) + .run(callerFingerprint, requestId) + } + + getMutationReceipt(callerFingerprint: string, requestId: string): MutationReceiptRow | undefined { + return this.db + .prepare( + `SELECT * FROM mutation_receipts + WHERE caller_fingerprint = ? AND request_id = ?` + ) + .get(callerFingerprint, requestId) as MutationReceiptRow | undefined + } + + // ── Runs ── + + createRun(params: { + objective: string + coordinatorHandle: string + coordinatorPaneKey: string + }): RunRow { + const id = generateId('run') + this.db.exec('BEGIN IMMEDIATE') + try { + this.unbindOtherRunsForPane(params.coordinatorPaneKey) + this.db + .prepare( + `INSERT INTO runs ( + id, objective, coordinator_handle, coordinator_pane_key, + consumer_generation, legacy + ) VALUES (?, ?, ?, ?, 1, 0)` + ) + .run(id, params.objective, params.coordinatorHandle, params.coordinatorPaneKey) + this.db.exec('COMMIT') + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + return this.getRun(id) as RunRow + } + + bindRun(params: { + runId: string + coordinatorHandle: string + coordinatorPaneKey: string + }): RunRow | undefined { + this.db.exec('BEGIN IMMEDIATE') + try { + const run = this.getRunRaw(params.runId) + if (!run || run.legacy === 1) { + this.db.exec('ROLLBACK') + return undefined + } + const sameBinding = + run.coordinator_pane_key !== null && + isEquivalentPaneKey(run.coordinator_pane_key, params.coordinatorPaneKey) + this.unbindOtherRunsForPane(params.coordinatorPaneKey, params.runId) + if (!sameBinding || run.coordinator_handle !== params.coordinatorHandle) { + this.db + .prepare( + `UPDATE runs + SET coordinator_handle = ?, coordinator_pane_key = ?, + consumer_generation = consumer_generation + 1, + updated_at = datetime('now') + WHERE id = ?` + ) + .run(params.coordinatorHandle, params.coordinatorPaneKey, params.runId) + this.fenceOutstandingDelivery(params.runId) + } + this.db.exec('COMMIT') + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + return this.getRun(params.runId) + } + + getRun(id: string): RunRow | undefined { + const run = this.getRunRaw(id) + return run ? exposeRunTimestamps(run) : undefined + } + + listRuns(): RunRow[] { + return (this.db.prepare('SELECT * FROM runs ORDER BY created_at DESC').all() as RunRow[]).map( + exposeRunTimestamps + ) + } + + getCurrentRunForPane(paneKey: string): RunRow | undefined { + const runs = this.db + .prepare('SELECT * FROM runs WHERE coordinator_pane_key IS NOT NULL AND legacy = 0') + .all() as RunRow[] + const run = runs.find( + (candidate) => + candidate.coordinator_pane_key !== null && + isEquivalentPaneKey(candidate.coordinator_pane_key, paneKey) + ) + return run ? exposeRunTimestamps(run) : undefined + } + + private getRunRaw(id: string): RunRow | undefined { + return this.db.prepare('SELECT * FROM runs WHERE id = ?').get(id) as RunRow | undefined + } + + private unbindOtherRunsForPane(paneKey: string, exceptRunId?: string): void { + const bound = this.db + .prepare('SELECT * FROM runs WHERE coordinator_pane_key IS NOT NULL AND legacy = 0') + .all() as RunRow[] + for (const run of bound) { + if ( + run.id !== exceptRunId && + run.coordinator_pane_key && + isEquivalentPaneKey(run.coordinator_pane_key, paneKey) + ) { + this.db + .prepare( + `UPDATE runs + SET coordinator_handle = NULL, coordinator_pane_key = NULL, + consumer_generation = consumer_generation + 1, + updated_at = datetime('now') + WHERE id = ?` + ) + .run(run.id) + this.fenceOutstandingDelivery(run.id) + } + } + } + + private requireRun(runId: string): void { + if (!this.getRunRaw(runId)) { + throw new Error(`Run not found: ${runId}`) + } + } + + private fenceOutstandingDelivery(runId: string): void { + this.db + .prepare( + "UPDATE deliveries SET status = 'fenced' WHERE run_id = ? AND status = 'outstanding'" + ) + .run(runId) + } + + private requireCurrentConsumer(runId: string, consumerGeneration: number): RunRow { + const run = this.getRunRaw(runId) + if (!run || run.legacy === 1 || run.consumer_generation !== consumerGeneration) { + throw new OrchestrationError( + 'consumer_fenced', + 'This mailbox consumer has been replaced. Rebind with orchestration run-use.' + ) + } + return run + } + + private getDeliveryRaw(id: string): DeliveryRow | undefined { + return this.db.prepare('SELECT * FROM deliveries WHERE id = ?').get(id) as + | DeliveryRow + | undefined + } + + private getDeliveryMessages(delivery: DeliveryRow): MessageRow[] { + const ids = JSON.parse(delivery.message_ids) as string[] + if (ids.length === 0) { + return [] + } + const rows = this.db + .prepare(`SELECT * FROM messages WHERE id IN (${ids.map(() => '?').join(',')})`) + .all(...ids) as MessageRow[] + const byId = new Map(rows.map((row) => [row.id, row])) + return exposeMessageListTimestamps( + ids.map((id) => byId.get(id)).filter((row): row is MessageRow => row !== undefined) + ) + } + + getOrCreateRunDelivery(params: { + runId: string + consumerGeneration: number + limit?: number + wakeTypes?: MessageType[] + }): { delivery: DeliveryRow; messages: MessageRow[]; replayed: boolean } | undefined { + const limit = Math.min(Math.max(params.limit ?? 50, 1), 50) + this.db.exec('BEGIN IMMEDIATE') + try { + this.requireCurrentConsumer(params.runId, params.consumerGeneration) + const existing = this.db + .prepare("SELECT * FROM deliveries WHERE run_id = ? AND status = 'outstanding'") + .get(params.runId) as DeliveryRow | undefined + if (existing) { + if (existing.consumer_generation !== params.consumerGeneration) { + throw new OrchestrationError( + 'consumer_fenced', + 'This mailbox Delivery belongs to a fenced consumer generation.' + ) + } + const messages = this.getDeliveryMessages(existing) + this.db.exec('COMMIT') + return { delivery: exposeDeliveryTimestamps(existing), messages, replayed: true } + } + + const address = `run:${params.runId}` + if (params.wakeTypes && params.wakeTypes.length > 0) { + const placeholders = params.wakeTypes.map(() => '?').join(',') + const matching = this.db + .prepare( + `SELECT 1 FROM messages + WHERE run_id = ? AND to_handle = ? AND read = 0 + AND type IN (${placeholders}) LIMIT 1` + ) + .get(params.runId, address, ...params.wakeTypes) + if (!matching) { + this.db.exec('COMMIT') + return undefined + } + } + + const messages = exposeMessageListTimestamps( + this.db + .prepare( + `SELECT * FROM messages + WHERE run_id = ? AND to_handle = ? AND read = 0 + ORDER BY sequence ASC LIMIT ?` + ) + .all(params.runId, address, limit) as MessageRow[] + ) + if (messages.length === 0) { + this.db.exec('COMMIT') + return undefined + } + + const deliveryId = generateId('delivery') + this.db + .prepare( + `INSERT INTO deliveries (id, run_id, consumer_generation, message_ids) + VALUES (?, ?, ?, ?)` + ) + .run( + deliveryId, + params.runId, + params.consumerGeneration, + JSON.stringify(messages.map((message) => message.id)) + ) + const delivery = this.getDeliveryRaw(deliveryId) as DeliveryRow + this.db.exec('COMMIT') + return { delivery: exposeDeliveryTimestamps(delivery), messages, replayed: false } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + } + + acknowledgeRunDelivery(params: { + runId: string + consumerGeneration: number + deliveryId: string + }): { delivery: DeliveryRow; duplicate: boolean } { + this.db.exec('BEGIN IMMEDIATE') + try { + this.requireCurrentConsumer(params.runId, params.consumerGeneration) + const delivery = this.getDeliveryRaw(params.deliveryId) + if (!delivery || delivery.run_id !== params.runId) { + throw new OrchestrationError( + 'stale_delivery', + `Delivery ${params.deliveryId} does not belong to this Run.` + ) + } + if ( + delivery.consumer_generation !== params.consumerGeneration || + delivery.status === 'fenced' + ) { + throw new OrchestrationError( + 'consumer_fenced', + 'This mailbox Delivery belongs to a fenced consumer generation.' + ) + } + if (delivery.status === 'acknowledged') { + this.db.exec('COMMIT') + return { delivery: exposeDeliveryTimestamps(delivery), duplicate: true } + } + + const messageIds = JSON.parse(delivery.message_ids) as string[] + if (messageIds.length > 0) { + const placeholders = messageIds.map(() => '?').join(',') + this.db + .prepare(`UPDATE messages SET read = 1 WHERE id IN (${placeholders})`) + .run(...messageIds) + } + this.db + .prepare( + "UPDATE deliveries SET status = 'acknowledged', acknowledged_at = datetime('now') WHERE id = ?" + ) + .run(delivery.id) + const acknowledged = this.getDeliveryRaw(delivery.id) as DeliveryRow + this.db.exec('COMMIT') + return { delivery: exposeDeliveryTimestamps(acknowledged), duplicate: false } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + } + + getRunMailboxHistory(runId: string, limit = 100, types?: MessageType[]): MessageRow[] { + const address = `run:${runId}` + if (types && types.length > 0) { + const placeholders = types.map(() => '?').join(',') + return exposeMessageListTimestamps( + this.db + .prepare( + `SELECT * FROM messages WHERE run_id = ? AND to_handle = ? + AND type IN (${placeholders}) ORDER BY sequence DESC LIMIT ?` + ) + .all(runId, address, ...types, limit) as MessageRow[] + ) + } + return exposeMessageListTimestamps( + this.db + .prepare( + `SELECT * FROM messages WHERE run_id = ? AND to_handle = ? + ORDER BY sequence DESC LIMIT ?` + ) + .all(runId, address, limit) as MessageRow[] + ) + } + // ── Messages ── insertMessage(msg: { + id?: string from: string to: string subject: string @@ -332,14 +1195,18 @@ export class OrchestrationDb { threadId?: string payload?: string senderPaneKey?: string + runId?: string }): MessageRow { - const id = generateId('msg') + const runId = msg.runId ?? LEGACY_RUN_ID + this.requireRun(runId) + const id = msg.id ?? generateId('msg') const stmt = this.db.prepare(` - INSERT INTO messages (id, from_handle, to_handle, subject, body, type, priority, thread_id, payload, sender_pane_key) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO messages (id, run_id, from_handle, to_handle, subject, body, type, priority, thread_id, payload, sender_pane_key) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `) stmt.run( id, + runId, msg.from, msg.to, msg.subject, @@ -373,7 +1240,11 @@ export class OrchestrationDb { ) } - convertLifecycleMessageToRejection(messageId: string, reason: string): MessageRow | undefined { + convertLifecycleMessageToRejection( + messageId: string, + code: string, + reason: string + ): MessageRow | undefined { const message = this.getMessageById(messageId) if (!message || (message.type !== 'worker_done' && message.type !== 'heartbeat')) { return message @@ -381,7 +1252,7 @@ export class OrchestrationDb { const originalBody = message.body ? `\n\nOriginal body:\n${message.body}` : '' const body = `Orca rejected this ${message.type}: ${reason}${originalBody}` - const payload = addLifecycleRejectionMarker(message.payload, reason) + const payload = addLifecycleRejectionMarker(message.payload, code, reason) // Why: rejected lifecycle signals stay auditable but must not reach read paths as actionable completion/liveness events. this.db .prepare( @@ -508,6 +1379,156 @@ export class OrchestrationDb { ) } + createQuestion(params: { + runId: string + dispatchId: string + askerHandle: string + question: string + options?: string[] + }): { question: QuestionRow; message: MessageRow } { + this.db.exec('BEGIN IMMEDIATE') + try { + this.requireRun(params.runId) + const dispatch = this.getDispatchContextById(params.dispatchId) + if ( + !dispatch || + dispatch.run_id !== params.runId || + (dispatch.status !== 'pending' && dispatch.status !== 'dispatched') + ) { + throw new OrchestrationError( + 'dispatch_inactive', + `Dispatch ${params.dispatchId} is not active in Run ${params.runId}.` + ) + } + const message = this.insertMessage({ + from: `dispatch:${params.dispatchId}`, + to: `run:${params.runId}`, + subject: 'Question', + body: params.question, + type: 'question', + payload: JSON.stringify({ + taskId: dispatch.task_id, + dispatchId: dispatch.id, + question: params.question, + options: params.options ?? [] + }), + runId: params.runId + }) + this.db.prepare('UPDATE messages SET thread_id = ? WHERE id = ?').run(message.id, message.id) + this.db + .prepare( + `INSERT INTO question_threads ( + message_id, run_id, dispatch_id, asker_handle + ) VALUES (?, ?, ?, ?)` + ) + .run(message.id, params.runId, params.dispatchId, params.askerHandle) + const question = this.getQuestionRaw(message.id) as QuestionRow + const storedMessage = this.getMessageById(message.id) as MessageRow + this.db.exec('COMMIT') + return { question: exposeQuestionTimestamps(question), message: storedMessage } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + } + + getQuestion(messageId: string): QuestionRow | undefined { + const question = this.getQuestionRaw(messageId) + return question ? exposeQuestionTimestamps(question) : undefined + } + + private getQuestionRaw(messageId: string): QuestionRow | undefined { + return this.db.prepare('SELECT * FROM question_threads WHERE message_id = ?').get(messageId) as + | QuestionRow + | undefined + } + + answerQuestion(params: { + messageId: string + runId: string + consumerGeneration: number + body: string + }): { question: QuestionRow; message: MessageRow; duplicate: boolean } { + this.db.exec('BEGIN IMMEDIATE') + try { + this.requireCurrentConsumer(params.runId, params.consumerGeneration) + const question = this.getQuestionRaw(params.messageId) + if (!question || question.run_id !== params.runId) { + throw new OrchestrationError( + 'question_not_found', + `Question ${params.messageId} was not found in Run ${params.runId}.` + ) + } + if (question.status === 'closed') { + throw new OrchestrationError( + 'dispatch_inactive', + `Question ${params.messageId} is closed because its Dispatch is inactive.` + ) + } + if (question.status === 'answered') { + if (question.answer_body !== params.body || !question.answer_message_id) { + throw new OrchestrationError( + 'answer_conflict', + `Question ${params.messageId} already has a different answer.` + ) + } + const message = this.getMessageById(question.answer_message_id) + if (!message) { + throw new Error(`Recorded answer message ${question.answer_message_id} was not found.`) + } + this.db.exec('COMMIT') + return { question: exposeQuestionTimestamps(question), message, duplicate: true } + } + + const message = this.insertMessage({ + from: `run:${params.runId}`, + to: `dispatch:${question.dispatch_id}`, + subject: 'Re: Question', + body: params.body, + threadId: question.message_id, + runId: params.runId + }) + // Why: ask returns thread state directly; leaving its answer unread would deliver it again via check. + this.markAsRead([message.id]) + this.db + .prepare( + `UPDATE question_threads + SET status = 'answered', answer_message_id = ?, answer_body = ?, + answered_by_generation = ?, answered_at = datetime('now') + WHERE message_id = ? AND status = 'pending'` + ) + .run(message.id, params.body, params.consumerGeneration, question.message_id) + const answered = this.getQuestionRaw(question.message_id) as QuestionRow + const storedMessage = this.getMessageById(message.id) as MessageRow + this.db.exec('COMMIT') + return { + question: exposeQuestionTimestamps(answered), + message: storedMessage, + duplicate: false + } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + } + + closeQuestionsForDispatch(dispatchId: string): string[] { + const rows = this.db + .prepare( + "SELECT message_id FROM question_threads WHERE dispatch_id = ? AND status = 'pending'" + ) + .all(dispatchId) as { message_id: string }[] + if (rows.length === 0) { + return [] + } + this.db + .prepare( + "UPDATE question_threads SET status = 'closed', closed_at = datetime('now') WHERE dispatch_id = ? AND status = 'pending'" + ) + .run(dispatchId) + return rows.map((row) => row.message_id) + } + // ── Tasks ── createTask(task: { @@ -517,7 +1538,22 @@ export class OrchestrationDb { deps?: string[] parentId?: string createdByTerminalHandle?: string + runId?: string }): TaskRow { + const runId = task.runId ?? LEGACY_RUN_ID + this.requireRun(runId) + if (task.parentId) { + const parent = this.getTask(task.parentId) + if (!parent || parent.run_id !== runId) { + throw new Error(`Parent task ${task.parentId} must belong to run ${runId}`) + } + } + for (const depId of task.deps ?? []) { + const dependency = this.getTask(depId) + if (!dependency || dependency.run_id !== runId) { + throw new Error(`Dependency task ${depId} must belong to run ${runId}`) + } + } const id = generateId('task') const depsJson = JSON.stringify(task.deps ?? []) const hasDeps = (task.deps ?? []).length > 0 @@ -529,10 +1565,11 @@ export class OrchestrationDb { }) this.db .prepare( - 'INSERT INTO tasks (id, parent_id, created_by_terminal_handle, task_title, display_name, spec, status, deps) VALUES (?, ?, ?, ?, ?, ?, ?, ?)' + 'INSERT INTO tasks (id, run_id, parent_id, created_by_terminal_handle, task_title, display_name, spec, status, deps) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)' ) .run( id, + runId, task.parentId ?? null, task.createdByTerminalHandle ?? null, display.taskTitle || null, @@ -548,27 +1585,42 @@ export class OrchestrationDb { return this.db.prepare('SELECT * FROM tasks WHERE id = ?').get(id) as TaskRow | undefined } - listTasks(filter?: { status?: TaskStatus; ready?: boolean }): TaskRow[] { + listTasks(filter?: { status?: TaskStatus; ready?: boolean; runId?: string }): TaskRow[] { + const runWhere = filter?.runId ? 'run_id = ? AND ' : '' + const runParams: Database.BindValue[] = filter?.runId ? [filter.runId] : [] if (filter?.ready) { return this.db - .prepare("SELECT * FROM tasks WHERE status = 'ready' ORDER BY created_at") - .all() as TaskRow[] + .prepare(`SELECT * FROM tasks WHERE ${runWhere}status = 'ready' ORDER BY created_at`) + .all(...runParams) as TaskRow[] } if (filter?.status) { return this.db - .prepare('SELECT * FROM tasks WHERE status = ? ORDER BY created_at') - .all(filter.status) as TaskRow[] + .prepare(`SELECT * FROM tasks WHERE ${runWhere}status = ? ORDER BY created_at`) + .all(...runParams, filter.status) as TaskRow[] + } + if (filter?.runId) { + return this.db + .prepare('SELECT * FROM tasks WHERE run_id = ? ORDER BY created_at') + .all(filter.runId) as TaskRow[] } return this.db.prepare('SELECT * FROM tasks ORDER BY created_at').all() as TaskRow[] } // Why: LEFT JOIN keeps non-dispatched tasks (NULL assignee); the MAX(rowid) subquery matches getDispatchContext's most-recent-active-dispatch semantics. - listTasksWithDispatch(filter?: { status?: TaskStatus; ready?: boolean }): (TaskRow & { + listTasksWithDispatch(filter?: { + status?: TaskStatus + ready?: boolean + runId?: string + }): (TaskRow & { assignee_handle: string | null dispatch_id: string | null })[] { const whereClauses: string[] = [] const params: Database.BindValue[] = [] + if (filter?.runId) { + whereClauses.push('t.run_id = ?') + params.push(filter.runId) + } if (filter?.ready) { whereClauses.push("t.status = 'ready'") } else if (filter?.status) { @@ -642,6 +1694,1498 @@ export class OrchestrationDb { // ── Dispatch Contexts ── + createStartingWorkerDispatch(params: { + taskId: string + startOptions: unknown + retryOf?: string + runtimeEpoch?: string + federation?: { + environmentId: string + environmentName: string + peerFingerprint: string + protocolVersion: number + } + mutationReceipt?: { + callerFingerprint: string + requestId: string + method: string + payloadHash: string + } + }): { dispatch: DispatchContextRow; worker: WorkerDispatchRow } { + this.db.exec('BEGIN IMMEDIATE') + try { + if (params.mutationReceipt) { + const receipt = params.mutationReceipt + const existing = this.getMutationReceipt(receipt.callerFingerprint, receipt.requestId) + if (existing) { + if (existing.method !== receipt.method || existing.payload_hash !== receipt.payloadHash) { + throw new OrchestrationError( + 'request_mismatch', + `Mutation request ${receipt.requestId} was already used with different input.` + ) + } + throw new OrchestrationError( + 'operation_unknown', + `Mutation ${receipt.requestId} already has a durable acceptance record.` + ) + } + this.db + .prepare( + `INSERT INTO mutation_receipts ( + caller_fingerprint, request_id, method, payload_hash, state + ) VALUES (?, ?, ?, ?, 'pending')` + ) + .run(receipt.callerFingerprint, receipt.requestId, receipt.method, receipt.payloadHash) + } + const task = this.getTask(params.taskId) + if (!task) { + throw new OrchestrationError('task_not_found', `Task ${params.taskId} was not found.`) + } + if (params.retryOf) { + const prior = this.getDispatchContextById(params.retryOf) + const priorWorker = this.getWorkerDispatch(params.retryOf) + const latest = this.getDispatchContext(task.id) + if ( + !prior || + prior.task_id !== task.id || + latest?.id !== prior.id || + !priorWorker || + !['failed', 'stopped', 'abandoned'].includes(priorWorker.state) || + !['failed', 'blocked'].includes(task.status) + ) { + throw new OrchestrationError( + 'task_not_startable', + `Task ${task.id} cannot retry from Dispatch ${params.retryOf}.` + ) + } + } else if (task.status !== 'ready') { + throw new OrchestrationError( + 'task_not_startable', + `Task ${task.id} is ${task.status}; only a ready Task can start.` + ) + } + + const id = generateId('ctx') + if (params.mutationReceipt) { + this.db + .prepare( + `UPDATE mutation_receipts + SET receipt = ?, updated_at = datetime('now') + WHERE caller_fingerprint = ? AND request_id = ? AND state = 'pending'` + ) + .run( + JSON.stringify({ accepted: { dispatchId: id } }), + params.mutationReceipt.callerFingerprint, + params.mutationReceipt.requestId + ) + } + this.db + .prepare( + `INSERT INTO dispatch_contexts ( + id, run_id, task_id, status, dispatched_at + ) VALUES (?, ?, ?, 'pending', datetime('now'))` + ) + .run(id, task.run_id, task.id) + this.db + .prepare( + `INSERT INTO worker_dispatches ( + dispatch_id, runtime_epoch, state, stage, start_options + ) VALUES (?, ?, 'starting', 'accepted', ?)` + ) + .run(id, params.runtimeEpoch ?? null, JSON.stringify(params.startOptions)) + if (params.federation) { + this.db + .prepare( + `INSERT INTO federated_dispatches ( + dispatch_id, environment_id, environment_name, peer_fingerprint, protocol_version + ) VALUES (?, ?, ?, ?, ?)` + ) + .run( + id, + params.federation.environmentId, + params.federation.environmentName, + params.federation.peerFingerprint, + params.federation.protocolVersion + ) + } + this.db + .prepare( + "UPDATE tasks SET status = 'dispatched', result = NULL, completed_at = NULL WHERE id = ?" + ) + .run(task.id) + this.db.exec('COMMIT') + return { + dispatch: this.getDispatchContextById(id) as DispatchContextRow, + worker: this.getWorkerDispatch(id) as WorkerDispatchRow + } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + } + + recordWorkerStage(params: { + dispatchId: string + stage: string + worktreeId?: string + terminalHandle?: string + setupState?: string + effects?: unknown[] + residualResources?: unknown[] + lastError?: string + state?: WorkerDispatchState + }): WorkerDispatchRow { + const current = this.getWorkerDispatch(params.dispatchId) + if (!current) { + throw new OrchestrationError( + 'dispatch_not_found', + `Dispatch ${params.dispatchId} was not found.` + ) + } + this.db + .prepare( + `UPDATE worker_dispatches + SET stage = ?, state = ?, worktree_id = ?, agent_terminal_handle = ?, + setup_state = ?, effects = ?, residual_resources = ?, last_error = ?, + updated_at = datetime('now') + WHERE dispatch_id = ?` + ) + .run( + params.stage, + params.state ?? current.state, + params.worktreeId ?? current.worktree_id, + params.terminalHandle ?? current.agent_terminal_handle, + params.setupState ?? current.setup_state, + params.effects ? JSON.stringify(params.effects) : current.effects, + params.residualResources + ? JSON.stringify(params.residualResources) + : current.residual_resources, + params.lastError ?? current.last_error, + params.dispatchId + ) + return this.getWorkerDispatch(params.dispatchId) as WorkerDispatchRow + } + + updateWorkerSetupEvidence(params: { + dispatchId: string + setupState: string + effects: unknown[] + }): { worker: WorkerDispatchRow; changed: boolean } { + const current = this.getWorkerDispatch(params.dispatchId) + if (!current) { + throw new OrchestrationError( + 'dispatch_not_found', + `Dispatch ${params.dispatchId} was not found.` + ) + } + const effects = JSON.stringify(params.effects) + if (current.setup_state === params.setupState && current.effects === effects) { + return { worker: current, changed: false } + } + this.db + .prepare( + `UPDATE worker_dispatches + SET setup_state = ?, effects = ?, updated_at = datetime('now') + WHERE dispatch_id = ?` + ) + .run(params.setupState, effects, params.dispatchId) + return { + worker: this.getWorkerDispatch(params.dispatchId) as WorkerDispatchRow, + changed: true + } + } + + prepareStartingWorkerAuthority(params: { + dispatchId: string + handle: string + paneKey: string + processIncarnation: string + worktreeId: string + effects: unknown[] + setupState: string + }): string { + const dispatch = this.getDispatchContextById(params.dispatchId) + const worker = this.getWorkerDispatch(params.dispatchId) + if (!dispatch || dispatch.status !== 'pending' || worker?.state !== 'starting') { + throw new OrchestrationError( + 'dispatch_inactive', + `Dispatch ${params.dispatchId} is not starting.` + ) + } + const capability = `dcap_${randomBytes(32).toString('base64url')}` + this.db.exec('BEGIN IMMEDIATE') + try { + this.db + .prepare( + `UPDATE dispatch_contexts + SET assignee_handle = ?, assignee_pane_key = ?, process_incarnation = ?, + capability_hash = ?, capability_revoked_at = NULL + WHERE id = ? AND status = 'pending'` + ) + .run( + params.handle, + params.paneKey, + params.processIncarnation, + hashDispatchCapability(capability), + params.dispatchId + ) + this.db + .prepare( + `UPDATE worker_dispatches + SET stage = 'authority_attached', worktree_id = ?, agent_terminal_handle = ?, + setup_state = ?, effects = ?, residual_resources = ?, updated_at = datetime('now') + WHERE dispatch_id = ? AND state = 'starting'` + ) + .run( + params.worktreeId, + params.handle, + params.setupState, + JSON.stringify(params.effects), + JSON.stringify( + params.effects.filter((effect) => + Boolean( + effect && + typeof effect === 'object' && + ((effect as { action?: string }).action?.startsWith('created') || + (effect as { action?: string }).action === 'reused_agent_terminal') + ) + ) + ), + params.dispatchId + ) + this.db.exec('COMMIT') + return capability + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + } + + markWorkerDispatchReady(dispatchId: string, effects?: unknown[]): WorkerDispatchRow { + this.db.exec('BEGIN IMMEDIATE') + try { + const dispatch = this.getDispatchContextById(dispatchId) + const worker = this.getWorkerDispatch(dispatchId) + if (!dispatch || dispatch.status !== 'pending' || worker?.state !== 'starting') { + throw new OrchestrationError('dispatch_inactive', `Dispatch ${dispatchId} is not starting.`) + } + this.db + .prepare("UPDATE dispatch_contexts SET status = 'dispatched' WHERE id = ?") + .run(dispatchId) + this.db + .prepare( + `UPDATE worker_dispatches + SET state = 'ready', stage = 'input_accepted', + effects = COALESCE(?, effects), updated_at = datetime('now') + WHERE dispatch_id = ?` + ) + .run(effects ? JSON.stringify(effects) : null, dispatchId) + this.db.exec('COMMIT') + return this.getWorkerDispatch(dispatchId) as WorkerDispatchRow + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + } + + failWorkerStart(dispatchId: string, stage: string, reason: string): WorkerDispatchRow { + this.db.exec('BEGIN IMMEDIATE') + try { + const dispatch = this.getDispatchContextById(dispatchId) + const worker = this.getWorkerDispatch(dispatchId) + if (!dispatch || !worker || worker.state !== 'starting') { + throw new OrchestrationError('dispatch_inactive', `Dispatch ${dispatchId} is not starting.`) + } + this.db + .prepare( + `UPDATE dispatch_contexts + SET status = 'failed', last_failure = ?, completed_at = datetime('now'), + capability_revoked_at = COALESCE(capability_revoked_at, datetime('now')) + WHERE id = ?` + ) + .run(reason, dispatchId) + this.db + .prepare( + `UPDATE worker_dispatches + SET state = 'failed', stage = ?, last_error = ?, updated_at = datetime('now') + WHERE dispatch_id = ?` + ) + .run(stage, reason, dispatchId) + this.db + .prepare("UPDATE tasks SET status = 'failed', completed_at = datetime('now') WHERE id = ?") + .run(dispatch.task_id) + this.closeQuestionsForDispatch(dispatchId) + this.db.exec('COMMIT') + return this.getWorkerDispatch(dispatchId) as WorkerDispatchRow + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + } + + markWorkerStartUnknown(dispatchId: string, stage: string, reason: string): WorkerDispatchRow { + this.db.exec('BEGIN IMMEDIATE') + try { + const dispatch = this.getDispatchContextById(dispatchId) + const worker = this.getWorkerDispatch(dispatchId) + if (!dispatch || !worker || worker.state !== 'starting') { + throw new OrchestrationError('dispatch_inactive', `Dispatch ${dispatchId} is not starting.`) + } + this.db + .prepare( + `UPDATE worker_dispatches + SET state = 'start_unknown', stage = ?, last_error = ?, updated_at = datetime('now') + WHERE dispatch_id = ?` + ) + .run(stage, reason, dispatchId) + this.db + .prepare( + `UPDATE dispatch_contexts + SET capability_revoked_at = COALESCE(capability_revoked_at, datetime('now')) + WHERE id = ?` + ) + .run(dispatchId) + this.db.prepare("UPDATE tasks SET status = 'blocked' WHERE id = ?").run(dispatch.task_id) + this.closeQuestionsForDispatch(dispatchId) + this.db.exec('COMMIT') + return this.getWorkerDispatch(dispatchId) as WorkerDispatchRow + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + } + + reconcileFederatedWorkerStart(params: { + dispatchId: string + state: 'ready' | 'failed' | 'stopped' | 'start_unknown' + stage: string + lastError?: string | null + worktreeId?: string | null + terminalHandle?: string | null + setupState?: string + effects?: unknown[] + residualResources?: unknown[] + }): WorkerDispatchRow { + this.db.exec('BEGIN IMMEDIATE') + try { + const dispatch = this.getDispatchContextById(params.dispatchId) + const worker = this.getWorkerDispatch(params.dispatchId) + if (!dispatch || !worker) { + throw new OrchestrationError( + 'dispatch_not_found', + `Federated Dispatch ${params.dispatchId} was not found.` + ) + } + if (!['starting', 'start_unknown'].includes(worker.state)) { + this.db.exec('COMMIT') + return worker + } + + if (params.state === 'ready') { + this.db + .prepare( + `UPDATE worker_dispatches + SET state = 'ready', stage = ?, worktree_id = COALESCE(?, worktree_id), + agent_terminal_handle = COALESCE(?, agent_terminal_handle), setup_state = ?, + effects = ?, residual_resources = ?, last_error = NULL, + updated_at = datetime('now') + WHERE dispatch_id = ? AND state IN ('starting', 'start_unknown')` + ) + .run( + params.stage, + params.worktreeId ?? null, + params.terminalHandle ?? null, + params.setupState ?? worker.setup_state, + JSON.stringify(params.effects ?? JSON.parse(worker.effects)), + JSON.stringify(params.residualResources ?? JSON.parse(worker.residual_resources)), + params.dispatchId + ) + this.db + .prepare( + "UPDATE dispatch_contexts SET status = 'dispatched' WHERE id = ? AND status = 'pending'" + ) + .run(params.dispatchId) + this.db + .prepare( + "UPDATE tasks SET status = 'dispatched', completed_at = NULL WHERE id = ? AND status = 'blocked'" + ) + .run(dispatch.task_id) + } else if (params.state === 'start_unknown') { + this.db + .prepare( + `UPDATE worker_dispatches + SET stage = ?, last_error = ?, updated_at = datetime('now') + WHERE dispatch_id = ? AND state IN ('starting', 'start_unknown')` + ) + .run(params.stage, params.lastError ?? worker.last_error, params.dispatchId) + } else { + const reason = params.lastError ?? `The worker server reported ${params.state}.` + this.db + .prepare( + `UPDATE worker_dispatches + SET state = ?, stage = ?, last_error = ?, updated_at = datetime('now') + WHERE dispatch_id = ? AND state IN ('starting', 'start_unknown')` + ) + .run(params.state, params.stage, reason, params.dispatchId) + this.db + .prepare( + `UPDATE dispatch_contexts + SET status = 'failed', last_failure = ?, completed_at = datetime('now'), + capability_revoked_at = COALESCE(capability_revoked_at, datetime('now')) + WHERE id = ? AND status IN ('pending', 'dispatched')` + ) + .run(reason, params.dispatchId) + this.db + .prepare( + "UPDATE tasks SET status = 'failed', completed_at = datetime('now') WHERE id = ? AND status IN ('blocked', 'dispatched')" + ) + .run(dispatch.task_id) + this.closeQuestionsForDispatch(params.dispatchId) + } + this.db.exec('COMMIT') + return this.getWorkerDispatch(params.dispatchId) as WorkerDispatchRow + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + } + + getWorkerDispatch(dispatchId: string): WorkerDispatchRow | undefined { + return this.db + .prepare('SELECT * FROM worker_dispatches WHERE dispatch_id = ?') + .get(dispatchId) as WorkerDispatchRow | undefined + } + + getFederatedDispatch(dispatchId: string): FederatedDispatchRow | undefined { + return this.db + .prepare('SELECT * FROM federated_dispatches WHERE dispatch_id = ?') + .get(dispatchId) as FederatedDispatchRow | undefined + } + + listActiveFederatedDispatches(runId?: string): FederatedDispatchRow[] { + return this.db + .prepare( + `SELECT fd.* + FROM federated_dispatches fd + INNER JOIN dispatch_contexts dc ON dc.id = fd.dispatch_id + INNER JOIN worker_dispatches wd ON wd.dispatch_id = fd.dispatch_id + WHERE wd.state IN ('starting', 'ready', 'stopping', 'start_unknown', 'stop_unknown') + AND (? IS NULL OR dc.run_id = ?) + ORDER BY fd.rowid` + ) + .all(runId ?? null, runId ?? null) as FederatedDispatchRow[] + } + + updateFederatedDispatchResources(params: { + dispatchId: string + remoteRuntimeEpoch: string + worktreeId: string + terminalHandle: string + }): FederatedDispatchRow { + this.db + .prepare( + `UPDATE federated_dispatches + SET remote_runtime_epoch = ?, remote_worktree_id = ?, remote_terminal_handle = ?, + updated_at = datetime('now') + WHERE dispatch_id = ?` + ) + .run(params.remoteRuntimeEpoch, params.worktreeId, params.terminalHandle, params.dispatchId) + const row = this.getFederatedDispatch(params.dispatchId) + if (!row) { + throw new OrchestrationError( + 'dispatch_not_found', + `Federated Dispatch ${params.dispatchId} was not found.` + ) + } + return row + } + + createRemoteDispatchAttachment(params: { + dispatchId: string + taskId: string + homePeerFingerprint: string + protocolVersion: number + runtimeEpoch: string + mutationReceipt: { + callerFingerprint: string + requestId: string + method: string + payloadHash: string + } + }): RemoteDispatchAttachmentRow { + this.db.exec('BEGIN IMMEDIATE') + try { + if (params.homePeerFingerprint !== params.mutationReceipt.callerFingerprint) { + throw new OrchestrationError( + 'resource_server_mismatch', + 'The authenticated Run-home peer does not match the attachment request.' + ) + } + const existingReceipt = this.getMutationReceipt( + params.mutationReceipt.callerFingerprint, + params.mutationReceipt.requestId + ) + if (existingReceipt) { + throw new OrchestrationError( + existingReceipt.method === params.mutationReceipt.method && + existingReceipt.payload_hash === params.mutationReceipt.payloadHash + ? 'operation_unknown' + : 'request_mismatch', + `Remote attachment request ${params.mutationReceipt.requestId} already exists.` + ) + } + this.db + .prepare( + `INSERT INTO mutation_receipts ( + caller_fingerprint, request_id, method, payload_hash, state, receipt + ) VALUES (?, ?, ?, ?, 'pending', ?)` + ) + .run( + params.mutationReceipt.callerFingerprint, + params.mutationReceipt.requestId, + params.mutationReceipt.method, + params.mutationReceipt.payloadHash, + JSON.stringify({ accepted: { dispatchId: params.dispatchId } }) + ) + this.db + .prepare( + `INSERT INTO remote_dispatch_attachments ( + dispatch_id, task_id, home_peer_fingerprint, protocol_version, runtime_epoch + ) VALUES (?, ?, ?, ?, ?)` + ) + .run( + params.dispatchId, + params.taskId, + params.homePeerFingerprint, + params.protocolVersion, + params.runtimeEpoch + ) + this.db.exec('COMMIT') + return this.getRemoteDispatchAttachment(params.dispatchId) as RemoteDispatchAttachmentRow + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + } + + getRemoteDispatchAttachment(dispatchId: string): RemoteDispatchAttachmentRow | undefined { + return this.db + .prepare('SELECT * FROM remote_dispatch_attachments WHERE dispatch_id = ?') + .get(dispatchId) as RemoteDispatchAttachmentRow | undefined + } + + recordRemoteAttachmentStage(params: { + dispatchId: string + stage: string + state?: WorkerDispatchState + worktreeId?: string + terminalHandle?: string + setupState?: string + effects?: unknown[] + residualResources?: unknown[] + lastError?: string + }): RemoteDispatchAttachmentRow { + const current = this.getRemoteDispatchAttachment(params.dispatchId) + if (!current) { + throw new OrchestrationError( + 'dispatch_not_found', + `Remote Dispatch ${params.dispatchId} was not found.` + ) + } + this.db + .prepare( + `UPDATE remote_dispatch_attachments + SET stage = ?, state = ?, worktree_id = ?, terminal_handle = ?, setup_state = ?, + effects = ?, residual_resources = ?, last_error = ?, updated_at = datetime('now') + WHERE dispatch_id = ?` + ) + .run( + params.stage, + params.state ?? current.state, + params.worktreeId ?? current.worktree_id, + params.terminalHandle ?? current.terminal_handle, + params.setupState ?? current.setup_state, + params.effects ? JSON.stringify(params.effects) : current.effects, + params.residualResources + ? JSON.stringify(params.residualResources) + : current.residual_resources, + params.lastError ?? current.last_error, + params.dispatchId + ) + return this.getRemoteDispatchAttachment(params.dispatchId) as RemoteDispatchAttachmentRow + } + + updateRemoteAttachmentSetupEvidence(params: { + dispatchId: string + setupState: string + effects: unknown[] + }): { attachment: RemoteDispatchAttachmentRow; changed: boolean } { + const current = this.getRemoteDispatchAttachment(params.dispatchId) + if (!current) { + throw new OrchestrationError( + 'dispatch_not_found', + `Remote Dispatch ${params.dispatchId} was not found.` + ) + } + const effects = JSON.stringify(params.effects) + if (current.setup_state === params.setupState && current.effects === effects) { + return { attachment: current, changed: false } + } + this.db + .prepare( + `UPDATE remote_dispatch_attachments + SET setup_state = ?, effects = ?, updated_at = datetime('now') + WHERE dispatch_id = ?` + ) + .run(params.setupState, effects, params.dispatchId) + return { + attachment: this.getRemoteDispatchAttachment( + params.dispatchId + ) as RemoteDispatchAttachmentRow, + changed: true + } + } + + prepareRemoteAttachmentAuthority(params: { + dispatchId: string + paneKey: string + processIncarnation: string + worktreeId: string + terminalHandle: string + setupState: string + effects: unknown[] + }): string { + const attachment = this.getRemoteDispatchAttachment(params.dispatchId) + if (!attachment || attachment.state !== 'starting') { + throw new OrchestrationError( + 'dispatch_inactive', + `Remote Dispatch ${params.dispatchId} is not starting.` + ) + } + const capability = `dcap_${randomBytes(32).toString('base64url')}` + this.db + .prepare( + `UPDATE remote_dispatch_attachments + SET stage = 'authority_attached', capability_hash = ?, pane_key = ?, + process_incarnation = ?, worktree_id = ?, terminal_handle = ?, setup_state = ?, + effects = ?, residual_resources = ?, updated_at = datetime('now') + WHERE dispatch_id = ? AND state = 'starting'` + ) + .run( + hashDispatchCapability(capability), + params.paneKey, + params.processIncarnation, + params.worktreeId, + params.terminalHandle, + params.setupState, + JSON.stringify(params.effects), + JSON.stringify( + params.effects.filter((effect) => + Boolean( + effect && + typeof effect === 'object' && + ((effect as { action?: string }).action?.startsWith('created') || + (effect as { action?: string }).action === 'reused_agent_terminal') + ) + ) + ), + params.dispatchId + ) + return capability + } + + markRemoteAttachmentReady(dispatchId: string, effects?: unknown[]): RemoteDispatchAttachmentRow { + const result = this.db + .prepare( + `UPDATE remote_dispatch_attachments + SET state = 'ready', stage = 'input_accepted', + effects = COALESCE(?, effects), updated_at = datetime('now') + WHERE dispatch_id = ? AND state = 'starting'` + ) + .run(effects ? JSON.stringify(effects) : null, dispatchId) + if (result.changes !== 1) { + throw new OrchestrationError( + 'dispatch_inactive', + `Remote Dispatch ${dispatchId} is not starting.` + ) + } + return this.getRemoteDispatchAttachment(dispatchId) as RemoteDispatchAttachmentRow + } + + failRemoteAttachment( + dispatchId: string, + stage: string, + reason: string, + unknown: boolean + ): RemoteDispatchAttachmentRow { + const state = unknown ? 'start_unknown' : 'failed' + const result = this.db + .prepare( + `UPDATE remote_dispatch_attachments + SET state = ?, stage = ?, last_error = ?, capability_hash = NULL, + updated_at = datetime('now') + WHERE dispatch_id = ? AND state = 'starting'` + ) + .run(state, stage, reason, dispatchId) + if (result.changes !== 1) { + throw new OrchestrationError( + 'dispatch_inactive', + `Remote Dispatch ${dispatchId} is not starting.` + ) + } + return this.getRemoteDispatchAttachment(dispatchId) as RemoteDispatchAttachmentRow + } + + verifyRemoteAttachmentAuthority(params: { + dispatchId: string + capability: string | undefined + paneKey: string | null + processIncarnation: string | null + }): boolean { + const attachment = this.getRemoteDispatchAttachment(params.dispatchId) + if ( + !attachment?.capability_hash || + !params.capability || + !attachment.pane_key || + !params.paneKey || + !isEquivalentPaneKey(attachment.pane_key, params.paneKey) || + !attachment.process_incarnation || + attachment.process_incarnation !== params.processIncarnation + ) { + return false + } + const expected = Buffer.from(attachment.capability_hash, 'hex') + const observed = Buffer.from(hashDispatchCapability(params.capability), 'hex') + return expected.length === observed.length && timingSafeEqual(expected, observed) + } + + isRemoteAttachmentProcessCurrent(params: { + dispatchId: string + paneKey: string | null + processIncarnation: string | null + }): boolean { + const attachment = this.getRemoteDispatchAttachment(params.dispatchId) + return Boolean( + attachment?.pane_key && + params.paneKey && + isEquivalentPaneKey(attachment.pane_key, params.paneKey) && + attachment.process_incarnation && + attachment.process_incarnation === params.processIncarnation + ) + } + + beginRemoteAttachmentStop(dispatchId: string): RemoteDispatchAttachmentRow { + const attachment = this.getRemoteDispatchAttachment(dispatchId) + if (!attachment) { + throw new OrchestrationError( + 'dispatch_not_found', + `Remote Dispatch ${dispatchId} was not found.` + ) + } + if (['succeeded', 'failed', 'stopped', 'abandoned'].includes(attachment.state)) { + return attachment + } + if (!['ready', 'start_unknown'].includes(attachment.state)) { + throw new OrchestrationError( + 'dispatch_inactive', + `Remote Dispatch ${dispatchId} cannot stop from ${attachment.state}.` + ) + } + this.db + .prepare( + `UPDATE remote_dispatch_attachments + SET state = 'stopping', stage = 'stop_requested', capability_hash = NULL, + updated_at = datetime('now') + WHERE dispatch_id = ? AND state IN ('ready', 'start_unknown')` + ) + .run(dispatchId) + return this.getRemoteDispatchAttachment(dispatchId) as RemoteDispatchAttachmentRow + } + + settleRemoteAttachmentStop(dispatchId: string): RemoteDispatchAttachmentRow { + this.db + .prepare( + `UPDATE remote_dispatch_attachments + SET state = 'stopped', stage = 'process_stopped', updated_at = datetime('now') + WHERE dispatch_id = ? AND state = 'stopping'` + ) + .run(dispatchId) + return this.getRemoteDispatchAttachment(dispatchId) as RemoteDispatchAttachmentRow + } + + markRemoteAttachmentStopUnknown(dispatchId: string, reason: string): RemoteDispatchAttachmentRow { + this.db + .prepare( + `UPDATE remote_dispatch_attachments + SET state = 'stop_unknown', stage = 'stop_outcome_unknown', last_error = ?, + updated_at = datetime('now') + WHERE dispatch_id = ? AND state = 'stopping'` + ) + .run(reason, dispatchId) + return this.getRemoteDispatchAttachment(dispatchId) as RemoteDispatchAttachmentRow + } + + findActiveRemoteAttachmentForPane(paneKey: string): RemoteDispatchAttachmentRow | undefined { + const rows = this.db + .prepare( + `SELECT * FROM remote_dispatch_attachments + WHERE state IN ('starting', 'ready') AND pane_key IS NOT NULL + ORDER BY rowid DESC` + ) + .all() as RemoteDispatchAttachmentRow[] + return rows.find((row) => row.pane_key && isEquivalentPaneKey(row.pane_key, paneKey)) + } + + enqueueFederationRelay(params: { + dispatchId: string + direction: FederationRelayDirection + kind: string + payload: string + messageId?: string + settleRemoteOutcome?: WorkerReportOutcome + remoteQuestion?: true + }): FederationRelayItemRow { + const byteCount = Buffer.byteLength(params.payload, 'utf8') + const messageId = params.messageId ?? generateId('relay') + if (byteCount > 64 * 1024) { + throw new OrchestrationError( + 'relay_quota_exceeded', + 'A federated orchestration message cannot exceed 64 KiB.' + ) + } + this.db.exec('BEGIN IMMEDIATE') + try { + if (params.settleRemoteOutcome) { + const attachment = this.getRemoteDispatchAttachment(params.dispatchId) + if (!attachment || attachment.state !== 'ready') { + throw new OrchestrationError( + 'dispatch_inactive', + `Remote Dispatch ${params.dispatchId} is not active.` + ) + } + } + if (params.kind === 'heartbeat') { + const heartbeat = this.db + .prepare( + `SELECT * FROM federation_relay_items + WHERE dispatch_id = ? AND direction = ? AND kind = 'heartbeat' + AND acked_at IS NULL + ORDER BY sequence DESC LIMIT 1` + ) + .get(params.dispatchId, params.direction) as FederationRelayItemRow | undefined + if (heartbeat) { + this.db + .prepare( + `UPDATE federation_relay_items + SET payload = ?, byte_count = ?, created_at = datetime('now') + WHERE dispatch_id = ? AND direction = ? AND sequence = ?` + ) + .run(params.payload, byteCount, params.dispatchId, params.direction, heartbeat.sequence) + this.db.exec('COMMIT') + return this.getFederationRelayItem( + params.dispatchId, + params.direction, + heartbeat.sequence + ) as FederationRelayItemRow + } + } + const quota = this.db + .prepare( + `SELECT COUNT(*) AS count, COALESCE(SUM(byte_count), 0) AS bytes + FROM federation_relay_items + WHERE dispatch_id = ? AND direction = ? AND acked_at IS NULL` + ) + .get(params.dispatchId, params.direction) as { count: number; bytes: number } + if (quota.count >= 256 || quota.bytes + byteCount > 1024 * 1024) { + if (params.kind === 'worker_done') { + const heartbeat = this.db + .prepare( + `SELECT * FROM federation_relay_items + WHERE dispatch_id = ? AND direction = ? AND kind = 'heartbeat' + AND acked_at IS NULL + ORDER BY sequence LIMIT 1` + ) + .get(params.dispatchId, params.direction) as FederationRelayItemRow | undefined + if (heartbeat) { + this.db + .prepare( + `UPDATE federation_relay_items + SET message_id = ?, kind = ?, payload = ?, byte_count = ?, + created_at = datetime('now') + WHERE dispatch_id = ? AND direction = ? AND sequence = ?` + ) + .run( + messageId, + params.kind, + params.payload, + byteCount, + params.dispatchId, + params.direction, + heartbeat.sequence + ) + this.settleRemoteAttachmentInRelayTransaction( + params.dispatchId, + params.settleRemoteOutcome + ) + this.db.exec('COMMIT') + return this.getFederationRelayItem( + params.dispatchId, + params.direction, + heartbeat.sequence + ) as FederationRelayItemRow + } + } + throw new OrchestrationError( + 'relay_quota_exceeded', + `Federated Dispatch ${params.dispatchId} has no relay capacity.` + ) + } + const latest = this.db + .prepare( + `SELECT COALESCE(MAX(sequence), 0) AS sequence + FROM federation_relay_items WHERE dispatch_id = ? AND direction = ?` + ) + .get(params.dispatchId, params.direction) as { sequence: number } + const sequence = latest.sequence + 1 + this.db + .prepare( + `INSERT INTO federation_relay_items ( + dispatch_id, direction, sequence, message_id, kind, payload, byte_count + ) VALUES (?, ?, ?, ?, ?, ?, ?)` + ) + .run( + params.dispatchId, + params.direction, + sequence, + messageId, + params.kind, + params.payload, + byteCount + ) + if (params.remoteQuestion) { + this.db + .prepare( + `INSERT INTO remote_questions (message_id, dispatch_id) + VALUES (?, ?)` + ) + .run(messageId, params.dispatchId) + } + this.settleRemoteAttachmentInRelayTransaction(params.dispatchId, params.settleRemoteOutcome) + this.db.exec('COMMIT') + return this.getFederationRelayItem( + params.dispatchId, + params.direction, + sequence + ) as FederationRelayItemRow + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + } + + listFederationRelay(params: { + dispatchId: string + direction: FederationRelayDirection + afterSequence: number + limit?: number + }): FederationRelayItemRow[] { + return this.db + .prepare( + `SELECT * FROM federation_relay_items + WHERE dispatch_id = ? AND direction = ? AND sequence > ? + ORDER BY sequence LIMIT ?` + ) + .all( + params.dispatchId, + params.direction, + params.afterSequence, + Math.min(Math.max(params.limit ?? 50, 1), 50) + ) as FederationRelayItemRow[] + } + + listPendingFederationRelay( + dispatchId: string, + direction: FederationRelayDirection, + limit = 50 + ): FederationRelayItemRow[] { + return this.db + .prepare( + `SELECT * FROM federation_relay_items + WHERE dispatch_id = ? AND direction = ? AND acked_at IS NULL + ORDER BY sequence LIMIT ?` + ) + .all(dispatchId, direction, Math.min(Math.max(limit, 1), 50)) as FederationRelayItemRow[] + } + + acknowledgeFederationRelay(params: { + dispatchId: string + direction: FederationRelayDirection + throughSequence: number + }): void { + this.db + .prepare( + `UPDATE federation_relay_items SET acked_at = COALESCE(acked_at, datetime('now')) + WHERE dispatch_id = ? AND direction = ? AND sequence <= ?` + ) + .run(params.dispatchId, params.direction, params.throughSequence) + } + + setFederatedHomeImportSequence(dispatchId: string, sequence: number): void { + this.db + .prepare( + `UPDATE federated_dispatches + SET to_home_imported_sequence = ?, updated_at = datetime('now') + WHERE dispatch_id = ? AND to_home_imported_sequence < ?` + ) + .run(sequence, dispatchId, sequence) + } + + importFederatedRelayItem(params: { + dispatchId: string + sequence: number + message: { + id: string + runId: string + from: string + to: string + subject: string + body: string + type: MessageType + priority: MessagePriority + threadId?: string + payload?: string + } + lifecycle: + | { kind: 'none' } + | { kind: 'heartbeat'; at: string } + | { + kind: 'worker_report' + taskId: string + outcome: WorkerReportOutcome + result: string + } + | { kind: 'rejected'; code: string; reason: string } + }): { message: MessageRow; duplicate: boolean } { + this.db.exec('BEGIN IMMEDIATE') + try { + const federated = this.getFederatedDispatch(params.dispatchId) + if (!federated) { + throw new OrchestrationError( + 'dispatch_not_found', + `Federated Dispatch ${params.dispatchId} was not found.` + ) + } + if (params.sequence <= federated.to_home_imported_sequence) { + const existing = this.getMessageById(params.message.id) + if (!existing) { + throw new OrchestrationError( + 'operation_unknown', + `Federated relay sequence ${params.sequence} was committed without its message.` + ) + } + this.db.exec('COMMIT') + return { message: existing, duplicate: true } + } + if (params.sequence !== federated.to_home_imported_sequence + 1) { + throw new OrchestrationError( + 'operation_unknown', + `Federated relay for ${params.dispatchId} is not contiguous after sequence ${federated.to_home_imported_sequence}.` + ) + } + + let message = this.getMessageById(params.message.id) + if (!message) { + message = this.insertMessage(params.message) + } else if ( + message.run_id !== params.message.runId || + message.to_handle !== params.message.to || + message.type !== params.message.type + ) { + throw new OrchestrationError( + 'request_mismatch', + `Federated relay message ${params.message.id} conflicts with an existing message.` + ) + } + if (message.type === 'question') { + this.registerFederatedQuestion({ + messageId: message.id, + runId: params.message.runId, + dispatchId: params.dispatchId + }) + } + if (params.lifecycle.kind === 'heartbeat') { + this.recordHeartbeat(params.dispatchId, params.lifecycle.at) + } else if (params.lifecycle.kind === 'worker_report') { + const settlement = this.settleWorkerReportInTransaction({ + taskId: params.lifecycle.taskId, + dispatchId: params.dispatchId, + outcome: params.lifecycle.outcome, + result: params.lifecycle.result + }) + if (settlement.action === 'rejected') { + message = this.convertLifecycleMessageToRejection( + message.id, + settlement.code, + settlement.reason + ) as MessageRow + } + } else if (params.lifecycle.kind === 'rejected') { + message = this.convertLifecycleMessageToRejection( + message.id, + params.lifecycle.code, + params.lifecycle.reason + ) as MessageRow + } + this.setFederatedHomeImportSequence(params.dispatchId, params.sequence) + this.db.exec('COMMIT') + return { message, duplicate: false } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + } + + getRemoteQuestion(messageId: string): + | { + message_id: string + dispatch_id: string + status: 'pending' | 'answered' | 'closed' + answer_message_id: string | null + answer_body: string | null + } + | undefined { + return this.db.prepare('SELECT * FROM remote_questions WHERE message_id = ?').get(messageId) as + | { + message_id: string + dispatch_id: string + status: 'pending' | 'answered' | 'closed' + answer_message_id: string | null + answer_body: string | null + } + | undefined + } + + answerRemoteQuestion(params: { + messageId: string + dispatchId: string + answerMessageId: string + body: string + }): void { + const question = this.getRemoteQuestion(params.messageId) + if (!question || question.dispatch_id !== params.dispatchId) { + throw new OrchestrationError( + 'question_not_found', + `Remote Question ${params.messageId} was not found.` + ) + } + if (question.status === 'answered') { + if ( + question.answer_message_id !== params.answerMessageId || + question.answer_body !== params.body + ) { + throw new OrchestrationError( + 'answer_conflict', + `Remote Question ${params.messageId} already has a different answer.` + ) + } + return + } + this.db + .prepare( + `UPDATE remote_questions + SET status = 'answered', answer_message_id = ?, answer_body = ?, + answered_at = datetime('now') + WHERE message_id = ? AND status = 'pending'` + ) + .run(params.answerMessageId, params.body, params.messageId) + } + + setRemoteWorkerImportSequence(dispatchId: string, sequence: number): void { + this.db + .prepare( + `UPDATE remote_dispatch_attachments + SET to_worker_imported_sequence = ?, updated_at = datetime('now') + WHERE dispatch_id = ? AND to_worker_imported_sequence < ?` + ) + .run(sequence, dispatchId, sequence) + } + + registerFederatedQuestion(params: { + messageId: string + runId: string + dispatchId: string + }): void { + this.db + .prepare( + `INSERT OR IGNORE INTO question_threads ( + message_id, run_id, dispatch_id, asker_handle + ) VALUES (?, ?, ?, ?)` + ) + .run(params.messageId, params.runId, params.dispatchId, `dispatch:${params.dispatchId}`) + } + + private getFederationRelayItem( + dispatchId: string, + direction: FederationRelayDirection, + sequence: number + ): FederationRelayItemRow | undefined { + return this.db + .prepare( + `SELECT * FROM federation_relay_items + WHERE dispatch_id = ? AND direction = ? AND sequence = ?` + ) + .get(dispatchId, direction, sequence) as FederationRelayItemRow | undefined + } + + private settleRemoteAttachmentInRelayTransaction( + dispatchId: string, + outcome: WorkerReportOutcome | undefined + ): void { + if (!outcome) { + return + } + this.db + .prepare( + `UPDATE remote_dispatch_attachments + SET state = ?, stage = 'worker_report_queued', capability_hash = NULL, + updated_at = datetime('now') + WHERE dispatch_id = ? AND state = 'ready'` + ) + .run(outcome === 'succeeded' ? 'succeeded' : 'failed', dispatchId) + } + + isDispatchProcessCurrent(params: { + dispatchId: string + paneKey: string | null + processIncarnation: string | null + }): boolean { + const dispatch = this.getDispatchContextById(params.dispatchId) + return Boolean( + dispatch?.assignee_pane_key && + params.paneKey && + isEquivalentPaneKey(dispatch.assignee_pane_key, params.paneKey) && + dispatch.process_incarnation && + params.processIncarnation === dispatch.process_incarnation + ) + } + + beginWorkerStop( + dispatchId: string + ): + | { disposition: 'stopping'; worker: WorkerDispatchRow; dispatch: DispatchContextRow } + | { disposition: 'already_settled'; worker: WorkerDispatchRow; dispatch: DispatchContextRow } { + this.db.exec('BEGIN IMMEDIATE') + try { + const dispatch = this.getDispatchContextById(dispatchId) + const worker = this.getWorkerDispatch(dispatchId) + if (!dispatch || !worker) { + throw new OrchestrationError('dispatch_not_found', `Dispatch ${dispatchId} was not found.`) + } + if (['succeeded', 'failed', 'stopped', 'abandoned'].includes(worker.state)) { + this.db.exec('COMMIT') + return { disposition: 'already_settled', worker, dispatch } + } + if (!['ready', 'start_unknown'].includes(worker.state)) { + throw new OrchestrationError( + 'dispatch_inactive', + `Dispatch ${dispatchId} cannot stop from ${worker.state}.` + ) + } + this.db + .prepare( + `UPDATE worker_dispatches + SET state = 'stopping', stage = 'stop_requested', updated_at = datetime('now') + WHERE dispatch_id = ? AND state IN ('ready', 'start_unknown')` + ) + .run(dispatchId) + this.db + .prepare( + `UPDATE dispatch_contexts + SET capability_revoked_at = COALESCE(capability_revoked_at, datetime('now')) + WHERE id = ?` + ) + .run(dispatchId) + this.db.prepare("UPDATE tasks SET status = 'blocked' WHERE id = ?").run(dispatch.task_id) + this.closeQuestionsForDispatch(dispatchId) + this.db.exec('COMMIT') + return { + disposition: 'stopping', + worker: this.getWorkerDispatch(dispatchId) as WorkerDispatchRow, + dispatch: this.getDispatchContextById(dispatchId) as DispatchContextRow + } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + } + + settleWorkerStop(dispatchId: string): WorkerDispatchRow { + this.db.exec('BEGIN IMMEDIATE') + try { + const worker = this.getWorkerDispatch(dispatchId) + const dispatch = this.getDispatchContextById(dispatchId) + if (!worker || !dispatch || worker.state !== 'stopping') { + throw new OrchestrationError('dispatch_inactive', `Dispatch ${dispatchId} is not stopping.`) + } + this.db + .prepare( + `UPDATE worker_dispatches + SET state = 'stopped', stage = 'process_stopped', updated_at = datetime('now') + WHERE dispatch_id = ? AND state = 'stopping'` + ) + .run(dispatchId) + this.db + .prepare( + `UPDATE dispatch_contexts + SET status = 'failed', completed_at = datetime('now'), last_failure = 'stopped' + WHERE id = ? AND status IN ('pending', 'dispatched')` + ) + .run(dispatchId) + this.db.exec('COMMIT') + return this.getWorkerDispatch(dispatchId) as WorkerDispatchRow + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + } + + reconcileFederatedWorkerStop(dispatchId: string): WorkerDispatchRow { + this.db.exec('BEGIN IMMEDIATE') + try { + const worker = this.getWorkerDispatch(dispatchId) + const dispatch = this.getDispatchContextById(dispatchId) + if (!worker || !dispatch || !this.getFederatedDispatch(dispatchId)) { + throw new OrchestrationError( + 'dispatch_not_found', + `Federated Dispatch ${dispatchId} was not found.` + ) + } + if (worker.state === 'stopped') { + this.db.exec('COMMIT') + return worker + } + if (!['stopping', 'stop_unknown'].includes(worker.state)) { + throw new OrchestrationError( + 'dispatch_inactive', + `Federated Dispatch ${dispatchId} cannot reconcile stop from ${worker.state}.` + ) + } + this.db + .prepare( + `UPDATE worker_dispatches + SET state = 'stopped', stage = 'process_stopped', last_error = NULL, + updated_at = datetime('now') + WHERE dispatch_id = ? AND state IN ('stopping', 'stop_unknown')` + ) + .run(dispatchId) + this.db + .prepare( + `UPDATE dispatch_contexts + SET status = 'failed', completed_at = COALESCE(completed_at, datetime('now')), + last_failure = 'stopped' + WHERE id = ? AND status IN ('pending', 'dispatched')` + ) + .run(dispatchId) + this.db.exec('COMMIT') + return this.getWorkerDispatch(dispatchId) as WorkerDispatchRow + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + } + + resumeFederatedWorkerForTerminalRelay(dispatchId: string): WorkerDispatchRow { + this.db.exec('BEGIN IMMEDIATE') + try { + const worker = this.getWorkerDispatch(dispatchId) + const dispatch = this.getDispatchContextById(dispatchId) + if (!worker || !dispatch || worker.state !== 'stopping') { + throw new OrchestrationError('dispatch_inactive', `Dispatch ${dispatchId} is not stopping.`) + } + this.db + .prepare( + `UPDATE worker_dispatches + SET state = 'ready', stage = 'remote_report_pending', updated_at = datetime('now') + WHERE dispatch_id = ? AND state = 'stopping'` + ) + .run(dispatchId) + this.db + .prepare("UPDATE tasks SET status = 'dispatched' WHERE id = ? AND status = 'blocked'") + .run(dispatch.task_id) + this.db.exec('COMMIT') + return this.getWorkerDispatch(dispatchId) as WorkerDispatchRow + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + } + + markWorkerStopUnknown(dispatchId: string, reason: string): WorkerDispatchRow { + this.db + .prepare( + `UPDATE worker_dispatches + SET state = 'stop_unknown', stage = 'stop_outcome_unknown', last_error = ?, + updated_at = datetime('now') + WHERE dispatch_id = ? AND state = 'stopping'` + ) + .run(reason, dispatchId) + return this.getWorkerDispatch(dispatchId) as WorkerDispatchRow + } + + abandonWorkerDispatch(dispatchId: string): { + disposition: 'abandoned' | 'already_abandoned' | 'stale' + worker: WorkerDispatchRow + } { + this.db.exec('BEGIN IMMEDIATE') + try { + const worker = this.getWorkerDispatch(dispatchId) + const dispatch = this.getDispatchContextById(dispatchId) + if (!worker || !dispatch) { + throw new OrchestrationError('dispatch_not_found', `Dispatch ${dispatchId} was not found.`) + } + if (worker.state === 'abandoned') { + this.db.exec('COMMIT') + return { disposition: 'already_abandoned', worker } + } + if (this.getDispatchContext(dispatch.task_id)?.id !== dispatchId) { + this.db.exec('COMMIT') + return { disposition: 'stale', worker } + } + if (worker.state === 'succeeded') { + throw new OrchestrationError( + 'dispatch_inactive', + `Dispatch ${dispatchId} already succeeded and cannot be abandoned.` + ) + } + this.db + .prepare( + `UPDATE worker_dispatches + SET state = 'abandoned', stage = 'abandoned', updated_at = datetime('now') + WHERE dispatch_id = ?` + ) + .run(dispatchId) + this.db + .prepare( + `UPDATE dispatch_contexts + SET status = CASE WHEN status IN ('pending', 'dispatched') THEN 'failed' ELSE status END, + capability_revoked_at = COALESCE(capability_revoked_at, datetime('now')), + completed_at = COALESCE(completed_at, datetime('now')) + WHERE id = ?` + ) + .run(dispatchId) + this.db.prepare("UPDATE tasks SET status = 'blocked' WHERE id = ?").run(dispatch.task_id) + this.closeQuestionsForDispatch(dispatchId) + this.db.exec('COMMIT') + return { + disposition: 'abandoned', + worker: this.getWorkerDispatch(dispatchId) as WorkerDispatchRow + } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + } + createDispatchContext( taskId: string, assigneeHandle: string, @@ -674,10 +3218,10 @@ export class OrchestrationDb { const id = generateId('ctx') this.db .prepare( - `INSERT INTO dispatch_contexts (id, task_id, assignee_handle, assignee_pane_key, status, failure_count, dispatched_at) - VALUES (?, ?, ?, ?, 'dispatched', ?, datetime('now'))` + `INSERT INTO dispatch_contexts (id, run_id, task_id, assignee_handle, assignee_pane_key, status, failure_count, dispatched_at) + VALUES (?, ?, ?, ?, ?, 'dispatched', ?, datetime('now'))` ) - .run(id, taskId, assigneeHandle, assigneePaneKey ?? null, priorFailures) + .run(id, task.run_id, taskId, assigneeHandle, assigneePaneKey ?? null, priorFailures) this.hasAnyDispatchContextsCache = true this.db.prepare("UPDATE tasks SET status = 'dispatched' WHERE id = ?").run(taskId) @@ -699,6 +3243,86 @@ export class OrchestrationDb { | undefined } + mintDispatchCapability(params: { + dispatchId: string + paneKey: string + processIncarnation: string + }): string { + const dispatch = this.getDispatchContextById(params.dispatchId) + if (!dispatch || (dispatch.status !== 'pending' && dispatch.status !== 'dispatched')) { + throw new OrchestrationError( + 'dispatch_inactive', + `Dispatch ${params.dispatchId} is not active.` + ) + } + const capability = `dcap_${randomBytes(32).toString('base64url')}` + this.db + .prepare( + `UPDATE dispatch_contexts + SET capability_hash = ?, assignee_pane_key = ?, process_incarnation = ?, + capability_revoked_at = NULL + WHERE id = ?` + ) + .run( + hashDispatchCapability(capability), + params.paneKey, + params.processIncarnation, + params.dispatchId + ) + return capability + } + + verifyDispatchCapability(params: { + dispatchId: string + capability: string | undefined + paneKey: string | undefined + processIncarnation: string | undefined + }): { valid: true } | { valid: false; reason: string } { + const dispatch = this.getDispatchContextById(params.dispatchId) + if (!dispatch) { + return { valid: false, reason: `Dispatch ${params.dispatchId} was not found.` } + } + if (!dispatch.capability_hash) { + return { valid: false, reason: `Dispatch ${params.dispatchId} has no lifecycle capability.` } + } + if (dispatch.capability_revoked_at) { + return { valid: false, reason: `Dispatch ${params.dispatchId} capability is revoked.` } + } + if (!params.capability) { + return { valid: false, reason: 'The Dispatch capability is missing.' } + } + const expected = Buffer.from(dispatch.capability_hash, 'hex') + const observed = Buffer.from(hashDispatchCapability(params.capability), 'hex') + if (expected.length !== observed.length || !timingSafeEqual(expected, observed)) { + return { valid: false, reason: 'The Dispatch capability is invalid.' } + } + if ( + !dispatch.assignee_pane_key || + !params.paneKey || + !isEquivalentPaneKey(dispatch.assignee_pane_key, params.paneKey) + ) { + return { valid: false, reason: 'The caller is not the Dispatch pane.' } + } + if ( + !dispatch.process_incarnation || + !params.processIncarnation || + dispatch.process_incarnation !== params.processIncarnation + ) { + return { valid: false, reason: 'The Dispatch process incarnation changed.' } + } + return { valid: true } + } + + revokeDispatchCapability(dispatchId: string): void { + this.db + .prepare( + `UPDATE dispatch_contexts + SET capability_revoked_at = COALESCE(capability_revoked_at, datetime('now')) + WHERE id = ?` + ) + .run(dispatchId) + } + getActiveDispatchForTerminal(handle: string): DispatchContextRow | undefined { return this.findActiveDispatchForAssignee(handle) } @@ -717,6 +3341,10 @@ export class OrchestrationDb { return this.hasAnyDispatchContextsCache } + getActiveDispatchForIdentity(handle: string, paneKey?: string): DispatchContextRow | undefined { + return this.findActiveDispatchForAssignee(handle, paneKey) + } + private findActiveDispatchForAssignee( assigneeHandle: string, assigneePaneKey?: string @@ -759,7 +3387,7 @@ export class OrchestrationDb { completeDispatch(ctxId: string): void { this.db .prepare( - "UPDATE dispatch_contexts SET status = 'completed', completed_at = datetime('now') WHERE id = ?" + "UPDATE dispatch_contexts SET status = 'completed', completed_at = datetime('now'), capability_revoked_at = COALESCE(capability_revoked_at, datetime('now')) WHERE id = ?" ) .run(ctxId) } @@ -775,6 +3403,111 @@ export class OrchestrationDb { } } + settleWorkerReport(params: { + taskId: string + dispatchId: string + outcome: WorkerReportOutcome + result: string + }): WorkerReportSettlement { + this.db.exec('BEGIN IMMEDIATE') + try { + const settlement = this.settleWorkerReportInTransaction(params) + this.db.exec('COMMIT') + return settlement + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + } + + private settleWorkerReportInTransaction(params: { + taskId: string + dispatchId: string + outcome: WorkerReportOutcome + result: string + }): WorkerReportSettlement { + const task = this.getTask(params.taskId) + if (!task) { + return { action: 'rejected', code: 'unknown_task', reason: `Unknown task ${params.taskId}.` } + } + const dispatch = this.getDispatchContextById(params.dispatchId) + if (!dispatch) { + return { + action: 'rejected', + code: 'unknown_dispatch', + reason: `Unknown dispatch ${params.dispatchId}.` + } + } + if (dispatch.task_id !== params.taskId) { + return { + action: 'rejected', + code: 'task_dispatch_mismatch', + reason: `Dispatch ${params.dispatchId} belongs to task ${dispatch.task_id}, not ${params.taskId}.` + } + } + + const expectedDispatchStatus = params.outcome === 'succeeded' ? 'completed' : 'failed' + const expectedTaskStatus = params.outcome === 'succeeded' ? 'completed' : 'failed' + if (dispatch.status === expectedDispatchStatus && task.status === expectedTaskStatus) { + return { action: 'settled', outcome: params.outcome, duplicate: true } + } + if (dispatch.status !== 'dispatched' || task.status !== 'dispatched') { + return { + action: 'rejected', + code: 'inactive_dispatch', + reason: `inactive dispatch ${params.dispatchId}: it or task ${params.taskId} is already settled.` + } + } + const latest = this.getDispatchContext(params.taskId) + if (latest?.id !== params.dispatchId) { + return { + action: 'rejected', + code: 'stale_dispatch', + reason: `Dispatch ${params.dispatchId} is not the current dispatch for task ${params.taskId}.` + } + } + + this.db.exec('SAVEPOINT settle_worker_report') + const dispatchUpdate = this.db + .prepare( + `UPDATE dispatch_contexts + SET status = ?, completed_at = datetime('now'), + last_failure = CASE WHEN ? = 'failed' THEN ? ELSE last_failure END, + capability_revoked_at = COALESCE(capability_revoked_at, datetime('now')) + WHERE id = ? AND status = 'dispatched'` + ) + .run(expectedDispatchStatus, expectedDispatchStatus, params.result, params.dispatchId) + const taskUpdate = this.db + .prepare( + `UPDATE tasks + SET status = ?, result = ?, completed_at = datetime('now') + WHERE id = ? AND status = 'dispatched'` + ) + .run(expectedTaskStatus, params.result, params.taskId) + if (dispatchUpdate.changes !== 1 || taskUpdate.changes !== 1) { + this.db.exec('ROLLBACK TO settle_worker_report') + this.db.exec('RELEASE settle_worker_report') + return { + action: 'rejected', + code: 'inactive_dispatch', + reason: `Dispatch ${params.dispatchId} changed while its worker report was settling.` + } + } + this.db + .prepare( + `UPDATE worker_dispatches + SET state = ?, stage = 'settled', updated_at = datetime('now') + WHERE dispatch_id = ? AND state = 'ready'` + ) + .run(params.outcome === 'succeeded' ? 'succeeded' : 'failed', params.dispatchId) + this.closeQuestionsForDispatch(params.dispatchId) + if (params.outcome === 'succeeded') { + this.promoteReadyTasks(params.taskId) + } + this.db.exec('RELEASE settle_worker_report') + return { action: 'settled', outcome: params.outcome, duplicate: false } + } + failActiveDispatchForTask(taskId: string, error: string): DispatchContextRow | undefined { const active = this.db .prepare( @@ -819,7 +3552,10 @@ export class OrchestrationDb { this.db .prepare( - 'UPDATE dispatch_contexts SET status = ?, failure_count = ?, last_failure = ? WHERE id = ?' + `UPDATE dispatch_contexts + SET status = ?, failure_count = ?, last_failure = ?, + capability_revoked_at = COALESCE(capability_revoked_at, datetime('now')) + WHERE id = ?` ) .run(newStatus, newFailureCount, error, ctxId) @@ -838,8 +3574,16 @@ export class OrchestrationDb { const id = generateId('gate') const optionsJson = JSON.stringify(gate.options ?? []) this.db - .prepare('INSERT INTO decision_gates (id, task_id, question, options) VALUES (?, ?, ?, ?)') - .run(id, gate.taskId, gate.question, optionsJson) + .prepare( + 'INSERT INTO decision_gates (id, run_id, task_id, question, options) VALUES (?, ?, ?, ?, ?)' + ) + .run( + id, + this.getTask(gate.taskId)?.run_id ?? LEGACY_RUN_ID, + gate.taskId, + gate.question, + optionsJson + ) this.completeActiveDispatchForTask(gate.taskId) this.db.prepare("UPDATE tasks SET status = 'blocked' WHERE id = ?").run(gate.taskId) @@ -973,25 +3717,62 @@ export class OrchestrationDb { // ── Lifecycle ── + private runResetTransaction(statements: string): void { + this.db.exec('BEGIN IMMEDIATE') + try { + this.db.exec(statements) + this.db.exec('COMMIT') + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + } + resetAll(): void { - this.db.exec('DELETE FROM coordinator_runs') - this.db.exec('DELETE FROM decision_gates') - this.db.exec('DELETE FROM dispatch_contexts') - this.db.exec('DELETE FROM tasks') - this.db.exec('DELETE FROM messages') + // Why: retain mutation receipts so a lost reset response cannot replay as a new mutation. + this.runResetTransaction(` + DELETE FROM coordinator_runs; + DELETE FROM decision_gates; + DELETE FROM remote_questions; + DELETE FROM question_threads; + DELETE FROM deliveries; + DELETE FROM federation_relay_items; + DELETE FROM remote_dispatch_attachments; + DELETE FROM federated_dispatches; + DELETE FROM worker_dispatches; + DELETE FROM dispatch_contexts; + DELETE FROM tasks; + DELETE FROM messages; + DELETE FROM runs; + INSERT INTO runs (id, objective, home_database, consumer_generation, legacy) + VALUES ('${LEGACY_RUN_ID}', 'Legacy orchestration state (inspect only)', 'this_database', 0, 1); + `) this.hasAnyDispatchContextsCache = undefined } resetTasks(): void { - this.db.exec('DELETE FROM coordinator_runs') - this.db.exec('DELETE FROM decision_gates') - this.db.exec('DELETE FROM dispatch_contexts') - this.db.exec('DELETE FROM tasks') + this.runResetTransaction(` + DELETE FROM coordinator_runs; + DELETE FROM decision_gates; + DELETE FROM remote_questions; + DELETE FROM question_threads; + DELETE FROM federation_relay_items; + DELETE FROM remote_dispatch_attachments; + DELETE FROM federated_dispatches; + DELETE FROM worker_dispatches; + DELETE FROM dispatch_contexts; + DELETE FROM tasks; + `) this.hasAnyDispatchContextsCache = undefined } resetMessages(): void { - this.db.exec('DELETE FROM messages') + // Why: relay rows carry contiguous cross-server cursors, not just inbox history. + this.runResetTransaction(` + DELETE FROM question_threads; + DELETE FROM deliveries; + DELETE FROM messages; + `) } close(): void { diff --git a/src/main/runtime/orchestration/environment-transport.ts b/src/main/runtime/orchestration/environment-transport.ts new file mode 100644 index 00000000000..cb52c30d392 --- /dev/null +++ b/src/main/runtime/orchestration/environment-transport.ts @@ -0,0 +1,26 @@ +import { createHash } from 'node:crypto' +import type { + RuntimeOrchestrationEnvelope, + RuntimeRpcResponse +} from '../../../shared/runtime-rpc-envelope' + +export type OrchestrationWorkerServer = { + environmentId: string + name: string + peerFingerprint: string +} + +export type OrchestrationEnvironmentTransport = { + resolve(selector: string): OrchestrationWorkerServer + call( + selector: string, + method: string, + params: unknown, + timeoutMs?: number, + envelope?: RuntimeOrchestrationEnvelope + ): Promise> +} + +export function fingerprintOrchestrationPeer(publicKeyB64: string): string { + return createHash('sha256').update(Buffer.from(publicKeyB64, 'base64')).digest('base64url') +} diff --git a/src/main/runtime/orchestration/federation-control-message.ts b/src/main/runtime/orchestration/federation-control-message.ts new file mode 100644 index 00000000000..bcab04f99b8 --- /dev/null +++ b/src/main/runtime/orchestration/federation-control-message.ts @@ -0,0 +1,94 @@ +import { MESSAGE_TYPES, type MessagePriority, type MessageType } from './types' +import type { OrchestrationDb } from './db' +import { OrchestrationError } from './orchestration-error' + +const MESSAGE_TYPE_SET = new Set(MESSAGE_TYPES) + +export type FederatedControlMessage = { + from: string + subject: string + body: string + type: MessageType + priority: MessagePriority + threadId: string | null + payload: string | null +} + +export function encodeFederatedControlMessage(message: FederatedControlMessage): string { + return JSON.stringify(message) +} + +export function parseFederatedControlMessage(payload: string): FederatedControlMessage { + let parsed: unknown + try { + parsed = JSON.parse(payload) + } catch { + throw new OrchestrationError('invalid_argument', 'Federated control message is invalid JSON.') + } + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new OrchestrationError('invalid_argument', 'Federated control message is invalid.') + } + const message = parsed as Partial + if ( + typeof message.from !== 'string' || + typeof message.subject !== 'string' || + typeof message.body !== 'string' || + typeof message.type !== 'string' || + !MESSAGE_TYPE_SET.has(message.type as MessageType) + ) { + throw new OrchestrationError('invalid_argument', 'Federated control message is incomplete.') + } + return { + from: message.from, + subject: message.subject, + body: message.body, + type: message.type as MessageType, + priority: + message.priority === 'high' || message.priority === 'urgent' ? message.priority : 'normal', + threadId: typeof message.threadId === 'string' ? message.threadId : null, + payload: typeof message.payload === 'string' ? message.payload : null + } +} + +export function importFederatedControlMessage( + db: OrchestrationDb, + params: { + dispatchId: string + messageId: string + payload: string + } +): { imported: boolean; type: MessageType } { + const message = parseFederatedControlMessage(params.payload) + const recipient = `dispatch:${params.dispatchId}` + const existing = db.getMessageById(params.messageId) + if (existing) { + if ( + existing.to_handle !== recipient || + existing.from_handle !== message.from || + existing.subject !== message.subject || + existing.body !== message.body || + existing.type !== message.type || + existing.priority !== message.priority || + existing.thread_id !== message.threadId || + existing.payload !== message.payload + ) { + throw new OrchestrationError( + 'request_mismatch', + `Federated control message ${params.messageId} conflicts with an existing message.` + ) + } + return { imported: false, type: message.type } + } + db.insertMessage({ + id: params.messageId, + from: message.from, + to: recipient, + subject: message.subject, + body: message.body, + type: message.type, + priority: message.priority, + threadId: message.threadId ?? undefined, + payload: message.payload ?? undefined + }) + return { imported: true, type: message.type } +} diff --git a/src/main/runtime/orchestration/federation-sync.test.ts b/src/main/runtime/orchestration/federation-sync.test.ts new file mode 100644 index 00000000000..339b1a55f27 --- /dev/null +++ b/src/main/runtime/orchestration/federation-sync.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from 'vitest' +import { parseRelayedMessage } from './federation-sync' + +describe('federation relay parsing', () => { + it('accepts a supported message type', () => { + expect( + parseRelayedMessage( + JSON.stringify({ subject: 'done', body: 'Finished', type: 'worker_done' }) + ) + ).toMatchObject({ type: 'worker_done', priority: 'normal' }) + }) + + it('rejects an unsupported type before it reaches the database constraint', () => { + expect(() => + parseRelayedMessage(JSON.stringify({ subject: 'bad', body: 'Blocked', type: 'invented' })) + ).toThrowError('Federated relay message type invented is not supported.') + }) +}) diff --git a/src/main/runtime/orchestration/federation-sync.ts b/src/main/runtime/orchestration/federation-sync.ts new file mode 100644 index 00000000000..85856883aa9 --- /dev/null +++ b/src/main/runtime/orchestration/federation-sync.ts @@ -0,0 +1,253 @@ +import { + MESSAGE_TYPES, + type MessagePriority, + type MessageType, + type WorkerReportOutcome +} from './types' +import type { OrcaRuntimeService } from '../orca-runtime' +import { OrchestrationError } from './orchestration-error' + +const MESSAGE_TYPE_SET = new Set(MESSAGE_TYPES) + +function isMessageType(value: unknown): value is MessageType { + return typeof value === 'string' && MESSAGE_TYPE_SET.has(value as MessageType) +} + +type PulledRelayItem = { + dispatch_id: string + direction: 'to_home' + sequence: number + message_id: string + kind: string + payload: string +} + +type RelayedMessage = { + from: string + subject: string + body: string + type: MessageType + priority: MessagePriority + threadId: string | null + payload: string | null +} + +export async function syncFederatedDispatch( + runtime: OrcaRuntimeService, + dispatchId: string +): Promise<{ imported: number; acknowledgedThrough: number }> { + const db = runtime.getOrchestrationDb() + const federated = db.getFederatedDispatch(dispatchId) + const dispatch = db.getDispatchContextById(dispatchId) + if (!federated || !dispatch) { + throw new OrchestrationError( + 'dispatch_not_found', + `Federated Dispatch ${dispatchId} was not found.` + ) + } + const currentServer = runtime.resolveOrchestrationWorkerServer(federated.environment_id) + if (currentServer.peerFingerprint !== federated.peer_fingerprint) { + throw new OrchestrationError( + 'peer_changed', + `Saved environment ${federated.environment_name} now identifies a different Orca server.` + ) + } + + const pulled = (await runtime.callOrchestrationWorkerServer( + federated.environment_id, + 'orchestration.federationPull', + { + dispatchId, + afterSequence: federated.to_home_imported_sequence, + limit: 50 + }, + 15_000 + )) as { runtimeEpoch: string; items: PulledRelayItem[] } + let cursor = federated.to_home_imported_sequence + let imported = 0 + for (const item of pulled.items) { + if (item.dispatch_id !== dispatchId || item.sequence !== cursor + 1) { + throw new OrchestrationError( + 'operation_unknown', + `Federated relay for ${dispatchId} is not contiguous after sequence ${cursor}.` + ) + } + const message = parseRelayedMessage(item.payload) + const stored = db.importFederatedRelayItem({ + dispatchId, + sequence: item.sequence, + message: { + id: item.message_id, + runId: dispatch.run_id, + from: `dispatch:${dispatchId}`, + to: `run:${dispatch.run_id}`, + subject: message.subject, + body: message.body, + type: message.type, + priority: message.priority, + threadId: message.threadId ?? undefined, + payload: message.payload ?? undefined + }, + lifecycle: parseFederatedLifecycle(message, item.message_id, dispatchId, dispatch.task_id) + }) + cursor = item.sequence + runtime.notifyMessageArrived(stored.message.to_handle, stored.message.type) + imported += stored.duplicate ? 0 : 1 + } + + if (cursor > 0) { + await runtime.callOrchestrationWorkerServer( + federated.environment_id, + 'orchestration.federationAck', + { dispatchId, throughSequence: cursor }, + 15_000, + { orchestrationRequestId: `relay_ack_${dispatchId}_${cursor}` } + ) + } + const toWorker = + db.getWorkerDispatch(dispatchId)?.state === 'ready' + ? db.listPendingFederationRelay(dispatchId, 'to_worker') + : [] + if (toWorker.length > 0) { + const delivered = (await runtime.callOrchestrationWorkerServer( + federated.environment_id, + 'orchestration.federationImport', + { dispatchId, items: toWorker }, + 15_000, + { + orchestrationRequestId: `relay_import_${dispatchId}_${toWorker.at(-1)?.sequence ?? 0}` + } + )) as { acknowledgedThrough: number } + db.acknowledgeFederationRelay({ + dispatchId, + direction: 'to_worker', + throughSequence: delivered.acknowledgedThrough + }) + } + return { imported, acknowledgedThrough: cursor } +} + +export function parseRelayedMessage(payload: string): RelayedMessage { + let parsed: unknown + try { + parsed = JSON.parse(payload) + } catch { + throw new OrchestrationError('invalid_argument', 'Federated relay payload is invalid JSON.') + } + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new OrchestrationError('invalid_argument', 'Federated relay payload is not a message.') + } + const message = parsed as Partial + if (typeof message.subject !== 'string' || typeof message.body !== 'string') { + throw new OrchestrationError('invalid_argument', 'Federated relay message is incomplete.') + } + if (!isMessageType(message.type)) { + throw new OrchestrationError( + 'invalid_argument', + `Federated relay message type ${String(message.type)} is not supported.` + ) + } + return { + from: typeof message.from === 'string' ? message.from : 'remote-worker', + subject: message.subject, + body: message.body, + type: message.type, + priority: + message.priority === 'high' || message.priority === 'urgent' ? message.priority : 'normal', + threadId: typeof message.threadId === 'string' ? message.threadId : null, + payload: typeof message.payload === 'string' ? message.payload : null + } +} + +function parseFederatedLifecycle( + message: RelayedMessage, + messageId: string, + dispatchId: string, + taskId: string +): + | { kind: 'none' } + | { kind: 'heartbeat'; at: string } + | { + kind: 'worker_report' + taskId: string + outcome: WorkerReportOutcome + result: string + } + | { kind: 'rejected'; code: string; reason: string } { + if (message.type === 'heartbeat') { + return { kind: 'heartbeat', at: new Date().toISOString() } + } + if (message.type !== 'worker_done') { + return { kind: 'none' } + } + let payload + try { + payload = parseWorkerReportPayload(message.payload) + } catch (error) { + return { + kind: 'rejected', + code: 'invalid_payload', + reason: error instanceof Error ? error.message : String(error) + } + } + if (payload.dispatchId !== dispatchId || payload.taskId !== taskId) { + return { + kind: 'rejected', + code: 'task_dispatch_mismatch', + reason: `Federated report does not match Dispatch ${dispatchId}.` + } + } + const result = JSON.stringify({ + provenance: 'worker_report', + outcome: payload.outcome, + messageId, + reportedBy: `dispatch:${dispatchId}`, + subject: message.subject, + body: message.body, + completedBy: `dispatch:${dispatchId}`, + filesModified: payload.filesModified, + reportPath: payload.reportPath, + completedAt: new Date().toISOString() + }) + return { + kind: 'worker_report', + taskId: payload.taskId, + outcome: payload.outcome, + result + } +} + +function parseWorkerReportPayload(payload: string | null): { + taskId: string + dispatchId: string + outcome: WorkerReportOutcome + filesModified: string[] + reportPath: string | null +} { + let parsed: unknown + try { + parsed = payload ? JSON.parse(payload) : null + } catch { + parsed = null + } + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new OrchestrationError('invalid_argument', 'Federated worker report is invalid.') + } + const report = parsed as Record + if ( + typeof report.taskId !== 'string' || + typeof report.dispatchId !== 'string' || + (report.outcome !== 'succeeded' && report.outcome !== 'failed') + ) { + throw new OrchestrationError('invalid_argument', 'Federated worker report is incomplete.') + } + return { + taskId: report.taskId, + dispatchId: report.dispatchId, + outcome: report.outcome, + filesModified: Array.isArray(report.filesModified) + ? report.filesModified.filter((file): file is string => typeof file === 'string') + : [], + reportPath: typeof report.reportPath === 'string' ? report.reportPath : null + } +} diff --git a/src/main/runtime/orchestration/formatter.test.ts b/src/main/runtime/orchestration/formatter.test.ts index ea8ae94bc19..de71c21ded5 100644 --- a/src/main/runtime/orchestration/formatter.test.ts +++ b/src/main/runtime/orchestration/formatter.test.ts @@ -5,6 +5,7 @@ import type { MessageRow } from './types' function makeMessage(overrides: Partial = {}): MessageRow { return { id: 'msg_test1', + run_id: 'run_test', from_handle: 'term_abc123', to_handle: 'term_coord', subject: 'Auth API implementation complete', diff --git a/src/main/runtime/orchestration/lifecycle-reconciliation.test.ts b/src/main/runtime/orchestration/lifecycle-reconciliation.test.ts index 8fed046d421..3a916751533 100644 --- a/src/main/runtime/orchestration/lifecycle-reconciliation.test.ts +++ b/src/main/runtime/orchestration/lifecycle-reconciliation.test.ts @@ -17,7 +17,7 @@ describe('lifecycle reconciliation', () => { to: 'term_coordinator', subject: 'Done', type: 'worker_done', - payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id }) + payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id, outcome: 'succeeded' }) }) expect(reconcileLifecycleMessage(db, message, (line) => logs.push(line))).toMatchObject({ @@ -43,7 +43,7 @@ describe('lifecycle reconciliation', () => { to: 'term_coordinator', subject: 'Done', type: 'worker_done', - payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id }), + payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id, outcome: 'succeeded' }), senderPaneKey: `tab_w:${LEAF_A}` }) @@ -51,6 +51,94 @@ describe('lifecycle reconciliation', () => { expect(db.getTask(task.id)?.status).toBe('completed') }) + it('fails both the dispatch and task from an authenticated failed worker report', () => { + db = new OrchestrationDb(':memory:') + const task = db.createTask({ spec: 'work' }) + const dispatch = db.createDispatchContext(task.id, 'term_worker', `tab_w:${LEAF_A}`) + const message = db.insertMessage({ + from: 'term_worker', + to: 'term_coordinator', + subject: 'Failed: tests cannot start', + body: 'I attempted the work. The required service is unavailable. No files changed.', + type: 'worker_done', + payload: JSON.stringify({ + taskId: task.id, + dispatchId: dispatch.id, + outcome: 'failed', + filesModified: [] + }), + senderPaneKey: `tab_w:${LEAF_A}` + }) + + expect(reconcileLifecycleMessage(db, message)).toEqual({ + action: 'failed', + taskId: task.id, + dispatchId: dispatch.id + }) + expect(db.getTask(task.id)).toMatchObject({ status: 'failed' }) + expect(db.getDispatchContextById(dispatch.id)).toMatchObject({ status: 'failed' }) + expect(JSON.parse(db.getTask(task.id)?.result ?? '{}')).toMatchObject({ + provenance: 'worker_report', + outcome: 'failed', + messageId: message.id + }) + }) + + it('replays an identical terminal outcome without mutating settled state', () => { + db = new OrchestrationDb(':memory:') + const task = db.createTask({ spec: 'work' }) + const dispatch = db.createDispatchContext(task.id, 'term_worker') + const makeMessage = () => + db.insertMessage({ + from: 'term_worker', + to: 'term_coordinator', + subject: 'Done', + type: 'worker_done', + payload: JSON.stringify({ + taskId: task.id, + dispatchId: dispatch.id, + outcome: 'succeeded' + }) + }) + + expect(reconcileLifecycleMessage(db, makeMessage()).action).toBe('completed') + const result = db.getTask(task.id)?.result + expect(reconcileLifecycleMessage(db, makeMessage()).action).toBe('completed') + expect(db.getTask(task.id)?.result).toBe(result) + }) + + it.each([ + { payload: undefined, code: 'invalid_payload' }, + { payload: '{', code: 'invalid_payload' }, + { + payload: JSON.stringify({ dispatchId: 'ctx_1', outcome: 'succeeded' }), + code: 'missing_task_id' + }, + { + payload: JSON.stringify({ taskId: 'task_1', outcome: 'succeeded' }), + code: 'missing_dispatch_id' + }, + { + payload: JSON.stringify({ taskId: 'task_1', dispatchId: 'ctx_1', outcome: 'maybe' }), + code: 'invalid_outcome' + } + ])('rejects malformed worker reports with $code', ({ payload, code }) => { + db = new OrchestrationDb(':memory:') + const message = db.insertMessage({ + from: 'term_worker', + to: 'term_coordinator', + subject: 'Done', + type: 'worker_done', + payload + }) + + expect(reconcileLifecycleMessage(db, message)).toMatchObject({ action: 'rejected', code }) + expect(db.getMessageById(message.id)).toMatchObject({ + priority: 'high', + subject: 'Rejected worker_done: Done' + }) + }) + it('completes worker_done from the same leaf after a pane break-out changed the tab half', () => { db = new OrchestrationDb(':memory:') const task = db.createTask({ spec: 'work' }) @@ -62,7 +150,7 @@ describe('lifecycle reconciliation', () => { to: 'term_coordinator', subject: 'Done', type: 'worker_done', - payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id }), + payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id, outcome: 'succeeded' }), senderPaneKey: `tab_old:${LEAF_A}` }) @@ -79,7 +167,7 @@ describe('lifecycle reconciliation', () => { to: 'term_coordinator', subject: 'Done', type: 'worker_done', - payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id }), + payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id, outcome: 'succeeded' }), senderPaneKey: 'tab_w:42' }) @@ -96,7 +184,7 @@ describe('lifecycle reconciliation', () => { to: 'term_coordinator', subject: 'Done', type: 'worker_done', - payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id }), + payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id, outcome: 'succeeded' }), senderPaneKey: `tab_w2:${LEAF_B}` }) @@ -142,6 +230,7 @@ describe('lifecycle reconciliation', () => { payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id, + outcome: 'succeeded', _orcaLifecycleRejection: { code: 'sender_not_assignee', reason: 'caller supplied' @@ -167,7 +256,7 @@ describe('lifecycle reconciliation', () => { to: 'term_coordinator', subject: 'Done', type: 'worker_done', - payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id }) + payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id, outcome: 'succeeded' }) }) expect(reconcileLifecycleMessage(db, message)).toMatchObject({ @@ -186,7 +275,11 @@ describe('lifecycle reconciliation', () => { to: 'term_coordinator', subject: 'Done', type: 'worker_done', - payload: JSON.stringify({ taskId: acceptedTask.id, dispatchId: acceptedDispatch.id }) + payload: JSON.stringify({ + taskId: acceptedTask.id, + dispatchId: acceptedDispatch.id, + outcome: 'succeeded' + }) }) expect(reconcileLifecycleMessage(db, accepted).action).toBe('completed') @@ -197,7 +290,11 @@ describe('lifecycle reconciliation', () => { to: 'term_coordinator', subject: 'Done', type: 'worker_done', - payload: JSON.stringify({ taskId: rejectedTask.id, dispatchId: rejectedDispatch.id }) + payload: JSON.stringify({ + taskId: rejectedTask.id, + dispatchId: rejectedDispatch.id, + outcome: 'succeeded' + }) }) expect(reconcileLifecycleMessage(db, rejected)).toMatchObject({ action: 'rejected', @@ -211,7 +308,11 @@ describe('lifecycle reconciliation', () => { const parent = db.createTask({ spec: 'parent' }) const child = db.createTask({ spec: 'child', deps: [parent.id] }) const dispatch = db.createDispatchContext(parent.id, 'term_worker', `tab_w:${LEAF_A}`) - const payload = JSON.stringify({ taskId: parent.id, dispatchId: dispatch.id }) + const payload = JSON.stringify({ + taskId: parent.id, + dispatchId: dispatch.id, + outcome: 'succeeded' + }) const foreign = db.insertMessage({ from: 'term_coordinator', @@ -243,7 +344,11 @@ describe('lifecycle reconciliation', () => { db = new OrchestrationDb(':memory:') const task = db.createTask({ spec: 'work' }) const dispatch = db.createDispatchContext(task.id, 'term_worker', `tab_w:${LEAF_A}`) - const payload = JSON.stringify({ taskId: task.id, dispatchId: dispatch.id }) + const payload = JSON.stringify({ + taskId: task.id, + dispatchId: dispatch.id, + outcome: 'succeeded' + }) const owner = db.insertMessage({ from: 'term_worker', to: 'term_coordinator', @@ -280,7 +385,7 @@ describe('lifecycle reconciliation', () => { to: 'term_coordinator', subject: 'Done', type: 'worker_done', - payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id }), + payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id, outcome: 'succeeded' }), senderPaneKey: `tab_w2:${LEAF_B}` }) @@ -390,7 +495,7 @@ describe('lifecycle reconciliation', () => { to: 'term_coordinator', subject: 'Done', type: 'worker_done', - payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id }) + payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id, outcome: 'succeeded' }) }) reconcileLifecycleMessage(db, done) diff --git a/src/main/runtime/orchestration/lifecycle-reconciliation.ts b/src/main/runtime/orchestration/lifecycle-reconciliation.ts index 96917babaf6..3d2793413b7 100644 --- a/src/main/runtime/orchestration/lifecycle-reconciliation.ts +++ b/src/main/runtime/orchestration/lifecycle-reconciliation.ts @@ -1,5 +1,5 @@ import type { OrchestrationDb } from './db' -import type { MessageRow } from './types' +import type { MessageRow, WorkerReportOutcome } from './types' import { parsePaneKey } from '../../../shared/stable-pane-id' // Why: the tab half can change on pane break-out, while opaque legacy keys @@ -35,11 +35,25 @@ export type LifecycleReconciliationResult = | { action: 'suppressed' } | LifecycleRejectionResult | { action: 'completed'; taskId: string; dispatchId: string } + | { action: 'failed'; taskId: string; dispatchId: string } | { action: 'heartbeat_recorded'; dispatchId: string } +export type LifecycleRejectionCode = + | 'sender_not_assignee' + | 'dispatch_capability_invalid' + | 'invalid_payload' + | 'missing_task_id' + | 'missing_dispatch_id' + | 'invalid_outcome' + | 'unknown_task' + | 'unknown_dispatch' + | 'task_dispatch_mismatch' + | 'inactive_dispatch' + | 'stale_dispatch' + export type LifecycleRejectionResult = { action: 'rejected' - code: 'sender_not_assignee' + code: LifecycleRejectionCode reason: string } @@ -54,7 +68,11 @@ function parseObjectPayload(msg: MessageRow, onInvalidJson: () => void): Record< try { const parsed: unknown = JSON.parse(msg.payload) - return parsed && typeof parsed === 'object' ? (parsed as Record) : {} + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { + return parsed as Record + } + onInvalidJson() + return {} } catch { onInvalidJson() return {} @@ -68,7 +86,7 @@ function getPersistedLifecycleRejection( if ( !rejection || typeof rejection !== 'object' || - (rejection as { code?: unknown }).code !== 'sender_not_assignee' || + typeof (rejection as { code?: unknown }).code !== 'string' || typeof (rejection as { reason?: unknown }).reason !== 'string' ) { return undefined @@ -77,7 +95,7 @@ function getPersistedLifecycleRejection( // also prevents caller-supplied markers from turning lifecycle sends into success. return { action: 'rejected', - code: 'sender_not_assignee', + code: (rejection as { code: LifecycleRejectionCode }).code, reason: (rejection as { reason: string }).reason } } @@ -98,6 +116,7 @@ export function reconcileLifecycleMessage( case 'escalation': case 'handoff': case 'decision_gate': + case 'question': return { action: 'ignored' } } } @@ -142,7 +161,7 @@ function reconcileHeartbeatMessage( // a hung assignee behind another agent's timer. const reason = buildLifecycleAuthorityRejectionReason(dispatchId, dispatch, msg) onLog(`Heartbeat rejected: ${reason}`) - db.convertLifecycleMessageToRejection(msg.id, reason) + db.convertLifecycleMessageToRejection(msg.id, 'sender_not_assignee', reason) return { action: 'rejected', code: 'sender_not_assignee', reason } } @@ -159,7 +178,9 @@ function reconcileWorkerDoneMessage( ): LifecycleReconciliationResult { onLog(`Worker done: ${msg.from_handle} — ${msg.subject}`) + let invalidPayload = false const payload = parseObjectPayload(msg, () => { + invalidPayload = true onLog(`Warning: invalid payload in worker_done from ${msg.from_handle}`) }) const persistedRejection = getPersistedLifecycleRejection(payload) @@ -169,58 +190,83 @@ function reconcileWorkerDoneMessage( onLog(`Warning: worker_done rejected: ${persistedRejection.reason}`) return persistedRejection } + if (invalidPayload || !msg.payload) { + return rejectLifecycleMessage( + db, + msg, + 'invalid_payload', + 'worker_done requires a JSON object payload.', + onLog + ) + } const taskId = payload.taskId if (typeof taskId !== 'string' || taskId.length === 0) { - onLog(`Warning: worker_done without taskId from ${msg.from_handle}`) - return { action: 'ignored' } + return rejectLifecycleMessage(db, msg, 'missing_task_id', 'worker_done requires taskId.', onLog) } const dispatchId = payload.dispatchId if (typeof dispatchId !== 'string' || dispatchId.length === 0) { - onLog(`Warning: worker_done without dispatchId from ${msg.from_handle}`) - return { action: 'ignored' } + return rejectLifecycleMessage( + db, + msg, + 'missing_dispatch_id', + 'worker_done requires dispatchId.', + onLog + ) + } + + const outcome = payload.outcome + if (outcome !== 'succeeded' && outcome !== 'failed') { + return rejectLifecycleMessage( + db, + msg, + 'invalid_outcome', + 'worker_done requires outcome=succeeded or outcome=failed.', + onLog + ) } const task = db.getTask(taskId) if (!task) { - onLog(`Warning: worker_done for unknown task ${taskId}`) - return { action: 'ignored' } + return rejectLifecycleMessage( + db, + msg, + 'unknown_task', + `worker_done references unknown task ${taskId}.`, + onLog + ) } // Why: taskId alone is not a completion authority; retried tasks can have // stale worker_done messages racing the current active dispatch. const dispatch = db.getDispatchContextById(dispatchId) if (!dispatch) { - onLog(`Warning: worker_done for unknown dispatch ${dispatchId}`) - return { action: 'ignored' } + return rejectLifecycleMessage( + db, + msg, + 'unknown_dispatch', + `worker_done references unknown dispatch ${dispatchId}.`, + onLog + ) } if (dispatch.task_id !== taskId) { - onLog( - `Warning: worker_done dispatch ${dispatchId} belongs to ${dispatch.task_id}, not ${taskId}` + return rejectLifecycleMessage( + db, + msg, + 'task_dispatch_mismatch', + `worker_done dispatch ${dispatchId} belongs to ${dispatch.task_id}, not ${taskId}.`, + onLog ) - return { action: 'ignored' } } if (!hasLifecycleAuthority(dispatch, msg)) { const reason = buildLifecycleAuthorityRejectionReason(dispatchId, dispatch, msg) onLog(`Warning: worker_done rejected: ${reason}`) - db.convertLifecycleMessageToRejection(msg.id, reason) + db.convertLifecycleMessageToRejection(msg.id, 'sender_not_assignee', reason) return { action: 'rejected', code: 'sender_not_assignee', reason } } // Why: `orchestration.send` can release the DB lock before waking the // coordinator; the later coordinator read still needs to observe completion. - if (dispatch.status === 'completed' && task.status === 'completed') { - return { action: 'completed', taskId, dispatchId } - } - if (dispatch.status !== 'dispatched') { - onLog(`Warning: worker_done for inactive dispatch ${dispatchId} ignored`) - return { action: 'ignored' } - } - if (db.getDispatchContext(taskId)?.id !== dispatchId || task.status !== 'dispatched') { - onLog(`Warning: worker_done for stale dispatch ${dispatchId} ignored`) - return { action: 'ignored' } - } - const filesModified = Array.isArray(payload.filesModified) && payload.filesModified.every((file) => typeof file === 'string') @@ -228,17 +274,48 @@ function reconcileWorkerDoneMessage( : [] const result = JSON.stringify({ + provenance: 'worker_report', + outcome, + messageId: msg.id, + reportedBy: msg.from_handle, + subject: msg.subject, + body: msg.body, completedBy: msg.from_handle, filesModified, + reportPath: typeof payload.reportPath === 'string' ? payload.reportPath : null, completedAt: new Date().toISOString() }) - db.updateTaskStatus(taskId, 'completed', result) + const settlement = db.settleWorkerReport({ + taskId, + dispatchId, + outcome: outcome as WorkerReportOutcome, + result + }) + if (settlement.action === 'rejected') { + return rejectLifecycleMessage(db, msg, settlement.code, settlement.reason, onLog) + } suppressEarlierHeartbeats(db, msg, dispatchId) - onLog(`Task ${taskId} completed`) + if (outcome === 'failed') { + onLog(`Task ${taskId} failed by worker report`) + return { action: 'failed', taskId, dispatchId } + } + onLog(`Task ${taskId} completed by worker report`) return { action: 'completed', taskId, dispatchId } } +function rejectLifecycleMessage( + db: OrchestrationDb, + msg: MessageRow, + code: LifecycleRejectionCode, + reason: string, + onLog: LogFn +): LifecycleRejectionResult { + onLog(`Warning: ${msg.type} rejected: ${reason}`) + db.convertLifecycleMessageToRejection(msg.id, code, reason) + return { action: 'rejected', code, reason } +} + function buildLifecycleAuthorityRejectionReason( dispatchId: string, dispatch: { assignee_handle: string | null; assignee_pane_key: string | null }, diff --git a/src/main/runtime/orchestration/orchestration-db-permissions.test.ts b/src/main/runtime/orchestration/orchestration-db-permissions.test.ts new file mode 100644 index 00000000000..07285ef35b1 --- /dev/null +++ b/src/main/runtime/orchestration/orchestration-db-permissions.test.ts @@ -0,0 +1,27 @@ +import { mkdtempSync, rmSync, statSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { OrchestrationDb } from './db' + +describe.skipIf(process.platform === 'win32')('orchestration database permissions', () => { + let directory: string | undefined + let db: OrchestrationDb | undefined + + afterEach(() => { + db?.close() + if (directory) { + rmSync(directory, { recursive: true, force: true }) + } + }) + + it('restricts the database and live SQLite sidecars to the current user', () => { + directory = mkdtempSync(join(tmpdir(), 'orca-orchestration-permissions-')) + const dbPath = join(directory, 'orchestration.db') + db = new OrchestrationDb(dbPath) + + for (const path of [dbPath, `${dbPath}-wal`, `${dbPath}-shm`]) { + expect(statSync(path).mode & 0o777).toBe(0o600) + } + }) +}) diff --git a/src/main/runtime/orchestration/orchestration-error.ts b/src/main/runtime/orchestration/orchestration-error.ts new file mode 100644 index 00000000000..41b5e88102e --- /dev/null +++ b/src/main/runtime/orchestration/orchestration-error.ts @@ -0,0 +1,11 @@ +export class OrchestrationError extends Error { + readonly code: string + readonly data?: unknown + + constructor(code: string, message: string, data?: unknown) { + super(message) + this.name = 'OrchestrationError' + this.code = code + this.data = data + } +} diff --git a/src/main/runtime/orchestration/orchestration-mutation-question-db.test.ts b/src/main/runtime/orchestration/orchestration-mutation-question-db.test.ts new file mode 100644 index 00000000000..a6039c381f9 --- /dev/null +++ b/src/main/runtime/orchestration/orchestration-mutation-question-db.test.ts @@ -0,0 +1,167 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { OrchestrationDb } from './db' + +describe('OrchestrationDb mutation and question state', () => { + let db: OrchestrationDb | undefined + + afterEach(() => { + db?.close() + }) + + function createDb(): OrchestrationDb { + db = new OrchestrationDb(':memory:') + return db + } + + describe('durable mutation receipts', () => { + it('replays completed input and rejects request ID reuse with changed input', () => { + const d = createDb() + const started = d.beginMutationReceipt({ + callerFingerprint: 'caller_a', + requestId: 'request_1', + method: 'orchestration.send', + payloadHash: 'hash_a' + }) + expect(started.disposition).toBe('started') + + d.completeMutationReceipt({ + callerFingerprint: 'caller_a', + requestId: 'request_1', + method: 'orchestration.send', + payloadHash: 'hash_a', + receipt: '{"messageId":"msg_1"}' + }) + expect( + d.beginMutationReceipt({ + callerFingerprint: 'caller_a', + requestId: 'request_1', + method: 'orchestration.send', + payloadHash: 'hash_a' + }) + ).toMatchObject({ + disposition: 'completed', + row: { receipt: '{"messageId":"msg_1"}' } + }) + + expect(() => + d.beginMutationReceipt({ + callerFingerprint: 'caller_a', + requestId: 'request_1', + method: 'orchestration.send', + payloadHash: 'hash_b' + }) + ).toThrow('already used with different input') + }) + + it('keeps caller namespaces separate and can discard only pending work', () => { + const d = createDb() + for (const callerFingerprint of ['caller_a', 'caller_b']) { + d.beginMutationReceipt({ + callerFingerprint, + requestId: 'same_request', + method: 'orchestration.send', + payloadHash: 'same_hash' + }) + } + expect(d.getMutationReceipt('caller_a', 'same_request')?.state).toBe('pending') + expect(d.getMutationReceipt('caller_b', 'same_request')?.state).toBe('pending') + + d.discardPendingMutationReceipt('caller_a', 'same_request') + expect(d.getMutationReceipt('caller_a', 'same_request')).toBeUndefined() + expect(d.getMutationReceipt('caller_b', 'same_request')?.state).toBe('pending') + }) + }) + + describe('question threads', () => { + it('accepts a question message in the fresh canonical schema', () => { + const d = createDb() + const message = d.insertMessage({ + from: 'worker', + to: 'run:run_1', + subject: 'Need input', + type: 'question' + }) + + expect(message.type).toBe('question') + }) + + it('uses the original message ID and records one durable answer', () => { + const d = createDb() + const run = d.createRun({ + objective: 'Questions', + coordinatorHandle: 'term_coord', + coordinatorPaneKey: 'tab_coord:11111111-1111-4111-8111-111111111111' + }) + const task = d.createTask({ spec: 'ask', runId: run.id }) + const dispatch = d.createDispatchContext(task.id, 'term_worker') + const created = d.createQuestion({ + runId: run.id, + dispatchId: dispatch.id, + askerHandle: 'term_worker', + question: 'Which format?', + options: ['old', 'new'] + }) + + expect(created.question.message_id).toBe(created.message.id) + expect(created.message).toMatchObject({ + run_id: run.id, + to_handle: `run:${run.id}`, + type: 'question', + thread_id: created.message.id + }) + const answer = d.answerQuestion({ + messageId: created.message.id, + runId: run.id, + consumerGeneration: run.consumer_generation, + body: 'old' + }) + const replay = d.answerQuestion({ + messageId: created.message.id, + runId: run.id, + consumerGeneration: run.consumer_generation, + body: 'old' + }) + + expect(answer.message.to_handle).toBe(`dispatch:${dispatch.id}`) + expect(answer.question.status).toBe('answered') + expect(replay.message.id).toBe(answer.message.id) + expect(replay.duplicate).toBe(true) + expect(() => + d.answerQuestion({ + messageId: created.message.id, + runId: run.id, + consumerGeneration: run.consumer_generation, + body: 'new' + }) + ).toThrow(/different answer/) + }) + + it('closes pending questions with their Dispatch', () => { + const d = createDb() + const run = d.createRun({ + objective: 'Close questions', + coordinatorHandle: 'term_coord', + coordinatorPaneKey: 'tab_coord:11111111-1111-4111-8111-111111111111' + }) + const task = d.createTask({ spec: 'ask', runId: run.id }) + const dispatch = d.createDispatchContext(task.id, 'term_worker') + const created = d.createQuestion({ + runId: run.id, + dispatchId: dispatch.id, + askerHandle: 'term_worker', + question: 'Still active?' + }) + + expect(d.closeQuestionsForDispatch(dispatch.id)).toEqual([created.message.id]) + expect(d.getQuestion(created.message.id)?.status).toBe('closed') + expect(() => + d.answerQuestion({ + messageId: created.message.id, + runId: run.id, + consumerGeneration: run.consumer_generation, + body: 'late' + }) + ).toThrow(/inactive/) + }) + }) +}) diff --git a/src/main/runtime/orchestration/orchestration-reset-db.test.ts b/src/main/runtime/orchestration/orchestration-reset-db.test.ts new file mode 100644 index 00000000000..e98ec4230b5 --- /dev/null +++ b/src/main/runtime/orchestration/orchestration-reset-db.test.ts @@ -0,0 +1,101 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { LEGACY_RUN_ID, OrchestrationDb } from './db' + +describe('OrchestrationDb reset scopes', () => { + let db: OrchestrationDb | undefined + + afterEach(() => db?.close()) + + function createState() { + db = new OrchestrationDb(':memory:') + const run = db.createRun({ + objective: 'Reset contract', + coordinatorHandle: 'term_coord', + coordinatorPaneKey: 'tab_coord:11111111-1111-4111-8111-111111111111' + }) + const task = db.createTask({ spec: 'work', runId: run.id }) + const started = db.createStartingWorkerDispatch({ + taskId: task.id, + startOptions: { worktree: 'current' }, + runtimeEpoch: 'runtime_1', + federation: { + environmentId: 'environment_1', + environmentName: 'Windows', + peerFingerprint: 'peer_1', + protocolVersion: 1 + }, + mutationReceipt: { + callerFingerprint: 'caller_1', + requestId: 'request_1', + method: 'orchestration.workerStart', + payloadHash: 'hash_1' + } + }) + const message = db.insertMessage({ + runId: run.id, + from: 'worker', + to: `run:${run.id}`, + subject: 'status' + }) + db.enqueueFederationRelay({ + dispatchId: started.dispatch.id, + direction: 'to_home', + kind: 'question', + payload: '{}', + messageId: 'question_1', + remoteQuestion: true + }) + return { run, task, started, message } + } + + it('resetAll clears Runs, worker/federation state, and messages', () => { + const state = createState() + + db!.resetAll() + + expect(db!.listRuns()).toEqual([expect.objectContaining({ id: LEGACY_RUN_ID, legacy: 1 })]) + expect(db!.getTask(state.task.id)).toBeUndefined() + expect(db!.getWorkerDispatch(state.started.dispatch.id)).toBeUndefined() + expect(db!.getFederatedDispatch(state.started.dispatch.id)).toBeUndefined() + // The ledger survives so a lost reset response cannot replay as a new mutation. + expect(db!.getMutationReceipt('caller_1', 'request_1')).toBeDefined() + expect(db!.getInbox()).toEqual([]) + expect( + db!.listFederationRelay({ + dispatchId: state.started.dispatch.id, + direction: 'to_home', + afterSequence: 0 + }) + ).toEqual([]) + }) + + it('resetTasks preserves Runs and messages while clearing every worker attachment', () => { + const state = createState() + + db!.resetTasks() + + expect(db!.getRun(state.run.id)).toBeDefined() + expect(db!.getMessageById(state.message.id)).toBeDefined() + expect(db!.getTask(state.task.id)).toBeUndefined() + expect(db!.getWorkerDispatch(state.started.dispatch.id)).toBeUndefined() + expect(db!.getFederatedDispatch(state.started.dispatch.id)).toBeUndefined() + expect(db!.getRemoteQuestion('question_1')).toBeUndefined() + }) + + it('resetMessages preserves active relay cursors while clearing the Run inbox', () => { + const state = createState() + + db!.resetMessages() + + expect(db!.getTask(state.task.id)).toBeDefined() + expect(db!.getInbox()).toEqual([]) + expect(db!.getRemoteQuestion('question_1')).toBeDefined() + expect( + db!.listFederationRelay({ + dispatchId: state.started.dispatch.id, + direction: 'to_home', + afterSequence: 0 + }) + ).toHaveLength(1) + }) +}) diff --git a/src/main/runtime/orchestration/orchestration-run-delivery-db.test.ts b/src/main/runtime/orchestration/orchestration-run-delivery-db.test.ts new file mode 100644 index 00000000000..acd6e936f47 --- /dev/null +++ b/src/main/runtime/orchestration/orchestration-run-delivery-db.test.ts @@ -0,0 +1,279 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { LEGACY_RUN_ID, OrchestrationDb } from './db' + +describe('OrchestrationDb Run state', () => { + let db: OrchestrationDb | undefined + + afterEach(() => { + db?.close() + }) + + function createDb(): OrchestrationDb { + db = new OrchestrationDb(':memory:') + return db + } + + function createBoundRun(d: OrchestrationDb) { + return d.createRun({ + objective: 'Mailbox test', + coordinatorHandle: 'term_coord', + coordinatorPaneKey: 'tab_coord:11111111-1111-4111-8111-111111111111' + }) + } + + describe('Run deliveries', () => { + it('returns one bounded FIFO batch and replays it until acknowledgment', () => { + const d = createDb() + const run = createBoundRun(d) + for (let index = 0; index < 55; index++) { + d.insertMessage({ + from: 'worker', + to: `run:${run.id}`, + subject: `message ${index}`, + runId: run.id + }) + } + + const first = d.getOrCreateRunDelivery({ + runId: run.id, + consumerGeneration: run.consumer_generation + }) + const replay = d.getOrCreateRunDelivery({ + runId: run.id, + consumerGeneration: run.consumer_generation + }) + + expect(first?.messages).toHaveLength(50) + expect(first?.messages[0].subject).toBe('message 0') + expect(first?.messages[49].subject).toBe('message 49') + expect(replay?.delivery.id).toBe(first?.delivery.id) + expect(replay?.replayed).toBe(true) + + d.acknowledgeRunDelivery({ + runId: run.id, + consumerGeneration: run.consumer_generation, + deliveryId: first!.delivery.id + }) + const next = d.getOrCreateRunDelivery({ + runId: run.id, + consumerGeneration: run.consumer_generation + }) + expect(next?.messages.map((message) => message.subject)).toEqual([ + 'message 50', + 'message 51', + 'message 52', + 'message 53', + 'message 54' + ]) + }) + + it('acknowledges the whole batch idempotently without consuming newer mail', () => { + const d = createDb() + const run = createBoundRun(d) + d.insertMessage({ from: 'a', to: `run:${run.id}`, subject: 'first', runId: run.id }) + const delivery = d.getOrCreateRunDelivery({ + runId: run.id, + consumerGeneration: run.consumer_generation + })! + d.insertMessage({ from: 'b', to: `run:${run.id}`, subject: 'newer', runId: run.id }) + + const firstAck = d.acknowledgeRunDelivery({ + runId: run.id, + consumerGeneration: run.consumer_generation, + deliveryId: delivery.delivery.id + }) + const duplicateAck = d.acknowledgeRunDelivery({ + runId: run.id, + consumerGeneration: run.consumer_generation, + deliveryId: delivery.delivery.id + }) + + expect(firstAck.duplicate).toBe(false) + expect(duplicateAck.duplicate).toBe(true) + expect( + d + .getOrCreateRunDelivery({ + runId: run.id, + consumerGeneration: run.consumer_generation + }) + ?.messages.map((message) => message.subject) + ).toEqual(['newer']) + }) + + it('uses type filters only as wake predicates and returns the full oldest batch', () => { + const d = createDb() + const run = createBoundRun(d) + d.insertMessage({ from: 'a', to: `run:${run.id}`, subject: 'status', runId: run.id }) + expect( + d.getOrCreateRunDelivery({ + runId: run.id, + consumerGeneration: run.consumer_generation, + wakeTypes: ['worker_done'] + }) + ).toBeUndefined() + d.insertMessage({ + from: 'b', + to: `run:${run.id}`, + subject: 'done', + type: 'worker_done', + runId: run.id + }) + + const delivery = d.getOrCreateRunDelivery({ + runId: run.id, + consumerGeneration: run.consumer_generation, + wakeTypes: ['worker_done'] + }) + expect(delivery?.messages.map((message) => message.subject)).toEqual(['status', 'done']) + }) + + it('fences an outstanding batch when the Run consumer changes', () => { + const d = createDb() + const run = createBoundRun(d) + d.insertMessage({ from: 'a', to: `run:${run.id}`, subject: 'one', runId: run.id }) + const oldDelivery = d.getOrCreateRunDelivery({ + runId: run.id, + consumerGeneration: run.consumer_generation + })! + const rebound = d.bindRun({ + runId: run.id, + coordinatorHandle: 'term_new', + coordinatorPaneKey: 'tab_new:22222222-2222-4222-9222-222222222222' + })! + + let fencedError: unknown + try { + d.acknowledgeRunDelivery({ + runId: run.id, + consumerGeneration: run.consumer_generation, + deliveryId: oldDelivery.delivery.id + }) + } catch (error) { + fencedError = error + } + expect(fencedError).toMatchObject({ code: 'consumer_fenced' }) + const replacement = d.getOrCreateRunDelivery({ + runId: run.id, + consumerGeneration: rebound.consumer_generation + }) + expect(replacement?.delivery.id).not.toBe(oldDelivery.delivery.id) + expect(replacement?.messages.map((message) => message.subject)).toEqual(['one']) + }) + + it('replays an outstanding batch after reopening the database', () => { + const dir = mkdtempSync(join(tmpdir(), 'orca-delivery-')) + const dbPath = join(dir, 'orchestration.db') + try { + const firstDb = new OrchestrationDb(dbPath) + const run = createBoundRun(firstDb) + firstDb.insertMessage({ + from: 'a', + to: `run:${run.id}`, + subject: 'survives', + runId: run.id + }) + const first = firstDb.getOrCreateRunDelivery({ + runId: run.id, + consumerGeneration: run.consumer_generation + })! + firstDb.close() + + const reopened = new OrchestrationDb(dbPath) + db = reopened + const replay = reopened.getOrCreateRunDelivery({ + runId: run.id, + consumerGeneration: run.consumer_generation + }) + expect(replay?.delivery.id).toBe(first.delivery.id) + expect(replay?.messages[0].subject).toBe('survives') + } finally { + db?.close() + db = undefined + rmSync(dir, { recursive: true, force: true }) + } + }) + }) + + describe('lightweight Run scope', () => { + it('binds creation to one pane and fences that pane when it creates another Run', () => { + const d = createDb() + const first = d.createRun({ + objective: 'First objective', + coordinatorHandle: 'term_first', + coordinatorPaneKey: 'tab_a:11111111-1111-4111-8111-111111111111' + }) + expect(first).toMatchObject({ consumer_generation: 1, legacy: 0 }) + expect(d.getCurrentRunForPane('tab_reminted:11111111-1111-4111-8111-111111111111')?.id).toBe( + first.id + ) + + const second = d.createRun({ + objective: 'Second objective', + coordinatorHandle: 'term_second', + coordinatorPaneKey: 'tab_b:11111111-1111-4111-8111-111111111111' + }) + expect(d.getRun(first.id)).toMatchObject({ + coordinator_handle: null, + coordinator_pane_key: null, + consumer_generation: 2 + }) + expect(d.getCurrentRunForPane('tab_b:11111111-1111-4111-8111-111111111111')?.id).toBe( + second.id + ) + }) + + it('rebinds a Run by incrementing its consumer generation', () => { + const d = createDb() + const run = d.createRun({ + objective: 'Move coordinator', + coordinatorHandle: 'term_old', + coordinatorPaneKey: 'tab_old:11111111-1111-4111-8111-111111111111' + }) + + expect( + d.bindRun({ + runId: run.id, + coordinatorHandle: 'term_new', + coordinatorPaneKey: 'tab_new:22222222-2222-4222-9222-222222222222' + }) + ).toMatchObject({ + coordinator_handle: 'term_new', + consumer_generation: 2 + }) + expect(d.getCurrentRunForPane('tab_old:11111111-1111-4111-8111-111111111111')).toBeUndefined() + expect( + d.bindRun({ + runId: LEGACY_RUN_ID, + coordinatorHandle: 'term_new', + coordinatorPaneKey: 'tab_new:22222222-2222-4222-9222-222222222222' + }) + ).toBeUndefined() + }) + + it('associates task, dispatch, message, and gate rows with the selected Run', () => { + const d = createDb() + const run = d.createRun({ + objective: 'Scoped work', + coordinatorHandle: 'term_coord', + coordinatorPaneKey: 'tab_coord:11111111-1111-4111-8111-111111111111' + }) + const task = d.createTask({ spec: 'work', runId: run.id }) + const dispatch = d.createDispatchContext(task.id, 'term_worker') + const message = d.insertMessage({ + runId: run.id, + from: 'term_worker', + to: 'term_coord', + subject: 'status' + }) + const gate = d.createGate({ taskId: task.id, question: 'Continue?' }) + + expect(task.run_id).toBe(run.id) + expect(dispatch.run_id).toBe(run.id) + expect(message.run_id).toBe(run.id) + expect(gate.run_id).toBe(run.id) + }) + }) +}) diff --git a/src/main/runtime/orchestration/orchestration-worker-dispatch-db.test.ts b/src/main/runtime/orchestration/orchestration-worker-dispatch-db.test.ts new file mode 100644 index 00000000000..34e9e456f6a --- /dev/null +++ b/src/main/runtime/orchestration/orchestration-worker-dispatch-db.test.ts @@ -0,0 +1,274 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { OrchestrationDb } from './db' + +describe('OrchestrationDb worker Dispatch state', () => { + let db: OrchestrationDb | undefined + + afterEach(() => { + db?.close() + }) + + function createDb(): OrchestrationDb { + db = new OrchestrationDb(':memory:') + return db + } + + it('creates and activates a composed worker Dispatch transactionally', () => { + const d = createDb() + const task = d.createTask({ spec: 'worker' }) + const started = d.createStartingWorkerDispatch({ + taskId: task.id, + startOptions: { topology: 'current', agent: 'codex' } + }) + expect(started).toMatchObject({ + dispatch: { status: 'pending' }, + worker: { state: 'starting', stage: 'accepted' } + }) + expect(d.getTask(task.id)?.status).toBe('dispatched') + + const capability = d.prepareStartingWorkerAuthority({ + dispatchId: started.dispatch.id, + handle: 'term_worker', + paneKey: 'tab_worker:leaf_worker', + processIncarnation: 'runtime:pty:1', + worktreeId: 'repo::worktree', + setupState: 'not_applicable', + effects: [{ kind: 'terminal', action: 'created', id: 'term_worker' }] + }) + expect(capability).toMatch(/^dcap_/) + expect(d.markWorkerDispatchReady(started.dispatch.id)).toMatchObject({ + state: 'ready', + stage: 'input_accepted' + }) + expect(d.getDispatchContextById(started.dispatch.id)).toMatchObject({ + status: 'dispatched', + assignee_handle: 'term_worker' + }) + }) + + it('commits worker-start mutation acceptance with the starting Dispatch', () => { + const d = createDb() + const task = d.createTask({ spec: 'atomic acceptance' }) + const mutationReceipt = { + callerFingerprint: 'caller_fingerprint', + requestId: 'worker_start_request', + method: 'orchestration.workerStart', + payloadHash: 'payload_hash' + } + + const started = d.createStartingWorkerDispatch({ + taskId: task.id, + startOptions: { topology: 'current' }, + mutationReceipt + }) + + expect(d.getMutationReceipt('caller_fingerprint', 'worker_start_request')).toMatchObject({ + state: 'pending', + method: 'orchestration.workerStart' + }) + expect(d.getWorkerDispatch(started.dispatch.id)).toMatchObject({ + state: 'starting', + stage: 'accepted' + }) + expect(d.getTask(task.id)?.status).toBe('dispatched') + }) + + it('rolls back worker-start mutation acceptance when the Task cannot start', () => { + const d = createDb() + + expect(() => + d.createStartingWorkerDispatch({ + taskId: 'task_missing', + startOptions: {}, + mutationReceipt: { + callerFingerprint: 'caller_fingerprint', + requestId: 'invalid_worker_start', + method: 'orchestration.workerStart', + payloadHash: 'payload_hash' + } + }) + ).toThrow('was not found') + expect(d.getMutationReceipt('caller_fingerprint', 'invalid_worker_start')).toBeUndefined() + }) + + it('fails a composed start without losing residual resource receipts', () => { + const d = createDb() + const task = d.createTask({ spec: 'worker' }) + const started = d.createStartingWorkerDispatch({ taskId: task.id, startOptions: {} }) + d.recordWorkerStage({ + dispatchId: started.dispatch.id, + stage: 'terminal_created', + effects: [{ kind: 'terminal', action: 'created', id: 'term_worker' }], + residualResources: [{ kind: 'terminal', id: 'term_worker' }] + }) + + expect(d.failWorkerStart(started.dispatch.id, 'agent_readiness', 'timed out')).toMatchObject({ + state: 'failed', + stage: 'agent_readiness', + last_error: 'timed out', + residual_resources: expect.stringContaining('term_worker') + }) + expect(d.getTask(task.id)?.status).toBe('failed') + }) + + it('allows retry only from the Task current terminal Dispatch', () => { + const d = createDb() + const task = d.createTask({ spec: 'retry current' }) + const first = d.createStartingWorkerDispatch({ taskId: task.id, startOptions: {} }) + d.failWorkerStart(first.dispatch.id, 'agent_readiness', 'first failed') + const second = d.createStartingWorkerDispatch({ + taskId: task.id, + retryOf: first.dispatch.id, + startOptions: {} + }) + d.failWorkerStart(second.dispatch.id, 'agent_readiness', 'second failed') + + expect(() => + d.createStartingWorkerDispatch({ + taskId: task.id, + retryOf: first.dispatch.id, + startOptions: {} + }) + ).toThrow('cannot retry') + expect( + d.createStartingWorkerDispatch({ + taskId: task.id, + retryOf: second.dispatch.id, + startOptions: {} + }).worker.state + ).toBe('starting') + }) + + it('treats abandon of a superseded Dispatch as a no-op', () => { + const d = createDb() + const task = d.createTask({ spec: 'stale abandon' }) + const first = d.createStartingWorkerDispatch({ taskId: task.id, startOptions: {} }) + d.failWorkerStart(first.dispatch.id, 'agent_readiness', 'first failed') + const second = d.createStartingWorkerDispatch({ + taskId: task.id, + retryOf: first.dispatch.id, + startOptions: {} + }) + d.prepareStartingWorkerAuthority({ + dispatchId: second.dispatch.id, + handle: 'term_replacement', + paneKey: 'tab_replacement:leaf_replacement', + processIncarnation: 'runtime:pty:2', + worktreeId: 'repo::worktree', + setupState: 'not_applicable', + effects: [] + }) + d.markWorkerDispatchReady(second.dispatch.id) + + expect(d.abandonWorkerDispatch(first.dispatch.id)).toMatchObject({ + disposition: 'stale', + worker: { state: 'failed' } + }) + expect(d.getTask(task.id)?.status).toBe('dispatched') + expect(d.getWorkerDispatch(second.dispatch.id)?.state).toBe('ready') + expect( + d.settleWorkerReport({ + taskId: task.id, + dispatchId: second.dispatch.id, + outcome: 'succeeded', + result: '{}' + }) + ).toMatchObject({ action: 'settled' }) + expect(d.getTask(task.id)?.status).toBe('completed') + }) + + it('lets the stop fence win before a late worker completion', () => { + const d = createDb() + const task = d.createTask({ spec: 'race' }) + const started = d.createStartingWorkerDispatch({ taskId: task.id, startOptions: {} }) + d.prepareStartingWorkerAuthority({ + dispatchId: started.dispatch.id, + handle: 'term_worker', + paneKey: 'tab_worker:leaf_worker', + processIncarnation: 'runtime:pty:1', + worktreeId: 'repo::worktree', + setupState: 'not_applicable', + effects: [] + }) + d.markWorkerDispatchReady(started.dispatch.id) + + expect(d.beginWorkerStop(started.dispatch.id).disposition).toBe('stopping') + expect( + d.settleWorkerReport({ + taskId: task.id, + dispatchId: started.dispatch.id, + outcome: 'succeeded', + result: '{}' + }) + ).toMatchObject({ action: 'rejected', code: 'inactive_dispatch' }) + expect(d.settleWorkerStop(started.dispatch.id).state).toBe('stopped') + expect(d.getTask(task.id)?.status).toBe('blocked') + }) + + it('allows explicit stop recovery from uncertain local and remote starts', () => { + const d = createDb() + const task = d.createTask({ spec: 'uncertain local start' }) + const started = d.createStartingWorkerDispatch({ taskId: task.id, startOptions: {} }) + d.markWorkerStartUnknown(started.dispatch.id, 'agent_readiness', 'connection lost') + + expect(d.beginWorkerStop(started.dispatch.id)).toMatchObject({ + disposition: 'stopping', + worker: { state: 'stopping' } + }) + + d.createRemoteDispatchAttachment({ + dispatchId: 'ctx_remote_unknown', + taskId: 'task_remote_unknown', + homePeerFingerprint: 'home_peer', + protocolVersion: 1, + runtimeEpoch: 'worker_epoch', + mutationReceipt: { + callerFingerprint: 'home_peer', + requestId: 'remote_unknown_start', + method: 'orchestration.federationAttachStart', + payloadHash: 'remote_unknown_payload' + } + }) + d.recordRemoteAttachmentStage({ + dispatchId: 'ctx_remote_unknown', + stage: 'agent_readiness', + state: 'start_unknown', + terminalHandle: 'term_remote_worker' + }) + + expect(d.beginRemoteAttachmentStop('ctx_remote_unknown')).toMatchObject({ + state: 'stopping', + stage: 'stop_requested', + capability_hash: null + }) + }) + + it('returns already-settled when completion wins before stop', () => { + const d = createDb() + const task = d.createTask({ spec: 'race' }) + const started = d.createStartingWorkerDispatch({ taskId: task.id, startOptions: {} }) + d.prepareStartingWorkerAuthority({ + dispatchId: started.dispatch.id, + handle: 'term_worker', + paneKey: 'tab_worker:leaf_worker', + processIncarnation: 'runtime:pty:1', + worktreeId: 'repo::worktree', + setupState: 'not_applicable', + effects: [] + }) + d.markWorkerDispatchReady(started.dispatch.id) + expect( + d.settleWorkerReport({ + taskId: task.id, + dispatchId: started.dispatch.id, + outcome: 'succeeded', + result: '{}' + }) + ).toMatchObject({ action: 'settled' }) + + expect(d.beginWorkerStop(started.dispatch.id)).toMatchObject({ + disposition: 'already_settled', + worker: { state: 'succeeded' } + }) + }) +}) diff --git a/src/main/runtime/orchestration/preamble.test.ts b/src/main/runtime/orchestration/preamble.test.ts index ef133551034..4497d9c8ad5 100644 --- a/src/main/runtime/orchestration/preamble.test.ts +++ b/src/main/runtime/orchestration/preamble.test.ts @@ -45,9 +45,12 @@ describe('buildDispatchPreamble', () => { expect(result).toContain('reportPath') expect(result).toContain('--task-id task_abc123') expect(result).toContain('--dispatch-id ctx_def456') + expect(result).toContain('--outcome succeeded') + expect(result).toContain('replace it with --outcome failed') expect(result).toContain('--files-modified "path/a,path/b"') expect(result).toContain('--report-path ""') - expect(result).toMatch(/orchestration send --to term_coord --from term_worker/) + expect(result).toMatch(/orchestration send --from term_worker/) + expect(result).not.toContain('orchestration send --to term_coord') }) it( @@ -85,12 +88,12 @@ describe('buildDispatchPreamble', () => { expect(result).toContain('--task-id task_abc123') expect(result).toContain('--dispatch-id ctx_def456') expect(result).toContain('--phase ""') - expect(result).toMatch(/orchestration send --to term_coord --from term_worker/) + expect(result).toMatch(/orchestration send --from term_worker/) }) it('includes ask block with BEHAVIOR RULE #1 forbidding AskUserQuestion', () => { const result = buildDispatchPreamble(baseParams()) - expect(result).toMatch(/orchestration ask --to term_coord --from term_worker/) + expect(result).toMatch(/orchestration ask --from term_worker/) expect(result).toContain('--question') expect(result).toContain('--timeout-ms 600000') // Why: the exact phrase is asserted so the rule can't be trimmed away by @@ -101,21 +104,27 @@ describe('buildDispatchPreamble', () => { // else (e.g., not in an example payload or header). Count occurrences // of the exact token as a sanity check. const occurrences = (result.match(/AskUserQuestion/g) ?? []).length - // Three mentions: the one-liner ban, the TUI-prompt rationale, and the - // "when tempted to reach for AskUserQuestion" closing line. - expect(occurrences).toBe(3) + expect(occurrences).toBe(2) }) it('binds every injected worker command to the dispatched terminal', () => { const result = buildDispatchPreamble(baseParams()) - expect(result).toMatch(/orchestration ask --to term_coord --from term_worker/) - expect(result).toMatch( - /orchestration send --to term_coord --from term_worker \\\n --type escalation/ - ) + expect(result).toMatch(/orchestration ask --from term_worker/) + expect(result).toMatch(/orchestration send --from term_worker \\\n --type escalation/) expect(result).toContain('orchestration check --terminal term_worker') }) + it('carries the minted Dispatch capability on lifecycle and question commands', () => { + const result = buildDispatchPreamble({ + ...baseParams(), + dispatchCapability: 'dcap_test_secret' + }) + + expect(result.match(/--dispatch-capability dcap_test_secret/g)).toHaveLength(4) + expect(result).not.toContain('"dispatchCapability"') + }) + it('tells prompt-returning workers to idle without post-done polling', () => { const result = buildDispatchPreamble(baseParams()) const section = afterWorkerDoneSection(result) diff --git a/src/main/runtime/orchestration/preamble.ts b/src/main/runtime/orchestration/preamble.ts index a8012c56854..4c1864f5507 100644 --- a/src/main/runtime/orchestration/preamble.ts +++ b/src/main/runtime/orchestration/preamble.ts @@ -8,6 +8,7 @@ export type PreambleParams = { // prevents stale messages from a previously-failed dispatch from completing // or refreshing the retry. dispatchId: string + dispatchCapability?: string taskSpec: string coordinatorHandle: string workerHandle: string @@ -52,6 +53,9 @@ export function buildDispatchPreamble(params: PreambleParams): string { cli, workerKind: params.workerKind ?? 'prompt-returning-agent' }) + const capabilityFlag = params.dispatchCapability + ? ` --dispatch-capability ${params.dispatchCapability}` + : '' const header = `You are working inside Orca, a multi-agent IDE. You are a dispatched worker. Your coordinator's terminal handle is: ${params.coordinatorHandle} @@ -62,7 +66,7 @@ Slack, GitHub comments, or any other channel to reach a human during the run. === CLI COMMANDS === - # Report task completion (REQUIRED when done — even on failure). + # Report the terminal task outcome (REQUIRED exactly once). # # RULE: --body must be a 3-sentence executive summary (what you did, # what you found, what's left). Never send an empty body; the coordinator @@ -70,14 +74,15 @@ Slack, GitHub comments, or any other channel to reach a human during the run. # If you produced a long-form artifact, include its path as # payload.reportPath so the coordinator can find it without a file search. # - # RULE: send worker_done exactly once. Failure is still a worker_done - # with subject like "Failed: " — never silently exit. + # RULE: send worker_done exactly once. Use --outcome succeeded when the + # requested work is done, or replace it with --outcome failed when it is not. + # Never encode failure only in prose and never silently exit. # Include BOTH taskId and dispatchId in the payload so a late completion # from a failed retry cannot complete the current dispatch. - ${cli} orchestration send --to ${params.coordinatorHandle} --from ${params.workerHandle} \\ + ${cli} orchestration send --from ${params.workerHandle}${capabilityFlag} \\ --type worker_done --subject "" \\ --body "<3-sentence summary: what you did, what you found, what's left>" \\ - --task-id ${params.taskId} --dispatch-id ${params.dispatchId} \\ + --task-id ${params.taskId} --dispatch-id ${params.dispatchId} --outcome succeeded \\ --files-modified "path/a,path/b" \\ --report-path "" @@ -91,7 +96,7 @@ Slack, GitHub comments, or any other channel to reach a human during the run. # attributes the heartbeat to the specific dispatch context, not just # the task, so a straggler heartbeat from a previously-failed dispatch # cannot mask a hung retry. - ${cli} orchestration send --to ${params.coordinatorHandle} --from ${params.workerHandle} \\ + ${cli} orchestration send --from ${params.workerHandle}${capabilityFlag} \\ --type heartbeat --subject "alive" \\ --task-id ${params.taskId} --dispatch-id ${params.dispatchId} \\ --phase "" @@ -104,18 +109,18 @@ Slack, GitHub comments, or any other channel to reach a human during the run. # coordinator cannot see and cannot answer — your session will hang forever # waiting on a human. Every interactive question goes through \`ask\` below. # - # The \`ask\` verb is a thin wrapper: it sends a decision_gate message and - # blocks on \`check --wait\` until the coordinator replies, then prints the - # reply body. Use it anywhere you would otherwise have reached for - # AskUserQuestion. - ${cli} orchestration ask --to ${params.coordinatorHandle} --from ${params.workerHandle} \\ + # The \`ask\` verb durably records a question in this Dispatch's Run and + # blocks until the coordinator replies, then prints the reply body. If the + # call times out or disconnects, resume with the returned message ID instead + # of creating a duplicate question. + ${cli} orchestration ask --from ${params.workerHandle}${capabilityFlag} \\ --question "" \\ --options "" \\ --timeout-ms 600000 # Escalate a blocker or failure (pre-completion, when you need the # coordinator to do something before you can continue): - ${cli} orchestration send --to ${params.coordinatorHandle} --from ${params.workerHandle} \\ + ${cli} orchestration send --from ${params.workerHandle}${capabilityFlag} \\ --type escalation --subject "Blocked: " \\ --body "
" \\ --task-id ${params.taskId} diff --git a/src/main/runtime/orchestration/setup-completion-signal.test.ts b/src/main/runtime/orchestration/setup-completion-signal.test.ts new file mode 100644 index 00000000000..0f2b3976f1e --- /dev/null +++ b/src/main/runtime/orchestration/setup-completion-signal.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it, vi } from 'vitest' +import { buildObservedSetupCommand, createSetupCompletionScanner } from './setup-completion-signal' + +describe('orchestration setup completion signal', () => { + it('preserves a POSIX setup exit code in a visible completion signal', () => { + const { command } = buildObservedSetupCommand( + '/repo/.git/orca/setup-runner.sh', + 'posix', + 'token-posix' + ) + + expect(command).toContain('bash /repo/.git/orca/setup-runner.sh') + expect(command).toContain('__ORCA_SETUP_COMPLETE__:token-posix:%s\\n') + expect(command).toContain('"$status"') + expect(command).toContain('exit "$status"') + }) + + it('preserves a native Windows setup path and exit code without shell interpolation', () => { + const runnerPath = 'C:\\repo %name%!^&\\.git\\orca\\setup-runner.cmd' + const observed = buildObservedSetupCommand(runnerPath, 'windows', 'token-windows') + const encodedCommand = observed.command.split(' ').at(-1) + const script = Buffer.from(encodedCommand ?? '', 'base64').toString('utf16le') + + expect(observed.command).toContain('powershell.exe -NoLogo -NoProfile -NonInteractive') + expect(observed.env).toEqual({ ORCA_SETUP_RUNNER_PATH: runnerPath }) + expect(script).toContain('& $runner') + expect(script).toContain('__ORCA_SETUP_COMPLETE__:token-windows:') + expect(script).toContain('exit $status') + expect(script).not.toContain(runnerPath) + }) + + it('keeps a WSL runner on the POSIX completion path', () => { + const { command } = buildObservedSetupCommand( + '\\\\wsl.localhost\\Ubuntu\\repo\\.git\\orca\\setup-runner.sh', + 'windows', + 'token-wsl' + ) + + expect(command).toContain('bash /repo/.git/orca/setup-runner.sh') + expect(command).toContain('__ORCA_SETUP_COMPLETE__:token-wsl:%s\\n') + expect(command).toContain('exit "$status"') + }) + + it('recognizes one completion signal across output chunk boundaries', () => { + const onComplete = vi.fn() + const scanner = createSetupCompletionScanner('token-chunks', onComplete) + + scanner.scan('installing...\r\n__ORCA_SETUP_COMPLETE__:wrong:0\r\n__ORCA_SETUP_COMP') + scanner.scan('LETE__:token-chunks:1') + expect(onComplete).not.toHaveBeenCalled() + scanner.scan('7\r') + expect(onComplete).not.toHaveBeenCalled() + scanner.scan('\nPS C:\\repo>') + scanner.scan('__ORCA_SETUP_COMPLETE__:token-chunks:0\r\n') + + expect(onComplete).toHaveBeenCalledOnce() + expect(onComplete).toHaveBeenCalledWith(17) + }) +}) diff --git a/src/main/runtime/orchestration/setup-completion-signal.ts b/src/main/runtime/orchestration/setup-completion-signal.ts new file mode 100644 index 00000000000..17819c12a0c --- /dev/null +++ b/src/main/runtime/orchestration/setup-completion-signal.ts @@ -0,0 +1,80 @@ +import { + resolveSetupRunnerCommand, + type SetupRunnerCommandPlatform +} from '../../../shared/setup-runner-command' + +const SETUP_COMPLETION_PREFIX = '__ORCA_SETUP_COMPLETE__:' +const SETUP_COMPLETION_CARRY_LENGTH = SETUP_COMPLETION_PREFIX.length + 96 +const WINDOWS_SETUP_RUNNER_ENV = 'ORCA_SETUP_RUNNER_PATH' + +export function buildObservedSetupCommand( + runnerScriptPath: string, + platform: SetupRunnerCommandPlatform, + completionToken: string +): { command: string; env?: Record } { + const resolution = resolveSetupRunnerCommand(runnerScriptPath, platform) + if (resolution.shell === 'windows') { + const script = [ + `$runner = $env:${WINDOWS_SETUP_RUNNER_ENV}`, + '& $runner', + '$succeeded = $?', + '$status = $LASTEXITCODE', + 'if ($null -eq $status) { $status = if ($succeeded) { 0 } else { 1 } }', + `Write-Output ('${completionPrefix(completionToken)}' + $status)`, + 'exit $status' + ].join('; ') + return { + command: `powershell.exe -NoLogo -NoProfile -NonInteractive -EncodedCommand ${Buffer.from( + script, + 'utf16le' + ).toString('base64')}`, + env: { [WINDOWS_SETUP_RUNNER_ENV]: resolution.runnerScriptPathForShell } + } + } + + const script = [ + `( ${resolution.command} )`, + 'status=$?', + `printf '\\n${completionPrefix(completionToken)}%s\\n' "$status"`, + 'exit "$status"' + ].join('; ') + return { command: `bash -lc ${quotePosixArg(script)}` } +} + +export function createSetupCompletionScanner( + completionToken: string, + onComplete: (exitCode: number) => void +): { + scan: (data: string) => void +} { + const expectedPrefix = completionPrefix(completionToken) + let carry = '' + let completed = false + return { + scan(data: string): void { + if (completed || data.length === 0) { + return + } + const combined = `${carry}${data}` + const markerIndex = combined.lastIndexOf(expectedPrefix) + if (markerIndex >= 0) { + const suffix = combined.slice(markerIndex + expectedPrefix.length) + const match = suffix.match(/^(-?\d+)\r?\n/) + if (match) { + completed = true + onComplete(Number.parseInt(match[1], 10)) + return + } + } + carry = combined.slice(-SETUP_COMPLETION_CARRY_LENGTH) + } + } +} + +function completionPrefix(completionToken: string): string { + return `${SETUP_COMPLETION_PREFIX}${completionToken}:` +} + +function quotePosixArg(value: string): string { + return `'${value.replace(/'/g, `'\\''`)}'` +} diff --git a/src/main/runtime/orchestration/types.ts b/src/main/runtime/orchestration/types.ts index 58b86e41b08..eac7868da20 100644 --- a/src/main/runtime/orchestration/types.ts +++ b/src/main/runtime/orchestration/types.ts @@ -1,12 +1,16 @@ -export type MessageType = - | 'status' - | 'dispatch' - | 'worker_done' - | 'merge_ready' - | 'escalation' - | 'handoff' - | 'decision_gate' - | 'heartbeat' +export const MESSAGE_TYPES = [ + 'status', + 'dispatch', + 'worker_done', + 'merge_ready', + 'escalation', + 'handoff', + 'decision_gate', + 'question', + 'heartbeat' +] as const + +export type MessageType = (typeof MESSAGE_TYPES)[number] export type MessagePriority = 'normal' | 'high' | 'urgent' @@ -14,12 +18,158 @@ export type TaskStatus = 'pending' | 'ready' | 'dispatched' | 'completed' | 'fai export type DispatchStatus = 'pending' | 'dispatched' | 'completed' | 'failed' | 'circuit_broken' +export type WorkerReportOutcome = 'succeeded' | 'failed' + +export type WorkerReportSettlement = + | { action: 'settled'; outcome: WorkerReportOutcome; duplicate: boolean } + | { + action: 'rejected' + code: + | 'unknown_task' + | 'unknown_dispatch' + | 'task_dispatch_mismatch' + | 'inactive_dispatch' + | 'stale_dispatch' + reason: string + } + export type GateStatus = 'pending' | 'resolved' | 'timeout' export type CoordinatorStatus = 'idle' | 'running' | 'completed' | 'failed' +export type RunRow = { + id: string + objective: string + home_database: string + coordinator_handle: string | null + coordinator_pane_key: string | null + consumer_generation: number + legacy: number + created_at: string + updated_at: string +} + +export type DeliveryStatus = 'outstanding' | 'acknowledged' | 'fenced' + +export type DeliveryRow = { + id: string + run_id: string + consumer_generation: number + message_ids: string + status: DeliveryStatus + created_at: string + acknowledged_at: string | null +} + +export type QuestionStatus = 'pending' | 'answered' | 'closed' + +export type QuestionRow = { + message_id: string + run_id: string + dispatch_id: string + asker_handle: string + status: QuestionStatus + answer_message_id: string | null + answer_body: string | null + answered_by_generation: number | null + created_at: string + answered_at: string | null + closed_at: string | null +} + +export type MutationState = 'pending' | 'completed' + +export type MutationReceiptRow = { + caller_fingerprint: string + request_id: string + method: string + payload_hash: string + state: MutationState + receipt: string | null + created_at: string + updated_at: string +} + +export type WorkerDispatchState = + | 'starting' + | 'ready' + | 'start_unknown' + | 'failed' + | 'succeeded' + | 'stopping' + | 'stop_unknown' + | 'stopped' + | 'abandoned' + +export type WorkerDispatchRow = { + dispatch_id: string + runtime_epoch: string | null + state: WorkerDispatchState + stage: string + worktree_id: string | null + agent_terminal_handle: string | null + setup_state: string + effects: string + residual_resources: string + start_options: string + last_error: string | null + created_at: string + updated_at: string +} + +export type FederatedDispatchRow = { + dispatch_id: string + environment_id: string + environment_name: string + peer_fingerprint: string + remote_runtime_epoch: string | null + protocol_version: number + remote_worktree_id: string | null + remote_terminal_handle: string | null + to_home_imported_sequence: number + created_at: string + updated_at: string +} + +export type RemoteDispatchAttachmentRow = { + dispatch_id: string + task_id: string + home_peer_fingerprint: string + protocol_version: number + runtime_epoch: string + capability_hash: string | null + pane_key: string | null + process_incarnation: string | null + state: WorkerDispatchState + stage: string + worktree_id: string | null + terminal_handle: string | null + setup_state: string + effects: string + residual_resources: string + to_worker_imported_sequence: number + last_error: string | null + created_at: string + updated_at: string +} + +export type FederationRelayDirection = 'to_home' | 'to_worker' + +export type FederationRelayItemRow = { + dispatch_id: string + direction: FederationRelayDirection + sequence: number + message_id: string + kind: string + payload: string + byte_count: number + acked_at: string | null + created_at: string +} + export type MessageRow = { id: string + run_id: string from_handle: string to_handle: string subject: string @@ -37,6 +187,7 @@ export type MessageRow = { export type TaskRow = { id: string + run_id: string parent_id: string | null created_by_terminal_handle: string | null task_title: string | null @@ -51,9 +202,13 @@ export type TaskRow = { export type DispatchContextRow = { id: string + run_id: string task_id: string assignee_handle: string | null assignee_pane_key: string | null + capability_hash: string | null + process_incarnation: string | null + capability_revoked_at: string | null status: DispatchStatus failure_count: number last_failure: string | null @@ -65,6 +220,7 @@ export type DispatchContextRow = { export type DecisionGateRow = { id: string + run_id: string task_id: string question: string options: string diff --git a/src/main/runtime/orchestration/worker-output-cursor.test.ts b/src/main/runtime/orchestration/worker-output-cursor.test.ts new file mode 100644 index 00000000000..e109699a8d7 --- /dev/null +++ b/src/main/runtime/orchestration/worker-output-cursor.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from 'vitest' +import { decodeWorkerOutputCursor, encodeWorkerOutputCursor } from './worker-output-cursor' + +describe('worker output cursors', () => { + it('round-trips a source-pinned cursor without exposing source details', () => { + const cursor = encodeWorkerOutputCursor('dispatch_1', 'transcript', 'source_digest', 42) + + expect(cursor).toMatch(/^owr1_/) + expect(cursor).not.toContain('source_digest') + expect(decodeWorkerOutputCursor(cursor, 'dispatch_1')).toEqual({ + source: 'transcript', + sourceIdentity: 'source_digest', + position: 42, + legacy: false + }) + }) + + it('accepts legacy numeric terminal cursors', () => { + expect(decodeWorkerOutputCursor(0, 'dispatch_1')).toEqual({ + source: 'terminal', + sourceIdentity: null, + position: 0, + legacy: true + }) + expect(decodeWorkerOutputCursor('17', 'dispatch_1')).toMatchObject({ + source: 'terminal', + position: 17, + legacy: true + }) + }) + + it('rejects another Dispatch and malformed cursor data', () => { + const cursor = encodeWorkerOutputCursor('dispatch_1', 'terminal', 'terminal_digest', 1) + + expect(() => decodeWorkerOutputCursor(cursor, 'dispatch_2')).toThrow( + expect.objectContaining({ code: 'cursor_dispatch_mismatch' }) + ) + expect(() => decodeWorkerOutputCursor('owr1_not-json', 'dispatch_1')).toThrow( + expect.objectContaining({ code: 'cursor_invalid' }) + ) + }) +}) diff --git a/src/main/runtime/orchestration/worker-output-cursor.ts b/src/main/runtime/orchestration/worker-output-cursor.ts new file mode 100644 index 00000000000..31e67f4b5f9 --- /dev/null +++ b/src/main/runtime/orchestration/worker-output-cursor.ts @@ -0,0 +1,122 @@ +import { createHash } from 'node:crypto' +import { OrchestrationError } from './orchestration-error' + +const WORKER_OUTPUT_CURSOR_PREFIX = 'owr1_' +const WORKER_OUTPUT_CURSOR_MAX_LENGTH = 2_048 + +type WorkerOutputCursorPayload = { + v: 1 + d: string + s: 'terminal' | 'transcript' + i: string + p: number +} + +export type DecodedWorkerOutputCursor = + | { + source: 'terminal' + sourceIdentity: string | null + position: number + legacy: boolean + } + | { + source: 'transcript' + sourceIdentity: string + position: number + legacy: false + } + +export function createWorkerOutputSourceIdentity(fields: readonly string[]): string { + return createHash('sha256').update(JSON.stringify(fields)).digest('base64url').slice(0, 32) +} + +export function encodeWorkerOutputCursor( + dispatchId: string, + source: WorkerOutputCursorPayload['s'], + sourceIdentity: string, + position: number +): string { + const payload: WorkerOutputCursorPayload = { + v: 1, + d: dispatchId, + s: source, + i: sourceIdentity, + p: position + } + return `${WORKER_OUTPUT_CURSOR_PREFIX}${Buffer.from(JSON.stringify(payload)).toString('base64url')}` +} + +export function decodeWorkerOutputCursor( + cursor: string | number | undefined, + dispatchId: string +): DecodedWorkerOutputCursor | null { + if (cursor === undefined) { + return null + } + if (typeof cursor === 'number') { + return decodeLegacyTerminalCursor(cursor) + } + if (/^\d+$/.test(cursor)) { + return decodeLegacyTerminalCursor(Number.parseInt(cursor, 10)) + } + if ( + cursor.length > WORKER_OUTPUT_CURSOR_MAX_LENGTH || + !cursor.startsWith(WORKER_OUTPUT_CURSOR_PREFIX) + ) { + throw invalidCursor() + } + let parsed: unknown + try { + parsed = JSON.parse( + Buffer.from(cursor.slice(WORKER_OUTPUT_CURSOR_PREFIX.length), 'base64url').toString('utf8') + ) + } catch { + throw invalidCursor() + } + if (!isWorkerOutputCursorPayload(parsed)) { + throw invalidCursor() + } + if (parsed.d !== dispatchId) { + throw new OrchestrationError( + 'cursor_dispatch_mismatch', + 'The worker-read cursor belongs to a different Dispatch.' + ) + } + return { + source: parsed.s, + sourceIdentity: parsed.i, + position: parsed.p, + legacy: false + } +} + +function decodeLegacyTerminalCursor(position: number): DecodedWorkerOutputCursor { + if (!Number.isSafeInteger(position) || position < 0) { + throw invalidCursor() + } + return { source: 'terminal', sourceIdentity: null, position, legacy: true } +} + +function isWorkerOutputCursorPayload(value: unknown): value is WorkerOutputCursorPayload { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + return false + } + const payload = value as Record + return ( + payload.v === 1 && + typeof payload.d === 'string' && + payload.d.length > 0 && + payload.d.length <= 512 && + (payload.s === 'terminal' || payload.s === 'transcript') && + typeof payload.i === 'string' && + payload.i.length > 0 && + payload.i.length <= 128 && + typeof payload.p === 'number' && + Number.isSafeInteger(payload.p) && + payload.p >= 0 + ) +} + +function invalidCursor(): OrchestrationError { + return new OrchestrationError('cursor_invalid', 'The worker-read cursor is invalid.') +} diff --git a/src/main/runtime/orchestration/worker-provider-session.test.ts b/src/main/runtime/orchestration/worker-provider-session.test.ts new file mode 100644 index 00000000000..0d36c65e732 --- /dev/null +++ b/src/main/runtime/orchestration/worker-provider-session.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it } from 'vitest' +import type { AgentStatusIpcPayload } from '../../../shared/agent-status-types' +import { selectExactWorkerProviderSession } from './worker-provider-session' + +function status( + paneKey: string, + sessionId: string, + overrides: Partial = {} +): AgentStatusIpcPayload { + return { + paneKey, + connectionId: null, + receivedAt: 200, + stateStartedAt: 190, + state: 'working', + prompt: '', + agentType: 'codex', + providerSession: { key: 'session_id', id: sessionId }, + ...overrides + } +} + +describe('exact worker provider session selection', () => { + it('selects only the current pane, connection, and observation window', () => { + const selected = selectExactWorkerProviderSession({ + paneKey: 'tab:worker', + processIncarnation: 'pty:incarnation', + connectionId: 'ssh-windows', + launchToken: undefined, + observedAfter: 150, + statuses: [ + status('tab:sibling', 'sibling', { connectionId: 'ssh-windows', receivedAt: 300 }), + status('tab:worker', 'old', { connectionId: 'ssh-windows', receivedAt: 100 }), + status('tab:worker', 'wrong-host', { connectionId: 'ssh-mac', receivedAt: 400 }), + status('tab:worker', 'exact', { connectionId: 'ssh-windows', receivedAt: 250 }) + ] + }) + + expect(selected).toEqual({ + paneKey: 'tab:worker', + processIncarnation: 'pty:incarnation', + agent: 'codex', + providerSession: { key: 'session_id', id: 'exact' }, + observedAt: 250 + }) + }) + + it('rejects stale and provider-session-only rows', () => { + expect( + selectExactWorkerProviderSession({ + paneKey: 'tab:worker', + processIncarnation: 'pty:incarnation', + connectionId: null, + launchToken: undefined, + observedAfter: 300, + statuses: [ + status('tab:worker', 'stale', { receivedAt: 200 }), + status('tab:worker', 'identity-only', { + receivedAt: 400, + providerSessionOnly: true + }) + ] + }) + ).toBeNull() + }) + + it('rejects a prior process snapshot when the launch token changed', () => { + expect( + selectExactWorkerProviderSession({ + paneKey: 'tab:worker', + processIncarnation: 'pty:new-incarnation', + connectionId: null, + launchToken: 'launch-new', + observedAfter: 0, + statuses: [status('tab:worker', 'prior', { launchToken: 'launch-old' })] + }) + ).toBeNull() + }) +}) diff --git a/src/main/runtime/orchestration/worker-provider-session.ts b/src/main/runtime/orchestration/worker-provider-session.ts new file mode 100644 index 00000000000..eb3e7064583 --- /dev/null +++ b/src/main/runtime/orchestration/worker-provider-session.ts @@ -0,0 +1,34 @@ +import type { AgentStatusIpcPayload } from '../../../shared/agent-status-types' +import type { ExactWorkerProviderSession } from '../../../shared/orchestration-worker-output' + +export function selectExactWorkerProviderSession(args: { + paneKey: string + processIncarnation: string + connectionId: string | null | undefined + launchToken: string | null | undefined + observedAfter: number + statuses: readonly AgentStatusIpcPayload[] +}): ExactWorkerProviderSession | null { + const status = args.statuses + .filter( + (entry) => + entry.paneKey === args.paneKey && + (args.connectionId === undefined || entry.connectionId === args.connectionId) && + (!args.launchToken || entry.launchToken === args.launchToken) && + entry.providerSessionOnly !== true && + entry.providerSession !== undefined && + entry.agentType !== undefined && + entry.receivedAt >= args.observedAfter + ) + .sort((left, right) => right.receivedAt - left.receivedAt)[0] + if (!status?.providerSession || !status.agentType) { + return null + } + return { + paneKey: args.paneKey, + processIncarnation: args.processIncarnation, + agent: status.agentType, + providerSession: { ...status.providerSession }, + observedAt: status.receivedAt + } +} diff --git a/src/main/runtime/orchestration/worker-transcript-payload.test.ts b/src/main/runtime/orchestration/worker-transcript-payload.test.ts new file mode 100644 index 00000000000..94f9506db09 --- /dev/null +++ b/src/main/runtime/orchestration/worker-transcript-payload.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, it } from 'vitest' +import { + boundWorkerTranscriptMessages, + redactWorkerTerminalLines +} from './worker-transcript-payload' + +describe('worker transcript wire bounds', () => { + it('clips oversized blocks and omits local image paths', () => { + const result = boundWorkerTranscriptMessages([ + { + id: 'message-1', + role: 'assistant', + timestamp: null, + source: 'transcript', + blocks: [ + { type: 'text', text: 'x'.repeat(5_000) }, + { type: 'image-ref', path: 'C:\\Users\\worker\\secret.png', alt: 'screenshot' } + ] + } + ]) + + expect(result.messages[0]?.blocks[0]).toMatchObject({ + type: 'text', + text: expect.stringContaining('… (truncated)') + }) + expect(result.messages[0]?.blocks[1]).toEqual({ + type: 'image-ref', + alt: 'screenshot' + }) + expect(JSON.stringify(result)).not.toContain('C:\\\\Users') + expect(result.warnings).toContain('Local image paths were omitted from transcript output.') + }) + + it('keeps fallback identifiers stable without exposing the transcript path', () => { + const transcriptPath = 'C:\\Users\\worker\\.codex\\session.jsonl' + const message = { + id: `${transcriptPath}:0000000000000042`, + turnId: `${transcriptPath}:0000000000000001`, + role: 'assistant' as const, + timestamp: null, + source: 'transcript' as const, + blocks: [{ type: 'image-ref' as const, url: `file:///${transcriptPath}` }] + } + + const first = boundWorkerTranscriptMessages([message], transcriptPath) + const second = boundWorkerTranscriptMessages([message], transcriptPath) + + expect(first.messages).toEqual(second.messages) + expect(first.messages[0]?.id).toMatch(/^worker-message-/) + expect(first.messages[0]?.turnId).toMatch(/^worker-message-/) + expect(first.messages[0]?.blocks[0]).toEqual({ type: 'image-ref' }) + expect(JSON.stringify(first)).not.toContain('Users') + expect(first.warnings).toEqual( + expect.arrayContaining([ + 'Transcript-backed message identifiers were made opaque.', + 'Local image paths were omitted from transcript output.' + ]) + ) + }) + + it('redacts dispatch capabilities from prose and tool payloads', () => { + const capability = `dcap_${'A'.repeat(43)}` + const result = boundWorkerTranscriptMessages([ + { + id: 'message-secret', + role: 'assistant', + timestamp: null, + source: 'transcript', + blocks: [ + { type: 'text', text: `Use --dispatch-capability ${capability}` }, + { + type: 'tool-call', + name: 'exec_command', + input: { + cmd: `orca orchestration send --dispatch-capability ${capability}`, + [capability]: 'secret key' + } + }, + { type: 'tool-result', output: `echoed ${capability}` } + ] + } + ]) + + expect(JSON.stringify(result)).not.toContain(capability) + expect(JSON.stringify(result.messages)).toContain('[dispatch capability redacted]') + expect(result.warnings).toContain( + 'Dispatch capability tokens were redacted from transcript output.' + ) + }) + + it('redacts dispatch capabilities from terminal fallback lines', () => { + const capability = `dcap_${'A'.repeat(43)}` + + expect(redactWorkerTerminalLines([`send --dispatch-capability ${capability}`, 'safe'])).toEqual( + { + lines: ['send --dispatch-capability [dispatch capability redacted]', 'safe'], + warnings: ['Dispatch capability tokens were redacted from terminal output.'] + } + ) + }) +}) diff --git a/src/main/runtime/orchestration/worker-transcript-payload.ts b/src/main/runtime/orchestration/worker-transcript-payload.ts new file mode 100644 index 00000000000..e4a5c0b3a58 --- /dev/null +++ b/src/main/runtime/orchestration/worker-transcript-payload.ts @@ -0,0 +1,226 @@ +import { createHash } from 'node:crypto' +import type { NativeChatBlock, NativeChatMessage } from '../../../shared/native-chat-types' + +export const DEFAULT_WORKER_TRANSCRIPT_MESSAGE_LIMIT = 40 +export const MAX_WORKER_TRANSCRIPT_MESSAGE_LIMIT = 50 +const MAX_WORKER_TRANSCRIPT_BLOCKS = 6 +const MAX_WORKER_TRANSCRIPT_BLOCK_CHARS = 1_200 +const MAX_WORKER_TRANSCRIPT_INPUT_ITEMS = 20 +const MAX_WORKER_TRANSCRIPT_INPUT_NODES = 100 +const MAX_WORKER_TRANSCRIPT_RESPONSE_BYTES = 512 * 1024 +const TRUNCATION_MARKER = '\n… (truncated)' +const DISPATCH_CAPABILITY_PATTERN = /\bdcap_[A-Za-z0-9_-]{20,}\b/g +const DISPATCH_CAPABILITY_REDACTION = '[dispatch capability redacted]' + +export function clampWorkerTranscriptLimit(limit: number | undefined): number { + if (!Number.isFinite(limit) || (limit ?? 0) <= 0) { + return DEFAULT_WORKER_TRANSCRIPT_MESSAGE_LIMIT + } + return Math.min(Math.floor(limit!), MAX_WORKER_TRANSCRIPT_MESSAGE_LIMIT) +} + +export function redactWorkerTerminalLines(lines: readonly string[]): { + lines: string[] + warnings: string[] +} { + let redacted = false + const bounded = lines.map((line) => { + const result = replaceDispatchCapabilities(line) + redacted ||= result.redacted + return result.value + }) + return { + lines: bounded, + warnings: redacted ? ['Dispatch capability tokens were redacted from terminal output.'] : [] + } +} + +export function boundWorkerTranscriptMessages( + messages: readonly NativeChatMessage[], + transcriptPath?: string +): { + messages: NativeChatMessage[] + limited: boolean + warnings: string[] +} { + const warnings = new Set() + const bounded: NativeChatMessage[] = [] + let bytes = 2 + for (const message of messages) { + const next = boundMessage(message, transcriptPath, warnings) + const serializedBytes = Buffer.byteLength(JSON.stringify(next), 'utf8') + 1 + if (bounded.length > 0 && bytes + serializedBytes > MAX_WORKER_TRANSCRIPT_RESPONSE_BYTES) { + warnings.add('Transcript response was clipped to the wire-size limit.') + return { messages: bounded, limited: true, warnings: [...warnings] } + } + bounded.push(next) + bytes += serializedBytes + } + return { messages: bounded, limited: false, warnings: [...warnings] } +} + +function boundMessage( + message: NativeChatMessage, + transcriptPath: string | undefined, + warnings: Set +): NativeChatMessage { + const blocks = message.blocks.slice(0, MAX_WORKER_TRANSCRIPT_BLOCKS) + if (blocks.length < message.blocks.length) { + warnings.add('Some transcript blocks were omitted from oversized messages.') + } + return { + ...message, + id: boundIdentifier(message.id, transcriptPath, warnings), + ...(message.turnId + ? { turnId: boundIdentifier(message.turnId, transcriptPath, warnings) } + : {}), + blocks: blocks.map((block) => boundBlock(block, warnings)) + } +} + +function boundBlock(block: NativeChatBlock, warnings: Set): NativeChatBlock { + if (block.type === 'text') { + return { ...block, text: clipText(block.text, warnings) } + } + if (block.type === 'tool-result') { + return { ...block, output: clipText(block.output, warnings) } + } + if (block.type === 'tool-call') { + const budget = { + remaining: MAX_WORKER_TRANSCRIPT_BLOCK_CHARS, + nodes: MAX_WORKER_TRANSCRIPT_INPUT_NODES + } + return { + ...block, + name: clipMetadata(block.name, warnings), + input: boundToolInput(block.input, budget, 0, warnings) + } + } + if (block.path || (block.url && isLocalFileLocator(block.url))) { + warnings.add('Local image paths were omitted from transcript output.') + return { + type: 'image-ref', + ...(block.alt ? { alt: clipText(block.alt, warnings) } : {}) + } + } + return { + ...block, + ...(block.url ? { url: clipMetadata(block.url, warnings) } : {}), + ...(block.alt ? { alt: clipText(block.alt, warnings) } : {}) + } +} + +function boundIdentifier( + value: string, + transcriptPath: string | undefined, + warnings: Set +): string { + if (transcriptPath && value.includes(transcriptPath)) { + warnings.add('Transcript-backed message identifiers were made opaque.') + return `worker-message-${createHash('sha256').update(value).digest('base64url').slice(0, 32)}` + } + return clipMetadata(value, warnings) +} + +function isLocalFileLocator(value: string): boolean { + return ( + /^file:/i.test(value) || + /^[a-z]:[\\/]/i.test(value) || + value.startsWith('/') || + value.startsWith('\\\\') + ) +} + +function clipMetadata(value: string, warnings: Set): string { + const redacted = redactSensitiveText(value, warnings) + if (redacted.length <= 512) { + return redacted + } + warnings.add('Oversized transcript metadata was clipped.') + return redacted.slice(0, 512) +} + +function clipText(value: string, warnings: Set): string { + const redacted = redactSensitiveText(value, warnings) + if (redacted.length <= MAX_WORKER_TRANSCRIPT_BLOCK_CHARS) { + return redacted + } + warnings.add('Oversized transcript text was clipped.') + return `${redacted.slice(0, MAX_WORKER_TRANSCRIPT_BLOCK_CHARS)}${TRUNCATION_MARKER}` +} + +function boundToolInput( + value: unknown, + budget: { remaining: number; nodes: number }, + depth: number, + warnings: Set +): unknown { + budget.nodes-- + if (budget.nodes < 0 || budget.remaining <= 0) { + warnings.add('Oversized tool input was clipped.') + return '… (truncated)' + } + if (typeof value === 'string') { + const redacted = redactSensitiveText(value, warnings) + const length = Math.min(redacted.length, budget.remaining) + budget.remaining -= length + if (length < redacted.length) { + warnings.add('Oversized tool input was clipped.') + return `${redacted.slice(0, length)}… (truncated)` + } + return redacted + } + if (!value || typeof value !== 'object') { + return value + } + if (depth >= 5) { + warnings.add('Deep tool input was clipped.') + return '… (truncated)' + } + if (Array.isArray(value)) { + const result = value + .slice(0, MAX_WORKER_TRANSCRIPT_INPUT_ITEMS) + .map((item) => boundToolInput(item, budget, depth + 1, warnings)) + if (value.length > MAX_WORKER_TRANSCRIPT_INPUT_ITEMS) { + warnings.add('Oversized tool input was clipped.') + result.push('… (truncated)') + } + return result + } + const result: Record = Object.create(null) + let count = 0 + for (const [rawKey, entry] of Object.entries(value)) { + if (count >= MAX_WORKER_TRANSCRIPT_INPUT_ITEMS || budget.remaining <= 0) { + warnings.add('Oversized tool input was clipped.') + result['…'] = 'truncated' + break + } + const redactedKey = redactSensitiveText(rawKey, warnings) + const key = redactedKey.slice(0, Math.min(redactedKey.length, budget.remaining, 128)) + budget.remaining -= key.length + result[key] = boundToolInput(entry, budget, depth + 1, warnings) + count++ + } + return result +} + +function redactSensitiveText(value: string, warnings: Set): string { + const result = replaceDispatchCapabilities(value) + if (!result.redacted) { + return result.value + } + warnings.add('Dispatch capability tokens were redacted from transcript output.') + return result.value +} + +function replaceDispatchCapabilities(value: string): { value: string; redacted: boolean } { + DISPATCH_CAPABILITY_PATTERN.lastIndex = 0 + const redacted = DISPATCH_CAPABILITY_PATTERN.test(value) + DISPATCH_CAPABILITY_PATTERN.lastIndex = 0 + return { + value: redacted + ? value.replace(DISPATCH_CAPABILITY_PATTERN, DISPATCH_CAPABILITY_REDACTION) + : value, + redacted + } +} diff --git a/src/main/runtime/orchestration/worker-transcript-read.test.ts b/src/main/runtime/orchestration/worker-transcript-read.test.ts new file mode 100644 index 00000000000..0bcceda11f5 --- /dev/null +++ b/src/main/runtime/orchestration/worker-transcript-read.test.ts @@ -0,0 +1,188 @@ +import { appendFile, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { readWorkerTranscript } from './worker-transcript-read' + +function codexMessage(id: string, text: string): string { + return JSON.stringify({ + timestamp: '2026-07-24T12:00:00.000Z', + type: 'event_msg', + payload: { id, type: 'agent_message', message: text } + }) +} + +function grokMessage(id: string, text: string): string { + return JSON.stringify({ + id, + timestamp: '2026-07-24T12:00:00.000Z', + type: 'assistant', + content: text + }) +} + +describe('worker transcript reads', () => { + let directory: string + let transcriptPath: string + + beforeEach(async () => { + directory = await mkdtemp(join(tmpdir(), 'orca-worker-transcript-')) + transcriptPath = join(directory, 'rollout-session.jsonl') + }) + + afterEach(async () => { + await rm(directory, { recursive: true, force: true }) + }) + + it('returns a bounded tail followed by new messages from the exact file', async () => { + await writeFile( + transcriptPath, + [codexMessage('one', 'first'), codexMessage('two', 'second'), codexMessage('three', 'third')] + .join('\n') + .concat('\n') + ) + + const initial = await readWorkerTranscript({ + agent: 'codex', + sessionId: 'session-exact', + transcriptPath, + limit: 2 + }) + expect(initial).toMatchObject({ + ok: true, + messages: [ + { id: 'two', blocks: [{ type: 'text', text: 'second' }] }, + { id: 'three', blocks: [{ type: 'text', text: 'third' }] } + ], + limited: true + }) + if (!initial.ok) { + throw new Error('Expected the initial transcript page') + } + + await appendFile(transcriptPath, `{malformed}\n${codexMessage('four', 'fourth')}\n`) + const appended = await readWorkerTranscript({ + agent: 'codex', + sessionId: 'session-exact', + transcriptPath, + offset: initial.nextOffset, + limit: 2 + }) + + expect(appended).toMatchObject({ + ok: true, + messages: [{ id: 'four', blocks: [{ type: 'text', text: 'fourth' }] }], + limited: false, + warnings: ['1 malformed transcript record(s) were skipped.'] + }) + }) + + it('reports source changes and unsupported providers without guessing', async () => { + await writeFile(transcriptPath, `${codexMessage('one', 'first')}\n`) + + await expect( + readWorkerTranscript({ + agent: 'codex', + sessionId: 'session-exact', + transcriptPath, + offset: 10_000, + limit: 2 + }) + ).resolves.toMatchObject({ ok: false, reason: 'source_changed' }) + + await expect( + readWorkerTranscript({ + agent: 'gemini', + sessionId: 'session-other', + transcriptPath, + limit: 2 + }) + ).resolves.toEqual({ ok: false, reason: 'provider_unsupported', warnings: [] }) + }) + + it('reuses the Native Chat Grok decoder', async () => { + await writeFile(transcriptPath, `${grokMessage('grok-one', 'Grok structured output')}\n`) + + await expect( + readWorkerTranscript({ + agent: 'grok', + sessionId: 'session-grok', + transcriptPath, + limit: 2 + }) + ).resolves.toMatchObject({ + ok: true, + messages: [ + { + role: 'assistant', + blocks: [{ type: 'text', text: 'Grok structured output' }] + } + ] + }) + }) + + it('makes file-position fallback IDs opaque', async () => { + await writeFile( + transcriptPath, + `${JSON.stringify({ + timestamp: '2026-07-24T12:00:00.000Z', + type: 'event_msg', + payload: { type: 'agent_message', message: 'no provider id' } + })}\n` + ) + + const result = await readWorkerTranscript({ + agent: 'codex', + sessionId: 'session-exact', + transcriptPath, + limit: 2 + }) + + expect(result).toMatchObject({ + ok: true, + messages: [{ id: expect.stringMatching(/^worker-message-/) }], + warnings: ['Transcript-backed message identifiers were made opaque.'] + }) + expect(result.ok && JSON.stringify(result.messages)).not.toContain(transcriptPath) + }) + + it('advances past a record larger than the forward scan window', async () => { + await writeFile(transcriptPath, 'x'.repeat(8 * 1024 * 1024 + 10)) + + const oversized = await readWorkerTranscript({ + agent: 'codex', + sessionId: 'session-exact', + transcriptPath, + offset: 0, + limit: 2 + }) + expect(oversized).toMatchObject({ + ok: true, + messages: [], + limited: true, + warnings: expect.arrayContaining([ + '1 oversized transcript record(s) were skipped.', + 'Transcript scanning stopped at the bounded byte limit; continue with the cursor.' + ]) + }) + if (!oversized.ok) { + throw new Error('Expected the oversized transcript page') + } + expect(oversized.nextOffset).toBe(8 * 1024 * 1024) + + await appendFile(transcriptPath, `\n${codexMessage('after', 'after oversized')}\n`) + const continued = await readWorkerTranscript({ + agent: 'codex', + sessionId: 'session-exact', + transcriptPath, + offset: oversized.nextOffset, + limit: 2 + }) + + expect(continued).toMatchObject({ + ok: true, + messages: [{ id: 'after', blocks: [{ type: 'text', text: 'after oversized' }] }], + limited: false + }) + }) +}) diff --git a/src/main/runtime/orchestration/worker-transcript-read.ts b/src/main/runtime/orchestration/worker-transcript-read.ts new file mode 100644 index 00000000000..d893c35f995 --- /dev/null +++ b/src/main/runtime/orchestration/worker-transcript-read.ts @@ -0,0 +1,263 @@ +import { open, stat } from 'node:fs/promises' +import type { AgentType, NativeChatMessage } from '../../../shared/native-chat-types' +import { resolveNativeChatTranscriptAgent } from '../../../shared/native-chat-agent-support' +import type { OrchestrationWorkerReadFallbackReason } from '../../../shared/orchestration-worker-output' +import { resolveSessionFilePath } from '../../native-chat/session-file-resolver' +import { + MAX_NATIVE_CHAT_TRANSCRIPT_RECORD_BYTES, + nativeChatLineDecoderForAgent, + readNativeChatTranscriptTailFile, + type NativeChatLineDecoder +} from '../../native-chat/transcript-tail-reader' +import { transcriptFallbackId } from '../../native-chat/transcript-fallback-id' +import { + boundWorkerTranscriptMessages, + clampWorkerTranscriptLimit +} from './worker-transcript-payload' + +const MAX_FORWARD_TRANSCRIPT_SCAN_BYTES = 8 * 1024 * 1024 + +type WorkerTranscriptReadFailure = { + ok: false + reason: OrchestrationWorkerReadFallbackReason | 'source_changed' + warnings: string[] +} + +type WorkerTranscriptReadSuccess = { + ok: true + filePath: string + messages: NativeChatMessage[] + nextOffset: number + limited: boolean + warnings: string[] +} + +export type WorkerTranscriptReadResult = WorkerTranscriptReadFailure | WorkerTranscriptReadSuccess + +export async function readWorkerTranscript(args: { + agent: AgentType + sessionId: string + transcriptPath?: string + offset?: number + limit?: number +}): Promise { + const transcriptAgent = resolveNativeChatTranscriptAgent(args.agent) + if (!transcriptAgent) { + return { ok: false, reason: 'provider_unsupported', warnings: [] } + } + const decode = nativeChatLineDecoderForAgent(args.agent) + if (!decode) { + return { ok: false, reason: 'provider_unsupported', warnings: [] } + } + let filePath: string | null + try { + filePath = await resolveSessionFilePath(args.agent, args.sessionId, { + transcriptPath: args.transcriptPath + }) + } catch { + return { ok: false, reason: 'transcript_unreadable', warnings: [] } + } + if (!filePath) { + return { ok: false, reason: 'transcript_missing', warnings: [] } + } + const limit = clampWorkerTranscriptLimit(args.limit) + try { + const page = + args.offset === undefined + ? await readInitialPage(filePath, limit, decode) + : await readForwardPage(filePath, args.offset, limit, decode) + if (!page.ok) { + return page + } + const bounded = boundWorkerTranscriptMessages(page.messages, filePath) + return { + ok: true, + filePath, + messages: bounded.messages, + nextOffset: page.nextOffset, + limited: page.limited || bounded.limited, + warnings: [...page.warnings, ...bounded.warnings] + } + } catch (error) { + const code = (error as NodeJS.ErrnoException | null)?.code + return { + ok: false, + reason: + code === 'ENOENT' + ? 'transcript_missing' + : code === 'EACCES' || code === 'EPERM' + ? 'transcript_unreadable' + : 'transcript_parse_failed', + warnings: [] + } + } +} + +async function readInitialPage( + filePath: string, + limit: number, + decode: NativeChatLineDecoder +): Promise { + const page = await readNativeChatTranscriptTailFile(filePath, limit, decode, false) + return { + ok: true, + filePath, + messages: page.messages, + nextOffset: page.consumedTo, + limited: page.hasMore, + warnings: recordWarnings(page.malformedRecordCount, page.oversizedRecordCount) + } +} + +async function readForwardPage( + filePath: string, + startOffset: number, + limit: number, + decode: NativeChatLineDecoder +): Promise { + const fileSize = (await stat(filePath)).size + if (startOffset > fileSize) { + return { ok: false, reason: 'source_changed', warnings: [] } + } + if (startOffset === fileSize) { + return { + ok: true, + filePath, + messages: [], + nextOffset: startOffset, + limited: false, + warnings: [] + } + } + const scanEnd = Math.min(fileSize, startOffset + MAX_FORWARD_TRANSCRIPT_SCAN_BYTES) + const handle = await open(filePath, 'r') + const messages: NativeChatMessage[] = [] + let pendingChunks: Buffer[] = [] + let pendingBytes = 0 + let pendingStart = startOffset + let droppingOversizedRecord = await startsInsideRecord(handle, startOffset) + let malformedRecordCount = 0 + let oversizedRecordCount = 0 + let nextOffset = startOffset + try { + const stream = handle.createReadStream({ + start: startOffset, + end: scanEnd - 1, + autoClose: false + }) + let absoluteOffset = startOffset + for await (const rawChunk of stream) { + const chunk = Buffer.isBuffer(rawChunk) ? rawChunk : Buffer.from(rawChunk) + let segmentStart = 0 + let newline = chunk.indexOf(0x0a) + while (newline >= 0) { + retainPart(chunk.subarray(segmentStart, newline)) + const lineEnd = absoluteOffset + newline + 1 + if (!droppingOversizedRecord) { + decodeLine() + } + resetLine(lineEnd) + nextOffset = lineEnd + if (messages.length >= limit) { + return successfulPage(lineEnd < fileSize) + } + segmentStart = newline + 1 + newline = chunk.indexOf(0x0a, segmentStart) + } + if (segmentStart < chunk.length) { + retainPart(chunk.subarray(segmentStart)) + } + absoluteOffset += chunk.length + } + if (droppingOversizedRecord) { + nextOffset = scanEnd + } + return successfulPage(scanEnd < fileSize, scanEnd < fileSize) + } finally { + await handle.close() + } + + function retainPart(part: Buffer): void { + if (droppingOversizedRecord) { + return + } + pendingBytes += part.length + if (pendingBytes > MAX_NATIVE_CHAT_TRANSCRIPT_RECORD_BYTES) { + pendingChunks = [] + droppingOversizedRecord = true + oversizedRecordCount++ + return + } + pendingChunks.push(part) + } + + function resetLine(nextStart: number): void { + pendingChunks = [] + pendingBytes = 0 + droppingOversizedRecord = false + pendingStart = nextStart + } + + function decodeLine(): void { + let line = Buffer.concat(pendingChunks).toString('utf8') + if (line.endsWith('\r')) { + line = line.slice(0, -1) + } + if (!line) { + return + } + try { + JSON.parse(line) + } catch { + malformedRecordCount++ + return + } + const message = decode(line, transcriptFallbackId(filePath, pendingStart)) + if (message) { + messages.push(message) + } + } + + function successfulPage(limited: boolean, scanLimited = false): WorkerTranscriptReadSuccess { + return { + ok: true, + filePath, + messages, + nextOffset, + limited, + warnings: recordWarnings(malformedRecordCount, oversizedRecordCount, scanLimited) + } + } +} + +async function startsInsideRecord( + handle: Awaited>, + offset: number +): Promise { + if (offset === 0) { + return false + } + const previousByte = Buffer.allocUnsafe(1) + const { bytesRead } = await handle.read(previousByte, 0, 1, offset - 1) + return bytesRead === 1 && previousByte[0] !== 0x0a +} + +function recordWarnings( + malformedRecordCount = 0, + oversizedRecordCount = 0, + scanLimited = false +): string[] { + const warnings: string[] = [] + if (malformedRecordCount > 0) { + warnings.push(`${malformedRecordCount} malformed transcript record(s) were skipped.`) + } + if (oversizedRecordCount > 0) { + warnings.push(`${oversizedRecordCount} oversized transcript record(s) were skipped.`) + } + if (scanLimited) { + warnings.push( + 'Transcript scanning stopped at the bounded byte limit; continue with the cursor.' + ) + } + return warnings +} diff --git a/src/main/runtime/rpc/core.ts b/src/main/runtime/rpc/core.ts index 061ff895a4a..e56e8600916 100644 --- a/src/main/runtime/rpc/core.ts +++ b/src/main/runtime/rpc/core.ts @@ -44,6 +44,9 @@ export type RpcRequest = { authToken: string method: string params?: unknown + orchestrationCapability?: string + orchestrationContractVersion?: number + orchestrationRequestId?: string } export type RpcContext = { @@ -60,6 +63,19 @@ export type RpcContext = { pairedDeviceId?: string // Why: lets handlers gate mobile payload truncation to phones only; undefined for in-process callers → treat as full-class (no clip). clientKind?: 'mobile' | 'runtime' + // Why: Dispatch authority rides in the authenticated RPC envelope, never in user payload fields. + orchestrationCapability?: string + // Why: long-lived mutations such as ask can durably expose acceptance before their waiter settles. + recordMutationReceipt?: (receipt: unknown) => void + // Why: worker-start commits this identity with its starting Dispatch so crash recovery always has an inspectable operation. + orchestrationMutation?: { + callerFingerprint: string + requestId: string + method: string + payloadHash: string + } + // Why: federation pins the authenticated saved-environment caller without exposing its token to handlers or storage. + authenticatedCallerFingerprint?: string pairing?: PairingRpcContext // Why: mobile terminal traffic bypasses JSON streaming; undefined on Unix/socket and non-E2EE WebSocket paths. sendBinary?: (bytes: Uint8Array) => boolean | void diff --git a/src/main/runtime/rpc/dispatcher-feature-interactions.test.ts b/src/main/runtime/rpc/dispatcher-feature-interactions.test.ts index 6b4bcf3455e..64dc5ee4f7f 100644 --- a/src/main/runtime/rpc/dispatcher-feature-interactions.test.ts +++ b/src/main/runtime/rpc/dispatcher-feature-interactions.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from 'vitest' import { z } from 'zod' import type { PersistedUIState } from '../../../shared/types' import { getDefaultUIState } from '../../../shared/constants' +import { ORCHESTRATION_CONTRACT_VERSION } from '../../../shared/protocol-version' import { ORCA_RUNTIME_RPC_BROWSER_UI_SOURCE, ORCA_RUNTIME_RPC_FEATURE_INTERACTION_SOURCE_KEY @@ -11,7 +12,15 @@ import { defineMethod, defineStreamingMethod, type RpcRequest } from './core' import type { OrcaRuntimeService } from '../orca-runtime' function makeRequest(method: string, params: unknown = {}): RpcRequest { - return { id: 'req-1', authToken: 'tok', method, params } + return { + id: 'req-1', + authToken: 'tok', + method, + params, + ...(method.startsWith('orchestration.') + ? { orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION } + : {}) + } } function makeRuntime(ui: PersistedUIState = getDefaultUIState()): OrcaRuntimeService { diff --git a/src/main/runtime/rpc/dispatcher.ts b/src/main/runtime/rpc/dispatcher.ts index 5b677088074..b90b697d2e7 100644 --- a/src/main/runtime/rpc/dispatcher.ts +++ b/src/main/runtime/rpc/dispatcher.ts @@ -15,9 +15,9 @@ import { type RpcRequest, type RpcResponse } from './core' + import type { TerminalStreamFrame } from '../../../shared/terminal-stream-protocol' import type { FeatureInteractionId } from '../../../shared/feature-interactions' -import { isBrowserPaneUiRuntimeRpcParams } from '../../../shared/runtime-rpc-feature-interaction-source' import { computerErrorData, errorResponse, @@ -29,6 +29,13 @@ import { import { ALL_RPC_METHODS } from './methods' import { emulatorProbe, emulatorProbeError } from '../../emulator/emulator-probe' import type { OrcaRuntimeService } from '../orca-runtime' +import { + OrchestrationMutationExecutor, + authenticatedCallerFingerprint, + type DurableMutationInvocation +} from './orchestration-mutation-executor' +import { orchestrationMigrationFence } from './orchestration-contract-fence' +import { getRuntimeFeatureInteractionId } from './runtime-feature-interaction' export type DispatcherOptions = { runtime: OrcaRuntimeService @@ -38,10 +45,12 @@ export type DispatcherOptions = { export class RpcDispatcher { private readonly runtime: OrcaRuntimeService private readonly registry: RpcRegistry + private readonly orchestrationMutations: OrchestrationMutationExecutor constructor({ runtime, methods = ALL_RPC_METHODS }: DispatcherOptions) { this.runtime = runtime this.registry = buildRegistry(methods) + this.orchestrationMutations = new OrchestrationMutationExecutor(runtime) } async dispatch(request: RpcRequest, options?: { signal?: AbortSignal }): Promise { @@ -56,6 +65,11 @@ export class RpcDispatcher { ) } + const migrationFence = orchestrationMigrationFence(request, meta) + if (migrationFence) { + return migrationFence + } + const parsedParams = this.parseParams(request, method, meta) if (parsedParams.error) { return parsedParams.error @@ -78,10 +92,17 @@ export class RpcDispatcher { emulatorProbe(`rpc ${request.method}`, request.params) } try { - const result = await method.handler(parsedParams.value, { - runtime: this.runtime, - signal: options?.signal - }) + const invoke = (mutation?: DurableMutationInvocation) => + method.handler(parsedParams.value, { + runtime: this.runtime, + signal: options?.signal, + requestId: request.id, + orchestrationCapability: request.orchestrationCapability, + authenticatedCallerFingerprint: authenticatedCallerFingerprint(request), + recordMutationReceipt: mutation?.recordReceipt, + orchestrationMutation: mutation?.identity + }) + const result = await this.orchestrationMutations.run(request, parsedParams.value, invoke) this.recordRuntimeFeatureInteraction(request.method, result, undefined, request.params) return successResponse(request.id, meta, result) } catch (error) { @@ -123,6 +144,12 @@ export class RpcDispatcher { return } + const migrationFence = orchestrationMigrationFence(request, meta) + if (migrationFence) { + reply(JSON.stringify(migrationFence)) + return + } + const parsedParams = this.parseParams(request, method, meta) if (parsedParams.error) { reply(JSON.stringify(parsedParams.error)) @@ -131,18 +158,24 @@ export class RpcDispatcher { if (!isStreamingMethod(method)) { try { - const result = await method.handler(parsedParams.value, { - runtime: this.runtime, - signal: options?.signal, - requestId: request.id, - connectionId: options?.connectionId, - clientId: options?.clientId, - pairedDeviceId: options?.pairedDeviceId, - clientKind: options?.clientKind, - pairing: options?.pairing, - sendBinary: options?.sendBinary, - registerBinaryStreamHandler: options?.registerBinaryStreamHandler - }) + const invoke = (mutation?: DurableMutationInvocation) => + method.handler(parsedParams.value, { + runtime: this.runtime, + signal: options?.signal, + requestId: request.id, + connectionId: options?.connectionId, + clientId: options?.clientId, + pairedDeviceId: options?.pairedDeviceId, + clientKind: options?.clientKind, + orchestrationCapability: request.orchestrationCapability, + authenticatedCallerFingerprint: authenticatedCallerFingerprint(request), + recordMutationReceipt: mutation?.recordReceipt, + orchestrationMutation: mutation?.identity, + pairing: options?.pairing, + sendBinary: options?.sendBinary, + registerBinaryStreamHandler: options?.registerBinaryStreamHandler + }) + const result = await this.orchestrationMutations.run(request, parsedParams.value, invoke) this.recordRuntimeFeatureInteraction(request.method, result, undefined, request.params) reply(JSON.stringify(successResponse(request.id, meta, result))) } catch (error) { @@ -270,50 +303,3 @@ export class RpcDispatcher { } } } - -function getRuntimeFeatureInteractionId( - method: string, - result: unknown, - rawParams?: unknown -): FeatureInteractionId | null { - if (method === 'browser.profileImportFromBrowser') { - return hasBooleanResult(result, 'ok') ? 'cookie-import' : null - } - if (method === 'browser.profileClearDefaultCookies') { - return hasBooleanResult(result, 'cleared') ? 'cookie-import' : null - } - if (method === 'browser.screencast.unsubscribe') { - return null - } - if (method.startsWith('browser.') && isBrowserPaneUiRuntimeRpcParams(rawParams)) { - return null - } - if (method.startsWith('browser.') && !method.startsWith('browser.profile')) { - return 'agent-browser-use' - } - if (method.startsWith('emulator.')) { - // Emulator commands are allowed from terminal/CLI (workspace-scoped, like other automation). - // Return null to indicate no special feature-interaction restriction (or add 'emulator-use' later). - return null - } - if (method === 'computer.permissions') { - return 'computer-use-setup' - } - if ( - method.startsWith('computer.') && - method !== 'computer.capabilities' && - method !== 'computer.permissionsStatus' - ) { - return 'computer-use' - } - if (method.startsWith('orchestration.')) { - return 'agent-orchestration' - } - return null -} - -function hasBooleanResult(value: unknown, key: string): boolean { - return ( - value !== null && typeof value === 'object' && (value as Record)[key] === true - ) -} diff --git a/src/main/runtime/rpc/errors.test.ts b/src/main/runtime/rpc/errors.test.ts index 0e4519656cb..ddd0541c57b 100644 --- a/src/main/runtime/rpc/errors.test.ts +++ b/src/main/runtime/rpc/errors.test.ts @@ -30,6 +30,18 @@ describe('mapRuntimeError', () => { }) }) + it.each(['remote_runtime_unavailable', 'runtime_timeout', 'invalid_runtime_response'])( + 'preserves structured remote transport failure %s', + (code) => { + const error = Object.assign(new Error(`Remote transport failed: ${code}`), { code }) + + expect(mapRuntimeError('req_1', { runtimeId: 'runtime-1' }, error)).toMatchObject({ + ok: false, + error: { code, message: `Remote transport failed: ${code}` } + }) + } + ) + it.each([ ['window_not_focused', 'keyboard input requires focus', 'restore-window'], ['permission_denied', 'missing DBUS_SESSION_BUS_ADDRESS', 'permissions'], diff --git a/src/main/runtime/rpc/errors.ts b/src/main/runtime/rpc/errors.ts index 39190c6dfcb..d76b0e851c2 100644 --- a/src/main/runtime/rpc/errors.ts +++ b/src/main/runtime/rpc/errors.ts @@ -60,7 +60,41 @@ const RUNTIME_PASSTHROUGH_CODES: ReadonlySet = new Set([ const COMPUTER_PASSTHROUGH_CODES: ReadonlySet = new Set(Object.values(COMPUTER_ERROR_CODES)) const LINEAR_PASSTHROUGH_CODES: ReadonlySet = new Set(LINEAR_ERROR_CODES) const STRUCTURED_RUNTIME_PASSTHROUGH_CODES: ReadonlySet = new Set([ - 'worktree_id_requires_full_path' + 'worktree_id_requires_full_path', + 'run_not_found', + 'run_required', + 'stable_pane_required', + 'consumer_fenced', + 'task_not_found', + 'task_not_startable', + 'dispatch_not_found', + 'dispatch_run_mismatch', + 'dispatch_inactive', + 'worker_identity_changed', + 'cursor_invalid', + 'cursor_dispatch_mismatch', + 'source_changed', + 'transcript_required', + 'server_required', + 'worktree_not_found_on_server', + 'resource_server_mismatch', + 'peer_changed', + 'remote_runtime_unavailable', + 'runtime_timeout', + 'invalid_runtime_response', + 'capability_unsupported', + 'relay_quota_exceeded', + 'dispatch_capability_invalid', + 'agent_unconfigured', + 'terminal_worktree_mismatch', + 'request_mismatch', + 'orchestration_migration_required', + 'operation_unknown', + 'question_not_found', + 'answer_conflict', + 'stale_delivery', + 'waiter_exists', + 'invalid_argument' ]) export function mapRuntimeError(id: string, meta: RpcEnvelopeMeta, error: unknown): RpcFailure { diff --git a/src/main/runtime/rpc/methods/orchestration-federated-message-targeting.test.ts b/src/main/runtime/rpc/methods/orchestration-federated-message-targeting.test.ts new file mode 100644 index 00000000000..bf1a9223f82 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-federated-message-targeting.test.ts @@ -0,0 +1,104 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { ORCHESTRATION_CONTRACT_VERSION } from '../../../../shared/protocol-version' +import { OrcaRuntimeService } from '../../orca-runtime' +import { OrchestrationDb } from '../../orchestration/db' +import type { RpcRequest } from '../core' +import { RpcDispatcher } from '../dispatcher' +import { ORCHESTRATION_METHODS } from './orchestration' + +describe('orchestration federated message targeting', () => { + let db: OrchestrationDb | undefined + let runtime: OrcaRuntimeService | undefined + + afterEach(() => { + runtime?.stopOrchestrationFederationRelay() + db?.close() + }) + + it('rejects explicit send and ask targets without enqueueing a relay', async () => { + db = new OrchestrationDb(':memory:') + runtime = new OrcaRuntimeService() + runtime.setOrchestrationDb(db) + const paneKey = 'tab_worker:leaf_worker' + const processIncarnation = 'worker_epoch:pty:1' + const dispatchId = 'ctx_remote_targeting' + vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue(paneKey) + vi.spyOn(runtime, 'getTerminalProcessIncarnation').mockReturnValue(processIncarnation) + db.createRemoteDispatchAttachment({ + dispatchId, + taskId: 'task_remote_targeting', + homePeerFingerprint: 'home_peer', + protocolVersion: 1, + runtimeEpoch: runtime.getRuntimeId(), + mutationReceipt: { + callerFingerprint: 'home_peer', + requestId: 'attach_request', + method: 'orchestration.federationAttachStart', + payloadHash: 'attach_payload' + } + }) + const capability = db.prepareRemoteAttachmentAuthority({ + dispatchId, + paneKey, + processIncarnation, + worktreeId: 'repo::remote-worktree', + terminalHandle: 'term_remote_worker', + setupState: 'not_applicable', + effects: [] + }) + db.markRemoteAttachmentReady(dispatchId) + const dispatcher = new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS }) + const requests: RpcRequest[] = [ + request('send_to', capability, 'orchestration.send', { + from: 'term_remote_worker', + to: 'run:explicit', + subject: 'Wrong explicit target' + }), + request('send_run', capability, 'orchestration.send', { + from: 'term_remote_worker', + run: 'run_explicit', + subject: 'Wrong explicit Run' + }), + request('ask_to', capability, 'orchestration.ask', { + from: 'term_remote_worker', + to: 'run:explicit', + question: 'Wrong explicit target?' + }), + request('ask_run', capability, 'orchestration.ask', { + from: 'term_remote_worker', + run: 'run_explicit', + question: 'Wrong explicit Run?' + }) + ] + + for (const item of requests) { + await expect(dispatcher.dispatch(item)).resolves.toMatchObject({ + ok: false, + error: { + code: 'invalid_argument', + message: 'Federated Dispatch messages route to their Run home; omit --to and --run.' + } + }) + } + expect( + db.listFederationRelay({ dispatchId, direction: 'to_home', afterSequence: 0 }) + ).toHaveLength(0) + }) +}) + +function request( + id: string, + capability: string, + method: 'orchestration.send' | 'orchestration.ask', + params: Record +): RpcRequest { + return { + id: `rpc_${id}`, + authToken: 'worker-token', + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationRequestId: `request_${id}`, + orchestrationCapability: capability, + method, + params + } +} diff --git a/src/main/runtime/rpc/methods/orchestration-federated-worker-start.ts b/src/main/runtime/rpc/methods/orchestration-federated-worker-start.ts new file mode 100644 index 00000000000..538dadc96d0 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-federated-worker-start.ts @@ -0,0 +1,292 @@ +import { isTuiAgent } from '../../../../shared/tui-agent-config' +import type { RuntimeStatus } from '../../../../shared/runtime-types' +import { + ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY, + ORCHESTRATION_FEDERATION_CONTROL_MAIL_PROTOCOL_VERSION, + ORCHESTRATION_FEDERATION_CONTROL_MAIL_RUNTIME_CAPABILITY, + ORCHESTRATION_FEDERATION_RUNTIME_CAPABILITY +} from '../../../../shared/protocol-version' +import { orchestrationMigrationData } from '../../../../shared/orchestration-rpc-contract' +import type { OrcaRuntimeService } from '../../orca-runtime' +import type { OrchestrationDb } from '../../orchestration/db' +import { OrchestrationError } from '../../orchestration/orchestration-error' +import type { WorkerStartInput } from './orchestration-worker-start-schema' + +export async function startFederatedWorker(args: { + params: WorkerStartInput + runtime: OrcaRuntimeService + db: OrchestrationDb + runId: string + task: { id: string; spec: string; status: string } + orchestrationMutation?: { + callerFingerprint: string + requestId: string + method: string + payloadHash: string + } +}): Promise { + const { params, runtime, db, task, runId, orchestrationMutation } = args + if (!orchestrationMutation) { + throw new OrchestrationError( + 'invalid_argument', + 'Remote worker-start requires a durable retry request.' + ) + } + const worktree = params.worktree ?? 'current' + if (worktree === 'current' || worktree === 'new-child') { + throw new OrchestrationError( + 'invalid_argument', + '--on requires an exact remote worktree selector or new-top-level.' + ) + } + const createsWorktree = worktree === 'new-top-level' + validateRemoteWorkerStart(params, createsWorktree) + + const server = runtime.resolveOrchestrationWorkerServer(params.on as string) + const status = (await runtime.callOrchestrationWorkerServer( + server.environmentId, + 'status.get', + undefined, + params.timeoutMs + )) as RuntimeStatus + if (!status.capabilities?.includes(ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY)) { + throw new OrchestrationError( + 'orchestration_migration_required', + `Connected server ${server.name} does not support the current orchestration contract. No effects were applied.`, + orchestrationMigrationData('runtime_capability_missing') + ) + } + if (!status.capabilities?.includes(ORCHESTRATION_FEDERATION_RUNTIME_CAPABILITY)) { + throw new OrchestrationError( + 'capability_unsupported', + `Connected server ${server.name} does not support orchestration federation.` + ) + } + const federationProtocolVersion = status.capabilities?.includes( + ORCHESTRATION_FEDERATION_CONTROL_MAIL_RUNTIME_CAPABILITY + ) + ? ORCHESTRATION_FEDERATION_CONTROL_MAIL_PROTOCOL_VERSION + : 1 + + const setupDecision = createsWorktree ? (params.setup ?? 'run') : 'not_applicable' + const started = db.createStartingWorkerDispatch({ + taskId: task.id, + retryOf: params.retryOf, + startOptions: { + on: server.environmentId, + serverName: server.name, + worktree, + name: params.name ?? null, + repo: params.repo ?? null, + baseBranch: params.baseBranch ?? null, + terminal: params.terminal ?? null, + agent: params.agent ?? null, + timeoutMs: params.timeoutMs ?? 60_000, + setup: setupDecision, + setupSource: createsWorktree + ? params.setup + ? 'explicit_request' + : 'orchestration_default' + : 'existing_worktree' + }, + runtimeEpoch: runtime.getRuntimeId(), + mutationReceipt: orchestrationMutation, + federation: { + environmentId: server.environmentId, + environmentName: server.name, + peerFingerprint: server.peerFingerprint, + protocolVersion: federationProtocolVersion + } + }) + db.recordWorkerStage({ dispatchId: started.dispatch.id, stage: 'remote_attach_requested' }) + try { + const remote = (await runtime.callOrchestrationWorkerServer( + server.environmentId, + 'orchestration.federationAttachStart', + { + dispatchId: started.dispatch.id, + taskId: task.id, + taskSpec: task.spec, + protocolVersion: federationProtocolVersion, + worktree, + name: params.name, + repo: params.repo, + baseBranch: params.baseBranch, + displayName: params.displayName, + comment: params.comment, + setup: createsWorktree ? (params.setup ?? 'run') : undefined, + setupSource: createsWorktree + ? params.setup + ? 'explicit_request' + : 'orchestration_default' + : undefined, + terminal: params.terminal, + agent: params.agent, + timeoutMs: params.timeoutMs, + devMode: params.devMode + }, + (params.timeoutMs ?? 60_000) + 15_000, + { orchestrationRequestId: orchestrationMutation.requestId } + )) as RemoteStartReceipt + if (remote.dispatchId !== started.dispatch.id) { + throw new OrchestrationError( + 'resource_server_mismatch', + 'The worker server returned a different Dispatch attachment.' + ) + } + if (remote.state === 'ready' && remote.worktreeId && remote.terminalHandle) { + db.updateFederatedDispatchResources({ + dispatchId: started.dispatch.id, + remoteRuntimeEpoch: remote.runtimeEpoch, + worktreeId: remote.worktreeId, + terminalHandle: remote.terminalHandle + }) + db.recordWorkerStage({ + dispatchId: started.dispatch.id, + stage: 'remote_input_accepted', + worktreeId: remote.worktreeId, + terminalHandle: remote.terminalHandle, + setupState: remote.setup?.state, + effects: remote.effects, + residualResources: remote.residualResources + }) + const readyWorker = db.markWorkerDispatchReady(started.dispatch.id) + runtime.ensureOrchestrationFederationRelay(runId) + return { + runId, + taskId: task.id, + dispatchId: started.dispatch.id, + state: 'ready', + stage: readyWorker.stage, + server: { environmentId: server.environmentId, name: server.name }, + setup: remote.setup, + timeoutMs: params.timeoutMs ?? 60_000, + effects: remote.effects ?? [], + residualResources: remote.residualResources ?? [] + } + } + if (remote.state === 'outcome_unknown') { + const worker = db.markWorkerStartUnknown( + started.dispatch.id, + remote.failedStage ?? 'remote_attach', + remote.lastError ?? 'The worker server reported an unknown start outcome.' + ) + return federatedUnknownReceipt(worker, task.id, server.name) + } + const worker = db.failWorkerStart( + started.dispatch.id, + remote.failedStage ?? 'remote_attach', + remote.lastError ?? `The worker server returned ${remote.state}.` + ) + return { + runId, + taskId: task.id, + dispatchId: started.dispatch.id, + state: worker.state, + stage: worker.stage, + server: { environmentId: server.environmentId, name: server.name }, + failedStage: worker.stage, + lastError: worker.last_error, + setup: remote.setup, + effects: remote.effects ?? [], + residualResources: remote.residualResources ?? [] + } + } catch (error) { + const reason = error instanceof Error ? error.message : String(error) + if (error instanceof OrchestrationError && isKnownRemoteStartFailure(error.code)) { + const worker = db.failWorkerStart(started.dispatch.id, 'remote_attach', reason) + return { + runId, + taskId: task.id, + dispatchId: started.dispatch.id, + state: worker.state, + stage: worker.stage, + server: { environmentId: server.environmentId, name: server.name }, + failedStage: worker.stage, + lastError: worker.last_error, + effects: [], + residualResources: [] + } + } + const worker = db.markWorkerStartUnknown(started.dispatch.id, 'remote_attach', reason) + return federatedUnknownReceipt(worker, task.id, server.name) + } +} + +type RemoteStartReceipt = { + dispatchId: string + state: string + runtimeEpoch: string + worktreeId?: string + terminalHandle?: string + setup?: { state: string } + effects?: unknown[] + residualResources?: unknown[] + failedStage?: string + lastError?: string +} + +function validateRemoteWorkerStart(params: WorkerStartInput, createsWorktree: boolean): void { + if (createsWorktree && (!params.name || !params.repo)) { + throw new OrchestrationError( + 'invalid_argument', + 'Remote new-top-level requires --name and an explicit --repo from remote discovery.' + ) + } + if (createsWorktree && params.terminal) { + throw new OrchestrationError( + 'invalid_argument', + '--terminal cannot combine with remote new-worktree creation.' + ) + } + if (!createsWorktree && (params.name || params.repo || params.baseBranch || params.setup)) { + throw new OrchestrationError( + 'invalid_argument', + 'Creation and setup options apply only to remote new-top-level worktrees.' + ) + } + if (params.terminal && params.agent) { + throw new OrchestrationError( + 'invalid_argument', + '--terminal reuses an existing agent and cannot combine with --agent.' + ) + } + if (!params.terminal && (!params.agent || !isTuiAgent(params.agent))) { + throw new OrchestrationError( + 'agent_unconfigured', + 'A configured --agent is required when remote worker-start creates a terminal.' + ) + } +} + +function isKnownRemoteStartFailure(code: string): boolean { + return [ + 'invalid_argument', + 'agent_unconfigured', + 'worktree_not_found_on_server', + 'terminal_worktree_mismatch', + 'capability_unsupported' + ].includes(code) +} + +function federatedUnknownReceipt( + worker: { dispatch_id: string; state: string; stage: string; last_error: string | null }, + taskId: string, + serverName: string +): unknown { + return { + taskId, + dispatchId: worker.dispatch_id, + state: 'outcome_unknown', + stage: worker.stage, + server: { name: serverName }, + failedStage: worker.stage, + lastError: worker.last_error, + effects: [], + residualResources: [], + nextCommands: [ + `orca orchestration worker-show --dispatch ${worker.dispatch_id} --json`, + `orca orchestration worker-abandon --dispatch ${worker.dispatch_id} --json` + ] + } +} diff --git a/src/main/runtime/rpc/methods/orchestration-federation-control-mail.test.ts b/src/main/runtime/rpc/methods/orchestration-federation-control-mail.test.ts new file mode 100644 index 00000000000..8a5111047cb --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-federation-control-mail.test.ts @@ -0,0 +1,345 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { ORCHESTRATION_CONTRACT_VERSION } from '../../../../shared/protocol-version' +import type { RuntimeRpcResponse } from '../../../../shared/runtime-rpc-envelope' +import { OrcaRuntimeService } from '../../orca-runtime' +import { OrchestrationDb } from '../../orchestration/db' +import type { OrchestrationEnvironmentTransport } from '../../orchestration/environment-transport' +import type { RpcRequest } from '../core' +import { RpcDispatcher } from '../dispatcher' +import { authenticatedCallerFingerprint } from '../orchestration-mutation-executor' +import { ORCHESTRATION_METHODS } from './orchestration' + +describe('orchestration federation control mail', () => { + const homeToken = 'run-home-device-token' + const workerToken = 'worker-local-token' + const workerPeerFingerprint = 'worker-peer' + const coordinatorPaneKey = 'tab_coord:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' + const workerPaneKey = 'tab_worker:bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb' + const processIncarnation = 'worker-runtime:pty:1' + let homeDb: OrchestrationDb + let workerDb: OrchestrationDb + let homeRuntime: OrcaRuntimeService + let workerRuntime: OrcaRuntimeService + let homeDispatcher: RpcDispatcher + let workerDispatcher: RpcDispatcher + let dispatchId: string + let runId: string + + beforeEach(() => { + workerDb = new OrchestrationDb(':memory:') + workerRuntime = new OrcaRuntimeService() + workerRuntime.setOrchestrationDb(workerDb) + vi.spyOn(workerRuntime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === 'term_worker' ? workerPaneKey : null + ) + vi.spyOn(workerRuntime, 'getTerminalProcessIncarnation').mockImplementation((handle) => + handle === 'term_worker' ? processIncarnation : null + ) + workerDispatcher = new RpcDispatcher({ + runtime: workerRuntime, + methods: ORCHESTRATION_METHODS + }) + + const transport: OrchestrationEnvironmentTransport = { + resolve: () => ({ + environmentId: 'environment_worker', + name: 'worker', + peerFingerprint: workerPeerFingerprint + }), + call: async (_selector, method, params, _timeoutMs, envelope) => { + if (method === 'status.get') { + return { + id: 'status', + ok: true, + result: workerRuntime.getStatus(), + _meta: { runtimeId: workerRuntime.getRuntimeId() } + } + } + const response = (await workerDispatcher.dispatch({ + id: `remote_${method}`, + authToken: homeToken, + method, + params, + orchestrationContractVersion: envelope?.orchestrationContractVersion, + orchestrationRequestId: envelope?.orchestrationRequestId + })) as RuntimeRpcResponse + return response + } + } + homeDb = new OrchestrationDb(':memory:') + homeRuntime = new OrcaRuntimeService(null, undefined, { + orchestrationEnvironmentTransport: transport + }) + homeRuntime.setOrchestrationDb(homeDb) + vi.spyOn(homeRuntime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === 'term_coord' ? coordinatorPaneKey : null + ) + homeDispatcher = new RpcDispatcher({ + runtime: homeRuntime, + methods: ORCHESTRATION_METHODS + }) + + const run = homeDb.createRun({ + objective: 'Federated control mail', + coordinatorHandle: 'term_coord', + coordinatorPaneKey + }) + runId = run.id + const task = homeDb.createTask({ spec: 'Wait for coordinator guidance', runId }) + const started = homeDb.createStartingWorkerDispatch({ + taskId: task.id, + startOptions: {}, + federation: { + environmentId: 'environment_worker', + environmentName: 'worker', + peerFingerprint: workerPeerFingerprint, + protocolVersion: 2 + } + }) + dispatchId = started.dispatch.id + homeDb.markWorkerDispatchReady(dispatchId) + + const homeFingerprint = authenticatedCallerFingerprint({ + id: 'home', + authToken: homeToken, + method: 'orchestration.federationImport' + }) + workerDb.createRemoteDispatchAttachment({ + dispatchId, + taskId: task.id, + homePeerFingerprint: homeFingerprint, + protocolVersion: 2, + runtimeEpoch: workerRuntime.getRuntimeId(), + mutationReceipt: { + callerFingerprint: homeFingerprint, + requestId: 'attach-worker', + method: 'orchestration.federationAttachStart', + payloadHash: 'attach-worker-payload' + } + }) + workerDb.prepareRemoteAttachmentAuthority({ + dispatchId, + paneKey: workerPaneKey, + processIncarnation, + worktreeId: 'repo::worker', + terminalHandle: 'term_worker', + setupState: 'not_applicable', + effects: [] + }) + workerDb.markRemoteAttachmentReady(dispatchId) + }) + + afterEach(() => { + homeRuntime.stopOrchestrationFederationRelay() + homeDb.close() + workerDb.close() + }) + + it('routes an exact Dispatch message through the durable relay', async () => { + vi.spyOn(homeRuntime, 'ensureOrchestrationFederationRelay').mockImplementation(() => {}) + const sent = await homeDispatcher.dispatch({ + id: 'send-control', + authToken: 'coordinator-token', + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationRequestId: 'send-control-request', + method: 'orchestration.send', + params: { + from: 'term_coord', + to: `dispatch:${dispatchId}`, + subject: 'Continue', + body: 'Run the focused follow-up.', + type: 'status' + } + }) + + expect(sent).toMatchObject({ + ok: true, + result: { relay: { dispatchId, accepted: true } } + }) + expect(homeDb.listPendingFederationRelay(dispatchId, 'to_worker')).toHaveLength(1) + + await homeRuntime.syncOrchestrationFederation() + const checked = await workerDispatcher.dispatch(checkRequest('check-imported')) + + expect(checked).toMatchObject({ + ok: true, + result: { + dispatchId, + count: 1, + messages: [ + { + to_handle: `dispatch:${dispatchId}`, + subject: 'Continue', + body: 'Run the focused follow-up.' + } + ] + } + }) + expect(homeDb.listPendingFederationRelay(dispatchId, 'to_worker')).toHaveLength(0) + }) + + it('wakes a remote worker waiter when control mail imports', async () => { + const waiting = workerDispatcher.dispatch(checkRequest('wait-for-control', true)) + await Promise.resolve() + + const imported = await workerDispatcher.dispatch( + importRequest('import-control', 1, 'relay-control') + ) + + expect(imported).toMatchObject({ + ok: true, + result: { acknowledgedThrough: 1, imported: 1 } + }) + await expect(waiting).resolves.toMatchObject({ + ok: true, + result: { + dispatchId, + count: 1, + messages: [{ id: 'relay-control', subject: 'Continue' }] + } + }) + }) + + it('accepts a repeated import after a lost acknowledgment without duplicating mail', async () => { + const first = await workerDispatcher.dispatch(importRequest('first-import', 1, 'relay-control')) + const repeated = await workerDispatcher.dispatch( + importRequest('repeated-import', 1, 'different-message-id') + ) + + expect(first).toMatchObject({ ok: true, result: { imported: 1 } }) + expect(repeated).toMatchObject({ ok: true, result: { imported: 0 } }) + expect(workerDb.getUnreadMessages(`dispatch:${dispatchId}`)).toHaveLength(1) + }) + + it('does not deliver pending control mail after worker completion', async () => { + vi.spyOn(homeRuntime, 'ensureOrchestrationFederationRelay').mockImplementation(() => {}) + await homeDispatcher.dispatch({ + id: 'send-stale-control', + authToken: 'coordinator-token', + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationRequestId: 'send-stale-control-request', + method: 'orchestration.send', + params: { + from: 'term_coord', + to: `dispatch:${dispatchId}`, + subject: 'Stale follow-up', + body: 'This must not arrive after completion.', + type: 'status' + } + }) + const waiting = workerDispatcher.dispatch(checkRequest('wait-before-completion', true, 30)) + await Promise.resolve() + + const taskId = homeDb.getDispatchContextById(dispatchId)!.task_id + workerDb.enqueueFederationRelay({ + dispatchId, + direction: 'to_home', + kind: 'worker_done', + payload: JSON.stringify({ + from: `dispatch:${dispatchId}`, + subject: 'Done', + body: 'Completed before the follow-up arrived.', + type: 'worker_done', + priority: 'normal', + threadId: null, + payload: JSON.stringify({ + taskId, + dispatchId, + outcome: 'succeeded', + filesModified: [] + }) + }), + settleRemoteOutcome: 'succeeded' + }) + + await homeRuntime.syncOrchestrationFederation() + + expect(homeDb.getWorkerDispatch(dispatchId)?.state).toBe('succeeded') + expect(workerDb.getUnreadMessages(`dispatch:${dispatchId}`)).toHaveLength(0) + expect(homeDb.listPendingFederationRelay(dispatchId, 'to_worker')).toHaveLength(1) + await expect( + workerDispatcher.dispatch(importRequest('late-direct-import', 1, 'late-control')) + ).resolves.toMatchObject({ + ok: false, + error: { code: 'dispatch_inactive' } + }) + await expect(waiting).resolves.toMatchObject({ + ok: true, + result: { count: 0, timedOut: true } + }) + }) + + it('wakes only waiters whose filter matches an imported control message', async () => { + const escalationWaiter = workerDispatcher.dispatch( + checkRequest('wait-escalation', true, 1_000, 'escalation') + ) + const statusWaiter = workerDispatcher.dispatch(checkRequest('wait-status', true, 30, 'status')) + await Promise.resolve() + + await workerDispatcher.dispatch( + importRequest('import-escalation', 1, 'relay-escalation', 'escalation') + ) + + await expect(escalationWaiter).resolves.toMatchObject({ + ok: true, + result: { + count: 1, + messages: [{ id: 'relay-escalation', type: 'escalation' }] + } + }) + await expect(statusWaiter).resolves.toMatchObject({ + ok: true, + result: { count: 0, timedOut: true } + }) + }) + + function checkRequest(id: string, wait = false, timeoutMs = 5_000, types?: string): RpcRequest { + return { + id, + authToken: workerToken, + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + method: 'orchestration.check', + params: { + terminal: 'term_worker', + wait, + timeoutMs, + types + } + } + } + + function importRequest( + id: string, + sequence: number, + messageId: string, + type = 'status' + ): RpcRequest { + return { + id, + authToken: homeToken, + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + method: 'orchestration.federationImport', + params: { + dispatchId, + items: [ + { + dispatch_id: dispatchId, + direction: 'to_worker', + sequence, + message_id: messageId, + kind: 'control_message', + payload: JSON.stringify({ + from: `run:${runId}`, + subject: 'Continue', + body: 'Run the focused follow-up.', + type, + priority: 'normal', + threadId: null, + payload: null + }) + } + ] + } + } + } +}) diff --git a/src/main/runtime/rpc/methods/orchestration-federation-control.ts b/src/main/runtime/rpc/methods/orchestration-federation-control.ts new file mode 100644 index 00000000000..215a45c5676 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-federation-control.ts @@ -0,0 +1,207 @@ +import { z } from 'zod' +import { ORCHESTRATION_WORKER_READ_SOURCES } from '../../../../shared/orchestration-worker-output' +import type { OrcaRuntimeService } from '../../orca-runtime' +import { OrchestrationError } from '../../orchestration/orchestration-error' +import type { RemoteDispatchAttachmentRow } from '../../orchestration/types' +import { defineMethod, type RpcMethod } from '../core' +import { OptionalFiniteNumber, requiredString } from '../schemas' +import { readExactWorkerOutput } from './orchestration-worker-output' + +const FederationDispatchParams = z.object({ + dispatchId: requiredString('Missing Dispatch ID') +}) +const FederationReadParams = FederationDispatchParams.extend({ + cursor: OptionalFiniteNumber, + limit: OptionalFiniteNumber +}) +const FederationOutputReadParams = FederationDispatchParams.extend({ + cursor: z.union([z.number().int().nonnegative(), z.string().min(1).max(2_048)]).optional(), + limit: OptionalFiniteNumber, + source: z.enum(ORCHESTRATION_WORKER_READ_SOURCES).optional() +}) + +export const ORCHESTRATION_FEDERATION_CONTROL_METHODS: RpcMethod[] = [ + defineMethod({ + name: 'orchestration.federationShow', + params: FederationDispatchParams, + handler: async (params, { runtime, authenticatedCallerFingerprint }) => { + const attachment = requireHomeAttachment( + runtime, + params.dispatchId, + authenticatedCallerFingerprint + ) + const observation = await inspectRemoteAttachment(runtime, params.dispatchId) + return { + dispatchId: params.dispatchId, + runtimeEpoch: runtime.getRuntimeId(), + attachment: exposeRemoteAttachment(attachment), + terminal: observation.exact ? observation.terminal : null, + observation: { status: observation.status, exactWorker: observation.exact } + } + } + }), + defineMethod({ + name: 'orchestration.federationRead', + params: FederationReadParams, + handler: async (params, { runtime, authenticatedCallerFingerprint }) => { + requireHomeAttachment(runtime, params.dispatchId, authenticatedCallerFingerprint) + const observation = await inspectRemoteAttachment(runtime, params.dispatchId) + if (!observation.exact || !observation.terminal || observation.status !== 'running') { + throw new OrchestrationError( + 'worker_identity_changed', + `Remote Dispatch ${params.dispatchId} no longer resolves to its exact process.` + ) + } + return { + dispatchId: params.dispatchId, + runtimeEpoch: runtime.getRuntimeId(), + terminal: await runtime.readTerminal(observation.terminal.handle, { + cursor: params.cursor, + limit: params.limit + }) + } + } + }), + defineMethod({ + name: 'orchestration.federationReadOutput', + params: FederationOutputReadParams, + handler: async (params, { runtime, authenticatedCallerFingerprint }) => { + const attachment = requireHomeAttachment( + runtime, + params.dispatchId, + authenticatedCallerFingerprint + ) + const observation = await inspectRemoteAttachment(runtime, params.dispatchId) + if (!observation.exact || !observation.terminal) { + throw new OrchestrationError( + 'worker_identity_changed', + `Remote Dispatch ${params.dispatchId} no longer resolves to its exact process.` + ) + } + const output = await readExactWorkerOutput({ + runtime, + dispatchId: params.dispatchId, + terminalHandle: observation.terminal.handle, + workerState: attachment.state, + terminalStatus: observation.status === 'exited' ? 'exited' : 'running', + attachedAt: attachment.created_at, + source: params.source, + cursor: params.cursor, + limit: params.limit + }) + const afterRead = await inspectRemoteAttachment(runtime, params.dispatchId) + if (!afterRead.exact) { + throw new OrchestrationError( + 'worker_identity_changed', + `Remote Dispatch ${params.dispatchId} changed process while output was read.` + ) + } + return { + dispatchId: params.dispatchId, + runtimeEpoch: runtime.getRuntimeId(), + output + } + } + }), + defineMethod({ + name: 'orchestration.federationStop', + params: FederationDispatchParams, + handler: async (params, { runtime, authenticatedCallerFingerprint }) => { + requireHomeAttachment(runtime, params.dispatchId, authenticatedCallerFingerprint) + const db = runtime.getOrchestrationDb() + const begun = db.beginRemoteAttachmentStop(params.dispatchId) + if (['succeeded', 'failed', 'stopped', 'abandoned'].includes(begun.state)) { + return { + dispatchId: params.dispatchId, + state: begun.state, + alreadySettled: true, + processAction: 'none' + } + } + const observation = await inspectRemoteAttachment(runtime, params.dispatchId) + if (!observation.exact || !observation.terminal) { + const attachment = db.markRemoteAttachmentStopUnknown( + params.dispatchId, + `The recorded worker process is ${observation.status}; no terminal was closed.` + ) + return { + dispatchId: params.dispatchId, + state: attachment.state, + alreadySettled: false, + processAction: 'none', + lastError: attachment.last_error + } + } + try { + const close = await runtime.closeTerminal(observation.terminal.handle) + const attachment = db.settleRemoteAttachmentStop(params.dispatchId) + return { + dispatchId: params.dispatchId, + state: attachment.state, + alreadySettled: false, + processAction: 'closed_agent_terminal', + close + } + } catch (error) { + const reason = error instanceof Error ? error.message : String(error) + const attachment = db.markRemoteAttachmentStopUnknown(params.dispatchId, reason) + return { + dispatchId: params.dispatchId, + state: attachment.state, + alreadySettled: false, + processAction: 'unknown', + lastError: reason + } + } + } + }) +] + +function requireHomeAttachment( + runtime: OrcaRuntimeService, + dispatchId: string, + callerFingerprint: string | undefined +): RemoteDispatchAttachmentRow { + const attachment = runtime.getOrchestrationDb().getRemoteDispatchAttachment(dispatchId) + if (!attachment || attachment.home_peer_fingerprint !== callerFingerprint) { + throw new OrchestrationError( + 'dispatch_not_found', + `Remote Dispatch ${dispatchId} was not found for this Run home.` + ) + } + return attachment +} + +async function inspectRemoteAttachment(runtime: OrcaRuntimeService, dispatchId: string) { + const db = runtime.getOrchestrationDb() + const attachment = db.getRemoteDispatchAttachment(dispatchId) + if (!attachment?.terminal_handle) { + return { terminal: null, exact: false, status: 'unattached' as const } + } + const terminal = await runtime.showTerminal(attachment.terminal_handle).catch(() => null) + if (!terminal) { + return { terminal: null, exact: false, status: 'missing' as const } + } + const exact = db.isRemoteAttachmentProcessCurrent({ + dispatchId, + paneKey: runtime.getTerminalPaneKey(attachment.terminal_handle), + processIncarnation: runtime.getTerminalProcessIncarnation(attachment.terminal_handle) + }) + return { + terminal, + exact, + status: exact + ? terminal.connected === false + ? ('exited' as const) + : ('running' as const) + : ('identity_changed' as const) + } +} + +function exposeRemoteAttachment(attachment: RemoteDispatchAttachmentRow) { + return { + ...attachment, + effects: JSON.parse(attachment.effects) as unknown[], + residualResources: JSON.parse(attachment.residual_resources) as unknown[] + } +} diff --git a/src/main/runtime/rpc/methods/orchestration-federation-effects.test.ts b/src/main/runtime/rpc/methods/orchestration-federation-effects.test.ts new file mode 100644 index 00000000000..b4c1cd37589 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-federation-effects.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest' +import { + appendFederationSetupEffect, + appendFederationTerminalEffects, + type FederationEffect +} from './orchestration-federation-effects' + +describe('orchestration federation effects', () => { + it('uses exact terminal handles instead of display titles for setup identity', () => { + const effects: FederationEffect[] = [] + + appendFederationTerminalEffects( + effects, + [ + { handle: 'term_agent', title: 'Codex' }, + { handle: 'term_configured', title: 'Setup' }, + { handle: 'term_setup', title: 'PowerShell' } + ], + 'term_agent', + 'term_setup' + ) + appendFederationSetupEffect(effects, { + requested: 'run', + effective: 'run', + source: 'orchestration_default', + hookFound: true, + startupPolicy: 'start-immediately', + state: 'running' + }) + + expect(effects).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: 'term_configured', role: 'configured_tab' }), + expect.objectContaining({ id: 'term_setup', role: 'setup' }), + expect.objectContaining({ kind: 'setup', terminalId: 'term_setup' }) + ]) + ) + }) +}) diff --git a/src/main/runtime/rpc/methods/orchestration-federation-effects.ts b/src/main/runtime/rpc/methods/orchestration-federation-effects.ts new file mode 100644 index 00000000000..da0bcf92894 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-federation-effects.ts @@ -0,0 +1,79 @@ +export type FederationEffect = { + kind: 'worktree' | 'terminal' | 'setup' | 'dispatch_input' + action?: string + role?: string + id?: string + state?: string + tabId?: string + leafId?: string + requested?: string + effective?: string + source?: string + hookFound?: boolean + startupPolicy?: string + terminalId?: string +} + +export function appendFederationTerminalEffects( + effects: FederationEffect[], + terminals: { handle: string; title: string | null; tabId?: string; leafId?: string }[], + agentHandle: string, + setupHandle?: string +): void { + for (const terminal of terminals) { + effects.push({ + kind: 'terminal', + role: + terminal.handle === agentHandle + ? 'agent' + : terminal.handle === setupHandle + ? 'setup' + : 'configured_tab', + action: terminal.handle === agentHandle ? 'reused_agent_terminal' : 'created', + id: terminal.handle, + tabId: terminal.tabId, + leafId: terminal.leafId + }) + } +} + +export function appendFederationSetupEffect( + effects: FederationEffect[], + setup: { + requested: string + effective: string + source: string + hookFound: boolean + startupPolicy: string + state: string + } +): void { + const setupTerminal = effects.find( + (effect) => effect.kind === 'terminal' && effect.role === 'setup' + ) + effects.push({ + kind: 'setup', + action: setup.requested, + ...setup, + terminalId: setupTerminal?.id + }) +} + +export function isFederationResidualEffect(effect: FederationEffect): boolean { + return Boolean(effect.action?.startsWith('created') || effect.action === 'reused_agent_terminal') +} + +export function isFederationEffectUnknown(error: unknown, stage: string): boolean { + const code = + error && typeof error === 'object' && typeof (error as { code?: unknown }).code === 'string' + ? (error as { code: string }).code + : '' + if (code === 'operation_unknown') { + return true + } + if (!['worktree_create', 'terminal_create', 'dispatch_input'].includes(stage)) { + return false + } + const message = error instanceof Error ? error.message : String(error) + return /connection|disconnect|timed?\s*out|runtime changed|outcome unknown/i.test(message) +} diff --git a/src/main/runtime/rpc/methods/orchestration-federation-folder-placement.test.ts b/src/main/runtime/rpc/methods/orchestration-federation-folder-placement.test.ts new file mode 100644 index 00000000000..8806c92fe47 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-federation-folder-placement.test.ts @@ -0,0 +1,57 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { OrcaRuntimeService } from '../../orca-runtime' +import { OrchestrationDb } from '../../orchestration/db' +import { ORCHESTRATION_METHODS } from './orchestration' + +describe('orchestration federated folder placement', () => { + let db: OrchestrationDb | undefined + + afterEach(() => db?.close()) + + it('rejects a new folder workspace before accepting the remote attachment', async () => { + db = new OrchestrationDb(':memory:') + const runtime = new OrcaRuntimeService() + runtime.setOrchestrationDb(db) + vi.spyOn(runtime, 'validateOrchestrationAgentLauncher').mockImplementation(() => {}) + vi.spyOn(runtime, 'showRepo').mockResolvedValue({ + id: 'folder-repo', + kind: 'folder' + } as never) + const method = ORCHESTRATION_METHODS.find( + (candidate) => candidate.name === 'orchestration.federationAttachStart' + ) + if (!method) { + throw new Error('federationAttachStart method is not registered') + } + + await expect( + method.handler( + method.params!.parse({ + dispatchId: 'ctx_folder', + taskId: 'task_folder', + taskSpec: 'work in folder', + protocolVersion: 1, + worktree: 'new-top-level', + repo: 'folder-repo', + name: 'folder-worker', + agent: 'codex' + }), + { + runtime, + orchestrationMutation: { + callerFingerprint: 'home_peer', + requestId: 'request_folder', + method: 'orchestration.federationAttachStart', + payloadHash: 'folder_payload' + } + } + ) + ).rejects.toMatchObject({ + code: 'invalid_argument', + message: + 'Folder projects cannot create orchestration worktrees; use an exact existing folder workspace.' + }) + expect(db.getRemoteDispatchAttachment('ctx_folder')).toBeUndefined() + expect(db.getMutationReceipt('home_peer', 'request_folder')).toBeUndefined() + }) +}) diff --git a/src/main/runtime/rpc/methods/orchestration-federation-methods.ts b/src/main/runtime/rpc/methods/orchestration-federation-methods.ts new file mode 100644 index 00000000000..171125602a0 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-federation-methods.ts @@ -0,0 +1,10 @@ +import type { RpcMethod } from '../core' +import { ORCHESTRATION_FEDERATION_CONTROL_METHODS } from './orchestration-federation-control' +import { ORCHESTRATION_FEDERATION_RELAY_METHODS } from './orchestration-federation-relay' +import { ORCHESTRATION_FEDERATION_ATTACH_METHODS } from './orchestration-federation' + +export const ORCHESTRATION_FEDERATION_METHODS: RpcMethod[] = [ + ...ORCHESTRATION_FEDERATION_ATTACH_METHODS, + ...ORCHESTRATION_FEDERATION_RELAY_METHODS, + ...ORCHESTRATION_FEDERATION_CONTROL_METHODS +] diff --git a/src/main/runtime/rpc/methods/orchestration-federation-output.test.ts b/src/main/runtime/rpc/methods/orchestration-federation-output.test.ts new file mode 100644 index 00000000000..9617c05c125 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-federation-output.test.ts @@ -0,0 +1,312 @@ +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { RuntimeRpcResponse } from '../../../../shared/runtime-rpc-envelope' +import { ORCHESTRATION_CONTRACT_VERSION } from '../../../../shared/protocol-version' +import { OrcaRuntimeService } from '../../orca-runtime' +import { OrchestrationDb } from '../../orchestration/db' +import type { OrchestrationEnvironmentTransport } from '../../orchestration/environment-transport' +import type { RpcRequest } from '../core' +import { RpcDispatcher } from '../dispatcher' +import { ORCHESTRATION_METHODS } from './orchestration' + +describe('orchestration federated worker output', () => { + const databases: OrchestrationDb[] = [] + let homeDb: OrchestrationDb + let workerDb: OrchestrationDb + let homeRuntime: OrcaRuntimeService + let workerRuntime: OrcaRuntimeService + let homeDispatcher: RpcDispatcher + let workerDispatcher: RpcDispatcher + let workerSupportsStructuredRead: boolean + + beforeEach(() => { + homeDb = new OrchestrationDb(':memory:') + workerDb = new OrchestrationDb(':memory:') + databases.push(homeDb, workerDb) + workerRuntime = new OrcaRuntimeService() + workerRuntime.setOrchestrationDb(workerDb) + workerDispatcher = new RpcDispatcher({ + runtime: workerRuntime, + methods: ORCHESTRATION_METHODS + }) + workerSupportsStructuredRead = true + const transport: OrchestrationEnvironmentTransport = { + resolve: () => ({ + environmentId: 'environment_windows', + name: 'windows', + peerFingerprint: 'windows_peer_fingerprint' + }), + call: async (_selector, method, params, _timeoutMs, envelope) => { + if (method === 'status.get') { + return { + id: 'status', + ok: true, + result: workerRuntime.getStatus(), + _meta: { runtimeId: workerRuntime.getRuntimeId() } + } + } + if (method === 'orchestration.federationReadOutput' && !workerSupportsStructuredRead) { + return { + id: `remote_${method}`, + ok: false, + error: { code: 'method_not_found', message: `Unknown method: ${method}` } + } + } + return (await workerDispatcher.dispatch({ + id: `remote_${method}`, + authToken: 'run-home-device-token', + method, + params, + orchestrationContractVersion: envelope?.orchestrationContractVersion, + orchestrationRequestId: envelope?.orchestrationRequestId, + orchestrationCapability: envelope?.orchestrationCapability + })) as RuntimeRpcResponse + } + } + homeRuntime = new OrcaRuntimeService(null, undefined, { + orchestrationEnvironmentTransport: transport + }) + homeRuntime.setOrchestrationDb(homeDb) + homeDispatcher = new RpcDispatcher({ + runtime: homeRuntime, + methods: ORCHESTRATION_METHODS + }) + vi.spyOn(homeRuntime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === 'term_coord' ? 'tab_coord:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' : null + ) + configureWorkerRuntime(workerRuntime) + }) + + afterEach(() => { + homeRuntime.stopOrchestrationFederationRelay() + for (const db of databases.splice(0)) { + db.close() + } + }) + + function createHomeTask() { + const run = homeDb.createRun({ + objective: 'Mac to Windows output', + coordinatorHandle: 'term_coord', + coordinatorPaneKey: 'tab_coord:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' + }) + return homeDb.createTask({ spec: 'Read Windows worker output', runId: run.id }) + } + + function startRequest(taskId: string): RpcRequest { + return { + id: 'rpc_worker_start', + authToken: 'coordinator-token', + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationRequestId: 'request_windows_worker', + method: 'orchestration.workerStart', + params: { + task: taskId, + from: 'term_coord', + on: 'windows', + worktree: 'new-top-level', + repo: 'id:windows-repo', + name: 'windows-output', + agent: 'codex' + } + } + } + + function configureWorkerRuntime(runtime: OrcaRuntimeService): void { + vi.spyOn(runtime, 'validateOrchestrationAgentLauncher').mockImplementation(() => {}) + vi.spyOn(runtime, 'showRepo').mockResolvedValue({ + id: 'windows-repo', + kind: 'git' + } as never) + vi.spyOn(runtime, 'createManagedWorktree').mockResolvedValue({ + worktree: { id: 'repo::windows-worktree', repoId: 'repo' }, + startupTerminal: { spawned: true, handle: 'term_windows_worker' }, + setupReceipt: { + requested: 'run', + hookFound: false, + startupPolicy: 'start-immediately', + state: 'not_configured' + } + } as never) + vi.spyOn(runtime, 'listTerminals').mockResolvedValue({ + terminals: [{ handle: 'term_windows_worker', title: 'Codex' }], + totalCount: 1, + truncated: false + } as never) + vi.spyOn(runtime, 'waitForTerminal').mockResolvedValue({ + handle: 'term_windows_worker', + condition: 'tui-idle', + satisfied: true, + status: 'running', + exitCode: null + }) + vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue( + 'tab_worker:bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb' + ) + vi.spyOn(runtime, 'getTerminalProcessIncarnation').mockReturnValue('windows_runtime:pty:1') + vi.spyOn(runtime, 'getTerminalOrchestrationCliCommand').mockReturnValue('orca') + vi.spyOn(runtime, 'sendTerminalAgentPrompt').mockResolvedValue({ + handle: 'term_windows_worker', + accepted: true, + bytesWritten: 1 + }) + vi.spyOn(runtime, 'showTerminal').mockResolvedValue({ + handle: 'term_windows_worker', + worktreeId: 'repo::windows-worktree', + status: 'running' + } as never) + vi.spyOn(runtime, 'readTerminal').mockResolvedValue({ + handle: 'term_windows_worker', + status: 'running', + tail: ['remote output'], + truncated: false, + nextCursor: '1' + }) + } + + async function startRemoteWorker(): Promise { + const task = createHomeTask() + await homeDispatcher.dispatch(startRequest(task.id)) + return homeDb.getDispatchContext(task.id)!.id + } + + it('routes show and read by Dispatch without repeating the worker server', async () => { + const dispatchId = await startRemoteWorker() + + const shown = await homeDispatcher.dispatch({ + id: 'rpc_remote_show', + authToken: 'coordinator-token', + method: 'orchestration.workerShow', + params: { dispatch: dispatchId } + }) + const read = await homeDispatcher.dispatch({ + id: 'rpc_remote_read', + authToken: 'coordinator-token', + method: 'orchestration.workerRead', + params: { dispatch: dispatchId, limit: 20 } + }) + + expect(shown).toMatchObject({ + ok: true, + result: { + server: { environmentId: 'environment_windows', name: 'windows' }, + observation: { status: 'running', exactWorker: true }, + terminal: { handle: 'term_windows_worker' } + } + }) + expect(read).toMatchObject({ + ok: true, + result: { + source: 'terminal', + fallbackReason: 'session_not_reported', + server: { environmentId: 'environment_windows', name: 'windows' }, + terminal: { tail: ['remote output'] } + } + }) + }) + + it('keeps an opaque terminal cursor across mixed server versions', async () => { + const dispatchId = await startRemoteWorker() + workerSupportsStructuredRead = false + + const automatic = await homeDispatcher.dispatch({ + id: 'rpc_remote_legacy_read', + authToken: 'coordinator-token', + method: 'orchestration.workerRead', + params: { dispatch: dispatchId } + }) + const cursor = (automatic as { result: { cursor: string } }).result.cursor + const continued = await homeDispatcher.dispatch({ + id: 'rpc_remote_legacy_continue', + authToken: 'coordinator-token', + method: 'orchestration.workerRead', + params: { dispatch: dispatchId, cursor } + }) + const required = await homeDispatcher.dispatch({ + id: 'rpc_remote_legacy_transcript', + authToken: 'coordinator-token', + method: 'orchestration.workerRead', + params: { dispatch: dispatchId, source: 'transcript' } + }) + + expect(automatic).toMatchObject({ + ok: true, + result: { + source: 'terminal', + fallbackReason: 'remote_capability_unavailable', + terminal: { tail: ['remote output'] } + } + }) + expect(cursor).toMatch(/^owr1_/) + expect(continued).toMatchObject({ + ok: true, + result: { + source: 'terminal', + fallbackReason: 'remote_capability_unavailable' + } + }) + expect((continued as { result: { cursor: string } }).result.cursor).toMatch(/^owr1_/) + expect(required).toMatchObject({ + ok: false, + error: { + code: 'transcript_required', + data: { reason: 'remote_capability_unavailable' } + } + }) + }) + + it('reads the exact transcript on the worker server without leaking its path home', async () => { + const dispatchId = await startRemoteWorker() + const directory = await mkdtemp(join(tmpdir(), 'orca-federated-worker-output-')) + const transcriptPath = join(directory, 'windows-session.jsonl') + await writeFile( + transcriptPath, + `${JSON.stringify({ + type: 'event_msg', + payload: { id: 'remote-message', type: 'agent_message', message: 'Windows result' } + })}\n` + ) + vi.spyOn(workerRuntime, 'getExactWorkerProviderSession').mockReturnValue({ + paneKey: 'tab_worker:bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb', + processIncarnation: 'windows_runtime:pty:1', + agent: 'codex', + providerSession: { + key: 'session_id', + id: 'windows-session', + transcriptPath + }, + observedAt: Date.now() + }) + + try { + const response = await homeDispatcher.dispatch({ + id: 'rpc_remote_transcript_read', + authToken: 'coordinator-token', + method: 'orchestration.workerRead', + params: { dispatch: dispatchId } + }) + + expect(response).toMatchObject({ + ok: true, + result: { + source: 'transcript', + provider: 'codex', + server: { environmentId: 'environment_windows' }, + transcript: { + messages: [ + { + id: 'remote-message', + blocks: [{ type: 'text', text: 'Windows result' }] + } + ] + } + } + }) + expect(JSON.stringify(response)).not.toContain(transcriptPath) + } finally { + await rm(directory, { recursive: true, force: true }) + } + }) +}) diff --git a/src/main/runtime/rpc/methods/orchestration-federation-relay.ts b/src/main/runtime/rpc/methods/orchestration-federation-relay.ts new file mode 100644 index 00000000000..fbc3e97fb3b --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-federation-relay.ts @@ -0,0 +1,179 @@ +import { z } from 'zod' +import { ORCHESTRATION_FEDERATION_CONTROL_MAIL_PROTOCOL_VERSION } from '../../../../shared/protocol-version' +import { importFederatedControlMessage } from '../../orchestration/federation-control-message' +import { OrchestrationError } from '../../orchestration/orchestration-error' +import { defineMethod, type RpcMethod } from '../core' +import { OptionalFiniteNumber, requiredString } from '../schemas' + +const FederationPullParams = z.object({ + dispatchId: requiredString('Missing Dispatch ID'), + afterSequence: OptionalFiniteNumber, + limit: OptionalFiniteNumber +}) + +const FederationAckParams = z.object({ + dispatchId: requiredString('Missing Dispatch ID'), + throughSequence: z.number().int().nonnegative() +}) + +const FederationImportParams = z.object({ + dispatchId: requiredString('Missing Dispatch ID'), + items: z.array( + z.object({ + dispatch_id: requiredString('Missing item Dispatch ID'), + direction: z.literal('to_worker'), + sequence: z.number().int().positive(), + message_id: requiredString('Missing relay message ID'), + kind: requiredString('Missing relay kind'), + payload: requiredString('Missing relay payload') + }) + ) +}) + +export const ORCHESTRATION_FEDERATION_RELAY_METHODS: RpcMethod[] = [ + defineMethod({ + name: 'orchestration.federationPull', + params: FederationPullParams, + handler: (params, { runtime, authenticatedCallerFingerprint }) => { + requireHomeAttachment(runtime, params.dispatchId, authenticatedCallerFingerprint) + return { + dispatchId: params.dispatchId, + runtimeEpoch: runtime.getRuntimeId(), + items: runtime.getOrchestrationDb().listFederationRelay({ + dispatchId: params.dispatchId, + direction: 'to_home', + afterSequence: params.afterSequence ?? 0, + limit: params.limit + }) + } + } + }), + defineMethod({ + name: 'orchestration.federationAck', + params: FederationAckParams, + handler: (params, { runtime, authenticatedCallerFingerprint }) => { + requireHomeAttachment(runtime, params.dispatchId, authenticatedCallerFingerprint) + runtime.getOrchestrationDb().acknowledgeFederationRelay({ + dispatchId: params.dispatchId, + direction: 'to_home', + throughSequence: params.throughSequence + }) + return { dispatchId: params.dispatchId, acknowledgedThrough: params.throughSequence } + } + }), + defineMethod({ + name: 'orchestration.federationImport', + params: FederationImportParams, + handler: (params, { runtime, authenticatedCallerFingerprint }) => { + const db = runtime.getOrchestrationDb() + const attachment = requireHomeAttachment( + runtime, + params.dispatchId, + authenticatedCallerFingerprint + ) + let cursor = attachment.to_worker_imported_sequence + let imported = 0 + for (const item of params.items) { + if (item.dispatch_id !== params.dispatchId || item.sequence > cursor + 1) { + throw new OrchestrationError( + 'operation_unknown', + `Home relay for ${params.dispatchId} is not contiguous after sequence ${cursor}.` + ) + } + if (item.sequence <= cursor) { + continue + } + const currentAttachment = requireHomeAttachment( + runtime, + params.dispatchId, + authenticatedCallerFingerprint + ) + if (currentAttachment.state !== 'ready') { + throw new OrchestrationError( + 'dispatch_inactive', + `Remote Dispatch ${params.dispatchId} is not active.` + ) + } + if (item.kind === 'reply') { + const reply = parseFederatedReply(item.payload) + db.answerRemoteQuestion({ + messageId: reply.questionId, + dispatchId: params.dispatchId, + answerMessageId: reply.answerMessageId, + body: reply.body + }) + runtime.notifyMessageArrived(`dispatch:${params.dispatchId}`, 'status') + } else if (item.kind === 'control_message') { + if ( + currentAttachment.protocol_version < + ORCHESTRATION_FEDERATION_CONTROL_MAIL_PROTOCOL_VERSION + ) { + throw new OrchestrationError( + 'capability_unsupported', + `Remote Dispatch ${params.dispatchId} does not support coordinator control mail.` + ) + } + const controlMessage = importFederatedControlMessage(db, { + dispatchId: params.dispatchId, + messageId: item.message_id, + payload: item.payload + }) + imported += controlMessage.imported ? 1 : 0 + if (controlMessage.imported) { + runtime.notifyMessageArrived(`dispatch:${params.dispatchId}`, controlMessage.type) + } + } else { + throw new OrchestrationError( + 'invalid_argument', + `Federated worker relay kind ${item.kind} is not supported.` + ) + } + cursor = item.sequence + db.setRemoteWorkerImportSequence(params.dispatchId, cursor) + } + return { dispatchId: params.dispatchId, acknowledgedThrough: cursor, imported } + } + }) +] + +function requireHomeAttachment( + runtime: Parameters[1]['runtime'], + dispatchId: string, + callerFingerprint: string | undefined +) { + const attachment = runtime.getOrchestrationDb().getRemoteDispatchAttachment(dispatchId) + if (!attachment || attachment.home_peer_fingerprint !== callerFingerprint) { + throw new OrchestrationError( + 'dispatch_not_found', + `Remote Dispatch ${dispatchId} was not found for this Run home.` + ) + } + return attachment +} + +function parseFederatedReply(payload: string): { + questionId: string + answerMessageId: string + body: string +} { + let parsed: unknown + try { + parsed = JSON.parse(payload) + } catch { + throw new OrchestrationError('invalid_argument', 'Federated reply payload is invalid JSON.') + } + const reply = parsed as Record | null + if ( + !reply || + typeof reply.questionId !== 'string' || + typeof reply.answerMessageId !== 'string' || + typeof reply.body !== 'string' + ) { + throw new OrchestrationError('invalid_argument', 'Federated reply payload is incomplete.') + } + return { + questionId: reply.questionId, + answerMessageId: reply.answerMessageId, + body: reply.body + } +} diff --git a/src/main/runtime/rpc/methods/orchestration-federation-setup.test.ts b/src/main/runtime/rpc/methods/orchestration-federation-setup.test.ts new file mode 100644 index 00000000000..70c08e2ce95 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-federation-setup.test.ts @@ -0,0 +1,196 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { OrcaRuntimeService } from '../../orca-runtime' +import { OrchestrationDb } from '../../orchestration/db' +import { ORCHESTRATION_METHODS } from './orchestration' +import { monitorFederatedSetup } from './orchestration-federation-setup' + +describe('orchestration federated setup evidence', () => { + const databases: OrchestrationDb[] = [] + const runtimes: OrcaRuntimeService[] = [] + + afterEach(() => { + for (const runtime of runtimes.splice(0)) { + runtime.stopOrchestrationFederationRelay() + } + for (const db of databases.splice(0)) { + db.close() + } + }) + + function createRuntime(): { db: OrchestrationDb; runtime: OrcaRuntimeService } { + const db = new OrchestrationDb(':memory:') + const runtime = new OrcaRuntimeService() + runtime.setOrchestrationDb(db) + databases.push(db) + runtimes.push(runtime) + return { db, runtime } + } + + it('records remote setup evidence once without changing attachment lifecycle', async () => { + const { db, runtime } = createRuntime() + const dispatchId = 'ctx_remote_setup' + const effects = [ + { + kind: 'terminal' as const, + role: 'setup', + action: 'created', + id: 'term_remote_setup' + }, + { + kind: 'setup' as const, + action: 'run', + state: 'running' + }, + { + kind: 'dispatch_input' as const, + role: 'agent', + id: 'term_remote_worker', + state: 'accepted' + } + ] + db.createRemoteDispatchAttachment({ + dispatchId, + taskId: 'task_remote_setup', + homePeerFingerprint: 'home_peer', + protocolVersion: 1, + runtimeEpoch: runtime.getRuntimeId(), + mutationReceipt: { + callerFingerprint: 'home_peer', + requestId: 'request_remote_setup', + method: 'orchestration.federationAttachStart', + payloadHash: 'remote_setup_payload' + } + }) + db.prepareRemoteAttachmentAuthority({ + dispatchId, + paneKey: 'tab_worker:leaf_worker', + processIncarnation: 'worker_epoch:pty:1', + worktreeId: 'repo::remote-worktree', + terminalHandle: 'term_remote_worker', + setupState: 'running', + effects + }) + db.markRemoteAttachmentReady(dispatchId) + vi.spyOn(runtime, 'waitForSetupTerminalCompletion').mockResolvedValue({ exitCode: 1 }) + const monitorArgs = { + runtime, + db, + dispatchId, + worktreeId: 'repo::remote-worktree', + terminalHandle: 'term_remote_worker', + setup: { + requested: 'run' as const, + effective: 'run' as const, + source: 'orchestration_default', + hookFound: true, + startupPolicy: 'start-immediately' as const, + state: 'running' as const + }, + effects + } + + monitorFederatedSetup(monitorArgs) + monitorFederatedSetup(monitorArgs) + + await vi.waitFor(() => + expect(db.getRemoteDispatchAttachment(dispatchId)).toMatchObject({ + state: 'ready', + stage: 'input_accepted', + setup_state: 'failed' + }) + ) + expect( + db.listFederationRelay({ dispatchId, direction: 'to_home', afterSequence: 0 }) + ).toHaveLength(1) + expect(JSON.parse(db.getRemoteDispatchAttachment(dispatchId)?.effects ?? '[]')).toEqual( + expect.arrayContaining([ + expect.objectContaining({ kind: 'dispatch_input', state: 'accepted' }) + ]) + ) + }) + + it('refreshes home setup evidence without changing a ready Dispatch lifecycle', async () => { + const { db, runtime } = createRuntime() + const run = db.createRun({ + objective: 'Observe remote setup', + coordinatorHandle: 'term_coord', + coordinatorPaneKey: 'tab_coord:leaf_coord' + }) + const task = db.createTask({ spec: 'remote setup', runId: run.id }) + const started = db.createStartingWorkerDispatch({ + taskId: task.id, + startOptions: {}, + runtimeEpoch: runtime.getRuntimeId(), + federation: { + environmentId: 'environment_windows', + environmentName: 'windows', + peerFingerprint: 'windows_peer', + protocolVersion: 1 + } + }) + db.recordWorkerStage({ + dispatchId: started.dispatch.id, + stage: 'terminal_readying', + setupState: 'running', + effects: [ + { kind: 'setup', action: 'run', state: 'running' }, + { + kind: 'dispatch_input', + role: 'agent', + id: 'term_remote_worker', + state: 'accepted' + } + ] + }) + db.markWorkerDispatchReady(started.dispatch.id) + vi.spyOn(runtime, 'resolveOrchestrationWorkerServer').mockReturnValue({ + environmentId: 'environment_windows', + name: 'windows', + peerFingerprint: 'windows_peer' + }) + vi.spyOn(runtime, 'callOrchestrationWorkerServer').mockResolvedValue({ + runtimeEpoch: 'windows_epoch', + attachment: { + state: 'ready', + stage: 'input_accepted', + last_error: null, + worktree_id: 'repo::remote-worktree', + terminal_handle: 'term_remote_worker', + setup_state: 'failed', + effects: [ + { kind: 'setup', action: 'run', state: 'failed' }, + { + kind: 'dispatch_input', + role: 'agent', + id: 'term_remote_worker', + state: 'accepted' + } + ], + residualResources: [] + }, + terminal: { handle: 'term_remote_worker', connected: true }, + observation: { status: 'running', exactWorker: true } + }) + const workerShow = ORCHESTRATION_METHODS.find( + (method) => method.name === 'orchestration.workerShow' + ) + if (!workerShow) { + throw new Error('workerShow method is not registered') + } + + await expect( + workerShow.handler(workerShow.params!.parse({ dispatch: started.dispatch.id }), { runtime }) + ).resolves.toMatchObject({ + worker: { + state: 'ready', + stage: 'input_accepted', + setup_state: 'failed', + effects: expect.arrayContaining([ + expect.objectContaining({ kind: 'setup', state: 'failed' }), + expect.objectContaining({ kind: 'dispatch_input', state: 'accepted' }) + ]) + } + }) + expect(db.getTask(task.id)?.status).toBe('dispatched') + }) +}) diff --git a/src/main/runtime/rpc/methods/orchestration-federation-setup.ts b/src/main/runtime/rpc/methods/orchestration-federation-setup.ts new file mode 100644 index 00000000000..3f35e2c46ca --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-federation-setup.ts @@ -0,0 +1,99 @@ +import type { OrcaRuntimeService } from '../../orca-runtime' +import type { OrchestrationDb } from '../../orchestration/db' +import { applyWaitForSetupOutcome, type WorkerSetupReceipt } from './orchestration-worker-topology' +import { + isFederationResidualEffect, + type FederationEffect +} from './orchestration-federation-effects' + +type FederationSetupStageArgs = { + db: OrchestrationDb + dispatchId: string + worktreeId: string + terminalHandle: string + setup: WorkerSetupReceipt + effects: FederationEffect[] +} + +function recordStage(args: FederationSetupStageArgs, stage: string): void { + args.db.recordRemoteAttachmentStage({ + dispatchId: args.dispatchId, + stage, + worktreeId: args.worktreeId, + terminalHandle: args.terminalHandle, + setupState: args.setup.state, + effects: args.effects, + residualResources: args.effects.filter(isFederationResidualEffect) + }) +} + +export function persistFederatedReadinessStage(args: FederationSetupStageArgs): void { + recordStage(args, 'terminal_readying') +} + +export function persistFederatedSetupSpawnFailure(args: FederationSetupStageArgs): boolean { + if (args.setup.startupPolicy !== 'wait-for-setup' || args.setup.state !== 'spawn_failed') { + return false + } + recordStage(args, 'setup_start') + return true +} + +export function persistFederatedSetupWaitOutcome( + args: FederationSetupStageArgs & { wait: { satisfied: boolean; status: string } } +): void { + applyWaitForSetupOutcome(args.setup, args.effects, args.wait) + if (args.setup.startupPolicy === 'wait-for-setup') { + recordStage(args, args.setup.state === 'failed' ? 'setup_failed' : 'setup_settled') + } +} + +export function monitorFederatedSetup( + args: FederationSetupStageArgs & { runtime: OrcaRuntimeService } +): void { + const setupTerminal = args.effects.find( + (effect) => effect.kind === 'terminal' && effect.role === 'setup' && effect.id + ) + if ( + !setupTerminal?.id || + args.setup.startupPolicy !== 'start-immediately' || + args.setup.state !== 'running' + ) { + return + } + void args.runtime + .waitForSetupTerminalCompletion(setupTerminal.id) + .then((completion) => { + const setupState = completion.exitCode === 0 ? 'succeeded' : 'failed' + const effects = args.effects.map((effect) => + effect.kind === 'setup' ? { ...effect, state: setupState } : effect + ) + const evidence = args.db.updateRemoteAttachmentSetupEvidence({ + dispatchId: args.dispatchId, + setupState, + effects + }) + if (!evidence.changed) { + return + } + args.db.enqueueFederationRelay({ + dispatchId: args.dispatchId, + direction: 'to_home', + kind: 'status', + payload: JSON.stringify({ + from: `dispatch:${args.dispatchId}`, + subject: `Setup ${setupState} for worker ${args.dispatchId}`, + body: '', + type: 'status', + priority: setupState === 'failed' ? 'high' : 'normal', + threadId: null, + payload: JSON.stringify({ + dispatchId: args.dispatchId, + setupState, + terminalHandle: setupTerminal.id + }) + }) + }) + }) + .catch(() => undefined) +} diff --git a/src/main/runtime/rpc/methods/orchestration-federation-start-receipt.ts b/src/main/runtime/rpc/methods/orchestration-federation-start-receipt.ts new file mode 100644 index 00000000000..b7e2da76531 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-federation-start-receipt.ts @@ -0,0 +1,32 @@ +import type { OrchestrationDb } from '../../orchestration/db' +import { isFederationEffectUnknown } from './orchestration-federation-effects' +import type { WorkerSetupReceipt } from './orchestration-worker-topology' + +export function failFederatedAttachmentWithReceipt(args: { + db: OrchestrationDb + dispatchId: string + runtimeEpoch: string + failedStage: string + error: unknown + setup: WorkerSetupReceipt +}): unknown { + const reason = args.error instanceof Error ? args.error.message : String(args.error) + const unknown = isFederationEffectUnknown(args.error, args.failedStage) + const attachment = args.db.failRemoteAttachment( + args.dispatchId, + args.failedStage, + reason, + unknown + ) + return { + dispatchId: args.dispatchId, + state: attachment.state === 'start_unknown' ? 'outcome_unknown' : attachment.state, + stage: attachment.stage, + runtimeEpoch: args.runtimeEpoch, + failedStage: args.failedStage, + lastError: reason, + setup: args.setup, + effects: JSON.parse(attachment.effects) as unknown[], + residualResources: JSON.parse(attachment.residual_resources) as unknown[] + } +} diff --git a/src/main/runtime/rpc/methods/orchestration-federation-start-schema.ts b/src/main/runtime/rpc/methods/orchestration-federation-start-schema.ts new file mode 100644 index 00000000000..a657ff7d09c --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-federation-start-schema.ts @@ -0,0 +1,21 @@ +import { z } from 'zod' +import { OptionalFiniteNumber, OptionalString, requiredString } from '../schemas' + +export const FederationAttachStartParams = z.object({ + dispatchId: requiredString('Missing Dispatch ID'), + taskId: requiredString('Missing Task ID'), + taskSpec: requiredString('Missing Task spec'), + protocolVersion: z.union([z.literal(1), z.literal(2)]), + worktree: requiredString('Missing remote worktree selector'), + name: OptionalString, + repo: OptionalString, + baseBranch: OptionalString, + displayName: OptionalString, + comment: OptionalString, + setup: z.enum(['run', 'skip', 'inherit']).optional(), + setupSource: z.enum(['explicit_request', 'orchestration_default']).optional(), + terminal: OptionalString, + agent: OptionalString, + timeoutMs: OptionalFiniteNumber, + devMode: z.boolean().optional() +}) diff --git a/src/main/runtime/rpc/methods/orchestration-federation.test.ts b/src/main/runtime/rpc/methods/orchestration-federation.test.ts new file mode 100644 index 00000000000..eb76d094368 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-federation.test.ts @@ -0,0 +1,862 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { RuntimeRpcResponse } from '../../../../shared/runtime-rpc-envelope' +import { + ORCHESTRATION_CONTRACT_VERSION, + ORCHESTRATION_FEDERATION_CONTROL_MAIL_RUNTIME_CAPABILITY +} from '../../../../shared/protocol-version' +import { OrcaRuntimeService } from '../../orca-runtime' +import { OrchestrationDb } from '../../orchestration/db' +import type { OrchestrationEnvironmentTransport } from '../../orchestration/environment-transport' +import { RpcDispatcher } from '../dispatcher' +import type { RpcRequest } from '../core' +import { ORCHESTRATION_METHODS } from './orchestration' + +describe('orchestration federation', () => { + const databases: OrchestrationDb[] = [] + let homeDb: OrchestrationDb + let workerDb: OrchestrationDb + let homeRuntime: OrcaRuntimeService + let workerRuntime: OrcaRuntimeService + let homeDispatcher: RpcDispatcher + let workerDispatcher: RpcDispatcher + let workerCapabilities: string[] + let workerPeerFingerprint: string + let loseNextAckResponse: boolean + + beforeEach(() => { + homeDb = new OrchestrationDb(':memory:') + workerDb = new OrchestrationDb(':memory:') + databases.push(homeDb, workerDb) + workerRuntime = new OrcaRuntimeService() + workerRuntime.setOrchestrationDb(workerDb) + workerDispatcher = new RpcDispatcher({ + runtime: workerRuntime, + methods: ORCHESTRATION_METHODS + }) + workerCapabilities = [...(workerRuntime.getStatus().capabilities ?? [])] + workerPeerFingerprint = 'windows_peer_fingerprint' + loseNextAckResponse = false + const transport: OrchestrationEnvironmentTransport = { + resolve: () => ({ + environmentId: 'environment_windows', + name: 'windows', + peerFingerprint: workerPeerFingerprint + }), + call: async (_selector, method, params, _timeoutMs, envelope) => { + if (method === 'status.get') { + return { + id: 'status', + ok: true, + result: { ...workerRuntime.getStatus(), capabilities: workerCapabilities }, + _meta: { runtimeId: workerRuntime.getRuntimeId() } + } + } + const response = (await workerDispatcher.dispatch({ + id: `remote_${method}`, + authToken: 'run-home-device-token', + method, + params, + orchestrationContractVersion: envelope?.orchestrationContractVersion, + orchestrationRequestId: envelope?.orchestrationRequestId, + orchestrationCapability: envelope?.orchestrationCapability + })) as RuntimeRpcResponse + if (method === 'orchestration.federationAck' && loseNextAckResponse) { + loseNextAckResponse = false + throw new Error('connection lost after acknowledgment') + } + return response + } + } + homeRuntime = new OrcaRuntimeService(null, undefined, { + orchestrationEnvironmentTransport: transport + }) + homeRuntime.setOrchestrationDb(homeDb) + homeDispatcher = new RpcDispatcher({ + runtime: homeRuntime, + methods: ORCHESTRATION_METHODS + }) + vi.spyOn(homeRuntime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === 'term_coord' ? 'tab_coord:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' : null + ) + configureWorkerRuntime(workerRuntime) + }) + + afterEach(() => { + homeRuntime.stopOrchestrationFederationRelay() + for (const db of databases.splice(0)) { + db.close() + } + }) + + function createHomeTask() { + const run = homeDb.createRun({ + objective: 'Mac to Windows', + coordinatorHandle: 'term_coord', + coordinatorPaneKey: 'tab_coord:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' + }) + return homeDb.createTask({ spec: 'Audit Windows behavior', runId: run.id }) + } + + function startRequest(taskId: string, overrides: Record = {}): RpcRequest { + return { + id: 'rpc_worker_start', + authToken: 'coordinator-token', + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationRequestId: 'request_windows_worker', + method: 'orchestration.workerStart', + params: { + task: taskId, + from: 'term_coord', + on: 'windows', + worktree: 'new-top-level', + repo: 'id:windows-repo', + name: 'windows-audit', + agent: 'codex', + ...overrides + } + } + } + + function configureWorkerRuntime(runtime: OrcaRuntimeService): void { + vi.spyOn(runtime, 'validateOrchestrationAgentLauncher').mockImplementation(() => {}) + vi.spyOn(runtime, 'showRepo').mockResolvedValue({ + id: 'windows-repo', + kind: 'git' + } as never) + vi.spyOn(runtime, 'createManagedWorktree').mockResolvedValue({ + worktree: { id: 'repo::windows-worktree', repoId: 'repo' }, + startupTerminal: { spawned: true, handle: 'term_windows_worker' }, + setupReceipt: { + requested: 'run', + hookFound: true, + startupPolicy: 'start-immediately', + state: 'running' + } + } as never) + vi.spyOn(runtime, 'listTerminals').mockResolvedValue({ + terminals: [ + { handle: 'term_windows_worker', title: 'Codex' }, + { handle: 'term_windows_setup', title: 'Setup' } + ], + totalCount: 2, + truncated: false + } as never) + vi.spyOn(runtime, 'waitForTerminal').mockResolvedValue({ + handle: 'term_windows_worker', + condition: 'tui-idle', + satisfied: true, + status: 'running', + exitCode: null + }) + vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue( + 'tab_worker:bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb' + ) + vi.spyOn(runtime, 'getTerminalProcessIncarnation').mockReturnValue('windows_runtime:pty:1') + vi.spyOn(runtime, 'getTerminalOrchestrationCliCommand').mockReturnValue('orca') + vi.spyOn(runtime, 'sendTerminalAgentPrompt').mockResolvedValue({ + handle: 'term_windows_worker', + accepted: true, + bytesWritten: 1 + }) + vi.spyOn(runtime, 'showTerminal').mockResolvedValue({ + handle: 'term_windows_worker', + worktreeId: 'repo::windows-worktree', + status: 'running' + } as never) + vi.spyOn(runtime, 'readTerminal').mockResolvedValue({ + handle: 'term_windows_worker', + status: 'running', + entries: [{ cursor: 1, text: 'remote output' }], + nextCursor: '1', + limited: false + } as never) + vi.spyOn(runtime, 'closeTerminal').mockResolvedValue({ + handle: 'term_windows_worker', + closed: true + } as never) + } + + function restartWorkerRuntime(): void { + workerRuntime = new OrcaRuntimeService() + workerRuntime.setOrchestrationDb(workerDb) + configureWorkerRuntime(workerRuntime) + workerDispatcher = new RpcDispatcher({ + runtime: workerRuntime, + methods: ORCHESTRATION_METHODS + }) + workerCapabilities = [...(workerRuntime.getStatus().capabilities ?? [])] + } + + it('starts a remote worker while keeping authoritative Task state at home', async () => { + const task = createHomeTask() + + const response = await homeDispatcher.dispatch(startRequest(task.id)) + + expect(response).toMatchObject({ + ok: true, + result: { + taskId: task.id, + state: 'ready', + server: { environmentId: 'environment_windows', name: 'windows' }, + setup: { source: 'orchestration_default' }, + mutation: { requestId: 'request_windows_worker' } + } + }) + const dispatch = homeDb.getDispatchContext(task.id)! + expect(homeDb.getTask(task.id)?.status).toBe('dispatched') + expect(homeDb.getFederatedDispatch(dispatch.id)).toMatchObject({ + environment_id: 'environment_windows', + environment_name: 'windows', + peer_fingerprint: 'windows_peer_fingerprint', + remote_worktree_id: 'repo::windows-worktree', + remote_terminal_handle: 'term_windows_worker' + }) + expect(workerDb.getRemoteDispatchAttachment(dispatch.id)).toMatchObject({ + task_id: task.id, + protocol_version: 2, + state: 'ready', + worktree_id: 'repo::windows-worktree', + terminal_handle: 'term_windows_worker' + }) + expect(JSON.parse(workerDb.getRemoteDispatchAttachment(dispatch.id)?.effects ?? '[]')).toEqual( + expect.arrayContaining([ + expect.objectContaining({ kind: 'dispatch_input', state: 'accepted' }) + ]) + ) + expect(workerDb.listTasks()).toHaveLength(0) + expect(workerRuntime.sendTerminalAgentPrompt).toHaveBeenCalledWith( + 'term_windows_worker', + expect.stringContaining(`Your task ID is: ${task.id}`) + ) + }) + + it('preserves wait-for-setup gating on the connected worker server', async () => { + vi.mocked(workerRuntime.createManagedWorktree).mockResolvedValueOnce({ + worktree: { id: 'repo::windows-worktree', repoId: 'repo' }, + startupTerminal: { spawned: true, handle: 'term_windows_worker' }, + setupReceipt: { + requested: 'run', + hookFound: true, + startupPolicy: 'wait-for-setup', + state: 'running' + } + } as never) + const task = createHomeTask() + + const response = await homeDispatcher.dispatch(startRequest(task.id, { setup: 'run' })) + + expect(response).toMatchObject({ + ok: true, + result: { + state: 'ready', + setup: { startupPolicy: 'wait-for-setup', state: 'succeeded' }, + effects: expect.arrayContaining([ + expect.objectContaining({ kind: 'setup', state: 'succeeded' }), + expect.objectContaining({ kind: 'dispatch_input', state: 'accepted' }) + ]) + } + }) + expect(response).toHaveProperty('result.setup.source', 'explicit_request') + expect(workerRuntime.sendTerminalAgentPrompt).toHaveBeenCalledOnce() + }) + + it('fails before remote task input when wait-for-setup fails', async () => { + vi.mocked(workerRuntime.createManagedWorktree).mockResolvedValueOnce({ + worktree: { id: 'repo::windows-worktree', repoId: 'repo' }, + startupTerminal: { spawned: true, handle: 'term_windows_worker' }, + setupReceipt: { + requested: 'run', + hookFound: true, + startupPolicy: 'wait-for-setup', + state: 'running' + } + } as never) + vi.mocked(workerRuntime.waitForTerminal).mockResolvedValueOnce({ + handle: 'term_windows_worker', + condition: 'tui-idle', + satisfied: false, + status: 'exited', + exitCode: 1 + }) + const task = createHomeTask() + + const response = await homeDispatcher.dispatch(startRequest(task.id)) + + expect(response).toMatchObject({ + ok: true, + result: { + state: 'failed', + failedStage: 'setup_wait', + setup: { state: 'failed' }, + effects: expect.arrayContaining([ + expect.objectContaining({ kind: 'setup', state: 'failed' }) + ]) + } + }) + expect(homeDb.getTask(task.id)?.status).toBe('failed') + expect(workerRuntime.sendTerminalAgentPrompt).not.toHaveBeenCalled() + }) + + it('rejects control mail before queueing when the worker lacks that capability', async () => { + workerCapabilities = workerCapabilities.filter( + (capability) => capability !== ORCHESTRATION_FEDERATION_CONTROL_MAIL_RUNTIME_CAPABILITY + ) + const task = createHomeTask() + const started = await homeDispatcher.dispatch(startRequest(task.id)) + expect(started).toMatchObject({ ok: true, result: { state: 'ready' } }) + const dispatch = homeDb.getDispatchContext(task.id)! + + const sent = await homeDispatcher.dispatch({ + id: 'send-control-to-old-worker', + authToken: 'coordinator-token', + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationRequestId: 'send-control-to-old-worker-request', + method: 'orchestration.send', + params: { + from: 'term_coord', + to: `dispatch:${dispatch.id}`, + subject: 'Continue', + body: 'This worker cannot receive control mail yet.', + type: 'status' + } + }) + + expect(sent).toMatchObject({ + ok: false, + error: { code: 'capability_unsupported' } + }) + expect(homeDb.listPendingFederationRelay(dispatch.id, 'to_worker')).toHaveLength(0) + }) + + it('durably relays remote completion into the home Run and acknowledges it', async () => { + const task = createHomeTask() + const started = await homeDispatcher.dispatch(startRequest(task.id)) + expect(started.ok).toBe(true) + const dispatch = homeDb.getDispatchContext(task.id)! + const prompt = vi.mocked(workerRuntime.sendTerminalAgentPrompt).mock.calls[0]?.[1] ?? '' + const capability = prompt.match(/--dispatch-capability (dcap_[A-Za-z0-9_-]+)/)?.[1] + expect(capability).toBeTruthy() + + const sent = await workerDispatcher.dispatch({ + id: 'rpc_worker_done', + authToken: 'worker-local-token', + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationRequestId: 'worker_done_request', + orchestrationCapability: capability, + method: 'orchestration.send', + params: { + from: 'term_windows_worker', + subject: 'Windows audit complete', + body: 'Audited Windows behavior. Found no blocker. Nothing remains.', + type: 'worker_done', + payload: JSON.stringify({ + taskId: task.id, + dispatchId: dispatch.id, + outcome: 'succeeded', + filesModified: [] + }) + } + }) + expect(sent).toMatchObject({ + ok: true, + result: { relay: { dispatchId: dispatch.id, accepted: true } } + }) + expect(homeDb.getTask(task.id)?.status).toBe('dispatched') + + await homeRuntime.syncOrchestrationFederation() + + expect(homeDb.getTask(task.id)?.status).toBe('completed') + expect(homeDb.getWorkerDispatch(dispatch.id)?.state).toBe('succeeded') + expect(homeDb.getRunMailboxHistory(task.run_id, 10)).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: expect.stringMatching(/^relay_/), + type: 'worker_done', + subject: 'Windows audit complete' + }) + ]) + ) + expect( + workerDb.listFederationRelay({ + dispatchId: dispatch.id, + direction: 'to_home', + afterSequence: 0 + })[0] + ).toMatchObject({ acked_at: expect.any(String) }) + }) + + it('relays a worker question home and the coordinator answer back', async () => { + const task = createHomeTask() + await homeDispatcher.dispatch(startRequest(task.id)) + const dispatch = homeDb.getDispatchContext(task.id)! + const prompt = vi.mocked(workerRuntime.sendTerminalAgentPrompt).mock.calls[0]?.[1] ?? '' + const capability = prompt.match(/--dispatch-capability (dcap_[A-Za-z0-9_-]+)/)?.[1] + const ask = workerDispatcher.dispatch({ + id: 'rpc_remote_ask', + authToken: 'worker-local-token', + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationRequestId: 'remote_question_request', + orchestrationCapability: capability, + method: 'orchestration.ask', + params: { + from: 'term_windows_worker', + question: 'Should I include slow integration tests?', + options: 'yes,no', + timeoutMs: 60_000 + } + }) + await vi.waitFor(() => + expect( + workerDb.listFederationRelay({ + dispatchId: dispatch.id, + direction: 'to_home', + afterSequence: 0 + }) + ).toHaveLength(1) + ) + + await homeRuntime.syncOrchestrationFederation() + const question = homeDb + .getRunMailboxHistory(task.run_id, 10) + .find((message) => message.type === 'question') + expect(question).toMatchObject({ + body: 'Should I include slow integration tests?' + }) + + const reply = await homeDispatcher.dispatch({ + id: 'rpc_home_reply', + authToken: 'coordinator-token', + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationRequestId: 'home_reply_request', + method: 'orchestration.reply', + params: { + id: question!.id, + body: 'yes', + from: 'term_coord' + } + }) + expect(reply).toMatchObject({ ok: true, result: { question: { status: 'answered' } } }) + await homeRuntime.syncOrchestrationFederation() + + await expect(ask).resolves.toMatchObject({ + ok: true, + result: { + answer: 'yes', + messageId: question!.id, + timedOut: false + } + }) + expect( + homeDb.listFederationRelay({ + dispatchId: dispatch.id, + direction: 'to_worker', + afterSequence: 0 + })[0] + ).toMatchObject({ acked_at: expect.any(String) }) + }) + + it('keeps a timed-out remote question resumable', async () => { + const task = createHomeTask() + await homeDispatcher.dispatch(startRequest(task.id)) + const prompt = vi.mocked(workerRuntime.sendTerminalAgentPrompt).mock.calls[0]?.[1] ?? '' + const capability = prompt.match(/--dispatch-capability (dcap_[A-Za-z0-9_-]+)/)?.[1] + const timedOut = await workerDispatcher.dispatch({ + id: 'rpc_remote_ask_timeout', + authToken: 'worker-local-token', + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationRequestId: 'remote_question_timeout_request', + orchestrationCapability: capability, + method: 'orchestration.ask', + params: { + from: 'term_windows_worker', + question: 'Resume this later?', + timeoutMs: 1 + } + }) + expect(timedOut).toMatchObject({ + ok: true, + result: { timedOut: true, messageId: expect.stringMatching(/^relay_/) } + }) + const questionId = (timedOut as { result: { messageId: string } }).result.messageId + + await homeRuntime.syncOrchestrationFederation() + await homeDispatcher.dispatch({ + id: 'rpc_home_late_reply', + authToken: 'coordinator-token', + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationRequestId: 'home_late_reply_request', + method: 'orchestration.reply', + params: { id: questionId, body: 'yes', from: 'term_coord' } + }) + restartWorkerRuntime() + const resumed = workerDispatcher.dispatch({ + id: 'rpc_remote_ask_resume', + authToken: 'worker-local-token', + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationRequestId: 'remote_question_resume_request', + orchestrationCapability: capability, + method: 'orchestration.ask', + params: { from: 'term_windows_worker', resume: questionId, timeoutMs: 5_000 } + }) + await homeRuntime.syncOrchestrationFederation() + + await expect(resumed).resolves.toMatchObject({ + ok: true, + result: { answer: 'yes', messageId: questionId, timedOut: false } + }) + }) + + it('retries a lost relay acknowledgment without duplicating the home message', async () => { + const task = createHomeTask() + await homeDispatcher.dispatch(startRequest(task.id)) + const dispatch = homeDb.getDispatchContext(task.id)! + const prompt = vi.mocked(workerRuntime.sendTerminalAgentPrompt).mock.calls[0]?.[1] ?? '' + const capability = prompt.match(/--dispatch-capability (dcap_[A-Za-z0-9_-]+)/)?.[1] + await workerDispatcher.dispatch({ + id: 'rpc_remote_status', + authToken: 'worker-local-token', + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationRequestId: 'remote_status_request', + orchestrationCapability: capability, + method: 'orchestration.send', + params: { + from: 'term_windows_worker', + subject: 'Checkpoint', + body: 'One durable update', + type: 'status' + } + }) + loseNextAckResponse = true + + await expect(homeRuntime.syncOrchestrationFederation()).resolves.toBeUndefined() + await homeRuntime.syncOrchestrationFederation() + + expect( + homeDb + .getRunMailboxHistory(task.run_id, 10) + .filter((message) => message.subject === 'Checkpoint') + ).toHaveLength(1) + expect(homeDb.getFederatedDispatch(dispatch.id)?.to_home_imported_sequence).toBe(1) + }) + + it('rejects a reordered relay gap, then converges without loss or duplication', async () => { + const task = createHomeTask() + await homeDispatcher.dispatch(startRequest(task.id)) + const dispatch = homeDb.getDispatchContext(task.id)! + + expect(() => + homeDb.importFederatedRelayItem({ + dispatchId: dispatch.id, + sequence: 2, + message: { + id: 'relay_gap', + runId: task.run_id, + from: `dispatch:${dispatch.id}`, + to: `run:${task.run_id}`, + subject: 'Gap', + body: 'Out of order', + type: 'status', + priority: 'normal' + }, + lifecycle: { kind: 'none' } + }) + ).toThrow(/not contiguous/) + expect(homeDb.getMessageById('relay_gap')).toBeUndefined() + expect(homeDb.getFederatedDispatch(dispatch.id)?.to_home_imported_sequence).toBe(0) + + homeDb.importFederatedRelayItem({ + dispatchId: dispatch.id, + sequence: 1, + message: { + id: 'relay_first', + runId: task.run_id, + from: `dispatch:${dispatch.id}`, + to: `run:${task.run_id}`, + subject: 'First', + body: 'Arrived after the gap was rejected', + type: 'status', + priority: 'normal' + }, + lifecycle: { kind: 'none' } + }) + const recovered = homeDb.importFederatedRelayItem({ + dispatchId: dispatch.id, + sequence: 2, + message: { + id: 'relay_gap', + runId: task.run_id, + from: `dispatch:${dispatch.id}`, + to: `run:${task.run_id}`, + subject: 'Gap', + body: 'Out of order', + type: 'status', + priority: 'normal' + }, + lifecycle: { kind: 'none' } + }) + const duplicate = homeDb.importFederatedRelayItem({ + dispatchId: dispatch.id, + sequence: 2, + message: { + id: 'relay_gap', + runId: task.run_id, + from: `dispatch:${dispatch.id}`, + to: `run:${task.run_id}`, + subject: 'Gap', + body: 'Out of order', + type: 'status', + priority: 'normal' + }, + lifecycle: { kind: 'none' } + }) + + expect(recovered.duplicate).toBe(false) + expect(duplicate.duplicate).toBe(true) + expect(homeDb.getFederatedDispatch(dispatch.id)?.to_home_imported_sequence).toBe(2) + expect( + homeDb + .getRunMailboxHistory(task.run_id, 10) + .filter((message) => ['relay_first', 'relay_gap'].includes(message.id)) + ).toHaveLength(2) + }) + + it('restarts relay polling when a federated worker is shown', async () => { + const task = createHomeTask() + await homeDispatcher.dispatch(startRequest(task.id)) + const dispatch = homeDb.getDispatchContext(task.id)! + const prompt = vi.mocked(workerRuntime.sendTerminalAgentPrompt).mock.calls[0]?.[1] ?? '' + const capability = prompt.match(/--dispatch-capability (dcap_[A-Za-z0-9_-]+)/)?.[1] + homeRuntime.stopOrchestrationFederationRelay() + await workerDispatcher.dispatch({ + id: 'rpc_restart_status', + authToken: 'worker-local-token', + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationRequestId: 'restart_status_request', + orchestrationCapability: capability, + method: 'orchestration.send', + params: { + from: 'term_windows_worker', + subject: 'After home restart', + body: 'Relay me after worker-show', + type: 'status' + } + }) + + await homeDispatcher.dispatch({ + id: 'rpc_restart_show', + authToken: 'coordinator-token', + method: 'orchestration.workerShow', + params: { dispatch: dispatch.id } + }) + + await vi.waitFor(() => + expect( + homeDb + .getRunMailboxHistory(task.run_id, 10) + .some((message) => message.subject === 'After home restart') + ).toBe(true) + ) + }) + + it('treats a worker runtime ID change as an epoch, not a new server', async () => { + const task = createHomeTask() + await homeDispatcher.dispatch(startRequest(task.id)) + const dispatch = homeDb.getDispatchContext(task.id)! + const oldEpoch = homeDb.getFederatedDispatch(dispatch.id)?.remote_runtime_epoch + restartWorkerRuntime() + + const shown = await homeDispatcher.dispatch({ + id: 'rpc_worker_restart_show', + authToken: 'coordinator-token', + method: 'orchestration.workerShow', + params: { dispatch: dispatch.id } + }) + + expect(shown).toMatchObject({ + ok: true, + result: { observation: { status: 'running', exactWorker: true } } + }) + expect(homeDb.getFederatedDispatch(dispatch.id)?.remote_runtime_epoch).not.toBe(oldEpoch) + expect(homeDb.getFederatedDispatch(dispatch.id)?.peer_fingerprint).toBe( + 'windows_peer_fingerprint' + ) + }) + + it('stops only the exact remote agent terminal', async () => { + const task = createHomeTask() + await homeDispatcher.dispatch(startRequest(task.id)) + const dispatch = homeDb.getDispatchContext(task.id)! + + const stopped = await homeDispatcher.dispatch({ + id: 'rpc_remote_stop', + authToken: 'coordinator-token', + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationRequestId: 'request_remote_stop', + method: 'orchestration.workerStop', + params: { dispatch: dispatch.id } + }) + + expect(stopped).toMatchObject({ + ok: true, + result: { state: 'stopped', processAction: 'closed_agent_terminal' } + }) + expect(workerRuntime.closeTerminal).toHaveBeenCalledTimes(1) + expect(workerRuntime.closeTerminal).toHaveBeenCalledWith('term_windows_worker') + expect(homeDb.getTask(task.id)?.status).toBe('blocked') + + vi.mocked(workerRuntime.showTerminal).mockResolvedValue({ + handle: 'term_windows_worker', + worktreeId: 'repo::windows-worktree', + connected: false, + writable: false + } as never) + const shown = await homeDispatcher.dispatch({ + id: 'rpc_remote_show_after_stop', + authToken: 'coordinator-token', + method: 'orchestration.workerShow', + params: { dispatch: dispatch.id } + }) + expect(shown).toMatchObject({ + ok: true, + result: { observation: { status: 'exited', exactWorker: true } } + }) + }) + + it('rejects a re-paired server before show or stop effects', async () => { + const task = createHomeTask() + await homeDispatcher.dispatch(startRequest(task.id)) + const dispatch = homeDb.getDispatchContext(task.id)! + workerPeerFingerprint = 'replacement_windows_peer' + + const shown = await homeDispatcher.dispatch({ + id: 'rpc_changed_peer_show', + authToken: 'coordinator-token', + method: 'orchestration.workerShow', + params: { dispatch: dispatch.id } + }) + const stopped = await homeDispatcher.dispatch({ + id: 'rpc_changed_peer_stop', + authToken: 'coordinator-token', + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationRequestId: 'request_changed_peer_stop', + method: 'orchestration.workerStop', + params: { dispatch: dispatch.id } + }) + + expect(shown).toMatchObject({ ok: false, error: { code: 'peer_changed' } }) + expect(stopped).toMatchObject({ ok: false, error: { code: 'peer_changed' } }) + expect(homeDb.getWorkerDispatch(dispatch.id)?.state).toBe('ready') + expect(workerRuntime.closeTerminal).not.toHaveBeenCalled() + }) + + it('coalesces overlapping relay polls for the same Dispatch', async () => { + const task = createHomeTask() + await homeDispatcher.dispatch(startRequest(task.id)) + await homeRuntime.syncOrchestrationFederation() + homeRuntime.stopOrchestrationFederationRelay() + + let releasePull!: () => void + const blockedPull = new Promise((resolve) => { + releasePull = resolve + }) + let pullCount = 0 + vi.spyOn(homeRuntime, 'callOrchestrationWorkerServer').mockImplementation( + async (_selector, method) => { + if (method !== 'orchestration.federationPull') { + throw new Error(`Unexpected relay method ${method}`) + } + pullCount += 1 + await blockedPull + return { runtimeEpoch: workerRuntime.getRuntimeId(), items: [] } + } + ) + + const first = homeRuntime.syncOrchestrationFederation() + const second = homeRuntime.syncOrchestrationFederation() + await vi.waitFor(() => expect(pullCount).toBe(1)) + releasePull() + await Promise.all([first, second]) + + expect(pullCount).toBe(1) + }) + + it('warns once while a federated Dispatch remains unreachable', async () => { + const task = createHomeTask() + await homeDispatcher.dispatch(startRequest(task.id)) + await homeRuntime.syncOrchestrationFederation() + homeRuntime.stopOrchestrationFederationRelay() + vi.spyOn(homeRuntime, 'callOrchestrationWorkerServer').mockRejectedValue( + new Error('worker server offline') + ) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + + await homeRuntime.syncOrchestrationFederation() + await homeRuntime.syncOrchestrationFederation() + + expect(warn).toHaveBeenCalledTimes(1) + expect(warn).toHaveBeenCalledWith( + expect.stringContaining('Federation sync failed'), + expect.any(Error) + ) + warn.mockRestore() + }) + + it('returns stop_unknown when the worker server disconnects after the home fence', async () => { + const task = createHomeTask() + await homeDispatcher.dispatch(startRequest(task.id)) + const dispatch = homeDb.getDispatchContext(task.id)! + vi.spyOn(homeRuntime, 'callOrchestrationWorkerServer').mockRejectedValueOnce( + new Error('connection lost') + ) + + const stopped = await homeDispatcher.dispatch({ + id: 'rpc_disconnected_stop', + authToken: 'coordinator-token', + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationRequestId: 'request_disconnected_stop', + method: 'orchestration.workerStop', + params: { dispatch: dispatch.id } + }) + + expect(stopped).toMatchObject({ + ok: true, + result: { state: 'stop_unknown', processAction: 'unknown' } + }) + expect(homeDb.getTask(task.id)?.status).toBe('blocked') + expect(workerRuntime.closeTerminal).not.toHaveBeenCalled() + }) + + it('never reads or closes a same-looking replacement process', async () => { + const task = createHomeTask() + await homeDispatcher.dispatch(startRequest(task.id)) + const dispatch = homeDb.getDispatchContext(task.id)! + vi.mocked(workerRuntime.getTerminalProcessIncarnation).mockReturnValue( + 'windows_runtime:pty:replacement' + ) + + const read = await homeDispatcher.dispatch({ + id: 'rpc_replacement_read', + authToken: 'coordinator-token', + method: 'orchestration.workerRead', + params: { dispatch: dispatch.id } + }) + const stopped = await homeDispatcher.dispatch({ + id: 'rpc_replacement_stop', + authToken: 'coordinator-token', + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationRequestId: 'request_replacement_stop', + method: 'orchestration.workerStop', + params: { dispatch: dispatch.id } + }) + + expect(read).toMatchObject({ + ok: false, + error: { code: 'worker_identity_changed' } + }) + expect(stopped).toMatchObject({ + ok: true, + result: { state: 'stop_unknown', processAction: 'none' } + }) + expect(workerRuntime.closeTerminal).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/runtime/rpc/methods/orchestration-federation.ts b/src/main/runtime/rpc/methods/orchestration-federation.ts new file mode 100644 index 00000000000..b8a130adb79 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-federation.ts @@ -0,0 +1,297 @@ +import { isTuiAgent } from '../../../../shared/tui-agent-config' +import type { TuiAgent } from '../../../../shared/types' +import { buildDispatchPreamble } from '../../orchestration/preamble' +import { OrchestrationError } from '../../orchestration/orchestration-error' +import { defineMethod, type RpcMethod } from '../core' +import { assertOrchestrationWorktreeCreationSupported } from './orchestration-folder-worktree-placement' +import { + appendFederationSetupEffect, + appendFederationTerminalEffects, + type FederationEffect +} from './orchestration-federation-effects' +import type { WorkerSetupReceipt } from './orchestration-worker-topology' +import { + monitorFederatedSetup, + persistFederatedReadinessStage, + persistFederatedSetupSpawnFailure, + persistFederatedSetupWaitOutcome +} from './orchestration-federation-setup' +import { FederationAttachStartParams } from './orchestration-federation-start-schema' +import { failFederatedAttachmentWithReceipt } from './orchestration-federation-start-receipt' + +export const ORCHESTRATION_FEDERATION_ATTACH_METHODS: RpcMethod[] = [ + defineMethod({ + name: 'orchestration.federationAttachStart', + params: FederationAttachStartParams, + handler: async (params, { runtime, orchestrationMutation }) => { + if (!orchestrationMutation) { + throw new OrchestrationError( + 'invalid_argument', + 'Federated worker attachment requires a durable retry request.' + ) + } + if (params.worktree === 'current' || params.worktree === 'new-child') { + throw new OrchestrationError( + 'invalid_argument', + 'A remote worker requires an exact existing worktree or new-top-level.' + ) + } + const createsWorktree = params.worktree === 'new-top-level' + if (createsWorktree && (!params.name || !params.repo)) { + throw new OrchestrationError( + 'invalid_argument', + 'A remote new-top-level worktree requires --name and an explicit --repo.' + ) + } + if (createsWorktree && params.terminal) { + throw new OrchestrationError( + 'invalid_argument', + '--terminal cannot combine with remote new-worktree creation.' + ) + } + if ( + !createsWorktree && + (params.name || params.repo || params.baseBranch || params.setup || params.setupSource) + ) { + throw new OrchestrationError( + 'invalid_argument', + 'Creation and setup options apply only to remote new-top-level worktrees.' + ) + } + if (params.terminal && params.agent) { + throw new OrchestrationError( + 'invalid_argument', + '--terminal reuses an existing agent and cannot combine with --agent.' + ) + } + const agent = params.agent + if (!params.terminal && (!agent || !isTuiAgent(agent))) { + throw new OrchestrationError( + 'agent_unconfigured', + 'A configured --agent is required when federated worker-start creates a terminal.' + ) + } + if (agent) { + runtime.validateOrchestrationAgentLauncher(agent as TuiAgent) + } + if (createsWorktree) { + await assertOrchestrationWorktreeCreationSupported({ + runtime, + repoSelector: params.repo as string, + existingPlacement: 'an exact existing folder workspace' + }) + } + + const db = runtime.getOrchestrationDb() + db.createRemoteDispatchAttachment({ + dispatchId: params.dispatchId, + taskId: params.taskId, + homePeerFingerprint: orchestrationMutation.callerFingerprint, + protocolVersion: params.protocolVersion, + runtimeEpoch: runtime.getRuntimeId(), + mutationReceipt: orchestrationMutation + }) + const effects: FederationEffect[] = [] + let failedStage = createsWorktree ? 'worktree_create' : 'worktree_resolve' + let worktree + let terminalHandle = params.terminal + const setupSource = createsWorktree + ? (params.setupSource ?? (params.setup ? 'explicit_request' : 'orchestration_default')) + : 'existing_worktree' + let setup: WorkerSetupReceipt = { + requested: createsWorktree ? (params.setup ?? 'run') : 'not_applicable', + effective: createsWorktree ? (params.setup ?? 'run') : 'not_applicable', + source: setupSource, + hookFound: false, + startupPolicy: 'start-immediately', + state: createsWorktree ? 'not_configured' : 'not_applicable' + } + try { + if (createsWorktree) { + db.recordRemoteAttachmentStage({ + dispatchId: params.dispatchId, + stage: 'worktree_creating' + }) + const setupDecision = params.setup ?? 'run' + const created = await runtime.createManagedWorktree({ + repoSelector: params.repo as string, + name: params.name as string, + baseBranch: params.baseBranch, + displayName: params.displayName, + comment: params.comment, + runHooks: setupDecision === 'run', + setupDecision, + awaitTerminalProvisioning: true, + observeSetupCompletion: true, + createdWithAgent: agent as TuiAgent, + startupAgent: agent as TuiAgent, + activate: false, + lineage: { noParent: true } + }) + worktree = created.worktree + terminalHandle = created.startupTerminal?.handle + effects.push({ + kind: 'worktree', + action: 'created_top_level', + id: created.worktree.id + }) + setup = { + requested: setupDecision, + effective: setupDecision, + source: setupSource, + hookFound: created.setupReceipt?.hookFound ?? false, + startupPolicy: created.setupReceipt?.startupPolicy ?? 'start-immediately', + state: created.setupReceipt?.state ?? 'not_configured' + } + if (!terminalHandle) { + throw new Error( + created.warning ?? 'Agent-first worktree creation returned no terminal.' + ) + } + const listed = await runtime.listTerminals(`id:${created.worktree.id}`) + appendFederationTerminalEffects( + effects, + listed.terminals, + terminalHandle, + created.setupReceipt?.terminalHandle + ) + appendFederationSetupEffect(effects, setup) + } else { + worktree = await runtime.showManagedWorktree(params.worktree).catch(() => { + throw new OrchestrationError( + 'worktree_not_found_on_server', + `Worktree ${params.worktree} was not found on the selected worker server.` + ) + }) + effects.push( + { kind: 'worktree', action: 'reused', id: worktree.id }, + { kind: 'setup', action: 'not_applicable', state: 'not_applicable' } + ) + if (terminalHandle) { + const terminal = await runtime.showTerminal(terminalHandle) + if (terminal.worktreeId !== worktree.id) { + throw new OrchestrationError( + 'terminal_worktree_mismatch', + `Terminal ${terminalHandle} does not belong to worktree ${worktree.id}.` + ) + } + if (!(await runtime.isTerminalRunningAgent(terminalHandle))) { + throw new OrchestrationError( + 'agent_unconfigured', + `Terminal ${terminalHandle} is not running a recognized agent.` + ) + } + effects.push({ + kind: 'terminal', + role: 'agent', + action: 'reused', + id: terminalHandle + }) + } else { + failedStage = 'terminal_create' + const terminal = await runtime.createTerminal(`id:${worktree.id}`, { + command: agent, + title: `worker-${params.taskId}`, + presentation: 'background' + }) + terminalHandle = terminal.handle + effects.push({ + kind: 'terminal', + role: 'agent', + action: 'created', + id: terminal.handle + }) + } + } + if (!worktree || !terminalHandle) { + throw new Error('Federated worker topology did not resolve.') + } + const setupStage = { + db, + dispatchId: params.dispatchId, + worktreeId: worktree.id, + terminalHandle, + setup, + effects + } + if (persistFederatedSetupSpawnFailure(setupStage)) { + failedStage = 'setup_start' + throw new Error('Setup terminal failed to start before the gated agent launch.') + } + persistFederatedReadinessStage(setupStage) + failedStage = 'agent_readiness' + const wait = await runtime.waitForTerminal(terminalHandle, { + condition: 'tui-idle', + timeoutMs: params.timeoutMs ?? 60_000 + }) + persistFederatedSetupWaitOutcome({ ...setupStage, wait }) + if (!wait.satisfied) { + if (setup.state === 'failed') { + failedStage = 'setup_wait' + } + throw new Error( + wait.blockedReason + ? `Agent startup blocked: ${wait.blockedReason}` + : `Agent did not become ready (${wait.status}).` + ) + } + const paneKey = runtime.getTerminalPaneKey(terminalHandle) + const processIncarnation = runtime.getTerminalProcessIncarnation(terminalHandle) + if (!paneKey || !processIncarnation) { + throw new Error('stable_pane_required') + } + const capability = db.prepareRemoteAttachmentAuthority({ + dispatchId: params.dispatchId, + paneKey, + processIncarnation, + worktreeId: worktree.id, + terminalHandle, + setupState: setup.state, + effects + }) + failedStage = 'dispatch_input' + await runtime.sendTerminalAgentPrompt( + terminalHandle, + buildDispatchPreamble({ + taskId: params.taskId, + dispatchId: params.dispatchId, + taskSpec: params.taskSpec, + coordinatorHandle: 'Run home (relayed by Orca)', + workerHandle: terminalHandle, + dispatchCapability: capability, + devMode: params.devMode, + cliCommand: runtime.getTerminalOrchestrationCliCommand(terminalHandle) + }) + ) + effects.push({ + kind: 'dispatch_input', + role: 'agent', + id: terminalHandle, + state: 'accepted' + }) + const attachment = db.markRemoteAttachmentReady(params.dispatchId, effects) + monitorFederatedSetup({ ...setupStage, runtime }) + return { + dispatchId: params.dispatchId, + state: attachment.state, + stage: attachment.stage, + runtimeEpoch: runtime.getRuntimeId(), + worktreeId: worktree.id, + terminalHandle, + setup, + effects, + residualResources: [] + } + } catch (error) { + return failFederatedAttachmentWithReceipt({ + db, + dispatchId: params.dispatchId, + runtimeEpoch: runtime.getRuntimeId(), + failedStage, + error, + setup + }) + } + } + }) +] diff --git a/src/main/runtime/rpc/methods/orchestration-folder-worktree-placement.ts b/src/main/runtime/rpc/methods/orchestration-folder-worktree-placement.ts new file mode 100644 index 00000000000..5f9a65a2390 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-folder-worktree-placement.ts @@ -0,0 +1,17 @@ +import { isFolderRepo } from '../../../../shared/repo-kind' +import type { OrcaRuntimeService } from '../../orca-runtime' +import { OrchestrationError } from '../../orchestration/orchestration-error' + +export async function assertOrchestrationWorktreeCreationSupported(args: { + runtime: OrcaRuntimeService + repoSelector: string + existingPlacement: string +}): Promise { + if (!isFolderRepo(await args.runtime.showRepo(args.repoSelector))) { + return + } + throw new OrchestrationError( + 'invalid_argument', + `Folder projects cannot create orchestration worktrees; use ${args.existingPlacement}.` + ) +} diff --git a/src/main/runtime/rpc/methods/orchestration-migration-behavior.test.ts b/src/main/runtime/rpc/methods/orchestration-migration-behavior.test.ts new file mode 100644 index 00000000000..ce76d7b611f --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-migration-behavior.test.ts @@ -0,0 +1,181 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY, + ORCHESTRATION_FEDERATION_RUNTIME_CAPABILITY +} from '../../../../shared/protocol-version' +import { OrcaRuntimeService } from '../../orca-runtime' +import { OrchestrationDb } from '../../orchestration/db' +import { RpcDispatcher } from '../dispatcher' +import { ORCHESTRATION_METHODS } from './orchestration' +import { startFederatedWorker } from './orchestration-federated-worker-start' + +describe('orchestration migration behavior', () => { + const databases: OrchestrationDb[] = [] + + afterEach(() => { + for (const database of databases.splice(0)) { + database.close() + } + }) + + function createRuntime(): { db: OrchestrationDb; runtime: OrcaRuntimeService } { + const db = new OrchestrationDb(':memory:') + const runtime = new OrcaRuntimeService() + runtime.setOrchestrationDb(db) + databases.push(db) + return { db, runtime } + } + + it('lists an explicitly selected legacy Run without binding or mutation', async () => { + const { db, runtime } = createRuntime() + const task = db.createTask({ spec: 'pre-upgrade work' }) + const taskList = ORCHESTRATION_METHODS.find( + (method) => method.name === 'orchestration.taskList' + )! + + const listed = (await taskList.handler(taskList.params!.parse({ run: 'run_legacy_local' }), { + runtime + })) as { + runId: string + legacyReadOnly: boolean + tasks: { id: string }[] + } + + expect(listed).toMatchObject({ + runId: 'run_legacy_local', + legacyReadOnly: true, + tasks: [{ id: task.id }] + }) + expect(db.getTask(task.id)?.status).toBe('ready') + }) + + it('rejects a pre-contract worker_done before message or lifecycle mutation', async () => { + const { db, runtime } = createRuntime() + const run = db.createRun({ + objective: 'legacy worker', + coordinatorHandle: 'term_coord', + coordinatorPaneKey: 'tab_coord:leaf_coord' + }) + const task = db.createTask({ spec: 'legacy worker', runId: run.id }) + const dispatch = db.createDispatchContext(task.id, 'term_worker', 'tab_worker:leaf_worker') + const dispatcher = new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS }) + + const response = await dispatcher.dispatch({ + id: 'legacy_worker_done', + authToken: 'worker-token', + method: 'orchestration.send', + params: { + from: 'term_worker', + subject: 'done', + type: 'worker_done', + payload: JSON.stringify({ + taskId: task.id, + dispatchId: dispatch.id, + outcome: 'succeeded' + }) + } + }) + + expect(response).toMatchObject({ + ok: false, + error: { + code: 'orchestration_migration_required', + data: { effectsApplied: false } + } + }) + expect(db.getInbox(100)).toHaveLength(0) + expect(db.getTask(task.id)?.status).toBe('dispatched') + expect(db.getDispatchContextById(dispatch.id)?.status).toBe('dispatched') + }) + + it('rejects a connected server missing the contract before home or remote effects', async () => { + const { db, runtime } = createRuntime() + const run = db.createRun({ + objective: 'mixed-version worker', + coordinatorHandle: 'term_coord', + coordinatorPaneKey: 'tab_coord:leaf_coord' + }) + const task = db.createTask({ spec: 'remote work', runId: run.id }) + vi.spyOn(runtime, 'resolveOrchestrationWorkerServer').mockReturnValue({ + environmentId: 'environment_windows', + name: 'windows', + peerFingerprint: 'windows_peer' + }) + vi.spyOn(runtime, 'callOrchestrationWorkerServer').mockResolvedValue({ + capabilities: [ORCHESTRATION_FEDERATION_RUNTIME_CAPABILITY] + }) + + await expect( + startFederatedWorker({ + params: { + task: task.id, + from: 'term_coord', + on: 'windows', + worktree: 'new-top-level', + repo: 'id:windows-repo', + name: 'remote-work', + agent: 'codex' + }, + runtime, + db, + runId: run.id, + task, + orchestrationMutation: { + callerFingerprint: 'caller', + requestId: 'remote_start', + method: 'orchestration.workerStart', + payloadHash: 'payload' + } + }) + ).rejects.toMatchObject({ + code: 'orchestration_migration_required', + data: { reason: 'runtime_capability_missing', effectsApplied: false } + }) + expect(db.getTask(task.id)?.status).toBe('ready') + expect(db.getDispatchContext(task.id)).toBeUndefined() + }) + + it('rejects a connected server missing federation support before Task mutation', async () => { + const { db, runtime } = createRuntime() + const run = db.createRun({ + objective: 'unsupported worker', + coordinatorHandle: 'term_coord', + coordinatorPaneKey: 'tab_coord:leaf_coord' + }) + const task = db.createTask({ spec: 'remote work', runId: run.id }) + vi.spyOn(runtime, 'resolveOrchestrationWorkerServer').mockReturnValue({ + environmentId: 'environment_windows', + name: 'windows', + peerFingerprint: 'windows_peer' + }) + vi.spyOn(runtime, 'callOrchestrationWorkerServer').mockResolvedValue({ + capabilities: [ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY] + }) + + await expect( + startFederatedWorker({ + params: { + task: task.id, + from: 'term_coord', + on: 'windows', + worktree: 'new-top-level', + repo: 'id:windows-repo', + name: 'remote-work', + agent: 'codex' + }, + runtime, + db, + runId: run.id, + task, + orchestrationMutation: { + callerFingerprint: 'caller', + requestId: 'remote_start', + method: 'orchestration.workerStart', + payloadHash: 'payload' + } + }) + ).rejects.toMatchObject({ code: 'capability_unsupported' }) + expect(db.getTask(task.id)?.status).toBe('ready') + expect(db.getDispatchContext(task.id)).toBeUndefined() + }) +}) diff --git a/src/main/runtime/rpc/methods/orchestration-runs.ts b/src/main/runtime/rpc/methods/orchestration-runs.ts new file mode 100644 index 00000000000..498d6228365 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-runs.ts @@ -0,0 +1,100 @@ +import { z } from 'zod' +import { defineMethod, type RpcMethod } from '../core' +import { OptionalString, requiredString } from '../schemas' +import type { OrcaRuntimeService } from '../../orca-runtime' +import { OrchestrationError } from '../../orchestration/orchestration-error' + +const RunCreateParams = z.object({ + objective: requiredString('Missing --objective'), + from: requiredString('Missing coordinator terminal') +}) + +const RunUseParams = z.object({ + id: requiredString('Missing --id'), + from: requiredString('Missing coordinator terminal') +}) + +const RunCurrentParams = z.object({ from: requiredString('Missing coordinator terminal') }) +const RunListParams = z.object({}) +const RunShowParams = z.object({ id: requiredString('Missing --id'), from: OptionalString }) + +function requireCallerPane(runtime: OrcaRuntimeService, handle: string): string { + const paneKey = runtime.getTerminalPaneKey(handle) + if (!paneKey) { + throw new OrchestrationError( + 'stable_pane_required', + 'The coordinator terminal has no stable pane identity. Run this command inside a live Orca terminal.' + ) + } + return paneKey +} + +export const ORCHESTRATION_RUN_METHODS: RpcMethod[] = [ + defineMethod({ + name: 'orchestration.runCreate', + params: RunCreateParams, + handler: (params, { runtime }) => { + const paneKey = requireCallerPane(runtime, params.from) + const db = runtime.getOrchestrationDb() + const priorRun = db.getCurrentRunForPane(paneKey) + const run = db.createRun({ + objective: params.objective, + coordinatorHandle: params.from, + coordinatorPaneKey: paneKey + }) + if (priorRun) { + runtime.cancelMessageWaiters(`run:${priorRun.id}`) + } + return { run, binding: { consumerGeneration: run.consumer_generation } } + } + }), + defineMethod({ + name: 'orchestration.runUse', + params: RunUseParams, + handler: (params, { runtime }) => { + const paneKey = requireCallerPane(runtime, params.from) + const db = runtime.getOrchestrationDb() + const priorRun = db.getCurrentRunForPane(paneKey) + const run = db.bindRun({ + runId: params.id, + coordinatorHandle: params.from, + coordinatorPaneKey: paneKey + }) + if (!run) { + throw new OrchestrationError( + 'run_not_found', + `Run ${params.id} was not found or is inspect-only.` + ) + } + runtime.cancelMessageWaiters(`run:${params.id}`) + if (priorRun && priorRun.id !== params.id) { + runtime.cancelMessageWaiters(`run:${priorRun.id}`) + } + return { run, binding: { consumerGeneration: run.consumer_generation } } + } + }), + defineMethod({ + name: 'orchestration.runCurrent', + params: RunCurrentParams, + handler: (params, { runtime }) => { + const paneKey = requireCallerPane(runtime, params.from) + return { run: runtime.getOrchestrationDb().getCurrentRunForPane(paneKey) ?? null } + } + }), + defineMethod({ + name: 'orchestration.runList', + params: RunListParams, + handler: (_params, { runtime }) => ({ runs: runtime.getOrchestrationDb().listRuns() }) + }), + defineMethod({ + name: 'orchestration.runShow', + params: RunShowParams, + handler: (params, { runtime }) => { + const run = runtime.getOrchestrationDb().getRun(params.id) + if (!run) { + throw new OrchestrationError('run_not_found', `Run ${params.id} was not found.`) + } + return { run } + } + }) +] diff --git a/src/main/runtime/rpc/methods/orchestration-worker-control.ts b/src/main/runtime/rpc/methods/orchestration-worker-control.ts new file mode 100644 index 00000000000..ae64f967e06 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-worker-control.ts @@ -0,0 +1,294 @@ +import { z } from 'zod' +import { + ORCHESTRATION_WORKER_READ_SOURCES, + type OrchestrationWorkerReadResult +} from '../../../../shared/orchestration-worker-output' +import type { RuntimeTerminalRead } from '../../../../shared/runtime-types' +import { OrchestrationError } from '../../orchestration/orchestration-error' +import { syncFederatedDispatch } from '../../orchestration/federation-sync' +import { + createWorkerOutputSourceIdentity, + decodeWorkerOutputCursor, + encodeWorkerOutputCursor +} from '../../orchestration/worker-output-cursor' +import { defineMethod, type RpcMethod } from '../core' +import { OptionalFiniteNumber, requiredString } from '../schemas' +import { + callFederatedWorkerShow, + exposeWorker, + inspectWorkerTerminal, + resolvePinnedFederatedServer +} from './orchestration-worker-observation' +import { readExactWorkerOutput } from './orchestration-worker-output' + +const WorkerDispatchParams = z.object({ dispatch: requiredString('Missing --dispatch') }) +const WorkerReadParams = WorkerDispatchParams.extend({ + cursor: z.union([z.number().int().nonnegative(), z.string().min(1).max(2_048)]).optional(), + limit: OptionalFiniteNumber, + source: z.enum(ORCHESTRATION_WORKER_READ_SOURCES).optional() +}) + +export const ORCHESTRATION_WORKER_CONTROL_METHODS: RpcMethod[] = [ + defineMethod({ + name: 'orchestration.workerShow', + params: WorkerDispatchParams, + handler: async (params, { runtime }) => { + const db = runtime.getOrchestrationDb() + const dispatch = db.getDispatchContextById(params.dispatch) + let worker = db.getWorkerDispatch(params.dispatch) + if (!dispatch || !worker) { + throw new OrchestrationError( + 'dispatch_not_found', + `Worker Dispatch ${params.dispatch} was not found.` + ) + } + const federated = db.getFederatedDispatch(params.dispatch) + if (federated) { + const server = resolvePinnedFederatedServer(runtime, federated) + runtime.ensureOrchestrationFederationRelay(dispatch.run_id) + const remote = await callFederatedWorkerShow(runtime, federated) + const attachment = remote.attachment + worker = db.updateWorkerSetupEvidence({ + dispatchId: params.dispatch, + setupState: attachment.setup_state, + effects: attachment.effects + }).worker + if ( + attachment.state === 'succeeded' || + (attachment.state === 'failed' && attachment.stage === 'worker_report_queued') + ) { + await syncFederatedDispatch(runtime, params.dispatch).catch(() => undefined) + } else if ( + attachment.state === 'stopped' && + ['stopping', 'stop_unknown'].includes(worker.state) + ) { + worker = db.reconcileFederatedWorkerStop(params.dispatch) + } else if (['ready', 'failed', 'stopped', 'start_unknown'].includes(attachment.state)) { + worker = db.reconcileFederatedWorkerStart({ + dispatchId: params.dispatch, + state: attachment.state as 'ready' | 'failed' | 'stopped' | 'start_unknown', + stage: attachment.stage, + lastError: attachment.last_error, + worktreeId: attachment.worktree_id, + terminalHandle: attachment.terminal_handle, + setupState: attachment.setup_state, + effects: attachment.effects, + residualResources: attachment.residualResources + }) + if ( + attachment.state === 'ready' && + attachment.worktree_id && + attachment.terminal_handle + ) { + db.updateFederatedDispatchResources({ + dispatchId: params.dispatch, + remoteRuntimeEpoch: remote.runtimeEpoch, + worktreeId: attachment.worktree_id, + terminalHandle: attachment.terminal_handle + }) + } + } + worker = db.getWorkerDispatch(params.dispatch) + if (!worker) { + throw new OrchestrationError( + 'dispatch_not_found', + `Worker Dispatch ${params.dispatch} was not found after remote reconciliation.` + ) + } + return { + dispatch: db.getDispatchContextById(params.dispatch), + worker: exposeWorker(worker), + server: { environmentId: server.environmentId, name: server.name }, + remoteRuntimeEpoch: remote.runtimeEpoch, + terminal: remote.terminal, + observation: remote.observation + } + } + if (worker.runtime_epoch && worker.runtime_epoch !== runtime.getRuntimeId()) { + if (worker.state === 'starting') { + worker = db.markWorkerStartUnknown( + params.dispatch, + worker.stage, + 'The runtime restarted before worker-start reached a terminal receipt.' + ) + } else if (worker.state === 'stopping') { + worker = db.markWorkerStopUnknown( + params.dispatch, + 'The runtime restarted before worker-stop reached a terminal receipt.' + ) + } + } + const observation = await inspectWorkerTerminal(runtime, db, params.dispatch) + return { + dispatch, + worker: exposeWorker(worker), + terminal: observation.exact ? observation.terminal : null, + observation: { status: observation.status, exactWorker: observation.exact } + } + } + }), + defineMethod({ + name: 'orchestration.workerRead', + params: WorkerReadParams, + handler: async (params, { runtime }) => { + const db = runtime.getOrchestrationDb() + const federated = db.getFederatedDispatch(params.dispatch) + if (federated) { + const server = resolvePinnedFederatedServer(runtime, federated) + try { + const remote = (await runtime.callOrchestrationWorkerServer( + server.environmentId, + 'orchestration.federationReadOutput', + { + dispatchId: params.dispatch, + cursor: params.cursor, + limit: params.limit, + source: params.source + }, + 15_000 + )) as { runtimeEpoch: string; output: OrchestrationWorkerReadResult } + return { + ...remote.output, + server: { environmentId: server.environmentId, name: server.name }, + remoteRuntimeEpoch: remote.runtimeEpoch + } + } catch (error) { + if (!(error instanceof OrchestrationError) || error.code !== 'method_not_found') { + throw error + } + return readLegacyFederatedTerminal({ + runtime, + server, + federated, + workerState: db.getWorkerDispatch(params.dispatch)?.state ?? 'unknown', + dispatchId: params.dispatch, + source: params.source, + cursor: params.cursor, + limit: params.limit + }) + } + } + const worker = db.getWorkerDispatch(params.dispatch) + if (!worker?.agent_terminal_handle) { + throw new OrchestrationError( + 'dispatch_not_found', + `Worker Dispatch ${params.dispatch} has no agent terminal.` + ) + } + const observation = await inspectWorkerTerminal(runtime, db, params.dispatch) + if (!observation.exact) { + throw new OrchestrationError( + 'worker_identity_changed', + `Worker Dispatch ${params.dispatch} no longer resolves to its exact process.` + ) + } + const output = await readExactWorkerOutput({ + runtime, + dispatchId: params.dispatch, + terminalHandle: worker.agent_terminal_handle, + workerState: worker.state, + terminalStatus: observation.status === 'exited' ? 'exited' : 'running', + attachedAt: worker.created_at, + source: params.source, + cursor: params.cursor, + limit: params.limit + }) + const afterRead = await inspectWorkerTerminal(runtime, db, params.dispatch) + if (!afterRead.exact) { + throw new OrchestrationError( + 'worker_identity_changed', + `Worker Dispatch ${params.dispatch} changed process while output was read.` + ) + } + return output + } + }), + defineMethod({ + name: 'orchestration.workerAbandon', + params: WorkerDispatchParams, + handler: (params, { runtime }) => { + const abandoned = runtime.getOrchestrationDb().abandonWorkerDispatch(params.dispatch) + const worker = abandoned.worker + if (abandoned.disposition === 'abandoned') { + runtime.notifyMessageArrived(`dispatch:${params.dispatch}`, 'status') + } + return { + dispatchId: params.dispatch, + state: worker.state, + alreadySettled: abandoned.disposition !== 'abandoned', + stale: abandoned.disposition === 'stale', + processAction: 'none', + warning: + abandoned.disposition === 'stale' + ? 'The Dispatch is no longer current; no state or process changed.' + : 'Possibly-live resources were retained; no process was stopped or deleted.', + residualResources: JSON.parse(worker.residual_resources) as unknown[] + } + } + }) +] + +async function readLegacyFederatedTerminal(args: { + runtime: Parameters[0] + server: ReturnType + federated: Parameters[1] + workerState: string + dispatchId: string + source: (typeof ORCHESTRATION_WORKER_READ_SOURCES)[number] | undefined + cursor: string | number | undefined + limit: number | undefined +}) { + const cursor = decodeWorkerOutputCursor(args.cursor, args.dispatchId) + if (args.source === 'transcript' || cursor?.source === 'transcript') { + throw new OrchestrationError( + 'transcript_required', + `Connected server ${args.server.name} does not support structured worker output.`, + { reason: 'remote_capability_unavailable' } + ) + } + const remote = (await args.runtime.callOrchestrationWorkerServer( + args.server.environmentId, + 'orchestration.federationRead', + { + dispatchId: args.dispatchId, + cursor: cursor?.source === 'terminal' ? cursor.position : undefined, + limit: args.limit + }, + 15_000 + )) as { runtimeEpoch: string; terminal: RuntimeTerminalRead } + const sourceIdentity = createWorkerOutputSourceIdentity([ + 'legacy-remote-terminal', + args.federated.peer_fingerprint, + args.dispatchId, + remote.runtimeEpoch + ]) + if ( + cursor?.source === 'terminal' && + cursor.sourceIdentity !== null && + cursor.sourceIdentity !== sourceIdentity + ) { + throw new OrchestrationError( + 'source_changed', + 'The worker output source changed. Start a fresh worker-read without the old cursor.' + ) + } + const nextPosition = + remote.terminal.nextCursor !== null && /^\d+$/.test(remote.terminal.nextCursor) + ? Number.parseInt(remote.terminal.nextCursor, 10) + : null + return { + dispatchId: args.dispatchId, + source: 'terminal' as const, + sourceIdentity, + terminal: remote.terminal, + cursor: + nextPosition === null + ? null + : encodeWorkerOutputCursor(args.dispatchId, 'terminal', sourceIdentity, nextPosition), + status: { worker: args.workerState, terminal: remote.terminal.status }, + fallbackReason: 'remote_capability_unavailable' as const, + warnings: [], + server: { environmentId: args.server.environmentId, name: args.server.name }, + remoteRuntimeEpoch: remote.runtimeEpoch + } +} diff --git a/src/main/runtime/rpc/methods/orchestration-worker-methods.ts b/src/main/runtime/rpc/methods/orchestration-worker-methods.ts new file mode 100644 index 00000000000..341521323fd --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-worker-methods.ts @@ -0,0 +1,10 @@ +import type { RpcMethod } from '../core' +import { ORCHESTRATION_WORKER_CONTROL_METHODS } from './orchestration-worker-control' +import { ORCHESTRATION_WORKER_STOP_METHODS } from './orchestration-worker-stop' +import { ORCHESTRATION_WORKER_START_METHODS } from './orchestration-workers' + +export const ORCHESTRATION_WORKER_METHODS: RpcMethod[] = [ + ...ORCHESTRATION_WORKER_START_METHODS, + ...ORCHESTRATION_WORKER_CONTROL_METHODS, + ...ORCHESTRATION_WORKER_STOP_METHODS +] diff --git a/src/main/runtime/rpc/methods/orchestration-worker-observation.ts b/src/main/runtime/rpc/methods/orchestration-worker-observation.ts new file mode 100644 index 00000000000..7148fe32dcc --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-worker-observation.ts @@ -0,0 +1,82 @@ +import type { OrcaRuntimeService } from '../../orca-runtime' +import type { OrchestrationDb } from '../../orchestration/db' +import { OrchestrationError } from '../../orchestration/orchestration-error' +import type { FederatedDispatchRow, WorkerDispatchRow } from '../../orchestration/types' + +export async function inspectWorkerTerminal( + runtime: OrcaRuntimeService, + db: OrchestrationDb, + dispatchId: string +): Promise<{ + terminal: Awaited> | null + exact: boolean + status: 'unattached' | 'missing' | 'identity_changed' | 'running' | 'exited' +}> { + const worker = db.getWorkerDispatch(dispatchId) + if (!worker?.agent_terminal_handle) { + return { terminal: null, exact: false, status: 'unattached' } + } + const terminal = await runtime.showTerminal(worker.agent_terminal_handle).catch(() => null) + if (!terminal) { + return { terminal: null, exact: false, status: 'missing' } + } + const exact = db.isDispatchProcessCurrent({ + dispatchId, + paneKey: runtime.getTerminalPaneKey(worker.agent_terminal_handle), + processIncarnation: runtime.getTerminalProcessIncarnation(worker.agent_terminal_handle) + }) + return { + terminal, + exact, + status: exact ? (terminal.connected === false ? 'exited' : 'running') : 'identity_changed' + } +} + +export function exposeWorker(worker: WorkerDispatchRow) { + return { + ...worker, + effects: JSON.parse(worker.effects) as unknown[], + residualResources: JSON.parse(worker.residual_resources) as unknown[], + startOptions: JSON.parse(worker.start_options) as unknown + } +} + +export function resolvePinnedFederatedServer( + runtime: OrcaRuntimeService, + federated: FederatedDispatchRow +) { + const server = runtime.resolveOrchestrationWorkerServer(federated.environment_id) + if (server.peerFingerprint !== federated.peer_fingerprint) { + throw new OrchestrationError( + 'peer_changed', + `Saved environment ${federated.environment_name} now identifies a different Orca server.` + ) + } + return server +} + +export async function callFederatedWorkerShow( + runtime: OrcaRuntimeService, + federated: FederatedDispatchRow +): Promise<{ + runtimeEpoch: string + attachment: { + state: string + stage: string + last_error: string | null + worktree_id: string | null + terminal_handle: string | null + setup_state: string + effects: unknown[] + residualResources: unknown[] + } + terminal: unknown + observation: { status: string; exactWorker: boolean } +}> { + return (await runtime.callOrchestrationWorkerServer( + federated.environment_id, + 'orchestration.federationShow', + { dispatchId: federated.dispatch_id }, + 15_000 + )) as Awaited> +} diff --git a/src/main/runtime/rpc/methods/orchestration-worker-output.test.ts b/src/main/runtime/rpc/methods/orchestration-worker-output.test.ts new file mode 100644 index 00000000000..6db96efe33d --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-worker-output.test.ts @@ -0,0 +1,205 @@ +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { OrcaRuntimeService } from '../../orca-runtime' +import { readExactWorkerOutput } from './orchestration-worker-output' + +function codexMessage(id: string, text: string): string { + return JSON.stringify({ + type: 'event_msg', + payload: { id, type: 'agent_message', message: text } + }) +} + +describe('exact orchestration worker output', () => { + let directory: string + let transcriptA: string + let transcriptB: string + let providerSession: ReturnType + let runtime: OrcaRuntimeService + const readTerminal = vi.fn() + + beforeEach(async () => { + directory = await mkdtemp(join(tmpdir(), 'orca-worker-output-')) + transcriptA = join(directory, 'session-a.jsonl') + transcriptB = join(directory, 'session-b.jsonl') + await writeFile(transcriptA, `${codexMessage('a', 'worker A only')}\n`) + await writeFile(transcriptB, `${codexMessage('b', 'worker B only')}\n`) + providerSession = { + paneKey: 'tab:worker', + processIncarnation: 'pty:incarnation-1', + agent: 'codex', + providerSession: { + key: 'session_id', + id: 'session-a', + transcriptPath: transcriptA + }, + observedAt: Date.now() + } + readTerminal.mockReset() + readTerminal.mockResolvedValue({ + handle: 'term_worker', + status: 'running', + tail: ['terminal output'], + truncated: false, + nextCursor: '9' + }) + runtime = { + getExactWorkerProviderSession: vi.fn(() => providerSession), + getTerminalProcessIncarnation: vi.fn(() => 'pty:incarnation-1'), + getTerminalPaneKey: vi.fn(() => 'tab:worker'), + readTerminal + } as unknown as OrcaRuntimeService + }) + + afterEach(async () => { + await rm(directory, { recursive: true, force: true }) + }) + + const read = (overrides: Partial[0]> = {}) => + readExactWorkerOutput({ + runtime, + dispatchId: 'dispatch_1', + terminalHandle: 'term_worker', + workerState: 'ready', + terminalStatus: 'running', + attachedAt: '2026-07-24 00:00:00', + ...overrides + }) + + it('reads only the exact pane session and keeps its local path private', async () => { + const result = await read() + + expect(result).toMatchObject({ + source: 'transcript', + provider: 'codex', + transcript: { + messages: [{ id: 'a', blocks: [{ type: 'text', text: 'worker A only' }] }] + } + }) + expect(JSON.stringify(result)).not.toContain(transcriptA) + expect(JSON.stringify(result)).not.toContain('worker B only') + expect(readTerminal).not.toHaveBeenCalled() + }) + + it('reads Grok through the shared Native Chat transcript decoder', async () => { + await writeFile( + transcriptA, + `${JSON.stringify({ + id: 'grok-a', + type: 'assistant', + content: 'Grok worker only' + })}\n` + ) + providerSession = { + ...providerSession!, + agent: 'grok', + providerSession: { + key: 'session_id', + id: 'session-grok', + transcriptPath: transcriptA + } + } + + const result = await read() + + expect(result).toMatchObject({ + source: 'transcript', + provider: 'grok', + transcript: { + messages: [{ role: 'assistant', blocks: [{ type: 'text', text: 'Grok worker only' }] }] + } + }) + expect(readTerminal).not.toHaveBeenCalled() + }) + + it('labels OpenCode as a terminal fallback when no transcript decoder exists', async () => { + const capability = `dcap_${'A'.repeat(43)}` + readTerminal.mockResolvedValue({ + handle: 'term_worker', + status: 'running', + tail: [`opencode --dispatch-capability ${capability}`], + truncated: false, + nextCursor: '9' + }) + providerSession = { + ...providerSession!, + agent: 'opencode', + providerSession: { + key: 'session_id', + id: 'session-opencode', + transcriptPath: transcriptA + } + } + + const result = await read() + + expect(result).toMatchObject({ + source: 'terminal', + fallbackReason: 'provider_unsupported', + terminal: { tail: ['opencode --dispatch-capability [dispatch capability redacted]'] }, + warnings: ['Dispatch capability tokens were redacted from terminal output.'] + }) + expect(JSON.stringify(result)).not.toContain(capability) + }) + + it('rejects an old cursor after the exact provider session changes', async () => { + const initial = await read() + if (initial.source !== 'transcript') { + throw new Error('Expected transcript output') + } + providerSession = { + ...providerSession!, + providerSession: { + key: 'session_id', + id: 'session-b', + transcriptPath: transcriptB + } + } + + await expect(read({ cursor: initial.cursor })).rejects.toMatchObject({ + code: 'source_changed' + }) + }) + + it('uses a labeled terminal fallback and keeps its cursor pinned', async () => { + providerSession = null + const fallback = await read() + + expect(fallback).toMatchObject({ + source: 'terminal', + fallbackReason: 'session_not_reported', + terminal: { tail: ['terminal output'] } + }) + expect(fallback.cursor).toMatch(/^owr1_/) + + providerSession = { + paneKey: 'tab:worker', + processIncarnation: 'pty:incarnation-1', + agent: 'codex', + providerSession: { + key: 'session_id', + id: 'session-a', + transcriptPath: transcriptA + }, + observedAt: Date.now() + } + await read({ cursor: fallback.cursor ?? undefined }) + + expect(readTerminal).toHaveBeenLastCalledWith('term_worker', { + cursor: 9, + limit: undefined + }) + }) + + it('fails instead of falling back when transcript output is required', async () => { + providerSession = null + + await expect(read({ source: 'transcript' })).rejects.toMatchObject({ + code: 'transcript_required', + data: { reason: 'session_not_reported' } + }) + expect(readTerminal).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/runtime/rpc/methods/orchestration-worker-output.ts b/src/main/runtime/rpc/methods/orchestration-worker-output.ts new file mode 100644 index 00000000000..60e3f942839 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-worker-output.ts @@ -0,0 +1,208 @@ +import type { + OrchestrationWorkerReadFallbackReason, + OrchestrationWorkerReadResult, + OrchestrationWorkerReadSource +} from '../../../../shared/orchestration-worker-output' +import type { RuntimeTerminalState } from '../../../../shared/runtime-types' +import type { OrcaRuntimeService } from '../../orca-runtime' +import { OrchestrationError } from '../../orchestration/orchestration-error' +import { + createWorkerOutputSourceIdentity, + decodeWorkerOutputCursor, + encodeWorkerOutputCursor +} from '../../orchestration/worker-output-cursor' +import { redactWorkerTerminalLines } from '../../orchestration/worker-transcript-payload' +import { readWorkerTranscript } from '../../orchestration/worker-transcript-read' + +export async function readExactWorkerOutput(args: { + runtime: OrcaRuntimeService + dispatchId: string + terminalHandle: string + workerState: string + terminalStatus: RuntimeTerminalState + attachedAt: string + source?: OrchestrationWorkerReadSource + cursor?: string | number + limit?: number +}): Promise { + const source = args.source ?? 'auto' + const cursor = decodeWorkerOutputCursor(args.cursor, args.dispatchId) + assertCursorSourceMatchesRequest(cursor?.source, source) + + if (cursor?.source === 'terminal' || source === 'terminal') { + return readTerminalOutput(args, cursor) + } + + const observedAfter = orchestrationTimestampToMs(args.attachedAt) + const session = args.runtime.getExactWorkerProviderSession(args.terminalHandle, observedAfter) + if (!session) { + if (cursor?.source === 'transcript') { + throw sourceChanged() + } + return fallbackOrThrow(args, 'session_not_reported') + } + const transcript = await readWorkerTranscript({ + agent: session.agent, + sessionId: session.providerSession.id, + transcriptPath: session.providerSession.transcriptPath, + offset: cursor?.source === 'transcript' ? cursor.position : undefined, + limit: args.limit + }) + if (!transcript.ok) { + if (transcript.reason === 'source_changed') { + throw sourceChanged() + } + if (cursor?.source === 'transcript') { + throw transcriptRequired(args.dispatchId, transcript.reason) + } + return fallbackOrThrow(args, transcript.reason, transcript.warnings) + } + const sourceIdentity = createWorkerOutputSourceIdentity([ + 'transcript', + session.processIncarnation, + session.agent, + session.providerSession.key, + session.providerSession.id, + transcript.filePath + ]) + if (cursor?.source === 'transcript' && cursor.sourceIdentity !== sourceIdentity) { + throw sourceChanged() + } + const sessionAfterRead = args.runtime.getExactWorkerProviderSession( + args.terminalHandle, + observedAfter + ) + if ( + !sessionAfterRead || + sessionAfterRead.processIncarnation !== session.processIncarnation || + sessionAfterRead.agent !== session.agent || + sessionAfterRead.providerSession.key !== session.providerSession.key || + sessionAfterRead.providerSession.id !== session.providerSession.id || + sessionAfterRead.providerSession.transcriptPath !== session.providerSession.transcriptPath + ) { + throw sourceChanged() + } + const nextCursor = encodeWorkerOutputCursor( + args.dispatchId, + 'transcript', + sourceIdentity, + transcript.nextOffset + ) + return { + dispatchId: args.dispatchId, + source: 'transcript', + sourceIdentity, + provider: session.agent, + transcript: { + messages: transcript.messages, + nextCursor, + limited: transcript.limited, + returnedMessageCount: transcript.messages.length + }, + cursor: nextCursor, + status: { worker: args.workerState, terminal: args.terminalStatus }, + fallbackReason: null, + warnings: transcript.warnings + } +} + +async function readTerminalOutput( + args: Parameters[0], + cursor: ReturnType +): Promise { + const processIncarnation = args.runtime.getTerminalProcessIncarnation(args.terminalHandle) + const paneKey = args.runtime.getTerminalPaneKey(args.terminalHandle) + if (!processIncarnation || !paneKey) { + throw new OrchestrationError( + 'worker_identity_changed', + `Worker Dispatch ${args.dispatchId} no longer resolves to its exact process.` + ) + } + const sourceIdentity = createWorkerOutputSourceIdentity(['terminal', processIncarnation, paneKey]) + if ( + cursor?.source === 'terminal' && + cursor.sourceIdentity !== null && + cursor.sourceIdentity !== sourceIdentity + ) { + throw sourceChanged() + } + const terminal = await args.runtime.readTerminal(args.terminalHandle, { + cursor: cursor?.source === 'terminal' ? cursor.position : undefined, + limit: args.limit + }) + const redactedTerminal = redactWorkerTerminalLines(terminal.tail) + const position = + terminal.nextCursor !== null && /^\d+$/.test(terminal.nextCursor) + ? Number.parseInt(terminal.nextCursor, 10) + : null + const nextCursor = + position === null + ? null + : encodeWorkerOutputCursor(args.dispatchId, 'terminal', sourceIdentity, position) + return { + dispatchId: args.dispatchId, + source: 'terminal', + sourceIdentity, + terminal: { ...terminal, tail: redactedTerminal.lines }, + cursor: nextCursor, + status: { worker: args.workerState, terminal: terminal.status }, + fallbackReason: null, + warnings: redactedTerminal.warnings + } +} + +async function fallbackOrThrow( + args: Parameters[0], + reason: OrchestrationWorkerReadFallbackReason, + warnings: string[] = [] +): Promise { + if (args.source === 'transcript') { + throw transcriptRequired(args.dispatchId, reason) + } + const fallback = await readTerminalOutput(args, null) + return fallback.source === 'terminal' + ? { + ...fallback, + fallbackReason: reason, + warnings: [...new Set([...fallback.warnings, ...warnings])] + } + : fallback +} + +function assertCursorSourceMatchesRequest( + cursorSource: 'terminal' | 'transcript' | undefined, + requestedSource: OrchestrationWorkerReadSource +): void { + if (cursorSource && requestedSource !== 'auto' && cursorSource !== requestedSource) { + throw new OrchestrationError( + 'cursor_invalid', + `The worker-read cursor is pinned to ${cursorSource} output.` + ) + } +} + +function orchestrationTimestampToMs(value: string): number { + const normalized = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/.test(value) + ? `${value.replace(' ', 'T')}Z` + : value + const parsed = Date.parse(normalized) + return Number.isFinite(parsed) ? parsed : 0 +} + +function sourceChanged(): OrchestrationError { + return new OrchestrationError( + 'source_changed', + 'The worker output source changed. Start a fresh worker-read without the old cursor.' + ) +} + +function transcriptRequired( + dispatchId: string, + reason: OrchestrationWorkerReadFallbackReason +): OrchestrationError { + return new OrchestrationError( + 'transcript_required', + `Structured output is unavailable for Dispatch ${dispatchId}: ${reason}.`, + { reason } + ) +} diff --git a/src/main/runtime/rpc/methods/orchestration-worker-setup-gate.ts b/src/main/runtime/rpc/methods/orchestration-worker-setup-gate.ts new file mode 100644 index 00000000000..35dc38d6dd9 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-worker-setup-gate.ts @@ -0,0 +1,67 @@ +import type { OrchestrationDb } from '../../orchestration/db' +import { + applyWaitForSetupOutcome, + type WorkerEffect, + type WorkerSetupReceipt +} from './orchestration-worker-topology' + +function residualWorkerEffects(effects: WorkerEffect[]): WorkerEffect[] { + return effects.filter( + (effect) => effect.action?.startsWith('created') || effect.action === 'reused_agent_terminal' + ) +} + +type WorkerSetupStageArgs = { + db: OrchestrationDb + dispatchId: string + worktreeId: string + terminalHandle: string + setup: WorkerSetupReceipt + effects: WorkerEffect[] +} + +export function persistWorkerReadinessStage(args: WorkerSetupStageArgs): void { + args.db.recordWorkerStage({ + dispatchId: args.dispatchId, + stage: 'terminal_readying', + worktreeId: args.worktreeId, + terminalHandle: args.terminalHandle, + setupState: args.setup.state, + effects: args.effects, + residualResources: residualWorkerEffects(args.effects) + }) +} + +export function persistGatedSetupSpawnFailure(args: WorkerSetupStageArgs): boolean { + if (args.setup.startupPolicy !== 'wait-for-setup' || args.setup.state !== 'spawn_failed') { + return false + } + args.db.recordWorkerStage({ + dispatchId: args.dispatchId, + stage: 'setup_start', + worktreeId: args.worktreeId, + terminalHandle: args.terminalHandle, + setupState: args.setup.state, + effects: args.effects, + residualResources: residualWorkerEffects(args.effects) + }) + return true +} + +export function persistWorkerSetupWaitOutcome( + args: WorkerSetupStageArgs & { wait: { satisfied: boolean; status: string } } +): void { + applyWaitForSetupOutcome(args.setup, args.effects, args.wait) + if (args.setup.startupPolicy !== 'wait-for-setup') { + return + } + args.db.recordWorkerStage({ + dispatchId: args.dispatchId, + stage: args.setup.state === 'failed' ? 'setup_failed' : 'setup_settled', + worktreeId: args.worktreeId, + terminalHandle: args.terminalHandle, + setupState: args.setup.state, + effects: args.effects, + residualResources: residualWorkerEffects(args.effects) + }) +} diff --git a/src/main/runtime/rpc/methods/orchestration-worker-start-receipt.ts b/src/main/runtime/rpc/methods/orchestration-worker-start-receipt.ts new file mode 100644 index 00000000000..167d4610984 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-worker-start-receipt.ts @@ -0,0 +1,41 @@ +import type { OrchestrationDb } from '../../orchestration/db' +import { + isUnknownWorkerStartOutcome, + type WorkerSetupReceipt +} from './orchestration-worker-topology' + +export function failWorkerStartWithReceipt(args: { + db: OrchestrationDb + runId: string + taskId: string + dispatchId: string + failedStage: string + error: unknown + setup: WorkerSetupReceipt +}): unknown { + const reason = args.error instanceof Error ? args.error.message : String(args.error) + const unknown = isUnknownWorkerStartOutcome(args.error, args.failedStage) + const worker = unknown + ? args.db.markWorkerStartUnknown(args.dispatchId, args.failedStage, reason) + : args.db.failWorkerStart(args.dispatchId, args.failedStage, reason) + return { + runId: args.runId, + taskId: args.taskId, + dispatchId: args.dispatchId, + state: worker.state === 'start_unknown' ? 'outcome_unknown' : worker.state, + stage: worker.stage, + failedStage: args.failedStage, + lastError: reason, + setup: args.setup, + effects: JSON.parse(worker.effects) as unknown[], + residualResources: JSON.parse(worker.residual_resources) as unknown[], + ...(unknown + ? { + nextCommands: [ + `orca orchestration worker-show --dispatch ${args.dispatchId} --json`, + `orca orchestration worker-abandon --dispatch ${args.dispatchId} --json` + ] + } + : {}) + } +} diff --git a/src/main/runtime/rpc/methods/orchestration-worker-start-schema.ts b/src/main/runtime/rpc/methods/orchestration-worker-start-schema.ts new file mode 100644 index 00000000000..d577598c966 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-worker-start-schema.ts @@ -0,0 +1,23 @@ +import { z } from 'zod' +import { OptionalFiniteNumber, OptionalString, requiredString } from '../schemas' + +export const WorkerStartParams = z.object({ + task: requiredString('Missing --task'), + on: OptionalString, + run: OptionalString, + from: requiredString('Missing --from'), + worktree: OptionalString, + name: OptionalString, + repo: OptionalString, + baseBranch: OptionalString, + displayName: OptionalString, + comment: OptionalString, + setup: z.enum(['run', 'skip', 'inherit']).optional(), + terminal: OptionalString, + agent: OptionalString, + retryOf: OptionalString, + timeoutMs: OptionalFiniteNumber, + devMode: z.boolean().optional() +}) + +export type WorkerStartInput = z.infer diff --git a/src/main/runtime/rpc/methods/orchestration-worker-stop.ts b/src/main/runtime/rpc/methods/orchestration-worker-stop.ts new file mode 100644 index 00000000000..91984c03ee5 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-worker-stop.ts @@ -0,0 +1,148 @@ +import { z } from 'zod' +import { OrchestrationError } from '../../orchestration/orchestration-error' +import { syncFederatedDispatch } from '../../orchestration/federation-sync' +import { defineMethod, type RpcMethod } from '../core' +import { requiredString } from '../schemas' +import { + inspectWorkerTerminal, + resolvePinnedFederatedServer +} from './orchestration-worker-observation' + +const WorkerDispatchParams = z.object({ dispatch: requiredString('Missing --dispatch') }) + +export const ORCHESTRATION_WORKER_STOP_METHODS: RpcMethod[] = [ + defineMethod({ + name: 'orchestration.workerStop', + params: WorkerDispatchParams, + handler: async (params, { runtime, orchestrationMutation }) => { + const db = runtime.getOrchestrationDb() + const federated = db.getFederatedDispatch(params.dispatch) + if (federated) { + if (!orchestrationMutation) { + throw new OrchestrationError( + 'invalid_argument', + 'Remote worker-stop requires a durable retry request.' + ) + } + const server = resolvePinnedFederatedServer(runtime, federated) + const begun = db.beginWorkerStop(params.dispatch) + if (begun.disposition === 'already_settled') { + return settledReceipt(params.dispatch, begun.worker.state) + } + try { + const remote = (await runtime.callOrchestrationWorkerServer( + server.environmentId, + 'orchestration.federationStop', + { dispatchId: params.dispatch }, + 30_000, + { orchestrationRequestId: orchestrationMutation.requestId } + )) as RemoteStopReceipt + if (remote.state === 'stopped') { + const worker = db.reconcileFederatedWorkerStop(params.dispatch) + return { + dispatchId: params.dispatch, + state: worker.state, + alreadySettled: remote.alreadySettled, + processAction: remote.processAction, + close: remote.close + } + } + if (remote.state === 'succeeded' || remote.state === 'failed') { + db.resumeFederatedWorkerForTerminalRelay(params.dispatch) + await syncFederatedDispatch(runtime, params.dispatch).catch(() => undefined) + return { + dispatchId: params.dispatch, + state: db.getWorkerDispatch(params.dispatch)?.state ?? remote.state, + alreadySettled: true, + processAction: 'none' + } + } + return unknownReceipt( + params.dispatch, + db.markWorkerStopUnknown( + params.dispatch, + remote.lastError ?? `The worker server returned ${remote.state}.` + ), + remote.processAction + ) + } catch (error) { + const reason = error instanceof Error ? error.message : String(error) + return unknownReceipt( + params.dispatch, + db.markWorkerStopUnknown(params.dispatch, reason), + 'unknown' + ) + } + } + + const begun = db.beginWorkerStop(params.dispatch) + if (begun.disposition === 'already_settled') { + return settledReceipt(params.dispatch, begun.worker.state) + } + const handle = begun.worker.agent_terminal_handle + if (!handle) { + return unknownReceipt( + params.dispatch, + db.markWorkerStopUnknown(params.dispatch, 'The Dispatch has no recorded agent terminal.'), + 'unknown' + ) + } + const observation = await inspectWorkerTerminal(runtime, db, params.dispatch) + if (!observation.exact || observation.status !== 'running') { + return unknownReceipt( + params.dispatch, + db.markWorkerStopUnknown( + params.dispatch, + `The recorded worker process is ${observation.status}; no terminal was closed.` + ), + 'none' + ) + } + try { + const close = await runtime.closeTerminal(handle) + const worker = db.settleWorkerStop(params.dispatch) + runtime.notifyMessageArrived(`dispatch:${params.dispatch}`, 'status') + return { + dispatchId: params.dispatch, + state: worker.state, + alreadySettled: false, + processAction: 'closed_agent_terminal', + close + } + } catch (error) { + const reason = error instanceof Error ? error.message : String(error) + return unknownReceipt( + params.dispatch, + db.markWorkerStopUnknown(params.dispatch, reason), + 'unknown' + ) + } + } + }) +] + +type RemoteStopReceipt = { + state: string + alreadySettled: boolean + processAction: string + close?: unknown + lastError?: string | null +} + +function settledReceipt(dispatchId: string, state: string) { + return { dispatchId, state, alreadySettled: true, processAction: 'none' } +} + +function unknownReceipt( + dispatchId: string, + worker: { state: string; last_error: string | null }, + processAction: string +) { + return { + dispatchId, + state: worker.state, + alreadySettled: false, + processAction, + lastError: worker.last_error + } +} diff --git a/src/main/runtime/rpc/methods/orchestration-worker-topology.ts b/src/main/runtime/rpc/methods/orchestration-worker-topology.ts new file mode 100644 index 00000000000..62ed7816487 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-worker-topology.ts @@ -0,0 +1,228 @@ +import type { TuiAgent } from '../../../../shared/types' +import type { OrcaRuntimeService } from '../../orca-runtime' +import type { OrchestrationDb } from '../../orchestration/db' + +export type WorkerEffect = { + kind: 'worktree' | 'terminal' | 'setup' | 'dispatch_input' + action?: string + role?: string + id?: string + state?: string + tabId?: string + leafId?: string + requested?: string + effective?: string + source?: string + hookFound?: boolean + startupPolicy?: string + terminalId?: string +} + +export type WorkerSetupReceipt = { + requested: 'run' | 'skip' | 'inherit' | 'not_applicable' + effective: 'run' | 'skip' | 'inherit' | 'not_applicable' + source: string + hookFound: boolean + startupPolicy: 'start-immediately' | 'wait-for-setup' + state: + | 'running' + | 'succeeded' + | 'failed' + | 'skipped' + | 'not_configured' + | 'spawn_failed' + | 'not_applicable' +} + +export function applyWaitForSetupOutcome( + receipt: WorkerSetupReceipt, + effects: WorkerEffect[], + wait: { satisfied: boolean; status: string } +): void { + if (receipt.startupPolicy !== 'wait-for-setup' || receipt.state !== 'running') { + return + } + if (wait.satisfied) { + receipt.state = 'succeeded' + } else if (wait.status === 'exited') { + receipt.state = 'failed' + } else { + return + } + const setupEffect = effects.find((effect) => effect.kind === 'setup') + if (setupEffect) { + setupEffect.state = receipt.state + } +} + +export async function createWorkerWorktree(args: { + runtime: OrcaRuntimeService + db: OrchestrationDb + dispatchId: string + requestedWorktree: string + coordinatorWorktree: Awaited> + params: { + repo?: string + name?: string + baseBranch?: string + displayName?: string + comment?: string + setup?: 'run' | 'skip' | 'inherit' + from: string + } + agent: TuiAgent + effects: WorkerEffect[] +}): Promise<{ + worktree: Awaited> + terminalHandle: string + setupReceipt: WorkerSetupReceipt +}> { + const { runtime, db, dispatchId, requestedWorktree, coordinatorWorktree, params, effects } = args + const setupDecision = params.setup ?? 'run' + db.recordWorkerStage({ dispatchId, stage: 'worktree_creating', effects }) + const created = await runtime.createManagedWorktree({ + repoSelector: params.repo ?? coordinatorWorktree.repoId, + name: params.name as string, + baseBranch: params.baseBranch, + displayName: params.displayName, + comment: params.comment, + runHooks: setupDecision === 'run', + setupDecision, + awaitTerminalProvisioning: true, + observeSetupCompletion: true, + createdWithAgent: args.agent, + startupAgent: args.agent, + activate: false, + lineage: { + parentWorktree: requestedWorktree === 'new-child' ? coordinatorWorktree.id : undefined, + noParent: requestedWorktree === 'new-top-level', + callerTerminalHandle: params.from + } + }) + const terminalHandle = created.startupTerminal?.handle + effects.push({ + kind: 'worktree', + action: requestedWorktree === 'new-child' ? 'created_child' : 'created_top_level', + id: created.worktree.id + }) + db.recordWorkerStage({ + dispatchId, + stage: 'worktree_created', + worktreeId: created.worktree.id, + effects, + residualResources: effects + }) + const setupReceipt = { + requested: setupDecision, + effective: setupDecision, + source: params.setup ? 'explicit_request' : 'orchestration_default', + hookFound: created.setupReceipt?.hookFound ?? false, + startupPolicy: created.setupReceipt?.startupPolicy ?? 'start-immediately', + state: created.setupReceipt?.state ?? 'not_configured' + } + if (!terminalHandle) { + throw new Error(created.warning ?? 'Agent-first worktree creation returned no terminal.') + } + const listed = await runtime.listTerminals(`id:${created.worktree.id}`) + const setupTerminalHandle = created.setupReceipt?.terminalHandle + for (const terminal of listed.terminals) { + effects.push({ + kind: 'terminal', + role: + terminal.handle === terminalHandle + ? 'agent' + : terminal.handle === setupTerminalHandle + ? 'setup' + : 'configured_tab', + action: terminal.handle === terminalHandle ? 'reused_agent_terminal' : 'created', + id: terminal.handle, + tabId: terminal.tabId, + leafId: terminal.leafId + }) + } + const setupTerminal = effects.find( + (effect) => effect.kind === 'terminal' && effect.role === 'setup' + ) + effects.push({ + kind: 'setup', + action: setupDecision, + requested: setupReceipt.requested, + effective: setupReceipt.effective, + source: setupReceipt.source, + hookFound: setupReceipt.hookFound, + startupPolicy: setupReceipt.startupPolicy, + state: setupReceipt.state, + terminalId: setupTerminalHandle ?? setupTerminal?.id + }) + return { + worktree: created.worktree as Awaited>, + terminalHandle, + setupReceipt + } +} + +export function monitorWorkerSetup(args: { + runtime: OrcaRuntimeService + db: OrchestrationDb + runId: string + dispatchId: string + setupReceipt: WorkerSetupReceipt + effects: WorkerEffect[] +}): void { + const setupTerminal = args.effects.find( + (effect) => effect.kind === 'terminal' && effect.role === 'setup' && effect.id + ) + if ( + !setupTerminal?.id || + args.setupReceipt.startupPolicy !== 'start-immediately' || + args.setupReceipt.state !== 'running' + ) { + return + } + // Why: setup is intentionally non-gating, but command completion remains durable evidence. + void args.runtime + .waitForSetupTerminalCompletion(setupTerminal.id) + .then((completion) => { + const setupState = completion.exitCode === 0 ? 'succeeded' : 'failed' + const evidence = args.db.updateWorkerSetupEvidence({ + dispatchId: args.dispatchId, + setupState, + effects: args.effects.map((effect) => + effect.kind === 'setup' ? { ...effect, state: setupState } : effect + ) + }) + if (!evidence.changed) { + return + } + const message = args.db.insertMessage({ + runId: args.runId, + from: `dispatch:${args.dispatchId}`, + to: `run:${args.runId}`, + subject: `Setup ${setupState} for worker ${args.dispatchId}`, + type: 'status', + priority: setupState === 'failed' ? 'high' : 'normal', + payload: JSON.stringify({ + dispatchId: args.dispatchId, + setupState, + terminalHandle: setupTerminal.id + }) + }) + args.runtime.notifyMessageArrived(message.to_handle, message.type) + }) + .catch(() => undefined) +} + +export function isUnknownWorkerStartOutcome(error: unknown, stage: string): boolean { + const code = + error && typeof error === 'object' && typeof (error as { code?: unknown }).code === 'string' + ? (error as { code: string }).code + : '' + if (code === 'operation_unknown') { + return true + } + if (stage !== 'worktree_create') { + return false + } + const message = error instanceof Error ? error.message : String(error) + return /connection|disconnect|timed?\s*out|runtime changed|outcome unknown/i.test(message) +} diff --git a/src/main/runtime/rpc/methods/orchestration-workers-new-worktree.test.ts b/src/main/runtime/rpc/methods/orchestration-workers-new-worktree.test.ts new file mode 100644 index 00000000000..0bfadd7d351 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-workers-new-worktree.test.ts @@ -0,0 +1,614 @@ +import { createHash } from 'node:crypto' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { ORCHESTRATION_CONTRACT_VERSION } from '../../../../shared/protocol-version' +import { OrcaRuntimeService } from '../../orca-runtime' +import { OrchestrationDb } from '../../orchestration/db' +import { RpcDispatcher } from '../dispatcher' +import type { RpcRequest } from '../core' +import { ORCHESTRATION_METHODS } from './orchestration' + +describe('orchestration new-worktree workers', () => { + type CreateWorktreeResult = Awaited> + const coordinatorPaneKey = 'tab_coord:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' + let db: OrchestrationDb + let runtime: OrcaRuntimeService + let runId: string + + beforeEach(() => { + db = new OrchestrationDb(':memory:') + runtime = new OrcaRuntimeService() + runtime.setOrchestrationDb(db) + runId = db.createRun({ + objective: 'Test new-worktree workers', + coordinatorHandle: 'term_coord', + coordinatorPaneKey + }).id + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === 'term_coord' + ? coordinatorPaneKey + : handle === 'term_worker' + ? 'tab_worker:bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb' + : null + ) + vi.spyOn(runtime, 'getTerminalProcessIncarnation').mockImplementation((handle) => + handle === 'term_worker' ? 'runtime_test:term_worker:1' : null + ) + vi.spyOn(runtime, 'validateOrchestrationAgentLauncher').mockImplementation(() => {}) + vi.spyOn(runtime, 'showTerminal').mockResolvedValue({ + handle: 'term_coord', + worktreeId: 'repo::parent', + status: 'running' + } as never) + vi.spyOn(runtime, 'showManagedWorktree').mockResolvedValue({ + id: 'repo::parent', + repoId: 'repo' + } as never) + vi.spyOn(runtime, 'showRepo').mockResolvedValue({ + id: 'repo', + kind: 'git' + } as never) + vi.spyOn(runtime, 'createTerminal') + vi.spyOn(runtime, 'listTerminals').mockResolvedValue({ + terminals: [{ handle: 'term_worker', title: 'Codex' }], + totalCount: 1, + truncated: false + } as never) + vi.spyOn(runtime, 'waitForTerminal').mockResolvedValue({ + handle: 'term_worker', + condition: 'tui-idle', + satisfied: true, + status: 'running', + exitCode: null + }) + vi.spyOn(runtime, 'waitForSetupTerminalCompletion').mockReturnValue( + new Promise(() => undefined) + ) + vi.spyOn(runtime, 'getTerminalOrchestrationCliCommand').mockReturnValue('orca') + vi.spyOn(runtime, 'sendTerminalAgentPrompt').mockResolvedValue({ + handle: 'term_worker', + accepted: true, + bytesWritten: 1 + }) + }) + + afterEach(() => db.close()) + + async function startWorker(overrides: Record = {}) { + const task = db.createTask({ spec: 'new-worktree task', runId }) + const method = ORCHESTRATION_METHODS.find( + (candidate) => candidate.name === 'orchestration.workerStart' + ) + if (!method) { + throw new Error('workerStart method is not registered') + } + const params = method.params!.parse({ + task: task.id, + from: 'term_coord', + worktree: 'new-child', + name: 'new-worker', + agent: 'codex', + ...overrides + }) + const result = await method.handler(params, { runtime }) + return { result, task } + } + + function mockCreatedWorktree(options?: { + hookFound?: boolean + startupPolicy?: 'start-immediately' | 'wait-for-setup' + state?: 'running' | 'skipped' | 'not_configured' | 'spawn_failed' + terminals?: { handle: string; title: string }[] + setupTerminalHandle?: string + }) { + const hookFound = options?.hookFound ?? true + const state = options?.state ?? (hookFound ? 'running' : 'not_configured') + vi.spyOn(runtime, 'createManagedWorktree').mockResolvedValue({ + worktree: { id: 'repo::created', repoId: 'repo' }, + startupTerminal: { spawned: true, handle: 'term_worker' }, + setupReceipt: { + requested: state === 'skipped' ? 'skip' : 'run', + hookFound, + startupPolicy: options?.startupPolicy ?? 'start-immediately', + state, + terminalHandle: + options?.setupTerminalHandle ?? + options?.terminals?.find((terminal) => terminal.title === 'Setup')?.handle + } + } as never) + if (options?.terminals) { + vi.mocked(runtime.listTerminals).mockResolvedValue({ + terminals: options.terminals, + totalCount: options.terminals.length, + truncated: false + } as never) + } + } + + it('creates an independent top-level worktree and reuses its agent terminal', async () => { + mockCreatedWorktree() + + const { result } = await startWorker({ worktree: 'new-top-level' }) + + expect(runtime.createManagedWorktree).toHaveBeenCalledWith( + expect.objectContaining({ + startupAgent: 'codex', + awaitTerminalProvisioning: true, + observeSetupCompletion: true, + lineage: expect.objectContaining({ noParent: true, parentWorktree: undefined }) + }) + ) + expect(result).toMatchObject({ state: 'ready' }) + expect(result).toHaveProperty( + 'effects', + expect.arrayContaining([ + expect.objectContaining({ + kind: 'worktree', + action: 'created_top_level', + id: 'repo::created' + }), + expect.objectContaining({ + kind: 'terminal', + role: 'agent', + action: 'reused_agent_terminal', + id: 'term_worker' + }) + ]) + ) + expect(runtime.createTerminal).not.toHaveBeenCalled() + }) + + it('rejects a new worktree for a folder project before creating effects', async () => { + vi.mocked(runtime.showRepo).mockResolvedValue({ + id: 'repo', + kind: 'folder' + } as never) + const createWorktree = vi.spyOn(runtime, 'createManagedWorktree') + const task = db.createTask({ spec: 'folder task', runId }) + const method = ORCHESTRATION_METHODS.find( + (candidate) => candidate.name === 'orchestration.workerStart' + ) + if (!method) { + throw new Error('workerStart method is not registered') + } + + await expect( + method.handler( + method.params!.parse({ + task: task.id, + from: 'term_coord', + worktree: 'new-child', + name: 'folder-worker', + agent: 'codex' + }), + { runtime } + ) + ).rejects.toMatchObject({ + code: 'invalid_argument', + message: + 'Folder projects cannot create orchestration worktrees; use current or an exact existing folder workspace.' + }) + expect(createWorktree).not.toHaveBeenCalled() + expect(db.getTask(task.id)?.status).toBe('ready') + expect(db.getDispatchContext(task.id)).toBeUndefined() + }) + + it('injects the execution host CLI command and Dispatch capability together', async () => { + mockCreatedWorktree() + vi.mocked(runtime.getTerminalOrchestrationCliCommand).mockReturnValue('orca-ide') + + await startWorker({ worktree: 'new-top-level' }) + + const prompt = vi.mocked(runtime.sendTerminalAgentPrompt).mock.calls[0]?.[1] ?? '' + expect(prompt).toContain('orca-ide orchestration send') + expect(prompt).toMatch(/--dispatch-capability dcap_[A-Za-z0-9_-]+/) + expect(prompt).not.toMatch(/(^|\s)orca orchestration send/) + }) + + it('passes exact repo, base, metadata, lineage, and setup choices to worktree creation', async () => { + mockCreatedWorktree({ state: 'skipped' }) + + await startWorker({ + worktree: 'new-top-level', + repo: 'id:repo-explicit', + baseBranch: 'origin/release', + displayName: 'Windows release audit', + comment: 'Created for a supervised audit', + setup: 'skip' + }) + + expect(runtime.createManagedWorktree).toHaveBeenCalledWith( + expect.objectContaining({ + repoSelector: 'id:repo-explicit', + baseBranch: 'origin/release', + displayName: 'Windows release audit', + comment: 'Created for a supervised audit', + setupDecision: 'skip', + runHooks: false, + lineage: expect.objectContaining({ noParent: true, parentWorktree: undefined }) + }) + ) + }) + + it('reports an absent setup hook as not configured without failing the start', async () => { + mockCreatedWorktree({ hookFound: false }) + + const { result } = await startWorker() + + expect(result).toMatchObject({ + state: 'ready', + setup: { + requested: 'run', + effective: 'run', + source: 'orchestration_default', + hookFound: false, + state: 'not_configured' + } + }) + }) + + it.each([ + ['skip', 'skipped'], + ['inherit', 'not_configured'], + ['run', 'running'] + ] as const)('passes explicit setup=%s through with a truthful receipt', async (setup, state) => { + mockCreatedWorktree({ hookFound: setup === 'run', state }) + + const { result } = await startWorker({ setup }) + + expect(runtime.createManagedWorktree).toHaveBeenCalledWith( + expect.objectContaining({ setupDecision: setup, runHooks: setup === 'run' }) + ) + expect(result).toMatchObject({ + state: 'ready', + setup: { requested: setup, effective: setup, source: 'explicit_request', state } + }) + }) + + it('records a later setup failure without gating a start-immediately worker', async () => { + mockCreatedWorktree({ + terminals: [ + { handle: 'term_worker', title: 'Codex' }, + { handle: 'term_setup', title: 'Setup' } + ] + }) + let finishSetup: ((result: { exitCode: number | null }) => void) | undefined + vi.mocked(runtime.waitForSetupTerminalCompletion).mockImplementation( + async () => + await new Promise((resolve) => { + finishSetup = resolve + }) + ) + + const { result, task } = await startWorker() + const dispatchId = (result as { dispatchId: string }).dispatchId + + expect(result).toMatchObject({ state: 'ready', setup: { state: 'running' } }) + expect( + db.settleWorkerReport({ + taskId: task.id, + dispatchId, + outcome: 'succeeded', + result: '{}' + }) + ).toMatchObject({ action: 'settled' }) + finishSetup?.({ exitCode: 1 }) + await vi.waitFor(() => expect(db.getWorkerDispatch(dispatchId)?.setup_state).toBe('failed')) + expect(db.getWorkerDispatch(dispatchId)).toMatchObject({ + state: 'succeeded', + stage: 'settled', + setup_state: 'failed' + }) + expect(JSON.parse(db.getWorkerDispatch(dispatchId)?.effects ?? '[]')).toEqual( + expect.arrayContaining([ + expect.objectContaining({ kind: 'dispatch_input', state: 'accepted' }) + ]) + ) + expect(runtime.sendTerminalAgentPrompt).toHaveBeenCalledOnce() + expect(db.getInbox(10).filter((message) => message.run_id === runId)).toEqual( + expect.arrayContaining([expect.objectContaining({ type: 'status', priority: 'high' })]) + ) + }) + + it('uses the exact setup handle instead of a configured tab title', async () => { + mockCreatedWorktree({ + setupTerminalHandle: 'term_actual_setup', + terminals: [ + { handle: 'term_worker', title: 'Codex' }, + { handle: 'term_configured_setup', title: 'Setup' }, + { handle: 'term_actual_setup', title: 'PowerShell' } + ] + }) + + const { result } = await startWorker() + + expect(result).toMatchObject({ + effects: expect.arrayContaining([ + expect.objectContaining({ + kind: 'terminal', + id: 'term_configured_setup', + role: 'configured_tab' + }), + expect.objectContaining({ kind: 'terminal', id: 'term_actual_setup', role: 'setup' }), + expect.objectContaining({ kind: 'setup', terminalId: 'term_actual_setup' }) + ]) + }) + }) + + it('records wait-for-setup success before task input is accepted', async () => { + mockCreatedWorktree({ startupPolicy: 'wait-for-setup', state: 'running' }) + + const { result } = await startWorker() + const dispatchId = (result as { dispatchId: string }).dispatchId + + expect(result).toMatchObject({ + state: 'ready', + setup: { startupPolicy: 'wait-for-setup', state: 'succeeded' }, + effects: expect.arrayContaining([ + expect.objectContaining({ kind: 'setup', state: 'succeeded' }), + expect.objectContaining({ kind: 'dispatch_input', state: 'accepted' }) + ]) + }) + expect(vi.mocked(runtime.waitForTerminal).mock.invocationCallOrder[0]).toBeLessThan( + vi.mocked(runtime.sendTerminalAgentPrompt).mock.invocationCallOrder[0]! + ) + expect(JSON.parse(db.getWorkerDispatch(dispatchId)?.effects ?? '[]')).toEqual( + expect.arrayContaining([ + expect.objectContaining({ kind: 'dispatch_input', state: 'accepted' }) + ]) + ) + }) + + it('does not inject task input when the gated setup terminal fails to start', async () => { + mockCreatedWorktree({ startupPolicy: 'wait-for-setup', state: 'spawn_failed' }) + vi.mocked(runtime.waitForTerminal).mockResolvedValue({ + handle: 'term_worker', + condition: 'tui-idle', + satisfied: false, + status: 'exited', + exitCode: 1 + }) + + const { result, task } = await startWorker() + + expect(result).toMatchObject({ + state: 'failed', + failedStage: 'setup_start', + setup: { startupPolicy: 'wait-for-setup', state: 'spawn_failed' }, + effects: expect.arrayContaining([ + expect.objectContaining({ kind: 'setup', state: 'spawn_failed' }) + ]) + }) + expect(db.getTask(task.id)?.status).toBe('failed') + expect(runtime.sendTerminalAgentPrompt).not.toHaveBeenCalled() + }) + + it('does not inject task input when the gated setup script fails', async () => { + mockCreatedWorktree({ startupPolicy: 'wait-for-setup', state: 'running' }) + vi.mocked(runtime.waitForTerminal).mockResolvedValue({ + handle: 'term_worker', + condition: 'tui-idle', + satisfied: false, + status: 'exited', + exitCode: 1 + }) + + const { result } = await startWorker() + + expect(result).toMatchObject({ + state: 'failed', + failedStage: 'setup_wait', + setup: { state: 'failed' }, + effects: expect.arrayContaining([expect.objectContaining({ kind: 'setup', state: 'failed' })]) + }) + expect(runtime.sendTerminalAgentPrompt).not.toHaveBeenCalled() + }) + + it('does not mislabel a wait-for-setup timeout as setup failure', async () => { + mockCreatedWorktree({ startupPolicy: 'wait-for-setup', state: 'running' }) + vi.mocked(runtime.waitForTerminal).mockResolvedValue({ + handle: 'term_worker', + condition: 'tui-idle', + satisfied: false, + status: 'running', + exitCode: null + }) + + const { result } = await startWorker() + + expect(result).toMatchObject({ + state: 'failed', + failedStage: 'agent_readiness', + setup: { state: 'running' } + }) + expect(runtime.sendTerminalAgentPrompt).not.toHaveBeenCalled() + }) + + it('distinguishes no-effect failure, unknown acceptance, and durable residual effects', async () => { + vi.spyOn(runtime, 'createManagedWorktree').mockRejectedValueOnce( + new Error('repository validation failed before creation') + ) + const noEffect = await startWorker({ name: 'no-effect' }) + expect(noEffect.result).toMatchObject({ + state: 'failed', + failedStage: 'worktree_create', + effects: [], + residualResources: [] + }) + + vi.mocked(runtime.createManagedWorktree).mockRejectedValueOnce( + Object.assign(new Error('connection lost after possible acceptance'), { + code: 'operation_unknown' + }) + ) + const unknown = await startWorker({ name: 'unknown-effect' }) + expect(unknown.result).toMatchObject({ + state: 'outcome_unknown', + failedStage: 'worktree_create', + effects: [], + residualResources: [] + }) + + mockCreatedWorktree() + vi.mocked(runtime.waitForTerminal).mockResolvedValueOnce({ + handle: 'term_worker', + condition: 'tui-idle', + satisfied: false, + status: 'exited', + exitCode: 1 + }) + const durableEffect = await startWorker({ name: 'durable-effect' }) + expect(durableEffect.result).toMatchObject({ + state: 'failed', + failedStage: 'agent_readiness', + effects: expect.arrayContaining([ + expect.objectContaining({ kind: 'worktree', id: 'repo::created' }), + expect.objectContaining({ kind: 'terminal', id: 'term_worker' }) + ]), + residualResources: expect.arrayContaining([ + expect.objectContaining({ kind: 'worktree', id: 'repo::created' }), + expect.objectContaining({ kind: 'terminal', id: 'term_worker' }) + ]) + }) + }) + + it('returns outcome unknown when worktree creation may have been accepted remotely', async () => { + vi.spyOn(runtime, 'createManagedWorktree').mockRejectedValue( + Object.assign(new Error('connection closed after request acceptance'), { + code: 'operation_unknown' + }) + ) + + const { result, task } = await startWorker() + + expect(result).toMatchObject({ + state: 'outcome_unknown', + failedStage: 'worktree_create', + nextCommands: expect.arrayContaining([ + expect.stringContaining('worker-show --dispatch'), + expect.stringContaining('worker-abandon --dispatch') + ]) + }) + expect(db.getTask(task.id)?.status).toBe('blocked') + }) + + it('persists the retry request with the starting Dispatch before worktree effects', async () => { + const task = db.createTask({ spec: 'atomic worker acceptance', runId }) + let finishCreate: ((value: CreateWorktreeResult) => void) | undefined + vi.spyOn(runtime, 'createManagedWorktree').mockImplementation( + async () => + await new Promise((resolve) => { + finishCreate = resolve + }) + ) + const dispatcher = new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS }) + const request: RpcRequest = { + id: 'rpc_worker_start', + authToken: 'caller-token', + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationRequestId: 'worker_start_request', + method: 'orchestration.workerStart', + params: { + task: task.id, + from: 'term_coord', + worktree: 'new-child', + name: 'atomic-worker', + agent: 'codex' + } + } + + const pending = dispatcher.dispatch(request) + await vi.waitFor(() => expect(db.getDispatchContext(task.id)).toBeDefined()) + const acceptedDispatch = db.getDispatchContext(task.id)! + const callerFingerprint = createHash('sha256').update('caller-token').digest('hex') + const receipt = db.getMutationReceipt(callerFingerprint, 'worker_start_request') + + expect(receipt).toMatchObject({ + request_id: 'worker_start_request', + method: 'orchestration.workerStart', + state: 'pending' + }) + expect(db.getWorkerDispatch(acceptedDispatch.id)).toMatchObject({ + state: 'starting', + stage: 'worktree_creating' + }) + + finishCreate?.({ + worktree: { id: 'repo::created', repoId: 'repo' }, + startupTerminal: { spawned: true, handle: 'term_worker' }, + setupReceipt: { + requested: 'run', + hookFound: false, + startupPolicy: 'start-immediately', + state: 'not_configured' + } + } as CreateWorktreeResult) + await expect(pending).resolves.toMatchObject({ + ok: true, + result: { state: 'ready', mutation: { requestId: 'worker_start_request' } } + }) + expect(db.getMutationReceipt(callerFingerprint, 'worker_start_request')).toMatchObject({ + state: 'completed' + }) + }) + + it('persists pre-effect, post-effect, and post-input stages in order', async () => { + mockCreatedWorktree({ hookFound: false }) + let finishWait: + | ((value: Awaited>) => void) + | undefined + let finishPrompt: + | ((value: Awaited>) => void) + | undefined + vi.mocked(runtime.waitForTerminal).mockImplementationOnce( + async () => + await new Promise((resolve) => { + finishWait = resolve + }) + ) + vi.mocked(runtime.sendTerminalAgentPrompt).mockImplementationOnce( + async () => + await new Promise((resolve) => { + finishPrompt = resolve + }) + ) + + const pending = startWorker({ name: 'staged-worker' }) + await vi.waitFor(() => { + const task = db.listTasks()[0] + const dispatch = task ? db.getDispatchContext(task.id) : undefined + expect(dispatch && db.getWorkerDispatch(dispatch.id)).toMatchObject({ + state: 'starting', + stage: 'terminal_readying', + worktree_id: 'repo::created', + agent_terminal_handle: 'term_worker' + }) + }) + const dispatch = db.getDispatchContext(db.listTasks()[0]!.id)! + expect(JSON.parse(db.getWorkerDispatch(dispatch.id)!.residual_resources)).toEqual( + expect.arrayContaining([expect.objectContaining({ kind: 'worktree', id: 'repo::created' })]) + ) + + finishWait?.({ + handle: 'term_worker', + condition: 'tui-idle', + satisfied: true, + status: 'running', + exitCode: null + }) + await vi.waitFor(() => + expect(db.getWorkerDispatch(dispatch.id)).toMatchObject({ + state: 'starting', + stage: 'authority_attached' + }) + ) + + finishPrompt?.({ handle: 'term_worker', accepted: true, bytesWritten: 1 }) + await expect(pending).resolves.toMatchObject({ + result: { state: 'ready', stage: 'input_accepted' } + }) + expect(db.getWorkerDispatch(dispatch.id)).toMatchObject({ + state: 'ready', + stage: 'input_accepted' + }) + }) +}) diff --git a/src/main/runtime/rpc/methods/orchestration-workers-recovery.test.ts b/src/main/runtime/rpc/methods/orchestration-workers-recovery.test.ts new file mode 100644 index 00000000000..880620eee43 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-workers-recovery.test.ts @@ -0,0 +1,246 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { OrcaRuntimeService } from '../../orca-runtime' +import { OrchestrationDb } from '../../orchestration/db' +import { ORCHESTRATION_METHODS } from './orchestration' + +describe('orchestration worker recovery', () => { + let db: OrchestrationDb + let runtime: OrcaRuntimeService + + beforeEach(() => { + db = new OrchestrationDb(':memory:') + runtime = new OrcaRuntimeService() + runtime.setOrchestrationDb(db) + vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue( + 'tab_worker:bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb' + ) + vi.spyOn(runtime, 'getTerminalProcessIncarnation').mockReturnValue('runtime:pty:1') + vi.spyOn(runtime, 'showTerminal').mockResolvedValue({ + handle: 'term_worker', + worktreeId: 'repo::worktree', + connected: true, + status: 'running' + } as never) + vi.spyOn(runtime, 'readTerminal').mockResolvedValue({ + handle: 'term_worker', + status: 'running', + tail: ['working'], + truncated: false, + nextCursor: null + }) + vi.spyOn(runtime, 'closeTerminal').mockResolvedValue({ + handle: 'term_worker', + closed: true + } as never) + }) + + afterEach(() => db.close()) + + async function call(name: string, params: Record) { + const method = ORCHESTRATION_METHODS.find((candidate) => candidate.name === name) + if (!method) { + throw new Error(`Method not found: ${name}`) + } + return method.handler(method.params!.parse(params), { runtime }) + } + + function createWorker(runtimeEpoch = runtime.getRuntimeId(), ready = true) { + const run = db.createRun({ + objective: 'Recovery', + coordinatorHandle: 'term_coord', + coordinatorPaneKey: 'tab_coord:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' + }) + const task = db.createTask({ spec: 'recover worker', runId: run.id }) + const started = db.createStartingWorkerDispatch({ + taskId: task.id, + startOptions: {}, + runtimeEpoch + }) + db.prepareStartingWorkerAuthority({ + dispatchId: started.dispatch.id, + handle: 'term_worker', + paneKey: 'tab_worker:bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb', + processIncarnation: 'runtime:pty:1', + worktreeId: 'repo::worktree', + setupState: 'not_applicable', + effects: [{ kind: 'terminal', action: 'created', id: 'term_worker' }] + }) + if (ready) { + db.markWorkerDispatchReady(started.dispatch.id) + } else { + db.markWorkerStartUnknown(started.dispatch.id, 'dispatch_input', 'connection lost') + } + return { run, task, dispatch: started.dispatch } + } + + it('shows and reads only the exact attached worker process', async () => { + const { dispatch } = createWorker() + + await expect( + call('orchestration.workerShow', { dispatch: dispatch.id }) + ).resolves.toMatchObject({ + worker: { state: 'ready' }, + observation: { status: 'running', exactWorker: true }, + terminal: { handle: 'term_worker' } + }) + await expect( + call('orchestration.workerRead', { dispatch: dispatch.id }) + ).resolves.toMatchObject({ + dispatchId: dispatch.id, + terminal: { tail: ['working'] } + }) + }) + + it('stops an exact worker whose start receipt is unknown', async () => { + const { task, dispatch } = createWorker(runtime.getRuntimeId(), false) + + await expect( + call('orchestration.workerStop', { dispatch: dispatch.id }) + ).resolves.toMatchObject({ + state: 'stopped', + processAction: 'closed_agent_terminal' + }) + expect(runtime.closeTerminal).toHaveBeenCalledWith('term_worker') + expect(db.getTask(task.id)?.status).toBe('blocked') + }) + + it('does not adopt or stop a same-looking pane with a new process incarnation', async () => { + const { task, dispatch } = createWorker() + vi.mocked(runtime.getTerminalProcessIncarnation).mockReturnValue('runtime:pty:2') + + await expect( + call('orchestration.workerShow', { dispatch: dispatch.id }) + ).resolves.toMatchObject({ + worker: { state: 'ready' }, + observation: { status: 'identity_changed', exactWorker: false }, + terminal: null + }) + await expect(call('orchestration.workerRead', { dispatch: dispatch.id })).rejects.toMatchObject( + { + code: 'worker_identity_changed' + } + ) + await expect( + call('orchestration.workerStop', { dispatch: dispatch.id }) + ).resolves.toMatchObject({ + state: 'stop_unknown', + processAction: 'none' + }) + expect(runtime.closeTerminal).not.toHaveBeenCalled() + expect(db.getTask(task.id)?.status).toBe('blocked') + }) + + it('labels an exact but disconnected worker as exited and does not close it again', async () => { + const { task, dispatch } = createWorker() + vi.mocked(runtime.showTerminal).mockResolvedValue({ + handle: 'term_worker', + worktreeId: 'repo::worktree', + connected: false, + writable: false + } as never) + + await expect( + call('orchestration.workerShow', { dispatch: dispatch.id }) + ).resolves.toMatchObject({ + observation: { status: 'exited', exactWorker: true }, + terminal: { handle: 'term_worker', connected: false } + }) + await expect( + call('orchestration.workerStop', { dispatch: dispatch.id }) + ).resolves.toMatchObject({ + state: 'stop_unknown', + processAction: 'none' + }) + expect(runtime.closeTerminal).not.toHaveBeenCalled() + expect(db.getTask(task.id)?.status).toBe('blocked') + }) + + it('turns an interrupted start into inspectable unknown after runtime restart', async () => { + const run = db.createRun({ + objective: 'Interrupted start', + coordinatorHandle: 'term_coord', + coordinatorPaneKey: 'tab_coord:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' + }) + const task = db.createTask({ spec: 'interrupted', runId: run.id }) + const started = db.createStartingWorkerDispatch({ + taskId: task.id, + startOptions: {}, + runtimeEpoch: 'previous_runtime' + }) + db.recordWorkerStage({ + dispatchId: started.dispatch.id, + stage: 'worktree_creating' + }) + + await expect( + call('orchestration.workerShow', { dispatch: started.dispatch.id }) + ).resolves.toMatchObject({ + worker: { state: 'start_unknown', stage: 'worktree_creating' }, + observation: { status: 'unattached', exactWorker: false } + }) + expect(db.getTask(task.id)?.status).toBe('blocked') + }) + + it('turns an interrupted stop into unknown after runtime restart', async () => { + const { task, dispatch } = createWorker('previous_runtime') + db.beginWorkerStop(dispatch.id) + + await expect( + call('orchestration.workerShow', { dispatch: dispatch.id }) + ).resolves.toMatchObject({ + worker: { state: 'stop_unknown' } + }) + expect(db.getTask(task.id)?.status).toBe('blocked') + }) + + it('reconciles a stop_unknown Dispatch from an authoritative remote stopped receipt', async () => { + const run = db.createRun({ + objective: 'Lost remote stop response', + coordinatorHandle: 'term_coord', + coordinatorPaneKey: 'tab_coord:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' + }) + const task = db.createTask({ spec: 'stop remote worker', runId: run.id }) + const started = db.createStartingWorkerDispatch({ + taskId: task.id, + startOptions: {}, + runtimeEpoch: runtime.getRuntimeId(), + federation: { + environmentId: 'environment_windows', + environmentName: 'windows', + peerFingerprint: 'windows_peer', + protocolVersion: 1 + } + }) + db.markWorkerStartUnknown(started.dispatch.id, 'remote_attach', 'response lost') + db.beginWorkerStop(started.dispatch.id) + db.markWorkerStopUnknown(started.dispatch.id, 'stop response lost') + vi.spyOn(runtime, 'resolveOrchestrationWorkerServer').mockReturnValue({ + environmentId: 'environment_windows', + name: 'windows', + peerFingerprint: 'windows_peer' + }) + vi.spyOn(runtime, 'callOrchestrationWorkerServer').mockResolvedValue({ + runtimeEpoch: 'windows_epoch', + attachment: { + state: 'stopped', + stage: 'process_stopped', + last_error: null, + worktree_id: 'repo::windows-worktree', + terminal_handle: 'term_windows_worker', + setup_state: 'running', + effects: [], + residualResources: [] + }, + terminal: { handle: 'term_windows_worker', connected: false }, + observation: { status: 'exited', exactWorker: true } + }) + + await expect( + call('orchestration.workerShow', { dispatch: started.dispatch.id }) + ).resolves.toMatchObject({ + worker: { state: 'stopped', stage: 'process_stopped', last_error: null }, + observation: { status: 'exited', exactWorker: true } + }) + expect(db.getTask(task.id)?.status).toBe('blocked') + }) +}) diff --git a/src/main/runtime/rpc/methods/orchestration-workers.ts b/src/main/runtime/rpc/methods/orchestration-workers.ts new file mode 100644 index 00000000000..0b2b19a18e0 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-workers.ts @@ -0,0 +1,305 @@ +import { isTuiAgent } from '../../../../shared/tui-agent-config' +import type { TuiAgent } from '../../../../shared/types' +import { buildDispatchPreamble } from '../../orchestration/preamble' +import { OrchestrationError } from '../../orchestration/orchestration-error' +import { defineMethod, type RpcMethod } from '../core' +import { startFederatedWorker } from './orchestration-federated-worker-start' +import { assertOrchestrationWorktreeCreationSupported } from './orchestration-folder-worktree-placement' +import { WorkerStartParams } from './orchestration-worker-start-schema' +import { + createWorkerWorktree, + monitorWorkerSetup, + type WorkerEffect, + type WorkerSetupReceipt +} from './orchestration-worker-topology' +import { + persistGatedSetupSpawnFailure, + persistWorkerReadinessStage, + persistWorkerSetupWaitOutcome +} from './orchestration-worker-setup-gate' +import { failWorkerStartWithReceipt } from './orchestration-worker-start-receipt' + +export const ORCHESTRATION_WORKER_START_METHODS: RpcMethod[] = [ + defineMethod({ + name: 'orchestration.workerStart', + params: WorkerStartParams, + handler: async (params, { runtime, orchestrationMutation }) => { + const db = runtime.getOrchestrationDb() + const coordinatorPane = runtime.getTerminalPaneKey(params.from) + const run = coordinatorPane ? db.getCurrentRunForPane(coordinatorPane) : undefined + if (!run || (params.run && params.run !== run.id)) { + throw new OrchestrationError( + 'consumer_fenced', + 'worker-start requires the coordinator terminal currently bound to the Task Run.' + ) + } + const task = db.getTask(params.task) + if (!task || task.run_id !== run.id) { + throw new OrchestrationError( + 'task_not_found', + `Task ${params.task} was not found in Run ${run.id}.` + ) + } + + if (params.on) { + return startFederatedWorker({ + params, + runtime, + db, + runId: run.id, + task, + orchestrationMutation + }) + } + + const requestedWorktree = params.worktree ?? 'current' + const createsWorktree = + requestedWorktree === 'new-child' || requestedWorktree === 'new-top-level' + if (params.terminal && params.agent) { + throw new OrchestrationError( + 'invalid_argument', + '--terminal reuses an existing agent and cannot combine with --agent.' + ) + } + if (createsWorktree && params.terminal) { + throw new OrchestrationError( + 'invalid_argument', + '--terminal cannot combine with new-worktree creation.' + ) + } + if (createsWorktree && !params.name) { + throw new OrchestrationError('invalid_argument', 'New worktrees require --name.') + } + if (!createsWorktree && (params.name || params.repo || params.baseBranch || params.setup)) { + throw new OrchestrationError( + 'invalid_argument', + 'Creation and setup options apply only to new-child or new-top-level worktrees.' + ) + } + const agent = params.agent + if (!params.terminal && (!agent || !isTuiAgent(agent))) { + throw new OrchestrationError( + 'agent_unconfigured', + 'A configured --agent is required when worker-start creates a terminal.' + ) + } + if (agent) { + runtime.validateOrchestrationAgentLauncher(agent as TuiAgent) + } + + const coordinatorTerminal = await runtime.showTerminal(params.from) + const coordinatorWorktree = await runtime.showManagedWorktree( + `id:${coordinatorTerminal.worktreeId}` + ) + if (createsWorktree) { + await assertOrchestrationWorktreeCreationSupported({ + runtime, + repoSelector: params.repo ?? coordinatorWorktree.repoId, + existingPlacement: 'current or an exact existing folder workspace' + }) + } + let resolvedWorktree = createsWorktree + ? undefined + : requestedWorktree === 'current' + ? coordinatorWorktree + : await runtime.showManagedWorktree(requestedWorktree) + let explicitTerminal + if (params.terminal) { + explicitTerminal = await runtime.showTerminal(params.terminal) + if (explicitTerminal.worktreeId !== resolvedWorktree?.id) { + throw new OrchestrationError( + 'terminal_worktree_mismatch', + `Terminal ${params.terminal} does not belong to worktree ${resolvedWorktree?.id}.` + ) + } + if (!(await runtime.isTerminalRunningAgent(params.terminal))) { + throw new OrchestrationError( + 'agent_unconfigured', + `Terminal ${params.terminal} is not running a recognized agent.` + ) + } + } + + const startOptions = { + worktree: requestedWorktree, + resolvedWorktreeId: resolvedWorktree?.id ?? null, + name: params.name ?? null, + repo: params.repo ?? (createsWorktree ? coordinatorWorktree.repoId : null), + baseBranch: params.baseBranch ?? null, + terminal: params.terminal ?? null, + agent: agent ?? null, + timeoutMs: params.timeoutMs ?? 60_000, + setup: createsWorktree ? (params.setup ?? 'run') : 'not_applicable', + setupSource: createsWorktree + ? params.setup + ? 'explicit_request' + : 'orchestration_default' + : 'existing_worktree' + } + const started = db.createStartingWorkerDispatch({ + taskId: task.id, + retryOf: params.retryOf, + startOptions, + runtimeEpoch: runtime.getRuntimeId(), + mutationReceipt: orchestrationMutation + }) + const effects: WorkerEffect[] = [] + if (resolvedWorktree) { + effects.push( + { kind: 'worktree', action: 'reused', id: resolvedWorktree.id }, + { kind: 'setup', action: 'not_applicable', state: 'not_applicable' } + ) + } + let terminalHandle = params.terminal + let failedStage = 'terminal_create' + let setupReceipt: WorkerSetupReceipt = { + requested: 'not_applicable', + effective: 'not_applicable', + source: 'existing_worktree', + hookFound: false, + startupPolicy: 'start-immediately', + state: 'not_applicable' + } + try { + if (createsWorktree) { + failedStage = 'worktree_create' + const created = await createWorkerWorktree({ + runtime, + db, + dispatchId: started.dispatch.id, + requestedWorktree, + coordinatorWorktree, + params, + agent: agent as TuiAgent, + effects + }) + resolvedWorktree = created.worktree + terminalHandle = created.terminalHandle + setupReceipt = created.setupReceipt + } else if (!terminalHandle) { + db.recordWorkerStage({ + dispatchId: started.dispatch.id, + stage: 'terminal_creating', + worktreeId: resolvedWorktree!.id, + effects + }) + const terminal = await runtime.createTerminal(`id:${resolvedWorktree!.id}`, { + command: agent, + title: `worker-${task.id}`, + presentation: 'background' + }) + terminalHandle = terminal.handle + effects.push({ + kind: 'terminal', + role: 'agent', + action: 'created', + id: terminal.handle + }) + } else { + effects.push({ + kind: 'terminal', + role: 'agent', + action: 'reused', + id: terminalHandle + }) + } + if (!resolvedWorktree || !terminalHandle) { + throw new Error('Worker topology did not resolve an agent terminal and worktree.') + } + const setupStage = { + db, + dispatchId: started.dispatch.id, + worktreeId: resolvedWorktree.id, + terminalHandle, + setup: setupReceipt, + effects + } + if (persistGatedSetupSpawnFailure(setupStage)) { + failedStage = 'setup_start' + throw new Error('Setup terminal failed to start before the gated agent launch.') + } + persistWorkerReadinessStage(setupStage) + + failedStage = 'agent_readiness' + const wait = await runtime.waitForTerminal(terminalHandle, { + condition: 'tui-idle', + timeoutMs: params.timeoutMs ?? 60_000 + }) + persistWorkerSetupWaitOutcome({ ...setupStage, wait }) + if (!wait.satisfied) { + if (setupReceipt.state === 'failed') { + failedStage = 'setup_wait' + } + throw new Error( + wait.blockedReason + ? `Agent startup blocked: ${wait.blockedReason}` + : `Agent did not become ready (${wait.status}).` + ) + } + const paneKey = runtime.getTerminalPaneKey(terminalHandle) + const processIncarnation = runtime.getTerminalProcessIncarnation(terminalHandle) + if (!paneKey || !processIncarnation) { + throw new Error('stable_pane_required') + } + const capability = db.prepareStartingWorkerAuthority({ + dispatchId: started.dispatch.id, + handle: terminalHandle, + paneKey, + processIncarnation, + worktreeId: resolvedWorktree.id, + effects, + setupState: setupReceipt.state + }) + + failedStage = 'dispatch_input' + const preamble = buildDispatchPreamble({ + taskId: task.id, + dispatchId: started.dispatch.id, + taskSpec: task.spec, + coordinatorHandle: params.from, + workerHandle: terminalHandle, + dispatchCapability: capability, + devMode: params.devMode, + cliCommand: runtime.getTerminalOrchestrationCliCommand(terminalHandle) + }) + await runtime.sendTerminalAgentPrompt(terminalHandle, preamble) + effects.push({ + kind: 'dispatch_input', + role: 'agent', + id: terminalHandle, + state: 'accepted' + }) + const worker = db.markWorkerDispatchReady(started.dispatch.id, effects) + monitorWorkerSetup({ + runtime, + db, + runId: run.id, + dispatchId: started.dispatch.id, + setupReceipt, + effects + }) + return { + runId: run.id, + taskId: task.id, + dispatchId: started.dispatch.id, + state: worker.state, + stage: worker.stage, + setup: setupReceipt, + timeoutMs: params.timeoutMs ?? 60_000, + effects, + residualResources: [] + } + } catch (error) { + return failWorkerStartWithReceipt({ + db, + runId: run.id, + taskId: task.id, + dispatchId: started.dispatch.id, + failedStage, + error, + setup: setupReceipt + }) + } + } + }) +] diff --git a/src/main/runtime/rpc/methods/orchestration.test.ts b/src/main/runtime/rpc/methods/orchestration.test.ts index 780cb2efd0a..899a94fa190 100644 --- a/src/main/runtime/rpc/methods/orchestration.test.ts +++ b/src/main/runtime/rpc/methods/orchestration.test.ts @@ -4,12 +4,14 @@ import { ORCHESTRATION_METHODS } from './orchestration' import { RpcDispatcher } from '../dispatcher' import { buildRegistry, type RpcContext, type RpcRequest } from '../core' import { OrchestrationDb } from '../../orchestration/db' +import { reconcileLifecycleMessage } from '../../orchestration/lifecycle-reconciliation' import { OrcaRuntimeService } from '../../orca-runtime' import type { RuntimeTerminalSummary } from '../../../../shared/runtime-types' import { ORCHESTRATION_ASK_MAX_TIMEOUT_MS } from '../../../../shared/orchestration-ask-timeout' +import { ORCHESTRATION_CONTRACT_VERSION } from '../../../../shared/protocol-version' function lifecycleGroupRecipientError(type: 'worker_done' | 'heartbeat'): string { - return `${type} messages must be sent to a concrete coordinator terminal handle, not a group address.` + return `${type} messages belong to one exact Dispatch and cannot target a group address.` } describe('orchestration RPC methods', () => { @@ -17,12 +19,33 @@ describe('orchestration RPC methods', () => { let dbOpen = false let runtime: OrcaRuntimeService let ctx: RpcContext + let activeRunId: string | undefined - function setup(): void { + const coordinatorPaneKey = 'tab_coord:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' + + function setup(withBoundRun = true): void { db = new OrchestrationDb(':memory:') dbOpen = true runtime = new OrcaRuntimeService() runtime.setOrchestrationDb(db) + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === 'term_coord' ? coordinatorPaneKey : null + ) + vi.spyOn(runtime, 'getTerminalProcessIncarnation').mockImplementation((handle) => + handle.startsWith('term_') ? `runtime_test:${handle}:1` : null + ) + if (withBoundRun) { + activeRunId = db.createRun({ + objective: 'Test Run', + coordinatorHandle: 'term_coord', + coordinatorPaneKey + }).id + const createTask = db.createTask.bind(db) + // Why: legacy tests exercise the RPC behavior under test; default their direct fixture rows to the bound Run. + db.createTask = (task) => createTask({ ...task, runId: task.runId ?? activeRunId }) + } else { + activeRunId = undefined + } ctx = { runtime } } @@ -47,17 +70,40 @@ describe('orchestration RPC methods', () => { async function call(name: string, params: Record) { const method = findMethod(name) - const parsed = method.params ? method.params.parse(params) : undefined + const scopedParams = { ...params } + if (activeRunId) { + if (name === 'orchestration.taskCreate' || name === 'orchestration.taskUpdate') { + scopedParams.run ??= activeRunId + scopedParams.callerTerminalHandle ??= 'term_coord' + } else if (name === 'orchestration.taskList') { + scopedParams.run ??= activeRunId + } else if (name === 'orchestration.dispatch') { + scopedParams.run ??= activeRunId + scopedParams.from ??= 'term_coord' + } + } + const parsed = method.params ? method.params.parse(scopedParams) : undefined return method.handler(parsed, ctx) } function makeRequest(method: string, params: Record): RpcRequest { - return { id: 'req_1', authToken: 'token', method, params } + return { + id: 'req_1', + authToken: 'token', + method, + params, + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION + } } it('registers all expected methods', () => { const registry = buildRegistry(ORCHESTRATION_METHODS) - expect(registry.size).toBe(16) + expect(registry.size).toBe(34) + expect(registry.has('orchestration.runCreate')).toBe(true) + expect(registry.has('orchestration.runUse')).toBe(true) + expect(registry.has('orchestration.runCurrent')).toBe(true) + expect(registry.has('orchestration.runList')).toBe(true) + expect(registry.has('orchestration.runShow')).toBe(true) expect(registry.has('orchestration.send')).toBe(true) expect(registry.has('orchestration.check')).toBe(true) expect(registry.has('orchestration.reply')).toBe(true) @@ -67,6 +113,19 @@ describe('orchestration RPC methods', () => { expect(registry.has('orchestration.taskUpdate')).toBe(true) expect(registry.has('orchestration.dispatch')).toBe(true) expect(registry.has('orchestration.dispatchShow')).toBe(true) + expect(registry.has('orchestration.workerStart')).toBe(true) + expect(registry.has('orchestration.workerShow')).toBe(true) + expect(registry.has('orchestration.workerRead')).toBe(true) + expect(registry.has('orchestration.workerStop')).toBe(true) + expect(registry.has('orchestration.workerAbandon')).toBe(true) + expect(registry.has('orchestration.federationAttachStart')).toBe(true) + expect(registry.has('orchestration.federationPull')).toBe(true) + expect(registry.has('orchestration.federationAck')).toBe(true) + expect(registry.has('orchestration.federationImport')).toBe(true) + expect(registry.has('orchestration.federationShow')).toBe(true) + expect(registry.has('orchestration.federationRead')).toBe(true) + expect(registry.has('orchestration.federationReadOutput')).toBe(true) + expect(registry.has('orchestration.federationStop')).toBe(true) expect(registry.has('orchestration.ask')).toBe(true) expect(registry.has('orchestration.run')).toBe(true) expect(registry.has('orchestration.runStop')).toBe(true) @@ -76,23 +135,207 @@ describe('orchestration RPC methods', () => { expect(registry.has('orchestration.reset')).toBe(true) }) + describe('lightweight Runs', () => { + it('creates and binds a Run to the runtime-resolved caller pane', async () => { + setup(false) + vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue( + 'tab_coord:11111111-1111-4111-8111-111111111111' + ) + + const created = (await call('orchestration.runCreate', { + objective: 'Coordinate reviews', + from: 'term_coord' + })) as { run: { id: string; consumer_generation: number } } + const current = (await call('orchestration.runCurrent', { from: 'term_coord' })) as { + run: { id: string } | null + } + + expect(created.run.consumer_generation).toBe(1) + expect(current.run?.id).toBe(created.run.id) + }) + + it('requires runtime-observed stable pane identity for binding', async () => { + setup(false) + vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue(null) + + await expect( + call('orchestration.runCreate', { objective: 'No pane', from: 'term_stale' }) + ).rejects.toMatchObject({ code: 'stable_pane_required' }) + expect(db.listRuns().filter((run) => run.legacy === 0)).toHaveLength(0) + }) + + it('rebinds explicitly, lists Runs, and keeps the legacy Run inspect-only', async () => { + setup(false) + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === 'term_old' + ? 'tab_old:11111111-1111-4111-8111-111111111111' + : 'tab_new:22222222-2222-4222-9222-222222222222' + ) + const created = (await call('orchestration.runCreate', { + objective: 'Move me', + from: 'term_old' + })) as { run: { id: string } } + const rebound = (await call('orchestration.runUse', { + id: created.run.id, + from: 'term_new' + })) as { run: { consumer_generation: number } } + const listed = (await call('orchestration.runList', {})) as { + runs: { id: string; legacy: number }[] + } + + expect(rebound.run.consumer_generation).toBe(2) + expect(listed.runs).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: created.run.id, legacy: 0 }), + expect.objectContaining({ id: 'run_legacy_local', legacy: 1 }) + ]) + ) + await expect( + call('orchestration.runUse', { id: 'run_legacy_local', from: 'term_new' }) + ).rejects.toMatchObject({ code: 'run_not_found' }) + }) + + it('requires an explicit binding before task mutation', async () => { + setup(false) + vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue(coordinatorPaneKey) + + await expect( + call('orchestration.taskCreate', { + spec: 'must not become global', + callerTerminalHandle: 'term_coord' + }) + ).rejects.toMatchObject({ + code: 'run_required', + data: { + effectsApplied: false, + nextCommandArgs: ['skills', 'get', 'orchestration', '--full'] + } + }) + expect(db.listTasks()).toHaveLength(0) + }) + + it('scopes task listing and fences the old coordinator after run-use', async () => { + setup(false) + const oldPane = 'tab_old:11111111-1111-4111-8111-111111111111' + const newPane = 'tab_new:22222222-2222-4222-9222-222222222222' + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === 'term_old' ? oldPane : newPane + ) + const runA = db.createRun({ + objective: 'A', + coordinatorHandle: 'term_old', + coordinatorPaneKey: oldPane + }) + const runB = db.createRun({ + objective: 'B', + coordinatorHandle: 'term_other', + coordinatorPaneKey: newPane + }) + const taskA = db.createTask({ spec: 'A work', runId: runA.id }) + db.createTask({ spec: 'B work', runId: runB.id }) + + const listed = (await call('orchestration.taskList', { run: runA.id })) as { + tasks: { id: string }[] + } + expect(listed.tasks.map((task) => task.id)).toEqual([taskA.id]) + + db.bindRun({ + runId: runA.id, + coordinatorHandle: 'term_new', + coordinatorPaneKey: newPane + }) + await expect( + call('orchestration.taskCreate', { + spec: 'stale write', + run: runA.id, + callerTerminalHandle: 'term_old' + }) + ).rejects.toMatchObject({ code: 'consumer_fenced' }) + }) + + it('cancels and fences the old Run waiter when run-use rebinds', async () => { + setup(false) + const oldPane = 'tab_old:11111111-1111-4111-8111-111111111111' + const newPane = 'tab_new:22222222-2222-4222-9222-222222222222' + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === 'term_old' ? oldPane : newPane + ) + const created = (await call('orchestration.runCreate', { + objective: 'Wait fencing', + from: 'term_old' + })) as { run: { id: string } } + const oldWait = call('orchestration.check', { + terminal: 'term_old', + wait: true, + timeoutMs: 5_000 + }) + const fenced = expect(oldWait).rejects.toMatchObject({ code: 'consumer_fenced' }) + await Promise.resolve() + + await call('orchestration.runUse', { + id: created.run.id, + from: 'term_new' + }) + + await fenced + }) + }) + describe('orchestration.send', () => { it('sends a message', async () => { setup() vi.spyOn(runtime, 'deliverPendingMessagesForHandle').mockImplementation(() => {}) const result = (await call('orchestration.send', { - from: 'term_a', - to: 'term_b', + from: 'term_coord', + to: `run:${activeRunId}`, subject: 'hello' - })) as { message: { id: string; from_handle: string } } + })) as { message: { id: string; from_handle: string; run_id: string } } expect(result.message.id).toMatch(/^msg_/) - expect(result.message.from_handle).toBe('term_a') - expect(runtime.deliverPendingMessagesForHandle).toHaveBeenCalledWith('term_b') + expect(result.message.from_handle).toBe('term_coord') + expect(result.message.run_id).toBe(activeRunId) + expect(runtime.deliverPendingMessagesForHandle).not.toHaveBeenCalled() }) - it('stores the sender pane key on the message row', async () => { + it('routes exact Dispatch mail independently of terminal handles', async () => { setup() + const task = db.createTask({ spec: 'controlled worker' }) + const dispatch = db.createDispatchContext(task.id, 'term_worker') + + const result = (await call('orchestration.send', { + from: 'term_coord', + to: `dispatch:${dispatch.id}`, + subject: 'Pause after this step' + })) as { message: { to_handle: string; run_id: string } } + + expect(result.message).toMatchObject({ + to_handle: `dispatch:${dispatch.id}`, + run_id: activeRunId + }) + + const workerCheck = (await call('orchestration.check', { + terminal: 'term_worker' + })) as { dispatchId: string; messages: { subject: string }[] } + expect(workerCheck).toMatchObject({ + dispatchId: dispatch.id, + messages: [{ subject: 'Pause after this step' }] + }) + }) + + it('rejects hidden task-recipient retargeting', async () => { + setup() + await expect( + call('orchestration.send', { + from: 'term_coord', + to: 'task:task_1', + subject: 'ambiguous' + }) + ).rejects.toMatchObject({ code: 'invalid_argument' }) + }) + + it('stores the runtime-observed sender pane key on the message row', async () => { + setup() + vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue('tab_runtime:leaf_runtime') vi.spyOn(runtime, 'deliverPendingMessagesForHandle').mockImplementation(() => {}) vi.spyOn(runtime, 'notifyMessageArrived').mockImplementation(() => {}) @@ -103,7 +346,7 @@ describe('orchestration RPC methods', () => { senderPaneKey: 'tab_a:leaf_a' })) as { message: { id: string } } - expect(db.getMessageById(result.message.id)?.sender_pane_key).toBe('tab_a:leaf_a') + expect(db.getMessageById(result.message.id)?.sender_pane_key).toBe('tab_runtime:leaf_runtime') }) it('recovers missing sender pane identity from the resolved handle', async () => { @@ -135,7 +378,11 @@ describe('orchestration RPC methods', () => { to: 'term_coord', subject: 'Done', type: 'worker_done', - payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id }) + payload: JSON.stringify({ + taskId: task.id, + dispatchId: dispatch.id, + outcome: 'succeeded' + }) }) expect(db.getTask(task.id)?.status).toBe('completed') @@ -157,14 +404,18 @@ describe('orchestration RPC methods', () => { to: 'term_coord', subject: 'Done', type: 'worker_done', - payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id }) + payload: JSON.stringify({ + taskId: task.id, + dispatchId: dispatch.id, + outcome: 'succeeded' + }) }) expect(db.getTask(task.id)?.status).toBe('dispatched') expect(db.getTask(dependent.id)?.status).toBe('pending') }) - it('does not replace a foreign sender pane with its claimed assignee handle pane', async () => { + it('ignores caller-supplied pane claims and uses the runtime-observed pane', async () => { setup() const task = db.createTask({ spec: 'work' }) const dispatch = db.createDispatchContext(task.id, 'term_worker', 'tab_worker:leaf_worker') @@ -178,26 +429,108 @@ describe('orchestration RPC methods', () => { subject: 'Done', type: 'worker_done', senderPaneKey: 'tab_foreign:leaf_foreign', - payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id }) + payload: JSON.stringify({ + taskId: task.id, + dispatchId: dispatch.id, + outcome: 'succeeded' + }) })) as { message: { id: string; type: string; subject: string } lifecycle: { action: string; code: string; reason: string } } - expect(db.getTask(task.id)?.status).toBe('dispatched') - expect(result.lifecycle).toMatchObject({ - action: 'rejected', - code: 'sender_not_assignee', - reason: expect.stringContaining('expected handle term_worker') - }) + expect(db.getTask(task.id)?.status).toBe('completed') + expect(result.lifecycle).toBeUndefined() expect(result.message).toMatchObject({ type: 'worker_done', - subject: 'Rejected worker_done: Done' + subject: 'Done' }) - expect(db.getUnreadMessages('term_coord')).toEqual([ + expect(db.getUnreadMessages(`run:${activeRunId}`)).toEqual([ expect.objectContaining({ id: result.message.id, type: 'worker_done' }) ]) - expect(runtime.notifyMessageArrived).toHaveBeenCalledWith('term_coord', 'worker_done') + expect(runtime.notifyMessageArrived).toHaveBeenCalledWith(`run:${activeRunId}`, 'worker_done') + }) + + it('requires the minted capability, exact pane, and process incarnation', async () => { + setup() + const task = db.createTask({ spec: 'capability work' }) + const dispatch = db.createDispatchContext(task.id, 'term_worker', 'tab_worker:leaf_worker') + const capability = db.mintDispatchCapability({ + dispatchId: dispatch.id, + paneKey: 'tab_worker:leaf_worker', + processIncarnation: 'runtime_test:term_worker:1' + }) + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === 'term_worker' ? 'tab_worker:leaf_worker' : coordinatorPaneKey + ) + const payload = JSON.stringify({ + taskId: task.id, + dispatchId: dispatch.id, + outcome: 'succeeded' + }) + + const rejected = (await call('orchestration.send', { + from: 'term_worker', + subject: 'Done', + type: 'worker_done', + payload + })) as { lifecycle: { code: string }; message: { subject: string } } + expect(rejected).toMatchObject({ + lifecycle: { code: 'dispatch_capability_invalid' }, + message: { subject: 'Rejected worker_done: Done' } + }) + expect(db.getTask(task.id)?.status).toBe('dispatched') + + ctx = { runtime, orchestrationCapability: 'dcap_wrong' } + const wrongToken = (await call('orchestration.send', { + from: 'term_worker', + subject: 'Done', + type: 'worker_done', + payload + })) as { lifecycle: { code: string } } + expect(wrongToken.lifecycle.code).toBe('dispatch_capability_invalid') + + ctx = { runtime, orchestrationCapability: capability } + vi.mocked(runtime.getTerminalPaneKey).mockImplementation((handle) => + handle === 'term_worker' ? 'tab_foreign:leaf_foreign' : coordinatorPaneKey + ) + const wrongPane = (await call('orchestration.send', { + from: 'term_worker', + subject: 'Done', + type: 'worker_done', + payload + })) as { lifecycle: { code: string } } + expect(wrongPane.lifecycle.code).toBe('dispatch_capability_invalid') + + vi.mocked(runtime.getTerminalPaneKey).mockImplementation((handle) => + handle === 'term_worker' ? 'tab_worker:leaf_worker' : coordinatorPaneKey + ) + vi.mocked(runtime.getTerminalProcessIncarnation).mockReturnValue('runtime_test:term_worker:2') + const wrongProcess = (await call('orchestration.send', { + from: 'term_worker', + subject: 'Done', + type: 'worker_done', + payload + })) as { lifecycle: { code: string } } + expect(wrongProcess.lifecycle.code).toBe('dispatch_capability_invalid') + + vi.mocked(runtime.getTerminalProcessIncarnation).mockReturnValue('runtime_test:term_worker:1') + await call('orchestration.send', { + from: 'term_worker', + subject: 'Done', + type: 'worker_done', + payload + }) + expect(db.getTask(task.id)?.status).toBe('completed') + expect(db.getDispatchContextById(dispatch.id)?.capability_revoked_at).toBeTruthy() + + const revoked = (await call('orchestration.send', { + from: 'term_worker', + subject: 'Done again', + type: 'worker_done', + payload + })) as { lifecycle: { code: string } } + expect(revoked.lifecycle.code).toBe('dispatch_capability_invalid') }) it('does not wake waiters for a heartbeat suppressed at send time', async () => { @@ -235,12 +568,12 @@ describe('orchestration RPC methods', () => { payload: JSON.stringify({ dispatchId: dispatch.id }) }) - expect(notify).toHaveBeenCalledWith('term_coord', 'heartbeat') + expect(notify).toHaveBeenCalledWith(`run:${activeRunId}`, 'heartbeat') }) - it('rejects missing --to', () => { + it('allows an omitted recipient so an active Dispatch can default to its Run', () => { const method = findMethod('orchestration.send') - expect(() => method.params!.parse({ subject: 'hi' })).toThrow() + expect(method.params!.parse({ subject: 'hi' })).toMatchObject({ subject: 'hi' }) }) it('rejects missing --subject', () => { @@ -403,18 +736,26 @@ describe('orchestration RPC methods', () => { it('continues to send worker_done to a concrete terminal handle', async () => { setup() + const task = db.createTask({ spec: 'work' }) + const dispatch = db.createDispatchContext(task.id, 'term_worker') const result = (await call('orchestration.send', { from: 'term_worker', to: 'term_coord', subject: 'done', type: 'worker_done', - payload: JSON.stringify({ taskId: 'task_1', dispatchId: 'ctx_1' }) + payload: JSON.stringify({ + taskId: task.id, + dispatchId: dispatch.id, + outcome: 'succeeded' + }) })) as { message: { to_handle: string; type: string; payload: string | null } } - expect(result.message.to_handle).toBe('term_coord') + expect(result.message.to_handle).toBe(`run:${activeRunId}`) expect(result.message.type).toBe('worker_done') - expect(result.message.payload).toBe(JSON.stringify({ taskId: 'task_1', dispatchId: 'ctx_1' })) + expect(result.message.payload).toBe( + JSON.stringify({ taskId: task.id, dispatchId: dispatch.id, outcome: 'succeeded' }) + ) }) it('fans out @idle to only idle agents', async () => { @@ -545,8 +886,8 @@ describe('orchestration RPC methods', () => { const task = db.createTask({ spec: 'lock-release work' }) const dispatch = db.createDispatchContext(task.id, 'term_worker') - // Why: assert lock is already gone at delivery time, not just after the call. - vi.spyOn(runtime, 'deliverPendingMessagesForHandle').mockImplementation(() => { + // Why: waiter notification must observe the settled Dispatch, not stale lifecycle state. + vi.spyOn(runtime, 'notifyMessageArrived').mockImplementation(() => { expect(db.getActiveDispatchForTerminal('term_worker')).toBeUndefined() }) @@ -555,7 +896,11 @@ describe('orchestration RPC methods', () => { to: 'term_coord', subject: 'done', type: 'worker_done', - payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id }) + payload: JSON.stringify({ + taskId: task.id, + dispatchId: dispatch.id, + outcome: 'succeeded' + }) })) as { message: { type: string } } expect(result.message.type).toBe('worker_done') @@ -628,18 +973,21 @@ describe('orchestration RPC methods', () => { if (params.dispatchId !== undefined) { payload.dispatchId = params.dispatchId } + payload.outcome = 'succeeded' if (params.filesModified !== undefined) { payload.filesModified = params.filesModified } - db.insertMessage({ + const message = db.insertMessage({ from: params.from ?? 'term_worker', - to: params.to ?? 'term_coord', + to: params.to ?? `run:${activeRunId}`, subject: 'Done', type: 'worker_done', payload: JSON.stringify(payload), - senderPaneKey: params.senderPaneKey + senderPaneKey: params.senderPaneKey, + runId: activeRunId }) + reconcileLifecycleMessage(db, message) } it('returns unread messages for a terminal', async () => { @@ -655,13 +1003,53 @@ describe('orchestration RPC methods', () => { expect(result.count).toBe(2) }) - it('returns formatted output with --inject', async () => { + it('never mixes two bound Run mailboxes', async () => { + setup(false) + const paneA = 'tab_a:11111111-1111-4111-8111-111111111111' + const paneB = 'tab_b:22222222-2222-4222-9222-222222222222' + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === 'term_a' ? paneA : paneB + ) + const runA = db.createRun({ + objective: 'A', + coordinatorHandle: 'term_a', + coordinatorPaneKey: paneA + }) + const runB = db.createRun({ + objective: 'B', + coordinatorHandle: 'term_b', + coordinatorPaneKey: paneB + }) + db.insertMessage({ + from: 'worker_a', + to: `run:${runA.id}`, + subject: 'A only', + runId: runA.id + }) + db.insertMessage({ + from: 'worker_b', + to: `run:${runB.id}`, + subject: 'B only', + runId: runB.id + }) + + const inboxA = (await call('orchestration.check', { terminal: 'term_a' })) as { + messages: { subject: string }[] + } + const inboxB = (await call('orchestration.check', { terminal: 'term_b' })) as { + messages: { subject: string }[] + } + expect(inboxA.messages.map((message) => message.subject)).toEqual(['A only']) + expect(inboxB.messages.map((message) => message.subject)).toEqual(['B only']) + }) + + it('returns formatted output with --format', async () => { setup() db.insertMessage({ from: 'a', to: 'b', subject: 'test' }) const result = (await call('orchestration.check', { terminal: 'b', - inject: true + format: true })) as { formatted: string; count: number } expect(result.formatted).toContain('Subject: test') @@ -681,6 +1069,68 @@ describe('orchestration RPC methods', () => { expect(result.count).toBe(1) }) + it('returns typed timeout and rejects a second actionable waiter', async () => { + setup() + vi.spyOn(runtime, 'waitForMessage').mockResolvedValueOnce('timed_out') + + const timedOut = (await call('orchestration.check', { + terminal: 'term_coord', + wait: true, + timeoutMs: 10 + })) as { timedOut: boolean; cancelled: boolean; count: number } + expect(timedOut).toMatchObject({ timedOut: true, cancelled: false, count: 0 }) + + vi.mocked(runtime.waitForMessage).mockResolvedValueOnce('waiter_exists') + await expect( + call('orchestration.check', { + terminal: 'term_coord', + wait: true, + timeoutMs: 10 + }) + ).rejects.toMatchObject({ code: 'waiter_exists' }) + }) + + it('rejects stale Delivery acknowledgment without consuming queued mail', async () => { + setup() + db.insertMessage({ + from: 'worker', + to: `run:${activeRunId}`, + subject: 'queued', + runId: activeRunId + }) + + await expect( + call('orchestration.check', { + terminal: 'term_coord', + ack: 'delivery_missing' + }) + ).rejects.toMatchObject({ code: 'stale_delivery' }) + expect(db.getUnreadMessages(`run:${activeRunId}`)).toHaveLength(1) + }) + + it('acknowledges a Run Delivery before returning --peek history', async () => { + setup() + db.insertMessage({ + from: 'worker', + to: `run:${activeRunId}`, + subject: 'queued', + runId: activeRunId + }) + + const first = (await call('orchestration.check', { + terminal: 'term_coord' + })) as { count: number; deliveryId: string } + const peeked = (await call('orchestration.check', { + terminal: 'term_coord', + ack: first.deliveryId, + peek: true + })) as { acknowledged: string | null; count: number } + + expect(first.count).toBe(1) + expect(peeked).toMatchObject({ acknowledged: first.deliveryId, count: 0 }) + expect(db.getUnreadMessages(`run:${activeRunId}`)).toHaveLength(0) + }) + it('reconciles worker_done returned by a waiting manual check', async () => { setup() const { task, dispatch } = createDispatchedTask() @@ -690,6 +1140,7 @@ describe('orchestration RPC methods', () => { dispatchId: dispatch.id, filesModified: ['src/file.ts'] }) + return 'notified' }) const result = (await call('orchestration.check', { @@ -697,13 +1148,13 @@ describe('orchestration RPC methods', () => { wait: true, timeoutMs: 100, types: 'worker_done,escalation,decision_gate' - })) as { count: number; messages: { type: string }[] } + })) as { count: number; messages: { type: string }[]; deliveryId: string } expect(result.count).toBe(1) expect(result.messages[0].type).toBe('worker_done') expect(db.getTask(task.id)?.status).toBe('completed') expect(db.getDispatchContextById(dispatch.id)?.status).toBe('completed') - expect(db.getUnreadMessages('term_coord')).toHaveLength(0) + expect(db.getUnreadMessages(`run:${activeRunId}`)).toHaveLength(1) const taskList = (await call('orchestration.taskList', {})) as { tasks: { id: string @@ -726,13 +1177,20 @@ describe('orchestration RPC methods', () => { const repeated = (await call('orchestration.check', { terminal: 'term_coord', types: 'worker_done' + })) as { count: number; deliveryId: string } + expect(repeated.count).toBe(1) + expect(repeated.deliveryId).toBe(result.deliveryId) + const acknowledged = (await call('orchestration.check', { + terminal: 'term_coord', + ack: repeated.deliveryId, + types: 'worker_done' })) as { count: number } - expect(repeated.count).toBe(0) + expect(acknowledged.count).toBe(0) expect(db.getTask(task.id)?.completed_at).toBe(completedAt) expect(db.getTask(task.id)?.result).toBe(taskResult) }) - it('keeps check --all read-only for lifecycle messages', async () => { + it('keeps check --all read-only while lifecycle settles at acceptance', async () => { setup() const { task, dispatch } = createDispatchedTask() insertWorkerDone({ taskId: task.id, dispatchId: dispatch.id }) @@ -744,9 +1202,9 @@ describe('orchestration RPC methods', () => { })) as { count: number } expect(result.count).toBe(1) - expect(db.getTask(task.id)?.status).toBe('dispatched') - expect(db.getDispatchContextById(dispatch.id)?.status).toBe('dispatched') - expect(db.getUnreadMessages('term_coord', ['worker_done'])).toHaveLength(1) + expect(db.getTask(task.id)?.status).toBe('completed') + expect(db.getDispatchContextById(dispatch.id)?.status).toBe('completed') + expect(db.getUnreadMessages(`run:${activeRunId}`, ['worker_done'])).toHaveLength(1) }) it('does not complete worker_done missing taskId or dispatchId', async () => { @@ -842,11 +1300,13 @@ describe('orchestration RPC methods', () => { const { task, dispatch } = createDispatchedTask() const msg = db.insertMessage({ from: 'term_worker', - to: 'term_coord', + to: `run:${activeRunId}`, subject: 'alive', type: 'heartbeat', - payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id }) + payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id }), + runId: activeRunId }) + reconcileLifecycleMessage(db, msg) const result = (await call('orchestration.check', { terminal: 'term_coord', @@ -999,6 +1459,7 @@ describe('orchestration RPC methods', () => { vi.spyOn(runtime, 'waitForMessage').mockImplementation(async () => { db.insertMessage({ from: 'a', to: 'b', subject: 'arrived during close' }) abortController.abort() + return 'cancelled' }) const result = (await call('orchestration.check', { @@ -1087,6 +1548,49 @@ describe('orchestration RPC methods', () => { 'Message not found' ) }) + + it('records one idempotent answer from the current Run consumer', async () => { + setup() + const task = db.createTask({ spec: 'question work' }) + const dispatch = db.createDispatchContext(task.id, 'term_worker') + const created = db.createQuestion({ + runId: activeRunId!, + dispatchId: dispatch.id, + askerHandle: 'term_worker', + question: 'Proceed?' + }) + const notify = vi.spyOn(runtime, 'notifyMessageArrived').mockImplementation(() => {}) + + const first = (await call('orchestration.reply', { + id: created.message.id, + body: 'Yes', + from: 'term_coord' + })) as { message: { id: string; to_handle: string }; duplicate: boolean } + const repeated = (await call('orchestration.reply', { + id: created.message.id, + body: 'Yes', + from: 'term_coord' + })) as { message: { id: string }; duplicate: boolean } + + expect(first.message.to_handle).toBe(`dispatch:${dispatch.id}`) + expect(first.duplicate).toBe(false) + expect(repeated).toMatchObject({ + message: { id: first.message.id }, + duplicate: true + }) + expect(notify).toHaveBeenCalledWith(`dispatch:${dispatch.id}`, 'status') + expect(db.getQuestion(created.message.id)).toMatchObject({ + status: 'answered', + answer_body: 'Yes' + }) + await expect( + call('orchestration.reply', { + id: created.message.id, + body: 'No', + from: 'term_coord' + }) + ).rejects.toMatchObject({ code: 'answer_conflict' }) + }) }) describe('orchestration.inbox', () => { @@ -1161,6 +1665,9 @@ describe('orchestration RPC methods', () => { it('records the caller terminal handle when creating a task', async () => { setup() + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === 'term_creator' ? coordinatorPaneKey : null + ) const result = (await call('orchestration.taskCreate', { spec: 'spawn related workspace', callerTerminalHandle: 'term_creator' @@ -1282,11 +1789,17 @@ describe('orchestration RPC methods', () => { setup() await expect( call('orchestration.taskUpdate', { id: 'task_fake', status: 'completed' }) - ).rejects.toThrow('Task not found') + ).rejects.toThrow('was not found') }) }) describe('orchestration.dispatch', () => { + function provideInjectIdentity(handle = 'term_a'): void { + vi.mocked(runtime.getTerminalPaneKey).mockImplementation((candidate) => + candidate === handle ? `tab_worker:${handle}` : coordinatorPaneKey + ) + } + it('dispatches a task to a terminal', async () => { setup() const task = db.createTask({ spec: 'work' }) @@ -1302,7 +1815,9 @@ describe('orchestration RPC methods', () => { it('records the assignee pane key on the dispatch context', async () => { setup() - vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue('tab_w:leaf_w') + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === 'term_a' ? 'tab_w:leaf_w' : coordinatorPaneKey + ) const task = db.createTask({ spec: 'work' }) const result = (await call('orchestration.dispatch', { @@ -1329,6 +1844,7 @@ describe('orchestration RPC methods', () => { it('rolls back active dispatch when injection fails', async () => { setup() + provideInjectIdentity() const task = db.createTask({ spec: 'work' }) vi.spyOn(runtime, 'isTerminalRunningAgent').mockResolvedValue(true) vi.spyOn(runtime, 'sendTerminalAgentPrompt').mockRejectedValue( @@ -1349,6 +1865,7 @@ describe('orchestration RPC methods', () => { it('uses caller-provided dev mode for injected preamble', async () => { setup() + provideInjectIdentity() const task = db.createTask({ spec: 'work' }) vi.spyOn(runtime, 'isTerminalRunningAgent').mockResolvedValue(true) const send = vi.spyOn(runtime, 'sendTerminalAgentPrompt').mockResolvedValue({ @@ -1388,6 +1905,7 @@ describe('orchestration RPC methods', () => { it('injects preamble through the agent prompt path instead of raw terminal send', async () => { setup() + provideInjectIdentity() const task = db.createTask({ spec: 'line one\nline two' }) vi.spyOn(runtime, 'isTerminalRunningAgent').mockResolvedValue(true) const agentPrompt = vi.spyOn(runtime, 'sendTerminalAgentPrompt').mockResolvedValue({ @@ -1481,6 +1999,303 @@ describe('orchestration RPC methods', () => { }) }) + describe('composed workers', () => { + function mockCurrentWorkerStart(options?: { ready?: boolean }): void { + vi.mocked(runtime.getTerminalPaneKey).mockImplementation((handle) => + handle === 'term_coord' + ? coordinatorPaneKey + : handle === 'term_worker' + ? 'tab_worker:leaf_worker' + : null + ) + vi.spyOn(runtime, 'validateOrchestrationAgentLauncher').mockImplementation(() => {}) + vi.spyOn(runtime, 'showTerminal').mockImplementation( + async (handle) => ({ handle, worktreeId: 'repo::worktree', status: 'running' }) as never + ) + vi.spyOn(runtime, 'showManagedWorktree').mockResolvedValue({ + id: 'repo::worktree' + } as never) + vi.spyOn(runtime, 'createTerminal').mockResolvedValue({ + handle: 'term_worker', + worktreeId: 'repo::worktree', + title: 'worker' + }) + vi.spyOn(runtime, 'waitForTerminal').mockResolvedValue({ + handle: 'term_worker', + condition: 'tui-idle', + satisfied: options?.ready !== false, + status: 'running', + exitCode: null + }) + vi.mocked(runtime.getTerminalProcessIncarnation).mockImplementation((handle) => + handle === 'term_worker' ? 'runtime_test:term_worker:1' : null + ) + vi.spyOn(runtime, 'getTerminalOrchestrationCliCommand').mockReturnValue('orca') + vi.spyOn(runtime, 'sendTerminalAgentPrompt').mockResolvedValue({ + handle: 'term_worker', + accepted: true, + bytesWritten: 1 + }) + } + + it('starts a fresh agent in the coordinator current worktree', async () => { + setup() + mockCurrentWorkerStart() + const task = db.createTask({ spec: 'implement worker start' }) + + const result = (await call('orchestration.workerStart', { + task: task.id, + from: 'term_coord', + agent: 'codex' + })) as { + dispatchId: string + state: string + effects: { kind: string; role?: string; action?: string; state?: string }[] + } + + expect(result.state).toBe('ready') + expect(result.effects).toEqual( + expect.arrayContaining([ + expect.objectContaining({ kind: 'worktree', action: 'reused' }), + expect.objectContaining({ kind: 'terminal', role: 'agent', action: 'created' }), + expect.objectContaining({ kind: 'dispatch_input', state: 'accepted' }) + ]) + ) + expect(db.getTask(task.id)?.status).toBe('dispatched') + expect(db.getWorkerDispatch(result.dispatchId)?.state).toBe('ready') + expect(runtime.sendTerminalAgentPrompt).toHaveBeenCalledWith( + 'term_worker', + expect.stringContaining('--dispatch-capability dcap_') + ) + }) + + it('starts a fresh agent in an exact existing worktree without replaying setup', async () => { + setup() + mockCurrentWorkerStart() + const createWorktree = vi.spyOn(runtime, 'createManagedWorktree') + vi.mocked(runtime.showManagedWorktree).mockImplementation( + async (selector) => + ({ + id: selector === 'id:repo::other' ? 'repo::other' : 'repo::worktree', + repoId: 'repo' + }) as never + ) + const task = db.createTask({ spec: 'existing worktree worker' }) + + const result = (await call('orchestration.workerStart', { + task: task.id, + from: 'term_coord', + worktree: 'id:repo::other', + agent: 'codex' + })) as { state: string; setup: { state: string }; effects: unknown[] } + + expect(result).toMatchObject({ state: 'ready' }) + expect(result.effects).toEqual( + expect.arrayContaining([ + expect.objectContaining({ kind: 'worktree', action: 'reused', id: 'repo::other' }), + expect.objectContaining({ kind: 'setup', action: 'not_applicable' }) + ]) + ) + expect(runtime.createTerminal).toHaveBeenCalledWith( + 'id:repo::other', + expect.objectContaining({ command: 'codex' }) + ) + expect(createWorktree).not.toHaveBeenCalled() + }) + + it('reuses only an explicitly selected existing agent terminal', async () => { + setup() + mockCurrentWorkerStart() + const createWorktree = vi.spyOn(runtime, 'createManagedWorktree') + vi.spyOn(runtime, 'isTerminalRunningAgent').mockResolvedValue(true) + const task = db.createTask({ spec: 'reuse exact worker' }) + + const result = (await call('orchestration.workerStart', { + task: task.id, + from: 'term_coord', + terminal: 'term_worker' + })) as { state: string; effects: unknown[] } + + expect(result).toMatchObject({ state: 'ready' }) + expect(result.effects).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + kind: 'terminal', + role: 'agent', + action: 'reused', + id: 'term_worker' + }) + ]) + ) + expect(runtime.createTerminal).not.toHaveBeenCalled() + expect(createWorktree).not.toHaveBeenCalled() + }) + + it('returns a failed receipt and preserves a created terminal as residual', async () => { + setup() + mockCurrentWorkerStart({ ready: false }) + const task = db.createTask({ spec: 'worker timeout' }) + + const result = (await call('orchestration.workerStart', { + task: task.id, + from: 'term_coord', + agent: 'codex' + })) as { state: string; failedStage: string; residualResources: { id: string }[] } + + expect(result).toMatchObject({ state: 'failed', failedStage: 'agent_readiness' }) + expect(result.residualResources).toEqual([expect.objectContaining({ id: 'term_worker' })]) + expect(db.getTask(task.id)?.status).toBe('failed') + expect(runtime.sendTerminalAgentPrompt).not.toHaveBeenCalled() + }) + + it('returns a no-effect failure when terminal creation fails', async () => { + setup() + mockCurrentWorkerStart() + vi.mocked(runtime.createTerminal).mockRejectedValueOnce(new Error('terminal spawn rejected')) + const task = db.createTask({ spec: 'terminal failure' }) + + const result = (await call('orchestration.workerStart', { + task: task.id, + from: 'term_coord', + agent: 'codex' + })) as { state: string; failedStage: string; residualResources: unknown[] } + + expect(result).toMatchObject({ + state: 'failed', + failedStage: 'terminal_create', + residualResources: [] + }) + expect(runtime.sendTerminalAgentPrompt).not.toHaveBeenCalled() + }) + + it('preserves the exact attached terminal when task input is rejected', async () => { + setup() + mockCurrentWorkerStart() + vi.mocked(runtime.sendTerminalAgentPrompt).mockRejectedValueOnce( + new Error('agent input rejected') + ) + const task = db.createTask({ spec: 'input failure' }) + + const result = (await call('orchestration.workerStart', { + task: task.id, + from: 'term_coord', + agent: 'codex' + })) as { + state: string + failedStage: string + residualResources: { kind: string; id: string }[] + } + + expect(result).toMatchObject({ state: 'failed', failedStage: 'dispatch_input' }) + expect(result.residualResources).toEqual( + expect.arrayContaining([expect.objectContaining({ kind: 'terminal', id: 'term_worker' })]) + ) + }) + + it.each(['codex-update-prompt', 'codex-trust-workspace'] as const)( + 'returns a truthful readiness failure for %s', + async (blockedReason) => { + setup() + mockCurrentWorkerStart() + vi.mocked(runtime.waitForTerminal).mockResolvedValueOnce({ + handle: 'term_worker', + condition: 'tui-idle', + satisfied: false, + status: 'running', + exitCode: null, + blockedReason + }) + const task = db.createTask({ spec: 'blocked startup prompt' }) + + const result = (await call('orchestration.workerStart', { + task: task.id, + from: 'term_coord', + agent: 'codex' + })) as { state: string; failedStage: string; lastError: string } + + expect(result).toMatchObject({ + state: 'failed', + failedStage: 'agent_readiness', + lastError: `Agent startup blocked: ${blockedReason}` + }) + expect(runtime.sendTerminalAgentPrompt).not.toHaveBeenCalled() + } + ) + + it('creates a child worktree agent-first with setup run by default', async () => { + setup() + mockCurrentWorkerStart() + vi.mocked(runtime.showManagedWorktree).mockResolvedValue({ + id: 'repo::parent', + repoId: 'repo' + } as never) + vi.spyOn(runtime, 'showRepo').mockResolvedValue({ + id: 'repo', + kind: 'git' + } as never) + const create = vi.spyOn(runtime, 'createManagedWorktree').mockResolvedValue({ + worktree: { id: 'repo::child', repoId: 'repo' }, + startupTerminal: { spawned: true, handle: 'term_worker' }, + setupReceipt: { + requested: 'run', + hookFound: true, + startupPolicy: 'start-immediately', + state: 'running', + terminalHandle: 'term_setup' + } + } as never) + vi.spyOn(runtime, 'listTerminals').mockResolvedValue({ + terminals: [ + { handle: 'term_worker', title: 'Codex' }, + { handle: 'term_setup', title: 'Setup' }, + { handle: 'term_logs', title: 'Logs' } + ], + totalCount: 3, + truncated: false + } as never) + const task = db.createTask({ spec: 'child worker' }) + + const result = (await call('orchestration.workerStart', { + task: task.id, + from: 'term_coord', + worktree: 'new-child', + name: 'child-worker', + agent: 'codex' + })) as { + state: string + setup: { requested: string; startupPolicy: string; state: string } + effects: { role?: string; action?: string }[] + } + + expect(result).toMatchObject({ + state: 'ready', + setup: { + requested: 'run', + startupPolicy: 'start-immediately', + state: 'running' + } + }) + expect(create).toHaveBeenCalledWith( + expect.objectContaining({ + repoSelector: 'repo', + name: 'child-worker', + runHooks: true, + setupDecision: 'run', + startupAgent: 'codex', + lineage: expect.objectContaining({ parentWorktree: 'repo::parent', noParent: false }) + }) + ) + expect(result.effects).toEqual( + expect.arrayContaining([ + expect.objectContaining({ role: 'agent', action: 'reused_agent_terminal' }), + expect.objectContaining({ role: 'setup', action: 'created' }), + expect.objectContaining({ role: 'configured_tab', action: 'created' }) + ]) + ) + expect(runtime.createTerminal).not.toHaveBeenCalled() + }) + }) + describe('orchestration.dispatchShow', () => { it('shows dispatch context for a task', async () => { setup() @@ -1643,33 +2458,38 @@ describe('orchestration RPC methods', () => { }) describe('orchestration.ask', () => { - it('sends a decision_gate and returns the first thread reply', async () => { + function createAskingDispatch(handle = 'term_worker') { + const task = db.createTask({ spec: 'question work' }) + const dispatch = db.createDispatchContext(task.id, handle) + return { task, dispatch } + } + + it('persists a Run question and returns its first durable answer', async () => { setup() - vi.spyOn(runtime, 'deliverPendingMessagesForHandle').mockImplementation(() => {}) + const { dispatch } = createAskingDispatch() vi.spyOn(runtime, 'notifyMessageArrived').mockImplementation(() => {}) vi.spyOn(runtime, 'waitForMessage').mockImplementation(async () => { - // Simulate coordinator replying in the thread during the wait - const outbound = db.getInbox(10).find((m) => m.type === 'decision_gate') + const outbound = db.getInbox(10).find((message) => message.type === 'question') if (outbound) { - db.insertMessage({ - from: 'term_coord', - to: 'term_worker', - subject: 'Re: Question', - body: 'go ahead', - threadId: outbound.id + db.answerQuestion({ + messageId: outbound.id, + runId: activeRunId!, + consumerGeneration: db.getRun(activeRunId!)!.consumer_generation, + body: 'go ahead' }) } + return 'notified' }) const result = (await call('orchestration.ask', { from: 'term_worker', - to: 'term_coord', question: 'proceed?', options: 'yes, no', timeoutMs: 500 })) as { answer: string messageId: string + answerMessageId: string threadId: string timedOut: boolean } @@ -1678,45 +2498,119 @@ describe('orchestration RPC methods', () => { expect(result.answer).toBe('go ahead') expect(result.messageId).toMatch(/^msg_/) - // Outbound decision_gate message was persisted with parsed options. - const outbound = db.getInbox(10).find((m) => m.type === 'decision_gate') + const outbound = db.getInbox(10).find((message) => message.type === 'question') expect(outbound).toBeTruthy() + expect(outbound?.to_handle).toBe(`run:${activeRunId}`) expect(outbound?.subject).toBe('Question') expect(outbound?.body).toBe('proceed?') const payload = JSON.parse(outbound!.payload ?? '{}') expect(payload.question).toBe('proceed?') expect(payload.options).toEqual(['yes', 'no']) + expect(db.getQuestion(outbound!.id)).toMatchObject({ + dispatch_id: dispatch.id, + status: 'answered', + answer_body: 'go ahead' + }) + expect(db.getMessageById(result.answerMessageId)).toMatchObject({ + to_handle: `dispatch:${dispatch.id}`, + read: 1 + }) + await expect(call('orchestration.check', { terminal: 'term_worker' })).resolves.toMatchObject( + { count: 0, messages: [] } + ) + }) + + it('requires the Dispatch capability before creating a question', async () => { + setup() + const { dispatch } = createAskingDispatch() + const capability = db.mintDispatchCapability({ + dispatchId: dispatch.id, + paneKey: 'tab_worker:leaf_worker', + processIncarnation: 'runtime_test:term_worker:1' + }) + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === 'term_worker' ? 'tab_worker:leaf_worker' : coordinatorPaneKey + ) + + await expect( + call('orchestration.ask', { + from: 'term_worker', + question: 'unauthorized', + timeoutMs: 1 + }) + ).rejects.toMatchObject({ code: 'dispatch_capability_invalid' }) + expect(db.getInbox(100).filter((message) => message.type === 'question')).toHaveLength(0) + + ctx = { runtime, orchestrationCapability: capability } + vi.spyOn(runtime, 'waitForMessage').mockResolvedValue('timed_out') + const accepted = (await call('orchestration.ask', { + from: 'term_worker', + question: 'authorized', + timeoutMs: 1 + })) as { messageId: string; timedOut: boolean } + expect(accepted.messageId).toMatch(/^msg_/) + expect(accepted.timedOut).toBe(true) }) it('returns timedOut when no reply arrives in the window', async () => { setup() - vi.spyOn(runtime, 'deliverPendingMessagesForHandle').mockImplementation(() => {}) + createAskingDispatch() vi.spyOn(runtime, 'notifyMessageArrived').mockImplementation(() => {}) - vi.spyOn(runtime, 'waitForMessage').mockResolvedValue() + vi.spyOn(runtime, 'waitForMessage').mockResolvedValue('timed_out') const result = (await call('orchestration.ask', { from: 'term_worker', - to: 'term_coord', question: 'still there?', timeoutMs: 1 })) as { answer: string | null; timedOut: boolean; messageId: string | null } expect(result.timedOut).toBe(true) expect(result.answer).toBeNull() - expect(result.messageId).toBeNull() - // Outbound message still persisted (coordinator can still see it). - const outbound = db.getInbox(10).find((m) => m.type === 'decision_gate') + expect(result.messageId).toMatch(/^msg_/) + const outbound = db.getInbox(10).find((message) => message.type === 'question') expect(outbound).toBeTruthy() + expect(db.getQuestion(outbound!.id)?.status).toBe('pending') + }) + + it('resumes the original question without creating a duplicate', async () => { + setup() + const { dispatch } = createAskingDispatch() + const created = db.createQuestion({ + runId: activeRunId!, + dispatchId: dispatch.id, + askerHandle: 'term_worker', + question: 'Resume me' + }) + db.answerQuestion({ + messageId: created.message.id, + runId: activeRunId!, + consumerGeneration: db.getRun(activeRunId!)!.consumer_generation, + body: 'recorded answer' + }) + const messageCount = db.getInbox(100).length + + const result = (await call('orchestration.ask', { + from: 'term_worker', + resume: created.message.id, + timeoutMs: 500 + })) as { answer: string; messageId: string; timedOut: boolean } + + expect(result).toMatchObject({ + answer: 'recorded answer', + messageId: created.message.id, + timedOut: false + }) + expect(db.getInbox(100)).toHaveLength(messageCount) }) it('returns promptly when the RPC signal aborts while waiting', async () => { setup() + createAskingDispatch() vi.useFakeTimers() const controller = new AbortController() const method = findMethod('orchestration.ask') const parsed = method.params!.parse({ from: 'term_worker', - to: 'term_coord', question: 'still there?', timeoutMs: 60_000 }) @@ -1725,17 +2619,17 @@ describe('orchestration RPC methods', () => { const promise = method.handler(parsed, { runtime, signal: controller.signal - }) as Promise<{ timedOut: boolean }> + }) as Promise<{ timedOut: boolean; cancelled: boolean }> controller.abort() const outcomePromise = Promise.race([ - promise.then((result) => (result.timedOut ? 'aborted' : 'answered')), + promise.then((result) => (result.cancelled ? 'cancelled' : 'answered')), new Promise<'pending'>((resolve) => setTimeout(() => resolve('pending'), 0)) ]) await vi.advanceTimersByTimeAsync(0) const outcome = await outcomePromise - expect(outcome).toBe('aborted') + expect(outcome).toBe('cancelled') } finally { vi.useRealTimers() } @@ -1753,38 +2647,35 @@ describe('orchestration RPC methods', () => { expect(db.getInbox(10)).toHaveLength(0) }) - it('does not return distractor messages on a different thread', async () => { + it('ignores unrelated wakes until the durable question is answered', async () => { setup() - vi.spyOn(runtime, 'deliverPendingMessagesForHandle').mockImplementation(() => {}) + createAskingDispatch() vi.spyOn(runtime, 'notifyMessageArrived').mockImplementation(() => {}) let wakeCount = 0 vi.spyOn(runtime, 'waitForMessage').mockImplementation(async () => { wakeCount++ - const outbound = db.getInbox(20).find((m) => m.type === 'decision_gate') + const outbound = db.getInbox(20).find((message) => message.type === 'question') if (wakeCount === 1 && outbound) { - // First wake: distractor in a DIFFERENT thread — must be ignored. db.insertMessage({ - from: 'term_coord', - to: 'term_worker', + from: 'unrelated', + to: `dispatch:${db.getQuestion(outbound.id)!.dispatch_id}`, subject: 'unrelated', body: 'other', - threadId: 'thread_other' + runId: activeRunId }) } else if (wakeCount === 2 && outbound) { - // Second wake: correct thread reply. - db.insertMessage({ - from: 'term_coord', - to: 'term_worker', - subject: 'Re: Question', - body: 'correct answer', - threadId: outbound.id + db.answerQuestion({ + messageId: outbound.id, + runId: activeRunId!, + consumerGeneration: db.getRun(activeRunId!)!.consumer_generation, + body: 'correct answer' }) } + return 'notified' }) const result = (await call('orchestration.ask', { from: 'term_worker', - to: 'term_coord', question: 'filter?', timeoutMs: 2_000 })) as { answer: string; timedOut: boolean } @@ -1799,20 +2690,22 @@ describe('orchestration RPC methods', () => { [Number.MAX_SAFE_INTEGER, ORCHESTRATION_ASK_MAX_TIMEOUT_MS] ])('applies effective timeout %s at the RPC handler boundary', async (requested, expected) => { setup() - vi.spyOn(runtime, 'deliverPendingMessagesForHandle').mockImplementation(() => {}) + createAskingDispatch() vi.spyOn(runtime, 'notifyMessageArrived').mockImplementation(() => {}) let observedTimeoutMs: number | undefined vi.spyOn(runtime, 'waitForMessage').mockImplementation(async (_handle, options) => { observedTimeoutMs = options?.timeoutMs - const outbound = db.getInbox(10).find((m) => m.type === 'decision_gate') + // End the wait loop so the assertion runs against the first budget slice. + const outbound = db.getInbox(10).find((message) => message.type === 'question') + // Why: without a reply the handler's while(true) spins on this mock until vitest times out, hanging instead of failing. expect(outbound).toBeDefined() - db.insertMessage({ - from: 'term_coord', - to: 'term_worker', - subject: 'Re: Question', - body: 'ok', - threadId: outbound!.id + db.answerQuestion({ + messageId: outbound!.id, + runId: activeRunId!, + consumerGeneration: db.getRun(activeRunId!)!.consumer_generation, + body: 'ok' }) + return 'notified' }) const result = (await call('orchestration.ask', { @@ -1829,6 +2722,7 @@ describe('orchestration RPC methods', () => { it('returns a zero effective timeout without entering the waiter', async () => { setup() + createAskingDispatch() const waitForMessage = vi.spyOn(runtime, 'waitForMessage') const result = (await call('orchestration.ask', { @@ -1844,19 +2738,18 @@ describe('orchestration RPC methods', () => { it('parses options CSV with whitespace and empty entries', async () => { setup() - vi.spyOn(runtime, 'deliverPendingMessagesForHandle').mockImplementation(() => {}) + createAskingDispatch('w') vi.spyOn(runtime, 'notifyMessageArrived').mockImplementation(() => {}) - vi.spyOn(runtime, 'waitForMessage').mockResolvedValue() + vi.spyOn(runtime, 'waitForMessage').mockResolvedValue('timed_out') await call('orchestration.ask', { from: 'w', - to: 'c', question: 'q', options: 'a, b ,,c', timeoutMs: 1 }) - const outbound = db.getInbox(10).find((m) => m.type === 'decision_gate') + const outbound = db.getInbox(10).find((message) => message.type === 'question') const payload = JSON.parse(outbound!.payload ?? '{}') expect(payload.options).toEqual(['a', 'b', 'c']) }) diff --git a/src/main/runtime/rpc/methods/orchestration.ts b/src/main/runtime/rpc/methods/orchestration.ts index ddf86825e6f..128520f4c33 100644 --- a/src/main/runtime/rpc/methods/orchestration.ts +++ b/src/main/runtime/rpc/methods/orchestration.ts @@ -3,24 +3,23 @@ import { z } from 'zod' import { defineMethod, type RpcMethod } from '../core' import { OptionalFiniteNumber, OptionalString, OptionalBoolean, requiredString } from '../schemas' import type { MessageType, MessagePriority, TaskStatus } from '../../orchestration/db' +import { MESSAGE_TYPES } from '../../orchestration/types' import { buildDispatchPreamble } from '../../orchestration/preamble' import { formatMessageBanner } from '../../orchestration/formatter' import { isGroupAddress, resolveGroupAddress } from '../../orchestration/groups' import { reconcileLifecycleMessage } from '../../orchestration/lifecycle-reconciliation' import { abbreviateOrchestrationTasks } from '../../../../shared/orchestration-task-summary' +import { orchestrationSkillRecoveryData } from '../../../../shared/orchestration-rpc-contract' import { clampOrchestrationAskTimeoutMs } from '../../../../shared/orchestration-ask-timeout' import { ORCHESTRATION_GATE_METHODS } from './orchestration-gates' - -const MESSAGE_TYPES: MessageType[] = [ - 'status', - 'dispatch', - 'worker_done', - 'merge_ready', - 'escalation', - 'handoff', - 'decision_gate', - 'heartbeat' -] +import { ORCHESTRATION_RUN_METHODS } from './orchestration-runs' +import { ORCHESTRATION_WORKER_METHODS } from './orchestration-worker-methods' +import { ORCHESTRATION_FEDERATION_METHODS } from './orchestration-federation-methods' +import { OrchestrationError } from '../../orchestration/orchestration-error' +import type { OrcaRuntimeService } from '../../orca-runtime' +import type { RunRow } from '../../orchestration/types' +import { encodeFederatedControlMessage } from '../../orchestration/federation-control-message' +import { ORCHESTRATION_FEDERATION_CONTROL_MAIL_PROTOCOL_VERSION } from '../../../../shared/protocol-version' const TASK_STATUSES: TaskStatus[] = [ 'pending', @@ -32,12 +31,26 @@ const TASK_STATUSES: TaskStatus[] = [ ] function getLifecycleGroupRecipientError(type: 'worker_done' | 'heartbeat'): string { - return `${type} messages must be sent to a concrete coordinator terminal handle, not a group address.` + return `${type} messages belong to one exact Dispatch and cannot target a group address.` +} + +function parseRemoteWorkerPayload(payload: string | undefined): Record { + if (!payload) { + return {} + } + try { + const parsed: unknown = JSON.parse(payload) + return parsed && typeof parsed === 'object' && !Array.isArray(parsed) + ? (parsed as Record) + : {} + } catch { + throw new OrchestrationError('invalid_argument', 'Message payload must be valid JSON.') + } } const SendParams = z .object({ - to: requiredString('Missing --to'), + to: OptionalString, subject: requiredString('Missing --subject'), from: OptionalString, body: OptionalString, @@ -50,6 +63,7 @@ const SendParams = z 'escalation', 'handoff', 'decision_gate', + 'question', 'heartbeat' ]) .optional(), @@ -58,11 +72,13 @@ const SendParams = z payload: OptionalString, // Why: pane key is the remint-stable identity used to verify worker_done/heartbeat ownership; the from handle stays routing metadata. senderPaneKey: OptionalString, + run: OptionalString, devMode: OptionalBoolean }) .superRefine((params, ctx) => { if ( (params.type !== 'worker_done' && params.type !== 'heartbeat') || + !params.to || !isGroupAddress(params.to) ) { return @@ -83,7 +99,11 @@ const CheckParams = z // Why: `all` surfaces every message and skips mark-read; legacy encoding was the `{unread: false}` trick (design doc §3.2/§3.3). all: OptionalBoolean, types: OptionalString, + format: OptionalBoolean, + // Why: one-release RPC compatibility only; the public CLI uses --format because no terminal input is injected. inject: OptionalBoolean, + ack: OptionalString, + run: OptionalString, wait: OptionalBoolean, timeoutMs: OptionalFiniteNumber }) @@ -105,7 +125,8 @@ const CheckParams = z const ReplyParams = z.object({ id: requiredString('Missing --id'), body: requiredString('Missing --body'), - from: OptionalString + from: OptionalString, + run: OptionalString }) const InboxParams = z.object({ @@ -120,14 +141,17 @@ const TaskCreateParams = z.object({ displayName: OptionalString, deps: OptionalString, parent: OptionalString, - callerTerminalHandle: OptionalString + callerTerminalHandle: OptionalString, + run: OptionalString }) const TaskListParams = z.object({ status: z.enum(['pending', 'ready', 'dispatched', 'completed', 'failed', 'blocked']).optional(), ready: OptionalBoolean, // Why: server-side truncation keeps --brief cheap over SSH/relay instead of shipping full specs the CLI throws away. - brief: OptionalBoolean + brief: OptionalBoolean, + run: OptionalString, + callerTerminalHandle: OptionalString }) const TaskUpdateParams = z.object({ @@ -145,7 +169,9 @@ const TaskUpdateParams = z.object({ message: 'Missing --status' }) ), - result: OptionalString + result: OptionalString, + run: OptionalString, + callerTerminalHandle: OptionalString }) const DispatchParams = z.object({ @@ -156,7 +182,8 @@ const DispatchParams = z.object({ inject: OptionalBoolean, dryRun: OptionalBoolean, returnPreamble: OptionalBoolean, - devMode: OptionalBoolean + devMode: OptionalBoolean, + run: OptionalString }) const DispatchShowParams = z.object({ @@ -166,13 +193,24 @@ const DispatchShowParams = z.object({ devMode: OptionalBoolean }) -const AskParams = z.object({ - to: requiredString('Missing --to'), - question: requiredString('Missing --question'), - options: OptionalString, - timeoutMs: OptionalFiniteNumber, - from: OptionalString -}) +const AskParams = z + .object({ + to: OptionalString, + question: OptionalString, + resume: OptionalString, + options: OptionalString, + timeoutMs: OptionalFiniteNumber, + from: OptionalString, + run: OptionalString + }) + .superRefine((params, ctx) => { + if ((params.question ? 1 : 0) + (params.resume ? 1 : 0) !== 1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Choose exactly one of --question or --resume.' + }) + } + }) const ResetParams = z .object({ @@ -192,29 +230,351 @@ const ResetParams = z } }) +function resolveRunScope( + runtime: OrcaRuntimeService, + params: { + runId?: string + callerTerminalHandle?: string + requireCurrentConsumer: boolean + } +): RunRow { + const db = runtime.getOrchestrationDb() + const explicit = params.runId ? db.getRun(params.runId) : undefined + if (params.runId && (!explicit || explicit.legacy === 1)) { + throw new OrchestrationError('run_not_found', `Run ${params.runId} was not found.`) + } + + if (!params.requireCurrentConsumer && explicit) { + return explicit + } + if (!params.callerTerminalHandle) { + throw new OrchestrationError( + 'run_required', + 'No Run is bound. Use orchestration run-create or run-use first. No effects were applied.', + orchestrationSkillRecoveryData() + ) + } + const paneKey = runtime.getTerminalPaneKey(params.callerTerminalHandle) + if (!paneKey) { + throw new OrchestrationError( + 'stable_pane_required', + 'The coordinator terminal has no stable pane identity.' + ) + } + const current = db.getCurrentRunForPane(paneKey) + if (!current) { + if (explicit) { + throw new OrchestrationError( + 'consumer_fenced', + `This coordinator terminal is no longer bound to Run ${explicit.id}.` + ) + } + throw new OrchestrationError( + 'run_required', + 'No Run is bound. Use orchestration run-create or run-use first. No effects were applied.', + orchestrationSkillRecoveryData() + ) + } + if (explicit && current.id !== explicit.id) { + throw new OrchestrationError( + 'consumer_fenced', + `This coordinator terminal is bound to ${current.id}, not ${explicit.id}.` + ) + } + return current +} + +function parseMessageTypes(rawTypes: string | undefined): MessageType[] | undefined { + const types = rawTypes + ?.split(',') + .map((type) => type.trim()) + .filter(Boolean) as MessageType[] | undefined + const invalidTypes = types?.filter((type) => !MESSAGE_TYPES.includes(type)) + if (invalidTypes && invalidTypes.length > 0) { + throw new OrchestrationError('invalid_argument', `Invalid --types: ${invalidTypes.join(',')}`) + } + return types && types.length > 0 ? types : undefined +} + +function resolveMessageRun( + runtime: OrcaRuntimeService, + params: { + from?: string + senderPaneKey?: string + to?: string + runId?: string + payload?: string + } +): { run: RunRow | undefined; dispatchId: string | undefined } { + const db = runtime.getOrchestrationDb() + let dispatchId: string | undefined + if (params.payload) { + try { + const payload: unknown = JSON.parse(params.payload) + if ( + payload && + typeof payload === 'object' && + !Array.isArray(payload) && + typeof (payload as { dispatchId?: unknown }).dispatchId === 'string' + ) { + dispatchId = (payload as { dispatchId: string }).dispatchId + } + } catch { + // Lifecycle validation owns malformed payload errors; routing simply cannot derive a Dispatch. + } + } + if (!dispatchId && params.to?.startsWith('dispatch:')) { + dispatchId = params.to.slice('dispatch:'.length) + } + + const dispatch = dispatchId + ? db.getDispatchContextById(dispatchId) + : params.from + ? db.getActiveDispatchForIdentity(params.from, params.senderPaneKey) + : undefined + if (params.to?.startsWith('dispatch:') && !dispatch) { + throw new OrchestrationError( + 'dispatch_not_found', + `Dispatch ${dispatchId ?? ''} was not found.` + ) + } + const targetRunId = params.to?.startsWith('run:') ? params.to.slice('run:'.length) : undefined + const resolvedRunId = params.runId ?? targetRunId ?? dispatch?.run_id + let run = resolvedRunId ? db.getRun(resolvedRunId) : undefined + + if (!run && params.from) { + const paneKey = params.senderPaneKey ?? runtime.getTerminalPaneKey(params.from) + run = paneKey ? db.getCurrentRunForPane(paneKey) : undefined + } + if (resolvedRunId && (!run || run.legacy === 1)) { + throw new OrchestrationError('run_not_found', `Run ${resolvedRunId} was not found.`) + } + if (run && targetRunId && targetRunId !== run.id) { + throw new OrchestrationError('run_not_found', `Run ${targetRunId} was not found.`) + } + if (run && dispatch && dispatch.run_id !== run.id) { + throw new OrchestrationError( + 'dispatch_run_mismatch', + `Dispatch ${dispatch.id} belongs to Run ${dispatch.run_id}, not ${run.id}.` + ) + } + return { run, dispatchId: dispatch?.id ?? dispatchId } +} + +function rejectFederatedExplicitTarget(params: { to?: string; run?: string }): void { + if (params.to || params.run) { + throw new OrchestrationError( + 'invalid_argument', + 'Federated Dispatch messages route to their Run home; omit --to and --run.' + ) + } +} + export const ORCHESTRATION_METHODS: RpcMethod[] = [ + ...ORCHESTRATION_RUN_METHODS, + ...ORCHESTRATION_WORKER_METHODS, + ...ORCHESTRATION_FEDERATION_METHODS, defineMethod({ name: 'orchestration.send', params: SendParams, - handler: async (params, { runtime }) => { + handler: async (params, { runtime, orchestrationCapability }) => { const db = runtime.getOrchestrationDb() const from = params.from ?? 'unknown' - // Why: older shells may lack ORCA_PANE_KEY, but the runtime still knows the pane behind their handle; persist that authority. - const senderPaneKey = params.senderPaneKey ?? runtime.getTerminalPaneKey(from) ?? undefined + // Why: caller-supplied pane fields are only compatibility metadata; lifecycle authority uses the runtime-observed pane plus capability. + const senderPaneKey = runtime.getTerminalPaneKey(from) ?? undefined + const remoteAttachment = senderPaneKey + ? db.findActiveRemoteAttachmentForPane(senderPaneKey) + : undefined + if (remoteAttachment) { + rejectFederatedExplicitTarget(params) + const processIncarnation = runtime.getTerminalProcessIncarnation(from) + if ( + !db.verifyRemoteAttachmentAuthority({ + dispatchId: remoteAttachment.dispatch_id, + capability: orchestrationCapability, + paneKey: senderPaneKey ?? null, + processIncarnation + }) + ) { + throw new OrchestrationError( + 'dispatch_capability_invalid', + 'The remote Dispatch capability or exact worker process is invalid.' + ) + } + const type = (params.type ?? 'status') as MessageType + const payload = parseRemoteWorkerPayload(params.payload) + if ( + typeof payload.dispatchId === 'string' && + payload.dispatchId !== remoteAttachment.dispatch_id + ) { + throw new OrchestrationError( + 'dispatch_inactive', + `Dispatch ${payload.dispatchId} is not the active remote Dispatch for this pane.` + ) + } + const outcome = + type === 'worker_done' && + (payload.outcome === 'succeeded' || payload.outcome === 'failed') + ? payload.outcome + : undefined + if (type === 'worker_done' && !outcome) { + throw new OrchestrationError( + 'invalid_argument', + 'Remote worker_done requires outcome=succeeded|failed.' + ) + } + const relay = db.enqueueFederationRelay({ + dispatchId: remoteAttachment.dispatch_id, + direction: 'to_home', + kind: type, + payload: JSON.stringify({ + from, + subject: params.subject, + body: params.body ?? '', + type, + priority: params.priority ?? 'normal', + threadId: params.threadId ?? null, + payload: params.payload ?? null + }), + settleRemoteOutcome: outcome + }) + return { + relay: { + messageId: relay.message_id, + sequence: relay.sequence, + dispatchId: relay.dispatch_id, + destination: 'run_home', + accepted: true + }, + ...(outcome + ? { lifecycle: { action: outcome === 'succeeded' ? 'completed' : 'failed' } } + : {}) + } + } + const routing = resolveMessageRun(runtime, { + from, + senderPaneKey, + to: params.to, + runId: params.run, + payload: params.payload + }) + if (params.to?.startsWith('task:')) { + throw new OrchestrationError( + 'invalid_argument', + 'Task recipients are intentionally unsupported; use run: or dispatch:.' + ) + } + let to = params.to + if ( + routing.run && + (!to || + ((params.type === 'worker_done' || params.type === 'heartbeat') && routing.dispatchId)) + ) { + to = `run:${routing.run.id}` + } + if (!to) { + throw new OrchestrationError( + 'run_required', + 'No recipient or active Dispatch Run could be resolved. No effects were applied.', + orchestrationSkillRecoveryData() + ) + } - if (!isGroupAddress(params.to)) { + if (!isGroupAddress(to)) { + const federatedDispatchId = routing.dispatchId + const federatedTarget = + federatedDispatchId && to === `dispatch:${federatedDispatchId}` + ? db.getFederatedDispatch(federatedDispatchId) + : undefined + if (federatedTarget && federatedDispatchId) { + const dispatchId = federatedDispatchId + if ( + federatedTarget.protocol_version < + ORCHESTRATION_FEDERATION_CONTROL_MAIL_PROTOCOL_VERSION + ) { + throw new OrchestrationError( + 'capability_unsupported', + `Federated Dispatch ${dispatchId} does not support coordinator control mail; start a fresh worker after updating its Orca server.` + ) + } + if (db.getWorkerDispatch(dispatchId)?.state !== 'ready') { + throw new OrchestrationError( + 'dispatch_inactive', + `Federated Dispatch ${dispatchId} is not active.` + ) + } + if (params.type === 'worker_done' || params.type === 'heartbeat') { + throw new OrchestrationError( + 'invalid_argument', + 'Coordinator-to-worker control mail cannot report worker lifecycle.' + ) + } + const relay = db.enqueueFederationRelay({ + dispatchId, + direction: 'to_worker', + kind: 'control_message', + payload: encodeFederatedControlMessage({ + from, + subject: params.subject, + body: params.body ?? '', + type: (params.type ?? 'status') as MessageType, + priority: (params.priority ?? 'normal') as MessagePriority, + threadId: params.threadId ?? null, + payload: params.payload ?? null + }) + }) + runtime.ensureOrchestrationFederationRelay(routing.run?.id) + return { + relay: { + messageId: relay.message_id, + sequence: relay.sequence, + dispatchId: relay.dispatch_id, + destination: 'worker', + accepted: true + } + } + } // Point-to-point — existing single-recipient behavior const msg = db.insertMessage({ from, - to: params.to, + to, subject: params.subject, body: params.body, type: params.type as MessageType, priority: params.priority as MessagePriority, threadId: params.threadId, payload: params.payload, - senderPaneKey + senderPaneKey, + runId: routing.run?.id }) + const dispatch = routing.dispatchId + ? db.getDispatchContextById(routing.dispatchId) + : undefined + if ((msg.type === 'worker_done' || msg.type === 'heartbeat') && dispatch?.capability_hash) { + const authority = db.verifyDispatchCapability({ + dispatchId: dispatch.id, + capability: orchestrationCapability, + paneKey: senderPaneKey, + processIncarnation: runtime.getTerminalProcessIncarnation(from) ?? undefined + }) + if (!authority.valid) { + const rejection = + db.convertLifecycleMessageToRejection( + msg.id, + 'dispatch_capability_invalid', + authority.reason + ) ?? msg + runtime.notifyMessageArrived(to, rejection.type) + return { + message: rejection, + lifecycle: { + action: 'rejected', + code: 'dispatch_capability_invalid', + reason: authority.reason + } + } + } + } // Why: reconcile releases the dispatch lock before waking recipients, else a woken coordinator re-dispatches while the lock is still held. if (msg.type === 'worker_done' || msg.type === 'heartbeat') { const reconciled = reconcileLifecycleMessage(db, msg) @@ -224,24 +584,22 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ } if (reconciled.action === 'rejected') { const rejection = db.getMessageById(msg.id) ?? msg - runtime.deliverPendingMessagesForHandle(params.to) - runtime.notifyMessageArrived(params.to, rejection.type) + runtime.notifyMessageArrived(to, rejection.type) return { message: rejection, lifecycle: reconciled } } } - runtime.deliverPendingMessagesForHandle(params.to) - runtime.notifyMessageArrived(params.to, msg.type) + runtime.notifyMessageArrived(to, msg.type) return { message: msg } } // Why: fan out one message per recipient (independent read-tracking) but share a thread_id for correlation (Section 4.5). const { terminals } = await runtime.listTerminals() - const handles = resolveGroupAddress(params.to, from, terminals, (handle: string) => + const handles = resolveGroupAddress(to, from, terminals, (handle: string) => runtime.getAgentStatusForHandle(handle) ) if (handles.length === 0) { - throw new Error(`No recipients resolved for group address: ${params.to}`) + throw new Error(`No recipients resolved for group address: ${to}`) } const threadId = params.threadId ?? `thread_${Date.now()}` @@ -255,11 +613,11 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ priority: params.priority as MessagePriority, threadId, payload: params.payload, - senderPaneKey + senderPaneKey, + runId: routing.run?.id }) ) for (const message of messages) { - runtime.deliverPendingMessagesForHandle(message.to_handle) runtime.notifyMessageArrived(message.to_handle, message.type) } @@ -273,15 +631,213 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ handler: async (params, { runtime, signal }) => { const db = runtime.getOrchestrationDb() const handle = params.terminal ?? 'unknown' - const typeFilter = params.types - ? (params.types - .split(',') - .map((t) => t.trim()) - .filter(Boolean) as MessageType[]) - : undefined - const invalidTypes = typeFilter?.filter((t) => !MESSAGE_TYPES.includes(t)) - if (invalidTypes && invalidTypes.length > 0) { - throw new Error(`Invalid --types: ${invalidTypes.join(',')}`) + const typeFilter = parseMessageTypes(params.types) + + const paneKey = runtime.getTerminalPaneKey(handle) + const boundRun = paneKey ? db.getCurrentRunForPane(paneKey) : undefined + if (params.run || boundRun) { + const run = resolveRunScope(runtime, { + runId: params.run, + callerTerminalHandle: handle, + requireCurrentConsumer: true + }) + const generation = run.consumer_generation + const address = `run:${run.id}` + runtime.ensureOrchestrationFederationRelay(run.id) + + const acknowledged = params.ack + ? db.acknowledgeRunDelivery({ + runId: run.id, + consumerGeneration: generation, + deliveryId: params.ack + }) + : undefined + if (params.peek || params.all || params.unread === false) { + const history = db.getRunMailboxHistory(run.id, 100, typeFilter) + const messages = + params.all || (params.unread === false && !params.peek) + ? history + : history.filter((message) => message.read === 0) + const result = { + messages, + count: messages.length, + acknowledged: acknowledged?.delivery.id ?? null + } + if (params.format || params.inject) { + return { + ...result, + formatted: messages.map(formatMessageBanner).join('\n\n'), + runId: run.id + } + } + return { ...result, runId: run.id } + } + + const readDelivery = (wakeTypes?: MessageType[]) => + db.getOrCreateRunDelivery({ + runId: run.id, + consumerGeneration: generation, + wakeTypes + }) + let current = readDelivery(params.wait ? typeFilter : undefined) + if (current) { + return { + runId: run.id, + deliveryId: current.delivery.id, + messages: current.messages, + count: current.messages.length, + replayed: current.replayed, + acknowledged: acknowledged?.delivery.id ?? null, + timedOut: false, + cancelled: false, + connectionLost: false, + ...(params.format || params.inject + ? { formatted: current.messages.map(formatMessageBanner).join('\n\n') } + : {}) + } + } + if (!params.wait) { + return { + runId: run.id, + deliveryId: null, + messages: [], + count: 0, + acknowledged: acknowledged?.delivery.id ?? null, + timedOut: false, + cancelled: false, + connectionLost: false + } + } + + const waitResult = await runtime.waitForMessage(address, { + typeFilter: typeFilter as string[] | undefined, + timeoutMs: params.timeoutMs ?? undefined, + signal, + exclusive: true + }) + const latestRun = db.getRun(run.id) + if (!latestRun || latestRun.consumer_generation !== generation) { + throw new OrchestrationError( + 'consumer_fenced', + 'This mailbox consumer was replaced while waiting.' + ) + } + if (waitResult === 'waiter_exists') { + throw new OrchestrationError( + 'waiter_exists', + `Run ${run.id} already has an active actionable waiter.` + ) + } + if (waitResult === 'timed_out') { + return { + runId: run.id, + deliveryId: null, + messages: [], + count: 0, + acknowledged: acknowledged?.delivery.id ?? null, + timedOut: true, + cancelled: false, + connectionLost: false + } + } + if (waitResult === 'cancelled') { + return { + runId: run.id, + deliveryId: null, + messages: [], + count: 0, + acknowledged: acknowledged?.delivery.id ?? null, + timedOut: false, + cancelled: true, + connectionLost: signal?.aborted === true + } + } + + current = readDelivery(typeFilter) + return { + runId: run.id, + deliveryId: current?.delivery.id ?? null, + messages: current?.messages ?? [], + count: current?.messages.length ?? 0, + replayed: current?.replayed ?? false, + acknowledged: acknowledged?.delivery.id ?? null, + timedOut: false, + cancelled: false, + connectionLost: false, + ...(params.format && current + ? { formatted: current.messages.map(formatMessageBanner).join('\n\n') } + : {}) + } + } + + const activeDispatch = db.getActiveDispatchForIdentity(handle, paneKey ?? undefined) + const remoteAttachment = + !activeDispatch && paneKey ? db.findActiveRemoteAttachmentForPane(paneKey) : undefined + if ( + remoteAttachment && + !db.isRemoteAttachmentProcessCurrent({ + dispatchId: remoteAttachment.dispatch_id, + paneKey, + processIncarnation: runtime.getTerminalProcessIncarnation(handle) + }) + ) { + throw new OrchestrationError( + 'dispatch_inactive', + `Dispatch ${remoteAttachment.dispatch_id} is no longer attached to this worker process.` + ) + } + const workerMailbox = activeDispatch + ? { dispatchId: activeDispatch.id, runId: activeDispatch.run_id } + : remoteAttachment + ? { dispatchId: remoteAttachment.dispatch_id, runId: undefined } + : undefined + if (workerMailbox) { + const address = `dispatch:${workerMailbox.dispatchId}` + const showAll = params.all === true || (params.unread === false && params.peek !== true) + const messages = showAll + ? db.getAllMessagesForHandle(address, 100, typeFilter) + : db.getUnreadMessages(address, typeFilter) + if (!showAll && params.peek !== true && messages.length > 0) { + db.markAsRead(messages.map((message) => message.id)) + } + if (messages.length > 0 || !params.wait) { + return { + ...(workerMailbox.runId ? { runId: workerMailbox.runId } : {}), + dispatchId: workerMailbox.dispatchId, + messages, + count: messages.length, + ...(params.format || params.inject + ? { formatted: messages.map(formatMessageBanner).join('\n\n') } + : {}) + } + } + const waitResult = await runtime.waitForMessage(address, { + typeFilter: typeFilter as string[] | undefined, + timeoutMs: params.timeoutMs ?? undefined, + signal + }) + if (waitResult === 'timed_out' || waitResult === 'cancelled') { + return { + ...(workerMailbox.runId ? { runId: workerMailbox.runId } : {}), + dispatchId: workerMailbox.dispatchId, + messages: [], + count: 0, + timedOut: waitResult === 'timed_out', + cancelled: waitResult === 'cancelled', + connectionLost: waitResult === 'cancelled' && signal?.aborted === true + } + } + const arrived = db.getUnreadMessages(address, typeFilter) + db.markAsRead(arrived.map((message) => message.id)) + return { + ...(workerMailbox.runId ? { runId: workerMailbox.runId } : {}), + dispatchId: workerMailbox.dispatchId, + messages: arrived, + count: arrived.length, + ...(params.format || params.inject + ? { formatted: arrived.map(formatMessageBanner).join('\n\n') } + : {}) + } } // Why: unread:false is honored for one release as a compat shim so in-flight callers don't break (design doc §5). @@ -305,7 +861,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ db.markAsRead(messages.map((m) => m.id)) } - if (params.inject) { + if (params.format || params.inject) { const formatted = visibleMessages.map(formatMessageBanner).join('\n\n') return { messages: visibleMessages, formatted, count: visibleMessages.length } } @@ -337,13 +893,49 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ defineMethod({ name: 'orchestration.reply', params: ReplyParams, - handler: (params, { runtime }) => { + handler: async (params, { runtime }) => { const db = runtime.getOrchestrationDb() const original = db.getMessageById(params.id) if (!original) { throw new Error(`Message not found: ${params.id}`) } + const question = db.getQuestion(params.id) + if (question) { + const run = resolveRunScope(runtime, { + runId: params.run ?? question.run_id, + callerTerminalHandle: params.from, + requireCurrentConsumer: true + }) + const answered = db.answerQuestion({ + messageId: question.message_id, + runId: run.id, + consumerGeneration: run.consumer_generation, + body: params.body + }) + const federated = db.getFederatedDispatch(question.dispatch_id) + if (federated) { + db.enqueueFederationRelay({ + dispatchId: question.dispatch_id, + direction: 'to_worker', + kind: 'reply', + payload: JSON.stringify({ + questionId: question.message_id, + answerMessageId: answered.message.id, + body: params.body + }) + }) + runtime.ensureOrchestrationFederationRelay(run.id) + } else { + runtime.notifyMessageArrived(`dispatch:${question.dispatch_id}`, 'status') + } + return { + message: answered.message, + question: answered.question, + duplicate: answered.duplicate + } + } + db.markAsRead([original.id]) const reply = db.insertMessage({ @@ -395,7 +987,12 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ displayName: params.displayName, deps, parentId: params.parent, - createdByTerminalHandle: params.callerTerminalHandle + createdByTerminalHandle: params.callerTerminalHandle, + runId: resolveRunScope(runtime, { + runId: params.run, + callerTerminalHandle: params.callerTerminalHandle, + requireCurrentConsumer: true + }).id }) return { task } } @@ -406,10 +1003,20 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ params: TaskListParams, handler: (params, { runtime }) => { const db = runtime.getOrchestrationDb() + const explicitRun = params.run ? db.getRun(params.run) : undefined + const run = + explicitRun?.legacy === 1 + ? explicitRun + : resolveRunScope(runtime, { + runId: params.run, + callerTerminalHandle: params.callerTerminalHandle, + requireCurrentConsumer: params.run === undefined + }) // Why: listTasksWithDispatch adds assignee_handle + dispatch_id (NULL for non-dispatched), so legacy-shape consumers are unaffected. const joined = db.listTasksWithDispatch({ status: params.status as TaskStatus, - ready: params.ready + ready: params.ready, + runId: run.id }) const tasks = joined.map((row) => { const { assignee_handle, dispatch_id, ...base } = row @@ -419,6 +1026,8 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ return base }) return { + runId: run.id, + legacyReadOnly: run.legacy === 1, tasks: params.brief ? abbreviateOrchestrationTasks(tasks) : tasks, count: tasks.length } @@ -430,6 +1039,18 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ params: TaskUpdateParams, handler: (params, { runtime }) => { const db = runtime.getOrchestrationDb() + const run = resolveRunScope(runtime, { + runId: params.run, + callerTerminalHandle: params.callerTerminalHandle, + requireCurrentConsumer: true + }) + const existing = db.getTask(params.id) + if (!existing || existing.run_id !== run.id) { + throw new OrchestrationError( + 'task_not_found', + `Task ${params.id} was not found in Run ${run.id}.` + ) + } const task = db.updateTaskStatus(params.id, params.status, params.result) if (!task) { throw new Error(`Task not found: ${params.id}`) @@ -447,6 +1068,17 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ if (!task) { throw new Error(`Task not found: ${params.task}`) } + const run = resolveRunScope(runtime, { + runId: params.run, + callerTerminalHandle: params.from, + requireCurrentConsumer: true + }) + if (task.run_id !== run.id) { + throw new OrchestrationError( + 'task_not_found', + `Task ${task.id} was not found in Run ${run.id}.` + ) + } // Why: dry-run previews the preamble without mutating state, so it skips the ready-status check and uses a placeholder dispatchId. if (params.dryRun) { @@ -485,11 +1117,23 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ } } - const ctx = db.createDispatchContext( - params.task, - to, - runtime.getTerminalPaneKey(to) ?? undefined - ) + const assigneePaneKey = runtime.getTerminalPaneKey(to) ?? undefined + const processIncarnation = runtime.getTerminalProcessIncarnation(to) ?? undefined + if (params.inject && (!assigneePaneKey || !processIncarnation)) { + throw new OrchestrationError( + 'stable_pane_required', + `Terminal ${to} has no stable pane/process incarnation for lifecycle authority.` + ) + } + + const ctx = db.createDispatchContext(params.task, to, assigneePaneKey) + const dispatchCapability = params.inject + ? db.mintDispatchCapability({ + dispatchId: ctx.id, + paneKey: assigneePaneKey as string, + processIncarnation: processIncarnation as string + }) + : undefined // Why: built after ctx so dispatchId is the real ctx.id, letting heartbeats attribute liveness to a specific dispatch context, not just a task. const preamble = buildDispatchPreamble({ @@ -498,6 +1142,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ taskSpec: task.spec, coordinatorHandle: params.from ?? 'coordinator', workerHandle: to, + dispatchCapability, devMode: params.devMode, cliCommand: runtime.getTerminalOrchestrationCliCommand(to) }) @@ -558,11 +1203,14 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ defineMethod({ name: 'orchestration.ask', params: AskParams, - handler: async (params, { runtime, signal }) => { - // Why: group addresses have no unambiguous answer semantics; rejecting avoids a silent timeout on a decision_gate no one subscribes to. - if (isGroupAddress(params.to)) { + handler: async ( + params, + { runtime, signal, orchestrationCapability, recordMutationReceipt } + ) => { + // Why: group addresses have no unambiguous first-answer authority. + if (params.to && isGroupAddress(params.to)) { throw new Error( - 'ask does not support group addresses; use send --type decision_gate for fan-out questions' + 'ask does not support group addresses; use send for non-blocking fan-out questions' ) } @@ -570,51 +1218,143 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ const from = params.from ?? 'unknown' // Why: echoed on every return so a clamped caller reports the budget actually waited, not the one it asked for. const timeoutMs = clampOrchestrationAskTimeoutMs(params.timeoutMs) + const paneKey = runtime.getTerminalPaneKey(from) ?? undefined + const remoteAttachment = paneKey ? db.findActiveRemoteAttachmentForPane(paneKey) : undefined + if (remoteAttachment) { + rejectFederatedExplicitTarget(params) + return askRemoteRunHome({ + params: { ...params, timeoutMs }, + runtime, + signal, + orchestrationCapability, + recordMutationReceipt, + from, + paneKey: paneKey as string, + dispatchId: remoteAttachment.dispatch_id, + taskId: remoteAttachment.task_id + }) + } + const activeDispatch = db.getActiveDispatchForIdentity(from, paneKey) + if (!activeDispatch) { + throw new OrchestrationError( + 'dispatch_inactive', + 'ask requires an active supervised Dispatch.' + ) + } + if (activeDispatch.capability_hash) { + const authority = db.verifyDispatchCapability({ + dispatchId: activeDispatch.id, + capability: orchestrationCapability, + paneKey, + processIncarnation: runtime.getTerminalProcessIncarnation(from) ?? undefined + }) + if (!authority.valid) { + throw new OrchestrationError('dispatch_capability_invalid', authority.reason) + } + } const options = params.options ?.split(',') .map((s) => s.trim()) .filter(Boolean) ?? [] + let question = params.resume ? db.getQuestion(params.resume) : undefined + if (params.resume) { + if (!question || question.dispatch_id !== activeDispatch.id) { + throw new OrchestrationError( + 'question_not_found', + `Question ${params.resume} does not belong to this active Dispatch.` + ) + } + } else { + const run = db.getRun(activeDispatch.run_id) + if (!run || run.legacy === 1) { + throw new OrchestrationError( + 'run_not_found', + `Run ${activeDispatch.run_id} was not found.` + ) + } + if (params.run && params.run !== run.id) { + throw new OrchestrationError( + 'dispatch_run_mismatch', + `Dispatch ${activeDispatch.id} belongs to Run ${run.id}, not ${params.run}.` + ) + } + if (params.to && params.to !== `run:${run.id}` && params.to !== run.coordinator_handle) { + throw new OrchestrationError( + 'dispatch_run_mismatch', + `ask from Dispatch ${activeDispatch.id} must target its owning Run ${run.id}.` + ) + } + const created = db.createQuestion({ + runId: run.id, + dispatchId: activeDispatch.id, + askerHandle: from, + question: params.question as string, + options + }) + question = created.question + runtime.notifyMessageArrived(`run:${run.id}`, created.message.type) + } - const payload = JSON.stringify({ question: params.question, options }) - const outbound = db.insertMessage({ - from, - to: params.to, - subject: 'Question', - body: params.question, - type: 'decision_gate', - payload + const questionId = question.message_id + recordMutationReceipt?.({ + accepted: true, + answer: null, + messageId: questionId, + threadId: questionId, + timedOut: false, + cancelled: false, + connectionLost: false, + timeoutMs }) - runtime.deliverPendingMessagesForHandle(params.to) - runtime.notifyMessageArrived(params.to, outbound.type) - - const threadId = outbound.id const deadline = Date.now() + timeoutMs - const afterSequence = outbound.sequence - - // Why: waitForMessage is handle-scoped, so re-query by thread each wake and bound by remaining budget so distractor messages can't loop forever. while (true) { - const replies = db.getThreadMessagesFor(threadId, from, afterSequence) - if (replies.length > 0) { - const reply = replies[0] - db.markAsRead([reply.id]) + const current = db.getQuestion(questionId) + if (!current || current.status === 'closed') { + throw new OrchestrationError( + 'dispatch_inactive', + `Question ${questionId} closed because its Dispatch is inactive.` + ) + } + if (current.status === 'answered') { return { - answer: reply.body, - messageId: reply.id, - threadId, + answer: current.answer_body, + messageId: questionId, + answerMessageId: current.answer_message_id, + threadId: questionId, timedOut: false, + cancelled: false, + connectionLost: false, timeoutMs } } if (signal?.aborted) { - return { answer: null, messageId: null, threadId, timedOut: true, timeoutMs } + return { + answer: null, + messageId: questionId, + threadId: questionId, + timedOut: false, + cancelled: true, + connectionLost: true, + timeoutMs + } } const remainingMs = deadline - Date.now() if (remainingMs <= 0) { - return { answer: null, messageId: null, threadId, timedOut: true, timeoutMs } + return { + answer: null, + messageId: questionId, + threadId: questionId, + timedOut: true, + cancelled: false, + connectionLost: false, + timeoutMs + } } - // Why: signal releases the waiter on client disconnect while the already-sent decision gate stays visible to the recipient. - await runtime.waitForMessage(from, { timeoutMs: remainingMs, signal }) + await runtime.waitForMessage(`dispatch:${activeDispatch.id}`, { + timeoutMs: remainingMs, + signal + }) } } }), @@ -642,3 +1382,127 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ } }) ] + +async function askRemoteRunHome(args: { + params: z.infer + runtime: OrcaRuntimeService + signal?: AbortSignal + orchestrationCapability?: string + recordMutationReceipt?: (receipt: unknown) => void + from: string + paneKey: string + dispatchId: string + taskId: string +}): Promise { + const db = args.runtime.getOrchestrationDb() + const timeoutMs = clampOrchestrationAskTimeoutMs(args.params.timeoutMs) + if ( + !db.verifyRemoteAttachmentAuthority({ + dispatchId: args.dispatchId, + capability: args.orchestrationCapability, + paneKey: args.paneKey, + processIncarnation: args.runtime.getTerminalProcessIncarnation(args.from) + }) + ) { + throw new OrchestrationError( + 'dispatch_capability_invalid', + 'The remote Dispatch capability or exact worker process is invalid.' + ) + } + const options = + args.params.options + ?.split(',') + .map((option) => option.trim()) + .filter(Boolean) ?? [] + let questionId = args.params.resume + if (questionId) { + const existing = db.getRemoteQuestion(questionId) + if (!existing || existing.dispatch_id !== args.dispatchId) { + throw new OrchestrationError( + 'question_not_found', + `Question ${questionId} does not belong to this remote Dispatch.` + ) + } + } else { + const relay = db.enqueueFederationRelay({ + dispatchId: args.dispatchId, + direction: 'to_home', + kind: 'question', + payload: JSON.stringify({ + from: args.from, + subject: 'Question', + body: args.params.question as string, + type: 'question', + priority: 'normal', + threadId: null, + payload: JSON.stringify({ + taskId: args.taskId, + dispatchId: args.dispatchId, + question: args.params.question, + options + }) + }), + remoteQuestion: true + }) + questionId = relay.message_id + } + args.recordMutationReceipt?.({ + accepted: true, + answer: null, + messageId: questionId, + threadId: questionId, + timedOut: false, + cancelled: false, + connectionLost: false, + timeoutMs + }) + const deadline = Date.now() + timeoutMs + while (true) { + const question = db.getRemoteQuestion(questionId) + if (!question || question.status === 'closed') { + throw new OrchestrationError( + 'dispatch_inactive', + `Question ${questionId} closed because its remote Dispatch is inactive.` + ) + } + if (question.status === 'answered') { + return { + answer: question.answer_body, + messageId: questionId, + answerMessageId: question.answer_message_id, + threadId: questionId, + timedOut: false, + cancelled: false, + connectionLost: false, + timeoutMs + } + } + if (args.signal?.aborted) { + return { + answer: null, + messageId: questionId, + threadId: questionId, + timedOut: false, + cancelled: true, + connectionLost: true, + timeoutMs + } + } + const remainingMs = deadline - Date.now() + if (remainingMs <= 0) { + return { + answer: null, + messageId: questionId, + threadId: questionId, + timedOut: true, + cancelled: false, + connectionLost: false, + timeoutMs + } + } + await args.runtime.waitForMessage(`dispatch:${args.dispatchId}`, { + timeoutMs: remainingMs, + signal: args.signal + }) + } +} diff --git a/src/main/runtime/rpc/orchestration-contract-fence.test.ts b/src/main/runtime/rpc/orchestration-contract-fence.test.ts new file mode 100644 index 00000000000..a6618542d9c --- /dev/null +++ b/src/main/runtime/rpc/orchestration-contract-fence.test.ts @@ -0,0 +1,138 @@ +import { createHash } from 'node:crypto' +import { z } from 'zod' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { ORCHESTRATION_CONTRACT_VERSION } from '../../../shared/protocol-version' +import { OrcaRuntimeService } from '../orca-runtime' +import { OrchestrationDb } from '../orchestration/db' +import { defineMethod, type RpcRequest } from './core' +import { RpcDispatcher } from './dispatcher' + +describe('orchestration contract fence', () => { + const databases: OrchestrationDb[] = [] + + afterEach(() => { + for (const database of databases.splice(0)) { + database.close() + } + }) + + function createHarness(method = 'orchestration.send') { + const database = new OrchestrationDb(':memory:') + databases.push(database) + const runtime = new OrcaRuntimeService() + runtime.setOrchestrationDb(database) + const effect = vi.fn(() => ({ accepted: true })) + const dispatcher = new RpcDispatcher({ + runtime, + methods: [ + defineMethod({ + name: method, + params: z.object({ subject: z.string() }), + handler: effect + }) + ] + }) + return { database, dispatcher, effect } + } + + function request(overrides: Partial = {}): RpcRequest { + return { + id: 'rpc_1', + authToken: 'caller-token', + method: 'orchestration.send', + params: { subject: 'hello' }, + orchestrationRequestId: 'mutation_1', + ...overrides + } + } + + it.each([ + [undefined, 'client_contract_missing'], + [0, 'client_contract_unsupported'], + [ORCHESTRATION_CONTRACT_VERSION + 1, 'client_contract_unsupported'] + ])( + 'rejects contract version %s before parsing, receipts, or effects', + async (version, reason) => { + const { database, dispatcher, effect } = createHarness() + const response = await dispatcher.dispatch( + request({ + params: { malformed: true }, + orchestrationContractVersion: version + }) + ) + + expect(response).toMatchObject({ + ok: false, + error: { + code: 'orchestration_migration_required', + data: { + reason, + effectsApplied: false, + nextCommandArgs: ['skills', 'get', 'orchestration', '--full'] + } + } + }) + expect(effect).not.toHaveBeenCalled() + const callerFingerprint = createHash('sha256').update('caller-token').digest('hex') + expect(database.getMutationReceipt(callerFingerprint, 'mutation_1')).toBeUndefined() + } + ) + + it('allows the current contract to reach the mutation executor', async () => { + const { dispatcher, effect } = createHarness() + const response = await dispatcher.dispatch( + request({ orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION }) + ) + + expect(response).toMatchObject({ ok: true, result: { accepted: true } }) + expect(effect).toHaveBeenCalledOnce() + }) + + it('keeps read-only orchestration inspection available without a contract', async () => { + const { dispatcher, effect } = createHarness('orchestration.taskList') + const response = await dispatcher.dispatch( + request({ + method: 'orchestration.taskList', + params: { subject: 'read' }, + orchestrationRequestId: undefined + }) + ) + + expect(response).toMatchObject({ ok: true, result: { accepted: true } }) + expect(effect).toHaveBeenCalledOnce() + }) + + it.each(['orchestration.run', 'orchestration.runStop'])( + 'retires %s even when the caller sends the current contract', + async (method) => { + const { dispatcher, effect } = createHarness(method) + const response = await dispatcher.dispatch( + request({ + method, + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION + }) + ) + + expect(response).toMatchObject({ + ok: false, + error: { + code: 'orchestration_migration_required', + data: { reason: 'command_retired', effectsApplied: false } + } + }) + expect(effect).not.toHaveBeenCalled() + } + ) + + it('applies the same pre-effect fence on WebSocket dispatch', async () => { + const { dispatcher, effect } = createHarness() + const replies: string[] = [] + await dispatcher.dispatchStreaming(request(), (reply) => replies.push(reply)) + + expect(JSON.parse(replies[0] ?? '{}')).toMatchObject({ + ok: false, + error: { code: 'orchestration_migration_required' } + }) + expect(effect).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/runtime/rpc/orchestration-contract-fence.ts b/src/main/runtime/rpc/orchestration-contract-fence.ts new file mode 100644 index 00000000000..ba266fb6d2c --- /dev/null +++ b/src/main/runtime/rpc/orchestration-contract-fence.ts @@ -0,0 +1,36 @@ +import { + isOrchestrationMutation, + isRetiredOrchestrationMethod, + orchestrationMigrationData, + type OrchestrationMigrationReason +} from '../../../shared/orchestration-rpc-contract' +import { ORCHESTRATION_CONTRACT_VERSION } from '../../../shared/protocol-version' +import type { RpcEnvelopeMeta, RpcRequest, RpcResponse } from './core' +import { errorResponse } from './errors' + +export function orchestrationMigrationFence( + request: RpcRequest, + meta: RpcEnvelopeMeta +): RpcResponse | undefined { + if (!isOrchestrationMutation(request.method, request.params)) { + return undefined + } + let reason: OrchestrationMigrationReason | undefined + if (isRetiredOrchestrationMethod(request.method)) { + reason = 'command_retired' + } else if (request.orchestrationContractVersion === undefined) { + reason = 'client_contract_missing' + } else if (request.orchestrationContractVersion !== ORCHESTRATION_CONTRACT_VERSION) { + reason = 'client_contract_unsupported' + } + if (!reason) { + return undefined + } + return errorResponse( + request.id, + meta, + 'orchestration_migration_required', + 'This orchestration mutation uses an obsolete contract. No effects were applied.', + orchestrationMigrationData(reason) + ) +} diff --git a/src/main/runtime/rpc/orchestration-mutation-executor.ts b/src/main/runtime/rpc/orchestration-mutation-executor.ts new file mode 100644 index 00000000000..0ec659cc02d --- /dev/null +++ b/src/main/runtime/rpc/orchestration-mutation-executor.ts @@ -0,0 +1,152 @@ +import { createHash } from 'node:crypto' +import { isOrchestrationMutation } from '../../../shared/orchestration-rpc-contract' +import type { OrcaRuntimeService } from '../orca-runtime' +import { OrchestrationError } from '../orchestration/orchestration-error' +import type { RpcRequest } from './core' + +export type DurableMutationInvocation = { + identity: { + callerFingerprint: string + requestId: string + method: string + payloadHash: string + } + recordReceipt: (receipt: unknown) => void +} + +export class OrchestrationMutationExecutor { + private readonly inFlight = new Map>() + + constructor(private readonly runtime: OrcaRuntimeService) {} + + async run( + request: RpcRequest, + params: unknown, + invoke: (mutation?: DurableMutationInvocation) => Promise | unknown + ): Promise { + const requestId = request.orchestrationRequestId + if (!requestId || !isOrchestrationMutation(request.method, params)) { + return await invoke() + } + const callerFingerprint = authenticatedCallerFingerprint(request) + const payloadHash = createHash('sha256') + .update(JSON.stringify(canonicalize({ method: request.method, params }))) + .digest('hex') + const key = `${callerFingerprint}:${requestId}` + const db = this.runtime.getOrchestrationDb() + const identity = { callerFingerprint, requestId, method: request.method, payloadHash } + const atomicWorkerAcceptance = + request.method === 'orchestration.workerStart' || + request.method === 'orchestration.federationAttachStart' + const begun = atomicWorkerAcceptance + ? (() => { + const row = db.getMutationReceipt(callerFingerprint, requestId) + if (!row) { + return { disposition: 'started' as const } + } + if (row.method !== request.method || row.payload_hash !== payloadHash) { + throw new OrchestrationError( + 'request_mismatch', + `Mutation request ${requestId} was already used with different input.` + ) + } + return { disposition: row.state, row } + })() + : db.beginMutationReceipt(identity) + + if (begun.disposition === 'completed') { + return attachMutationReceipt(JSON.parse(begun.row.receipt ?? 'null'), requestId, true) + } + if (begun.disposition === 'pending') { + const active = this.inFlight.get(key) + if (active) { + return attachMutationReceipt(await active, requestId, true) + } + const recovery = getPendingWorkerStartRecovery(request.method, begun.row.receipt) + throw new OrchestrationError( + 'operation_unknown', + recovery + ? `Worker start ${requestId} was accepted as Dispatch ${recovery.dispatchId} before restart. Inspect that Dispatch; do not start another worker.` + : `Mutation ${requestId} may have been accepted before restart. Retry inspection or recovery with the same request ID.`, + recovery + ? { + requestId, + dispatchId: recovery.dispatchId, + recoveryCommand: `orca orchestration worker-show --dispatch ${recovery.dispatchId} --json` + } + : { requestId } + ) + } + + const recordReceipt = (result: unknown): void => { + db.completeMutationReceipt({ + ...identity, + receipt: JSON.stringify(attachMutationReceipt(result, requestId, false)) + }) + } + const active = Promise.resolve().then(() => invoke({ identity, recordReceipt })) + this.inFlight.set(key, active) + try { + const result = await active + const receipted = attachMutationReceipt(result, requestId, false) + db.completeMutationReceipt({ ...identity, receipt: JSON.stringify(receipted) }) + return receipted + } catch (error) { + if (!(error instanceof OrchestrationError && error.code === 'operation_unknown')) { + db.discardPendingMutationReceipt(callerFingerprint, requestId) + } + throw error + } finally { + this.inFlight.delete(key) + } + } +} + +export function authenticatedCallerFingerprint(request: RpcRequest): string { + const callerToken = + request.authToken || + (request as RpcRequest & { deviceToken?: string }).deviceToken || + 'authenticated_transport' + return createHash('sha256').update(callerToken).digest('hex') +} + +function canonicalize(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(canonicalize) + } + if (!value || typeof value !== 'object') { + return value + } + const source = value as Record + const result: Record = {} + for (const key of Object.keys(source).sort()) { + if (source[key] !== undefined) { + result[key] = canonicalize(source[key]) + } + } + return result +} + +function attachMutationReceipt(result: unknown, requestId: string, replayed: boolean): unknown { + if (!result || typeof result !== 'object' || Array.isArray(result)) { + return { result, mutation: { requestId, replayed } } + } + return { ...(result as Record), mutation: { requestId, replayed } } +} + +function getPendingWorkerStartRecovery( + method: string, + receipt: string | null +): { dispatchId: string } | undefined { + if (method !== 'orchestration.workerStart' || !receipt) { + return undefined + } + try { + const parsed = JSON.parse(receipt) as { accepted?: { dispatchId?: unknown } } + return typeof parsed.accepted?.dispatchId === 'string' + ? { dispatchId: parsed.accepted.dispatchId } + : undefined + } catch { + return undefined + } +} diff --git a/src/main/runtime/rpc/orchestration-mutation-ledger.test.ts b/src/main/runtime/rpc/orchestration-mutation-ledger.test.ts new file mode 100644 index 00000000000..3407211ba6f --- /dev/null +++ b/src/main/runtime/rpc/orchestration-mutation-ledger.test.ts @@ -0,0 +1,309 @@ +import { createHash } from 'node:crypto' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { z } from 'zod' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { ORCHESTRATION_CONTRACT_VERSION } from '../../../shared/protocol-version' +import { OrcaRuntimeService } from '../orca-runtime' +import { OrchestrationDb } from '../orchestration/db' +import { defineMethod, type RpcRequest } from './core' +import { RpcDispatcher } from './dispatcher' +import { ORCHESTRATION_METHODS } from './methods/orchestration' + +const Params = z.object({ subject: z.string() }) + +function request(params: { + rpcId: string + mutationId: string + subject: string + authToken?: string +}): RpcRequest { + return { + id: params.rpcId, + authToken: params.authToken ?? 'caller-token', + method: 'orchestration.send', + params: { subject: params.subject }, + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationRequestId: params.mutationId + } +} + +describe('durable orchestration mutation ledger', () => { + const paths: string[] = [] + + afterEach(() => { + for (const path of paths.splice(0)) { + rmSync(path, { recursive: true, force: true }) + } + }) + + function createHarness(dbPath: string | ':memory:' = ':memory:') { + const db = new OrchestrationDb(dbPath) + const runtime = new OrcaRuntimeService() + runtime.setOrchestrationDb(db) + const effect = vi.fn((subject: string) => + db.insertMessage({ from: 'caller', to: 'recipient', subject }) + ) + const dispatcher = new RpcDispatcher({ + runtime, + methods: [ + defineMethod({ + name: 'orchestration.send', + params: Params, + handler: ({ subject }) => ({ message: effect(subject) }) + }) + ] + }) + return { db, runtime, dispatcher, effect } + } + + it('replays one completed receipt without repeating the effect', async () => { + const { db, dispatcher, effect } = createHarness() + const first = await dispatcher.dispatch( + request({ rpcId: 'rpc_1', mutationId: 'mutation_1', subject: 'hello' }) + ) + const replay = await dispatcher.dispatch( + request({ rpcId: 'rpc_2', mutationId: 'mutation_1', subject: 'hello' }) + ) + + expect(first).toMatchObject({ + ok: true, + result: { message: { subject: 'hello' }, mutation: { replayed: false } } + }) + expect(replay).toMatchObject({ + ok: true, + result: { message: { subject: 'hello' }, mutation: { replayed: true } } + }) + expect(effect).toHaveBeenCalledTimes(1) + expect(db.getInbox(10)).toHaveLength(1) + db.close() + }) + + it('rejects changed input for the same caller and request ID', async () => { + const { db, dispatcher } = createHarness() + await dispatcher.dispatch( + request({ rpcId: 'rpc_1', mutationId: 'mutation_1', subject: 'hello' }) + ) + const mismatch = await dispatcher.dispatch( + request({ rpcId: 'rpc_2', mutationId: 'mutation_1', subject: 'changed' }) + ) + + expect(mismatch).toMatchObject({ ok: false, error: { code: 'request_mismatch' } }) + db.close() + }) + + it('applies the same ledger on authenticated WebSocket dispatch', async () => { + const { db, dispatcher, effect } = createHarness() + const replies: string[] = [] + const firstRequest = request({ + rpcId: 'rpc_1', + mutationId: 'mutation_remote', + subject: 'remote' + }) as RpcRequest & { deviceToken?: string } + firstRequest.authToken = '' + firstRequest.deviceToken = 'paired-device' + await dispatcher.dispatchStreaming(firstRequest, (reply) => replies.push(reply)) + const replayRequest = { ...firstRequest, id: 'rpc_2' } + await dispatcher.dispatchStreaming(replayRequest, (reply) => replies.push(reply)) + + expect(JSON.parse(replies[0] ?? '{}')).toMatchObject({ + ok: true, + result: { mutation: { replayed: false } } + }) + expect(JSON.parse(replies[1] ?? '{}')).toMatchObject({ + ok: true, + result: { mutation: { replayed: true } } + }) + expect(effect).toHaveBeenCalledTimes(1) + db.close() + }) + + it('joins concurrent identical mutations', async () => { + const db = new OrchestrationDb(':memory:') + const runtime = new OrcaRuntimeService() + runtime.setOrchestrationDb(db) + let release: (() => void) | undefined + const gate = new Promise((resolve) => { + release = resolve + }) + const effect = vi.fn(async () => { + await gate + return { accepted: true } + }) + const dispatcher = new RpcDispatcher({ + runtime, + methods: [ + defineMethod({ + name: 'orchestration.send', + params: Params, + handler: effect + }) + ] + }) + const first = dispatcher.dispatch( + request({ rpcId: 'rpc_1', mutationId: 'mutation_join', subject: 'same' }) + ) + await Promise.resolve() + const second = dispatcher.dispatch( + request({ rpcId: 'rpc_2', mutationId: 'mutation_join', subject: 'same' }) + ) + release?.() + + expect(await first).toMatchObject({ ok: true, result: { mutation: { replayed: false } } }) + expect(await second).toMatchObject({ ok: true, result: { mutation: { replayed: true } } }) + expect(effect).toHaveBeenCalledTimes(1) + db.close() + }) + + it('replays a completed receipt after database and dispatcher restart', async () => { + const dir = mkdtempSync(join(tmpdir(), 'orca-mutation-ledger-')) + paths.push(dir) + const dbPath = join(dir, 'orchestration.db') + const first = createHarness(dbPath) + await first.dispatcher.dispatch( + request({ rpcId: 'rpc_1', mutationId: 'mutation_1', subject: 'durable' }) + ) + first.db.close() + + const second = createHarness(dbPath) + const replay = await second.dispatcher.dispatch( + request({ rpcId: 'rpc_2', mutationId: 'mutation_1', subject: 'durable' }) + ) + expect(replay).toMatchObject({ + ok: true, + result: { message: { subject: 'durable' }, mutation: { replayed: true } } + }) + expect(second.effect).not.toHaveBeenCalled() + second.db.close() + }) + + it('returns unknown for a pending receipt left by a previous process', async () => { + const { db, dispatcher } = createHarness() + db.beginMutationReceipt({ + callerFingerprint: createHash('sha256').update('caller-token').digest('hex'), + requestId: 'mutation_1', + method: 'orchestration.send', + payloadHash: createHash('sha256') + .update('{"method":"orchestration.send","params":{"subject":"hello"}}') + .digest('hex') + }) + + const result = await dispatcher.dispatch( + request({ rpcId: 'rpc_1', mutationId: 'mutation_1', subject: 'hello' }) + ) + expect(result).toMatchObject({ ok: false, error: { code: 'operation_unknown' } }) + db.close() + }) + + it('returns the accepted Dispatch when worker-start was interrupted by restart', async () => { + const db = new OrchestrationDb(':memory:') + const runtime = new OrcaRuntimeService() + runtime.setOrchestrationDb(db) + const params = { from: 'term_coord', task: db.createTask({ spec: 'restart' }).id } + const callerFingerprint = createHash('sha256').update('caller-token').digest('hex') + const payloadHash = createHash('sha256') + .update(JSON.stringify({ method: 'orchestration.workerStart', params })) + .digest('hex') + const started = db.createStartingWorkerDispatch({ + taskId: params.task, + startOptions: {}, + mutationReceipt: { + callerFingerprint, + requestId: 'mutation_worker_start', + method: 'orchestration.workerStart', + payloadHash + } + }) + const effect = vi.fn() + const dispatcher = new RpcDispatcher({ + runtime, + methods: [ + defineMethod({ + name: 'orchestration.workerStart', + params: z.object({ from: z.string(), task: z.string() }), + handler: effect + }) + ] + }) + + const result = await dispatcher.dispatch({ + id: 'rpc_worker_start_retry', + authToken: 'caller-token', + method: 'orchestration.workerStart', + params, + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationRequestId: 'mutation_worker_start' + }) + + expect(result).toMatchObject({ + ok: false, + error: { + code: 'operation_unknown', + data: { + requestId: 'mutation_worker_start', + dispatchId: started.dispatch.id, + recoveryCommand: `orca orchestration worker-show --dispatch ${started.dispatch.id} --json` + } + } + }) + expect(effect).not.toHaveBeenCalled() + db.close() + }) + + it('recovers a lost ask acceptance without creating a second question', async () => { + const db = new OrchestrationDb(':memory:') + const runtime = new OrcaRuntimeService() + runtime.setOrchestrationDb(db) + vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue('tab_worker:leaf_worker') + vi.spyOn(runtime, 'getTerminalProcessIncarnation').mockReturnValue('runtime:pty:1') + vi.spyOn(runtime, 'notifyMessageArrived').mockImplementation(() => {}) + vi.spyOn(runtime, 'waitForMessage').mockImplementation( + async (_address, options) => + await new Promise<'cancelled'>((resolve) => { + options?.signal?.addEventListener('abort', () => resolve('cancelled'), { once: true }) + }) + ) + const run = db.createRun({ + objective: 'Ask recovery', + coordinatorHandle: 'term_coord', + coordinatorPaneKey: 'tab_coord:leaf_coord' + }) + const task = db.createTask({ spec: 'ask', runId: run.id }) + const dispatch = db.createDispatchContext(task.id, 'term_worker', 'tab_worker:leaf_worker') + const capability = db.mintDispatchCapability({ + dispatchId: dispatch.id, + paneKey: 'tab_worker:leaf_worker', + processIncarnation: 'runtime:pty:1' + }) + const askRequest: RpcRequest = { + id: 'rpc_ask_1', + authToken: 'caller-token', + method: 'orchestration.ask', + params: { from: 'term_worker', question: 'Proceed?', timeoutMs: 60_000 }, + orchestrationCapability: capability, + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationRequestId: 'mutation_ask' + } + const controller = new AbortController() + const firstDispatcher = new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS }) + const first = firstDispatcher.dispatch(askRequest, { signal: controller.signal }) + await vi.waitFor(() => expect(db.getInbox(10)).toHaveLength(1)) + + const restartedDispatcher = new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS }) + const recovered = await restartedDispatcher.dispatch({ ...askRequest, id: 'rpc_ask_2' }) + expect(recovered).toMatchObject({ + ok: true, + result: { + accepted: true, + messageId: expect.stringMatching(/^msg_/), + mutation: { requestId: 'mutation_ask', replayed: true } + } + }) + expect(db.getInbox(10)).toHaveLength(1) + + controller.abort() + await first + db.close() + }) +}) diff --git a/src/main/runtime/rpc/runtime-feature-interaction.ts b/src/main/runtime/rpc/runtime-feature-interaction.ts new file mode 100644 index 00000000000..2a25bff1080 --- /dev/null +++ b/src/main/runtime/rpc/runtime-feature-interaction.ts @@ -0,0 +1,44 @@ +import type { FeatureInteractionId } from '../../../shared/feature-interactions' +import { isBrowserPaneUiRuntimeRpcParams } from '../../../shared/runtime-rpc-feature-interaction-source' + +export function getRuntimeFeatureInteractionId( + method: string, + result: unknown, + rawParams?: unknown +): FeatureInteractionId | null { + if (method === 'browser.profileImportFromBrowser') { + return hasBooleanResult(result, 'ok') ? 'cookie-import' : null + } + if (method === 'browser.profileClearDefaultCookies') { + return hasBooleanResult(result, 'cleared') ? 'cookie-import' : null + } + if (method === 'browser.screencast.unsubscribe') { + return null + } + if (method.startsWith('browser.') && isBrowserPaneUiRuntimeRpcParams(rawParams)) { + return null + } + if (method.startsWith('browser.') && !method.startsWith('browser.profile')) { + return 'agent-browser-use' + } + if (method.startsWith('emulator.')) { + return null + } + if (method === 'computer.permissions') { + return 'computer-use-setup' + } + if ( + method.startsWith('computer.') && + method !== 'computer.capabilities' && + method !== 'computer.permissionsStatus' + ) { + return 'computer-use' + } + return method.startsWith('orchestration.') ? 'agent-orchestration' : null +} + +function hasBooleanResult(value: unknown, key: string): boolean { + return ( + value !== null && typeof value === 'object' && (value as Record)[key] === true + ) +} diff --git a/src/main/runtime/runtime-rpc.test.ts b/src/main/runtime/runtime-rpc.test.ts index c143d4f75d8..2ed908bba4c 100644 --- a/src/main/runtime/runtime-rpc.test.ts +++ b/src/main/runtime/runtime-rpc.test.ts @@ -26,6 +26,7 @@ import { import { decrypt, deriveSharedKey, encrypt, generateKeyPair } from './rpc/e2ee-crypto' import { DeviceRegistry } from './device-registry' import { DEVICE_REGISTRY_FILENAME, E2EE_KEYPAIR_FILENAME } from './mobile-pairing-files' +import { ORCHESTRATION_CONTRACT_VERSION } from '../../shared/protocol-version' vi.mock('../git/worktree', () => ({ listWorktrees: vi.fn().mockResolvedValue([ @@ -60,7 +61,7 @@ async function sendRequest( resolve(JSON.parse(message) as Record) }) socket.on('connect', () => { - socket.write(`${JSON.stringify(request)}\n`) + socket.write(`${JSON.stringify(withCurrentOrchestrationContract(request))}\n`) }) }) } @@ -111,12 +112,20 @@ function openFramedSession(endpoint: string, request: Record): } }) socket.on('connect', () => { - socket.write(`${JSON.stringify(request)}\n`) + socket.write(`${JSON.stringify(withCurrentOrchestrationContract(request))}\n`) }) }) return { socket, frames, done } } +function withCurrentOrchestrationContract( + request: Record +): Record { + return typeof request.method === 'string' && request.method.startsWith('orchestration.') + ? { ...request, orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION } + : request +} + function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)) } @@ -131,6 +140,18 @@ async function waitFor(predicate: () => boolean, timeoutMs = 2_000): Promise { return new Promise((resolve, reject) => { const ws = new WebSocket(endpoint) @@ -1285,12 +1306,14 @@ describe('OrcaRuntimeRpcServer', () => { try { const first = server['handleWebSocketMessage']( - JSON.stringify({ - id: 'req_wait', - method: 'orchestration.check', - deviceToken: entry.token, - params: { terminal: 'term_wait', wait: true, timeoutMs: 10_000 } - }), + JSON.stringify( + withCurrentOrchestrationContract({ + id: 'req_wait', + method: 'orchestration.check', + deviceToken: entry.token, + params: { terminal: 'term_wait', wait: true, timeoutMs: 10_000 } + }) + ), (response) => replies.push(JSON.parse(response) as Record), () => {}, undefined, @@ -1300,12 +1323,14 @@ describe('OrcaRuntimeRpcServer', () => { await waitFor(() => server['activeLongPolls'] === 1) await server['handleWebSocketMessage']( - JSON.stringify({ - id: 'req_busy', - method: 'orchestration.check', - deviceToken: entry.token, - params: { terminal: 'term_busy', wait: true, timeoutMs: 10_000 } - }), + JSON.stringify( + withCurrentOrchestrationContract({ + id: 'req_busy', + method: 'orchestration.check', + deviceToken: entry.token, + params: { terminal: 'term_busy', wait: true, timeoutMs: 10_000 } + }) + ), (response) => replies.push(JSON.parse(response) as Record), () => {}, undefined, @@ -1338,6 +1363,7 @@ describe('OrcaRuntimeRpcServer', () => { const runtime = new OrcaRuntimeService() const db = new OrchestrationDb(':memory:') runtime.setOrchestrationDb(db) + seedSupervisedAskWorkers(db, ['term_w0', 'term_w1', 'term_w2']) // Why: cap 4 → ask sub-cap 2, so the third ask must be shed while waits keep the other half. const server = new OrcaRuntimeRpcServer({ runtime, @@ -1358,7 +1384,9 @@ describe('OrcaRuntimeRpcServer', () => { } const dispatch = (id: string, method: string, params: unknown): Promise => server['handleWebSocketMessage']( - JSON.stringify({ id, method, deviceToken: entry.token, params }), + JSON.stringify( + withCurrentOrchestrationContract({ id, method, deviceToken: entry.token, params }) + ), push, () => {}, undefined, @@ -4032,6 +4060,17 @@ describe('OrcaRuntimeRpcServer', () => { const runtime = new OrcaRuntimeService() const db = new OrchestrationDb(':memory:') runtime.setOrchestrationDb(db) + const askerPaneKey = 'tab_asker:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === 'term_asker' ? askerPaneKey : null + ) + const run = db.createRun({ + objective: 'Keepalive test', + coordinatorHandle: 'term_nobody', + coordinatorPaneKey: 'tab_coord:bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb' + }) + const task = db.createTask({ spec: 'Wait for an answer', runId: run.id }) + db.createDispatchContext(task.id, 'term_asker', askerPaneKey) const server = new OrcaRuntimeRpcServer({ runtime, userDataPath, @@ -4380,6 +4419,7 @@ describe('OrcaRuntimeRpcServer', () => { const runtime = new OrcaRuntimeService() const db = new OrchestrationDb(':memory:') runtime.setOrchestrationDb(db) + seedSupervisedAskWorkers(db, ['term_w0', 'term_w1', 'term_w2', 'term_w3']) // Why: cap 4 → ask sub-cap 2, so 4 concurrent asks can only take half the budget. const server = new OrcaRuntimeRpcServer({ runtime, diff --git a/src/main/ssh/ssh-remote-cli-error-response.ts b/src/main/ssh/ssh-remote-cli-error-response.ts new file mode 100644 index 00000000000..dacf9402a45 --- /dev/null +++ b/src/main/ssh/ssh-remote-cli-error-response.ts @@ -0,0 +1,10 @@ +import type { RpcResponse } from '../runtime/rpc/core' + +export function buildRemoteCliError(message: string, code = 'runtime_error'): RpcResponse { + return { + id: 'remote-cli-local', + ok: false, + error: { code, message }, + _meta: { runtimeId: 'unknown' } + } +} diff --git a/src/main/ssh/ssh-remote-orca-cli.test.ts b/src/main/ssh/ssh-remote-orca-cli.test.ts index 3d1c3e60793..6c9ce09244b 100644 --- a/src/main/ssh/ssh-remote-orca-cli.test.ts +++ b/src/main/ssh/ssh-remote-orca-cli.test.ts @@ -79,7 +79,8 @@ describe('runRemoteOrcaCli', () => { message.read_at = new Date(0).toISOString() } } - }) + }), + getActiveDispatchForIdentity: vi.fn(() => undefined) } const runtime = { getRuntimeId: () => 'runtime-test', @@ -155,7 +156,7 @@ describe('runRemoteOrcaCli', () => { expect(db.getUnreadMessages('term_windows')[0]?.from_handle).toBe('term_ssh') }) - it('forwards remote pane identity through the legacy orchestration fallback', async () => { + it('does not trust caller-supplied remote pane identity in the legacy fallback', async () => { const { runtime, db } = createRuntime() const result = await runRemoteOrcaCli( @@ -173,7 +174,7 @@ describe('runRemoteOrcaCli', () => { expect(result.exitCode).toBe(0) expect(db.insertMessage).toHaveBeenCalledWith( - expect.objectContaining({ senderPaneKey: 'tab_ssh:leaf_ssh' }) + expect.objectContaining({ senderPaneKey: undefined }) ) }) @@ -183,8 +184,14 @@ describe('runRemoteOrcaCli', () => { runtime.setOrchestrationDb(db) vi.spyOn(runtime, 'deliverPendingMessagesForHandle').mockImplementation(() => {}) vi.spyOn(runtime, 'notifyMessageArrived').mockImplementation(() => {}) - const task = db.createTask({ spec: 'remote work' }) + const run = db.createRun({ + objective: 'Remote lifecycle rejection', + coordinatorHandle: 'term_coord', + coordinatorPaneKey: 'tab_coord:leaf_coord' + }) + const task = db.createTask({ spec: 'remote work', runId: run.id }) const dispatch = db.createDispatchContext(task.id, 'term_ssh', 'tab_owner:leaf_owner') + vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue('tab_foreign:leaf_foreign') try { const result = await runRemoteOrcaCli( @@ -202,7 +209,11 @@ describe('runRemoteOrcaCli', () => { '--type', 'worker_done', '--payload', - JSON.stringify({ taskId: task.id, dispatchId: dispatch.id }), + JSON.stringify({ + taskId: task.id, + dispatchId: dispatch.id, + outcome: 'succeeded' + }), '--json' ], cwd: '/home/alice/repo', @@ -231,8 +242,14 @@ describe('runRemoteOrcaCli', () => { runtime.setOrchestrationDb(db) vi.spyOn(runtime, 'deliverPendingMessagesForHandle').mockImplementation(() => {}) vi.spyOn(runtime, 'notifyMessageArrived').mockImplementation(() => {}) - const task = db.createTask({ spec: 'remote work' }) + const run = db.createRun({ + objective: 'Remote lifecycle success', + coordinatorHandle: 'term_coord', + coordinatorPaneKey: 'tab_coord:leaf_coord' + }) + const task = db.createTask({ spec: 'remote work', runId: run.id }) const dispatch = db.createDispatchContext(task.id, 'term_ssh', 'tab_owner:leaf_owner') + vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue('tab_owner:leaf_owner') try { const result = await runRemoteOrcaCli( @@ -251,6 +268,8 @@ describe('runRemoteOrcaCli', () => { task.id, '--dispatch-id', dispatch.id, + '--outcome', + 'succeeded', '--files-modified', 'src/a.ts, src/b.ts', '--json' @@ -274,6 +293,70 @@ describe('runRemoteOrcaCli', () => { } }) + it('carries the Dispatch capability through the SSH envelope', async () => { + const db = new OrchestrationDb(':memory:') + const runtime = new OrcaRuntimeService() + runtime.setOrchestrationDb(db) + vi.spyOn(runtime, 'deliverPendingMessagesForHandle').mockImplementation(() => {}) + vi.spyOn(runtime, 'notifyMessageArrived').mockImplementation(() => {}) + vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue('tab_ssh:leaf_ssh') + vi.spyOn(runtime, 'getTerminalProcessIncarnation').mockReturnValue('ssh_runtime:pty:1') + const run = db.createRun({ + objective: 'SSH capability transport', + coordinatorHandle: 'term_coord', + coordinatorPaneKey: 'tab_coord:leaf_coord' + }) + const task = db.createTask({ spec: 'remote work', runId: run.id }) + const started = db.createStartingWorkerDispatch({ taskId: task.id, startOptions: {} }) + const capability = db.prepareStartingWorkerAuthority({ + dispatchId: started.dispatch.id, + handle: 'term_ssh', + paneKey: 'tab_ssh:leaf_ssh', + processIncarnation: 'ssh_runtime:pty:1', + worktreeId: 'repo::/home/alice/repo', + setupState: 'not_applicable', + effects: [] + }) + db.markWorkerDispatchReady(started.dispatch.id) + + try { + const result = await runRemoteOrcaCli( + runtime, + { + argv: [ + 'orchestration', + 'send', + '--type', + 'worker_done', + '--subject', + 'Done', + '--task-id', + task.id, + '--dispatch-id', + started.dispatch.id, + '--outcome', + 'succeeded', + '--dispatch-capability', + capability, + '--json' + ], + cwd: '/home/alice/repo', + env: { + ORCA_TERMINAL_HANDLE: 'term_ssh', + ORCA_PANE_KEY: 'tab_ssh:leaf_ssh' + } + }, + LEGACY_FALLBACK_OPTIONS + ) + + expect(result.exitCode).toBe(0) + expect(db.getTask(task.id)).toMatchObject({ status: 'completed' }) + expect(db.getWorkerDispatch(started.dispatch.id)).toMatchObject({ state: 'succeeded' }) + } finally { + db.close() + } + }) + it('rejects identity-less lifecycle sends in the legacy fallback', async () => { const { runtime, db } = createRuntime() diff --git a/src/main/ssh/ssh-remote-orca-cli.ts b/src/main/ssh/ssh-remote-orca-cli.ts index 14f5448d3d0..70c25943616 100644 --- a/src/main/ssh/ssh-remote-orca-cli.ts +++ b/src/main/ssh/ssh-remote-orca-cli.ts @@ -1,4 +1,6 @@ import type { CliStatusResult, RuntimeStatus } from '../../shared/runtime-types' +import type { RuntimeOrchestrationEnvelope } from '../../shared/runtime-rpc-envelope' +import { ORCHESTRATION_CONTRACT_VERSION } from '../../shared/protocol-version' import { RpcDispatcher } from '../runtime/rpc/dispatcher' import type { RpcResponse } from '../runtime/rpc/core' import type { OrcaRuntimeService } from '../runtime/orca-runtime' @@ -18,6 +20,7 @@ import { requiredRemoteCliString, resolveRemoteCliHandle } from './ssh-remote-cli-args' +import { buildRemoteCliError } from './ssh-remote-cli-error-response' import { getRemoteLinearHelp, tryDispatchRemoteLinearCli } from './ssh-remote-linear-cli' import { getRemoteOrchestrationPayload, @@ -51,7 +54,7 @@ export async function runRemoteOrcaCli( if (interactiveMessage) { if (json) { return { - stdout: `${JSON.stringify(buildLocalError(interactiveMessage, 'unsupported_over_ssh'), null, 2)}\n`, + stdout: `${JSON.stringify(buildRemoteCliError(interactiveMessage, 'unsupported_over_ssh'), null, 2)}\n`, stderr: '', exitCode: 1 } @@ -117,7 +120,7 @@ async function runLegacyRemoteOrcaCli( : 'runtime_error' if (json) { return { - stdout: `${JSON.stringify(buildLocalError(message, code), null, 2)}\n`, + stdout: `${JSON.stringify(buildRemoteCliError(message, code), null, 2)}\n`, stderr: '', exitCode: 1 } @@ -167,19 +170,27 @@ async function dispatchRemoteCli( }) case 'orchestration send': { const type = optionalRemoteCliString(parsed.flags, 'type') - return await call(dispatcher, 'orchestration.send', { - from: resolveRemoteOrchestrationSender(parsed.flags, env, type), - to: requiredRemoteCliString(parsed.flags, 'to'), - subject: requiredRemoteCliString(parsed.flags, 'subject'), - body: optionalRemoteCliString(parsed.flags, 'body'), - type, - priority: optionalRemoteCliString(parsed.flags, 'priority'), - threadId: optionalRemoteCliString(parsed.flags, 'thread-id'), - payload: getRemoteOrchestrationPayload(parsed.flags), - // Why: the legacy in-process bridge must preserve the same pane - // authority as the full host CLI passthrough. - senderPaneKey: env.ORCA_PANE_KEY || undefined - }) + return await call( + dispatcher, + 'orchestration.send', + { + from: resolveRemoteOrchestrationSender(parsed.flags, env, type), + to: optionalRemoteCliString(parsed.flags, 'to'), + subject: requiredRemoteCliString(parsed.flags, 'subject'), + body: optionalRemoteCliString(parsed.flags, 'body'), + type, + priority: optionalRemoteCliString(parsed.flags, 'priority'), + threadId: optionalRemoteCliString(parsed.flags, 'thread-id'), + payload: getRemoteOrchestrationPayload(parsed.flags), + // Why: the legacy in-process bridge must preserve the same pane + // authority as the full host CLI passthrough. + senderPaneKey: env.ORCA_PANE_KEY || undefined + }, + { + orchestrationCapability: optionalRemoteCliString(parsed.flags, 'dispatch-capability'), + orchestrationRequestId: optionalRemoteCliString(parsed.flags, 'retry-request') + } + ) } case 'orchestration check': return await call(dispatcher, 'orchestration.check', { @@ -215,21 +226,18 @@ async function dispatchRemoteCli( async function call( dispatcher: RpcDispatcher, method: string, - params?: Record + params?: Record, + envelope?: RuntimeOrchestrationEnvelope ): Promise { return await dispatcher.dispatch({ id: `remote-cli-${Date.now()}`, authToken: 'remote-cli', method, - params + params, + orchestrationCapability: envelope?.orchestrationCapability, + orchestrationContractVersion: method.startsWith('orchestration.') + ? ORCHESTRATION_CONTRACT_VERSION + : undefined, + orchestrationRequestId: envelope?.orchestrationRequestId }) } - -function buildLocalError(message: string, code = 'runtime_error'): RpcResponse { - return { - id: 'remote-cli-local', - ok: false, - error: { code, message }, - _meta: { runtimeId: 'unknown' } - } -} diff --git a/src/main/ssh/ssh-remote-orchestration-send.test.ts b/src/main/ssh/ssh-remote-orchestration-send.test.ts index 6efd9a18891..79ed89ddef0 100644 --- a/src/main/ssh/ssh-remote-orchestration-send.test.ts +++ b/src/main/ssh/ssh-remote-orchestration-send.test.ts @@ -38,6 +38,7 @@ describe('remote orchestration send compatibility', () => { new Map([ ['task-id', 'task_1'], ['dispatch-id', 'ctx_1'], + ['outcome', 'succeeded'], ['files-modified', 'src/a.ts, src/b.ts,'], ['report-path', 'report.md'], ['phase', 'reviewing'] @@ -47,6 +48,7 @@ describe('remote orchestration send compatibility', () => { expect(JSON.parse(payload ?? '{}')).toEqual({ taskId: 'task_1', dispatchId: 'ctx_1', + outcome: 'succeeded', filesModified: ['src/a.ts', 'src/b.ts'], reportPath: 'report.md', phase: 'reviewing' diff --git a/src/main/ssh/ssh-remote-orchestration-send.ts b/src/main/ssh/ssh-remote-orchestration-send.ts index 5e844a9c586..391be6499df 100644 --- a/src/main/ssh/ssh-remote-orchestration-send.ts +++ b/src/main/ssh/ssh-remote-orchestration-send.ts @@ -37,10 +37,11 @@ export function getRemoteOrchestrationPayload(flags: RemoteFlags): string | unde const rawPayload = optionalString(flags, 'payload') const taskId = optionalString(flags, 'task-id') const dispatchId = optionalString(flags, 'dispatch-id') + const outcome = optionalString(flags, 'outcome') const filesModified = optionalString(flags, 'files-modified') const reportPath = optionalString(flags, 'report-path') const phase = optionalString(flags, 'phase') - const hasStructuredPayload = [taskId, dispatchId, filesModified, reportPath, phase].some( + const hasStructuredPayload = [taskId, dispatchId, outcome, filesModified, reportPath, phase].some( (value) => value !== undefined ) if (!hasStructuredPayload) { @@ -62,6 +63,15 @@ export function getRemoteOrchestrationPayload(flags: RemoteFlags): string | unde if (dispatchId) { payload.dispatchId = dispatchId } + if (outcome) { + if (outcome !== 'succeeded' && outcome !== 'failed') { + throw new RemoteCliArgumentError( + 'invalid_argument', + 'Invalid --outcome. Expected succeeded or failed.' + ) + } + payload.outcome = outcome + } if (filesModified) { payload.filesModified = filesModified .split(',') diff --git a/src/shared/agent-prompt-injection.test.ts b/src/shared/agent-prompt-injection.test.ts index 5ad3e42d4d1..54b4d6cb544 100644 --- a/src/shared/agent-prompt-injection.test.ts +++ b/src/shared/agent-prompt-injection.test.ts @@ -4,6 +4,7 @@ import { AGENT_PROMPT_BRACKETED_PASTE_START, buildAgentPromptPasteBytes, buildAgentPromptSubmitBytes, + getAgentPromptSubmitDelayMs, iterateAgentPromptPasteChunks, sanitizeAgentPromptText } from './agent-prompt-injection' @@ -23,6 +24,12 @@ describe('agent prompt injection bytes', () => { expect(buildAgentPromptSubmitBytes()).toBe('\r') }) + it('gives Windows ConPTY more time to render before submit', () => { + expect(getAgentPromptSubmitDelayMs('win32')).toBe(1_500) + expect(getAgentPromptSubmitDelayMs('darwin')).toBe(500) + expect(getAgentPromptSubmitDelayMs('linux')).toBe(500) + }) + it('sanitizes embedded escape bytes before framing', () => { const bytes = buildAgentPromptPasteBytes('before\x1b[201~after\x1b') expect(bytes).toBe(`${BEGIN}before[201~after${END}`) diff --git a/src/shared/agent-prompt-injection.ts b/src/shared/agent-prompt-injection.ts index 1d57fb0607c..3db9d1cd39d 100644 --- a/src/shared/agent-prompt-injection.ts +++ b/src/shared/agent-prompt-injection.ts @@ -4,9 +4,17 @@ export const AGENT_PROMPT_BRACKETED_PASTE_START = '\x1b[200~' export const AGENT_PROMPT_BRACKETED_PASTE_END = '\x1b[201~' export const AGENT_PROMPT_SUBMIT = '\r' -// Why: Codex/Claude can need a render turn after bracketed-paste end before -// Enter is accepted as submit, not paste content. Match the proven runtime gap. -export const AGENT_PROMPT_SUBMIT_DELAY_MS = 500 +const DEFAULT_AGENT_PROMPT_SUBMIT_DELAY_MS = 500 +const WINDOWS_AGENT_PROMPT_SUBMIT_DELAY_MS = 1_500 + +// Why: ConPTY renders long bracketed pastes more slowly; an early Enter leaves the task in the agent input buffer. +export function getAgentPromptSubmitDelayMs(platform: NodeJS.Platform): number { + return platform === 'win32' + ? WINDOWS_AGENT_PROMPT_SUBMIT_DELAY_MS + : DEFAULT_AGENT_PROMPT_SUBMIT_DELAY_MS +} + +export const AGENT_PROMPT_SUBMIT_DELAY_MS = getAgentPromptSubmitDelayMs(process.platform) const ESCAPE = '\x1b' const INERT_ESCAPE = '' diff --git a/src/shared/orchestration-rpc-contract.test.ts b/src/shared/orchestration-rpc-contract.test.ts new file mode 100644 index 00000000000..90ba326d919 --- /dev/null +++ b/src/shared/orchestration-rpc-contract.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from 'vitest' +import { + isOrchestrationMutation, + isRetiredOrchestrationMethod, + orchestrationMigrationData +} from './orchestration-rpc-contract' + +describe('orchestration RPC contract', () => { + it.each([ + ['orchestration.runCreate', {}], + ['orchestration.runUse', {}], + ['orchestration.send', {}], + ['orchestration.reply', {}], + ['orchestration.taskCreate', {}], + ['orchestration.taskUpdate', {}], + ['orchestration.dispatch', {}], + ['orchestration.workerStart', {}], + ['orchestration.workerStop', {}], + ['orchestration.workerAbandon', {}], + ['orchestration.ask', {}], + ['orchestration.gateCreate', {}], + ['orchestration.gateResolve', {}], + ['orchestration.reset', {}], + ['orchestration.federationAttachStart', {}], + ['orchestration.federationAck', {}], + ['orchestration.federationImport', {}], + ['orchestration.federationStop', {}], + ['orchestration.check', {}], + ['orchestration.check', { wait: true }], + ['orchestration.check', { unread: true }], + ['orchestration.check', { peek: true, ack: 'delivery_1' }], + ['orchestration.run', {}], + ['orchestration.runStop', {}] + ])('classifies %s as a mutation', (method, params) => { + expect(isOrchestrationMutation(method, params)).toBe(true) + }) + + it.each([ + ['orchestration.runList', {}], + ['orchestration.runShow', {}], + ['orchestration.runCurrent', {}], + ['orchestration.inbox', {}], + ['orchestration.taskList', {}], + ['orchestration.dispatchShow', {}], + ['orchestration.gateList', {}], + ['orchestration.workerShow', {}], + ['orchestration.workerRead', {}], + ['orchestration.federationPull', {}], + ['orchestration.federationShow', {}], + ['orchestration.federationRead', {}], + ['orchestration.federationReadOutput', {}], + ['orchestration.dispatch', { dryRun: true }], + ['orchestration.check', { peek: true }], + ['orchestration.check', { all: true }], + ['orchestration.check', { unread: false }] + ])('classifies %s as read-only', (method, params) => { + expect(isOrchestrationMutation(method, params)).toBe(false) + }) + + it('keeps the retired scheduler methods explicit', () => { + expect(isRetiredOrchestrationMethod('orchestration.run')).toBe(true) + expect(isRetiredOrchestrationMethod('orchestration.runStop')).toBe(true) + expect(isRetiredOrchestrationMethod('orchestration.runCreate')).toBe(false) + }) + + it('returns executable skill recovery without hardcoding a binary name', () => { + expect(orchestrationMigrationData('client_contract_missing')).toMatchObject({ + reason: 'client_contract_missing', + effectsApplied: false, + requiredContractVersion: 1, + guide: { topic: 'orchestration', full: true }, + nextCommandArgs: ['skills', 'get', 'orchestration', '--full'] + }) + }) +}) diff --git a/src/shared/orchestration-rpc-contract.ts b/src/shared/orchestration-rpc-contract.ts new file mode 100644 index 00000000000..2fb372366cc --- /dev/null +++ b/src/shared/orchestration-rpc-contract.ts @@ -0,0 +1,119 @@ +import { ORCHESTRATION_CONTRACT_VERSION } from './protocol-version' + +export type OrchestrationMigrationReason = + | 'client_contract_missing' + | 'client_contract_unsupported' + | 'runtime_capability_missing' + | 'command_retired' + +export const ORCHESTRATION_SKILL_COMMAND_ARGS = [ + 'skills', + 'get', + 'orchestration', + '--full' +] as const + +const ORCHESTRATION_MUTATION_METHODS = new Set([ + 'orchestration.runCreate', + 'orchestration.runUse', + 'orchestration.send', + 'orchestration.reply', + 'orchestration.taskCreate', + 'orchestration.taskUpdate', + 'orchestration.dispatch', + 'orchestration.workerStart', + 'orchestration.workerStop', + 'orchestration.workerAbandon', + 'orchestration.ask', + 'orchestration.gateCreate', + 'orchestration.gateResolve', + 'orchestration.reset', + 'orchestration.federationAttachStart', + 'orchestration.federationAck', + 'orchestration.federationImport', + 'orchestration.federationStop' +]) + +const RETIRED_ORCHESTRATION_METHODS = new Set(['orchestration.run', 'orchestration.runStop']) + +export function isRetiredOrchestrationMethod(method: string): boolean { + return RETIRED_ORCHESTRATION_METHODS.has(method) +} + +export function isOrchestrationMutation(method: string, params: unknown): boolean { + if (isRetiredOrchestrationMethod(method)) { + return true + } + if (method === 'orchestration.check') { + if (hasStringProperty(params, 'ack')) { + return true + } + return !isExplicitReadOnlyCheck(params) + } + if (method === 'orchestration.dispatch') { + return !hasTrueProperty(params, 'dryRun') + } + return ORCHESTRATION_MUTATION_METHODS.has(method) +} + +export function orchestrationSkillRecoveryData(): { + effectsApplied: false + guide: { topic: 'orchestration'; full: true } + nextCommandArgs: typeof ORCHESTRATION_SKILL_COMMAND_ARGS + nextSteps: string[] +} { + return { + effectsApplied: false, + guide: { topic: 'orchestration', full: true }, + nextCommandArgs: ORCHESTRATION_SKILL_COMMAND_ARGS, + nextSteps: [ + 'Using this same Orca CLI executable, run: skills get orchestration --full', + 'Read the returned guide completely and do not retry the previous command unchanged.' + ] + } +} + +export function orchestrationMigrationData(reason: OrchestrationMigrationReason): ReturnType< + typeof orchestrationSkillRecoveryData +> & { + reason: OrchestrationMigrationReason + requiredContractVersion: number +} { + return { + reason, + requiredContractVersion: ORCHESTRATION_CONTRACT_VERSION, + ...orchestrationSkillRecoveryData() + } +} + +function isExplicitReadOnlyCheck(params: unknown): boolean { + return ( + hasTrueProperty(params, 'peek') || + hasTrueProperty(params, 'all') || + hasFalseProperty(params, 'unread') + ) +} + +function hasStringProperty(value: unknown, property: string): boolean { + return ( + typeof value === 'object' && + value !== null && + typeof (value as Record)[property] === 'string' + ) +} + +function hasTrueProperty(value: unknown, property: string): boolean { + return ( + typeof value === 'object' && + value !== null && + (value as Record)[property] === true + ) +} + +function hasFalseProperty(value: unknown, property: string): boolean { + return ( + typeof value === 'object' && + value !== null && + (value as Record)[property] === false + ) +} diff --git a/src/shared/orchestration-worker-output.ts b/src/shared/orchestration-worker-output.ts new file mode 100644 index 00000000000..250e8ccded9 --- /dev/null +++ b/src/shared/orchestration-worker-output.ts @@ -0,0 +1,65 @@ +import type { AgentProviderSessionMetadata } from './agent-session-resume' +import type { AgentType, NativeChatMessage } from './native-chat-types' +import type { RuntimeTerminalRead, RuntimeTerminalState } from './runtime-types' + +export const ORCHESTRATION_WORKER_READ_SOURCES = ['auto', 'transcript', 'terminal'] as const +export type OrchestrationWorkerReadSource = (typeof ORCHESTRATION_WORKER_READ_SOURCES)[number] + +export const ORCHESTRATION_WORKER_READ_FALLBACK_REASONS = [ + 'provider_unsupported', + 'session_not_reported', + 'transcript_missing', + 'transcript_unreadable', + 'transcript_parse_failed', + 'remote_capability_unavailable' +] as const +export type OrchestrationWorkerReadFallbackReason = + (typeof ORCHESTRATION_WORKER_READ_FALLBACK_REASONS)[number] + +export type ExactWorkerProviderSession = { + paneKey: string + processIncarnation: string + agent: AgentType + providerSession: AgentProviderSessionMetadata + observedAt: number +} + +export type OrchestrationWorkerTranscriptPage = { + messages: NativeChatMessage[] + nextCursor: string + limited: boolean + returnedMessageCount: number +} + +export type OrchestrationWorkerReadTranscriptResult = { + dispatchId: string + source: 'transcript' + sourceIdentity: string + provider: AgentType + transcript: OrchestrationWorkerTranscriptPage + cursor: string + status: { + worker: string + terminal: RuntimeTerminalState + } + fallbackReason: null + warnings: string[] +} + +export type OrchestrationWorkerReadTerminalResult = { + dispatchId: string + source: 'terminal' + sourceIdentity: string + terminal: RuntimeTerminalRead + cursor: string | null + status: { + worker: string + terminal: RuntimeTerminalState + } + fallbackReason: OrchestrationWorkerReadFallbackReason | null + warnings: string[] +} + +export type OrchestrationWorkerReadResult = + | OrchestrationWorkerReadTranscriptResult + | OrchestrationWorkerReadTerminalResult diff --git a/src/shared/protocol-version.ts b/src/shared/protocol-version.ts index eeff9fefd49..7ba2874c16f 100644 --- a/src/shared/protocol-version.ts +++ b/src/shared/protocol-version.ts @@ -27,6 +27,12 @@ export const PROJECT_HOST_SETUP_RUNTIME_CAPABILITY = 'project-host-setup.v1' as export const TASK_SOURCE_CONTEXT_RUNTIME_CAPABILITY = 'task-source-context.v1' as const export const WORKSPACE_RUN_CONTEXT_RUNTIME_CAPABILITY = 'workspace-run-context.v1' as const export const REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY = 'remote-runtime.shared-control.v1' as const +export const ORCHESTRATION_FEDERATION_RUNTIME_CAPABILITY = 'orchestration.federation.v1' as const +export const ORCHESTRATION_FEDERATION_CONTROL_MAIL_RUNTIME_CAPABILITY = + 'orchestration.federation-control-mail.v1' as const +export const ORCHESTRATION_FEDERATION_CONTROL_MAIL_PROTOCOL_VERSION = 2 as const +export const ORCHESTRATION_CONTRACT_VERSION = 1 as const +export const ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY = 'orchestration.contract.v1' as const export const FOLDER_WORKSPACE_PATH_STATUS_RUNTIME_CAPABILITY = 'folder-workspace.path-status.v1' as const export const LINEAR_ISSUE_ATTRIBUTE_FILTER_RUNTIME_CAPABILITY = @@ -71,6 +77,9 @@ export const RUNTIME_CAPABILITIES = [ 'runtime.status.compat.v1', 'runtime.environments.v1', REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY, + ORCHESTRATION_FEDERATION_RUNTIME_CAPABILITY, + ORCHESTRATION_FEDERATION_CONTROL_MAIL_RUNTIME_CAPABILITY, + ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY, 'browser.screencast.v1', 'terminal.binary-stream.v1', 'terminal.multiplex.v1', diff --git a/src/shared/remote-runtime-client.test.ts b/src/shared/remote-runtime-client.test.ts index 4c09c1e909d..fa15ca82467 100644 --- a/src/shared/remote-runtime-client.test.ts +++ b/src/shared/remote-runtime-client.test.ts @@ -241,6 +241,35 @@ describe('sendRemoteRuntimeRequest', () => { }) }) + it('sends orchestration authentication fields in the admitted encrypted request', async () => { + let receivedRequest: Record | null = null + const server = await createOneShotServer({ + onRequest: (request) => { + receivedRequest = request + } + }) + + await sendRemoteRuntimeRequest( + server.pairing, + 'orchestration.federationControl', + { dispatch: 'ctx_1' }, + 1000, + { + orchestrationCapability: 'capability', + orchestrationContractVersion: 1, + orchestrationRequestId: 'mutation_1' + } + ) + + expect(receivedRequest).toMatchObject({ + method: 'orchestration.federationControl', + params: { dispatch: 'ctx_1' }, + orchestrationCapability: 'capability', + orchestrationContractVersion: 1, + orchestrationRequestId: 'mutation_1' + }) + }) + it('detaches one-shot socket listeners after a successful response', async () => { const offSpy = vi.spyOn(WebSocketClient.prototype, 'off') try { @@ -387,6 +416,7 @@ async function createClosingServer( async function createOneShotServer( options: { response?: (requestId: string) => unknown + onRequest?: (request: Record) => void } = {} ): Promise<{ pairing: PairingOffer }> { const serverKeyPair = generateKeyPair() @@ -422,7 +452,8 @@ async function createOneShotServer( return } - const request = JSON.parse(plaintext) as { id: string } + const request = JSON.parse(plaintext) as { id: string } & Record + options.onRequest?.(request) const key = sharedKey const keepalive = setInterval(() => { sendEncrypted(ws, key, { _keepalive: true }) diff --git a/src/shared/remote-runtime-client.ts b/src/shared/remote-runtime-client.ts index da1fff5bb1b..4eaaeaf5bb8 100644 --- a/src/shared/remote-runtime-client.ts +++ b/src/shared/remote-runtime-client.ts @@ -18,6 +18,7 @@ import { import { isKeepaliveFrame, RuntimeRpcEnvelopeSchema, + type RuntimeOrchestrationEnvelope, type RuntimeRpcResponse } from './runtime-rpc-envelope' // Re-export so existing value importers of `RemoteRuntimeClientError` are @@ -81,7 +82,8 @@ export async function sendRemoteRuntimeRequest( pairing: PairingOffer, method: string, params: unknown, - timeoutMs: number + timeoutMs: number, + envelope?: RuntimeOrchestrationEnvelope ): Promise> { if (!isSafeTimerDelayMs(timeoutMs)) { throw new RemoteRuntimeClientError( @@ -96,11 +98,14 @@ export async function sendRemoteRuntimeRequest( }) const pendingRequest = { preparedRequest: prepareRemoteRuntimeRequest(new Map(), () => - serializeRemoteRuntimeRpcRequest({ - requestId, + serializeRemoteRuntimePayload({ + id: requestId, deviceToken: pairing.deviceToken, method, - params + params, + orchestrationCapability: envelope?.orchestrationCapability, + orchestrationContractVersion: envelope?.orchestrationContractVersion, + orchestrationRequestId: envelope?.orchestrationRequestId }) ) } diff --git a/src/shared/runtime-rpc-envelope.ts b/src/shared/runtime-rpc-envelope.ts index 8884d63313c..d12c762dd9d 100644 --- a/src/shared/runtime-rpc-envelope.ts +++ b/src/shared/runtime-rpc-envelope.ts @@ -74,6 +74,12 @@ export type RuntimeRpcFailure = { export type RuntimeRpcResponse = RuntimeRpcSuccess | RuntimeRpcFailure +export type RuntimeOrchestrationEnvelope = { + orchestrationCapability?: string + orchestrationContractVersion?: number + orchestrationRequestId?: string +} + export type RuntimeRpcKeepaliveFrame = z.infer export function isKeepaliveFrame(frame: unknown): frame is RuntimeRpcKeepaliveFrame { diff --git a/src/shared/types.ts b/src/shared/types.ts index 2983cbf32ac..e34ed5bc26d 100644 --- a/src/shared/types.ts +++ b/src/shared/types.ts @@ -2251,6 +2251,13 @@ export type CreateWorktreeResult = { workspaceLineage?: WorkspaceLineage | null warnings?: WorktreeLineageWarning[] setup?: WorktreeSetupLaunch + setupReceipt?: { + requested: 'run' | 'skip' | 'inherit' + hookFound: boolean + startupPolicy: 'start-immediately' | 'wait-for-setup' + state: 'running' | 'skipped' | 'not_configured' | 'spawn_failed' + terminalHandle?: string + } defaultTabs?: WorktreeDefaultTabsLaunch warning?: string initialBaseStatus?: WorktreeBaseStatusEvent