From 1cf24e5e6fd9f33311689e7dd358ab0873fe309a Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 22 Oct 2026 14:43:06 -0700 Subject: [PATCH] fix(orchestration): let a worker start helpers on the Run it works for (#29205) * fix(orchestration): let a worker start helpers on the Run it works for A dispatched worker could only start helper workers by taking a coordinator seat on a new Run, and taking a seat moved its inbox. worker-start now also accepts a worker on an active Dispatch: its helpers run in the same Run, report to it (worker_done, heartbeat, escalation, status, ask), and it answers their questions. The worker's own mailbox never moves. When it settles, whatever its helpers sent that it never handled goes to the Run's coordinator. Authority comes from the worker's own Dispatch, re-checked inside the reservation transaction. Helpers are fresh agents on this host; a worker may retry only its own helper's latest attempt. Remote workers keep the old own-Run instructions. No saved field or wire change. * fix(orchestration): read helper rows without type assertions * fix(orchestration): hand a settled helper creator's mail to the Run as live delivery When a worker that started helpers settled, its helpers' unhandled mail was re-addressed to the Run by settlement SQL alone. Nothing woke the Run's coordinator, delivered rows kept their old delivery stamp so pointers and the restart scan skipped them, payload-free status reports were left behind because selection required payload.dispatchId, and a question asked while the creator was being stopped stayed with the stopped creator. - Select handed-back mail by its recipient mailbox and Run (unread reports plus pending questions, even ones already read), reset delivery and pointer state on the moved rows, and fence the old mailbox's Deliveries. - After the settling transaction ends, the runtime re-reads exactly the moved rows and wakes the Run with their real types; a rollback wakes no one and a failure falls back to the existing repoint retry. Restart recovery is the existing unread-undelivered scan. - Hand mail back again at the final stop settlement, which also closes the stopped Dispatch's own questions asked during the stop. - A worker_done receipt replay re-offers the handed-back targets. - Scope markAsDelivered to the mailbox it pointed, so a late pointer stamp from the old owner cannot hide mail now owned by the Run. * test(orchestration): drop an unused inbox type * fix(orchestration): read only the Run when handing a settled worker's mail back Every settlement runs the hand-back, so a full dispatch row read there added a statement per settlement. Also expect the mailbox-scoped delivered stamp in two delivery tests from main. * fix(orchestration): re-offer handed-back Run mail on every worker_done replay A helper creator's worker_done hands its helpers' unread reports and pending questions to the Run. The send stored the Run's extra wake targets in the mutation receipt, but the executor rewrites that receipt on completion and on each replay, so an identical retry only woke the original destination. Derive the targets at replay time instead: for the settled Dispatch (when it has same-Run helpers), notify run: for each type of unread report or pending question there. Notification only; nothing is re-settled or inserted. The receipt augmentation is removed. * style(orchestration): format with the formatter version main now pins --- skill-guides/orchestration.md | 2 + .../references/coordinator-loop.md | 2 + .../references/messaging-and-gates.md | 4 +- .../references/worker-contract.md | 15 + src/cli/bundled-skill-guides.ts | 10 +- ...untime-test-orchestration-messages.spec.ts | 8 +- .../db-message-timestamp.test.ts | 2 +- .../runtime/orchestration/db-messages.test.ts | 2 +- .../db-undelivered-mailboxes.test.ts | 2 +- .../db/messages/helper-mailbox-hand-back.ts | 95 +++++ .../db/messages/message-inbox.ts | 11 +- .../orchestration/db/orchestration-db.ts | 5 + .../db/questions/question-threads.ts | 39 ++- .../worker-dispatch/worker-dispatch-start.ts | 12 + .../worker-dispatch/worker-dispatch-stop.ts | 3 + .../orchestration/dispatch-report-mailbox.ts | 41 +++ .../orchestration/mailbox-pointer-stage.ts | 2 +- .../message-batch-atomicity.test.ts | 6 +- .../runtime/orchestration/preamble.test.ts | 10 +- src/main/runtime/orchestration/preamble.ts | 30 +- ...tructured-mailbox-pointer-delivery.test.ts | 21 +- .../structured-mailbox-pointer-delivery.ts | 4 +- .../orchestration/federation/federation.ts | 1 + .../orchestration/messaging/ask-methods.ts | 10 +- .../orchestration/messaging/check.test.ts | 2 +- .../messaging/helper-question-answerer.ts | 38 ++ .../messaging/message-methods.ts | 23 ++ .../orchestration/messaging/send-methods.ts | 30 +- .../worker/local-worker-start.ts | 8 +- .../worker/worker-start-authority.ts | 82 +++++ .../orchestration/worker/worker-topology.ts | 4 +- .../methods/orchestration/worker/workers.ts | 20 +- .../structured-chat-worker-helpers.test.ts | 331 ++++++++++++++++++ .../layout/terminal-resize-listeners.ts | 3 +- .../automation-operations.ts | 31 ++ 35 files changed, 851 insertions(+), 58 deletions(-) create mode 100644 src/main/runtime/orchestration/db/messages/helper-mailbox-hand-back.ts create mode 100644 src/main/runtime/orchestration/dispatch-report-mailbox.ts create mode 100644 src/main/runtime/rpc/methods/orchestration/messaging/helper-question-answerer.ts create mode 100644 src/main/runtime/rpc/methods/orchestration/worker/worker-start-authority.ts create mode 100644 src/main/runtime/structured-chat-worker-helpers.test.ts diff --git a/skill-guides/orchestration.md b/skill-guides/orchestration.md index 728eeb473a0..1ab928de921 100644 --- a/skill-guides/orchestration.md +++ b/skill-guides/orchestration.md @@ -116,6 +116,8 @@ Use `task-create` plus `worker-start --task ` for planned fan-out with dependencies or a retry of a known Task. Use dependencies only for real ordering and prefer parallel waves over chains deeper than three or four steps; nested workers obey the depth limit, and a new Run does not reset the caller's depth. +A dispatched worker starts its own helpers with `worker-start --spec` on the Run +it works for, without `run-create`; the helpers report to that worker. A consuming `check` names its caller with `--terminal `, never `--from`; omit it inside the coordinator's own Orca terminal. It returns the bound Run's diff --git a/skill-guides/orchestration/references/coordinator-loop.md b/skill-guides/orchestration/references/coordinator-loop.md index 164f91e4548..a82925526ff 100644 --- a/skill-guides/orchestration/references/coordinator-loop.md +++ b/skill-guides/orchestration/references/coordinator-loop.md @@ -18,6 +18,8 @@ ORCA orchestration task-list --ready --brief --json `spec_truncated` identifies shortened rows. Omit it when full specs are needed or when an older CLI rejects the flag. A nested worker must respect `nested_worker_depth_exceeded`; creating another Run does not reset depth. +Helpers a worker starts appear in this Run's `worker-list` with their +`creatorDispatchId`, and report to that worker rather than to you. ## Launch preferences diff --git a/skill-guides/orchestration/references/messaging-and-gates.md b/skill-guides/orchestration/references/messaging-and-gates.md index a05ccf7786c..bd6a815aa04 100644 --- a/skill-guides/orchestration/references/messaging-and-gates.md +++ b/skill-guides/orchestration/references/messaging-and-gates.md @@ -55,7 +55,9 @@ Dispatch lifecycle messages never target groups. ## Questions and gates A worker uses `ask`; its timeout leaves one durable question pending, which the -worker resumes by message ID. The coordinator answers that message with `reply`. +worker resumes by message ID. The coordinator answers that message with `reply`; +a helper's question goes to the worker that started it, which answers it the +same way. Use a gate only for a coordinator-owned Task-DAG decision: diff --git a/skill-guides/orchestration/references/worker-contract.md b/skill-guides/orchestration/references/worker-contract.md index a2315b7ceee..4ee967dd228 100644 --- a/skill-guides/orchestration/references/worker-contract.md +++ b/skill-guides/orchestration/references/worker-contract.md @@ -53,6 +53,21 @@ you were replaced; `consumer_fenced` is the only way you learn that. If `send` or `ask` returns `consumer_fenced`, the command ran from another party's terminal (a coordinator or another worker); run it from your own terminal. +## Helper workers + +When the preamble has a SUB-DISPATCH section, start helpers on your own +Dispatch's Run. Do not `run-create` or `run-use` for them. + +```text +ORCA orchestration worker-start --from --spec "" --worktree current --agent --json +ORCA orchestration reply --from --id --body "" +``` + +Their questions, escalations, and `worker_done` arrive in your own `check`, next +to your coordinator's mail. Answer their questions with `reply`, and wait for +every helper's `worker_done` before sending yours. After you settle, a helper +still running reports to your coordinator's Run. + ## Escalation Escalate only before completion and only when the coordinator must intervene: diff --git a/src/cli/bundled-skill-guides.ts b/src/cli/bundled-skill-guides.ts index 71c7339d29a..e4f0b68092c 100644 --- a/src/cli/bundled-skill-guides.ts +++ b/src/cli/bundled-skill-guides.ts @@ -72,13 +72,13 @@ const ORCA_PER_WORKSPACE_ENV_SSH_HOST_REFERENCE_MARKDOWN = "# SSH connection mod const ORCA_PER_WORKSPACE_ENV_WINDOWS_SCRIPTS_REFERENCE_MARKDOWN = "# Windows local-side scripts\n\nLoad this when the user's desktop is Windows and you are scaffolding the local-side scripts. A bare\n`.sh` will not execute there. Either require WSL or Git Bash and point `orca.yaml` at a launcher such\nas `bash ./scripts/orca-vm/.sh` through a `.cmd` file, or scaffold PowerShell equivalents.\n\nThe remote-side commands you run inside the Linux environment stay bash regardless of the desktop OS.\n\n```powershell\n#requires -Version 5\n$ErrorActionPreference = 'Stop'\n# resolve env→state→fallback; run the provider CLI / ssh the same way;\n# capture provider output; build the result object for the chosen mode and write ONE line of JSON to stdout.\n# Orca-server mode: @{ schemaVersion=1; pairingCode=$pairingCode; projectRoot=$projectRoot; userData=@{...} }\n# SSH mode: @{ schemaVersion=1; connection=@{ type=\"ssh\"; projectRoot=$projectRoot;\n# target=@{ label=$label; host=$host; port=$port; username=$user } } }\n($result | ConvertTo-Json -Compress -Depth 6)\n# progress/errors → Write-Error / the error stream, never stdout.\n```\n\nThe doctor's executable-bit check is a POSIX concept and is skipped on Windows, so a script that is\nunusable on the user's machine for a different reason still has to be caught by the `--provision`\nself-test.\n" // oxfmt-ignore -const ORCHESTRATION_MARKDOWN = "---\nname: orchestration\ndescription: >-\n Coordinate supervised Orca workers: threaded messages, blocking ask/reply,\n task dispatch, worker_done/escalation waits, task DAGs, decision gates,\n coordinator loops, and decomposing work across agents. Use `orca-cli` for full\n ownership handoffs — \"hand off\", \"handoff\", \"handover\", \"give this to another\n agent\", \"another worktree\" — unless asked to supervise, monitor, or coordinate\n a DAG, and for terminal control, lightweight terminal prompts, shell commands,\n Orca worktree management, and reading or waiting on terminals.\n---\n\n# Orca orchestration\n\nOrchestration is Orca's structured coordination layer. It records who owns work,\nwhich attempt is authoritative, and when supervised work has settled.\n\n## Outcome\n\n**Result:** every in-scope Task has one explicit outcome and every settled worker\nterminal has a next owner or cleanup decision. **Next consumer:** the user who\nrequested supervision. **Done:** all expected Dispatches have settled, every\ndelivered message was processed before acknowledgment, each settled worker was\nreused, explicitly retained, or released, and the turn ends only when the report\nto that user names, per Task, its outcome, the evidence behind it, and any\nunresolved blocker.\n\n**Safe failure:** preserve work and authority and report the state as unknown or\n`unverifiable`. Only positive proof of exit authorizes stop, abandon, or retry,\nand only an accepted settlement authorizes release. Every other observation,\nabsence included, is a checkpoint.\n\n## Classify the role\n\n| Current context | Role | Route |\n| ---------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------- | ------------------------------------------------------------------------------ |\n| The user explicitly asks to supervise, monitor, wait for results, track completion, coordinate a DAG, use a decision gate, or manage ask/reply | Coordinator | Use the supervised loop below |\n| The current prompt contains a live injected preamble with Task and Dispatch IDs | Dispatched worker | Follow the preamble and the worker obligations below |\n| The user asks to hand off ownership or start another agent/worktree without supervision | Handoff owner | Use `orca-cli`; create no Run, Task, or Dispatch and do not monitor completion |\n| A message carries a legacy authority label | Compatibility operator | Load the legacy contract reference before any lifecycle mutation |\n| No live preamble and no explicit supervision | Ordinary terminal agent | Do not emit lifecycle messages; use `orca-cli` for terminal/worktree work |\n\nModel or effort selection does not make a handoff supervised: `orca-cli`'s\n`worktree create --agent` and `agent launch` take `--model` and `--effort`, and\nopen a chat or a terminal per the user's Chat UI setting, as `worker-start` does.\nNever substitute a non-Orca subagent tool when Orca orchestration provenance was\nrequested.\n\n## Authority and safety floor\n\n- A Run is a durable namespace and coordinator inbox; it does not schedule or\n place workers. A Task is work. A Dispatch is one authoritative Task attempt.\n- Lifecycle authority comes from the active Dispatch, not a terminal title,\n copied ID, old database row, provider transcript, or visible pane.\n- Workers use the exact executable, handle, Task ID, and Dispatch ID in the live\n preamble, plus any other flag it carries. Never reconstruct, translate, or\n broaden those arguments.\n- After remote start, address the worker by Dispatch ID. The execution host owns\n process, filesystem, transcript, stop, and cleanup facts. Preserve the verdicts\n `live` / `unverifiable` / `exited`; contact loss is not process death.\n- Liveness is layered: `worker-list`'s `projection.liveness` is the fleet verdict\n for the agent; `worker-show`'s `observation.status` is PTY liveness only. A live\n terminal can still hold a dead or stuck agent.\n- Folder workspaces are valid; never require Git or assume a worktree.\n- Clients and remote servers update independently. Treat unknown optional fields\n as absent. A new stream operation requires advertised capability because old\n decoders may silently drop unknown opcodes. Never fall back to local execution\n when remote authority or capability is unproven.\n- Use the executable you used to run `skills get` for the entire run. In the\n examples below, replace `ORCA` with it; do not create a shell variable or run\n `ORCA` literally. If it fails, report that exact error instead of switching.\n- `ORCA status --json` shows your Orca session ID as `caller.orcaSessionId` when you have one.\n- A successful `orchestration send` proves durable enqueue; its wake or nudge is\n best-effort attention only and does not prove the recipient read or accepted it.\n\n## Worker obligations\n\nThe injected preamble is authoritative. A dispatched worker must:\n\n1. Do only the current Task and use the preamble's `ask` command for a blocking\n coordinator question. Never open a local question TUI the coordinator cannot\n answer. Resume the same message ID after an ask timeout.\n2. Send heartbeats only at the cadence in the preamble. A heartbeat proves\n liveness, not completion.\n3. Read coordinator follow-ups at each natural checkpoint — before starting a\n new file, after a test run — and once more immediately before `worker_done`:\n `ORCA orchestration check --terminal --json`.\n4. Send `worker_done` exactly once, from the dispatched terminal, with a\n three-sentence executive summary, both lifecycle IDs, and explicit\n `--outcome succeeded` or `--outcome failed`. Never encode failure only in prose.\n5. Append `--files-modified` and `--report-path` only with real values when\n applicable. After `worker_done`, end the dispatched turn and idle; do not poll\n or start new work.\n\nA direct user instruction after completion starts new user-owned work and takes\nprecedence over the idle rule. Do not reuse the settled lifecycle IDs.\n\n## Canonical supervised loop\n\nConfirm the runtime, bind one Run, and start the full independent wave before\nwaiting. `worker-start --spec` creates the Task and its attempt in one call:\n\n```text\nORCA status --json\nORCA orchestration run-create --objective \"\" --json\nORCA orchestration worker-start --spec \"\" --worktree current --agent codex --json\nORCA orchestration worker-start --spec \"\" --worktree current --agent claude --json\nORCA orchestration check --wait --types \"worker_done,escalation,question\" --timeout-ms 900000 --json\n```\n\nIf `worker-start` exits non-zero, do not relaunch. Read the receipt's\n`failedStage` and `residualResources`, then load\n`references/recovery-and-cleanup.md`.\n\nUse `task-create` plus `worker-start --task ` for planned fan-out with\ndependencies or a retry of a known Task. Use dependencies only for real ordering\nand prefer parallel waves over chains deeper than three or four steps; nested\nworkers obey the depth limit, and a new Run does not reset the caller's depth.\n\nA consuming `check` names its caller with `--terminal `, never `--from`;\nomit it inside the coordinator's own Orca terminal. It returns the bound Run's\noldest FIFO Delivery and replays that batch until acknowledged. Process every\nmessage: reply to questions, validate each `worker_done` against the expected\nactive Dispatch, and decide each settled terminal's next owner before the ack:\n\n```text\nORCA orchestration reply --id --body \"\" --json\nORCA orchestration worker-release --dispatch --json\nORCA orchestration check --ack --wait --types \"worker_done,escalation,question\" --timeout-ms 900000 --json\n```\n\nKeep waiting until every expected Dispatch settles. A timeout or empty result is\na checkpoint, not a failure. Do not stop, retry, release, or launch a duplicate\neditor without the positive proof `## Outcome` requires.\n\nAfter three consecutive empty waits, stop waiting blindly and enumerate with\n`ORCA orchestration worker-list --include-remote --json` (defaults to the bound\nRun; `--run ` overrides; the receipt's `scope` names which), acting on\neach row's `projection.attention` categories, `projection.attention.requiresAction`, and literal `projection.nextAction` argv.\nRows come newest first and page at 100: while `page.hasMore`, follow `page.nextCursor` with `--cursor `.\nA `none` `nextAction` has no argv to run: read `liveness.reason` and keep waiting\nwith `check --wait`. Absence never earns an argv; settlement and pending work still do.\nLeave the wait only on positive proof the agent stopped: `exited` liveness, the\nworker's own observation of process exit, or a transcript whose final agent turn\nsent no `worker_done`. Then load `references/recovery-and-cleanup.md` and choose\n`worker-stop` or `worker-abandon` explicitly. `unverifiable` is absence,\nincluding when `worker-show` reports `agentWait` null. Absence never authorizes\nstop, abandon, retry, or release; keep waiting or inspect.\n\n`worker-start` is the normal path, composing placement, terminal readiness,\nprompt injection, and supervised resource ownership. `dispatch --inject` leaves\nan operator-created process unsupervised and is only for an expressiveness gap.\n\n## Task-spec contract\n\nEvery Task spec must be self-contained and name:\n\n- **Target:** the files, component, or environment in scope.\n- **Change:** the concrete result to produce.\n- **Constraints:** invariants, compatibility rules, and do-not-touch boundaries.\n- **Ownership:** what this worker may edit and any coordination boundary.\n- **Observable acceptance:** the test, output, or evidence that proves completion.\n\n## Completion accounting\n\nAfter an accepted success or failure report, immediately do exactly one:\n\n1. Reuse the same proven agent terminal for an immediate follow-up Dispatch.\n2. Record user-requested retention with `worker-retain`.\n3. Run `worker-release`.\n\nRelease is post-settlement cleanup, not cancellation. Only an accepted\nsettlement authorizes it; no other observation does. If release is uncertain,\nfollow its exact recovery receipt and never substitute `terminal close`.\n\nA valid `worker_done` settles the Task and Dispatch automatically; do not follow\nit with `task-update --status completed`. Enumerate the terminals still owing a\ndecision with `worker-list --run --terminal-state reclaimable --json`,\nand do not end the coordinator turn until it returns none.\n\n## Conditional references\n\nThis compact guide is sufficient for the normal local loop. At an action gate\nbelow, run `ORCA skills get orchestration --reference references/.md` and\nread only that document; `--references` lists the names. If the CLI rejects\n`--reference`, run `ORCA skills get orchestration --full` once instead: it\nreturns this exact kernel and every reference, so read only the named one. If an\nolder CLI rejects `--full`, keep this kernel's safety floor, use that command's\n`--help`, and never guess newer flags.\n\n| Action gate | Bundled reference |\n| ------------------------------------------------------------------------------------------------------------- | ----------------------------------------- |\n| Expanded DAG waves, launch model/effort, same-terminal reuse, or review ownership | `references/coordinator-loop.md` |\n| You are a dispatched worker and the live preamble does not answer your question, or `check` returned an error | `references/worker-contract.md` |\n| New worktree, exact workspace, SSH, WSL, or connected-server placement | `references/placement-and-remote.md` |\n| Inbox replay, follow-up messages, group addresses, or decision gates | `references/messaging-and-gates.md` |\n| Failed/stopped/unknown attempts, retry, stop, abandon, retain, or uncertain release | `references/recovery-and-cleanup.md` |\n| Custom argv or terminal topology that `worker-start` cannot express | `references/low-level-topology.md` |\n| Any legacy label, adopted Run, compatibility receipt, or takeover | `references/legacy-contract-migration.md` |\n\nRetired scheduler commands are not aliases for Run creation. Recovery commands\nmust provide their exact next action; follow it with the same selected executable.\n" +const ORCHESTRATION_MARKDOWN = "---\nname: orchestration\ndescription: >-\n Coordinate supervised Orca workers: threaded messages, blocking ask/reply,\n task dispatch, worker_done/escalation waits, task DAGs, decision gates,\n coordinator loops, and decomposing work across agents. Use `orca-cli` for full\n ownership handoffs — \"hand off\", \"handoff\", \"handover\", \"give this to another\n agent\", \"another worktree\" — unless asked to supervise, monitor, or coordinate\n a DAG, and for terminal control, lightweight terminal prompts, shell commands,\n Orca worktree management, and reading or waiting on terminals.\n---\n\n# Orca orchestration\n\nOrchestration is Orca's structured coordination layer. It records who owns work,\nwhich attempt is authoritative, and when supervised work has settled.\n\n## Outcome\n\n**Result:** every in-scope Task has one explicit outcome and every settled worker\nterminal has a next owner or cleanup decision. **Next consumer:** the user who\nrequested supervision. **Done:** all expected Dispatches have settled, every\ndelivered message was processed before acknowledgment, each settled worker was\nreused, explicitly retained, or released, and the turn ends only when the report\nto that user names, per Task, its outcome, the evidence behind it, and any\nunresolved blocker.\n\n**Safe failure:** preserve work and authority and report the state as unknown or\n`unverifiable`. Only positive proof of exit authorizes stop, abandon, or retry,\nand only an accepted settlement authorizes release. Every other observation,\nabsence included, is a checkpoint.\n\n## Classify the role\n\n| Current context | Role | Route |\n| ---------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------- | ------------------------------------------------------------------------------ |\n| The user explicitly asks to supervise, monitor, wait for results, track completion, coordinate a DAG, use a decision gate, or manage ask/reply | Coordinator | Use the supervised loop below |\n| The current prompt contains a live injected preamble with Task and Dispatch IDs | Dispatched worker | Follow the preamble and the worker obligations below |\n| The user asks to hand off ownership or start another agent/worktree without supervision | Handoff owner | Use `orca-cli`; create no Run, Task, or Dispatch and do not monitor completion |\n| A message carries a legacy authority label | Compatibility operator | Load the legacy contract reference before any lifecycle mutation |\n| No live preamble and no explicit supervision | Ordinary terminal agent | Do not emit lifecycle messages; use `orca-cli` for terminal/worktree work |\n\nModel or effort selection does not make a handoff supervised: `orca-cli`'s\n`worktree create --agent` and `agent launch` take `--model` and `--effort`, and\nopen a chat or a terminal per the user's Chat UI setting, as `worker-start` does.\nNever substitute a non-Orca subagent tool when Orca orchestration provenance was\nrequested.\n\n## Authority and safety floor\n\n- A Run is a durable namespace and coordinator inbox; it does not schedule or\n place workers. A Task is work. A Dispatch is one authoritative Task attempt.\n- Lifecycle authority comes from the active Dispatch, not a terminal title,\n copied ID, old database row, provider transcript, or visible pane.\n- Workers use the exact executable, handle, Task ID, and Dispatch ID in the live\n preamble, plus any other flag it carries. Never reconstruct, translate, or\n broaden those arguments.\n- After remote start, address the worker by Dispatch ID. The execution host owns\n process, filesystem, transcript, stop, and cleanup facts. Preserve the verdicts\n `live` / `unverifiable` / `exited`; contact loss is not process death.\n- Liveness is layered: `worker-list`'s `projection.liveness` is the fleet verdict\n for the agent; `worker-show`'s `observation.status` is PTY liveness only. A live\n terminal can still hold a dead or stuck agent.\n- Folder workspaces are valid; never require Git or assume a worktree.\n- Clients and remote servers update independently. Treat unknown optional fields\n as absent. A new stream operation requires advertised capability because old\n decoders may silently drop unknown opcodes. Never fall back to local execution\n when remote authority or capability is unproven.\n- Use the executable you used to run `skills get` for the entire run. In the\n examples below, replace `ORCA` with it; do not create a shell variable or run\n `ORCA` literally. If it fails, report that exact error instead of switching.\n- `ORCA status --json` shows your Orca session ID as `caller.orcaSessionId` when you have one.\n- A successful `orchestration send` proves durable enqueue; its wake or nudge is\n best-effort attention only and does not prove the recipient read or accepted it.\n\n## Worker obligations\n\nThe injected preamble is authoritative. A dispatched worker must:\n\n1. Do only the current Task and use the preamble's `ask` command for a blocking\n coordinator question. Never open a local question TUI the coordinator cannot\n answer. Resume the same message ID after an ask timeout.\n2. Send heartbeats only at the cadence in the preamble. A heartbeat proves\n liveness, not completion.\n3. Read coordinator follow-ups at each natural checkpoint — before starting a\n new file, after a test run — and once more immediately before `worker_done`:\n `ORCA orchestration check --terminal --json`.\n4. Send `worker_done` exactly once, from the dispatched terminal, with a\n three-sentence executive summary, both lifecycle IDs, and explicit\n `--outcome succeeded` or `--outcome failed`. Never encode failure only in prose.\n5. Append `--files-modified` and `--report-path` only with real values when\n applicable. After `worker_done`, end the dispatched turn and idle; do not poll\n or start new work.\n\nA direct user instruction after completion starts new user-owned work and takes\nprecedence over the idle rule. Do not reuse the settled lifecycle IDs.\n\n## Canonical supervised loop\n\nConfirm the runtime, bind one Run, and start the full independent wave before\nwaiting. `worker-start --spec` creates the Task and its attempt in one call:\n\n```text\nORCA status --json\nORCA orchestration run-create --objective \"\" --json\nORCA orchestration worker-start --spec \"\" --worktree current --agent codex --json\nORCA orchestration worker-start --spec \"\" --worktree current --agent claude --json\nORCA orchestration check --wait --types \"worker_done,escalation,question\" --timeout-ms 900000 --json\n```\n\nIf `worker-start` exits non-zero, do not relaunch. Read the receipt's\n`failedStage` and `residualResources`, then load\n`references/recovery-and-cleanup.md`.\n\nUse `task-create` plus `worker-start --task ` for planned fan-out with\ndependencies or a retry of a known Task. Use dependencies only for real ordering\nand prefer parallel waves over chains deeper than three or four steps; nested\nworkers obey the depth limit, and a new Run does not reset the caller's depth.\nA dispatched worker starts its own helpers with `worker-start --spec` on the Run\nit works for, without `run-create`; the helpers report to that worker.\n\nA consuming `check` names its caller with `--terminal `, never `--from`;\nomit it inside the coordinator's own Orca terminal. It returns the bound Run's\noldest FIFO Delivery and replays that batch until acknowledged. Process every\nmessage: reply to questions, validate each `worker_done` against the expected\nactive Dispatch, and decide each settled terminal's next owner before the ack:\n\n```text\nORCA orchestration reply --id --body \"\" --json\nORCA orchestration worker-release --dispatch --json\nORCA orchestration check --ack --wait --types \"worker_done,escalation,question\" --timeout-ms 900000 --json\n```\n\nKeep waiting until every expected Dispatch settles. A timeout or empty result is\na checkpoint, not a failure. Do not stop, retry, release, or launch a duplicate\neditor without the positive proof `## Outcome` requires.\n\nAfter three consecutive empty waits, stop waiting blindly and enumerate with\n`ORCA orchestration worker-list --include-remote --json` (defaults to the bound\nRun; `--run ` overrides; the receipt's `scope` names which), acting on\neach row's `projection.attention` categories, `projection.attention.requiresAction`, and literal `projection.nextAction` argv.\nRows come newest first and page at 100: while `page.hasMore`, follow `page.nextCursor` with `--cursor `.\nA `none` `nextAction` has no argv to run: read `liveness.reason` and keep waiting\nwith `check --wait`. Absence never earns an argv; settlement and pending work still do.\nLeave the wait only on positive proof the agent stopped: `exited` liveness, the\nworker's own observation of process exit, or a transcript whose final agent turn\nsent no `worker_done`. Then load `references/recovery-and-cleanup.md` and choose\n`worker-stop` or `worker-abandon` explicitly. `unverifiable` is absence,\nincluding when `worker-show` reports `agentWait` null. Absence never authorizes\nstop, abandon, retry, or release; keep waiting or inspect.\n\n`worker-start` is the normal path, composing placement, terminal readiness,\nprompt injection, and supervised resource ownership. `dispatch --inject` leaves\nan operator-created process unsupervised and is only for an expressiveness gap.\n\n## Task-spec contract\n\nEvery Task spec must be self-contained and name:\n\n- **Target:** the files, component, or environment in scope.\n- **Change:** the concrete result to produce.\n- **Constraints:** invariants, compatibility rules, and do-not-touch boundaries.\n- **Ownership:** what this worker may edit and any coordination boundary.\n- **Observable acceptance:** the test, output, or evidence that proves completion.\n\n## Completion accounting\n\nAfter an accepted success or failure report, immediately do exactly one:\n\n1. Reuse the same proven agent terminal for an immediate follow-up Dispatch.\n2. Record user-requested retention with `worker-retain`.\n3. Run `worker-release`.\n\nRelease is post-settlement cleanup, not cancellation. Only an accepted\nsettlement authorizes it; no other observation does. If release is uncertain,\nfollow its exact recovery receipt and never substitute `terminal close`.\n\nA valid `worker_done` settles the Task and Dispatch automatically; do not follow\nit with `task-update --status completed`. Enumerate the terminals still owing a\ndecision with `worker-list --run --terminal-state reclaimable --json`,\nand do not end the coordinator turn until it returns none.\n\n## Conditional references\n\nThis compact guide is sufficient for the normal local loop. At an action gate\nbelow, run `ORCA skills get orchestration --reference references/.md` and\nread only that document; `--references` lists the names. If the CLI rejects\n`--reference`, run `ORCA skills get orchestration --full` once instead: it\nreturns this exact kernel and every reference, so read only the named one. If an\nolder CLI rejects `--full`, keep this kernel's safety floor, use that command's\n`--help`, and never guess newer flags.\n\n| Action gate | Bundled reference |\n| ------------------------------------------------------------------------------------------------------------- | ----------------------------------------- |\n| Expanded DAG waves, launch model/effort, same-terminal reuse, or review ownership | `references/coordinator-loop.md` |\n| You are a dispatched worker and the live preamble does not answer your question, or `check` returned an error | `references/worker-contract.md` |\n| New worktree, exact workspace, SSH, WSL, or connected-server placement | `references/placement-and-remote.md` |\n| Inbox replay, follow-up messages, group addresses, or decision gates | `references/messaging-and-gates.md` |\n| Failed/stopped/unknown attempts, retry, stop, abandon, retain, or uncertain release | `references/recovery-and-cleanup.md` |\n| Custom argv or terminal topology that `worker-start` cannot express | `references/low-level-topology.md` |\n| Any legacy label, adopted Run, compatibility receipt, or takeover | `references/legacy-contract-migration.md` |\n\nRetired scheduler commands are not aliases for Run creation. Recovery commands\nmust provide their exact next action; follow it with the same selected executable.\n" // oxfmt-ignore -const ORCHESTRATION_FULL_MARKDOWN = "---\nname: orchestration\ndescription: >-\n Coordinate supervised Orca workers: threaded messages, blocking ask/reply,\n task dispatch, worker_done/escalation waits, task DAGs, decision gates,\n coordinator loops, and decomposing work across agents. Use `orca-cli` for full\n ownership handoffs — \"hand off\", \"handoff\", \"handover\", \"give this to another\n agent\", \"another worktree\" — unless asked to supervise, monitor, or coordinate\n a DAG, and for terminal control, lightweight terminal prompts, shell commands,\n Orca worktree management, and reading or waiting on terminals.\n---\n\n# Orca orchestration\n\nOrchestration is Orca's structured coordination layer. It records who owns work,\nwhich attempt is authoritative, and when supervised work has settled.\n\n## Outcome\n\n**Result:** every in-scope Task has one explicit outcome and every settled worker\nterminal has a next owner or cleanup decision. **Next consumer:** the user who\nrequested supervision. **Done:** all expected Dispatches have settled, every\ndelivered message was processed before acknowledgment, each settled worker was\nreused, explicitly retained, or released, and the turn ends only when the report\nto that user names, per Task, its outcome, the evidence behind it, and any\nunresolved blocker.\n\n**Safe failure:** preserve work and authority and report the state as unknown or\n`unverifiable`. Only positive proof of exit authorizes stop, abandon, or retry,\nand only an accepted settlement authorizes release. Every other observation,\nabsence included, is a checkpoint.\n\n## Classify the role\n\n| Current context | Role | Route |\n| ---------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------- | ------------------------------------------------------------------------------ |\n| The user explicitly asks to supervise, monitor, wait for results, track completion, coordinate a DAG, use a decision gate, or manage ask/reply | Coordinator | Use the supervised loop below |\n| The current prompt contains a live injected preamble with Task and Dispatch IDs | Dispatched worker | Follow the preamble and the worker obligations below |\n| The user asks to hand off ownership or start another agent/worktree without supervision | Handoff owner | Use `orca-cli`; create no Run, Task, or Dispatch and do not monitor completion |\n| A message carries a legacy authority label | Compatibility operator | Load the legacy contract reference before any lifecycle mutation |\n| No live preamble and no explicit supervision | Ordinary terminal agent | Do not emit lifecycle messages; use `orca-cli` for terminal/worktree work |\n\nModel or effort selection does not make a handoff supervised: `orca-cli`'s\n`worktree create --agent` and `agent launch` take `--model` and `--effort`, and\nopen a chat or a terminal per the user's Chat UI setting, as `worker-start` does.\nNever substitute a non-Orca subagent tool when Orca orchestration provenance was\nrequested.\n\n## Authority and safety floor\n\n- A Run is a durable namespace and coordinator inbox; it does not schedule or\n place workers. A Task is work. A Dispatch is one authoritative Task attempt.\n- Lifecycle authority comes from the active Dispatch, not a terminal title,\n copied ID, old database row, provider transcript, or visible pane.\n- Workers use the exact executable, handle, Task ID, and Dispatch ID in the live\n preamble, plus any other flag it carries. Never reconstruct, translate, or\n broaden those arguments.\n- After remote start, address the worker by Dispatch ID. The execution host owns\n process, filesystem, transcript, stop, and cleanup facts. Preserve the verdicts\n `live` / `unverifiable` / `exited`; contact loss is not process death.\n- Liveness is layered: `worker-list`'s `projection.liveness` is the fleet verdict\n for the agent; `worker-show`'s `observation.status` is PTY liveness only. A live\n terminal can still hold a dead or stuck agent.\n- Folder workspaces are valid; never require Git or assume a worktree.\n- Clients and remote servers update independently. Treat unknown optional fields\n as absent. A new stream operation requires advertised capability because old\n decoders may silently drop unknown opcodes. Never fall back to local execution\n when remote authority or capability is unproven.\n- Use the executable you used to run `skills get` for the entire run. In the\n examples below, replace `ORCA` with it; do not create a shell variable or run\n `ORCA` literally. If it fails, report that exact error instead of switching.\n- `ORCA status --json` shows your Orca session ID as `caller.orcaSessionId` when you have one.\n- A successful `orchestration send` proves durable enqueue; its wake or nudge is\n best-effort attention only and does not prove the recipient read or accepted it.\n\n## Worker obligations\n\nThe injected preamble is authoritative. A dispatched worker must:\n\n1. Do only the current Task and use the preamble's `ask` command for a blocking\n coordinator question. Never open a local question TUI the coordinator cannot\n answer. Resume the same message ID after an ask timeout.\n2. Send heartbeats only at the cadence in the preamble. A heartbeat proves\n liveness, not completion.\n3. Read coordinator follow-ups at each natural checkpoint — before starting a\n new file, after a test run — and once more immediately before `worker_done`:\n `ORCA orchestration check --terminal --json`.\n4. Send `worker_done` exactly once, from the dispatched terminal, with a\n three-sentence executive summary, both lifecycle IDs, and explicit\n `--outcome succeeded` or `--outcome failed`. Never encode failure only in prose.\n5. Append `--files-modified` and `--report-path` only with real values when\n applicable. After `worker_done`, end the dispatched turn and idle; do not poll\n or start new work.\n\nA direct user instruction after completion starts new user-owned work and takes\nprecedence over the idle rule. Do not reuse the settled lifecycle IDs.\n\n## Canonical supervised loop\n\nConfirm the runtime, bind one Run, and start the full independent wave before\nwaiting. `worker-start --spec` creates the Task and its attempt in one call:\n\n```text\nORCA status --json\nORCA orchestration run-create --objective \"\" --json\nORCA orchestration worker-start --spec \"\" --worktree current --agent codex --json\nORCA orchestration worker-start --spec \"\" --worktree current --agent claude --json\nORCA orchestration check --wait --types \"worker_done,escalation,question\" --timeout-ms 900000 --json\n```\n\nIf `worker-start` exits non-zero, do not relaunch. Read the receipt's\n`failedStage` and `residualResources`, then load\n`references/recovery-and-cleanup.md`.\n\nUse `task-create` plus `worker-start --task ` for planned fan-out with\ndependencies or a retry of a known Task. Use dependencies only for real ordering\nand prefer parallel waves over chains deeper than three or four steps; nested\nworkers obey the depth limit, and a new Run does not reset the caller's depth.\n\nA consuming `check` names its caller with `--terminal `, never `--from`;\nomit it inside the coordinator's own Orca terminal. It returns the bound Run's\noldest FIFO Delivery and replays that batch until acknowledged. Process every\nmessage: reply to questions, validate each `worker_done` against the expected\nactive Dispatch, and decide each settled terminal's next owner before the ack:\n\n```text\nORCA orchestration reply --id --body \"\" --json\nORCA orchestration worker-release --dispatch --json\nORCA orchestration check --ack --wait --types \"worker_done,escalation,question\" --timeout-ms 900000 --json\n```\n\nKeep waiting until every expected Dispatch settles. A timeout or empty result is\na checkpoint, not a failure. Do not stop, retry, release, or launch a duplicate\neditor without the positive proof `## Outcome` requires.\n\nAfter three consecutive empty waits, stop waiting blindly and enumerate with\n`ORCA orchestration worker-list --include-remote --json` (defaults to the bound\nRun; `--run ` overrides; the receipt's `scope` names which), acting on\neach row's `projection.attention` categories, `projection.attention.requiresAction`, and literal `projection.nextAction` argv.\nRows come newest first and page at 100: while `page.hasMore`, follow `page.nextCursor` with `--cursor `.\nA `none` `nextAction` has no argv to run: read `liveness.reason` and keep waiting\nwith `check --wait`. Absence never earns an argv; settlement and pending work still do.\nLeave the wait only on positive proof the agent stopped: `exited` liveness, the\nworker's own observation of process exit, or a transcript whose final agent turn\nsent no `worker_done`. Then load `references/recovery-and-cleanup.md` and choose\n`worker-stop` or `worker-abandon` explicitly. `unverifiable` is absence,\nincluding when `worker-show` reports `agentWait` null. Absence never authorizes\nstop, abandon, retry, or release; keep waiting or inspect.\n\n`worker-start` is the normal path, composing placement, terminal readiness,\nprompt injection, and supervised resource ownership. `dispatch --inject` leaves\nan operator-created process unsupervised and is only for an expressiveness gap.\n\n## Task-spec contract\n\nEvery Task spec must be self-contained and name:\n\n- **Target:** the files, component, or environment in scope.\n- **Change:** the concrete result to produce.\n- **Constraints:** invariants, compatibility rules, and do-not-touch boundaries.\n- **Ownership:** what this worker may edit and any coordination boundary.\n- **Observable acceptance:** the test, output, or evidence that proves completion.\n\n## Completion accounting\n\nAfter an accepted success or failure report, immediately do exactly one:\n\n1. Reuse the same proven agent terminal for an immediate follow-up Dispatch.\n2. Record user-requested retention with `worker-retain`.\n3. Run `worker-release`.\n\nRelease is post-settlement cleanup, not cancellation. Only an accepted\nsettlement authorizes it; no other observation does. If release is uncertain,\nfollow its exact recovery receipt and never substitute `terminal close`.\n\nA valid `worker_done` settles the Task and Dispatch automatically; do not follow\nit with `task-update --status completed`. Enumerate the terminals still owing a\ndecision with `worker-list --run --terminal-state reclaimable --json`,\nand do not end the coordinator turn until it returns none.\n\n## Conditional references\n\nThis compact guide is sufficient for the normal local loop. At an action gate\nbelow, run `ORCA skills get orchestration --reference references/.md` and\nread only that document; `--references` lists the names. If the CLI rejects\n`--reference`, run `ORCA skills get orchestration --full` once instead: it\nreturns this exact kernel and every reference, so read only the named one. If an\nolder CLI rejects `--full`, keep this kernel's safety floor, use that command's\n`--help`, and never guess newer flags.\n\n| Action gate | Bundled reference |\n| ------------------------------------------------------------------------------------------------------------- | ----------------------------------------- |\n| Expanded DAG waves, launch model/effort, same-terminal reuse, or review ownership | `references/coordinator-loop.md` |\n| You are a dispatched worker and the live preamble does not answer your question, or `check` returned an error | `references/worker-contract.md` |\n| New worktree, exact workspace, SSH, WSL, or connected-server placement | `references/placement-and-remote.md` |\n| Inbox replay, follow-up messages, group addresses, or decision gates | `references/messaging-and-gates.md` |\n| Failed/stopped/unknown attempts, retry, stop, abandon, retain, or uncertain release | `references/recovery-and-cleanup.md` |\n| Custom argv or terminal topology that `worker-start` cannot express | `references/low-level-topology.md` |\n| Any legacy label, adopted Run, compatibility receipt, or takeover | `references/legacy-contract-migration.md` |\n\nRetired scheduler commands are not aliases for Run creation. Recovery commands\nmust provide their exact next action; follow it with the same selected executable.\n\n---\n\n# Bundled references\n\nThese references belong to the version-matched guide above. Read only the documents named by its action gates.\n\n\n\n# Coordinator loop\n\nLoad this reference for expanded DAG waves, per-invocation launch preferences,\nsame-terminal reuse, or review ownership. The compact guide remains the source\nof truth for the loop order and completion boundary.\n\n## Ready waves\n\nCreate independent Tasks before the first wait. Encode only real dependencies,\nthen use the ready view as external memory:\n\n```text\nORCA orchestration task-create --spec \"\" --deps --json\nORCA orchestration task-list --ready --brief --json\n```\n\n`--brief` collapses whitespace and caps echoed specs at 160 characters;\n`spec_truncated` identifies shortened rows. Omit it when full specs are needed or\nwhen an older CLI rejects the flag. A nested worker must respect\n`nested_worker_depth_exceeded`; creating another Run does not reset depth.\n\n## Launch preferences\n\nFor a fresh Claude, Codex, Cursor, Antigravity, or Muse terminal, `--model`\naccepts an opaque provider model ID. Pass it only when the user named a model;\notherwise omit it so the worker inherits the user's configured agent default.\nAdd `--effort` only when that model supports it:\n\n```text\nORCA orchestration worker-start --task --worktree current --agent claude --model opus --effort high --json\nORCA orchestration worker-start --task --worktree current --agent muse --model muse-spark-1.3 --json\n```\n\nOpenCode also accepts `--model` in an existing worktree when the execution host\nverifies its CLI version and model availability. Creating a new worktree with an\nOpenCode model override is unsupported; use `--worktree current` or an existing\nworktree selector, or omit `--model`. OpenCode effort is unsupported. Omit the\nmodel to inherit its configuration; unsupported or unknown hosts refuse the\noverride explicitly.\n\n`--effort` requires `--model`; neither option combines with `--terminal`. A\nconnected worker server must advertise launch-preference support before Orca\nforwards either field. Compare `launch.requested` with `launch.effective`; never\nclaim a model or effort from requested arguments alone.\n\n## Reuse after settlement\n\nChoose the terminal's next owner before acknowledging the Delivery. When the\nsame exact agent has immediate follow-up work, recover the proven handle and\ntransfer cleanup ownership to the new Dispatch:\n\n```text\nORCA orchestration worker-show --dispatch --json\nORCA orchestration worker-start --task --terminal --json\n```\n\nOtherwise explicitly retain or release the settled worker. Do not leave it live\nonly to inspect output; archived output remains available through `worker-read`.\n\n## Review ownership\n\nA review-only `worker_done` authorizes synthesis of findings, not coordinator\nfile edits. Dispatch or hand off fixes unless the user explicitly assigned them\nto the coordinator. If the user's plan names a next owner, post-review fixes and\nPR preparation remain with that owner; the coordinator routes and synthesizes.\n\n\n\n# Legacy contract migration\n\nLoad this reference only for an authority label, adopted Run, compatibility or\nrecovery receipt, or explicit legacy takeover. A newly created attempt always\nuses the current grammar.\n\n## Authority labels\n\n- `[LEGACY COMPATIBILITY]` is live and attested. Run only the exact supported\n command printed with the message, using the same selected executable and\n arguments supplied by the original prompt.\n- `[LEGACY RECOVERY REPLAY — MAY HAVE BEEN SEEN]` is one bounded,\n at-least-once cutover replay. Process it idempotently and acknowledge only\n through the exact displayed guidance.\n- `[LEGACY READ-ONLY]` is inspection-only. It has no reply, acknowledgment, or\n lifecycle mutation.\n- An unlabeled current message uses the current guide and grammar.\n\nAn explicitly selected current Run, attested current binding, current Dispatch,\nor federated attachment takes precedence over legacy fallback. A retained\nadoption record alone does not grant mutation authority. If liveness, principal\nownership, capability, or the exact legacy contract is unproven, degrade to\nread-only inspection and never fall back to local execution.\n\nAdoption preserves the live agent process, PTY/session, terminal handle,\ntab/pane, worktree or folder workspace, Task, and Dispatch. It never restarts or\nreplaces the worker and never revives the retired scheduler. Loss of lifecycle\nauthority does not invalidate the existing process, assignment, or filesystem\nwork. Exact recovery may restore the same PTY once in its original inactive\nbackground tab; it must not spawn, write, signal, stop, switch, focus, split, or\ninject a terminal.\n\n## Compatibility recovery\n\nWhen a compatibility response returns structured next-step arguments, execute\nthose exact arguments with the same selected CLI executable. Do not translate\nfrom memory, broaden the recipient, or retry as a current mutation unless the\nreceipt explicitly authorizes it.\n\nA pending ask, reply, final Dispatch settlement, and consuming check have\ndurable recovery identities. Heartbeat and escalation remain at-least-once\nacross a manual contract-boundary retry. If an ask may already have been\nanswered, run the exact non-consuming recovery check printed by Orca before\ncreating any new question. Never guess among identical question threads.\n\nOn packaged Windows, a legacy ask uses a two-step commit/resume protocol. The\ninitial command commits the question, prints its exact\n`ask --resume ` command, and exits with launcher status `75`. Run\nthat exact resume after the launcher or update boundary. For an attested WSL\nlaunch, preserve the printed `orca-ide` executable and distro route. Older WSL\nworkers without launch proof remain lifecycle read-only even while their\nterminal and filesystem work continue.\n\n## Read-only inspection and takeover\n\nRead-only inspection does not consume mail:\n\n```text\nORCA orchestration run-list --json\nORCA orchestration run-show --id run_legacy_local --json\nORCA orchestration run-show --id --json\nORCA orchestration task-list --run --json\nORCA orchestration inbox --full --json\nORCA orchestration check --terminal --peek --format --json\nORCA terminal read --terminal --json\nORCA terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\n```\n\n`run_legacy_local` is an empty audit tombstone after adoption. Find the ordinary\nRun whose objective is `Recovered orchestration work from a contract update`.\n\nOnly when the original coordinator is unavailable or cannot prove retained\nauthority may a new live coordinator take over from its own terminal:\n\n```text\nORCA orchestration run-use --id --takeover-legacy --json\nORCA orchestration check --run --json\n```\n\nTakeover binds the authenticated invoking terminal; `--from` cannot nominate\nanother coordinator. It fences only the old coordinator and moves pending mail\ninto current Run delivery. It preserves live workers, Tasks, Dispatches, processes, and files.\nNever take over while the original coordinator is actively coordinating.\n\nDo not launch a replacement editor merely because Orca updated or authority is\nunclear. Keep the original worker as the only editor until a stable handoff\npoint, then use a fresh current Dispatch in a conflict-free placement.\n\n\n\n# Low-level topology\n\nLoad this reference only when `worker-start` cannot express required custom argv\nor terminal topology. It is not the normal supervised loop and is never a full\nhandoff recipe. A model or effort is not custom argv: `worker-start --model\n--effort` carries them and keeps the user's Chat UI setting.\n\n```text\nORCA terminal create --worktree active --title --command \"\" --json\nORCA terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\nORCA orchestration dispatch --task --to --inject --json\n```\n\nWait for readiness only when startup could lose injected input. Prefer\nagent-first `worker-start` whenever its argv and topology are sufficient.\n\n`dispatch --inject` creates authoritative Task/Dispatch context but deliberately\nkeeps an operator-created process unsupervised: it creates no supervised worker\nresource row. `worker-show`, `worker-read`, and `worker-list` report the lane as\n`unsupervised`; `worker-stop` and `worker-abandon` do not close that process, and\nsettled retain/release take no process action.\n\nUse `worker-start --terminal ` when lifecycle ownership of an existing\nagent terminal is required. Never imply that low-level dispatch retroactively\nowns a process, never use it to route around the nested-depth limit, and never\nuse it for an ownership handoff.\n\n\n\n# Messaging and gates\n\nLoad this reference for inbox replay, attempt-specific guidance, group\naddresses, blocking questions, or coordinator-managed DAG decisions.\n\nA successful `send` proves durable enqueue. Wake and nudge are best-effort\nattention only: neither proves the recipient read the message, began a turn, or\naccepted steering.\n\n## Coordinator delivery loop\n\n`check` names its caller with `--terminal ` and is the only verb that\nrejects `--from`. Omit `--terminal` inside an Orca terminal, where Orca resolves\nthe caller; pass it explicitly from anywhere else, including a dispatched\nworker reading coordinator follow-ups.\n\nA consuming coordinator `check` returns the bound Run's oldest FIFO Delivery,\nup to 50 messages, and replays that exact batch until acknowledged. Process\nevery row and required terminal ownership decision before `--ack`. Type filters\ndecide when a waiter wakes; they do not authorize skipping older actionable\nmail. A Delivery therefore always carries the whole FIFO batch whatever its\ntypes, and a `check` without `--wait` hands that batch over unfiltered.\n`--peek` and `--all` are read-only inspection, not progress through the\ncoordinator inbox.\n\nAn empty wait or timeout is a checkpoint. Continue rolling waits until every\nexpected Dispatch settles. Heartbeat or visible activity means alive, not done.\n\n## Addresses\n\nUse a stable Dispatch address for attempt-specific coordinator guidance:\n\n```text\nORCA orchestration send --to dispatch: --subject \"Follow-up\" --body \"\" --json\n```\n\nDo not substitute a remote terminal handle. Omit `--from` for ordinary\ncoordinator calls; a dispatched worker instead copies the exact `--from` and\nother arguments in its preamble. `check` is the exception: it identifies\nits caller with `--terminal`, never `--from`.\n\nGroup addresses include `@all`, `@idle`, `@claude`, `@codex`, `@opencode`,\n`@gemini`, `@droid`, `@grok`, `@cursor`, and `@worktree:`. Every group but\n`@worktree:` means the live Dispatches of the sender's own Run. Mail goes\nto each `dispatch:` mailbox, except a worker coordinating a child Run\nreceives it in that `run:` mailbox. A sender bound to no Run is refused;\n`--run` must match the group audience and never grants membership.\nA Run group excludes its owning coordinator; a worker raising a blocker sends\nto `run:`. A worker that created its own Run addresses that Run's workers,\nnot its siblings. `@worktree:` reaches matching workspace terminals,\nincluding coordinators. Use groups only for intentional fan-out status or\nquestions. `worker_done`, heartbeat, and other\nDispatch lifecycle messages never target groups.\n\n## Questions and gates\n\nA worker uses `ask`; its timeout leaves one durable question pending, which the\nworker resumes by message ID. The coordinator answers that message with `reply`.\n\nUse a gate only for a coordinator-owned Task-DAG decision:\n\n```text\nORCA orchestration gate-create --task --question \"\" --options --json\nORCA orchestration gate-resolve --id --resolution \"\" --json\nORCA orchestration gate-list --task --json\n```\n\nPass `json_array` using the quoting rules of the active shell; do not copy POSIX\nsingle-quote syntax into PowerShell or `cmd.exe`.\n\nDo not create a gate merely to answer a worker's `ask`.\n\n\n\n# Placement and remote execution\n\nLoad this reference before creating a new worktree or placing work through SSH,\nWSL, or another connected Orca server.\n\n## Placement choices\n\nA fresh worker means a fresh agent terminal, not a new Git worktree. Use the\ncurrent or an exact existing workspace by default. Create a worktree only when\nthe user requested one or a concrete checkout or filesystem conflict makes\nsharing unsafe.\n\n```text\n# Current workspace; setup is not rerun.\nORCA orchestration worker-start --task --worktree current --agent codex --json\n\n# Stacked child worktree.\nORCA orchestration worker-start --task --worktree new-child --name --agent codex --setup run --json\n\n# Independent top-level worktree.\nORCA orchestration worker-start --task --worktree new-top-level --name --agent codex --setup run --json\n```\n\nCurrent and exact existing workspaces create a fresh terminal unless\n`--terminal` is explicit. Folder workspaces are first-class; do not invoke Git\nor require worktree lineage when the selected workspace is a folder.\n\nRegister a folder workspace through project setup. `repo add --path `\nrequires a valid Git repository and rejects a plain directory:\n\n```text\nORCA project setup-existing-folder --project --host --path --kind folder --json\n```\n\nThen place work on the returned workspace with an exact selector. A worktree\nselector needs the full `::` value Orca returned, passed as\n`id:`; a bare repo id is not a worktree id. `new-child` and\n`new-top-level` are worktree creation and do not apply to a folder.\n\nNew worktrees use agent-first creation and run setup by default. Preserve the\nrepository's startup policy: `start-immediately` can report setup as `running`,\nwhile `wait-for-setup` gates prompt delivery on success. Orca lineage, Git base,\nfilesystem isolation, coordination parentage, UI grouping, and execution host\nare separate decisions.\n\n## Connected servers\n\nThe Run and Tasks remain authoritative on the current server. `--on` selects\nonly the worker's execution server and appears only on `worker-start`:\n\n```text\nORCA orchestration worker-start --task --on --worktree new-top-level --repo --name --agent codex --setup run --json\n```\n\nRemote `current` and `new-child` are invalid because they are ambiguous across\nservers. Use an exact discovered remote workspace, or `new-top-level` with an\nexact remote repository selector. After start, route every follow-up, read,\nstop, and cleanup by Dispatch ID; never repeat `--on` or substitute a remote\nterminal handle.\n\n```text\nORCA orchestration worker-show --dispatch --json\nORCA orchestration worker-read --dispatch --limit 50 --json\nORCA orchestration send --to dispatch: --subject \"Follow-up\" --body \"\" --json\nORCA orchestration worker-list --run --include-remote --json\n```\n\n`worker-list` reads local fleet state only; enumerate remote workers with\n`--include-remote` or every one of them reads `unverifiable`. Scope every list\nwith `--run `: unscoped, it reports every Dispatch this runtime has\nrecorded, and the workers you are waiting on are lost in that history.\n\n## Execution-host and mixed-version floor\n\nThe execution host owns process, filesystem, transcript, stop, and cleanup\nfacts. Render only `live`, `unverifiable`, or `exited`. Connection loss, relay\nabsence, missing client inventory, or timeout yields `unverifiable`, never\nsynthetic exit and never a client-local substitute action.\n\nClients and servers update independently. Optional response fields may be\nabsent. Forward model/effort, transcript reads, cleanup, or another new remote\noperation only when the peer advertises the relevant capability; unknown stream\nopcodes can be silently dropped. A narrow unsupported response may degrade to a\ndocumented older path, but must not broaden the target or cross the execution\nboundary. Changing host-published content reaches old clients even without a\nwire-shape change, so preserve established semantics or negotiate the behavior.\n\nFor WSL, use the exact executable and arguments returned by Orca so the distro\nand packaged launcher remain bound. Do not translate a printed `orca-ide`\nrecovery command into a PATH-resolved local command.\n\n\n\n# Recovery and cleanup\n\nLoad this reference only after a failed/stopped/unknown attempt, explicit retry\ndecision, stop/abandon request, retention request, or uncertain release.\n\n| Proven state | Safe action |\n| ----------------------- | ------------------------------------------------------------------ |\n| `ready` or active | Keep waiting; optionally read bounded output |\n| `failed` or `stopped` | Start a replacement with `--retry-of`; repeat placement explicitly |\n| `outcome_unknown` | Inspect, then choose `worker-stop` or explicit `worker-abandon` |\n| Accepted `worker_done` | Reuse, retain, or release |\n| Remote contact lost | Preserve `unverifiable`; do not stop or retry from absence alone |\n| `unverifiable` liveness | Keep waiting or inspect; never stop, abandon, retry, or release |\n| Proven `exited` agent | Enumerate with `worker-list`; follow its `nextAction` |\n\n## Inspect before acting\n\n```text\nORCA orchestration worker-list --run --json\nORCA orchestration worker-list --run --include-remote --json\nORCA orchestration worker-show --dispatch --json\nORCA orchestration worker-read --dispatch --limit 50 --json\n```\n\n`worker-list` is the enumerating command and the authority on agent liveness:\neach row carries `projection.liveness`, `projection.attention.categories`,\n`projection.attention.requiresAction`, and a literal `projection.nextAction`\nargv to run. Always scope it with `--run `; an unscoped list reports\nevery Dispatch this runtime has ever recorded and buries the live ones.\n`worker-show`'s `observation.status` is PTY liveness only, so a `live` terminal\nwhose agent died at a trust prompt still reads `live` there.\n\nWhen the two disagree, the fleet verdict decides — unless the fleet row is\n`unverifiable` for a reason that names a gap on this client rather than a fact\nabout the worker. `missing_status`, `host_unavailable`, and\n`capability_unsupported` are such gaps: the first means this runtime holds no\nstatus row, the second that it could not ask the execution host at all, and the\nthird that a stale peer answered but lacks the fleet-snapshot capability.\nAgainst any of them, a `worker-show` verdict sourced from the execution host is\nthe better evidence and outranks the row. Only `host_unavailable` is contact\nloss; the other two mean the host was never asked or answered without the\ncapability.\n\nThis never promotes absence. `unverifiable` from either command still authorizes\nnothing — only a positive `live` or `exited` verdict does.\n\nA worker started with `--on ` reads `unverifiable` until you\nenumerate with `--include-remote`, which asks its execution host for the\nverdict. Rows come newest first and past 100 the response pages, so follow\n`page.nextCursor` with `--cursor ` until `page.hasMore` is false.\n\n## Stall needs positive evidence\n\nLeave the wait only on positive proof the agent stopped: `exited` liveness, the\nworker's own observation of process exit, or a transcript whose final agent turn\nsent no `worker_done`. Only then choose `worker-stop` or `worker-abandon`.\n\n`unverifiable` is always absence — `missing_status`, `stale_status`,\n`restored_unconfirmed`, or a remote worker with no connection — and a null\n`agentWait` or an unchanged `worker-read` tail is that same absence seen again.\nAbsence never authorizes stop, abandon, retry, or release: keep waiting, or\ninspect until you hold one of the positive signals above. A `nextAction` that\nnames an inspecting command is asking for evidence, not for cleanup.\n\n`worker-read --source auto` uses a proven provider transcript when available and\notherwise returns bounded terminal output with a typed `fallbackReason`.\nContinue with its top-level cursor, which is pinned to that source. If Orca\nreports `source_changed`, restart without the old cursor. A bounded initial\ntranscript tail can return an EOF cursor that follows only newly appended records;\nread `contentComplete`, `clipping`, and `warnings` before assuming omitted older\nrecords are pageable. Never guess a provider session ID, transcript path, or\nremote terminal handle.\n\n## Was the mutation applied?\n\nWhen a mutation's response was lost and named no Dispatch, do not replay blind.\nEvery orchestration mutation accepts `--retry-request `, which reuses one\noperation identity so Orca can replay, join, or recover it instead of starting a\nduplicate. Ask what happened first:\n\n```text\nORCA orchestration request-show --request --json\n```\n\n`completed` means the mutation already took effect; read its recorded receipt\ninstead of rerunning. `pending` means the original mutation is still running or\nOrca restarted before recording its outcome; replay the original command with\n`--retry-request `. `absent` means this runtime holds no receipt\nunder your caller identity — that is not proof nothing happened, so inspect the\naffected Task, Dispatch, and terminal before deciding whether to retry.\n\nWhen a worker's terminal accepted input but the submit is unconfirmed, use\n`terminal send --wait-submit `: it observes the accepted prompt for that\nlong and, on timeout, returns the input-accepted receipt without resending.\n\n## Refused starts\n\n`dispatch` and `worker-start` refuse the following preflight cases with a stable\n`error.code`; read it before choosing a recovery, and treat `error.data.nextSteps`\nas the exact recovery text. Older hosts may omit `data`, so treat every field as\noptional.\n\n| Code | Meaning | Recovery |\n| -------------------- | --------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ |\n| `task_not_found` | No Task with that id, or not in the bound Run (`data.taskId`, `data.runId`) | Check `task-list --json`; create the Task with `task-create` if it does not exist |\n| `task_not_startable` | Task cannot start now: not `ready`, or invalid `--retry-of` (`data.status`, `data.unmetDependencies`, `data.retryOf`) | Wait for running dependencies with `check --wait`; retry or unblock failed ones; inspect `dispatch-show` if already dispatched |\n| `inject_rejected` | `--inject` refused because no recognized agent runs in the target (`data.terminal`, `data.reason`) | Start a recognized agent there or pick another terminal; or dispatch without `--inject` and use `terminal send` |\n| `runtime_error` | Any other failure, including a target terminal that already owns an active Dispatch | Read the message, inspect state, and do not retry unchanged |\n\n## Retry, stop, and abandon\n\nRetry only a positively proven failed or stopped attempt. Name the failed Task\nwith `--task`, since `--spec` creates a new one. Placement is never silently\ninherited:\n\n```text\nORCA orchestration worker-start --task --retry-of --worktree --agent --json\n```\n\nAfter three consecutive failures for one Task, its dispatch context\ncircuit-breaks and the Task is failed. Failed `worker-start` attempts count\nacross `--retry-of`; a worker's own `failed` report does not. `--retry-of` a\ncircuit-broken Dispatch is refused with `task_not_startable`: report the failure\nto the user instead. Do not route around that boundary with a new Run, Task, or\nunrelated Dispatch.\n\nFor `outcome_unknown`, inspect first, then make an explicit choice:\n\n```text\nORCA orchestration worker-stop --dispatch --json\nORCA orchestration worker-abandon --dispatch --json\n```\n\n`worker-stop` closes only the exact proven supervised agent terminal. It never\ndeletes the worktree, setup terminal, configured tabs, or unrelated processes.\n`worker-abandon` fences orchestration while accepting that resources may remain\nlive; it performs no remote, process, or filesystem action. When it settles a\nworker, it retains an owned terminal by the same rule as `worker-retain`, so Orca\nstops owing its release; a release already committed (`releasing`,\n`release_unknown`) is left as it is. An already-settled worker is left untouched.\n\nTo cancel a Task, settle its worker with `worker-stop` or `worker-abandon`, then\nrecord the reason. A cancelled Task is `failed`, so its dependents stay blocked\nuntil a `--retry-of` replacement completes it:\n\n```text\nORCA orchestration task-update --id --status failed --result cancelled --json\n```\n\n## Retain and release\n\n```text\nORCA orchestration worker-retain --dispatch --json\nORCA orchestration worker-release --dispatch --json\n```\n\nRetain only when the user explicitly wants the settled terminal kept live.\nRelease works after succeeded and failed reports, archives readable output, and\ncloses only the exact terminal owned by that settled Dispatch. Replays may call\nrelease again safely. Reused, pre-existing, setup, coordinator, active,\nuser-taken-over, and unproven terminals are retained.\n\nA `worker-start` that failed before its agent was ready still owns the terminal\nit created. Its receipt names `worker-release`, and `worker-list` reports that\nrow as `reclaimable`; release it there rather than closing the terminal by hand.\nA `--retry-of` replacement for a `failed` worker releases the terminal that\nworker still owns the same way.\n\nNever release because of timeout, TUI idle, heartbeat, status, question,\nescalation, or stale/rejected completion. If the receipt says `release_pending`\nor `release_unknown`, follow its exact recovery action. Never substitute\n`terminal close`.\n\n`orchestration reset` is destructive recovery. Do not run it during active\ncoordination unless the user explicitly abandons that state.\n\n\n\n# Worker contract\n\nThe injected preamble is authoritative. Copy its command rather than\nreconstructing flags. In particular, preserve the exact executable, worker\nhandle, Task ID, and Dispatch ID, and keep any other flag it carries (an older\nOrca host adds `--dispatch-capability`).\n\n## Heartbeat\n\nSend heartbeats only at the cadence required by the live preamble. Skip them\nwhile blocked inside `ask` or `check --wait`; those calls are liveness signals.\n\n```text\nORCA orchestration send --from --type heartbeat --subject \"alive\" --task-id --dispatch-id --phase \"\"\n```\n\nUse typed lifecycle flags, not a hand-written JSON payload. A heartbeat proves\nliveness, never completion.\n\n## Ask and resume\n\nUse Orca `ask` whenever the coordinator must answer. Never open a local question\nTUI the coordinator cannot answer.\n\n```text\nORCA orchestration ask --from --question \"\" --options \",\" --timeout-ms 600000\n\nORCA orchestration ask --from --resume --timeout-ms 600000\n```\n\nA timeout or disconnect leaves the original question pending. Resume its\nmessage ID; do not create a duplicate question.\n\n## Reading coordinator follow-ups\n\nThe coordinator steers a running worker with `send --to dispatch:`. That\nenqueue is durable but does not interrupt you, so nothing arrives unless you\nlook:\n\n```text\nORCA orchestration check --terminal --json\n```\n\nRun it at each natural checkpoint — before starting a new file, after a test\nrun — and once more immediately before `worker_done`, so a redirect or a\ncancellation lands before the Task settles. `check` names its caller with\n`--terminal`, never `--from`. Stop checking after `worker_done`.\n\nIf `check` returns `consumer_fenced`, this process no longer owns its Dispatch:\nthe Attempt was re-attached to another worker or settled without you. Stop, do\nnot send `worker_done`, and do not retry the check. An empty `check` never means\nyou were replaced; `consumer_fenced` is the only way you learn that.\nIf `send` or `ask` returns `consumer_fenced`, the command ran from another\nparty's terminal (a coordinator or another worker); run it from your own terminal.\n\n## Escalation\n\nEscalate only before completion and only when the coordinator must intervene:\n\n```text\nORCA orchestration send --from --type escalation --subject \"Blocked: \" --body \"
\" --task-id --dispatch-id \n```\n\n## Completion\n\nSend exactly one terminal report. `--body` is three sentences: what changed,\nwhat was found, and what remains. Use `--outcome failed` when the requested work\nis not complete; never hide failure in prose or silently exit.\n\nAppend `--files-modified` or `--report-path` only when applicable, using actual\npaths. Do not send documentation placeholders as metadata.\n\n```text\nORCA orchestration send --from --type worker_done --subject \"\" --body \"\" --task-id --dispatch-id --outcome succeeded\n```\n\nAfter `worker_done`, end the dispatched turn and idle. Do not poll, close your\nown terminal, or begin unrelated work. A later direct user instruction is new\nuser-owned work and must not reuse settled lifecycle IDs; a supervised follow-up\narrives with a fresh preamble and Task block.\n" +const ORCHESTRATION_FULL_MARKDOWN = "---\nname: orchestration\ndescription: >-\n Coordinate supervised Orca workers: threaded messages, blocking ask/reply,\n task dispatch, worker_done/escalation waits, task DAGs, decision gates,\n coordinator loops, and decomposing work across agents. Use `orca-cli` for full\n ownership handoffs — \"hand off\", \"handoff\", \"handover\", \"give this to another\n agent\", \"another worktree\" — unless asked to supervise, monitor, or coordinate\n a DAG, and for terminal control, lightweight terminal prompts, shell commands,\n Orca worktree management, and reading or waiting on terminals.\n---\n\n# Orca orchestration\n\nOrchestration is Orca's structured coordination layer. It records who owns work,\nwhich attempt is authoritative, and when supervised work has settled.\n\n## Outcome\n\n**Result:** every in-scope Task has one explicit outcome and every settled worker\nterminal has a next owner or cleanup decision. **Next consumer:** the user who\nrequested supervision. **Done:** all expected Dispatches have settled, every\ndelivered message was processed before acknowledgment, each settled worker was\nreused, explicitly retained, or released, and the turn ends only when the report\nto that user names, per Task, its outcome, the evidence behind it, and any\nunresolved blocker.\n\n**Safe failure:** preserve work and authority and report the state as unknown or\n`unverifiable`. Only positive proof of exit authorizes stop, abandon, or retry,\nand only an accepted settlement authorizes release. Every other observation,\nabsence included, is a checkpoint.\n\n## Classify the role\n\n| Current context | Role | Route |\n| ---------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------- | ------------------------------------------------------------------------------ |\n| The user explicitly asks to supervise, monitor, wait for results, track completion, coordinate a DAG, use a decision gate, or manage ask/reply | Coordinator | Use the supervised loop below |\n| The current prompt contains a live injected preamble with Task and Dispatch IDs | Dispatched worker | Follow the preamble and the worker obligations below |\n| The user asks to hand off ownership or start another agent/worktree without supervision | Handoff owner | Use `orca-cli`; create no Run, Task, or Dispatch and do not monitor completion |\n| A message carries a legacy authority label | Compatibility operator | Load the legacy contract reference before any lifecycle mutation |\n| No live preamble and no explicit supervision | Ordinary terminal agent | Do not emit lifecycle messages; use `orca-cli` for terminal/worktree work |\n\nModel or effort selection does not make a handoff supervised: `orca-cli`'s\n`worktree create --agent` and `agent launch` take `--model` and `--effort`, and\nopen a chat or a terminal per the user's Chat UI setting, as `worker-start` does.\nNever substitute a non-Orca subagent tool when Orca orchestration provenance was\nrequested.\n\n## Authority and safety floor\n\n- A Run is a durable namespace and coordinator inbox; it does not schedule or\n place workers. A Task is work. A Dispatch is one authoritative Task attempt.\n- Lifecycle authority comes from the active Dispatch, not a terminal title,\n copied ID, old database row, provider transcript, or visible pane.\n- Workers use the exact executable, handle, Task ID, and Dispatch ID in the live\n preamble, plus any other flag it carries. Never reconstruct, translate, or\n broaden those arguments.\n- After remote start, address the worker by Dispatch ID. The execution host owns\n process, filesystem, transcript, stop, and cleanup facts. Preserve the verdicts\n `live` / `unverifiable` / `exited`; contact loss is not process death.\n- Liveness is layered: `worker-list`'s `projection.liveness` is the fleet verdict\n for the agent; `worker-show`'s `observation.status` is PTY liveness only. A live\n terminal can still hold a dead or stuck agent.\n- Folder workspaces are valid; never require Git or assume a worktree.\n- Clients and remote servers update independently. Treat unknown optional fields\n as absent. A new stream operation requires advertised capability because old\n decoders may silently drop unknown opcodes. Never fall back to local execution\n when remote authority or capability is unproven.\n- Use the executable you used to run `skills get` for the entire run. In the\n examples below, replace `ORCA` with it; do not create a shell variable or run\n `ORCA` literally. If it fails, report that exact error instead of switching.\n- `ORCA status --json` shows your Orca session ID as `caller.orcaSessionId` when you have one.\n- A successful `orchestration send` proves durable enqueue; its wake or nudge is\n best-effort attention only and does not prove the recipient read or accepted it.\n\n## Worker obligations\n\nThe injected preamble is authoritative. A dispatched worker must:\n\n1. Do only the current Task and use the preamble's `ask` command for a blocking\n coordinator question. Never open a local question TUI the coordinator cannot\n answer. Resume the same message ID after an ask timeout.\n2. Send heartbeats only at the cadence in the preamble. A heartbeat proves\n liveness, not completion.\n3. Read coordinator follow-ups at each natural checkpoint — before starting a\n new file, after a test run — and once more immediately before `worker_done`:\n `ORCA orchestration check --terminal --json`.\n4. Send `worker_done` exactly once, from the dispatched terminal, with a\n three-sentence executive summary, both lifecycle IDs, and explicit\n `--outcome succeeded` or `--outcome failed`. Never encode failure only in prose.\n5. Append `--files-modified` and `--report-path` only with real values when\n applicable. After `worker_done`, end the dispatched turn and idle; do not poll\n or start new work.\n\nA direct user instruction after completion starts new user-owned work and takes\nprecedence over the idle rule. Do not reuse the settled lifecycle IDs.\n\n## Canonical supervised loop\n\nConfirm the runtime, bind one Run, and start the full independent wave before\nwaiting. `worker-start --spec` creates the Task and its attempt in one call:\n\n```text\nORCA status --json\nORCA orchestration run-create --objective \"\" --json\nORCA orchestration worker-start --spec \"\" --worktree current --agent codex --json\nORCA orchestration worker-start --spec \"\" --worktree current --agent claude --json\nORCA orchestration check --wait --types \"worker_done,escalation,question\" --timeout-ms 900000 --json\n```\n\nIf `worker-start` exits non-zero, do not relaunch. Read the receipt's\n`failedStage` and `residualResources`, then load\n`references/recovery-and-cleanup.md`.\n\nUse `task-create` plus `worker-start --task ` for planned fan-out with\ndependencies or a retry of a known Task. Use dependencies only for real ordering\nand prefer parallel waves over chains deeper than three or four steps; nested\nworkers obey the depth limit, and a new Run does not reset the caller's depth.\nA dispatched worker starts its own helpers with `worker-start --spec` on the Run\nit works for, without `run-create`; the helpers report to that worker.\n\nA consuming `check` names its caller with `--terminal `, never `--from`;\nomit it inside the coordinator's own Orca terminal. It returns the bound Run's\noldest FIFO Delivery and replays that batch until acknowledged. Process every\nmessage: reply to questions, validate each `worker_done` against the expected\nactive Dispatch, and decide each settled terminal's next owner before the ack:\n\n```text\nORCA orchestration reply --id --body \"\" --json\nORCA orchestration worker-release --dispatch --json\nORCA orchestration check --ack --wait --types \"worker_done,escalation,question\" --timeout-ms 900000 --json\n```\n\nKeep waiting until every expected Dispatch settles. A timeout or empty result is\na checkpoint, not a failure. Do not stop, retry, release, or launch a duplicate\neditor without the positive proof `## Outcome` requires.\n\nAfter three consecutive empty waits, stop waiting blindly and enumerate with\n`ORCA orchestration worker-list --include-remote --json` (defaults to the bound\nRun; `--run ` overrides; the receipt's `scope` names which), acting on\neach row's `projection.attention` categories, `projection.attention.requiresAction`, and literal `projection.nextAction` argv.\nRows come newest first and page at 100: while `page.hasMore`, follow `page.nextCursor` with `--cursor `.\nA `none` `nextAction` has no argv to run: read `liveness.reason` and keep waiting\nwith `check --wait`. Absence never earns an argv; settlement and pending work still do.\nLeave the wait only on positive proof the agent stopped: `exited` liveness, the\nworker's own observation of process exit, or a transcript whose final agent turn\nsent no `worker_done`. Then load `references/recovery-and-cleanup.md` and choose\n`worker-stop` or `worker-abandon` explicitly. `unverifiable` is absence,\nincluding when `worker-show` reports `agentWait` null. Absence never authorizes\nstop, abandon, retry, or release; keep waiting or inspect.\n\n`worker-start` is the normal path, composing placement, terminal readiness,\nprompt injection, and supervised resource ownership. `dispatch --inject` leaves\nan operator-created process unsupervised and is only for an expressiveness gap.\n\n## Task-spec contract\n\nEvery Task spec must be self-contained and name:\n\n- **Target:** the files, component, or environment in scope.\n- **Change:** the concrete result to produce.\n- **Constraints:** invariants, compatibility rules, and do-not-touch boundaries.\n- **Ownership:** what this worker may edit and any coordination boundary.\n- **Observable acceptance:** the test, output, or evidence that proves completion.\n\n## Completion accounting\n\nAfter an accepted success or failure report, immediately do exactly one:\n\n1. Reuse the same proven agent terminal for an immediate follow-up Dispatch.\n2. Record user-requested retention with `worker-retain`.\n3. Run `worker-release`.\n\nRelease is post-settlement cleanup, not cancellation. Only an accepted\nsettlement authorizes it; no other observation does. If release is uncertain,\nfollow its exact recovery receipt and never substitute `terminal close`.\n\nA valid `worker_done` settles the Task and Dispatch automatically; do not follow\nit with `task-update --status completed`. Enumerate the terminals still owing a\ndecision with `worker-list --run --terminal-state reclaimable --json`,\nand do not end the coordinator turn until it returns none.\n\n## Conditional references\n\nThis compact guide is sufficient for the normal local loop. At an action gate\nbelow, run `ORCA skills get orchestration --reference references/.md` and\nread only that document; `--references` lists the names. If the CLI rejects\n`--reference`, run `ORCA skills get orchestration --full` once instead: it\nreturns this exact kernel and every reference, so read only the named one. If an\nolder CLI rejects `--full`, keep this kernel's safety floor, use that command's\n`--help`, and never guess newer flags.\n\n| Action gate | Bundled reference |\n| ------------------------------------------------------------------------------------------------------------- | ----------------------------------------- |\n| Expanded DAG waves, launch model/effort, same-terminal reuse, or review ownership | `references/coordinator-loop.md` |\n| You are a dispatched worker and the live preamble does not answer your question, or `check` returned an error | `references/worker-contract.md` |\n| New worktree, exact workspace, SSH, WSL, or connected-server placement | `references/placement-and-remote.md` |\n| Inbox replay, follow-up messages, group addresses, or decision gates | `references/messaging-and-gates.md` |\n| Failed/stopped/unknown attempts, retry, stop, abandon, retain, or uncertain release | `references/recovery-and-cleanup.md` |\n| Custom argv or terminal topology that `worker-start` cannot express | `references/low-level-topology.md` |\n| Any legacy label, adopted Run, compatibility receipt, or takeover | `references/legacy-contract-migration.md` |\n\nRetired scheduler commands are not aliases for Run creation. Recovery commands\nmust provide their exact next action; follow it with the same selected executable.\n\n---\n\n# Bundled references\n\nThese references belong to the version-matched guide above. Read only the documents named by its action gates.\n\n\n\n# Coordinator loop\n\nLoad this reference for expanded DAG waves, per-invocation launch preferences,\nsame-terminal reuse, or review ownership. The compact guide remains the source\nof truth for the loop order and completion boundary.\n\n## Ready waves\n\nCreate independent Tasks before the first wait. Encode only real dependencies,\nthen use the ready view as external memory:\n\n```text\nORCA orchestration task-create --spec \"\" --deps --json\nORCA orchestration task-list --ready --brief --json\n```\n\n`--brief` collapses whitespace and caps echoed specs at 160 characters;\n`spec_truncated` identifies shortened rows. Omit it when full specs are needed or\nwhen an older CLI rejects the flag. A nested worker must respect\n`nested_worker_depth_exceeded`; creating another Run does not reset depth.\nHelpers a worker starts appear in this Run's `worker-list` with their\n`creatorDispatchId`, and report to that worker rather than to you.\n\n## Launch preferences\n\nFor a fresh Claude, Codex, Cursor, Antigravity, or Muse terminal, `--model`\naccepts an opaque provider model ID. Pass it only when the user named a model;\notherwise omit it so the worker inherits the user's configured agent default.\nAdd `--effort` only when that model supports it:\n\n```text\nORCA orchestration worker-start --task --worktree current --agent claude --model opus --effort high --json\nORCA orchestration worker-start --task --worktree current --agent muse --model muse-spark-1.3 --json\n```\n\nOpenCode also accepts `--model` in an existing worktree when the execution host\nverifies its CLI version and model availability. Creating a new worktree with an\nOpenCode model override is unsupported; use `--worktree current` or an existing\nworktree selector, or omit `--model`. OpenCode effort is unsupported. Omit the\nmodel to inherit its configuration; unsupported or unknown hosts refuse the\noverride explicitly.\n\n`--effort` requires `--model`; neither option combines with `--terminal`. A\nconnected worker server must advertise launch-preference support before Orca\nforwards either field. Compare `launch.requested` with `launch.effective`; never\nclaim a model or effort from requested arguments alone.\n\n## Reuse after settlement\n\nChoose the terminal's next owner before acknowledging the Delivery. When the\nsame exact agent has immediate follow-up work, recover the proven handle and\ntransfer cleanup ownership to the new Dispatch:\n\n```text\nORCA orchestration worker-show --dispatch --json\nORCA orchestration worker-start --task --terminal --json\n```\n\nOtherwise explicitly retain or release the settled worker. Do not leave it live\nonly to inspect output; archived output remains available through `worker-read`.\n\n## Review ownership\n\nA review-only `worker_done` authorizes synthesis of findings, not coordinator\nfile edits. Dispatch or hand off fixes unless the user explicitly assigned them\nto the coordinator. If the user's plan names a next owner, post-review fixes and\nPR preparation remain with that owner; the coordinator routes and synthesizes.\n\n\n\n# Legacy contract migration\n\nLoad this reference only for an authority label, adopted Run, compatibility or\nrecovery receipt, or explicit legacy takeover. A newly created attempt always\nuses the current grammar.\n\n## Authority labels\n\n- `[LEGACY COMPATIBILITY]` is live and attested. Run only the exact supported\n command printed with the message, using the same selected executable and\n arguments supplied by the original prompt.\n- `[LEGACY RECOVERY REPLAY — MAY HAVE BEEN SEEN]` is one bounded,\n at-least-once cutover replay. Process it idempotently and acknowledge only\n through the exact displayed guidance.\n- `[LEGACY READ-ONLY]` is inspection-only. It has no reply, acknowledgment, or\n lifecycle mutation.\n- An unlabeled current message uses the current guide and grammar.\n\nAn explicitly selected current Run, attested current binding, current Dispatch,\nor federated attachment takes precedence over legacy fallback. A retained\nadoption record alone does not grant mutation authority. If liveness, principal\nownership, capability, or the exact legacy contract is unproven, degrade to\nread-only inspection and never fall back to local execution.\n\nAdoption preserves the live agent process, PTY/session, terminal handle,\ntab/pane, worktree or folder workspace, Task, and Dispatch. It never restarts or\nreplaces the worker and never revives the retired scheduler. Loss of lifecycle\nauthority does not invalidate the existing process, assignment, or filesystem\nwork. Exact recovery may restore the same PTY once in its original inactive\nbackground tab; it must not spawn, write, signal, stop, switch, focus, split, or\ninject a terminal.\n\n## Compatibility recovery\n\nWhen a compatibility response returns structured next-step arguments, execute\nthose exact arguments with the same selected CLI executable. Do not translate\nfrom memory, broaden the recipient, or retry as a current mutation unless the\nreceipt explicitly authorizes it.\n\nA pending ask, reply, final Dispatch settlement, and consuming check have\ndurable recovery identities. Heartbeat and escalation remain at-least-once\nacross a manual contract-boundary retry. If an ask may already have been\nanswered, run the exact non-consuming recovery check printed by Orca before\ncreating any new question. Never guess among identical question threads.\n\nOn packaged Windows, a legacy ask uses a two-step commit/resume protocol. The\ninitial command commits the question, prints its exact\n`ask --resume ` command, and exits with launcher status `75`. Run\nthat exact resume after the launcher or update boundary. For an attested WSL\nlaunch, preserve the printed `orca-ide` executable and distro route. Older WSL\nworkers without launch proof remain lifecycle read-only even while their\nterminal and filesystem work continue.\n\n## Read-only inspection and takeover\n\nRead-only inspection does not consume mail:\n\n```text\nORCA orchestration run-list --json\nORCA orchestration run-show --id run_legacy_local --json\nORCA orchestration run-show --id --json\nORCA orchestration task-list --run --json\nORCA orchestration inbox --full --json\nORCA orchestration check --terminal --peek --format --json\nORCA terminal read --terminal --json\nORCA terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\n```\n\n`run_legacy_local` is an empty audit tombstone after adoption. Find the ordinary\nRun whose objective is `Recovered orchestration work from a contract update`.\n\nOnly when the original coordinator is unavailable or cannot prove retained\nauthority may a new live coordinator take over from its own terminal:\n\n```text\nORCA orchestration run-use --id --takeover-legacy --json\nORCA orchestration check --run --json\n```\n\nTakeover binds the authenticated invoking terminal; `--from` cannot nominate\nanother coordinator. It fences only the old coordinator and moves pending mail\ninto current Run delivery. It preserves live workers, Tasks, Dispatches, processes, and files.\nNever take over while the original coordinator is actively coordinating.\n\nDo not launch a replacement editor merely because Orca updated or authority is\nunclear. Keep the original worker as the only editor until a stable handoff\npoint, then use a fresh current Dispatch in a conflict-free placement.\n\n\n\n# Low-level topology\n\nLoad this reference only when `worker-start` cannot express required custom argv\nor terminal topology. It is not the normal supervised loop and is never a full\nhandoff recipe. A model or effort is not custom argv: `worker-start --model\n--effort` carries them and keeps the user's Chat UI setting.\n\n```text\nORCA terminal create --worktree active --title --command \"\" --json\nORCA terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\nORCA orchestration dispatch --task --to --inject --json\n```\n\nWait for readiness only when startup could lose injected input. Prefer\nagent-first `worker-start` whenever its argv and topology are sufficient.\n\n`dispatch --inject` creates authoritative Task/Dispatch context but deliberately\nkeeps an operator-created process unsupervised: it creates no supervised worker\nresource row. `worker-show`, `worker-read`, and `worker-list` report the lane as\n`unsupervised`; `worker-stop` and `worker-abandon` do not close that process, and\nsettled retain/release take no process action.\n\nUse `worker-start --terminal ` when lifecycle ownership of an existing\nagent terminal is required. Never imply that low-level dispatch retroactively\nowns a process, never use it to route around the nested-depth limit, and never\nuse it for an ownership handoff.\n\n\n\n# Messaging and gates\n\nLoad this reference for inbox replay, attempt-specific guidance, group\naddresses, blocking questions, or coordinator-managed DAG decisions.\n\nA successful `send` proves durable enqueue. Wake and nudge are best-effort\nattention only: neither proves the recipient read the message, began a turn, or\naccepted steering.\n\n## Coordinator delivery loop\n\n`check` names its caller with `--terminal ` and is the only verb that\nrejects `--from`. Omit `--terminal` inside an Orca terminal, where Orca resolves\nthe caller; pass it explicitly from anywhere else, including a dispatched\nworker reading coordinator follow-ups.\n\nA consuming coordinator `check` returns the bound Run's oldest FIFO Delivery,\nup to 50 messages, and replays that exact batch until acknowledged. Process\nevery row and required terminal ownership decision before `--ack`. Type filters\ndecide when a waiter wakes; they do not authorize skipping older actionable\nmail. A Delivery therefore always carries the whole FIFO batch whatever its\ntypes, and a `check` without `--wait` hands that batch over unfiltered.\n`--peek` and `--all` are read-only inspection, not progress through the\ncoordinator inbox.\n\nAn empty wait or timeout is a checkpoint. Continue rolling waits until every\nexpected Dispatch settles. Heartbeat or visible activity means alive, not done.\n\n## Addresses\n\nUse a stable Dispatch address for attempt-specific coordinator guidance:\n\n```text\nORCA orchestration send --to dispatch: --subject \"Follow-up\" --body \"\" --json\n```\n\nDo not substitute a remote terminal handle. Omit `--from` for ordinary\ncoordinator calls; a dispatched worker instead copies the exact `--from` and\nother arguments in its preamble. `check` is the exception: it identifies\nits caller with `--terminal`, never `--from`.\n\nGroup addresses include `@all`, `@idle`, `@claude`, `@codex`, `@opencode`,\n`@gemini`, `@droid`, `@grok`, `@cursor`, and `@worktree:`. Every group but\n`@worktree:` means the live Dispatches of the sender's own Run. Mail goes\nto each `dispatch:` mailbox, except a worker coordinating a child Run\nreceives it in that `run:` mailbox. A sender bound to no Run is refused;\n`--run` must match the group audience and never grants membership.\nA Run group excludes its owning coordinator; a worker raising a blocker sends\nto `run:`. A worker that created its own Run addresses that Run's workers,\nnot its siblings. `@worktree:` reaches matching workspace terminals,\nincluding coordinators. Use groups only for intentional fan-out status or\nquestions. `worker_done`, heartbeat, and other\nDispatch lifecycle messages never target groups.\n\n## Questions and gates\n\nA worker uses `ask`; its timeout leaves one durable question pending, which the\nworker resumes by message ID. The coordinator answers that message with `reply`;\na helper's question goes to the worker that started it, which answers it the\nsame way.\n\nUse a gate only for a coordinator-owned Task-DAG decision:\n\n```text\nORCA orchestration gate-create --task --question \"\" --options --json\nORCA orchestration gate-resolve --id --resolution \"\" --json\nORCA orchestration gate-list --task --json\n```\n\nPass `json_array` using the quoting rules of the active shell; do not copy POSIX\nsingle-quote syntax into PowerShell or `cmd.exe`.\n\nDo not create a gate merely to answer a worker's `ask`.\n\n\n\n# Placement and remote execution\n\nLoad this reference before creating a new worktree or placing work through SSH,\nWSL, or another connected Orca server.\n\n## Placement choices\n\nA fresh worker means a fresh agent terminal, not a new Git worktree. Use the\ncurrent or an exact existing workspace by default. Create a worktree only when\nthe user requested one or a concrete checkout or filesystem conflict makes\nsharing unsafe.\n\n```text\n# Current workspace; setup is not rerun.\nORCA orchestration worker-start --task --worktree current --agent codex --json\n\n# Stacked child worktree.\nORCA orchestration worker-start --task --worktree new-child --name --agent codex --setup run --json\n\n# Independent top-level worktree.\nORCA orchestration worker-start --task --worktree new-top-level --name --agent codex --setup run --json\n```\n\nCurrent and exact existing workspaces create a fresh terminal unless\n`--terminal` is explicit. Folder workspaces are first-class; do not invoke Git\nor require worktree lineage when the selected workspace is a folder.\n\nRegister a folder workspace through project setup. `repo add --path `\nrequires a valid Git repository and rejects a plain directory:\n\n```text\nORCA project setup-existing-folder --project --host --path --kind folder --json\n```\n\nThen place work on the returned workspace with an exact selector. A worktree\nselector needs the full `::` value Orca returned, passed as\n`id:`; a bare repo id is not a worktree id. `new-child` and\n`new-top-level` are worktree creation and do not apply to a folder.\n\nNew worktrees use agent-first creation and run setup by default. Preserve the\nrepository's startup policy: `start-immediately` can report setup as `running`,\nwhile `wait-for-setup` gates prompt delivery on success. Orca lineage, Git base,\nfilesystem isolation, coordination parentage, UI grouping, and execution host\nare separate decisions.\n\n## Connected servers\n\nThe Run and Tasks remain authoritative on the current server. `--on` selects\nonly the worker's execution server and appears only on `worker-start`:\n\n```text\nORCA orchestration worker-start --task --on --worktree new-top-level --repo --name --agent codex --setup run --json\n```\n\nRemote `current` and `new-child` are invalid because they are ambiguous across\nservers. Use an exact discovered remote workspace, or `new-top-level` with an\nexact remote repository selector. After start, route every follow-up, read,\nstop, and cleanup by Dispatch ID; never repeat `--on` or substitute a remote\nterminal handle.\n\n```text\nORCA orchestration worker-show --dispatch --json\nORCA orchestration worker-read --dispatch --limit 50 --json\nORCA orchestration send --to dispatch: --subject \"Follow-up\" --body \"\" --json\nORCA orchestration worker-list --run --include-remote --json\n```\n\n`worker-list` reads local fleet state only; enumerate remote workers with\n`--include-remote` or every one of them reads `unverifiable`. Scope every list\nwith `--run `: unscoped, it reports every Dispatch this runtime has\nrecorded, and the workers you are waiting on are lost in that history.\n\n## Execution-host and mixed-version floor\n\nThe execution host owns process, filesystem, transcript, stop, and cleanup\nfacts. Render only `live`, `unverifiable`, or `exited`. Connection loss, relay\nabsence, missing client inventory, or timeout yields `unverifiable`, never\nsynthetic exit and never a client-local substitute action.\n\nClients and servers update independently. Optional response fields may be\nabsent. Forward model/effort, transcript reads, cleanup, or another new remote\noperation only when the peer advertises the relevant capability; unknown stream\nopcodes can be silently dropped. A narrow unsupported response may degrade to a\ndocumented older path, but must not broaden the target or cross the execution\nboundary. Changing host-published content reaches old clients even without a\nwire-shape change, so preserve established semantics or negotiate the behavior.\n\nFor WSL, use the exact executable and arguments returned by Orca so the distro\nand packaged launcher remain bound. Do not translate a printed `orca-ide`\nrecovery command into a PATH-resolved local command.\n\n\n\n# Recovery and cleanup\n\nLoad this reference only after a failed/stopped/unknown attempt, explicit retry\ndecision, stop/abandon request, retention request, or uncertain release.\n\n| Proven state | Safe action |\n| ----------------------- | ------------------------------------------------------------------ |\n| `ready` or active | Keep waiting; optionally read bounded output |\n| `failed` or `stopped` | Start a replacement with `--retry-of`; repeat placement explicitly |\n| `outcome_unknown` | Inspect, then choose `worker-stop` or explicit `worker-abandon` |\n| Accepted `worker_done` | Reuse, retain, or release |\n| Remote contact lost | Preserve `unverifiable`; do not stop or retry from absence alone |\n| `unverifiable` liveness | Keep waiting or inspect; never stop, abandon, retry, or release |\n| Proven `exited` agent | Enumerate with `worker-list`; follow its `nextAction` |\n\n## Inspect before acting\n\n```text\nORCA orchestration worker-list --run --json\nORCA orchestration worker-list --run --include-remote --json\nORCA orchestration worker-show --dispatch --json\nORCA orchestration worker-read --dispatch --limit 50 --json\n```\n\n`worker-list` is the enumerating command and the authority on agent liveness:\neach row carries `projection.liveness`, `projection.attention.categories`,\n`projection.attention.requiresAction`, and a literal `projection.nextAction`\nargv to run. Always scope it with `--run `; an unscoped list reports\nevery Dispatch this runtime has ever recorded and buries the live ones.\n`worker-show`'s `observation.status` is PTY liveness only, so a `live` terminal\nwhose agent died at a trust prompt still reads `live` there.\n\nWhen the two disagree, the fleet verdict decides — unless the fleet row is\n`unverifiable` for a reason that names a gap on this client rather than a fact\nabout the worker. `missing_status`, `host_unavailable`, and\n`capability_unsupported` are such gaps: the first means this runtime holds no\nstatus row, the second that it could not ask the execution host at all, and the\nthird that a stale peer answered but lacks the fleet-snapshot capability.\nAgainst any of them, a `worker-show` verdict sourced from the execution host is\nthe better evidence and outranks the row. Only `host_unavailable` is contact\nloss; the other two mean the host was never asked or answered without the\ncapability.\n\nThis never promotes absence. `unverifiable` from either command still authorizes\nnothing — only a positive `live` or `exited` verdict does.\n\nA worker started with `--on ` reads `unverifiable` until you\nenumerate with `--include-remote`, which asks its execution host for the\nverdict. Rows come newest first and past 100 the response pages, so follow\n`page.nextCursor` with `--cursor ` until `page.hasMore` is false.\n\n## Stall needs positive evidence\n\nLeave the wait only on positive proof the agent stopped: `exited` liveness, the\nworker's own observation of process exit, or a transcript whose final agent turn\nsent no `worker_done`. Only then choose `worker-stop` or `worker-abandon`.\n\n`unverifiable` is always absence — `missing_status`, `stale_status`,\n`restored_unconfirmed`, or a remote worker with no connection — and a null\n`agentWait` or an unchanged `worker-read` tail is that same absence seen again.\nAbsence never authorizes stop, abandon, retry, or release: keep waiting, or\ninspect until you hold one of the positive signals above. A `nextAction` that\nnames an inspecting command is asking for evidence, not for cleanup.\n\n`worker-read --source auto` uses a proven provider transcript when available and\notherwise returns bounded terminal output with a typed `fallbackReason`.\nContinue with its top-level cursor, which is pinned to that source. If Orca\nreports `source_changed`, restart without the old cursor. A bounded initial\ntranscript tail can return an EOF cursor that follows only newly appended records;\nread `contentComplete`, `clipping`, and `warnings` before assuming omitted older\nrecords are pageable. Never guess a provider session ID, transcript path, or\nremote terminal handle.\n\n## Was the mutation applied?\n\nWhen a mutation's response was lost and named no Dispatch, do not replay blind.\nEvery orchestration mutation accepts `--retry-request `, which reuses one\noperation identity so Orca can replay, join, or recover it instead of starting a\nduplicate. Ask what happened first:\n\n```text\nORCA orchestration request-show --request --json\n```\n\n`completed` means the mutation already took effect; read its recorded receipt\ninstead of rerunning. `pending` means the original mutation is still running or\nOrca restarted before recording its outcome; replay the original command with\n`--retry-request `. `absent` means this runtime holds no receipt\nunder your caller identity — that is not proof nothing happened, so inspect the\naffected Task, Dispatch, and terminal before deciding whether to retry.\n\nWhen a worker's terminal accepted input but the submit is unconfirmed, use\n`terminal send --wait-submit `: it observes the accepted prompt for that\nlong and, on timeout, returns the input-accepted receipt without resending.\n\n## Refused starts\n\n`dispatch` and `worker-start` refuse the following preflight cases with a stable\n`error.code`; read it before choosing a recovery, and treat `error.data.nextSteps`\nas the exact recovery text. Older hosts may omit `data`, so treat every field as\noptional.\n\n| Code | Meaning | Recovery |\n| -------------------- | --------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ |\n| `task_not_found` | No Task with that id, or not in the bound Run (`data.taskId`, `data.runId`) | Check `task-list --json`; create the Task with `task-create` if it does not exist |\n| `task_not_startable` | Task cannot start now: not `ready`, or invalid `--retry-of` (`data.status`, `data.unmetDependencies`, `data.retryOf`) | Wait for running dependencies with `check --wait`; retry or unblock failed ones; inspect `dispatch-show` if already dispatched |\n| `inject_rejected` | `--inject` refused because no recognized agent runs in the target (`data.terminal`, `data.reason`) | Start a recognized agent there or pick another terminal; or dispatch without `--inject` and use `terminal send` |\n| `runtime_error` | Any other failure, including a target terminal that already owns an active Dispatch | Read the message, inspect state, and do not retry unchanged |\n\n## Retry, stop, and abandon\n\nRetry only a positively proven failed or stopped attempt. Name the failed Task\nwith `--task`, since `--spec` creates a new one. Placement is never silently\ninherited:\n\n```text\nORCA orchestration worker-start --task --retry-of --worktree --agent --json\n```\n\nAfter three consecutive failures for one Task, its dispatch context\ncircuit-breaks and the Task is failed. Failed `worker-start` attempts count\nacross `--retry-of`; a worker's own `failed` report does not. `--retry-of` a\ncircuit-broken Dispatch is refused with `task_not_startable`: report the failure\nto the user instead. Do not route around that boundary with a new Run, Task, or\nunrelated Dispatch.\n\nFor `outcome_unknown`, inspect first, then make an explicit choice:\n\n```text\nORCA orchestration worker-stop --dispatch --json\nORCA orchestration worker-abandon --dispatch --json\n```\n\n`worker-stop` closes only the exact proven supervised agent terminal. It never\ndeletes the worktree, setup terminal, configured tabs, or unrelated processes.\n`worker-abandon` fences orchestration while accepting that resources may remain\nlive; it performs no remote, process, or filesystem action. When it settles a\nworker, it retains an owned terminal by the same rule as `worker-retain`, so Orca\nstops owing its release; a release already committed (`releasing`,\n`release_unknown`) is left as it is. An already-settled worker is left untouched.\n\nTo cancel a Task, settle its worker with `worker-stop` or `worker-abandon`, then\nrecord the reason. A cancelled Task is `failed`, so its dependents stay blocked\nuntil a `--retry-of` replacement completes it:\n\n```text\nORCA orchestration task-update --id --status failed --result cancelled --json\n```\n\n## Retain and release\n\n```text\nORCA orchestration worker-retain --dispatch --json\nORCA orchestration worker-release --dispatch --json\n```\n\nRetain only when the user explicitly wants the settled terminal kept live.\nRelease works after succeeded and failed reports, archives readable output, and\ncloses only the exact terminal owned by that settled Dispatch. Replays may call\nrelease again safely. Reused, pre-existing, setup, coordinator, active,\nuser-taken-over, and unproven terminals are retained.\n\nA `worker-start` that failed before its agent was ready still owns the terminal\nit created. Its receipt names `worker-release`, and `worker-list` reports that\nrow as `reclaimable`; release it there rather than closing the terminal by hand.\nA `--retry-of` replacement for a `failed` worker releases the terminal that\nworker still owns the same way.\n\nNever release because of timeout, TUI idle, heartbeat, status, question,\nescalation, or stale/rejected completion. If the receipt says `release_pending`\nor `release_unknown`, follow its exact recovery action. Never substitute\n`terminal close`.\n\n`orchestration reset` is destructive recovery. Do not run it during active\ncoordination unless the user explicitly abandons that state.\n\n\n\n# Worker contract\n\nThe injected preamble is authoritative. Copy its command rather than\nreconstructing flags. In particular, preserve the exact executable, worker\nhandle, Task ID, and Dispatch ID, and keep any other flag it carries (an older\nOrca host adds `--dispatch-capability`).\n\n## Heartbeat\n\nSend heartbeats only at the cadence required by the live preamble. Skip them\nwhile blocked inside `ask` or `check --wait`; those calls are liveness signals.\n\n```text\nORCA orchestration send --from --type heartbeat --subject \"alive\" --task-id --dispatch-id --phase \"\"\n```\n\nUse typed lifecycle flags, not a hand-written JSON payload. A heartbeat proves\nliveness, never completion.\n\n## Ask and resume\n\nUse Orca `ask` whenever the coordinator must answer. Never open a local question\nTUI the coordinator cannot answer.\n\n```text\nORCA orchestration ask --from --question \"\" --options \",\" --timeout-ms 600000\n\nORCA orchestration ask --from --resume --timeout-ms 600000\n```\n\nA timeout or disconnect leaves the original question pending. Resume its\nmessage ID; do not create a duplicate question.\n\n## Reading coordinator follow-ups\n\nThe coordinator steers a running worker with `send --to dispatch:`. That\nenqueue is durable but does not interrupt you, so nothing arrives unless you\nlook:\n\n```text\nORCA orchestration check --terminal --json\n```\n\nRun it at each natural checkpoint — before starting a new file, after a test\nrun — and once more immediately before `worker_done`, so a redirect or a\ncancellation lands before the Task settles. `check` names its caller with\n`--terminal`, never `--from`. Stop checking after `worker_done`.\n\nIf `check` returns `consumer_fenced`, this process no longer owns its Dispatch:\nthe Attempt was re-attached to another worker or settled without you. Stop, do\nnot send `worker_done`, and do not retry the check. An empty `check` never means\nyou were replaced; `consumer_fenced` is the only way you learn that.\nIf `send` or `ask` returns `consumer_fenced`, the command ran from another\nparty's terminal (a coordinator or another worker); run it from your own terminal.\n\n## Helper workers\n\nWhen the preamble has a SUB-DISPATCH section, start helpers on your own\nDispatch's Run. Do not `run-create` or `run-use` for them.\n\n```text\nORCA orchestration worker-start --from --spec \"\" --worktree current --agent --json\nORCA orchestration reply --from --id --body \"\"\n```\n\nTheir questions, escalations, and `worker_done` arrive in your own `check`, next\nto your coordinator's mail. Answer their questions with `reply`, and wait for\nevery helper's `worker_done` before sending yours. After you settle, a helper\nstill running reports to your coordinator's Run.\n\n## Escalation\n\nEscalate only before completion and only when the coordinator must intervene:\n\n```text\nORCA orchestration send --from --type escalation --subject \"Blocked: \" --body \"
\" --task-id --dispatch-id \n```\n\n## Completion\n\nSend exactly one terminal report. `--body` is three sentences: what changed,\nwhat was found, and what remains. Use `--outcome failed` when the requested work\nis not complete; never hide failure in prose or silently exit.\n\nAppend `--files-modified` or `--report-path` only when applicable, using actual\npaths. Do not send documentation placeholders as metadata.\n\n```text\nORCA orchestration send --from --type worker_done --subject \"\" --body \"\" --task-id --dispatch-id --outcome succeeded\n```\n\nAfter `worker_done`, end the dispatched turn and idle. Do not poll, close your\nown terminal, or begin unrelated work. A later direct user instruction is new\nuser-owned work and must not reuse settled lifecycle IDs; a supervised follow-up\narrives with a fresh preamble and Task block.\n" // oxfmt-ignore -const ORCHESTRATION_COORDINATOR_LOOP_REFERENCE_MARKDOWN = "# Coordinator loop\n\nLoad this reference for expanded DAG waves, per-invocation launch preferences,\nsame-terminal reuse, or review ownership. The compact guide remains the source\nof truth for the loop order and completion boundary.\n\n## Ready waves\n\nCreate independent Tasks before the first wait. Encode only real dependencies,\nthen use the ready view as external memory:\n\n```text\nORCA orchestration task-create --spec \"\" --deps --json\nORCA orchestration task-list --ready --brief --json\n```\n\n`--brief` collapses whitespace and caps echoed specs at 160 characters;\n`spec_truncated` identifies shortened rows. Omit it when full specs are needed or\nwhen an older CLI rejects the flag. A nested worker must respect\n`nested_worker_depth_exceeded`; creating another Run does not reset depth.\n\n## Launch preferences\n\nFor a fresh Claude, Codex, Cursor, Antigravity, or Muse terminal, `--model`\naccepts an opaque provider model ID. Pass it only when the user named a model;\notherwise omit it so the worker inherits the user's configured agent default.\nAdd `--effort` only when that model supports it:\n\n```text\nORCA orchestration worker-start --task --worktree current --agent claude --model opus --effort high --json\nORCA orchestration worker-start --task --worktree current --agent muse --model muse-spark-1.3 --json\n```\n\nOpenCode also accepts `--model` in an existing worktree when the execution host\nverifies its CLI version and model availability. Creating a new worktree with an\nOpenCode model override is unsupported; use `--worktree current` or an existing\nworktree selector, or omit `--model`. OpenCode effort is unsupported. Omit the\nmodel to inherit its configuration; unsupported or unknown hosts refuse the\noverride explicitly.\n\n`--effort` requires `--model`; neither option combines with `--terminal`. A\nconnected worker server must advertise launch-preference support before Orca\nforwards either field. Compare `launch.requested` with `launch.effective`; never\nclaim a model or effort from requested arguments alone.\n\n## Reuse after settlement\n\nChoose the terminal's next owner before acknowledging the Delivery. When the\nsame exact agent has immediate follow-up work, recover the proven handle and\ntransfer cleanup ownership to the new Dispatch:\n\n```text\nORCA orchestration worker-show --dispatch --json\nORCA orchestration worker-start --task --terminal --json\n```\n\nOtherwise explicitly retain or release the settled worker. Do not leave it live\nonly to inspect output; archived output remains available through `worker-read`.\n\n## Review ownership\n\nA review-only `worker_done` authorizes synthesis of findings, not coordinator\nfile edits. Dispatch or hand off fixes unless the user explicitly assigned them\nto the coordinator. If the user's plan names a next owner, post-review fixes and\nPR preparation remain with that owner; the coordinator routes and synthesizes.\n" +const ORCHESTRATION_COORDINATOR_LOOP_REFERENCE_MARKDOWN = "# Coordinator loop\n\nLoad this reference for expanded DAG waves, per-invocation launch preferences,\nsame-terminal reuse, or review ownership. The compact guide remains the source\nof truth for the loop order and completion boundary.\n\n## Ready waves\n\nCreate independent Tasks before the first wait. Encode only real dependencies,\nthen use the ready view as external memory:\n\n```text\nORCA orchestration task-create --spec \"\" --deps --json\nORCA orchestration task-list --ready --brief --json\n```\n\n`--brief` collapses whitespace and caps echoed specs at 160 characters;\n`spec_truncated` identifies shortened rows. Omit it when full specs are needed or\nwhen an older CLI rejects the flag. A nested worker must respect\n`nested_worker_depth_exceeded`; creating another Run does not reset depth.\nHelpers a worker starts appear in this Run's `worker-list` with their\n`creatorDispatchId`, and report to that worker rather than to you.\n\n## Launch preferences\n\nFor a fresh Claude, Codex, Cursor, Antigravity, or Muse terminal, `--model`\naccepts an opaque provider model ID. Pass it only when the user named a model;\notherwise omit it so the worker inherits the user's configured agent default.\nAdd `--effort` only when that model supports it:\n\n```text\nORCA orchestration worker-start --task --worktree current --agent claude --model opus --effort high --json\nORCA orchestration worker-start --task --worktree current --agent muse --model muse-spark-1.3 --json\n```\n\nOpenCode also accepts `--model` in an existing worktree when the execution host\nverifies its CLI version and model availability. Creating a new worktree with an\nOpenCode model override is unsupported; use `--worktree current` or an existing\nworktree selector, or omit `--model`. OpenCode effort is unsupported. Omit the\nmodel to inherit its configuration; unsupported or unknown hosts refuse the\noverride explicitly.\n\n`--effort` requires `--model`; neither option combines with `--terminal`. A\nconnected worker server must advertise launch-preference support before Orca\nforwards either field. Compare `launch.requested` with `launch.effective`; never\nclaim a model or effort from requested arguments alone.\n\n## Reuse after settlement\n\nChoose the terminal's next owner before acknowledging the Delivery. When the\nsame exact agent has immediate follow-up work, recover the proven handle and\ntransfer cleanup ownership to the new Dispatch:\n\n```text\nORCA orchestration worker-show --dispatch --json\nORCA orchestration worker-start --task --terminal --json\n```\n\nOtherwise explicitly retain or release the settled worker. Do not leave it live\nonly to inspect output; archived output remains available through `worker-read`.\n\n## Review ownership\n\nA review-only `worker_done` authorizes synthesis of findings, not coordinator\nfile edits. Dispatch or hand off fixes unless the user explicitly assigned them\nto the coordinator. If the user's plan names a next owner, post-review fixes and\nPR preparation remain with that owner; the coordinator routes and synthesizes.\n" // oxfmt-ignore const ORCHESTRATION_LEGACY_CONTRACT_MIGRATION_REFERENCE_MARKDOWN = "# Legacy contract migration\n\nLoad this reference only for an authority label, adopted Run, compatibility or\nrecovery receipt, or explicit legacy takeover. A newly created attempt always\nuses the current grammar.\n\n## Authority labels\n\n- `[LEGACY COMPATIBILITY]` is live and attested. Run only the exact supported\n command printed with the message, using the same selected executable and\n arguments supplied by the original prompt.\n- `[LEGACY RECOVERY REPLAY — MAY HAVE BEEN SEEN]` is one bounded,\n at-least-once cutover replay. Process it idempotently and acknowledge only\n through the exact displayed guidance.\n- `[LEGACY READ-ONLY]` is inspection-only. It has no reply, acknowledgment, or\n lifecycle mutation.\n- An unlabeled current message uses the current guide and grammar.\n\nAn explicitly selected current Run, attested current binding, current Dispatch,\nor federated attachment takes precedence over legacy fallback. A retained\nadoption record alone does not grant mutation authority. If liveness, principal\nownership, capability, or the exact legacy contract is unproven, degrade to\nread-only inspection and never fall back to local execution.\n\nAdoption preserves the live agent process, PTY/session, terminal handle,\ntab/pane, worktree or folder workspace, Task, and Dispatch. It never restarts or\nreplaces the worker and never revives the retired scheduler. Loss of lifecycle\nauthority does not invalidate the existing process, assignment, or filesystem\nwork. Exact recovery may restore the same PTY once in its original inactive\nbackground tab; it must not spawn, write, signal, stop, switch, focus, split, or\ninject a terminal.\n\n## Compatibility recovery\n\nWhen a compatibility response returns structured next-step arguments, execute\nthose exact arguments with the same selected CLI executable. Do not translate\nfrom memory, broaden the recipient, or retry as a current mutation unless the\nreceipt explicitly authorizes it.\n\nA pending ask, reply, final Dispatch settlement, and consuming check have\ndurable recovery identities. Heartbeat and escalation remain at-least-once\nacross a manual contract-boundary retry. If an ask may already have been\nanswered, run the exact non-consuming recovery check printed by Orca before\ncreating any new question. Never guess among identical question threads.\n\nOn packaged Windows, a legacy ask uses a two-step commit/resume protocol. The\ninitial command commits the question, prints its exact\n`ask --resume ` command, and exits with launcher status `75`. Run\nthat exact resume after the launcher or update boundary. For an attested WSL\nlaunch, preserve the printed `orca-ide` executable and distro route. Older WSL\nworkers without launch proof remain lifecycle read-only even while their\nterminal and filesystem work continue.\n\n## Read-only inspection and takeover\n\nRead-only inspection does not consume mail:\n\n```text\nORCA orchestration run-list --json\nORCA orchestration run-show --id run_legacy_local --json\nORCA orchestration run-show --id --json\nORCA orchestration task-list --run --json\nORCA orchestration inbox --full --json\nORCA orchestration check --terminal --peek --format --json\nORCA terminal read --terminal --json\nORCA terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\n```\n\n`run_legacy_local` is an empty audit tombstone after adoption. Find the ordinary\nRun whose objective is `Recovered orchestration work from a contract update`.\n\nOnly when the original coordinator is unavailable or cannot prove retained\nauthority may a new live coordinator take over from its own terminal:\n\n```text\nORCA orchestration run-use --id --takeover-legacy --json\nORCA orchestration check --run --json\n```\n\nTakeover binds the authenticated invoking terminal; `--from` cannot nominate\nanother coordinator. It fences only the old coordinator and moves pending mail\ninto current Run delivery. It preserves live workers, Tasks, Dispatches, processes, and files.\nNever take over while the original coordinator is actively coordinating.\n\nDo not launch a replacement editor merely because Orca updated or authority is\nunclear. Keep the original worker as the only editor until a stable handoff\npoint, then use a fresh current Dispatch in a conflict-free placement.\n" @@ -87,7 +87,7 @@ const ORCHESTRATION_LEGACY_CONTRACT_MIGRATION_REFERENCE_MARKDOWN = "# Legacy con const ORCHESTRATION_LOW_LEVEL_TOPOLOGY_REFERENCE_MARKDOWN = "# Low-level topology\n\nLoad this reference only when `worker-start` cannot express required custom argv\nor terminal topology. It is not the normal supervised loop and is never a full\nhandoff recipe. A model or effort is not custom argv: `worker-start --model\n--effort` carries them and keeps the user's Chat UI setting.\n\n```text\nORCA terminal create --worktree active --title --command \"\" --json\nORCA terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\nORCA orchestration dispatch --task --to --inject --json\n```\n\nWait for readiness only when startup could lose injected input. Prefer\nagent-first `worker-start` whenever its argv and topology are sufficient.\n\n`dispatch --inject` creates authoritative Task/Dispatch context but deliberately\nkeeps an operator-created process unsupervised: it creates no supervised worker\nresource row. `worker-show`, `worker-read`, and `worker-list` report the lane as\n`unsupervised`; `worker-stop` and `worker-abandon` do not close that process, and\nsettled retain/release take no process action.\n\nUse `worker-start --terminal ` when lifecycle ownership of an existing\nagent terminal is required. Never imply that low-level dispatch retroactively\nowns a process, never use it to route around the nested-depth limit, and never\nuse it for an ownership handoff.\n" // oxfmt-ignore -const ORCHESTRATION_MESSAGING_AND_GATES_REFERENCE_MARKDOWN = "# Messaging and gates\n\nLoad this reference for inbox replay, attempt-specific guidance, group\naddresses, blocking questions, or coordinator-managed DAG decisions.\n\nA successful `send` proves durable enqueue. Wake and nudge are best-effort\nattention only: neither proves the recipient read the message, began a turn, or\naccepted steering.\n\n## Coordinator delivery loop\n\n`check` names its caller with `--terminal ` and is the only verb that\nrejects `--from`. Omit `--terminal` inside an Orca terminal, where Orca resolves\nthe caller; pass it explicitly from anywhere else, including a dispatched\nworker reading coordinator follow-ups.\n\nA consuming coordinator `check` returns the bound Run's oldest FIFO Delivery,\nup to 50 messages, and replays that exact batch until acknowledged. Process\nevery row and required terminal ownership decision before `--ack`. Type filters\ndecide when a waiter wakes; they do not authorize skipping older actionable\nmail. A Delivery therefore always carries the whole FIFO batch whatever its\ntypes, and a `check` without `--wait` hands that batch over unfiltered.\n`--peek` and `--all` are read-only inspection, not progress through the\ncoordinator inbox.\n\nAn empty wait or timeout is a checkpoint. Continue rolling waits until every\nexpected Dispatch settles. Heartbeat or visible activity means alive, not done.\n\n## Addresses\n\nUse a stable Dispatch address for attempt-specific coordinator guidance:\n\n```text\nORCA orchestration send --to dispatch: --subject \"Follow-up\" --body \"\" --json\n```\n\nDo not substitute a remote terminal handle. Omit `--from` for ordinary\ncoordinator calls; a dispatched worker instead copies the exact `--from` and\nother arguments in its preamble. `check` is the exception: it identifies\nits caller with `--terminal`, never `--from`.\n\nGroup addresses include `@all`, `@idle`, `@claude`, `@codex`, `@opencode`,\n`@gemini`, `@droid`, `@grok`, `@cursor`, and `@worktree:`. Every group but\n`@worktree:` means the live Dispatches of the sender's own Run. Mail goes\nto each `dispatch:` mailbox, except a worker coordinating a child Run\nreceives it in that `run:` mailbox. A sender bound to no Run is refused;\n`--run` must match the group audience and never grants membership.\nA Run group excludes its owning coordinator; a worker raising a blocker sends\nto `run:`. A worker that created its own Run addresses that Run's workers,\nnot its siblings. `@worktree:` reaches matching workspace terminals,\nincluding coordinators. Use groups only for intentional fan-out status or\nquestions. `worker_done`, heartbeat, and other\nDispatch lifecycle messages never target groups.\n\n## Questions and gates\n\nA worker uses `ask`; its timeout leaves one durable question pending, which the\nworker resumes by message ID. The coordinator answers that message with `reply`.\n\nUse a gate only for a coordinator-owned Task-DAG decision:\n\n```text\nORCA orchestration gate-create --task --question \"\" --options --json\nORCA orchestration gate-resolve --id --resolution \"\" --json\nORCA orchestration gate-list --task --json\n```\n\nPass `json_array` using the quoting rules of the active shell; do not copy POSIX\nsingle-quote syntax into PowerShell or `cmd.exe`.\n\nDo not create a gate merely to answer a worker's `ask`.\n" +const ORCHESTRATION_MESSAGING_AND_GATES_REFERENCE_MARKDOWN = "# Messaging and gates\n\nLoad this reference for inbox replay, attempt-specific guidance, group\naddresses, blocking questions, or coordinator-managed DAG decisions.\n\nA successful `send` proves durable enqueue. Wake and nudge are best-effort\nattention only: neither proves the recipient read the message, began a turn, or\naccepted steering.\n\n## Coordinator delivery loop\n\n`check` names its caller with `--terminal ` and is the only verb that\nrejects `--from`. Omit `--terminal` inside an Orca terminal, where Orca resolves\nthe caller; pass it explicitly from anywhere else, including a dispatched\nworker reading coordinator follow-ups.\n\nA consuming coordinator `check` returns the bound Run's oldest FIFO Delivery,\nup to 50 messages, and replays that exact batch until acknowledged. Process\nevery row and required terminal ownership decision before `--ack`. Type filters\ndecide when a waiter wakes; they do not authorize skipping older actionable\nmail. A Delivery therefore always carries the whole FIFO batch whatever its\ntypes, and a `check` without `--wait` hands that batch over unfiltered.\n`--peek` and `--all` are read-only inspection, not progress through the\ncoordinator inbox.\n\nAn empty wait or timeout is a checkpoint. Continue rolling waits until every\nexpected Dispatch settles. Heartbeat or visible activity means alive, not done.\n\n## Addresses\n\nUse a stable Dispatch address for attempt-specific coordinator guidance:\n\n```text\nORCA orchestration send --to dispatch: --subject \"Follow-up\" --body \"\" --json\n```\n\nDo not substitute a remote terminal handle. Omit `--from` for ordinary\ncoordinator calls; a dispatched worker instead copies the exact `--from` and\nother arguments in its preamble. `check` is the exception: it identifies\nits caller with `--terminal`, never `--from`.\n\nGroup addresses include `@all`, `@idle`, `@claude`, `@codex`, `@opencode`,\n`@gemini`, `@droid`, `@grok`, `@cursor`, and `@worktree:`. Every group but\n`@worktree:` means the live Dispatches of the sender's own Run. Mail goes\nto each `dispatch:` mailbox, except a worker coordinating a child Run\nreceives it in that `run:` mailbox. A sender bound to no Run is refused;\n`--run` must match the group audience and never grants membership.\nA Run group excludes its owning coordinator; a worker raising a blocker sends\nto `run:`. A worker that created its own Run addresses that Run's workers,\nnot its siblings. `@worktree:` reaches matching workspace terminals,\nincluding coordinators. Use groups only for intentional fan-out status or\nquestions. `worker_done`, heartbeat, and other\nDispatch lifecycle messages never target groups.\n\n## Questions and gates\n\nA worker uses `ask`; its timeout leaves one durable question pending, which the\nworker resumes by message ID. The coordinator answers that message with `reply`;\na helper's question goes to the worker that started it, which answers it the\nsame way.\n\nUse a gate only for a coordinator-owned Task-DAG decision:\n\n```text\nORCA orchestration gate-create --task --question \"\" --options --json\nORCA orchestration gate-resolve --id --resolution \"\" --json\nORCA orchestration gate-list --task --json\n```\n\nPass `json_array` using the quoting rules of the active shell; do not copy POSIX\nsingle-quote syntax into PowerShell or `cmd.exe`.\n\nDo not create a gate merely to answer a worker's `ask`.\n" // oxfmt-ignore const ORCHESTRATION_PLACEMENT_AND_REMOTE_REFERENCE_MARKDOWN = "# Placement and remote execution\n\nLoad this reference before creating a new worktree or placing work through SSH,\nWSL, or another connected Orca server.\n\n## Placement choices\n\nA fresh worker means a fresh agent terminal, not a new Git worktree. Use the\ncurrent or an exact existing workspace by default. Create a worktree only when\nthe user requested one or a concrete checkout or filesystem conflict makes\nsharing unsafe.\n\n```text\n# Current workspace; setup is not rerun.\nORCA orchestration worker-start --task --worktree current --agent codex --json\n\n# Stacked child worktree.\nORCA orchestration worker-start --task --worktree new-child --name --agent codex --setup run --json\n\n# Independent top-level worktree.\nORCA orchestration worker-start --task --worktree new-top-level --name --agent codex --setup run --json\n```\n\nCurrent and exact existing workspaces create a fresh terminal unless\n`--terminal` is explicit. Folder workspaces are first-class; do not invoke Git\nor require worktree lineage when the selected workspace is a folder.\n\nRegister a folder workspace through project setup. `repo add --path `\nrequires a valid Git repository and rejects a plain directory:\n\n```text\nORCA project setup-existing-folder --project --host --path --kind folder --json\n```\n\nThen place work on the returned workspace with an exact selector. A worktree\nselector needs the full `::` value Orca returned, passed as\n`id:`; a bare repo id is not a worktree id. `new-child` and\n`new-top-level` are worktree creation and do not apply to a folder.\n\nNew worktrees use agent-first creation and run setup by default. Preserve the\nrepository's startup policy: `start-immediately` can report setup as `running`,\nwhile `wait-for-setup` gates prompt delivery on success. Orca lineage, Git base,\nfilesystem isolation, coordination parentage, UI grouping, and execution host\nare separate decisions.\n\n## Connected servers\n\nThe Run and Tasks remain authoritative on the current server. `--on` selects\nonly the worker's execution server and appears only on `worker-start`:\n\n```text\nORCA orchestration worker-start --task --on --worktree new-top-level --repo --name --agent codex --setup run --json\n```\n\nRemote `current` and `new-child` are invalid because they are ambiguous across\nservers. Use an exact discovered remote workspace, or `new-top-level` with an\nexact remote repository selector. After start, route every follow-up, read,\nstop, and cleanup by Dispatch ID; never repeat `--on` or substitute a remote\nterminal handle.\n\n```text\nORCA orchestration worker-show --dispatch --json\nORCA orchestration worker-read --dispatch --limit 50 --json\nORCA orchestration send --to dispatch: --subject \"Follow-up\" --body \"\" --json\nORCA orchestration worker-list --run --include-remote --json\n```\n\n`worker-list` reads local fleet state only; enumerate remote workers with\n`--include-remote` or every one of them reads `unverifiable`. Scope every list\nwith `--run `: unscoped, it reports every Dispatch this runtime has\nrecorded, and the workers you are waiting on are lost in that history.\n\n## Execution-host and mixed-version floor\n\nThe execution host owns process, filesystem, transcript, stop, and cleanup\nfacts. Render only `live`, `unverifiable`, or `exited`. Connection loss, relay\nabsence, missing client inventory, or timeout yields `unverifiable`, never\nsynthetic exit and never a client-local substitute action.\n\nClients and servers update independently. Optional response fields may be\nabsent. Forward model/effort, transcript reads, cleanup, or another new remote\noperation only when the peer advertises the relevant capability; unknown stream\nopcodes can be silently dropped. A narrow unsupported response may degrade to a\ndocumented older path, but must not broaden the target or cross the execution\nboundary. Changing host-published content reaches old clients even without a\nwire-shape change, so preserve established semantics or negotiate the behavior.\n\nFor WSL, use the exact executable and arguments returned by Orca so the distro\nand packaged launcher remain bound. Do not translate a printed `orca-ide`\nrecovery command into a PATH-resolved local command.\n" @@ -96,7 +96,7 @@ const ORCHESTRATION_PLACEMENT_AND_REMOTE_REFERENCE_MARKDOWN = "# Placement and r const ORCHESTRATION_RECOVERY_AND_CLEANUP_REFERENCE_MARKDOWN = "# Recovery and cleanup\n\nLoad this reference only after a failed/stopped/unknown attempt, explicit retry\ndecision, stop/abandon request, retention request, or uncertain release.\n\n| Proven state | Safe action |\n| ----------------------- | ------------------------------------------------------------------ |\n| `ready` or active | Keep waiting; optionally read bounded output |\n| `failed` or `stopped` | Start a replacement with `--retry-of`; repeat placement explicitly |\n| `outcome_unknown` | Inspect, then choose `worker-stop` or explicit `worker-abandon` |\n| Accepted `worker_done` | Reuse, retain, or release |\n| Remote contact lost | Preserve `unverifiable`; do not stop or retry from absence alone |\n| `unverifiable` liveness | Keep waiting or inspect; never stop, abandon, retry, or release |\n| Proven `exited` agent | Enumerate with `worker-list`; follow its `nextAction` |\n\n## Inspect before acting\n\n```text\nORCA orchestration worker-list --run --json\nORCA orchestration worker-list --run --include-remote --json\nORCA orchestration worker-show --dispatch --json\nORCA orchestration worker-read --dispatch --limit 50 --json\n```\n\n`worker-list` is the enumerating command and the authority on agent liveness:\neach row carries `projection.liveness`, `projection.attention.categories`,\n`projection.attention.requiresAction`, and a literal `projection.nextAction`\nargv to run. Always scope it with `--run `; an unscoped list reports\nevery Dispatch this runtime has ever recorded and buries the live ones.\n`worker-show`'s `observation.status` is PTY liveness only, so a `live` terminal\nwhose agent died at a trust prompt still reads `live` there.\n\nWhen the two disagree, the fleet verdict decides — unless the fleet row is\n`unverifiable` for a reason that names a gap on this client rather than a fact\nabout the worker. `missing_status`, `host_unavailable`, and\n`capability_unsupported` are such gaps: the first means this runtime holds no\nstatus row, the second that it could not ask the execution host at all, and the\nthird that a stale peer answered but lacks the fleet-snapshot capability.\nAgainst any of them, a `worker-show` verdict sourced from the execution host is\nthe better evidence and outranks the row. Only `host_unavailable` is contact\nloss; the other two mean the host was never asked or answered without the\ncapability.\n\nThis never promotes absence. `unverifiable` from either command still authorizes\nnothing — only a positive `live` or `exited` verdict does.\n\nA worker started with `--on ` reads `unverifiable` until you\nenumerate with `--include-remote`, which asks its execution host for the\nverdict. Rows come newest first and past 100 the response pages, so follow\n`page.nextCursor` with `--cursor ` until `page.hasMore` is false.\n\n## Stall needs positive evidence\n\nLeave the wait only on positive proof the agent stopped: `exited` liveness, the\nworker's own observation of process exit, or a transcript whose final agent turn\nsent no `worker_done`. Only then choose `worker-stop` or `worker-abandon`.\n\n`unverifiable` is always absence — `missing_status`, `stale_status`,\n`restored_unconfirmed`, or a remote worker with no connection — and a null\n`agentWait` or an unchanged `worker-read` tail is that same absence seen again.\nAbsence never authorizes stop, abandon, retry, or release: keep waiting, or\ninspect until you hold one of the positive signals above. A `nextAction` that\nnames an inspecting command is asking for evidence, not for cleanup.\n\n`worker-read --source auto` uses a proven provider transcript when available and\notherwise returns bounded terminal output with a typed `fallbackReason`.\nContinue with its top-level cursor, which is pinned to that source. If Orca\nreports `source_changed`, restart without the old cursor. A bounded initial\ntranscript tail can return an EOF cursor that follows only newly appended records;\nread `contentComplete`, `clipping`, and `warnings` before assuming omitted older\nrecords are pageable. Never guess a provider session ID, transcript path, or\nremote terminal handle.\n\n## Was the mutation applied?\n\nWhen a mutation's response was lost and named no Dispatch, do not replay blind.\nEvery orchestration mutation accepts `--retry-request `, which reuses one\noperation identity so Orca can replay, join, or recover it instead of starting a\nduplicate. Ask what happened first:\n\n```text\nORCA orchestration request-show --request --json\n```\n\n`completed` means the mutation already took effect; read its recorded receipt\ninstead of rerunning. `pending` means the original mutation is still running or\nOrca restarted before recording its outcome; replay the original command with\n`--retry-request `. `absent` means this runtime holds no receipt\nunder your caller identity — that is not proof nothing happened, so inspect the\naffected Task, Dispatch, and terminal before deciding whether to retry.\n\nWhen a worker's terminal accepted input but the submit is unconfirmed, use\n`terminal send --wait-submit `: it observes the accepted prompt for that\nlong and, on timeout, returns the input-accepted receipt without resending.\n\n## Refused starts\n\n`dispatch` and `worker-start` refuse the following preflight cases with a stable\n`error.code`; read it before choosing a recovery, and treat `error.data.nextSteps`\nas the exact recovery text. Older hosts may omit `data`, so treat every field as\noptional.\n\n| Code | Meaning | Recovery |\n| -------------------- | --------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ |\n| `task_not_found` | No Task with that id, or not in the bound Run (`data.taskId`, `data.runId`) | Check `task-list --json`; create the Task with `task-create` if it does not exist |\n| `task_not_startable` | Task cannot start now: not `ready`, or invalid `--retry-of` (`data.status`, `data.unmetDependencies`, `data.retryOf`) | Wait for running dependencies with `check --wait`; retry or unblock failed ones; inspect `dispatch-show` if already dispatched |\n| `inject_rejected` | `--inject` refused because no recognized agent runs in the target (`data.terminal`, `data.reason`) | Start a recognized agent there or pick another terminal; or dispatch without `--inject` and use `terminal send` |\n| `runtime_error` | Any other failure, including a target terminal that already owns an active Dispatch | Read the message, inspect state, and do not retry unchanged |\n\n## Retry, stop, and abandon\n\nRetry only a positively proven failed or stopped attempt. Name the failed Task\nwith `--task`, since `--spec` creates a new one. Placement is never silently\ninherited:\n\n```text\nORCA orchestration worker-start --task --retry-of --worktree --agent --json\n```\n\nAfter three consecutive failures for one Task, its dispatch context\ncircuit-breaks and the Task is failed. Failed `worker-start` attempts count\nacross `--retry-of`; a worker's own `failed` report does not. `--retry-of` a\ncircuit-broken Dispatch is refused with `task_not_startable`: report the failure\nto the user instead. Do not route around that boundary with a new Run, Task, or\nunrelated Dispatch.\n\nFor `outcome_unknown`, inspect first, then make an explicit choice:\n\n```text\nORCA orchestration worker-stop --dispatch --json\nORCA orchestration worker-abandon --dispatch --json\n```\n\n`worker-stop` closes only the exact proven supervised agent terminal. It never\ndeletes the worktree, setup terminal, configured tabs, or unrelated processes.\n`worker-abandon` fences orchestration while accepting that resources may remain\nlive; it performs no remote, process, or filesystem action. When it settles a\nworker, it retains an owned terminal by the same rule as `worker-retain`, so Orca\nstops owing its release; a release already committed (`releasing`,\n`release_unknown`) is left as it is. An already-settled worker is left untouched.\n\nTo cancel a Task, settle its worker with `worker-stop` or `worker-abandon`, then\nrecord the reason. A cancelled Task is `failed`, so its dependents stay blocked\nuntil a `--retry-of` replacement completes it:\n\n```text\nORCA orchestration task-update --id --status failed --result cancelled --json\n```\n\n## Retain and release\n\n```text\nORCA orchestration worker-retain --dispatch --json\nORCA orchestration worker-release --dispatch --json\n```\n\nRetain only when the user explicitly wants the settled terminal kept live.\nRelease works after succeeded and failed reports, archives readable output, and\ncloses only the exact terminal owned by that settled Dispatch. Replays may call\nrelease again safely. Reused, pre-existing, setup, coordinator, active,\nuser-taken-over, and unproven terminals are retained.\n\nA `worker-start` that failed before its agent was ready still owns the terminal\nit created. Its receipt names `worker-release`, and `worker-list` reports that\nrow as `reclaimable`; release it there rather than closing the terminal by hand.\nA `--retry-of` replacement for a `failed` worker releases the terminal that\nworker still owns the same way.\n\nNever release because of timeout, TUI idle, heartbeat, status, question,\nescalation, or stale/rejected completion. If the receipt says `release_pending`\nor `release_unknown`, follow its exact recovery action. Never substitute\n`terminal close`.\n\n`orchestration reset` is destructive recovery. Do not run it during active\ncoordination unless the user explicitly abandons that state.\n" // oxfmt-ignore -const ORCHESTRATION_WORKER_CONTRACT_REFERENCE_MARKDOWN = "# Worker contract\n\nThe injected preamble is authoritative. Copy its command rather than\nreconstructing flags. In particular, preserve the exact executable, worker\nhandle, Task ID, and Dispatch ID, and keep any other flag it carries (an older\nOrca host adds `--dispatch-capability`).\n\n## Heartbeat\n\nSend heartbeats only at the cadence required by the live preamble. Skip them\nwhile blocked inside `ask` or `check --wait`; those calls are liveness signals.\n\n```text\nORCA orchestration send --from --type heartbeat --subject \"alive\" --task-id --dispatch-id --phase \"\"\n```\n\nUse typed lifecycle flags, not a hand-written JSON payload. A heartbeat proves\nliveness, never completion.\n\n## Ask and resume\n\nUse Orca `ask` whenever the coordinator must answer. Never open a local question\nTUI the coordinator cannot answer.\n\n```text\nORCA orchestration ask --from --question \"\" --options \",\" --timeout-ms 600000\n\nORCA orchestration ask --from --resume --timeout-ms 600000\n```\n\nA timeout or disconnect leaves the original question pending. Resume its\nmessage ID; do not create a duplicate question.\n\n## Reading coordinator follow-ups\n\nThe coordinator steers a running worker with `send --to dispatch:`. That\nenqueue is durable but does not interrupt you, so nothing arrives unless you\nlook:\n\n```text\nORCA orchestration check --terminal --json\n```\n\nRun it at each natural checkpoint — before starting a new file, after a test\nrun — and once more immediately before `worker_done`, so a redirect or a\ncancellation lands before the Task settles. `check` names its caller with\n`--terminal`, never `--from`. Stop checking after `worker_done`.\n\nIf `check` returns `consumer_fenced`, this process no longer owns its Dispatch:\nthe Attempt was re-attached to another worker or settled without you. Stop, do\nnot send `worker_done`, and do not retry the check. An empty `check` never means\nyou were replaced; `consumer_fenced` is the only way you learn that.\nIf `send` or `ask` returns `consumer_fenced`, the command ran from another\nparty's terminal (a coordinator or another worker); run it from your own terminal.\n\n## Escalation\n\nEscalate only before completion and only when the coordinator must intervene:\n\n```text\nORCA orchestration send --from --type escalation --subject \"Blocked: \" --body \"
\" --task-id --dispatch-id \n```\n\n## Completion\n\nSend exactly one terminal report. `--body` is three sentences: what changed,\nwhat was found, and what remains. Use `--outcome failed` when the requested work\nis not complete; never hide failure in prose or silently exit.\n\nAppend `--files-modified` or `--report-path` only when applicable, using actual\npaths. Do not send documentation placeholders as metadata.\n\n```text\nORCA orchestration send --from --type worker_done --subject \"\" --body \"\" --task-id --dispatch-id --outcome succeeded\n```\n\nAfter `worker_done`, end the dispatched turn and idle. Do not poll, close your\nown terminal, or begin unrelated work. A later direct user instruction is new\nuser-owned work and must not reuse settled lifecycle IDs; a supervised follow-up\narrives with a fresh preamble and Task block.\n" +const ORCHESTRATION_WORKER_CONTRACT_REFERENCE_MARKDOWN = "# Worker contract\n\nThe injected preamble is authoritative. Copy its command rather than\nreconstructing flags. In particular, preserve the exact executable, worker\nhandle, Task ID, and Dispatch ID, and keep any other flag it carries (an older\nOrca host adds `--dispatch-capability`).\n\n## Heartbeat\n\nSend heartbeats only at the cadence required by the live preamble. Skip them\nwhile blocked inside `ask` or `check --wait`; those calls are liveness signals.\n\n```text\nORCA orchestration send --from --type heartbeat --subject \"alive\" --task-id --dispatch-id --phase \"\"\n```\n\nUse typed lifecycle flags, not a hand-written JSON payload. A heartbeat proves\nliveness, never completion.\n\n## Ask and resume\n\nUse Orca `ask` whenever the coordinator must answer. Never open a local question\nTUI the coordinator cannot answer.\n\n```text\nORCA orchestration ask --from --question \"\" --options \",\" --timeout-ms 600000\n\nORCA orchestration ask --from --resume --timeout-ms 600000\n```\n\nA timeout or disconnect leaves the original question pending. Resume its\nmessage ID; do not create a duplicate question.\n\n## Reading coordinator follow-ups\n\nThe coordinator steers a running worker with `send --to dispatch:`. That\nenqueue is durable but does not interrupt you, so nothing arrives unless you\nlook:\n\n```text\nORCA orchestration check --terminal --json\n```\n\nRun it at each natural checkpoint — before starting a new file, after a test\nrun — and once more immediately before `worker_done`, so a redirect or a\ncancellation lands before the Task settles. `check` names its caller with\n`--terminal`, never `--from`. Stop checking after `worker_done`.\n\nIf `check` returns `consumer_fenced`, this process no longer owns its Dispatch:\nthe Attempt was re-attached to another worker or settled without you. Stop, do\nnot send `worker_done`, and do not retry the check. An empty `check` never means\nyou were replaced; `consumer_fenced` is the only way you learn that.\nIf `send` or `ask` returns `consumer_fenced`, the command ran from another\nparty's terminal (a coordinator or another worker); run it from your own terminal.\n\n## Helper workers\n\nWhen the preamble has a SUB-DISPATCH section, start helpers on your own\nDispatch's Run. Do not `run-create` or `run-use` for them.\n\n```text\nORCA orchestration worker-start --from --spec \"\" --worktree current --agent --json\nORCA orchestration reply --from --id --body \"\"\n```\n\nTheir questions, escalations, and `worker_done` arrive in your own `check`, next\nto your coordinator's mail. Answer their questions with `reply`, and wait for\nevery helper's `worker_done` before sending yours. After you settle, a helper\nstill running reports to your coordinator's Run.\n\n## Escalation\n\nEscalate only before completion and only when the coordinator must intervene:\n\n```text\nORCA orchestration send --from --type escalation --subject \"Blocked: \" --body \"
\" --task-id --dispatch-id \n```\n\n## Completion\n\nSend exactly one terminal report. `--body` is three sentences: what changed,\nwhat was found, and what remains. Use `--outcome failed` when the requested work\nis not complete; never hide failure in prose or silently exit.\n\nAppend `--files-modified` or `--report-path` only when applicable, using actual\npaths. Do not send documentation placeholders as metadata.\n\n```text\nORCA orchestration send --from --type worker_done --subject \"\" --body \"\" --task-id --dispatch-id --outcome succeeded\n```\n\nAfter `worker_done`, end the dispatched turn and idle. Do not poll, close your\nown terminal, or begin unrelated work. A later direct user instruction is new\nuser-owned work and must not reuse settled lifecycle IDs; a supervised follow-up\narrives with a fresh preamble and Task block.\n" // oxfmt-ignore export const BUNDLED_SKILL_GUIDES = [ diff --git a/src/main/runtime/orca-runtime-test-orchestration-messages.spec.ts b/src/main/runtime/orca-runtime-test-orchestration-messages.spec.ts index 56fa9ca5aac..ce6b82e82d5 100644 --- a/src/main/runtime/orca-runtime-test-orchestration-messages.spec.ts +++ b/src/main/runtime/orca-runtime-test-orchestration-messages.spec.ts @@ -267,11 +267,13 @@ export class InMemoryOrchestrationMessages { ) } - markAsDelivered(ids: string[]): void { - const deliveredIds = new Set(ids) + markAsDelivered(ids: string[], mailboxHandle: string): void { + const requested = new Set(ids) + const deliveredIds = new Set() for (const message of this.messages) { - if (deliveredIds.has(message.id)) { + if (requested.has(message.id) && message.to_handle === mailboxHandle) { message.delivered_at = '1970-01-01 00:00:00' + deliveredIds.add(message.id) } } this.clearMailboxPointerEnter(deliveredIds) diff --git a/src/main/runtime/orchestration/db-message-timestamp.test.ts b/src/main/runtime/orchestration/db-message-timestamp.test.ts index 3900904e5b1..3392f96c4b5 100644 --- a/src/main/runtime/orchestration/db-message-timestamp.test.ts +++ b/src/main/runtime/orchestration/db-message-timestamp.test.ts @@ -16,7 +16,7 @@ describe('orchestration message timestamps', () => { }) expect(message.created_at).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$/) - db.markAsDelivered([message.id]) + db.markAsDelivered([message.id], 'b') expect(db.getMessageById(message.id)?.delivered_at).toMatch( /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$/ ) diff --git a/src/main/runtime/orchestration/db-messages.test.ts b/src/main/runtime/orchestration/db-messages.test.ts index e803aad3640..96d9c7fed18 100644 --- a/src/main/runtime/orchestration/db-messages.test.ts +++ b/src/main/runtime/orchestration/db-messages.test.ts @@ -76,7 +76,7 @@ describe('OrchestrationDb', () => { const m1 = d.insertMessage({ runId, from: 'a', to: 'b', subject: 'one' }) const m2 = d.insertMessage({ runId, from: 'a', to: 'b', subject: 'two' }) - d.markAsDelivered([m1.id]) + d.markAsDelivered([m1.id], 'b') // Push delivery query: only undelivered, unread. const pending = d.getUndeliveredUnreadMessages('b') diff --git a/src/main/runtime/orchestration/db-undelivered-mailboxes.test.ts b/src/main/runtime/orchestration/db-undelivered-mailboxes.test.ts index b20dd625310..e6a15689e0e 100644 --- a/src/main/runtime/orchestration/db-undelivered-mailboxes.test.ts +++ b/src/main/runtime/orchestration/db-undelivered-mailboxes.test.ts @@ -22,7 +22,7 @@ describe('undelivered orchestration mailboxes', () => { }) db.insertMessage({ runId: 'run_legacy_local', from: 'a', to: 'pending', subject: 'first' }) db.insertMessage({ runId: 'run_legacy_local', from: 'a', to: 'pending', subject: 'second' }) - db.markAsDelivered([delivered.id]) + db.markAsDelivered([delivered.id], 'delivered') db.markAsRead([read.id]) expect(db.getUndeliveredUnreadMailboxHandles()).toEqual(['pending']) diff --git a/src/main/runtime/orchestration/db/messages/helper-mailbox-hand-back.ts b/src/main/runtime/orchestration/db/messages/helper-mailbox-hand-back.ts new file mode 100644 index 00000000000..63f0366edc1 --- /dev/null +++ b/src/main/runtime/orchestration/db/messages/helper-mailbox-hand-back.ts @@ -0,0 +1,95 @@ +import type { OrchestrationDb } from '../orchestration-db' + +/** Mail a settling helper creator never handled, now owned by its Run's mailbox. */ +export type MailHandBack = { mailbox: string; messageIds: string[]; types: string[] } + +/** + * A settled worker no longer reads its mailbox, so its helpers' unread reports and unanswered + * questions (even ones it read) become the Run coordinator's, delivered as if they just arrived. + */ +export function handHelperMailToRun(db: OrchestrationDb, dispatchId: string): void { + // Why: every settlement runs this, so read only the Run, not the whole dispatch row. + const runId = db.db + .prepare('SELECT run_id FROM dispatch_contexts WHERE id = ?') + .get(dispatchId)?.run_id + if (typeof runId !== 'string') { + return + } + const mailbox = `dispatch:${dispatchId}` + // Why: the recipient mailbox is the ownership fact; report payloads and senders are optional. + const rows = db.db + .prepare( + `SELECT id, type FROM messages + WHERE run_id = ? AND to_handle = ? AND ( + (read = 0 AND delivery_contract = 'current_delivery' + AND type IN ('worker_done', 'escalation', 'status')) + OR id IN ( + SELECT message_id FROM question_threads WHERE run_id = ? AND status = 'pending'))` + ) + .all(runId, mailbox, runId) + if ( + rows.length === 0 || + !db.db + .prepare( + 'SELECT 1 FROM dispatch_contexts WHERE creator_dispatch_id = ? AND run_id = ? LIMIT 1' + ) + .get(dispatchId, runId) + ) { + return + } + const handBack: MailHandBack = { + mailbox: `run:${runId}`, + messageIds: rows.map((row) => String(row.id)), + types: [...new Set(rows.map((row) => String(row.type)))] + } + // Why: the old mailbox's delivery stamp and pointer reservation would hide the rows from the Run. + db.db + .prepare( + `UPDATE messages + SET to_handle = ?, read = 0, delivered_at = NULL, pointer_enter_pending = 0, + pointer_pty_id = NULL, pointer_process_incarnation = NULL + WHERE id IN (SELECT value FROM json_each(?))` + ) + .run(handBack.mailbox, JSON.stringify(handBack.messageIds)) + db.fenceUnacknowledgedMailboxDeliveries(mailbox) + db.pendingMailHandBacks.push(handBack) + if (db.pendingMailHandBacks.length === 1) { + // Why: settlements are synchronous, so this runs after the outermost COMMIT or ROLLBACK. + queueMicrotask(() => { + for (const pending of db.pendingMailHandBacks.splice(0)) { + db.mailHandBackListener?.(pending) + } + }) + } +} + +/** The types of handed-back rows the Run still owes attention to, read after the settlement. */ +export function committedHandBackTypes(db: OrchestrationDb, handBack: MailHandBack): string[] { + return db.db + .prepare( + `SELECT DISTINCT type FROM messages + WHERE to_handle = ? AND read = 0 AND delivery_contract = 'current_delivery' + AND id IN (SELECT value FROM json_each(?)) + AND (type <> 'question' + OR id IN (SELECT message_id FROM question_threads WHERE status = 'pending'))` + ) + .all(handBack.mailbox, JSON.stringify(handBack.messageIds)) + .map((row) => String(row.type)) +} + +/** The Run mail a helper creator's settlement may have handed back that still owes attention. */ +export function handBackReplayTargets(db: OrchestrationDb, dispatchId: string) { + return db.db + .prepare( + `SELECT DISTINCT m.to_handle AS mailbox, m.type FROM dispatch_contexts AS d + JOIN messages AS m ON m.to_handle = 'run:' || d.run_id + WHERE d.id = ? AND m.read = 0 AND m.delivery_contract = 'current_delivery' + AND EXISTS (SELECT 1 FROM dispatch_contexts AS h + WHERE h.creator_dispatch_id = d.id AND h.run_id = d.run_id) + AND (m.type IN ('worker_done', 'escalation', 'status') OR (m.type = 'question' + AND m.id IN (SELECT message_id FROM question_threads + WHERE run_id = d.run_id AND status = 'pending')))` + ) + .all(dispatchId) + .map((row) => ({ to: String(row.mailbox), type: String(row.type) })) +} diff --git a/src/main/runtime/orchestration/db/messages/message-inbox.ts b/src/main/runtime/orchestration/db/messages/message-inbox.ts index e9b800d431e..3b7a3ea75c8 100644 --- a/src/main/runtime/orchestration/db/messages/message-inbox.ts +++ b/src/main/runtime/orchestration/db/messages/message-inbox.ts @@ -9,7 +9,8 @@ const MESSAGE_MUTATION_SAVEPOINT = 'message_id_mutation' function runBatchedMessageMutation( db: OrchestrationDb, ids: string[], - sqlForPlaceholders: (placeholders: string) => string + sqlForPlaceholders: (placeholders: string) => string, + leadingBindings: string[] = [] ): void { if (ids.length === 0) { return @@ -19,7 +20,7 @@ function runBatchedMessageMutation( for (let offset = 0; offset < ids.length; offset += MESSAGE_ID_UPDATE_BATCH_SIZE) { const batch = ids.slice(offset, offset + MESSAGE_ID_UPDATE_BATCH_SIZE) const placeholders = batch.map(() => '?').join(',') - db.db.prepare(sqlForPlaceholders(placeholders)).run(...batch) + db.db.prepare(sqlForPlaceholders(placeholders)).run(...leadingBindings, ...batch) } db.db.exec(`RELEASE ${MESSAGE_MUTATION_SAVEPOINT}`) } catch (error) { @@ -165,7 +166,8 @@ export function markAsRead(this: OrchestrationDb, ids: string[]): void { } // Why: use datetime('now') so delivered_at matches the space-format UTC shape of the table's other timestamps for correct ordering (§3.2). -export function markAsDelivered(this: OrchestrationDb, ids: string[]): void { +// Scoped to the pointed mailbox so a late stamp cannot mark mail since handed elsewhere. +export function markAsDelivered(this: OrchestrationDb, ids: string[], mailboxHandle: string): void { runBatchedMessageMutation( this, ids, @@ -173,7 +175,8 @@ export function markAsDelivered(this: OrchestrationDb, ids: string[]): void { `UPDATE messages SET delivered_at = datetime('now'), pointer_enter_pending = 0, pointer_pty_id = NULL, pointer_process_incarnation = NULL - WHERE id IN (${placeholders})` + WHERE to_handle = ? AND id IN (${placeholders})`, + [mailboxHandle] ) } diff --git a/src/main/runtime/orchestration/db/orchestration-db.ts b/src/main/runtime/orchestration/db/orchestration-db.ts index 10ec5a1b6c1..69366c585cf 100644 --- a/src/main/runtime/orchestration/db/orchestration-db.ts +++ b/src/main/runtime/orchestration/db/orchestration-db.ts @@ -1,5 +1,6 @@ import Database from '../../../sqlite/sync-database' import { attachOrchestrationDbMethods } from './attach-orchestration-db-methods' +import type { MailHandBack } from './messages/helper-mailbox-hand-back' import { hardenOrchestrationDatabaseFiles } from './database-file-permissions' import { backfillFederatedStubHomeRuns } from './federation/federated-stub-home-run-backfill' import type { OrchestrationDbMethods } from './orchestration-db-methods' @@ -24,6 +25,10 @@ class OrchestrationDbCore { // per-terminal fan-out. Only createDispatchContext flips this false→true. hasAnyDispatchContextsCache: boolean | undefined localMutationCallerFingerprint: string | undefined + // Why: in memory only; the re-armed rows themselves are what restart recovery reads. + pendingMailHandBacks: MailHandBack[] = [] + /** The runtime's wake for mail a settlement handed to a Run, called once the transaction ends. */ + mailHandBackListener: ((handBack: MailHandBack) => void) | undefined constructor(dbPath: (string & {}) | ':memory:') { this.db = new Database(dbPath) diff --git a/src/main/runtime/orchestration/db/questions/question-threads.ts b/src/main/runtime/orchestration/db/questions/question-threads.ts index 126ffa8e3b3..07f668fe4b2 100644 --- a/src/main/runtime/orchestration/db/questions/question-threads.ts +++ b/src/main/runtime/orchestration/db/questions/question-threads.ts @@ -2,6 +2,7 @@ import type { MessageRow, QuestionRow } from '../../types' import { OrchestrationError } from '../../orchestration-error' import { exposeQuestionTimestamps } from '../utc-timestamp' import type { OrchestrationDb } from '../orchestration-db' +import { handHelperMailToRun } from '../messages/helper-mailbox-hand-back' export function createQuestion( this: OrchestrationDb, @@ -11,6 +12,8 @@ export function createQuestion( askerHandle: string question: string options?: string[] + /** The asker's report mailbox; the Run when omitted. */ + to?: string } ): { question: QuestionRow; message: MessageRow } { this.db.exec('BEGIN IMMEDIATE') @@ -29,7 +32,7 @@ export function createQuestion( } const message = this.insertMessage({ from: `dispatch:${params.dispatchId}`, - to: `run:${params.runId}`, + to: params.to ?? `run:${params.runId}`, subject: 'Question', body: params.question, type: 'question', @@ -75,13 +78,21 @@ export function answerQuestion( params: { messageId: string runId: string - consumerGeneration: number body: string - } + } & ( + | { consumerGeneration: number } + /** A worker answering its own helper, fenced on its Dispatch instead of the Run's consumer. */ + | { answeredBy: { dispatchId: string; consumerGeneration: number } } + ) ): { question: QuestionRow; message: MessageRow; duplicate: boolean } { + const answeredBy = 'answeredBy' in params ? params.answeredBy : null + const byDispatch = answeredBy?.dispatchId ?? null + const generation = 'consumerGeneration' in params ? params.consumerGeneration : null this.db.exec('BEGIN IMMEDIATE') try { - this.requireCurrentConsumer(params.runId, params.consumerGeneration) + if (generation !== null) { + this.requireCurrentConsumer(params.runId, generation) + } const question = this.getQuestionRaw(params.messageId) if (!question || question.run_id !== params.runId) { throw new OrchestrationError( @@ -89,6 +100,20 @@ export function answerQuestion( `Question ${params.messageId} was not found in Run ${params.runId}.` ) } + if (answeredBy !== null) { + const answerer = this.getDispatchContextById(answeredBy.dispatchId) + if ( + this.getMessageById(question.message_id)?.to_handle !== + `dispatch:${answeredBy.dispatchId}` || + (answerer?.status !== 'pending' && answerer?.status !== 'dispatched') || + answerer.consumer_generation !== answeredBy.consumerGeneration + ) { + throw new OrchestrationError( + 'consumer_fenced', + `Question ${params.messageId} was not asked of active Dispatch ${answeredBy.dispatchId}.` + ) + } + } if (question.status === 'closed') { throw new OrchestrationError( 'dispatch_inactive', @@ -111,7 +136,7 @@ export function answerQuestion( } const message = this.insertMessage({ - from: `run:${params.runId}`, + from: byDispatch !== null ? `dispatch:${byDispatch}` : `run:${params.runId}`, to: `dispatch:${question.dispatch_id}`, subject: 'Re: Question', body: params.body, @@ -127,7 +152,7 @@ export function answerQuestion( answered_by_generation = ?, answered_at = datetime('now') WHERE message_id = ? AND status = 'pending'` ) - .run(message.id, params.body, params.consumerGeneration, question.message_id) + .run(message.id, params.body, generation, question.message_id) const answered = this.getQuestionRaw(question.message_id) as QuestionRow const storedMessage = this.getMessageById(message.id) as MessageRow this.db.exec('COMMIT') @@ -143,6 +168,8 @@ export function answerQuestion( } export function closeQuestionsForDispatch(this: OrchestrationDb, dispatchId: string): string[] { + // Every settlement path calls this, so it is also where helper mail moves to the Run. + handHelperMailToRun(this, dispatchId) const rows = this.db .prepare("SELECT message_id FROM question_threads WHERE dispatch_id = ? AND status = 'pending'") .all(dispatchId) as { message_id: string }[] diff --git a/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-start.ts b/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-start.ts index 1d77e59ba4e..7e7682a6a0f 100644 --- a/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-start.ts +++ b/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-start.ts @@ -41,6 +41,8 @@ export function createStartingWorkerDispatch( /** Who is dispatching, for nesting depth. Required so a new caller must decide. */ creator: DispatchCreator maxDepth: number + /** A worker's own Dispatch that authorized this helper; it must still be the creator here. */ + requiredCreatorDispatchId?: string } ): { dispatch: DispatchContextRow; worker: WorkerDispatchRow; task: TaskRow } { this.db.exec('BEGIN IMMEDIATE') @@ -133,6 +135,16 @@ export function createStartingWorkerDispatch( const id = generateId('ctx') const creatorDispatchId = this.resolveCreatorDispatchId(params.creator) + if ( + params.requiredCreatorDispatchId !== undefined && + creatorDispatchId !== params.requiredCreatorDispatchId + ) { + throw new OrchestrationError( + 'consumer_fenced', + `Dispatch ${params.requiredCreatorDispatchId} is no longer this caller's active work, so it cannot start a helper.`, + { effectsApplied: false } + ) + } if (params.mutationReceipt) { this.db .prepare( diff --git a/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-stop.ts b/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-stop.ts index 62ff9dba3c5..8b68d73dfe4 100644 --- a/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-stop.ts +++ b/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-stop.ts @@ -145,6 +145,8 @@ export function settleWorkerStop(this: OrchestrationDb, dispatchId: string): Wor }) } reconcileTaskAfterDispatchInterruption(this, dispatch.task_id, dispatchId) + // Why: helpers keep reporting to a stopping creator, so mail sent since begin-stop moves here. + this.closeQuestionsForDispatch(dispatchId) this.db.exec('COMMIT') return this.getWorkerDispatch(dispatchId) as WorkerDispatchRow } catch (error) { @@ -201,6 +203,7 @@ export function reconcileFederatedWorkerStop( }) } reconcileTaskAfterDispatchInterruption(this, dispatch.task_id, dispatchId) + this.closeQuestionsForDispatch(dispatchId) commitLifecycleWriteTransaction(this.db, transaction) return this.getWorkerDispatch(dispatchId) as WorkerDispatchRow } catch (error) { diff --git a/src/main/runtime/orchestration/dispatch-report-mailbox.ts b/src/main/runtime/orchestration/dispatch-report-mailbox.ts new file mode 100644 index 00000000000..5dcc431db8a --- /dev/null +++ b/src/main/runtime/orchestration/dispatch-report-mailbox.ts @@ -0,0 +1,41 @@ +import type { OrchestrationDb } from './db' +import type { DispatchContextRow, MessageType } from './types' + +/** Reports about the worker's own task; gates and handoffs stay with the Run's coordinator. */ +export function isDispatchReportType(type: MessageType | undefined): boolean { + return ( + type === undefined || + type === 'worker_done' || + type === 'heartbeat' || + type === 'escalation' || + type === 'status' + ) +} + +/** The live Dispatch that started this one in the same Run: the worker it reports to. */ +export function activeCreatorDispatch( + db: OrchestrationDb, + dispatch: Pick +): DispatchContextRow | undefined { + const creator = dispatch.creator_dispatch_id + ? db.getDispatchContextById(dispatch.creator_dispatch_id) + : undefined + return creator && + creator.run_id === dispatch.run_id && + (creator.status === 'pending' || creator.status === 'dispatched') + ? creator + : undefined +} + +/** + * Where a Dispatch's reports go: the worker that started it while that worker is still on its + * own task, otherwise the Run. Derived on every send, so a helper whose creator settled reports + * to the Run without anything being rewritten. + */ +export function dispatchReportMailbox( + db: OrchestrationDb, + dispatch: Pick +): string { + const creator = activeCreatorDispatch(db, dispatch) + return creator ? `dispatch:${creator.id}` : `run:${dispatch.run_id}` +} diff --git a/src/main/runtime/orchestration/mailbox-pointer-stage.ts b/src/main/runtime/orchestration/mailbox-pointer-stage.ts index 3c0bea2baad..918827b486b 100644 --- a/src/main/runtime/orchestration/mailbox-pointer-stage.ts +++ b/src/main/runtime/orchestration/mailbox-pointer-stage.ts @@ -152,7 +152,7 @@ function finishPointerWriteAndStageEnter { } rejectLastMessageUpdate(sqlite) - expect(() => db?.[method](messageIds)).toThrow('blocked') + expect(() => + method === 'markAsDelivered' + ? db?.markAsDelivered(messageIds, 'recipient') + : db?.[method](messageIds) + ).toThrow('blocked') const changed = sqlite.prepare(changedCountSql).get() as { count: number } expect(changed.count).toBe(0) diff --git a/src/main/runtime/orchestration/preamble.test.ts b/src/main/runtime/orchestration/preamble.test.ts index e6238e20578..32180049aad 100644 --- a/src/main/runtime/orchestration/preamble.test.ts +++ b/src/main/runtime/orchestration/preamble.test.ts @@ -86,8 +86,8 @@ describe('buildDispatchPreamble', () => { expect(headings).toHaveLength(0) expect(codeBlocks).toHaveLength(2) expect(codeBlocks[1]).toMatchObject({ lang: 'sh' }) - expect(codeBlocks[1].value).toContain('orchestration worker-start --task ') - expect(result).toContain('able to dispatch further.\n\n---') + expect(codeBlocks[1].value).toContain('orchestration worker-start --from') + expect(result).toContain('helpers of their own.\n\n---') expect(result).toContain('before starting.\n\n---') }) @@ -256,11 +256,11 @@ describe('sub-dispatch section', () => { ) }) - it('appears with the run-create sequence when budget remains', () => { + it('starts helpers on this Dispatch, without a Run of their own, when budget remains', () => { const preamble = buildDispatchPreamble({ ...base, canDispatchSubWorkers: true }) expect(preamble).toContain('=== SUB-DISPATCH ===') - expect(preamble).toContain('orchestration run-create') - expect(preamble).toContain('orchestration worker-start') + expect(preamble).toContain('orchestration worker-start --from term_worker --spec') + expect(preamble).not.toContain('orchestration run-create') }) it('keeps the task block last so the spec is not buried', () => { diff --git a/src/main/runtime/orchestration/preamble.ts b/src/main/runtime/orchestration/preamble.ts index 8c5d04aed44..fda5a9132be 100644 --- a/src/main/runtime/orchestration/preamble.ts +++ b/src/main/runtime/orchestration/preamble.ts @@ -36,6 +36,8 @@ export type PreambleParams = { workerKind?: 'prompt-returning-agent' | 'bare-shell' // Why gated: advertising a verb the depth cap will reject just burns a turn. canDispatchSubWorkers?: boolean + // Why: a remote worker has no Dispatch row on its own host to start helpers under. + subWorkersInOwnRun?: boolean } // Why: 5 minutes is frequent enough that the coordinator's stale-heartbeat @@ -155,7 +157,11 @@ ${postDoneInstructions}` const drift = params.baseDrift && params.baseDrift.behind > 0 ? buildDriftSection(params.baseDrift) : '' - const subDispatch = params.canDispatchSubWorkers ? buildSubDispatchSection(cli) : '' + const subDispatch = params.canDispatchSubWorkers + ? params.subWorkersInOwnRun + ? buildOwnRunSubDispatchSection(cli) + : buildSubDispatchSection(cli, params.workerHandle) + : '' return `${header}${drift}${subDispatch} @@ -235,7 +241,27 @@ ${chat ? CHAT_REDISPATCH_PARAGRAPH : TERMINAL_REDISPATCH_PARAGRAPH}` // as a blocker. // Why fenced + blank line before the closing `---`: unfenced `` are stripped as raw // HTML by the Chat UI, and a rule directly under a paragraph is a setext H2 (giant last sentence). -function buildSubDispatchSection(cli: string): string { +function buildSubDispatchSection(cli: string, workerHandle: string): string { + return ` + +=== SUB-DISPATCH === +You may start helper workers for this task. They work in this Dispatch's Run and +report to you: their questions, escalations, and worker_done arrive in your own +\`check\`, and your coordinator's mail keeps arriving there too. + +\`\`\`sh + ${cli} orchestration worker-start --from ${workerHandle} --spec "" --worktree current --agent --json + # Answer a helper's question by its message ID: + ${cli} orchestration reply --from ${workerHandle} --id --body "" +\`\`\` + +Wait for every helper's worker_done before reporting your own. Nesting is +capped; your helpers may not be able to start helpers of their own. + +---` +} + +function buildOwnRunSubDispatchSection(cli: string): string { return ` === SUB-DISPATCH === diff --git a/src/main/runtime/orchestration/structured-mailbox-pointer-delivery.test.ts b/src/main/runtime/orchestration/structured-mailbox-pointer-delivery.test.ts index c2f1ac9fb06..f89c129da53 100644 --- a/src/main/runtime/orchestration/structured-mailbox-pointer-delivery.test.ts +++ b/src/main/runtime/orchestration/structured-mailbox-pointer-delivery.test.ts @@ -162,7 +162,7 @@ describe('structured mailbox pointer delivery', () => { await flush() expect(send).toHaveBeenCalledTimes(1) expect(send.mock.calls[0]![0].operationId).toMatch(/^\d{13}-[0-9a-f]{32}$/) - expect(markAsDelivered).toHaveBeenCalledWith(['m1']) + expect(markAsDelivered).toHaveBeenCalledWith(['m1'], 'dispatch:d1') }) it('nudges through the worker`s own handle for direct peer mail outside a dispatch', async () => { @@ -180,7 +180,7 @@ describe('structured mailbox pointer delivery', () => { expect(send.mock.calls[0]![0].body.blocks[0]).toMatchObject({ text: expect.not.stringContaining('--run') }) - expect(markAsDelivered).toHaveBeenCalledWith(['m1']) + expect(markAsDelivered).toHaveBeenCalledWith(['m1'], IDENTITY.handle) }) it('releases its in-flight lane when reading the context fails', async () => { @@ -194,7 +194,7 @@ describe('structured mailbox pointer delivery', () => { delivery.deliverForHandle('dispatch:d1') await flush() expect(send).toHaveBeenCalledOnce() - expect(markAsDelivered).toHaveBeenCalledWith(['m1']) + expect(markAsDelivered).toHaveBeenCalledWith(['m1'], 'dispatch:d1') }) it('retains mail when the dispatch settles unknown', async () => { @@ -224,7 +224,7 @@ describe('structured mailbox pointer delivery', () => { delivery.onJournalActivity('session-1') await flush() expect(send).toHaveBeenCalledTimes(1) - expect(markAsDelivered).toHaveBeenCalledWith(['m1']) + expect(markAsDelivered).toHaveBeenCalledWith(['m1'], 'dispatch:d1') }) it('rechecks a held notice when the answer edge lands during its host call', async () => { @@ -238,7 +238,7 @@ describe('structured mailbox pointer delivery', () => { delivery.deliverForHandle('dispatch:d1') await flush() expect(send).toHaveBeenCalledTimes(2) - expect(markAsDelivered).toHaveBeenCalledWith(['m1']) + expect(markAsDelivered).toHaveBeenCalledWith(['m1'], 'dispatch:d1') }) it('sends the pointer through the chat, with who it is from, and counts it pointed once queued', async () => { @@ -259,7 +259,7 @@ describe('structured mailbox pointer delivery', () => { } }) // The chat's queue holds it now, as it holds the person's: the same mail is not pointed again. - expect(markAsDelivered).toHaveBeenCalledWith(['m1']) + expect(markAsDelivered).toHaveBeenCalledWith(['m1'], 'dispatch:d1') expect(stored.has('dispatch:d1')).toBe(false) }) @@ -305,7 +305,10 @@ describe('structured mailbox pointer delivery', () => { expect(send.mock.calls[1]![0].body.from).toMatchObject({ orchestration: { messages: [{ messageId: 'm2' }] } }) - expect(markAsDelivered.mock.calls).toEqual([[['m1']], [['m2']]]) + expect(markAsDelivered.mock.calls).toEqual([ + [['m1'], 'run:run_1'], + [['m2'], 'run:run_1'] + ]) delivery.onJournalActivity('session-1') delivery.deliverForHandle('run:run_1') await flush() @@ -323,7 +326,7 @@ describe('structured mailbox pointer delivery', () => { delivery.deliverForHandle('dispatch:d1') await flush() expect(send).toHaveBeenCalledTimes(1) - expect(markAsDelivered).toHaveBeenCalledWith(['m1']) + expect(markAsDelivered).toHaveBeenCalledWith(['m1'], 'dispatch:d1') }) it('does not re-nudge a mailbox still holding its own unacked batch', async () => { @@ -478,7 +481,7 @@ describe('structured mailbox pointer delivery', () => { delivery.onJournalActivity('session-1') await flush() expect(send).toHaveBeenCalledTimes(1) - expect(markAsDelivered).toHaveBeenCalledWith(['m1']) + expect(markAsDelivered).toHaveBeenCalledWith(['m1'], 'dispatch:d1') expect(stored.has('dispatch:d1')).toBe(false) }) diff --git a/src/main/runtime/orchestration/structured-mailbox-pointer-delivery.ts b/src/main/runtime/orchestration/structured-mailbox-pointer-delivery.ts index 8d206dca015..6927554081a 100644 --- a/src/main/runtime/orchestration/structured-mailbox-pointer-delivery.ts +++ b/src/main/runtime/orchestration/structured-mailbox-pointer-delivery.ts @@ -332,7 +332,7 @@ export class OrchestrationStructuredMailboxPointerDelivery< }) if (operation.kind === 'stamp') { // A send this lane gave up waiting on ran after all. - db.markAsDelivered(staged) + db.markAsDelivered(staged, mailboxHandle) db.deleteStructuredPointerOperation(mailboxHandle) this.sentOperationIds.delete(mailboxHandle) return true @@ -362,7 +362,7 @@ export class OrchestrationStructuredMailboxPointerDelivery< this.retain(mailboxHandle, sessionId, retainReasonForDispatch(outcome.state), reservedTypes) return false } - db.markAsDelivered(staged) + db.markAsDelivered(staged, mailboxHandle) db.deleteStructuredPointerOperation(mailboxHandle) this.sentOperationIds.delete(mailboxHandle) return true diff --git a/src/main/runtime/rpc/methods/orchestration/federation/federation.ts b/src/main/runtime/rpc/methods/orchestration/federation/federation.ts index 542ba45e958..459ba693574 100644 --- a/src/main/runtime/rpc/methods/orchestration/federation/federation.ts +++ b/src/main/runtime/rpc/methods/orchestration/federation/federation.ts @@ -211,6 +211,7 @@ export const ORCHESTRATION_FEDERATION_ATTACH_METHODS = [ // Why the worker host's own setting: enforcement runs here, with this // host's code, against this host's cap. canDispatchSubWorkers: (params.depth ?? 1) < runtime.getNestedWorkerMaxDepth(), + subWorkersInOwnRun: true, cliCommand: runtime.getTerminalOrchestrationCliCommand(terminalHandle) }) } diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/ask-methods.ts b/src/main/runtime/rpc/methods/orchestration/messaging/ask-methods.ts index 96919bae3c1..bb6ba8b1099 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/ask-methods.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/ask-methods.ts @@ -2,6 +2,7 @@ import { defineMethod } from '../../../core' import { OrchestrationError } from '../../../../orchestration/orchestration-error' import { clampOrchestrationAskTimeoutMs } from '../../../../../../shared/orchestration-ask-timeout' import { isGroupAddress } from '../../../../orchestration/groups' +import { dispatchReportMailbox } from '../../../../orchestration/dispatch-report-mailbox' import { AskParams } from '../schemas' import { rejectFederatedExplicitTarget } from '../routing' import { askRemoteRunHome } from './ask-remote' @@ -109,9 +110,11 @@ export const ORCHESTRATION_ASK_METHODS = [ `Dispatch ${activeDispatch.id} belongs to Run ${run.id}, not ${params.run}.` ) } + const reportTo = dispatchReportMailbox(db, activeDispatch) if ( params.to && params.to !== `run:${run.id}` && + params.to !== reportTo && resolveOrchestrationParty(params.to, db).address !== mailboxAddressOf(runCoordinatorKey(run)) ) { @@ -120,15 +123,18 @@ export const ORCHESTRATION_ASK_METHODS = [ `ask from Dispatch ${activeDispatch.id} must target its owning Run ${run.id}.` ) } + // An explicit Run or coordinator target still reaches the Run; otherwise the asker's creator. + const to = params.to && params.to !== reportTo ? `run:${run.id}` : reportTo const created = db.createQuestion({ runId: run.id, dispatchId: activeDispatch.id, askerHandle: from, question: params.question as string, - options + options, + to }) question = created.question - runtime.notifyMessageArrived(`run:${run.id}`, created.message.type) + runtime.notifyMessageArrived(to, created.message.type) } const questionId = question.message_id diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/check.test.ts b/src/main/runtime/rpc/methods/orchestration/messaging/check.test.ts index affdddd0ac4..223f6ea7d80 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/check.test.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/check.test.ts @@ -709,7 +709,7 @@ describe('orchestration RPC methods', () => { setup() const msg = db.insertMessage({ from: 'a', to: 'b', subject: 'hi' }) // Why: simulate push-on-idle stamping delivered_at without the runtime loop. - db.markAsDelivered([msg.id]) + db.markAsDelivered([msg.id], 'b') const result = (await call('orchestration.check', { terminal: 'b', diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/helper-question-answerer.ts b/src/main/runtime/rpc/methods/orchestration/messaging/helper-question-answerer.ts new file mode 100644 index 00000000000..76ec26e82d0 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration/messaging/helper-question-answerer.ts @@ -0,0 +1,38 @@ +import type { OrchestrationCompatibilityEvidence } from '../../../../../../shared/orchestration-compatibility-evidence' +import type { OrchestrationSessionCaller } from '../../../../orchestration/orchestration-caller-identity' +import type { MessageRow } from '../../../../orchestration/types' +import type { OrcaRuntimeService } from '../../../../orca-runtime' +import { assertCallerHandleMatchesEvidence, orchestrationCallerIdentity } from '../runs/run-scope' + +/** The caller's Dispatch when a helper asked this question of it; the Run answers otherwise. */ +export function helperQuestionAnswerer( + runtime: OrcaRuntimeService, + args: { + question: MessageRow + from: string | undefined + session: OrchestrationSessionCaller | undefined + evidence: OrchestrationCompatibilityEvidence | undefined + } +): { dispatchId: string; consumerGeneration: number } | undefined { + const askedOf = args.question.to_handle.startsWith('dispatch:') + ? args.question.to_handle.slice('dispatch:'.length) + : undefined + if (!askedOf || (!args.from && !args.session)) { + return undefined + } + const handle = args.from ?? '' + if (args.from) { + assertCallerHandleMatchesEvidence(runtime, args.from, args.evidence) + } + const caller = orchestrationCallerIdentity(runtime, { + handle, + session: args.session, + paneKey: args.session ? undefined : runtime.getTerminalPaneKey(handle) + }) + const own = runtime + .getOrchestrationDb() + .getActiveDispatchForIdentity(caller.address, caller.paneKey ?? undefined) + return own?.id === askedOf + ? { dispatchId: own.id, consumerGeneration: own.consumer_generation } + : undefined +} diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/message-methods.ts b/src/main/runtime/rpc/methods/orchestration/messaging/message-methods.ts index f8dd5776fa6..bedc1778272 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/message-methods.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/message-methods.ts @@ -13,6 +13,7 @@ import { exposeMessage } from './mailbox-message-receipt' import { resolveOrchestrationParty } from '../../../../orchestration/orchestration-party' import { recordReceiptBeforeNudge, replayMutationNudge } from './mutation-replay-nudge' import { resolveReplyRecipient } from './recipient-routing' +import { helperQuestionAnswerer } from './helper-question-answerer' import { ReplyParams, InboxParams, @@ -71,6 +72,28 @@ export const ORCHESTRATION_MESSAGE_METHODS = [ } const question = db.getQuestion(params.id) + const creatorAnswerer = + question && (params.run === undefined || params.run === question.run_id) + ? helperQuestionAnswerer(runtime, { + question: original, + from: params.from, + session: orchestrationCaller, + evidence: orchestrationCompatibilityEvidence + }) + : undefined + if (question && creatorAnswerer) { + const answered = db.answerQuestion({ + messageId: question.message_id, + runId: question.run_id, + answeredBy: creatorAnswerer, + body: params.body + }) + return recordReceiptBeforeNudge( + recordMutationReceipt, + { ...answered, message: exposeMessage(answered.message) }, + () => runtime.notifyMessageArrived(`dispatch:${question.dispatch_id}`, 'status') + ) + } if (question) { const run = resolveRunScope(runtime, { runId: params.run ?? question.run_id, diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/send-methods.ts b/src/main/runtime/rpc/methods/orchestration/messaging/send-methods.ts index d10ad61ae4d..c9a6ba396e6 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/send-methods.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/send-methods.ts @@ -1,6 +1,10 @@ import { defineMethod } from '../../../core' import { OrchestrationError } from '../../../../orchestration/orchestration-error' import { isGroupAddress } from '../../../../orchestration/groups' +import { + dispatchReportMailbox, + isDispatchReportType +} from '../../../../orchestration/dispatch-report-mailbox' import { orchestrationSkillRecoveryData } from '../../../../../../shared/orchestration-rpc-contract' import { SendParams, @@ -21,6 +25,7 @@ import { stripMutationReplayNudge } from '../../../orchestration-mutation-receipt' import { replayMutationNudge } from './mutation-replay-nudge' +import { handBackReplayTargets } from '../../../../orchestration/db/messages/helper-mailbox-hand-back' import { sendRemoteMessage } from './send-remote' import { mayNameSession } from './session-recipient' import { sendPointToPointMessage } from './send-point-to-point' @@ -62,6 +67,12 @@ export const ORCHESTRATION_SEND_METHODS = [ : undefined) if (replayNudge) { replayMutationNudge(runtime, replayNudge) + const settled = + params.type === 'worker_done' ? settledDispatchId(replayedMutationReceipt) : undefined + // Why: derived, since the executor's receipt rewrites drop stored hand-back targets. + for (const target of settled ? handBackReplayTargets(db, settled) : []) { + runtime.notifyMessageArrived(target.to, target.type) + } return stripMutationReplayNudge(replayedMutationReceipt) } const from = params.from ?? 'unknown' @@ -134,7 +145,11 @@ export const ORCHESTRATION_SEND_METHODS = [ (!to || ((params.type === 'worker_done' || params.type === 'heartbeat') && routing.dispatchId)) ) { - to = `run:${routing.run.id}` + const reporting = + routing.dispatchId && isDispatchReportType(params.type) + ? db.getDispatchContextById(routing.dispatchId) + : undefined + to = reporting ? dispatchReportMailbox(db, reporting) : `run:${routing.run.id}` } if (!to) { throw new OrchestrationError( @@ -247,3 +262,16 @@ export const ORCHESTRATION_SEND_METHODS = [ } }) ] + +function settledDispatchId(receipt: unknown): string | undefined { + if (!receipt || typeof receipt !== 'object' || !('lifecycle' in receipt)) { + return undefined + } + const { lifecycle } = receipt + return lifecycle && + typeof lifecycle === 'object' && + 'dispatchId' in lifecycle && + typeof lifecycle.dispatchId === 'string' + ? lifecycle.dispatchId + : undefined +} diff --git a/src/main/runtime/rpc/methods/orchestration/worker/local-worker-start.ts b/src/main/runtime/rpc/methods/orchestration/worker/local-worker-start.ts index 1ea7cf7e985..ae874b5f947 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/local-worker-start.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/local-worker-start.ts @@ -4,7 +4,7 @@ import { waitForWorkerAgentReady } from '../../../../launched-agent-composer-rea import type { OrcaRuntimeService } from '../../../../orca-runtime' import { describeTerminalWaitBlockedReason } from '../../../../../../shared/terminal-wait-blocked-reason-legacy-alias' import type { OrchestrationDb } from '../../../../orchestration/db' -import type { RunRow, TaskRow } from '../../../../orchestration/types' +import type { DispatchContextRow, RunRow, TaskRow } from '../../../../orchestration/types' import type { OrchestrationCallerIdentity, OrchestrationSessionCaller @@ -48,6 +48,8 @@ export async function startLocalWorker(args: { orchestrationMutation?: WorkerStartMutation /** Settings-driven; the executing host still gets to refuse below. */ mode: WorkerStartModeReceipt + /** Set when a worker starts its own helper; the helper reports to this Dispatch. */ + helperCreator?: DispatchContextRow }): Promise { const { runtime, db, run, coordinator, callerSession, existingTask, orchestrationMutation } = args // `--terminal` names a party: a chat by its `/clear` root address, as its Dispatch does. @@ -147,11 +149,13 @@ export async function startLocalWorker(args: { const started = db.createStartingWorkerDispatch({ creator: resolveDispatchCreator(runtime, params.from, callerSession), maxDepth: runtime.getNestedWorkerMaxDepth(), + requiredCreatorDispatchId: args.helperCreator?.id, taskId: existingTask?.id, taskSpec: params.spec, taskTitle: params.taskTitle, taskDeps: parseTaskDeps(params.deps), - taskParentId: params.parent, + // A helper's Task sits under its creator's Task, so the Run's Task tree shows who started it. + taskParentId: params.parent ?? args.helperCreator?.task_id, taskRunId: run.id, // A handle-less session creates root Tasks: Task lineage is recorded by terminal only. taskCreatedByTerminalHandle: coordinator?.terminalHandle ?? undefined, diff --git a/src/main/runtime/rpc/methods/orchestration/worker/worker-start-authority.ts b/src/main/runtime/rpc/methods/orchestration/worker/worker-start-authority.ts new file mode 100644 index 00000000000..3e2155dab9c --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration/worker/worker-start-authority.ts @@ -0,0 +1,82 @@ +import type { OrchestrationDb } from '../../../../orchestration/db' +import { OrchestrationError } from '../../../../orchestration/orchestration-error' +import type { + OrchestrationCallerIdentity, + OrchestrationSessionCaller +} from '../../../../orchestration/orchestration-caller-identity' +import type { DispatchContextRow, RunRow, TaskRow } from '../../../../orchestration/types' +import type { OrcaRuntimeService } from '../../../../orca-runtime' +import { resolveDispatchCreator } from '../runs/dispatch-creator' + +/** + * The Run a worker-start lands in, and who may start it. The Run's coordinator starts workers in + * it; a worker on an active Dispatch starts helpers in the Run it works for, as itself, so the + * helpers report to it and its own mailbox never moves. + */ +export function resolveWorkerStartRun( + runtime: OrcaRuntimeService, + args: { + coordinator: OrchestrationCallerIdentity | null + from: string + callerSession: OrchestrationSessionCaller | undefined + runId: string | undefined + } +): { run: RunRow; helperCreator?: DispatchContextRow } { + const db = runtime.getOrchestrationDb() + const coordinated = args.coordinator + ? db.getCurrentRunForCoordinator(args.coordinator) + : undefined + if (coordinated) { + if (args.runId && args.runId !== coordinated.id) { + throw fenced(`This caller coordinates Run ${coordinated.id}, not ${args.runId}.`) + } + return { run: coordinated } + } + // Same resolution that stamps the helper's creator, so authority and reporting always agree. + // It returns only an active, delegated Dispatch; a remote one has no row on this host. + const creatorId = db.resolveCreatorDispatchId( + resolveDispatchCreator(runtime, args.from, args.callerSession) + ) + const creator = creatorId ? db.getDispatchContextById(creatorId) : undefined + const run = creator ? db.getRun(creator.run_id) : undefined + if (!creator || !run || run.legacy === 1) { + throw fenced( + 'worker-start needs the coordinator bound to the Run, or a worker on an active Dispatch in it.' + ) + } + if (args.runId && args.runId !== run.id) { + throw fenced( + `This worker's Dispatch ${creator.id} is in Run ${run.id}; it starts helpers only there.` + ) + } + return { run, helperCreator: creator } +} + +/** + * What a worker may start: a fresh helper in a terminal Orca launches for it, on this host, or a + * retry of its own helper's latest attempt. Never the coordinator's Tasks or another party's pane. + */ +export function assertHelperStartScope( + db: OrchestrationDb, + creator: DispatchContextRow, + args: { task: TaskRow | undefined; params: { on?: string; terminal?: string } } +): void { + if (args.params.on || args.params.terminal) { + // Why --on: remote reports land in the Run home's Run mailbox, so the helper could not report back. + throw new OrchestrationError( + 'invalid_argument', + 'A worker starts helpers in a new agent on this host; omit --on and --terminal. No effects were applied.', + { effectsApplied: false } + ) + } + if (args.task && db.getDispatchContext(args.task.id)?.creator_dispatch_id !== creator.id) { + throw new OrchestrationError( + 'task_not_found', + `Task ${args.task.id} was not last run by a helper of Dispatch ${creator.id}; start new helper work with --spec.` + ) + } +} + +function fenced(message: string): OrchestrationError { + return new OrchestrationError('consumer_fenced', message) +} diff --git a/src/main/runtime/rpc/methods/orchestration/worker/worker-topology.ts b/src/main/runtime/rpc/methods/orchestration/worker/worker-topology.ts index a11eb4b4c00..9fe094038f1 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/worker-topology.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/worker-topology.ts @@ -4,6 +4,7 @@ import type { TuiAgent } from '../../../../../../shared/tui-agent' import type { OrcaRuntimeService } from '../../../../orca-runtime' import type { OrchestrationDb } from '../../../../orchestration/db' import { createStructuredWorkerSession } from '../../orchestration-structured-worker-session' +import { dispatchReportMailbox } from '../../../../orchestration/dispatch-report-mailbox' export type WorkerEffect = { kind: 'worktree' | 'terminal' | 'setup' | 'dispatch_input' @@ -175,10 +176,11 @@ export function monitorWorkerSetup(args: { if (!evidence.changed) { return } + const dispatch = args.db.getDispatchContextById(args.dispatchId) const message = args.db.insertMessage({ runId: args.runId, from: `dispatch:${args.dispatchId}`, - to: `run:${args.runId}`, + to: dispatch ? dispatchReportMailbox(args.db, dispatch) : `run:${args.runId}`, subject: `Setup ${setupState} for worker ${args.dispatchId}`, type: 'status', priority: setupState === 'failed' ? 'high' : 'normal', diff --git a/src/main/runtime/rpc/methods/orchestration/worker/workers.ts b/src/main/runtime/rpc/methods/orchestration/worker/workers.ts index 64d2b14e1bd..bfc26f43ef3 100644 --- a/src/main/runtime/rpc/methods/orchestration/worker/workers.ts +++ b/src/main/runtime/rpc/methods/orchestration/worker/workers.ts @@ -7,6 +7,7 @@ import { readWorkerStartModeSettings } from '../../orchestration-worker-start-mode' import { resolveOrchestrationCaller } from '../runs/run-scope' +import { assertHelperStartScope, resolveWorkerStartRun } from './worker-start-authority' import { WorkerStartParams } from '../../../../../../shared/rpc-contract/orchestration-worker-start-params' import { isWorkerStartTimeoutWithinTimerLimit, @@ -38,13 +39,12 @@ export const ORCHESTRATION_WORKER_START_METHODS = [ callerEvidence: orchestrationCompatibilityEvidence, callerSession: orchestrationCaller }) - const run = coordinator ? db.getCurrentRunForCoordinator(coordinator) : undefined - if (!run || (params.run && params.run !== run.id)) { - throw new OrchestrationError( - 'consumer_fenced', - 'worker-start requires the coordinator terminal currently bound to the Task Run.' - ) - } + const { run, helperCreator } = resolveWorkerStartRun(runtime, { + coordinator, + from: params.from, + callerSession: orchestrationCaller, + runId: params.run + }) const existingTask = params.task ? db.getTask(params.task) : undefined if (params.task && (!existingTask || existingTask.run_id !== run.id)) { throw new OrchestrationError( @@ -52,6 +52,9 @@ export const ORCHESTRATION_WORKER_START_METHODS = [ `Task ${params.task} was not found in Run ${run.id}.` ) } + if (helperCreator) { + assertHelperStartScope(db, helperCreator, { task: existingTask, params }) + } await assertWorkerStartTaskSpecWithinPromptBudget(params.spec ?? existingTask!.spec) const mode = decideWorkerStartMode({ params, @@ -88,7 +91,8 @@ export const ORCHESTRATION_WORKER_START_METHODS = [ callerSession: orchestrationCaller, existingTask, orchestrationMutation, - mode + mode, + helperCreator }) } }) diff --git a/src/main/runtime/structured-chat-worker-helpers.test.ts b/src/main/runtime/structured-chat-worker-helpers.test.ts new file mode 100644 index 00000000000..b801c7e7af9 --- /dev/null +++ b/src/main/runtime/structured-chat-worker-helpers.test.ts @@ -0,0 +1,331 @@ +import './rpc/unused-default-rpc-methods.test-fixture' +// A dispatched chat starting helper workers on the Run it works for, on the coordinator-mail rig: +// the helpers report to it, its own mailbox never moves, and whatever it never handled goes back +// to the Run's coordinator once it settles. + +import { describe, expect, it, vi } from 'vitest' +import { formatOrcaSessionAddress } from '../../shared/orca-session-address' +import { testOrcaSessionId } from '../../shared/orca-session-address-test-fixture' +import { idOf, resultOf } from './rpc/orchestration-session-caller-test-fixture' +import type { DispatchCreator } from './orchestration/db/dispatch-depth' +import { + COORDINATOR, + PEER_CHAT, + WAIT, + WORKER_2_PANE, + WORKER_PANE, + call, + coordinatorRunAndTask, + db, + dispatcher, + openChat, + request, + restartRuntime, + runtime, + turnText +} from './structured-chat-coordinator-mail-rig.test-fixture' + +const WORKER = formatOrcaSessionAddress(testOrcaSessionId(PEER_CHAT)) +const HELPER = 'term_worker' +/** A helper's own helper, two levels below the Run's coordinator. */ +const LEAF = 'term_worker_2' + +async function refusal(method: string, params: Record, sessionId: string) { + const response = await dispatcher.dispatch(request(method, params, { sessionId })) + if (response.ok) { + throw new Error(`expected a refusal, got ${JSON.stringify(response)}`) + } + return response.error +} + +/** The helper row worker-start reserves once the worker's authority resolved, in a live terminal. */ +function reserveHelper( + creatorDispatchId: string, + creator: DispatchCreator = { kind: 'session', orcaSessionId: testOrcaSessionId(PEER_CHAT) }, + handle = HELPER, + paneKey = WORKER_PANE +): { dispatchId: string; taskId: string } { + const started = db.createStartingWorkerDispatch({ + creator, + maxDepth: runtime.getNestedWorkerMaxDepth(), + requiredCreatorDispatchId: creatorDispatchId, + taskSpec: 'review the parser', + taskRunId: db.getDispatchContextById(creatorDispatchId)!.run_id, + taskParentId: db.getDispatchContextById(creatorDispatchId)!.task_id, + startOptions: {} + }) + db.prepareStartingWorkerAuthority({ + dispatchId: started.dispatch.id, + handle, + paneKey, + processIncarnation: `runtime_test:${handle}:1`, + worktreeId: 'repo::worker', + effects: [], + setupState: 'not_applicable' + }) + db.markWorkerDispatchReady(started.dispatch.id) + return { dispatchId: started.dispatch.id, taskId: started.task.id } +} + +function helperSend(type: string, subject: string, helper: { dispatchId: string; taskId: string }) { + return call('orchestration.send', { + from: HELPER, + type, + subject, + payload: JSON.stringify({ ...helper, outcome: 'succeeded' }) + }) +} + +function inboxOf(inbox: Record) { + const messages = Array.isArray(inbox.messages) + ? inbox.messages.map((message: Record) => ({ + id: String(message.id), + subject: String(message.subject), + type: String(message.type) + })) + : [] + return { deliveryId: inbox.deliveryId, messages } +} + +/** A chat checks by its session; a terminal helper by its handle. */ +async function check(caller: string, ack?: unknown) { + const byTerminal = caller.startsWith('term_') + return inboxOf( + await call( + 'orchestration.check', + { ...(byTerminal ? { terminal: caller } : {}), ...(ack ? { ack } : {}) }, + byTerminal ? {} : { sessionId: caller } + ) + ) +} + +async function subjects(caller: string, ack?: unknown): Promise { + return (await check(caller, ack)).messages.map((message) => message.subject) +} + +describe('a worker starting helpers on the Run it works for', () => { + it('gets their reports and questions in its own check, and hands the unhandled rest to the Run when it settles', async () => { + await openChat(COORDINATOR) + await openChat(PEER_CHAT) + const { runId, taskId } = await coordinatorRunAndTask() + const parent = idOf( + ( + await call( + 'orchestration.dispatch', + { task: taskId, to: WORKER }, + { sessionId: COORDINATOR } + ) + ).dispatch + ) + vi.spyOn(runtime, 'getNestedWorkerMaxDepth').mockReturnValue(2) + + // The coordinator seat no longer gates it: the worker reaches the helper-scope rules instead. + expect( + await refusal( + 'orchestration.workerStart', + { run: runId, spec: 'review', terminal: HELPER }, + PEER_CHAT + ) + ).toMatchObject({ code: 'invalid_argument', message: expect.stringContaining('--terminal') }) + + const helper = reserveHelper(parent) + expect(db.getDispatchContextById(helper.dispatchId)).toMatchObject({ + run_id: runId, + creator_dispatch_id: parent, + depth: 2 + }) + + await helperSend('escalation', 'Blocked on fixtures', helper) + const asked = call('orchestration.ask', { from: HELPER, question: 'Which parser?' }) + await vi.waitFor( + () => + expect(db.db.prepare('SELECT COUNT(*) AS n FROM question_threads').get()).toEqual({ n: 1 }), + WAIT + ) + const inbox = await check(PEER_CHAT) + const messages = inbox.messages + expect(messages.map((message) => message.subject)).toEqual(['Blocked on fixtures', 'Question']) + await call( + 'orchestration.reply', + { id: messages.find((message) => message.type === 'question')!.id, body: 'the TOML one' }, + { sessionId: PEER_CHAT } + ) + expect(await asked).toMatchObject({ answer: 'the TOML one' }) + + expect(await subjects(COORDINATOR)).toEqual([]) + expect(await call('orchestration.runCurrent', {}, { sessionId: PEER_CHAT })).toEqual({ + run: null + }) + + // The worker's last check acknowledges what it read; what arrives after goes to the Run. + expect(await subjects(PEER_CHAT, inbox.deliveryId)).toEqual([]) + await helperSend('escalation', 'Blocked again', helper) + await call( + 'orchestration.send', + { + type: 'worker_done', + subject: 'Parent done', + payload: JSON.stringify({ taskId, dispatchId: parent, outcome: 'succeeded' }) + }, + { sessionId: PEER_CHAT } + ) + await helperSend('worker_done', 'Helper done', helper) + expect(await subjects(COORDINATOR)).toEqual(['Blocked again', 'Parent done', 'Helper done']) + }) +}) + +/** The Run's coordinator, a dispatched chat, its terminal helper, and that helper's own helper. */ +async function threeLevels() { + const chat = await openChat(COORDINATOR) + await openChat(PEER_CHAT) + const { runId, taskId } = await coordinatorRunAndTask() + const parent = idOf( + (await call('orchestration.dispatch', { task: taskId, to: WORKER }, { sessionId: COORDINATOR })) + .dispatch + ) + vi.spyOn(runtime, 'getNestedWorkerMaxDepth').mockReturnValue(3) + const middle = reserveHelper(parent) + const leaf = reserveHelper( + middle.dispatchId, + { kind: 'terminal', handle: HELPER, paneKey: WORKER_PANE }, + LEAF, + WORKER_2_PANE + ) + return { chat, runId, middle, leaf } +} + +/** The coordinator waiting on questions only, as its loop does between steps. */ +async function parkRunQuestionCheck(runId: string) { + expect(await subjects(COORDINATOR)).toEqual([]) + const parked = call( + 'orchestration.check', + { wait: true, types: 'question', timeoutMs: 10_000 }, + { sessionId: COORDINATOR } + ) + await vi.waitFor( + () => expect(runtime.messageWaitersByHandle.has(`run:${runId}`)).toBe(true), + WAIT + ) + return { parked } +} + +async function answerFromRun(parked: Promise>, expected: string[]) { + const woke = inboxOf(await parked) + expect(woke.messages.map((message) => message.subject)).toEqual(expected) + const question = woke.messages.find((message) => message.type === 'question')! + await call( + 'orchestration.reply', + { id: question.id, body: 'the TOML one' }, + { sessionId: COORDINATOR } + ) + return woke +} + +describe('a helper whose creator settles between it and the Run', () => { + it('wakes the Run for its question, which the Run answers, and reports arrive once', async () => { + const { runId, middle } = await threeLevels() + await call('orchestration.send', { from: LEAF, type: 'status', subject: 'Read status' }) + const asked = call('orchestration.ask', { from: LEAF, question: 'Which parser?' }) + await vi.waitFor( + () => expect(db.getAllMessages(`dispatch:${middle.dispatchId}`)).toHaveLength(2), + WAIT + ) + // The middle worker reads both, but never answers. + const read = await check(HELPER) + expect(read.messages.map((message) => message.subject)).toEqual(['Read status', 'Question']) + expect(await subjects(HELPER, read.deliveryId)).toEqual([]) + await call('orchestration.send', { from: LEAF, type: 'status', subject: 'Unread status' }) + const { parked } = await parkRunQuestionCheck(runId) + + await helperSend('worker_done', 'Middle done', middle) + const woke = await answerFromRun(parked, ['Question', 'Unread status']) + expect(await asked).toMatchObject({ answer: 'the TOML one' }) + expect(await subjects(COORDINATOR, woke.deliveryId)).toEqual([]) + expect(await subjects(PEER_CHAT)).toEqual(['Middle done']) + }) + + it('also hands over a question asked while its creator was being stopped', async () => { + const { runId, middle } = await threeLevels() + const { parked } = await parkRunQuestionCheck(runId) + db.beginWorkerStop(middle.dispatchId, 'test-stop') + const asked = call('orchestration.ask', { from: LEAF, question: 'Which parser?' }) + await vi.waitFor( + () => expect(db.getAllMessages(`dispatch:${middle.dispatchId}`)).toHaveLength(1), + WAIT + ) + + db.settleWorkerStop(middle.dispatchId) + await answerFromRun(parked, ['Question']) + expect(await asked).toMatchObject({ answer: 'the TOML one' }) + }) + + it('re-offers the handed-back question to the Run on every replay of its settling report', async () => { + const { runId, middle, leaf } = await threeLevels() + db.createQuestion({ + runId, + dispatchId: leaf.dispatchId, + askerHandle: LEAF, + question: 'Which parser?', + to: `dispatch:${middle.dispatchId}` + }) + const done = request('orchestration.send', { + from: HELPER, + type: 'worker_done', + subject: 'Middle done', + payload: JSON.stringify({ ...middle, outcome: 'succeeded' }) + }) + resultOf(await dispatcher.dispatch(done)) + await new Promise((resolve) => setTimeout(resolve, 0)) + const countMessages = () => db.db.prepare('SELECT COUNT(*) AS n FROM messages').get() + const before = countMessages() + const notify = vi.spyOn(runtime, 'notifyMessageArrived') + + // The second replay reads the receipt the first replay rewrote. + for (const replays of [1, 2]) { + expect(resultOf(await dispatcher.dispatch(done))).toMatchObject({ + mutation: { replayed: true } + }) + expect( + notify.mock.calls.filter(([to, type]) => to === `run:${runId}` && type === 'question') + ).toHaveLength(replays) + } + expect(countMessages()).toEqual(before) + }) + + it('moves nothing on rollback; a restart offers a committed hand-back nothing woke', async () => { + const { chat, runId, middle, leaf } = await threeLevels() + const mailbox = `dispatch:${middle.dispatchId}` + db.createQuestion({ + runId, + dispatchId: leaf.dispatchId, + askerHandle: LEAF, + question: 'Which parser?', + to: mailbox + }) + await call('orchestration.send', { from: LEAF, type: 'status', subject: 'Unread status' }) + const ids = db.getAllMessages(mailbox).map((message) => message.id) + // The middle worker's pointer for both already landed. + db.markAsDelivered(ids, mailbox) + const notify = vi.spyOn(runtime, 'notifyMessageArrived') + + db.db.exec('BEGIN IMMEDIATE') + db.completeDispatch(middle.dispatchId) + db.db.exec('ROLLBACK') + await new Promise((resolve) => setTimeout(resolve, 0)) + expect(db.getAllMessages(mailbox)).toHaveLength(2) + expect(db.getUndeliveredUnreadMessages(mailbox)).toEqual([]) + expect(notify).not.toHaveBeenCalledWith(`run:${runId}`, expect.anything()) + + // Committed, but the process ends before anything wakes the Run; the old pointer lands late. + db.mailHandBackListener = undefined + db.completeDispatch(middle.dispatchId) + db.markAsDelivered(ids, mailbox) + expect(db.getUndeliveredUnreadMessages(`run:${runId}`).map((message) => message.id)).toEqual( + expect.arrayContaining(ids) + ) + + restartRuntime() + await vi.waitFor(() => expect(chat.turns).toHaveLength(1), WAIT) + expect(turnText(chat.turns[0]!)).toMatch(/orchestration message/) + }) +}) diff --git a/src/main/runtime/terminal/layout/terminal-resize-listeners.ts b/src/main/runtime/terminal/layout/terminal-resize-listeners.ts index ccecfb5f722..a97566f9515 100644 --- a/src/main/runtime/terminal/layout/terminal-resize-listeners.ts +++ b/src/main/runtime/terminal/layout/terminal-resize-listeners.ts @@ -21,6 +21,7 @@ import { classifyWorkerTerminalProcessIncarnation } from '../../orchestration/wo import { parseWorkerTerminalHostScope } from '../../../../shared/worker-terminal-host-scope' import { getRepoIdFromWorktreeId } from '../../../../shared/worktree/id' import { buildOrchestrationTaskDisplayMetadata } from '../../../../shared/orchestration-task-display' +import { dispatchReportMailbox } from '../../orchestration/dispatch-report-mailbox' import { LOCAL_EXECUTION_HOST_ID, toSshExecutionHostId } from '../../../../shared/execution-host' export abstract class OrcaRuntimeWithSubscribeToTerminalResize extends OrcaRuntimeWithApplyMobileDisplayMode { @@ -98,7 +99,7 @@ export abstract class OrcaRuntimeWithSubscribeToTerminalResize extends OrcaRunti const active = this._orchestrationDb.getActiveCoordinatorRun?.() const recipient = owningRun && owningRun.legacy !== 1 - ? { to: `run:${owningRun.id}`, runId: owningRun.id } + ? { to: dispatchReportMailbox(this._orchestrationDb, dispatch), runId: owningRun.id } : active ? { to: active.coordinator_handle, runId: undefined } : null diff --git a/src/main/runtime/workspaces/persisted-surfaces/automation-operations.ts b/src/main/runtime/workspaces/persisted-surfaces/automation-operations.ts index 967e08b2115..c6309666dd4 100644 --- a/src/main/runtime/workspaces/persisted-surfaces/automation-operations.ts +++ b/src/main/runtime/workspaces/persisted-surfaces/automation-operations.ts @@ -16,6 +16,10 @@ import { type AutomationListResult } from '../../../../shared/automation-list-scope' import { OrchestrationDb } from '../../orchestration/db' +import { + committedHandBackTypes, + type MailHandBack +} from '../../orchestration/db/messages/helper-mailbox-hand-back' import { join } from 'node:path' import { existsSync } from 'node:fs' import { getAppEnvironment } from '../../../../shared/app-environment' @@ -150,6 +154,7 @@ export abstract class OrcaRuntimeWithAutomationOperations extends OrcaRuntimeWit getOrchestrationDb(): OrchestrationDb { if (!this._orchestrationDb) { this._orchestrationDb = new OrchestrationDb(this.orchestrationDbPath()) + this.bindMailHandBack(this._orchestrationDb) this.ensureOrchestrationFederationRelay() this.scheduleRestoredMessageRepoints() } @@ -171,10 +176,36 @@ export abstract class OrcaRuntimeWithAutomationOperations extends OrcaRuntimeWit this.orchestrationFederation.resetForDatabaseChange() this.mailPointerRepointScheduler.clear() this._orchestrationDb = db + this.bindMailHandBack(db) this.ensureOrchestrationFederationRelay() this.scheduleRestoredMessageRepoints() } + private bindMailHandBack(db: OrchestrationDb): void { + db.mailHandBackListener = (handBack) => this.offerHandedBackMail(db, handBack) + } + + /** Wakes the Run for helper mail its creator's settlement moved there, once that committed. */ + private offerHandedBackMail(db: OrchestrationDb, handBack: MailHandBack): void { + // Why: a replaced database's consumers are not this runtime's to wake. + if (this._orchestrationDb !== db) { + return + } + try { + if (db.db.isTransaction) { + this.mailPointerRepointScheduler.schedule(handBack.mailbox) + return + } + // Why: re-reading the moved rows means a rolled-back settlement wakes no one. + for (const type of committedHandBackTypes(db, handBack)) { + this.notifyMessageArrived(handBack.mailbox, type) + } + } catch (error) { + console.warn('[orchestration] failed to offer handed-back helper mail', error) + this.mailPointerRepointScheduler.schedule(handBack.mailbox) + } + } + protected async flushWorkspaceSessionOrThrowAsync(): Promise { const store = this.store if (store?.flushPendingOrThrowAsync) {