From 8d70972c52d3ce16068a2e456ef05492c787f586 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 27 Aug 2026 13:10:57 -0700 Subject: [PATCH] feat(orchestration): add durable run capacity targets --- config/reliability-gates.jsonc | 107 ++++- skill-guides/orchestration.md | 24 ++ src/cli/bundled-skill-guides.ts | 2 +- src/cli/handler-group-manifest.ts | 4 + .../orchestration-capacity-cli.test.ts | 69 ++++ .../handlers/orchestration-worker-cli.test.ts | 55 ++- src/cli/handlers/orchestration.ts | 2 + .../orchestration/capacity-handlers.ts | 100 +++++ .../orchestration/worker-launch-handler.ts | 19 +- src/cli/specs/orchestration-worker-specs.ts | 6 +- src/cli/specs/orchestration.ts | 28 ++ ...hestration-capacity-cli-subprocess.test.ts | 196 +++++++++ .../db/attach-orchestration-db-methods.ts | 2 + .../orchestration/db/contract-constants.ts | 4 +- .../dispatch-context-store.ts | 1 + .../orchestration/db/dispatch-row-writer.ts | 48 ++- .../db/orchestration-db-methods.ts | 2 + .../db/run-capacity/run-capacity-store.ts | 223 ++++++++++ .../db/schema/create-core-tables-sql.ts | 1 + .../db/schema/create-graph-tables-sql.ts | 1 + .../orchestration/db/schema/migrate-v31.ts | 21 + .../orchestration/db/schema/migrate.ts | 2 + .../orchestration/db/tasks/task-store.ts | 6 +- .../worker-dispatch/worker-dispatch-start.ts | 2 + .../orchestration-schema-version-skew.ts | 9 +- .../run-capacity-migration.test.ts | 89 ++++ .../orchestration/run-capacity.test.ts | 391 ++++++++++++++++++ src/main/runtime/orchestration/types.ts | 2 + .../methods/orchestration-capacity.test.ts | 94 +++++ .../rpc/methods/orchestration-capacity.ts | 81 ++++ .../orchestration-federated-worker-start.ts | 4 +- .../orchestration-worker-start-schema.ts | 1 + .../rpc/methods/orchestration-workers.ts | 6 +- src/main/runtime/rpc/methods/orchestration.ts | 2 + src/shared/orchestration-rpc-contract.test.ts | 2 + src/shared/orchestration-rpc-contract.ts | 2 + src/shared/protocol-version.ts | 3 + 37 files changed, 1581 insertions(+), 30 deletions(-) create mode 100644 src/cli/handlers/orchestration-capacity-cli.test.ts create mode 100644 src/cli/handlers/orchestration/capacity-handlers.ts create mode 100644 src/main/runtime/orchestration-capacity-cli-subprocess.test.ts create mode 100644 src/main/runtime/orchestration/db/run-capacity/run-capacity-store.ts create mode 100644 src/main/runtime/orchestration/db/schema/migrate-v31.ts create mode 100644 src/main/runtime/orchestration/run-capacity-migration.test.ts create mode 100644 src/main/runtime/orchestration/run-capacity.test.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-capacity.test.ts create mode 100644 src/main/runtime/rpc/methods/orchestration-capacity.ts diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index ac37e3760b9..4327ce54008 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "updatedAt": "2026-08-23", + "updatedAt": "2026-08-27", "policy": { "maturityLevels": ["experimental", "soak", "blocking", "accepted-gap", "deprecated"], "blockingPromotion": { @@ -2148,9 +2148,7 @@ }, { "file": "src/cli/runtime/serve-signal-exit-diagnostic.test.ts", - "assertions": [ - "late update handoff failure cannot rearm termination after child exit" - ] + "assertions": ["late update handoff failure cannot rearm termination after child exit"] }, { "file": "src/main/serve-update-handoff.test.ts", @@ -16425,9 +16423,7 @@ }, { "file": "src/main/browser/agent-browser-bridge-tab-routing.test.ts", - "assertions": [ - "closing a tab retires the exact named agent-browser session" - ] + "assertions": ["closing a tab retires the exact named agent-browser session"] }, { "file": "src/main/startup/serve-signal-handlers.test.ts", @@ -16570,6 +16566,103 @@ "The sentinel changes a pane title within an existing layout; concurrent split and close conflicts remain separate coverage." ], "demotionRule": "Demote if a failed push suppresses an identical retry, a successful equal write resumes redundant churn, or the routed observer journey flakes without a diagnosed cause." + }, + { + "id": "orchestration.run-capacity-target", + "title": "Run capacity targets expose and enforce safe backfill slots", + "maturity": "experimental", + "protection": "partial", + "owner": "orchestration-runtime", + "layer": "run-home-database-and-cli", + "surfaces": [ + "Run target concurrency", + "explicit Task capacity enrollment", + "local and federated worker start", + "settled worker terminal accounting" + ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "ssh", "wsl", "remote-runtime"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["local"], + "coverageNotes": "Deterministic Run-home database and RPC contracts cover exact atomic admission, simultaneous settlement, dependency and decision-gate promotion, failed and blocked Tasks, empty eligible queues, retry circuit breaking, restart persistence, federated home admission, idempotent mutation replay, and retained, user-owned, reclaimable, and release-unknown terminal resources. Remote provider parity is architectural because all starts claim the capacity slot in the authoritative Run-home transaction before placement; live remote host journeys remain a gap.", + "motivatingLinks": [ + "https://linear.app/stably/issue/STA-3664", + "https://linear.app/stably/issue/STA-4693" + ], + "invariant": "A Run with target concurrency N never admits more than N pending or dispatched attempts; only ready Tasks explicitly enrolled in that Run's capacity pool are exposed for backfill, and settlement reopens the exact slot without treating retained, user-owned, or unverifiable terminals as active lanes or automatically closing them.", + "oracle": "Configure a target, enroll ready and dependency-blocked Tasks, and start concurrent local and federated attempts through both composed worker-start and low-level dispatch. Require the Run-home transaction to reject attempt N+1, require simultaneous worker_done settlement and dependency promotion to expose the exact reopened slots, require restart and replay to preserve target/enrollment without duplicate Dispatches, and require settled terminal cleanup debt to report reclaimable, retained/user-takeover, and release-unknown resources without closing them.", + "commands": [ + "pnpm test src/main/runtime/orchestration/run-capacity.test.ts src/main/runtime/rpc/methods/orchestration-capacity.test.ts src/cli/handlers/orchestration-capacity-cli.test.ts", + "pnpm tc:node && pnpm tc:cli" + ], + "testFiles": [ + "src/main/runtime/orchestration/run-capacity.test.ts", + "src/main/runtime/rpc/methods/orchestration-capacity.test.ts", + "src/cli/handlers/orchestration-capacity-cli.test.ts" + ], + "assertionRefs": [ + { + "file": "src/main/runtime/orchestration/run-capacity.test.ts", + "assertions": [ + "capacity admission is atomic and exact across composed and low-level dispatch paths", + "federated starts claim the exact slot at the authoritative Run home", + "simultaneous settlement and dependency promotion reopen the exact launchable slots", + "pending decision gates suppress launchability until resolution", + "launchable observation is index-backed and bounded by the available slots", + "ready Tasks with a pre-fix active split are never offered twice", + "restart persistence retains the target and explicit Task enrollment", + "settled terminal debt distinguishes reclaimable, user takeover, and release-unknown resources without closing them" + ] + }, + { + "file": "src/main/runtime/rpc/methods/orchestration-capacity.test.ts", + "assertions": ["capacity mutations stay scoped to the coordinator's authoritative Run"] + }, + { + "file": "src/cli/handlers/orchestration-capacity-cli.test.ts", + "assertions": [ + "capacity mutations carry replay identity and worker-start claims a slot only when explicitly requested" + ] + } + ], + "evidenceRuns": [ + { + "date": "2026-08-27", + "runner": "local", + "platform": "macos", + "result": "passed", + "durationSeconds": 19.45, + "command": "pnpm test src/main/runtime/orchestration/run-capacity.test.ts src/main/runtime/rpc/methods/orchestration-capacity.test.ts src/cli/handlers/orchestration-capacity-cli.test.ts", + "summary": "Focused database, RPC, and CLI contracts passed after the original four-test oracle failed because no Run capacity contract existed." + } + ], + "runtimeBudget": { + "p95Seconds": 25, + "scope": "focused in-memory database, RPC, and CLI contracts" + }, + "flakeHistory": { + "status": "not-started", + "evidence": "Fresh deterministic coverage has local candidate evidence only." + }, + "redGreenEvidence": { + "status": "complete", + "evidence": "The initial focused oracle failed because configureRunCapacity and the capacity CLI contract did not exist; the candidate passes the same behavior-level contracts." + }, + "performanceBudget": { + "required": true, + "evidence": "Capacity observation and admission are caller-driven and event-aligned. They add no timer, polling, subprocess, provider fanout, or wire opcode; admission uses one indexed Run-home count inside the existing BEGIN IMMEDIATE transaction, the launchable query is capped at the 64-slot maximum, and pending/debt queries use indexed filters instead of full historical inventories." + }, + "promotionCriteria": [ + "Collect 100 consecutive routed CI passes or 14 days without an unexplained flake.", + "Add live compiled-CLI Run-home and federated-server journeys on Windows, WSL, Linux, and physical SSH.", + "Keep exact capacity admission, dependency promotion, replay, restart, and terminal-debt assertions green." + ], + "knownGaps": [ + "Orca does not infer Task placement, worktree conflicts, agent choice, or setup policy; a coordinator loop must select those explicitly for each reported eligible Task.", + "Live federated Windows, WSL, Linux, and physical SSH journeys are not recorded; provider parity is currently proven at the shared Run-home admission boundary.", + "Settled retained and release-unknown terminals are reported as cleanup debt and intentionally remain open until their ownership is proved." + ], + "demotionRule": "Demote if any start path exceeds the target, an unenrolled or dependency-blocked Task appears launchable, restart or replay duplicates admission, or capacity accounting closes or hides an unproven terminal resource." } ] } diff --git a/skill-guides/orchestration.md b/skill-guides/orchestration.md index 5236a7f47fc..b0b262acd27 100644 --- a/skill-guides/orchestration.md +++ b/skill-guides/orchestration.md @@ -219,6 +219,30 @@ orca orchestration worker-start --task --worktree current --agent claud `current` and exact existing worktrees create a fresh agent terminal and do not rerun setup. Reuse an existing agent only with `--terminal `. +### Durable target concurrency and safe backfill + +A coordinator can declare a Run-wide execution ceiling and explicitly enroll independent Tasks in its capacity pool: + +```bash +orca orchestration capacity-set --target 5 --json +orca orchestration capacity-enroll --task --json +orca orchestration capacity-show --json +``` + +`capacity-show` reports the target, every active Dispatch in the Run, open slots, enrolled ready Tasks in stable creation order, enrolled pending/blocked Tasks, and settled terminal cleanup debt. Only Tasks explicitly enrolled with `capacity-enroll` appear as launchable; dependency promotion and decision-gate resolution must first make the Task `ready`. Failed, blocked, already dispatched, and circuit-broken work never appears as launchable. + +The runtime deliberately does not invent worker placement. For each Task returned under `launchableTasks`, the coordinator chooses the existing `worker-start` placement and claims one slot atomically: + +```bash +orca orchestration worker-start --task --capacity-slot --worktree current --agent codex --json +``` + +Once a target is nonzero, every new Dispatch in that Run—including low-level `dispatch` and Tasks outside the pool—must fit below it. `--capacity-slot` additionally requires explicit pool enrollment. Concurrent starts serialize at the Run home, so a stale `capacity-show` or idempotent coordinator replay can fail with `capacity_full` or `task_not_startable` but cannot exceed the target or duplicate a Task. A target of `0` disables capacity-slot launches without stopping existing work. + +After each accepted `worker_done`, release or reuse the settled worker as below, then call `capacity-show` again and start up to `launchableCount`. This closes the settled-lane backfill gap without a scheduler: Runs remain authoritative, placement remains explicit, and the same loop works after coordinator or runtime restart because target and enrollment are durable. Retained, user-owned, external, and `release_unknown` terminals are reported as cleanup debt; they are never auto-closed and do not masquerade as live execution lanes. + +Use `capacity-withdraw --task ` to remove a Task from future backfill. Enrollment changes neither Task status nor dependencies and never cancels an active Dispatch. + For a per-invocation Claude, Codex, or Cursor launch, pass an opaque provider model id with `--model`; add `--effort` only when that agent/model supports the level. These options apply only to fresh agent terminals, override general agent default arguments, and are reported under `launch.requested` and `launch.effective` in the receipt: ```bash diff --git a/src/cli/bundled-skill-guides.ts b/src/cli/bundled-skill-guides.ts index 4385841d2f8..14c0380d7f8 100644 --- a/src/cli/bundled-skill-guides.ts +++ b/src/cli/bundled-skill-guides.ts @@ -30,7 +30,7 @@ const ORCA_LINEAR_MARKDOWN = "---\nname: orca-linear\ndescription: >-\n Use Orc const ORCA_PER_WORKSPACE_ENV_MARKDOWN = "---\nname: orca-per-workspace-env\ndescription: >-\n Set up, review, debug, or validate Orca per-workspace environment recipes —\n on-demand, disposable runtimes (cloud sandboxes, VMs, or local) created fresh\n for each workspace. Covers first-time setup (provider prerequisites, the\n reusable base snapshot, the coding-agent auth snapshot, credentials, and\n state), not just the per-workspace lifecycle scripts. Use to stand up\n per-workspace environments, fix an `environmentRecipes` entry in `orca.yaml`, scaffold\n provider lifecycle scripts, or resolve an `orca vm recipe doctor` failure.\n---\n\n# Per-Workspace Environments\n\nHelp a user stand up and maintain a repo-owned per-workspace environment recipe end to end. Each\nworkspace gets its own on-demand, disposable runtime (a cloud sandbox, a VM, or a local one),\ncreated fresh and torn down after.\n\nOrca is a **thin wrapper**: you guide, detect, and scaffold; you never own the user's cloud account,\nbilling, images, or credentials.\n\n- **You DO:** sequence the setup, detect what's detectable (provider CLI present/logged-in? recipe\n present? `doctor` passing?), scaffold provider-templated scripts the user fills in, drive the slow\n snapshot/auth phases with the user, and always show the next action.\n- **You DO NOT:** create accounts, choose plans/regions, invent org/project/scope ids, store or print\n secrets, or run anything that spends money without an explicit user OK.\n\nFirst-time setup has **four phases before the per-workspace recipe runs** — easy to miss, so walk\nthem in order:\n\n1. **Prerequisites** — cloud account, provider CLI, scope/project, plan limits, git token (§2).\n2. **Base snapshot** — reusable image: tools + repo + headless build, snapshotted once (§3).\n3. **Agent-auth snapshot** — boot the base, run interactive device-auth, re-snapshot (§4).\n4. **State** — thread snapshot id / scope / project / port between phases via a state file (§6).\n\nThen the **per-workspace contract** (create/suspend/resume/destroy) runs fast (§8).\n\n**The one branch that shapes everything — connection mode:** **Orca-server** (`create` runs `orca serve`\nin the env and emits a `pairingCode`; §7c/§7f) vs **SSH** (`create` runs no server and emits a\n`connection.type:\"ssh\"` block Orca dials into; §7g/§7h). Settle this first — it changes the `create`\noutput shape and half the templates.\n\nKeep Orca's checkout behavior unchanged by default: omit `checkoutMode`, emit schema version 1, and\nlet Orca create a linked worktree. Only use `checkoutMode: provisioned-root` when the user explicitly\nwants one ephemeral machine to clone the finished workspace itself. This niche mode currently requires\ndirect SSH, an ordinary non-bare/non-sparse primary checkout at `projectRoot`, and schema version 2.\n\n**Quick-start (happy path):** interview the user (connection mode Orca-server vs SSH, provider, agent CLI,\ngit auth — §1.2) + read the provider's CLI docs → scaffold `scripts/orca-vm/` from §7 → run the\nbase-snapshot script, then the auth script (you invoke these by hand; not via `orca.yaml`) → wire\n`environmentRecipes` in `orca.yaml` → `orca vm recipe doctor --json` (free) → then the `--provision`\nself-test loop (§9) until it passes.\n\n---\n\n## 1. Setup workflow\n\nDrive these with the user. **[CHECKPOINT]** steps need explicit confirmation — they spend money, take\na long time, or need the user at the keyboard. Never create an Orca workspace or commit unless asked.\n\n1. **Inspect the repo** for an existing `environmentRecipes` entry, `scripts/orca-vm/`, a state file, or setup\n notes. If a working recipe exists, jump to Doctor (§9) instead of rebuilding.\n2. **Interview the user up front** — gather these choices and confirm them back before scaffolding\n anything. Don't pick for them (§11); don't guess.\n - **Connection mode:** how Orca attaches to the environment — an **Orca server** (the VM runs\n `orca serve` and Orca pairs over its pairing URL; worked example §7f) or **SSH** (Orca connects to\n the host over SSH; §7g). This decides the recipe's connection shape, so settle it first.\n - **Checkout ownership:** do not ask by default. Only when the user requires the environment to\n create the exact final checkout, confirm `provisioned-root` and direct SSH; otherwise omit it.\n - **Provider:** Vercel Sandbox, Fly, Modal, an existing SSH host, … For non-obvious providers, also\n ask scope/project/region and plan limits (§2). Then **read that provider's CLI/SDK docs** (or\n ` --help`) before scaffolding — you need its exact create/exec/snapshot/remove verbs.\n If a provider advertises `ssh`, verify whether it exposes a real dialable SSH target\n (host/port/user/key or proxy command) or only a provider-mediated interactive shell; Orca SSH mode\n needs the former.\n - **Coding-agent CLI + account:** which agent runs in the VM (`codex`, `claude`, …) and that the user\n has an account for it — it gets logged in during the Phase-3 auth snapshot (§4).\n - **Git auth:** the token source for cloning a private repo (`GH_TOKEN`/`GITHUB_TOKEN` or `gh auth\n token`; §5).\n3. **Check prerequisites (§2)** — detect the provider CLI + auth and confirm the items above are in\n place before any paid step.\n4. **Scaffold scripts + state file** from §7 (worked Vercel example: §7f; SSH host: §7g; Docker SSH:\n §7h; Windows: §7i), filling in the provider's real commands. Make them executable.\n5. **[CHECKPOINT] Build the base snapshot (§3)** — paid, slow.\n6. **[CHECKPOINT] Authenticate the agent (§4)** — interactive; the user follows a URL/code. **You cannot\n drive this step** — you run commands non-interactively, so there's no TTY for `docker exec -it` /\n `ssh -t` to prompt against. The **user** runs the Phase-3 login in their own terminal (or via the\n Claude Code harness bang-prefix — `! `, with the required space after `!`); you scaffold and drive\n the non-interactive phases around it. After kicking it off, **ask the user to report back once the login\n finishes** — you can't observe it completing, and you need that confirmation before resuming the\n non-interactive steps (base/auth commit, doctor, provision).\n7. **Wire the recipe** so `orca.yaml` points create/suspend/resume/destroy at the scripts (§8). The\n workspace composer reads `environmentRecipes` from the project's primary checkout of `orca.yaml`, **not** from\n a feature branch or worktree. So a recipe added only on a branch won't appear as a \"Run on\" option\n until that `orca.yaml` change is committed and merged to the project's primary branch. Tell the user\n this up front: `doctor`/`--provision` validate the scripts from the working copy on any branch, but\n creating a workspace from the recipe in the picker needs it on primary.\n8. **Dry-run doctor** — `orca vm recipe doctor --repo-path --json` (free, static; §9).\n Fix every failure before going live.\n9. **[CHECKPOINT] Live self-test** — get the user's OK once, then run\n `orca vm recipe doctor --provision --json` as a loop: it runs create → validates →\n destroys, and on failure returns a full transcript. Read it, fix the scripts, and re-run yourself until\n it passes (§9). Spends cloud money; the one approval covers the loop.\n10. **[CHECKPOINT] Optional workspace test** — only if asked: create a workspace via the picker, then\n verify sleep/wake/delete.\n\n---\n\n## 2. Phase 1 — Prerequisites\n\nThe user's responsibility; verify what's verifiable, ask for the rest, invent nothing. State which\nitems you verified vs. which the user asserted.\n\n- **Connection mode** (Orca server vs SSH) confirmed with the user — see §1 step 2; it shapes the recipe.\n- **Cloud account + plan** that allows sandboxes/VMs. Ask.\n- **Provider CLI installed + authenticated** — detect (`command -v `), check auth (e.g.\n `vercel whoami`). If missing, point at the provider's docs; don't log them in.\n- **Scope / project / region** the sandboxes live under. Ask; flows into every script via state.\n- **Plan / timeout / RAM caps.** Record them — e.g. Vercel Hobby caps sandbox timeout at **45m**,\n which limits both the base build and per-workspace runtime (see §10).\n- **Git token for private repos** (`GH_TOKEN`/`GITHUB_TOKEN`, or the provider's git auth; can fall back\n to `gh auth token`). See §5.\n- **Coding-agent CLI choice** (`codex`, `claude`…) and that the user has an account — it gets\n authenticated into the VM in Phase 3.\n\n---\n\n## 3. Phase 2 — Base snapshot (the reusable image)\n\nBuild **once**, snapshot, and every workspace boots from it in seconds instead of rebuilding.\nProvisioning + building takes a while (often ~20–30 min), so it runs behind a checkpoint. The script\nshape is §7a; key points:\n\n- Build the **headless Electron main only** (not the renderer) so it fits in plan RAM.\n- Use the VM image's package manager (`apt`/`dnf`/`apk`, per the base distro — not the provider brand).\n- Clone with the git token via `GIT_ASKPASS` (§5).\n- **Trap errors and remove the half-built sandbox** so a crash doesn't leave a paid resource running.\n- Snapshot the stopped sandbox, parse the snapshot id, and write it + scope/project/port/repo to state.\n\n---\n\n## 4. Phase 3 — Agent-auth snapshot (interactive)\n\nThe base snapshot has the agent CLI installed but **not logged in**, and per-workspace VMs are\nephemeral — so authenticate once and bake it into a second snapshot layer. Script shape is §7b:\n\n1. Boot a sandbox from the base `snapshotId` (from state).\n2. Run the agent's login **interactively** (`--interactive --tty`); the user completes the URL/code in\n their browser. On a **headless VM this must be the device-auth flow** (e.g. `codex login --device-auth`),\n **not** plain `codex login`: the default OAuth login starts a loopback callback server on a container\n port the host browser can't reach, so it hangs. Device-auth instead prints a URL + code the user opens\n on the **host**.\n3. Verify login; **refuse to snapshot an unauthenticated VM.** Prefer the status command's **exit code**\n (most agent CLIs exit non-zero when unauthenticated). If you grep instead, agent status often goes to\n **stderr** (e.g. `codex login status` prints \"Logged in using ChatGPT\" there), so **fold stderr first**\n (`... 2>&1 | grep …`) and match the agent's **exact success line** — never `grep -qi 'logged in'`, which\n also matches \"**not** logged in\" and would commit an unauthenticated image.\n4. Re-snapshot, parse the new id, and overwrite `snapshotId` in state to the authenticated image\n (recording `authSourceSnapshotId`). Remove the auth sandbox.\n\n**You can't drive step 2 yourself** (you run commands non-interactively — no TTY). The **user** runs it in\ntheir own terminal, or via the Claude Code harness bang-prefix (`! `, with the required space after\n`!`). You scaffold/boot the sandbox and run steps 3–4, but **you cannot observe the interactive login\nfinishing** — so **ask the user to tell you when it's done** before you verify and re-snapshot.\n\nIf the agent's credentials are short-lived, warn that the snapshot may need periodic re-auth (§10).\n\nFor disposable runtimes, do **not** treat a host agent config directory (for example `~/.codex`) as the\nauth snapshot by bind-mounting or copying it wholesale. Agent homes often contain sqlite state, hook\napproval state, caches, logs, and host-specific env/config. Instead, authenticate/configure the agent\ninside the disposable runtime and snapshot/commit that runtime layer.\n\n---\n\n## 5. Credentials\n\n- **Never** commit secrets or put them in `userData`, recipe JSON, comments, docs, or the state file.\n- **Git token:** read from env (`GH_TOKEN`/`GITHUB_TOKEN`), falling back to `gh auth token`. Pass to the\n VM only via the provider's ephemeral `--env`. Inside the VM, use a `GIT_ASKPASS` helper with\n `x-access-token` (not the token in the clone URL) and `GIT_TERMINAL_PROMPT=0` so a missing token fails\n fast instead of hanging. When you write the helper from inside `bash -lc` under `set -u`, escape the\n positional arg and the token (`\\$1`, `\\$GH_TOKEN`) so they land **literally** and resolve at git-runtime\n — an unescaped `$1` aborts with \"unbound variable\", and a literal `$GH_TOKEN` keeps the real token out of\n the written file. `rm -f` the helper after the clone/fetch.\n- **Provider auth:** rely on the provider CLI's logged-in session, not checked-in keys.\n- **Agent auth:** lives in the authenticated snapshot (Phase 3) — never a file you write or commit.\n- State holds only **non-secret** wiring (snapshot ids, scope, project, port, repo url/ref).\n\n---\n\n## 6. State file\n\nA repo-local JSON file (e.g. `scripts/orca-vm/-state.json`) threads non-secret values between\nphases. Each script resolves values as **env var → state → built-in fallback**, and merges its outputs\nback. Phase 2 writes the base `snapshotId`; Phase 3 overwrites it with the authenticated snapshot;\nper-workspace `create` boots from `snapshotId`.\n\n```json\n{\n \"baseName\": \"orca-base\",\n \"snapshotId\": \"snap_authenticated_image_id\",\n \"authSourceSnapshotId\": \"snap_base_image_id\",\n \"scope\": \"\",\n \"project\": \"\",\n \"port\": 7331,\n \"repoUrl\": \"https://host/org/repo.git\",\n \"repoRef\": \"main\",\n \"projectRoot\": \"/abs/path/on/remote/repo\"\n}\n```\n\n---\n\n## 7. Script templates (provider-agnostic shapes)\n\nScaffold under `scripts/orca-vm/`. These are **shapes** — fill in the provider's real commands. All\nreserve stdout for the final JSON and log progress to stderr. Include a shared `json_value ` /\n`env_value ` reader (env → state → fallback) in each.\n\n**Where each script runs:**\n\n- **Local-side** (`create`/`suspend`/`resume`/`destroy` + the base-snapshot/auth scripts the user\n invokes) runs **on the user's desktop**, so it must run on their OS. macOS/Linux: `#!/usr/bin/env\n bash`, `set -euo pipefail`, quoted paths. **Windows:** a bare `.sh` won't run — scaffold `.ps1`/`.cmd`\n or require WSL/Git-Bash and point `orca.yaml` at the right launcher.\n- **Remote-side** (commands you `exec` *inside* the Linux VM) always runs in the VM's Linux shell, so\n bash is fine there regardless of the user's OS.\n\n### 7a. Base-snapshot (`-base-snapshot.sh`) — Phase 2\n\n```bash\n#!/usr/bin/env bash\nset -euo pipefail\n# resolve base_name/repo_url/repo_ref/project_root/port/scope/project/timeout (env→state→fallback)\n# resolve gh token: GH_TOKEN | GITHUB_TOKEN | `gh auth token`\n# 1. provision a sandbox (timeout/vcpus/published port/snapshot retention); trap: remove on error\n# 2. remote exec (long timeout): install pkgs + gh + corepack/pnpm + agent CLI;\n# clone with GIT_ASKPASS(token); write headless main-only build config;\n# dev setup; pnpm install; build CLI; build headless electron main; smoke-check tools\n# 3. snapshot stopped sandbox; parse snapshot id (fail if unparseable)\n# 4. merge { baseName, snapshotId, projectRoot, repoUrl, repoRef, port, scope, project } into state\n# print only the state JSON to stdout\n```\n\nWorked Vercel commands for this phase are in §7f. You run this script by hand (not via `orca.yaml`),\nafter exporting the first-run inputs the state file doesn't have yet — e.g. provider scope/project, the\nrepo URL/ref, and a git token (`GH_TOKEN`); later runs read them back from state.\n\n### 7b. Auth (`-base-auth.sh`) — Phase 3\n\n```bash\n#!/usr/bin/env bash\nset -euo pipefail\n# read source snapshot from state.snapshotId (fail if absent); auth_name=\"${base_name}-auth\"\n# 1. boot sandbox from source snapshot; trap: remove on error\n# 2. INTERACTIVE/TTY remote exec: agent login — user completes URL/code. Headless VM: MUST use the\n# device-auth flow (e.g. `codex login --device-auth`) — plain OAuth login binds a loopback callback\n# port the host can't reach and hangs. User runs this themselves (you have no interactive TTY); ask\n# them to report back when it's done before continuing.\n# 3. verify login, then refuse to snapshot if not logged in. Prefer the status command's EXIT CODE (most\n# agent CLIs exit non-zero when unauthenticated) over string-matching. If you must grep, fold stderr\n# first (`status 2>&1 | grep …` — many agents print the success line there) and match the agent's exact\n# success line; never `grep -qi 'logged in'`, which also matches \"not logged in\". Codex example: §7f.\n# 4. snapshot; parse new id\n# 5. merge { snapshotId:, authSourceSnapshotId: } into state; remove auth sandbox\n# print only the state JSON to stdout\n```\n\n### 7c. Create (`-create.sh`) — per workspace\n\n```bash\n#!/usr/bin/env bash\nset -euo pipefail\n# read authenticated snapshotId/scope/project/port/repo*/project_root (env→state→fallback)\n# fail clearly if snapshotId is missing (point back to Phases 2–3)\n# name = orca-${ORCA_RECIPE_ID}-${ORCA_VM_INSTANCE_ID} (sanitized, length-capped)\n# 1. boot sandbox from snapshotId with a published port; capture the public URL → pairing address\n# (an externally reachable wss:// URL); trap: remove sandbox on error\n# 2. remote exec: ensure repo at desired commit; rebuild only if commit changed (cache marker)\n# 3. remote exec: start orca serve in the background and read the recipe JSON it writes (see below)\n# 4. print serve's JSON to stdout, optionally enriched with userData:\n# { schemaVersion:1, pairingCode, projectRoot, userData:{ provider, resourceId:name, snapshotId } }\n```\n\n**The exact `orca serve` invocation and its output (verified — do not improvise the flags).** Inside the\nVM, run:\n\n```bash\norca serve \\\n --port \"$PORT\" \\\n --project-root \"$ABS_REPO_PATH_ON_REMOTE\" \\\n --pairing-address \"$EXTERNAL_WSS_URL\" \\\n --recipe-json\n```\n\n**Binary name:** in a VM built from source (the Phase-2 flow), run it as `pnpm exec orca-dev serve …`\nfrom the repo root — `orca-dev` is the in-repo entrypoint and is what the §7f example uses. Plain\n`orca serve …` is the same command when the built CLI is installed on the VM's PATH. The flags/output\nare identical either way.\n\nThere is **no `--host` flag**. `--project-root` must be an absolute directory on the remote. With\n`--recipe-json` the server **stays running** and prints exactly this single object to **stdout**, then\nkeeps serving:\n\n```json\n{ \"schemaVersion\": 1, \"pairingCode\": \"\", \"projectRoot\": \"\" }\n```\n\n`pairingCode` is the pairing URL, already pointing at whatever you passed as `--pairing-address` — so set\n`--pairing-address` to the externally reachable address and **pass `pairingCode` through unchanged; never\nhand-rewrite it**. Because serve runs in the foreground and doesn't exit, redirect its stdout to a file\nand poll until that file parses as JSON (and bail if the process dies — dump its stderr log). Your\n`create` script then prints that JSON (optionally merging `userData`). Concrete pattern: §7f.\n\n### 7d. Suspend / resume / destroy — per workspace\n\n```bash\n#!/usr/bin/env bash\nset -euo pipefail\npayload=\"$(cat)\" # Orca passes lifecycle JSON on stdin\nresource_id=\"$(node -e 'const d=JSON.parse(process.argv[1]); process.stdout.write(d.recipeResult?.userData?.resourceId ?? \"\")' \"$payload\")\"\n[ -n \"$resource_id\" ] || { echo \"No resource id in lifecycle payload\" >&2; exit 1; }\n# suspend: provider suspend \"$resource_id\"\n# resume: provider resume \"$resource_id\"; then RE-EMIT fresh recipe JSON (pairing may change)\n# destroy: provider remove \"$resource_id\" (or set destroy: none in orca.yaml)\n```\n\n### 7e. State file — scaffold with scope/project/repo filled in and snapshot ids empty (§6).\n\n### 7f. Worked example — Vercel Sandbox (all three phases)\n\nA real, working shape (the Vercel surface is a CLI: `vercel sandbox create|exec|snapshot|remove`). Adapt\nnames; verify flags against `vercel sandbox --help` for the user's CLI version before relying on them.\nThese ground §7a (base snapshot) and §7b (auth), which are otherwise generic skeletons.\n\n**Phase 2 — base snapshot (§7a):** provision → install tools + clone + headless build → snapshot.\n\n```bash\n# provision a fresh build sandbox (retain a couple of snapshots); trap-remove on error\nvercel sandbox create --name \"$base\" --runtime node24 --timeout 30m --vcpus 4 --publish-port \"$port\" \\\n --snapshot-expiration 30d --keep-last-snapshots 2 \"${vercel_args[@]}\" >&2\n# remote build (long timeout): install pkgs+gh+pnpm+agent CLI, clone with GIT_ASKPASS (write the helper\n# with LITERAL \\$1/\\$GH_TOKEN so they resolve at git-runtime, not write-time — see §5/§7f create — then\n# `rm -f /tmp/askpass.sh`), write the headless main-only build config (drop the renderer), dev setup,\n# build CLI + headless main, smoke-check\nvercel sandbox exec \"$base\" \"${vercel_args[@]}\" --timeout 25m --env \"GH_TOKEN=$gh_token\" … -- bash -lc '…build…' >&2\n# snapshot the STOPPED sandbox and parse the id from CLI output (fail if unparseable)\nout=\"$(vercel sandbox snapshot \"$base\" --stop --expiration 30d \"${vercel_args[@]}\" 2>&1)\"; printf '%s\\n' \"$out\" >&2\nsnapshot_id=\"$(printf '%s\\n' \"$out\" | sed -nE 's/.*(snap_[A-Za-z0-9]+).*/\\1/p' | tail -1)\"\n# merge { baseName, snapshotId, scope, project, port, repoUrl, repoRef, projectRoot } into state; print state JSON\n```\n\n**Phase 3 — agent-auth snapshot (§7b):** boot the base, log the agent in interactively, re-snapshot.\n(`codex` below is an example — substitute the user's chosen agent's login/status verbs, e.g. `claude`.)\n\n```bash\nvercel sandbox create --name \"$auth\" --snapshot \"$snapshot_id\" --timeout 30m --publish-port \"$port\" \"${vercel_args[@]}\" >&2\n# INTERACTIVE — the USER runs this in their own terminal (you have no interactive TTY) and completes the\n# URL/code on the HOST. --device-auth is MANDATORY on a headless VM: plain `codex login` binds a loopback\n# callback port the host browser can't reach and hangs. Ask the user to report back when login finishes.\nvercel sandbox exec --interactive --tty \"$auth\" \"${vercel_args[@]}\" -- bash -lc 'codex login --device-auth'\n# refuse to snapshot an unauthenticated VM — fold stderr, match codex's exact success line (§4)\nvercel sandbox exec \"$auth\" \"${vercel_args[@]}\" --timeout 30s -- bash -lc 'codex login status 2>&1' | grep -Eqi 'Logged in using ChatGPT|Logged in via device' \\\n || { echo \"agent not logged in; not snapshotting\" >&2; exit 1; }\nout=\"$(vercel sandbox snapshot \"$auth\" --stop --expiration 30d \"${vercel_args[@]}\" 2>&1)\"; printf '%s\\n' \"$out\" >&2\nnew_id=\"$(printf '%s\\n' \"$out\" | sed -nE 's/.*(snap_[A-Za-z0-9]+).*/\\1/p' | tail -1)\"\n# overwrite state.snapshotId = new_id, record authSourceSnapshotId = snapshot_id; remove the auth sandbox\n```\n\n**Per-workspace `create`** (the fast path):\n\n```bash\n#!/usr/bin/env bash\nset -euo pipefail\n# resolve from env→state→fallback: snapshot_id, scope, project, port, repo_url, repo_ref, project_root\nvercel_args=(); [ -n \"$scope\" ] && vercel_args+=(--scope \"$scope\"); [ -n \"$project\" ] && vercel_args+=(--project \"$project\")\n[ -n \"$snapshot_id\" ] || { echo \"snapshotId missing — run Phases 2–3 first\" >&2; exit 1; }\ngh_token=\"${GH_TOKEN:-${GITHUB_TOKEN:-$(command -v gh >/dev/null 2>&1 && gh auth token 2>/dev/null || true)}}\"\nrecipe_id=\"${ORCA_RECIPE_ID:-vercel-sandbox}\"\nrecipe_id=\"${recipe_id//./-}\" # Vercel names forbid dots.\ninstance_id=\"${ORCA_VM_INSTANCE_ID:-$(date +%s)}\"\nmax_recipe_id_length=$((128 - ${#instance_id} - 6)) # Preserve the unique instance suffix.\n[ \"$max_recipe_id_length\" -gt 0 ] || { echo \"ORCA_VM_INSTANCE_ID is too long for a Vercel sandbox name\" >&2; exit 1; }\nname=\"orca-${recipe_id:0:max_recipe_id_length}-${instance_id}\"\n\n# Arm cleanup BEFORE create so a failing create can't leak a half-built paid sandbox.\ncleanup_on_error() { [ \"$?\" -ne 0 ] && vercel sandbox remove \"$name\" \"${vercel_args[@]}\" >/dev/null 2>&1 || true; }\ntrap cleanup_on_error EXIT\n\n# 1. boot from the authenticated snapshot, publish the serve port\ncreate_output=\"$(vercel sandbox create --name \"$name\" --snapshot \"$snapshot_id\" \\\n --timeout 30m --publish-port \"$port\" \"${vercel_args[@]}\" 2>&1)\"; printf '%s\\n' \"$create_output\" >&2\n# Vercel prints the published https URL; derive the external wss:// pairing address from it\npublic_url=\"$(printf '%s\\n' \"$create_output\" | sed -nE 's#.*(https://[^[:space:]]+\\.vercel\\.run).*#\\1#p' | head -1)\"\n[ -n \"$public_url\" ] || { echo \"no published URL in create output\" >&2; exit 1; }\npairing_ws=\"${public_url/https:\\/\\//wss://}\"\n\n# 2. (remote) ensure the repo is at the right commit; rebuild only if the commit changed (cache marker)\nvercel sandbox exec \"$name\" \"${vercel_args[@]}\" --timeout 20m \\\n --env \"GH_TOKEN=$gh_token\" --env \"ORCA_PROJECT_ROOT=$project_root\" \\\n --env \"ORCA_REPO_URL=$repo_url\" --env \"ORCA_REPO_REF=$repo_ref\" \\\n -- bash -lc 'set -euo pipefail; cd \"$ORCA_PROJECT_ROOT\"; \\\n # Re-establish git auth for the private-repo fetch (why + full rationale: §5); else it hangs on a prompt.\n # Load-bearing escaping: \\$1 and \\$GH_TOKEN must land LITERALLY and resolve at git-runtime. Test after\n # any edit here — reformatting the nested printf/node quoting silently breaks the fetch or leaks the token.\n if [ -n \"${GH_TOKEN:-}\" ]; then \\\n printf \"%s\\n\" \"#!/usr/bin/env bash\" \"case \\\"\\$1\\\" in *Username*) echo x-access-token;; *Password*) echo \\\"\\$GH_TOKEN\\\";; esac\" > /tmp/askpass.sh; \\\n chmod 700 /tmp/askpass.sh; export GIT_ASKPASS=/tmp/askpass.sh GIT_TERMINAL_PROMPT=0; fi; \\\n git fetch origin \"$ORCA_REPO_REF\"; \\\n git checkout -B \"$ORCA_REPO_REF\" FETCH_HEAD; \\\n rm -f /tmp/askpass.sh; \\\n c=\"$(git rev-parse HEAD)\"; [ -f .orca-built ] && [ \"$(cat .orca-built)\" = \"$c\" ] || { \\\n pnpm install --prefer-offline && pnpm run build:cli && \\\n node config/scripts/run-electron-vite-build.mjs --config config/electron-vite.vm-serve.config.ts && \\\n printf \"%s\" \"$c\" > .orca-built; }' >&2\n\n# 3. (remote) start orca serve in the background, writing recipe JSON to a file; poll until it parses\nrecipe_json=\"$(vercel sandbox exec \"$name\" \"${vercel_args[@]}\" --timeout 60s \\\n --env \"ORCA_PORT=$port\" --env \"ORCA_PROJECT_ROOT=$project_root\" --env \"ORCA_PAIRING_ADDRESS=$pairing_ws\" \\\n -- bash -lc 'set -euo pipefail; cd \"$ORCA_PROJECT_ROOT\"; rm -f /tmp/orca-recipe.json /tmp/orca-serve.log; \\\n nohup pnpm exec orca-dev serve --port \"$ORCA_PORT\" --project-root \"$ORCA_PROJECT_ROOT\" \\\n --pairing-address \"$ORCA_PAIRING_ADDRESS\" --recipe-json >/tmp/orca-recipe.json 2>/tmp/orca-serve.log /dev/null 2>&1 && { cat /tmp/orca-recipe.json; exit 0; }; \\\n kill -0 \"$pid\" 2>/dev/null || { cat /tmp/orca-serve.log >&2; exit 1; }; sleep 0.25; \\\n done; cat /tmp/orca-serve.log >&2; echo \"serve recipe JSON timed out\" >&2; exit 1')\"\n\n# 4. print serve's JSON enriched with userData (single object on stdout)\nnode -e 'const p=JSON.parse(process.argv[1]); console.log(JSON.stringify({...p, schemaVersion:1,\n userData:{...p.userData, provider:\"vercel-sandbox\", resourceId:process.argv[2], snapshotId:process.argv[3]}}))' \\\n \"$recipe_json\" \"$name\" \"$snapshot_id\"\ntrap - EXIT\n```\n\n`suspend`/`resume`/`destroy` use `vercel sandbox stop|...|remove \"$resource_id\"` reading\n`userData.resourceId` from stdin (§7d). This is the **Orca-server** connection mode (the recipe emits a\npairing URL). If the user chose **SSH** in the §1 interview, use §7g instead.\n\n### 7g. Worked example — existing SSH host (SSH connection mode)\n\nSSH mode is **fundamentally different from §7c/§7f**, not a relabeling of them:\n\n- **`create` does NOT run `orca serve` and does NOT emit a `pairingCode`.** Orca itself connects to the\n host over its SSH relay, brings up the git + filesystem providers, and imports the repo. The script's\n only job is to make the host ready and **print SSH connection details** Orca will dial.\n- The result uses a `connection` block with `type: \"ssh\"` and a `target`, **not** the flat\n `pairingCode`/`projectRoot` shape. Exact shape (Orca rejects anything else):\n\n```json\n{\n \"schemaVersion\": 1,\n \"connection\": {\n \"type\": \"ssh\",\n \"projectRoot\": \"/abs/path/to/repo/on/host\",\n \"target\": {\n \"label\": \"my-box\",\n \"host\": \"192.0.2.10\",\n \"port\": 22,\n \"username\": \"ubuntu\",\n \"identityFile\": \"~/.ssh/id_ed25519\",\n \"jumpHost\": \"bastion.example.com\",\n \"proxyCommand\": \"cloudflared access ssh --hostname %h\",\n \"relayGracePeriodSeconds\": 0,\n \"portForwards\": []\n }\n }\n}\n```\n\n`label`, `host`, `port`, `username` are required; the rest are optional — omit any you don't need.\n\nFor an explicitly requested one-VM-per-workspace checkout, the create script must read\n`ORCA_RECIPE_RESULT_SCHEMA_VERSION`, `ORCA_REPO_URL`, `ORCA_REPO_REF`, `ORCA_REPO_REF_HEAD`, and\n`ORCA_REPO_BRANCH`. Use `ORCA_REPO_REF` to fetch the selected source, but create\n`ORCA_REPO_BRANCH` at the exact `ORCA_REPO_REF_HEAD` commit; resolving the symbolic ref again can race\nwith an upstream update. `ORCA_REPO_URL` and `ORCA_REPO_REF` are a matched fetch pair, including when\nthe desktop source uses multiple remotes. Return that primary checkout at `projectRoot` and emit the\nsame SSH result with:\n\n```bash\n[ -n \"${ORCA_REPO_REF_HEAD:-}\" ] || { echo \"missing pinned source commit\" >&2; exit 1; }\ngit fetch origin \"$ORCA_REPO_REF\"\ngit cat-file -e \"${ORCA_REPO_REF_HEAD}^{commit}\"\ngit checkout -B \"$ORCA_REPO_BRANCH\" \"$ORCA_REPO_REF_HEAD\"\n```\n\n```json\n{\n \"schemaVersion\": 2,\n \"checkoutMode\": \"provisioned-root\",\n \"connection\": {\n \"type\": \"ssh\",\n \"projectRoot\": \"/abs/repo\",\n \"target\": { \"label\": \"my-box\", \"host\": \"192.0.2.10\", \"port\": 22, \"username\": \"ubuntu\" }\n }\n}\n```\n\nFail if the requested schema is not `2`; do not silently fall back to the ordinary recipe shape.\n\n**Networking → which `target` fields to set** (how *your desktop* reaches the box — there is no\n`orca serve` URL in SSH mode):\n\n- Public IP / DNS, or a Tailscale/VPN address → `host`; SSH port → `port` (usually 22).\n- Key auth → `identityFile` (add `identitiesOnly: true` if the agent has many keys).\n- Through a bastion → `jumpHost` (a `user@host` ProxyJump) **or** a full `proxyCommand` (e.g. an access\n proxy). Use one, not both.\n- A service port the workspace needs → add entries to `portForwards`.\n- `relayGracePeriodSeconds` (optional): how long Orca keeps the SSH relay alive after the workspace\n detaches before tearing it down; `0` = tear down immediately. Leave it off unless the user wants a\n reconnect grace window.\n\n**Toolchain & agent auth on a persistent (no-snapshot) host — do this ONCE, by hand, before wiring the\nrecipe** (there's no base image to bake; the host *is* the base). Run the §7f Phase-2 install steps and\nthe §7f Phase-3 ` login --device-auth` **directly over SSH on the host** (interactive, e.g.\n`ssh -t user@host ' login --device-auth'`). After that the host stays ready across workspaces.\n\n```bash\n#!/usr/bin/env bash\nset -euo pipefail\n# resolve from env→state→fallback (default unset optionals to \"\"): ssh_username, host,\n# ssh_port (default 22), identity_file, jump_host, proxy_command, project_root, repo_url, repo_ref\n: \"${identity_file:=}\"; : \"${jump_host:=}\"; : \"${proxy_command:=}\" # avoid set -u aborts on optionals\ngh_token=\"${GH_TOKEN:-${GITHUB_TOKEN:-$(command -v gh >/dev/null 2>&1 && gh auth token 2>/dev/null || true)}}\"\nssh_target=\"${ssh_username}@${host}\"\nssh_opts=(-p \"$ssh_port\"); [ -n \"$identity_file\" ] && ssh_opts+=(-i \"$identity_file\")\n# Why: a fresh host's key isn't in known_hosts; a StrictHostKeyChecking prompt would HANG a\n# non-interactive create. Pre-add the key (or set the option) so it can't block.\nssh-keyscan -p \"$ssh_port\" \"$host\" >> \"$HOME/.ssh/known_hosts\" 2>/dev/null || true\n\n# 1. ensure the repo is present and at the right commit on the host (NO orca serve here)\nssh \"${ssh_opts[@]}\" \"$ssh_target\" \\\n \"GH_TOKEN='$gh_token' GIT_TERMINAL_PROMPT=0 bash -lc '\n set -euo pipefail\n [ -d \\\"$project_root/.git\\\" ] || git clone \\\"$repo_url\\\" \\\"$project_root\\\"\n cd \\\"$project_root\\\" && git fetch origin \\\"$repo_ref\\\" && git checkout -B \\\"$repo_ref\\\" FETCH_HEAD\n '\" >&2\n\n# 2. print the SSH connection block (NO pairingCode, NO orca serve). host/port/username tell Orca's\n# relay how to dial in; identityFile/jumpHost/proxyCommand/portForwards are emitted when set.\nnode -e 'const [host,port,user,idf,jh,pc,root]=process.argv.slice(1);\n const target={ label:\"per-workspace-host\", host, port:Number(port), username:user };\n if(idf) target.identityFile=idf; if(jh) target.jumpHost=jh; if(pc) target.proxyCommand=pc;\n // add target.portForwards=[...] here if the workspace needs forwarded service ports\n console.log(JSON.stringify({ schemaVersion:1, connection:{ type:\"ssh\", projectRoot:root, target } }))' \\\n \"$host\" \"$ssh_port\" \"$ssh_username\" \"$identity_file\" \"$jump_host\" \"$proxy_command\" \"$project_root\"\n```\n\n`suspend`/`resume`/`destroy`: on a persistent host there's usually nothing to tear down — set\n`destroy: none` and omit suspend/resume. (Orca still disconnects/reconnects its own SSH relay on\nsleep/wake/delete — that's separate from these scripts.)\n\nIf the SSH host is instead an **ephemeral/snapshot-capable VM** (your hypervisor, or a cloud VM with\nimage support), keep the §7f Phase-2/3 base-image model for provisioning, but still emit the\n`connection.type:\"ssh\"` block above instead of starting `orca serve`.\n\n### 7h. Worked example — local Docker SSH (SSH connection mode)\n\nLocal Docker can model an ephemeral SSH VM without cloud cost: build a base image with `sshd`, tools,\nrepo prerequisites, and the agent CLI; run an **interactive auth container** once; then `docker commit`\nthat container as the authenticated image used by per-workspace `create`.\n\nKey points:\n\n- Publish container SSH to a random localhost port (`-p 127.0.0.1::22`) and emit\n `connection.type:\"ssh\"` with `host:\"127.0.0.1\"`, that port, `username`, `identityFile`, and\n `identitiesOnly:true`.\n- Generate a repo-local SSH key if needed, but gitignore the private/public key files.\n- **Bake SSH host keys into the base image** (`ssh-keygen -A` at **build** time; at runtime only generate\n if absent). Ephemeral containers all present the **same** host key, so `known_hosts` on `127.0.0.1`\n doesn't churn as the published port rotates across workspaces (otherwise every container's freshly\n generated key collides on `localhost` and trips host-key-changed warnings).\n- The auth image is the Docker equivalent of Phase 3: the **user** runs the agent login **inside** the\n container (you can't drive it — you have no interactive TTY), configures proxy env/config, approves\n hooks, and you commit once they report it's done. On a headless container use the **device-auth** flow\n (§4). Verify login before committing — exit code, or fold stderr and match the exact success line (§4).\n- Do not bind-mount or copy the host's full agent home into the image. Let each container have writable\n agent state; only the committed auth image should carry reusable authenticated state.\n- If committing from an interactive shell, force the runtime entrypoint back to `sshd`:\n `docker commit --change='ENTRYPOINT [\"/usr/local/bin/orca-docker-ssh-entrypoint\"]' …`.\n- `destroy` should read `recipeResult.userData.resourceId` and run `docker rm -f \"$resource_id\"`.\n\nValidation before wiring/live use:\n\n```bash\ndocker image inspect \"$auth_image\" --format '{{json .Config.Entrypoint}}'\ndocker run -d --name \"$name\" -p 127.0.0.1::22 -e \"ORCA_SSH_PUBLIC_KEY=$pubkey\" \"$auth_image\"\ndocker ps -a --filter \"name=$name\"\ndocker logs \"$name\"\nssh -i \"$key\" -p \"$port\" -o IdentitiesOnly=yes user@127.0.0.1 'codex --version'\n```\n\nIf the container exits immediately, inspect logs before the cleanup trap removes it; a committed\ninteractive image with `ENTRYPOINT [\"bash\"]` is a common cause.\n\nAlso confirm the **host key is stable** across containers: the SSH `ssh -i … 127.0.0.1` dial should not\ntrigger a host-key-changed warning when a second container reuses the port. If it does, the host keys\nweren't baked into the base image (see the `ssh-keygen -A` point above).\n\n### 7i. Windows local-side scripts\n\nThe local-side scripts run on the user's desktop. On **Windows**, a bare `.sh` won't execute. Either\nrequire WSL/Git-Bash (and point `orca.yaml` at e.g. `bash ./scripts/orca-vm/.sh` via a `.cmd`\nlauncher), or scaffold PowerShell equivalents. Minimal PowerShell shape:\n\n```powershell\n#requires -Version 5\n$ErrorActionPreference = 'Stop'\n# resolve env→state→fallback; run the provider CLI / ssh the same way;\n# capture provider output; build the result object for the chosen mode and write ONE line of JSON to stdout.\n# Orca-server mode: @{ schemaVersion=1; pairingCode=$pairingCode; projectRoot=$projectRoot; userData=@{...} }\n# SSH mode: @{ schemaVersion=1; connection=@{ type=\"ssh\"; projectRoot=$projectRoot;\n# target=@{ label=$label; host=$host; port=$port; username=$user } } } (see §7g/§7h)\n($result | ConvertTo-Json -Compress -Depth 6)\n# progress/errors → Write-Error / the error stream, never stdout.\n```\n\nThe remote-side commands you run *inside* the Linux VM stay bash regardless of the desktop OS.\n\n---\n\n## 8. Per-workspace recipe contract (the fast path)\n\nOnce the authenticated snapshot exists, this runs on every workspace create. Define recipes in\n`orca.yaml`:\n\n```yaml\nenvironmentRecipes:\n - id: cloud-sandbox\n name: Cloud Sandbox\n create: ./scripts/orca-vm/cloud-sandbox-create.sh\n suspend: ./scripts/orca-vm/cloud-sandbox-suspend.sh\n resume: ./scripts/orca-vm/cloud-sandbox-resume.sh\n destroy: ./scripts/orca-vm/cloud-sandbox-destroy.sh\n```\n\n`create` runs **locally from the repo root** and prints **one** JSON object to stdout. Its shape depends\non the connection mode chosen in §1:\n\n**Orca-server mode** — boot the env, start `orca serve` in it, and print serve's result:\n\n```json\n{\n \"schemaVersion\": 1,\n \"pairingCode\": \"orca-pairing-code-or-url\",\n \"projectRoot\": \"/absolute/path/to/repo/on/remote\",\n \"userData\": { \"provider\": \"example\", \"resourceId\": \"provider-resource-id\" }\n}\n```\n\nHere `pairingCode` (from `orca serve --recipe-json`) and `projectRoot` are required; `schemaVersion` (`1`)\nand `userData` are optional.\n\n**SSH mode** — do **not** run `orca serve`; print the `connection.type:\"ssh\"` block instead (full shape +\nworked script in §7g). `pairingCode` is **not** used in SSH mode.\n\n**Optional provisioned root** — only for direct SSH and only when explicitly requested. Add\n`checkoutMode: provisioned-root` to the recipe, require `ORCA_RECIPE_RESULT_SCHEMA_VERSION=2`, create\nthe requested `ORCA_REPO_BRANCH` at the pinned `ORCA_REPO_REF_HEAD` commit (use `ORCA_REPO_REF` only\nto fetch that commit) at the returned `projectRoot`, and emit schema version 2 with\n`checkoutMode: \"provisioned-root\"`. All recipes without this field retain the schema-v1 behavior above.\n\nLifecycle hooks (all run locally):\n\n- `create`: required. Prints recipe result JSON.\n- `suspend`: optional. Sleep; reads lifecycle payload on stdin.\n- `resume`: optional. Wake; reads payload on stdin and **prints fresh recipe JSON** (pairing may change).\n- `destroy`: optional unless `destroy: none`. Delete/cleanup; reads payload on stdin.\n\nStart Orca remotely with `orca serve --port \"$PORT\" --project-root \"$ABS_ROOT\" --pairing-address\n\"$EXTERNAL_WSS_URL\" --recipe-json` (exact flags + output in §7c). Set `--pairing-address` to the\nexternally reachable address so the emitted `pairingCode` is reachable; tunneling/port mapping is the\nscript's job.\n\nBackward compatibility: `command`→`create`, `cleanup`→`destroy`, `cleanup: none`→`destroy: none`.\nPrefer the lifecycle names.\n\n---\n\n## 9. Doctor and validation\n\nValidate in two stages — the cheap dry run first, then the live self-test.\n\n### Dry run (free, non-destructive) — always do this first\n\n`orca vm recipe doctor --repo-path --json` validates **static wiring only** — it does\n**not** boot anything. It checks: local-host execution (v1), repo path, recipe id exists,\ncreate/destroy/suspend/resume command paths resolve, suspend/resume are paired, and each script is\nexecutable (POSIX exec bit; skipped on Windows). Fix every failure here before spending any cloud money.\n\n### Live self-test (`--provision`) — diagnose and iterate yourself\n\n`orca vm recipe doctor --repo-path --provision --json` actually runs the recipe end\nto end: it executes `create`, validates the returned recipe JSON, then runs `destroy` to **tear the\nenvironment back down** (so the test leaves nothing running, as long as `destroy` works). It spends real\ncloud money, so get the user's OK **once** before starting — that one approval covers the whole loop\nbelow; do not re-ask before each run.\n\nOn failure, the JSON result includes a `provisionTranscript` with the **complete** captured output of\neach stage so you can self-diagnose without asking the user to relay logs:\n\n```json\n{\n \"ok\": false,\n \"checks\": [ { \"id\": \"recipe.provision\", \"status\": \"fail\", \"message\": \"…\" } ],\n \"provisionTranscript\": {\n \"provision\": { \"exitCode\": 0, \"signal\": null, \"stdout\": \"…\", \"stderr\": \"…\", \"parseError\": \"…\" },\n \"destroy\": { \"exitCode\": 0, \"signal\": null, \"stdout\": \"…\", \"stderr\": \"…\" }\n }\n}\n```\n\n**Run it as a loop:** read `provisionTranscript.provision.stderr` / `.stdout` / `.parseError` (and\n`destroy.*`), fix the script, and re-run `--provision` until `ok` is `true` — iterating on your own\nrather than waiting for the user to paste errors. Common reads: a non-empty `stderr` with `exitCode 0`\nplus a `parseError` means `create` ran but printed something other than the single recipe-result JSON on\nstdout (often a stray `echo` — route it to stderr, see §10); a non-zero `exitCode` is a provider/script\nfailure described in `stderr`. Each stream is redacted and capped (head+tail) — large logs keep both the\nsetup context and the failure.\n\nThe self-test cannot see provider-side truth beyond what the scripts print, so still confirm: state has a\npopulated **authenticated** `snapshotId` (Phases 2–3 done), and `destroy` is implemented/tested (or\nexplicitly `none` — in which case the self-test won't tear down, so clean up manually).\n\nFor SSH recipes, also smoke-test the exact emitted target before declaring success: dial the host/port\nwith the identity/proxy settings, run `pwd`, verify the repo path, check the agent binary, and confirm\n`destroy` removes the provider resource/container. For Docker, inspect the auth image entrypoint and do a\nstartup-only `docker run` before the full clone/install path.\n\n---\n\n## 10. Failure modes\n\n- **Build exceeds plan timeout (e.g. Hobby 45m).** Use enough vCPUs and a timeout covering the build;\n else split work or use a higher plan. The cap also limits per-workspace runtime — surface it.\n- **Build exceeds plan RAM.** Build the **headless main only** (drop the renderer) — the biggest fitter.\n- **Private-repo clone hangs/fails.** Wrong/missing token. Use `GIT_ASKPASS` + `GIT_TERMINAL_PROMPT=0`\n so it fails fast instead of prompting.\n- **`GIT_ASKPASS` helper aborts the clone with \"`$1: unbound variable`\".** The `printf`/heredoc that writes\n the helper inside `bash -lc` under `set -u` expanded `$1`/`$GH_TOKEN` at **write** time. Escape them\n (`\\$1`, `\\$GH_TOKEN`) so they land literally and resolve at git-runtime; this also keeps the real token\n out of the file. `rm -f` the helper afterward (§5, §7f).\n- **Agent verified as \"not logged in\" despite a good login.** `codex login status` (and similar) print\n \"Logged in …\" to **stderr**; an stdout-only `grep` misses it. Prefer the status **exit code**; if you\n grep, fold stderr first (`status 2>&1 | grep …`) and match the exact success line — not `grep -qi\n 'logged in'`, which also matches \"not logged in\".\n- **Headless agent login hangs.** Plain OAuth `login` starts a loopback callback server on a VM/container\n port the host browser can't reach. Use the **device-auth** flow (`login --device-auth`) — it prints a\n URL + code the user opens on the host.\n- **`known_hosts` host-key churn on local Docker.** Each ephemeral container regenerating its SSH host key\n collides on `127.0.0.1` as the published port rotates. Bake host keys into the base image at build time\n (`ssh-keygen -A`; runtime generates only if absent) so all containers share one stable key (§7h).\n- **Snapshot expired/evicted.** If `create` hits an unknown snapshot id, rerun Phases 2–3 and update\n `snapshotId`.\n- **Agent auth didn't persist.** Confirm `snapshotId` points at the **authenticated** snapshot; re-run\n Phase 3. Warn that short-lived tokens may need periodic re-auth.\n- **Agent auth copied from the host breaks.** Do not bind-mount/copy a full host agent home; sqlite\n files can be unwritable or host-specific, hooks may need approval again, and config may reference\n local-only env vars. Authenticate inside the runtime and snapshot/commit that layer.\n- **Docker auth image exits immediately.** Inspect `docker image inspect … .Config.Entrypoint` and\n `docker logs`. If the image was committed from an interactive shell, reset the entrypoint to the SSH\n entrypoint during `docker commit`.\n- **Leaked paid resource.** Every long script must trap errors and remove the sandbox it created.\n- **`create` emits non-JSON on stdout.** A stray `echo` corrupts the result — stdout is for the final\n JSON only; everything else to stderr. The `--provision` self-test surfaces this as `exitCode 0` + a\n `parseError` with the offending stdout in `provisionTranscript` (§9).\n\n---\n\n## 11. Boundaries\n\n- Don't create accounts, choose plans/regions, or invent scope/project/org/image/billing ids.\n- Don't invent or store credentials; no secrets in `userData`, state, comments, docs, or commits.\n- Don't run paid/long phases (base snapshot, auth, live test) without an explicit OK.\n- Don't hide provider errors behind generic messages — preserve actionable stderr.\n- Don't make Orca own provider lifecycle beyond invoking the configured scripts.\n- Don't commit or create an Orca workspace unless asked.\n" // oxfmt-ignore -const ORCHESTRATION_MARKDOWN = "---\nname: orchestration\ndescription: >-\n Use Orca orchestration for structured multi-agent coordination: threaded\n messages, blocking ask/reply flows, task dispatch, worker_done/escalation\n waits, task DAGs, decision gates, coordinator loops, or decomposing work\n across agents. Use `orca-cli` instead for full ownership handoffs, including\n requests phrased as \"hand off\", \"handoff\", \"handover\", \"give this to another\n agent\", or \"another worktree\" when the user did not explicitly ask to\n supervise, monitor, wait for results, or coordinate a DAG. Use `orca-cli` for\n ordinary terminal control, lightweight terminal prompts, shell commands, Orca\n worktree management, reading or waiting on terminals, and automation of the\n browser embedded inside Orca. Use Computer Use for browser windows, webviews,\n Orca app UI, or desktop UI outside Orca's embedded browser.\n---\n\n# Orca Inter-Agent Orchestration\n\nOrchestration is Orca's structured coordination layer for agent messages, task ownership, dispatch state, and worker completion tracking.\n\nUse this skill when coordination state matters. For lightweight terminal prompts or basic worktree/terminal/built-in-browser control, use `orca-cli`.\n\n## Tool Boundary\n\nIf a task says to use Orca orchestration, the coordinator must create or bind a Run, create the Task with `orca orchestration task-create`, then attach the worker with either the preferred `orca orchestration worker-start` composition or the low-level `orca orchestration dispatch --inject` path.\n\nDo not substitute non-Orca subagent tools, generic agent-spawn APIs, or chat-only parallel worker features. Those may create useful workers, but they do not create Orca task/dispatch provenance, injected lifecycle preambles, `worker_done` authority, or decision gates.\n\nBefore claiming a worker was orchestrated, verify the task/dispatch exists:\n\n```bash\norca orchestration task-list --json\norca orchestration dispatch-show --task --json\n```\n\nIf the work was accidentally run outside Orca orchestration, say so plainly. To repair provenance, rerun or revalidate the needed work through a fresh Orca terminal plus injected dispatch; do not retroactively describe the external worker as orchestrated.\n\n## When To Use\n\n- Send/reply/ask between agent terminals with persistent messages.\n- Dispatch structured tasks to workers and wait for `worker_done` or `escalation`.\n- Track task DAGs with dependencies.\n- Run coordinator loops or decision gates.\n\nDo not use orchestration merely because the user says \"hand off\", \"handoff\", \"handover\", \"give this to another agent\", or asks for another worktree/agent/model/effort. Those are full ownership transfers unless the user explicitly asks to supervise, monitor, wait for worker completion/results, coordinate a DAG, use decision gates, or keep a blocking ask/reply loop.\n\n## Preconditions\n\n- `orca status --json` should show a running runtime.\n- `orca` must be on PATH (`orca-ide` on Linux).\n- The orchestration experimental feature must be enabled in Settings > Experimental.\n- `orca orchestration` commands are RPC calls to the running Orca runtime.\n\n## Contract Migration\n\nOrca adopts a live pre-update orchestration assignment into an ordinary Run. Adoption preserves the existing agent process, PTY/session, terminal handle, tab/leaf/pane, worktree or folder workspace, Task, and Dispatch; it never restarts or replaces the worker. The retired scheduler is not revived, and a newly created attempt uses the current grammar.\n\nTreat the authority label on injected or formatted messages as definitive:\n\n- `[LEGACY COMPATIBILITY]` is live and attested. Run only the exact supported command printed with the message, using the same CLI executable and arguments that the original prompt supplied.\n- `[LEGACY RECOVERY REPLAY — MAY HAVE BEEN SEEN]` is one bounded, at-least-once cutover replay. Process it idempotently and acknowledge it only through the exact displayed guidance.\n- `[LEGACY READ-ONLY]` is inspection-only. It has no reply, acknowledgment, or lifecycle action.\n- An unlabeled current message uses the current guide and current grammar.\n\nAn explicitly selected current Run, attested current Run binding, current Dispatch, or federated attachment takes precedence over legacy fallback. A retained adoption record alone never turns a current command into a legacy call.\n\nDatabase provenance, an old-looking terminal, or a legacy Run ID does not prove mutation authority. If the runtime cannot prove liveness, principal ownership, capability, or the exact legacy contract, it degrades to read-only inspection and must not fall back to local execution. Exact recovery may restore the already-live PTY once in its original inactive background tab. It must not spawn, write, signal, stop, switch, focus, split, or inject a terminal. Loss of lifecycle authority does not invalidate the existing assignment, process, or filesystem work.\n\nCompatibility retries have narrow guarantees. A pending ask, a reply, a final Dispatch settlement, and a consuming check have durable recovery identities. A-era heartbeat and escalation calls remain at-least-once across a manual A-to-B retry because identical later signals may be intentional. If an A-era ask may already have been answered, run the exact non-consuming recovery check printed by the runtime first; after its answer is printed and acknowledged, a new invocation with the same question creates a new question. Never guess among multiple identical question threads.\n\nWhen a compatibility or recovery command returns structured next-step arguments, run those exact arguments with the same CLI executable. The arguments intentionally omit the executable name so the guidance works with `orca`, `orca-ide`, `orca-dev`, or another configured Orca CLI command. Do not translate the command from memory, broaden its recipient, or retry it as a current mutation unless the returned guidance explicitly says to.\n\nOn packaged Windows, a legacy ask uses a two-step commit/resume protocol. The initial command durably commits the question, prints its exact `ask --resume ` command, and exits with launcher status `75`; it does not wait for the answer. Run that exact resume command after the launcher or update boundary. Resume is idempotent and read-oriented: it waits for the already-committed question and does not create another one. For a WSL process that received compatibility proof at launch, use the printed executable `orca-ide` WSL resume command so the same distro and packaged launcher authority are preserved; do not substitute a PATH-resolved local CLI. Older WSL processes that never received the hidden launch token remain lifecycle read-only after the update, even while their terminal and filesystem work continue.\n\nLegacy inspection remains available without consuming mail:\n\n```bash\norca orchestration run-list --json\n# run_legacy_local is an empty audit tombstone after adoption.\norca orchestration run-show --id run_legacy_local --json\n# In run-list, find the ordinary Run whose objective is:\n# \"Recovered orchestration work from a contract update\"\norca orchestration run-show --id --json\norca orchestration task-list --run --json\norca orchestration inbox --full --json\norca orchestration check --terminal --peek --format --json\norca terminal read --terminal --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\n```\n\nIf the original coordinator is unavailable or cannot prove its retained authority, a current coordinator may explicitly take over the adopted Run from its own live agent terminal:\n\n```bash\norca orchestration run-use --id --takeover-legacy --json\norca orchestration check --run --json\n```\n\nTakeover fences only the old coordinator, binds the current one, and moves pending worker mail into current Run Delivery. It is bound to the authenticated invoking terminal; `--from` cannot name another coordinator. Live legacy workers keep their original Tasks, Dispatches, processes, filesystems, and old prompt commands; their later questions, escalations, and completion reports route to the current coordinator. Do not use takeover while the original coordinator is still actively coordinating, because its later lifecycle mutations are rejected.\n\nDo not launch a replacement editor merely because the desktop app or runtime was updated. If adoption cannot prove continuing authority, keep the original worker as the only editor until it reaches a stable handoff point, then use a new current Dispatch in a conflict-free placement for any remaining work.\n\n## Ownership\n\nNew orchestration messages and tasks belong to one explicitly bound Run. A Run is only a durable namespace and coordinator inbox; it never schedules or places workers. Lifecycle authority comes from the active Dispatch, and terminal handles remain routing metadata rather than durable identity. Send `worker_done` and `heartbeat` from the worker's own terminal; Orca routes them to that Dispatch's Run.\n\nClassify inherited context before sending lifecycle messages:\n\n- Coordinated subtask: a live coordinator owns the DAG and waits on this dispatch. Follow the preamble exactly, including `worker_done`, heartbeat/status, `ask`, and `escalation`.\n- Full handoff means ownership transfer, not supervised dispatch. The original actor is not monitoring a DAG, so do not create lifecycle obligations unless the user explicitly asks you to supervise.\n- Classify requests containing \"hand off\", \"handoff\", \"handover\", \"give this to another agent\", \"give this to another worktree\", \"another agent\", or \"another worktree\" as full handoffs by default, even when the user names a custom model or reasoning effort.\n- Use supervised orchestration only when the user explicitly asks you to \"supervise\", \"monitor\", \"wait\", \"track completion\", \"wait for worker_done\", return results, coordinate a DAG, use a decision gate, or manage ask/reply flow.\n- Do not use `orca orchestration dispatch --inject` for full handoffs. It injects a coordinator preamble that tells the worker to send `worker_done`, heartbeat, and `ask` messages, then end its turn under the original terminal's dispatch lifecycle.\n- Do not run `orca orchestration task-create`, `orca orchestration dispatch --inject`, or `orca orchestration check --wait` for full handoffs. Do not peek at terminal output after prompt delivery to monitor progress.\n- A review-only `worker_done` reports findings; it does not authorize coordinator file edits. After a review-only completion, synthesize findings, ask a decision gate if ownership is unclear, and dispatch or hand off fixes unless the user explicitly asked the coordinator to own fixes.\n- If the user's plan names a next owner agent (for example, \"then use opencode to create a PR\"), post-review corrections and PR prep belong to that named owner. The coordinator routes, synthesizes, asks decision gates when needed, and supervises; the named owner edits files and creates the PR.\n\nIf unclear, inspect orchestration state before sending lifecycle messages:\n\n```bash\norca orchestration task-list --json\norca terminal list --json\n# If inherited context includes a task id:\norca orchestration dispatch-show --task --json\n```\n\n## Messaging\n\n```bash\norca orchestration send --subject [--to ] [--from ] [--body ] [--type ] [--priority ] [--thread-id ] [--payload ] [--json]\norca orchestration check [--terminal ] [--ack ] [--peek|--all] [--types ] [--format] [--wait] [--timeout-ms ] [--json]\norca orchestration reply --id --body [--from ] [--json]\norca orchestration ask (--question |--resume ) [--options ] [--timeout-ms ] [--from ] [--json]\norca orchestration inbox [--limit ] [--json]\n```\n\nRules:\n\n- Omit `--from` unless impersonating another terminal; Orca auto-resolves it from the current terminal.\n- A coordinator `check` returns the bound Run's oldest FIFO Delivery (up to 50 messages) and replays that exact batch until `--ack `. Process every message before acknowledging; `check --ack --wait` acknowledges, checks, and waits in one operation.\n- Use `--peek` and `--all` only for read-only history/debugging. Type filters decide when a waiter wakes; the returned actionable Delivery is still the oldest full batch.\n- Use `dispatch:` for coordinator guidance to one supervised worker. Orca routes that stable address locally or through the connected-server relay; do not substitute a remote terminal handle.\n- Terminal handles remain appropriate for low-level pre-Dispatch messaging. Prefer `agentTerminalHandle` from the create response, fall back to `startupTerminal.handle` for older runtimes, then re-resolve with `orca terminal list --worktree ... --json` if missing or stale. Continue with the replacement handle only; never dual-send to old and new handles.\n- `terminal list --json` omits `visualLayouts` because handle recovery does not need topology. Add `--include-visual-layouts` only for explicit tab and pane inspection.\n- `orca orchestration check --peek --format --json` returns locally formatted unread mail without consuming it; it never writes to terminal input or remotely wakes another terminal. Use `orchestration dispatch --inject` to deliver a tracked task, or `terminal send` when an existing agent needs a free-form prompt.\n- While supervising workers manually, use `check --wait --types worker_done,escalation,question --timeout-ms ` instead of sleep/poll loops. Process the whole Delivery, reply to `question` messages with `orca orchestration reply --id --body --json`, then acknowledge and keep waiting.\n- Treat a `check --wait` timeout or `{count:0}` as a checkpoint, not a worker failure. Long coding tasks routinely run 15-60 minutes; keep using rolling waits unless you receive `worker_done`/`escalation`, the terminal exits or disappears, or the user explicitly asks you to stop.\n- Heartbeats and visible terminal activity mean the worker is alive, not done. Do not stop, close, kill, or restart a worker just because it has not produced a completion message yet.\n- Use `ask` when a worker needs a blocking answer from the coordinator; it defaults to the active Dispatch's Run. Timeout or disconnect leaves the question pending, so resume by its original message ID instead of asking again.\n- `check --wait` returns one bounded Delivery, not every future completion. Process every message, acknowledge it, then keep waiting until every expected Dispatch settles.\n- Group addresses include `@all`, `@idle`, `@claude`, `@codex`, `@opencode`, `@gemini`, `@droid`, `@grok`, `@cursor`, and `@worktree:`.\n- Message types include `status`, `dispatch`, `worker_done`, `merge_ready`, `escalation`, `handoff`, `question`, `decision_gate` (legacy/gates), and `heartbeat`.\n- Use group addresses only for messages that are genuinely useful to many terminals, such as `status` broadcasts or intentional fan-out questions. Do not send dispatch lifecycle messages to groups.\n- `worker_done` belongs to the active Dispatch and defaults to its Run mailbox; never target a group.\n- A valid `worker_done` for the active `taskId` + `dispatchId` marks the task and dispatch completed automatically. Do not follow it with `task-update --status completed`; reserve manual updates for explicit recovery or overrides.\n- `heartbeat` is also Dispatch-scoped. Include both IDs and omit `--to` so Orca uses the owning Run; use `status` for broad progress updates.\n\n## Tasks And Dispatch\n\nA Run is the namespace/inbox, a Task is the work item, and a Dispatch assigns one Task attempt to a terminal. Create or bind a Run once before the common loop.\n\n```bash\norca orchestration run-create --objective --json\norca orchestration task-create --spec [--deps ] [--parent ] [--json]\norca orchestration task-list [--status ] [--ready] [--brief] [--json]\norca orchestration task-update --id --status [--result ] [--json]\norca orchestration dispatch --task --to [--from ] [--inject] [--json]\norca orchestration dispatch-show --task [--json]\n```\n\nTask statuses: `pending`, `ready`, `dispatched`, `completed`, `failed`, `blocked`.\n\nDispatch rules:\n\n- `--inject` sends the task spec plus preamble into a recognized agent CLI so it can report `worker_done`.\n- If the target is a bare shell, omit `--inject`, dispatch for tracking if needed, then send the prompt manually with `orca terminal send --terminal --text --enter --json`.\n- After 3 consecutive failures on one task, the dispatch context circuit-breaks and the task is marked failed.\n- Use `task-list --brief --json` for coordinator sweeps; it collapses whitespace and caps each echoed spec at 160 characters (`spec_truncated` marks shortened rows). Omit `--brief` when the full spec is required, or when an older CLI rejects it as an unknown flag.\n\n## How deep workers can nest\n\nA dispatched worker normally cannot dispatch sub-workers. Attempting it fails with\n`nested_worker_depth_exceeded` and a message telling the worker to complete the task\nitself. Do that — do not try to route around it.\n\nThe limit is a number, not an on/off switch. `Settings -> Orchestration -> Nested worker depth`\nsets how many generations are allowed:\n\n- `1` (default): a coordinator dispatches workers; those workers do not dispatch.\n- `2`: workers may dispatch one further generation.\n\nDepth is counted from the terminal that issues the command, not from the Run. Creating a\nnew Run does not reset it — a worker that runs `run-create` then `worker-start` is still a\nworker, and still counted. This is the part that changed: the old behaviour rejected\nsub-dispatch only because a worker's terminal was not bound to a Run, so creating a Run was\nenough to slip past it.\n\nTwo limits worth knowing:\n\n- **It is a guardrail, not a security boundary.** A caller that declares another terminal's\n handle while its own launch evidence is unverifiable (an ordinary restored terminal, for\n example) can be counted as that terminal instead. Orca does not treat workers as hostile.\n- **It applies while a Dispatch is active.** After `worker_done`, or after a coordinator\n settles the task, the terminal is no longer a worker and is counted as a root again. The\n process may still be alive; that is the documented boundary, not an accident.\n\n## Preferred Supervised Worker Loop\n\nUse `worker-start` for the normal supervised path. It composes the existing worktree, terminal, readiness, and dispatch primitives while returning exact created/reused effects. Agents still choose placement and concurrency; Orca does not schedule workers or infer conflicts.\n\nCreate the Run and every independent Task first, then start all independent workers before waiting:\n\n```bash\norca orchestration run-create --objective \"\" --json\norca orchestration task-create --spec \"\" --json\norca orchestration task-create --spec \"\" --json\norca orchestration worker-start --task --worktree current --agent codex --json\norca orchestration worker-start --task --worktree current --agent claude --json\n```\n\n`current` and exact existing worktrees create a fresh agent terminal and do not rerun setup. Reuse an existing agent only with `--terminal `.\n\nFor a per-invocation Claude, Codex, or Cursor launch, pass an opaque provider model id with `--model`; add `--effort` only when that agent/model supports the level. These options apply only to fresh agent terminals, override general agent default arguments, and are reported under `launch.requested` and `launch.effective` in the receipt:\n\n```bash\norca orchestration worker-start --task --worktree current --agent claude --model opus --effort high --json\n```\n\n`--effort` requires `--model`, and neither option can combine with `--terminal`. A connected worker server must advertise launch-preference support before Orca forwards either option.\n\nFor a new worktree, setup runs by default and agent-first creation reuses the returned startup agent terminal:\n\n```bash\norca orchestration worker-start --task --worktree new-child --name --agent codex --setup run --json\n# Independent/top-level:\norca orchestration worker-start --task --worktree new-top-level --name --agent codex --setup run --json\n```\n\nSetup normally starts alongside the agent. Only a repository explicitly configured with `wait-for-setup` delays agent launch until setup succeeds. Use `--setup skip` or `--setup inherit` only for a concrete reason.\n\nRead the returned receipt before continuing: `ready` plus setup `running` is normal for start-immediately, while wait-for-setup returns setup `succeeded` before accepting task input. A failed or unknown start exits nonzero; inspect its `stage`, `effects`, and `residualResources` instead of guessing or automatically retrying. A wait-for-setup timeout can honestly leave setup `running`, which is not proof of failure.\n\nTo run the worker on another connected Orca server, add `--on `. The Run and Tasks remain authoritative on the current server; later commands route by Dispatch ID, so never repeat `--on`:\n\n```bash\n# Mac Run home -> Windows worker (the reverse is identical from a Windows Run home)\norca orchestration worker-start --task --on windows --worktree new-top-level --repo --name --agent codex --setup run --json\norca orchestration worker-show --dispatch --json\norca orchestration worker-read --dispatch --limit 50 --json\norca orchestration send --to dispatch: --subject \"Follow-up\" --body \"\" --json\n```\n\nRemote `current` and `new-child` are intentionally invalid because those words are ambiguous across servers. Use an exact discovered remote worktree selector or `new-top-level` with an explicit remote repo selector.\n\nThe follow-up is structured inbox mail, not prompt injection. The worker's next\n`orchestration check` receives it even when the Dispatch is on another connected Orca server.\n\n`worker-read` defaults to `--source auto`: Orca returns the exact hook-reported Codex, Claude, OpenClaude, or Grok transcript when it can prove the worker session, otherwise it returns bounded terminal output with `source: \"terminal\"` and a typed `fallbackReason`. Continue with the returned top-level `cursor`; it stays pinned to that exact source. If Orca reports `source_changed`, start a fresh read without the old cursor. Never supply or guess a provider session ID or transcript path.\n\nWait until every expected Dispatch settles, not for a fixed number of batches:\n\n```bash\norca orchestration check --wait --types worker_done,escalation,question --timeout-ms 900000 --json\n# Process every message. For each accepted worker_done that is not immediately reused:\norca orchestration worker-release --dispatch --json\n# Acknowledge only after every message and required release decision is handled:\norca orchestration check --ack --wait --types worker_done,escalation,question --timeout-ms 900000 --json\n```\n\nAfter processing each accepted `worker_done`, choose the terminal's next owner before you acknowledge the Delivery or wait again. If the same exact agent has an immediate follow-up Task, read the `worker.agent_terminal_handle` field of `worker-show --dispatch --json`, then run `orca orchestration worker-start --task --terminal --json` so Orca transfers cleanup ownership to the new Dispatch. Otherwise run `orca orchestration worker-release --dispatch --json`.\n\nRun `worker-release` after both succeeded and failed `worker_done` reports unless the user explicitly asked to keep that worker live. Release is post-completion cleanup, not cancellation: Orca first preserves inspectable output, then closes only the exact agent terminal owned by that settled Dispatch. Reused or pre-existing terminals, setup terminals, coordinators, active workers, user-taken-over terminals, and identities Orca cannot prove are retained. If the user explicitly asks to keep the live terminal for debugging, record that exception with `orca orchestration worker-retain --dispatch --json` instead of silently skipping cleanup. When the user is finished, the same Dispatch can be passed to `worker-release`, which clears the requested retention and releases the terminal.\n\nDo not release a worker because of a timeout, TUI idle state, heartbeat, status, question, escalation, or rejected/stale `worker_done`. If release returns `release_pending` or `release_unknown`, do not substitute `terminal close`; follow the exact recovery action in the receipt. A replayed Delivery may repeat `worker-release` safely.\n\nWorkers report exactly once using the IDs and capability injected by Orca; they do not supply Run/server/terminal identity:\n\n```bash\norca orchestration send --type worker_done --subject \"\" --body \"\" --task-id --dispatch-id --outcome succeeded --files-modified \"path/a,path/b\" --json\n# On failure, use --outcome failed; never encode failure only in prose.\n```\n\nA worker question defaults to its owning Run. Timeout leaves it pending:\n\n```bash\norca orchestration ask --question \"\" --options \"yes,no\" --timeout-ms 600000 --json\norca orchestration ask --resume --timeout-ms 600000 --json\n# Coordinator:\norca orchestration reply --id --body \"\" --json\n```\n\nRecovery is conditional, never a fixed destructive sequence:\n\n- `worker-show --dispatch ` says `ready`: keep waiting or read bounded output.\n- It proves `failed` or `stopped`: start a replacement with `worker-start --task --retry-of ` plus an explicit `--on`/`--worktree` and `--agent`/`--terminal` choice. Retry does not silently inherit placement.\n- It remains `outcome_unknown`: either `worker-stop --dispatch ` and inspect again, or explicitly `worker-abandon --dispatch ` while accepting that resources may still be live. Abandon performs no remote, process, or filesystem action.\n- `worker-stop` closes only the exact supervised agent terminal. It never deletes the worktree, setup terminal, configured tabs, or unrelated processes.\n\nLow-level `worktree create`, `terminal create`, and `dispatch --inject` remain valid recipes for custom argv or topology that `worker-start` does not express.\n\n`dispatch --inject` deliberately keeps an operator-started terminal unsupervised: it never creates a `worker_dispatches` row and `worker-stop`/`worker-abandon` never close that process. The dispatch context is still authoritative, so `worker-show`, `worker-read`, and `worker-list` report it as `unsupervised`; settled `worker-retain` and `worker-release` report `retained` with `no_owned_resource` and take no process action. Use `worker-start --terminal ` when supervision and worker lifecycle state are required.\n\n## Gates And Legacy Inspection\n\n```bash\norca orchestration gate-create --task --question [--options ] [--json]\norca orchestration gate-resolve --id --resolution [--json]\norca orchestration gate-list [--task ] [--status ] [--json]\n```\n\nUse `ask` for worker-to-coordinator questions; it creates a `question` message that the coordinator answers with `reply`. Use `gate-create` only for coordinator-managed task DAG decisions, not for answering a worker's `ask`.\n\n`coordinator-start`, `coordinator-stop`, `run`, and `run-stop` are retired scheduler commands. They perform no effects and return the current-skill recovery action. They are not aliases for lightweight Run creation or binding.\n\nRecovery only: `orca orchestration reset --tasks|--messages|--all --json` clears the selected local orchestration database state. Do not run it during active coordination unless explicitly abandoning that state.\n\n## Full Handoffs\n\nFor full ownership transfer, use non-lifecycle terminal/worktree commands and then stop monitoring unless the user asks for supervision.\n\nTreat these as full handoff requests by default: \"hand off\", \"handoff\", \"handover\", \"give this to another agent\", \"give this to another worktree\", \"send this to another agent\", \"another agent\", \"another worktree\", or \"launch another agent to own this.\" Custom model or reasoning effort words such as `gpt-5.5`, `high`, or `xhigh` do not make the handoff supervised.\n\nSupervised orchestration remains available only when the user explicitly asks for supervision or coordination: \"supervise\", \"monitor\", \"wait for worker_done\", \"wait for results\", \"track completion\", \"DAG\", \"decision gate\", \"ask/reply\", or \"coordinate workers.\"\n\nDo not run `orca orchestration task-create`, `orca orchestration dispatch --inject`, or `orca orchestration check --wait` for full handoffs. `task-create` is also forbidden because it records coordinator-owned tracking state; if a task row is needed, the user asked for supervised orchestration. Do not create a `taskId`/`dispatchId`, inject a lifecycle preamble, wait for completion, or read the worker terminal after prompt delivery except to avoid losing the initial prompt.\n\nNew top-level worktree handoff:\n\n```bash\norca worktree create --name --no-parent --agent codex --prompt \"\" --setup run --json\n```\n\nBefore creating a new worktree from an active feature branch, decide and state whether the desired Orca lineage is child or top-level. Use child worktree lineage only when the new work is conceptually stacked under or dependent on the active worktree. For independent repo-wide fixes, standalone feature work, or unrelated follow-up tasks, create a top-level worktree with `--no-parent`.\n\nExisting terminal handoff:\n\n```bash\norca terminal send --terminal --text \"\" --enter --json\n```\n\nCustom Codex model/effort handoff:\n\n`orca worktree create --agent codex --prompt ...` launches the known Codex agent but does not accept Codex-specific `--model` or `-c model_reasoning_effort=...` arguments. When the user asks for a specific Codex model or effort, create the independent worktree first, launch Codex with the requested command in that worktree, wait only for TUI readiness if prompt delivery would otherwise race startup, send the prompt, and stop.\n\nThe two-step custom-argv path cannot enforce a repository's explicit `wait-for-setup` startup policy because the later `terminal create` is not the startup owned by `worktree create`. Use it only when the repository starts agents immediately. If the repository requires `wait-for-setup`, use an agent-first configured launcher that can preserve sequencing, or stop and ask rather than silently bypassing the policy.\n\nNote: when no repo default-terminal configuration supplies a primary terminal, bare create opens a fallback shell before `terminal create` adds the agent. Configured default tabs are materialized instead and may run real commands. Prefer `--agent` whenever custom argv is not required. With the two-step path, target only the agent handle; close a prior terminal only after `terminal list` or `terminal show` confirms it is an unused shell.\n\nUse the exact full `::` worktree id returned by `orca worktree create --json`; a bare repo id cannot target the new worktree.\n\n```bash\norca worktree create --name --no-parent --setup run --json\norca terminal create --worktree id: --title --command 'codex --model gpt-5.5 -c model_reasoning_effort=\"xhigh\"' --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca terminal send --terminal --text \"\" --enter --json\n```\n\nWait only for `tui-idle` when needed to avoid losing the prompt. Do not monitor task completion.\n\n`--no-parent` only controls Orca lineage; it does not choose the Git base. If the work should start from the repo default base, omit `--base-branch` so Orca uses that default, or explicitly pass the repo default base (`origin/main`, `origin/master`, or the `orca repo show --repo --json` value); never base it on the current feature branch unless the user explicitly asks for stacked work or \"branch from current\". Put current-branch context in the prompt instead.\n\n## Worker Terminals\n\nChoose the worker location before creating a terminal. `Fresh worker` means a fresh agent session, not a new git worktree. For parallel work, create one fresh agent terminal per worker in the same required worktree, falling back to the active worktree when none is named. If the task says current worktree only, depends on uncommitted files/artifacts, or must validate/PR the current branch, keep every worker in the active worktree:\n\n```bash\norca terminal create --worktree active --title --command \"codex\" --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca orchestration dispatch --task --to --inject --json\n```\n\nReuse an idle agent in the required worktree only if the prompt allows reuse; otherwise create a fresh terminal there. Create a new worktree only when the user explicitly requests one or a concrete checkout or filesystem conflict makes sharing unsafe or impossible; if the user did not request it, state that conflict before running `worktree create`. Independent tasks, parallel execution, convenience, or a preference for separate checkouts are not isolation requirements.\n\nWhen a new worktree is allowed, use child lineage for isolated work that is stacked under or dependent on the active worktree, and use `--no-parent` when it is not stacked. Decide the Git base separately: `--no-parent` makes the worktree top-level in Orca, while omitted `--base-branch` uses the repo default base.\n\nFor every new worktree, pass `--setup run` so any configured repository setup hook runs. This does not mean waiting for setup before agent launch: preserve the repository's startup policy, whose default starts setup and the agent side by side. Use `--setup skip` or `--setup inherit` only when there is a concrete task-specific reason, and state that reason before creating the worktree. This rule does not rerun setup for current or existing worktrees.\n\n```bash\norca worktree create --name --agent codex --setup run --json\n# or: --agent claude | omp | pi | grok | ...\n# Read from agentTerminalHandle, falling back to startupTerminal.handle.\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca orchestration dispatch --task --to --inject --json\n```\n\nFor new-worktree workers, read the id and `agentTerminalHandle` from `worktree create`, falling back to `startupTerminal.handle` for older runtimes. Use that as the sole worker handle when present; otherwise use `terminal list` to resolve the agent handle. Omit `--repo` only inside an Orca-managed worktree; otherwise pass `--repo `.\n\n**For an allowed new worktree, use agent-first:** `--agent` reveals the new worktree and launches the selected agent **in its first terminal**, without adding a separate fallback shell for that worker. Pass `--setup run`; repo setup and default-terminal settings may add intentional tabs or splits. Do **not** run bare `worktree create` and then `terminal create --command ` for the same worker when agent-first create is available: without configured default tabs, that two-step path leaves a fallback shell + agent pair. Only use it when custom agent argv is required (for example Codex model/effort flags) or when an older CLI rejects `--agent`; if you must, message only the agent handle. Configured default tabs are intentional surfaces, so close a prior terminal only after `terminal list` or `terminal show` confirms it is an unused shell. Do not run `worktree create` when the task must stay in the current worktree.\n\nUse `orca worktree create --prompt ...` or `orca terminal send ...` for full handoffs or untracked/lightweight prompts. Those paths do not attach `taskId`/`dispatchId`; the worker should not send lifecycle messages unless the prompt supplies a live orchestration preamble.\n\nSidebar lineage and orchestration lifecycle are related but not identical. A same-worktree worker may appear as a peer under that worktree in the sidebar while remaining a child dispatch in orchestration state; only an actual child worktree creates visible parent/child worktree lineage.\n\nOther terminal commands coordinators often need:\n\n```bash\norca terminal list [--worktree ] [--include-visual-layouts] [--json]\norca terminal create [--worktree ] [--title ] [--command ] [--json]\norca terminal split --terminal [--direction horizontal|vertical] [--command ] [--json]\norca terminal wait --terminal --for tui-idle --timeout-ms --json\norca terminal read --terminal --json\norca terminal send --terminal --text --enter --json\n```\n\nIf an older CLI rejects `worktree create --agent`, create the worktree normally, then run `orca terminal create --worktree --command \"codex\" --json` or `--command \"claude\"`.\n\nWait for `tui-idle` before dispatching. Always pass `--timeout-ms`; real coding tasks can take 15-60 minutes. During supervision, use rolling `check --wait` windows. If a window returns no matching message, inspect `task-list`, `terminal read`, or `terminal wait --for tui-idle` as a liveness checkpoint; if the terminal is still working or producing activity, keep waiting instead of retrying the task.\n\n## Agent Guidance\n\n- Workers with a valid live preamble must send `worker_done` exactly once from their own terminal with an explicit `--outcome succeeded` or `--outcome failed`:\n `orca orchestration send --type worker_done --subject \"\" --body \"<3-sentence summary: what you did, what you found, what's left>\" --task-id --dispatch-id --outcome succeeded --files-modified \"path/a\" --report-path \"\" --json`\n- A failed outcome is still a terminal report, but Orca records both the Dispatch and Task as failed. Never encode failure only in the subject/body.\n- After sending `worker_done`, end that dispatched turn and idle at the agent prompt. Do not autonomously start more work, poll, or attempt to close the terminal yourself. A direct user instruction takes precedence and starts ordinary user-owned work: follow it without coordinator approval or a fresh Dispatch, never refuse it because of worker/coordinator roles, and do not reuse the settled Dispatch's lifecycle IDs. A coordinator-supervised follow-up still arrives with a fresh preamble + TASK block.\n- For long tasks, send heartbeat/status only when the preamble asks for it, including both IDs:\n `orca orchestration send --type heartbeat --subject \"alive\" --payload '{\"taskId\":\"\",\"dispatchId\":\"\",\"phase\":\"implementing\"}' --json`\n- If blocked before completion, use `ask`; use `escalation` only when ownership is valid and the coordinator must intervene.\n- Treat preambles inherited through terminal history or full handoffs as stale unless the current prompt explicitly keeps that coordinator in the loop.\n- Coordinators must account for every settled worker terminal before waiting again or ending the turn: immediately reuse the exact worker for a new Dispatch, explicitly retain it at the user's request with `worker-retain`, or run `worker-release`. Do not leave a completed worker live merely to inspect output; released workers remain readable through `worker-read`.\n- Coordinators should use `task-list --ready` as external memory, dispatch parallel waves, and avoid dependency chains deeper than 3-4 steps.\n\n## Example\n\n```bash\norca terminal create --worktree active --title login-css-worker --command \"claude\" --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca orchestration task-create --spec \"Fix the login button CSS\" --json\norca orchestration dispatch --task --to --inject --json\norca orchestration check --wait --types worker_done,escalation,question --timeout-ms 900000 --json\n```\n\n## Next Action\n\nCoordinator: confirm `orca status --json`, create or bind a Run, inspect `task-list`/`dispatch-show` if inheriting state, then use the explicit supervised loop (`task-create` -> `worker-start` -> `check --wait`). Use low-level terminal creation plus `dispatch --inject` only when the composed start does not express the needed topology. After every accepted `worker_done`, either transfer the exact terminal to an immediate follow-up Dispatch or run `worker-release` before the next wait.\n\nWorker: if the current prompt contains a live dispatch preamble, do the task, use `ask` for blocking questions, and send `worker_done` once with the required payload. If the preamble is stale or absent, do not send lifecycle messages; inspect state or treat the prompt as an ordinary handoff.\n" +const ORCHESTRATION_MARKDOWN = "---\nname: orchestration\ndescription: >-\n Use Orca orchestration for structured multi-agent coordination: threaded\n messages, blocking ask/reply flows, task dispatch, worker_done/escalation\n waits, task DAGs, decision gates, coordinator loops, or decomposing work\n across agents. Use `orca-cli` instead for full ownership handoffs, including\n requests phrased as \"hand off\", \"handoff\", \"handover\", \"give this to another\n agent\", or \"another worktree\" when the user did not explicitly ask to\n supervise, monitor, wait for results, or coordinate a DAG. Use `orca-cli` for\n ordinary terminal control, lightweight terminal prompts, shell commands, Orca\n worktree management, reading or waiting on terminals, and automation of the\n browser embedded inside Orca. Use Computer Use for browser windows, webviews,\n Orca app UI, or desktop UI outside Orca's embedded browser.\n---\n\n# Orca Inter-Agent Orchestration\n\nOrchestration is Orca's structured coordination layer for agent messages, task ownership, dispatch state, and worker completion tracking.\n\nUse this skill when coordination state matters. For lightweight terminal prompts or basic worktree/terminal/built-in-browser control, use `orca-cli`.\n\n## Tool Boundary\n\nIf a task says to use Orca orchestration, the coordinator must create or bind a Run, create the Task with `orca orchestration task-create`, then attach the worker with either the preferred `orca orchestration worker-start` composition or the low-level `orca orchestration dispatch --inject` path.\n\nDo not substitute non-Orca subagent tools, generic agent-spawn APIs, or chat-only parallel worker features. Those may create useful workers, but they do not create Orca task/dispatch provenance, injected lifecycle preambles, `worker_done` authority, or decision gates.\n\nBefore claiming a worker was orchestrated, verify the task/dispatch exists:\n\n```bash\norca orchestration task-list --json\norca orchestration dispatch-show --task --json\n```\n\nIf the work was accidentally run outside Orca orchestration, say so plainly. To repair provenance, rerun or revalidate the needed work through a fresh Orca terminal plus injected dispatch; do not retroactively describe the external worker as orchestrated.\n\n## When To Use\n\n- Send/reply/ask between agent terminals with persistent messages.\n- Dispatch structured tasks to workers and wait for `worker_done` or `escalation`.\n- Track task DAGs with dependencies.\n- Run coordinator loops or decision gates.\n\nDo not use orchestration merely because the user says \"hand off\", \"handoff\", \"handover\", \"give this to another agent\", or asks for another worktree/agent/model/effort. Those are full ownership transfers unless the user explicitly asks to supervise, monitor, wait for worker completion/results, coordinate a DAG, use decision gates, or keep a blocking ask/reply loop.\n\n## Preconditions\n\n- `orca status --json` should show a running runtime.\n- `orca` must be on PATH (`orca-ide` on Linux).\n- The orchestration experimental feature must be enabled in Settings > Experimental.\n- `orca orchestration` commands are RPC calls to the running Orca runtime.\n\n## Contract Migration\n\nOrca adopts a live pre-update orchestration assignment into an ordinary Run. Adoption preserves the existing agent process, PTY/session, terminal handle, tab/leaf/pane, worktree or folder workspace, Task, and Dispatch; it never restarts or replaces the worker. The retired scheduler is not revived, and a newly created attempt uses the current grammar.\n\nTreat the authority label on injected or formatted messages as definitive:\n\n- `[LEGACY COMPATIBILITY]` is live and attested. Run only the exact supported command printed with the message, using the same CLI executable and arguments that the original prompt supplied.\n- `[LEGACY RECOVERY REPLAY — MAY HAVE BEEN SEEN]` is one bounded, at-least-once cutover replay. Process it idempotently and acknowledge it only through the exact displayed guidance.\n- `[LEGACY READ-ONLY]` is inspection-only. It has no reply, acknowledgment, or lifecycle action.\n- An unlabeled current message uses the current guide and current grammar.\n\nAn explicitly selected current Run, attested current Run binding, current Dispatch, or federated attachment takes precedence over legacy fallback. A retained adoption record alone never turns a current command into a legacy call.\n\nDatabase provenance, an old-looking terminal, or a legacy Run ID does not prove mutation authority. If the runtime cannot prove liveness, principal ownership, capability, or the exact legacy contract, it degrades to read-only inspection and must not fall back to local execution. Exact recovery may restore the already-live PTY once in its original inactive background tab. It must not spawn, write, signal, stop, switch, focus, split, or inject a terminal. Loss of lifecycle authority does not invalidate the existing assignment, process, or filesystem work.\n\nCompatibility retries have narrow guarantees. A pending ask, a reply, a final Dispatch settlement, and a consuming check have durable recovery identities. A-era heartbeat and escalation calls remain at-least-once across a manual A-to-B retry because identical later signals may be intentional. If an A-era ask may already have been answered, run the exact non-consuming recovery check printed by the runtime first; after its answer is printed and acknowledged, a new invocation with the same question creates a new question. Never guess among multiple identical question threads.\n\nWhen a compatibility or recovery command returns structured next-step arguments, run those exact arguments with the same CLI executable. The arguments intentionally omit the executable name so the guidance works with `orca`, `orca-ide`, `orca-dev`, or another configured Orca CLI command. Do not translate the command from memory, broaden its recipient, or retry it as a current mutation unless the returned guidance explicitly says to.\n\nOn packaged Windows, a legacy ask uses a two-step commit/resume protocol. The initial command durably commits the question, prints its exact `ask --resume ` command, and exits with launcher status `75`; it does not wait for the answer. Run that exact resume command after the launcher or update boundary. Resume is idempotent and read-oriented: it waits for the already-committed question and does not create another one. For a WSL process that received compatibility proof at launch, use the printed executable `orca-ide` WSL resume command so the same distro and packaged launcher authority are preserved; do not substitute a PATH-resolved local CLI. Older WSL processes that never received the hidden launch token remain lifecycle read-only after the update, even while their terminal and filesystem work continue.\n\nLegacy inspection remains available without consuming mail:\n\n```bash\norca orchestration run-list --json\n# run_legacy_local is an empty audit tombstone after adoption.\norca orchestration run-show --id run_legacy_local --json\n# In run-list, find the ordinary Run whose objective is:\n# \"Recovered orchestration work from a contract update\"\norca orchestration run-show --id --json\norca orchestration task-list --run --json\norca orchestration inbox --full --json\norca orchestration check --terminal --peek --format --json\norca terminal read --terminal --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\n```\n\nIf the original coordinator is unavailable or cannot prove its retained authority, a current coordinator may explicitly take over the adopted Run from its own live agent terminal:\n\n```bash\norca orchestration run-use --id --takeover-legacy --json\norca orchestration check --run --json\n```\n\nTakeover fences only the old coordinator, binds the current one, and moves pending worker mail into current Run Delivery. It is bound to the authenticated invoking terminal; `--from` cannot name another coordinator. Live legacy workers keep their original Tasks, Dispatches, processes, filesystems, and old prompt commands; their later questions, escalations, and completion reports route to the current coordinator. Do not use takeover while the original coordinator is still actively coordinating, because its later lifecycle mutations are rejected.\n\nDo not launch a replacement editor merely because the desktop app or runtime was updated. If adoption cannot prove continuing authority, keep the original worker as the only editor until it reaches a stable handoff point, then use a new current Dispatch in a conflict-free placement for any remaining work.\n\n## Ownership\n\nNew orchestration messages and tasks belong to one explicitly bound Run. A Run is only a durable namespace and coordinator inbox; it never schedules or places workers. Lifecycle authority comes from the active Dispatch, and terminal handles remain routing metadata rather than durable identity. Send `worker_done` and `heartbeat` from the worker's own terminal; Orca routes them to that Dispatch's Run.\n\nClassify inherited context before sending lifecycle messages:\n\n- Coordinated subtask: a live coordinator owns the DAG and waits on this dispatch. Follow the preamble exactly, including `worker_done`, heartbeat/status, `ask`, and `escalation`.\n- Full handoff means ownership transfer, not supervised dispatch. The original actor is not monitoring a DAG, so do not create lifecycle obligations unless the user explicitly asks you to supervise.\n- Classify requests containing \"hand off\", \"handoff\", \"handover\", \"give this to another agent\", \"give this to another worktree\", \"another agent\", or \"another worktree\" as full handoffs by default, even when the user names a custom model or reasoning effort.\n- Use supervised orchestration only when the user explicitly asks you to \"supervise\", \"monitor\", \"wait\", \"track completion\", \"wait for worker_done\", return results, coordinate a DAG, use a decision gate, or manage ask/reply flow.\n- Do not use `orca orchestration dispatch --inject` for full handoffs. It injects a coordinator preamble that tells the worker to send `worker_done`, heartbeat, and `ask` messages, then end its turn under the original terminal's dispatch lifecycle.\n- Do not run `orca orchestration task-create`, `orca orchestration dispatch --inject`, or `orca orchestration check --wait` for full handoffs. Do not peek at terminal output after prompt delivery to monitor progress.\n- A review-only `worker_done` reports findings; it does not authorize coordinator file edits. After a review-only completion, synthesize findings, ask a decision gate if ownership is unclear, and dispatch or hand off fixes unless the user explicitly asked the coordinator to own fixes.\n- If the user's plan names a next owner agent (for example, \"then use opencode to create a PR\"), post-review corrections and PR prep belong to that named owner. The coordinator routes, synthesizes, asks decision gates when needed, and supervises; the named owner edits files and creates the PR.\n\nIf unclear, inspect orchestration state before sending lifecycle messages:\n\n```bash\norca orchestration task-list --json\norca terminal list --json\n# If inherited context includes a task id:\norca orchestration dispatch-show --task --json\n```\n\n## Messaging\n\n```bash\norca orchestration send --subject [--to ] [--from ] [--body ] [--type ] [--priority ] [--thread-id ] [--payload ] [--json]\norca orchestration check [--terminal ] [--ack ] [--peek|--all] [--types ] [--format] [--wait] [--timeout-ms ] [--json]\norca orchestration reply --id --body [--from ] [--json]\norca orchestration ask (--question |--resume ) [--options ] [--timeout-ms ] [--from ] [--json]\norca orchestration inbox [--limit ] [--json]\n```\n\nRules:\n\n- Omit `--from` unless impersonating another terminal; Orca auto-resolves it from the current terminal.\n- A coordinator `check` returns the bound Run's oldest FIFO Delivery (up to 50 messages) and replays that exact batch until `--ack `. Process every message before acknowledging; `check --ack --wait` acknowledges, checks, and waits in one operation.\n- Use `--peek` and `--all` only for read-only history/debugging. Type filters decide when a waiter wakes; the returned actionable Delivery is still the oldest full batch.\n- Use `dispatch:` for coordinator guidance to one supervised worker. Orca routes that stable address locally or through the connected-server relay; do not substitute a remote terminal handle.\n- Terminal handles remain appropriate for low-level pre-Dispatch messaging. Prefer `agentTerminalHandle` from the create response, fall back to `startupTerminal.handle` for older runtimes, then re-resolve with `orca terminal list --worktree ... --json` if missing or stale. Continue with the replacement handle only; never dual-send to old and new handles.\n- `terminal list --json` omits `visualLayouts` because handle recovery does not need topology. Add `--include-visual-layouts` only for explicit tab and pane inspection.\n- `orca orchestration check --peek --format --json` returns locally formatted unread mail without consuming it; it never writes to terminal input or remotely wakes another terminal. Use `orchestration dispatch --inject` to deliver a tracked task, or `terminal send` when an existing agent needs a free-form prompt.\n- While supervising workers manually, use `check --wait --types worker_done,escalation,question --timeout-ms ` instead of sleep/poll loops. Process the whole Delivery, reply to `question` messages with `orca orchestration reply --id --body --json`, then acknowledge and keep waiting.\n- Treat a `check --wait` timeout or `{count:0}` as a checkpoint, not a worker failure. Long coding tasks routinely run 15-60 minutes; keep using rolling waits unless you receive `worker_done`/`escalation`, the terminal exits or disappears, or the user explicitly asks you to stop.\n- Heartbeats and visible terminal activity mean the worker is alive, not done. Do not stop, close, kill, or restart a worker just because it has not produced a completion message yet.\n- Use `ask` when a worker needs a blocking answer from the coordinator; it defaults to the active Dispatch's Run. Timeout or disconnect leaves the question pending, so resume by its original message ID instead of asking again.\n- `check --wait` returns one bounded Delivery, not every future completion. Process every message, acknowledge it, then keep waiting until every expected Dispatch settles.\n- Group addresses include `@all`, `@idle`, `@claude`, `@codex`, `@opencode`, `@gemini`, `@droid`, `@grok`, `@cursor`, and `@worktree:`.\n- Message types include `status`, `dispatch`, `worker_done`, `merge_ready`, `escalation`, `handoff`, `question`, `decision_gate` (legacy/gates), and `heartbeat`.\n- Use group addresses only for messages that are genuinely useful to many terminals, such as `status` broadcasts or intentional fan-out questions. Do not send dispatch lifecycle messages to groups.\n- `worker_done` belongs to the active Dispatch and defaults to its Run mailbox; never target a group.\n- A valid `worker_done` for the active `taskId` + `dispatchId` marks the task and dispatch completed automatically. Do not follow it with `task-update --status completed`; reserve manual updates for explicit recovery or overrides.\n- `heartbeat` is also Dispatch-scoped. Include both IDs and omit `--to` so Orca uses the owning Run; use `status` for broad progress updates.\n\n## Tasks And Dispatch\n\nA Run is the namespace/inbox, a Task is the work item, and a Dispatch assigns one Task attempt to a terminal. Create or bind a Run once before the common loop.\n\n```bash\norca orchestration run-create --objective --json\norca orchestration task-create --spec [--deps ] [--parent ] [--json]\norca orchestration task-list [--status ] [--ready] [--brief] [--json]\norca orchestration task-update --id --status [--result ] [--json]\norca orchestration dispatch --task --to [--from ] [--inject] [--json]\norca orchestration dispatch-show --task [--json]\n```\n\nTask statuses: `pending`, `ready`, `dispatched`, `completed`, `failed`, `blocked`.\n\nDispatch rules:\n\n- `--inject` sends the task spec plus preamble into a recognized agent CLI so it can report `worker_done`.\n- If the target is a bare shell, omit `--inject`, dispatch for tracking if needed, then send the prompt manually with `orca terminal send --terminal --text --enter --json`.\n- After 3 consecutive failures on one task, the dispatch context circuit-breaks and the task is marked failed.\n- Use `task-list --brief --json` for coordinator sweeps; it collapses whitespace and caps each echoed spec at 160 characters (`spec_truncated` marks shortened rows). Omit `--brief` when the full spec is required, or when an older CLI rejects it as an unknown flag.\n\n## How deep workers can nest\n\nA dispatched worker normally cannot dispatch sub-workers. Attempting it fails with\n`nested_worker_depth_exceeded` and a message telling the worker to complete the task\nitself. Do that — do not try to route around it.\n\nThe limit is a number, not an on/off switch. `Settings -> Orchestration -> Nested worker depth`\nsets how many generations are allowed:\n\n- `1` (default): a coordinator dispatches workers; those workers do not dispatch.\n- `2`: workers may dispatch one further generation.\n\nDepth is counted from the terminal that issues the command, not from the Run. Creating a\nnew Run does not reset it — a worker that runs `run-create` then `worker-start` is still a\nworker, and still counted. This is the part that changed: the old behaviour rejected\nsub-dispatch only because a worker's terminal was not bound to a Run, so creating a Run was\nenough to slip past it.\n\nTwo limits worth knowing:\n\n- **It is a guardrail, not a security boundary.** A caller that declares another terminal's\n handle while its own launch evidence is unverifiable (an ordinary restored terminal, for\n example) can be counted as that terminal instead. Orca does not treat workers as hostile.\n- **It applies while a Dispatch is active.** After `worker_done`, or after a coordinator\n settles the task, the terminal is no longer a worker and is counted as a root again. The\n process may still be alive; that is the documented boundary, not an accident.\n\n## Preferred Supervised Worker Loop\n\nUse `worker-start` for the normal supervised path. It composes the existing worktree, terminal, readiness, and dispatch primitives while returning exact created/reused effects. Agents still choose placement and concurrency; Orca does not schedule workers or infer conflicts.\n\nCreate the Run and every independent Task first, then start all independent workers before waiting:\n\n```bash\norca orchestration run-create --objective \"\" --json\norca orchestration task-create --spec \"\" --json\norca orchestration task-create --spec \"\" --json\norca orchestration worker-start --task --worktree current --agent codex --json\norca orchestration worker-start --task --worktree current --agent claude --json\n```\n\n`current` and exact existing worktrees create a fresh agent terminal and do not rerun setup. Reuse an existing agent only with `--terminal `.\n\n### Durable target concurrency and safe backfill\n\nA coordinator can declare a Run-wide execution ceiling and explicitly enroll independent Tasks in its capacity pool:\n\n```bash\norca orchestration capacity-set --target 5 --json\norca orchestration capacity-enroll --task --json\norca orchestration capacity-show --json\n```\n\n`capacity-show` reports the target, every active Dispatch in the Run, open slots, enrolled ready Tasks in stable creation order, enrolled pending/blocked Tasks, and settled terminal cleanup debt. Only Tasks explicitly enrolled with `capacity-enroll` appear as launchable; dependency promotion and decision-gate resolution must first make the Task `ready`. Failed, blocked, already dispatched, and circuit-broken work never appears as launchable.\n\nThe runtime deliberately does not invent worker placement. For each Task returned under `launchableTasks`, the coordinator chooses the existing `worker-start` placement and claims one slot atomically:\n\n```bash\norca orchestration worker-start --task --capacity-slot --worktree current --agent codex --json\n```\n\nOnce a target is nonzero, every new Dispatch in that Run—including low-level `dispatch` and Tasks outside the pool—must fit below it. `--capacity-slot` additionally requires explicit pool enrollment. Concurrent starts serialize at the Run home, so a stale `capacity-show` or idempotent coordinator replay can fail with `capacity_full` or `task_not_startable` but cannot exceed the target or duplicate a Task. A target of `0` disables capacity-slot launches without stopping existing work.\n\nAfter each accepted `worker_done`, release or reuse the settled worker as below, then call `capacity-show` again and start up to `launchableCount`. This closes the settled-lane backfill gap without a scheduler: Runs remain authoritative, placement remains explicit, and the same loop works after coordinator or runtime restart because target and enrollment are durable. Retained, user-owned, external, and `release_unknown` terminals are reported as cleanup debt; they are never auto-closed and do not masquerade as live execution lanes.\n\nUse `capacity-withdraw --task ` to remove a Task from future backfill. Enrollment changes neither Task status nor dependencies and never cancels an active Dispatch.\n\nFor a per-invocation Claude, Codex, or Cursor launch, pass an opaque provider model id with `--model`; add `--effort` only when that agent/model supports the level. These options apply only to fresh agent terminals, override general agent default arguments, and are reported under `launch.requested` and `launch.effective` in the receipt:\n\n```bash\norca orchestration worker-start --task --worktree current --agent claude --model opus --effort high --json\n```\n\n`--effort` requires `--model`, and neither option can combine with `--terminal`. A connected worker server must advertise launch-preference support before Orca forwards either option.\n\nFor a new worktree, setup runs by default and agent-first creation reuses the returned startup agent terminal:\n\n```bash\norca orchestration worker-start --task --worktree new-child --name --agent codex --setup run --json\n# Independent/top-level:\norca orchestration worker-start --task --worktree new-top-level --name --agent codex --setup run --json\n```\n\nSetup normally starts alongside the agent. Only a repository explicitly configured with `wait-for-setup` delays agent launch until setup succeeds. Use `--setup skip` or `--setup inherit` only for a concrete reason.\n\nRead the returned receipt before continuing: `ready` plus setup `running` is normal for start-immediately, while wait-for-setup returns setup `succeeded` before accepting task input. A failed or unknown start exits nonzero; inspect its `stage`, `effects`, and `residualResources` instead of guessing or automatically retrying. A wait-for-setup timeout can honestly leave setup `running`, which is not proof of failure.\n\nTo run the worker on another connected Orca server, add `--on `. The Run and Tasks remain authoritative on the current server; later commands route by Dispatch ID, so never repeat `--on`:\n\n```bash\n# Mac Run home -> Windows worker (the reverse is identical from a Windows Run home)\norca orchestration worker-start --task --on windows --worktree new-top-level --repo --name --agent codex --setup run --json\norca orchestration worker-show --dispatch --json\norca orchestration worker-read --dispatch --limit 50 --json\norca orchestration send --to dispatch: --subject \"Follow-up\" --body \"\" --json\n```\n\nRemote `current` and `new-child` are intentionally invalid because those words are ambiguous across servers. Use an exact discovered remote worktree selector or `new-top-level` with an explicit remote repo selector.\n\nThe follow-up is structured inbox mail, not prompt injection. The worker's next\n`orchestration check` receives it even when the Dispatch is on another connected Orca server.\n\n`worker-read` defaults to `--source auto`: Orca returns the exact hook-reported Codex, Claude, OpenClaude, or Grok transcript when it can prove the worker session, otherwise it returns bounded terminal output with `source: \"terminal\"` and a typed `fallbackReason`. Continue with the returned top-level `cursor`; it stays pinned to that exact source. If Orca reports `source_changed`, start a fresh read without the old cursor. Never supply or guess a provider session ID or transcript path.\n\nWait until every expected Dispatch settles, not for a fixed number of batches:\n\n```bash\norca orchestration check --wait --types worker_done,escalation,question --timeout-ms 900000 --json\n# Process every message. For each accepted worker_done that is not immediately reused:\norca orchestration worker-release --dispatch --json\n# Acknowledge only after every message and required release decision is handled:\norca orchestration check --ack --wait --types worker_done,escalation,question --timeout-ms 900000 --json\n```\n\nAfter processing each accepted `worker_done`, choose the terminal's next owner before you acknowledge the Delivery or wait again. If the same exact agent has an immediate follow-up Task, read the `worker.agent_terminal_handle` field of `worker-show --dispatch --json`, then run `orca orchestration worker-start --task --terminal --json` so Orca transfers cleanup ownership to the new Dispatch. Otherwise run `orca orchestration worker-release --dispatch --json`.\n\nRun `worker-release` after both succeeded and failed `worker_done` reports unless the user explicitly asked to keep that worker live. Release is post-completion cleanup, not cancellation: Orca first preserves inspectable output, then closes only the exact agent terminal owned by that settled Dispatch. Reused or pre-existing terminals, setup terminals, coordinators, active workers, user-taken-over terminals, and identities Orca cannot prove are retained. If the user explicitly asks to keep the live terminal for debugging, record that exception with `orca orchestration worker-retain --dispatch --json` instead of silently skipping cleanup. When the user is finished, the same Dispatch can be passed to `worker-release`, which clears the requested retention and releases the terminal.\n\nDo not release a worker because of a timeout, TUI idle state, heartbeat, status, question, escalation, or rejected/stale `worker_done`. If release returns `release_pending` or `release_unknown`, do not substitute `terminal close`; follow the exact recovery action in the receipt. A replayed Delivery may repeat `worker-release` safely.\n\nWorkers report exactly once using the IDs and capability injected by Orca; they do not supply Run/server/terminal identity:\n\n```bash\norca orchestration send --type worker_done --subject \"\" --body \"\" --task-id --dispatch-id --outcome succeeded --files-modified \"path/a,path/b\" --json\n# On failure, use --outcome failed; never encode failure only in prose.\n```\n\nA worker question defaults to its owning Run. Timeout leaves it pending:\n\n```bash\norca orchestration ask --question \"\" --options \"yes,no\" --timeout-ms 600000 --json\norca orchestration ask --resume --timeout-ms 600000 --json\n# Coordinator:\norca orchestration reply --id --body \"\" --json\n```\n\nRecovery is conditional, never a fixed destructive sequence:\n\n- `worker-show --dispatch ` says `ready`: keep waiting or read bounded output.\n- It proves `failed` or `stopped`: start a replacement with `worker-start --task --retry-of ` plus an explicit `--on`/`--worktree` and `--agent`/`--terminal` choice. Retry does not silently inherit placement.\n- It remains `outcome_unknown`: either `worker-stop --dispatch ` and inspect again, or explicitly `worker-abandon --dispatch ` while accepting that resources may still be live. Abandon performs no remote, process, or filesystem action.\n- `worker-stop` closes only the exact supervised agent terminal. It never deletes the worktree, setup terminal, configured tabs, or unrelated processes.\n\nLow-level `worktree create`, `terminal create`, and `dispatch --inject` remain valid recipes for custom argv or topology that `worker-start` does not express.\n\n`dispatch --inject` deliberately keeps an operator-started terminal unsupervised: it never creates a `worker_dispatches` row and `worker-stop`/`worker-abandon` never close that process. The dispatch context is still authoritative, so `worker-show`, `worker-read`, and `worker-list` report it as `unsupervised`; settled `worker-retain` and `worker-release` report `retained` with `no_owned_resource` and take no process action. Use `worker-start --terminal ` when supervision and worker lifecycle state are required.\n\n## Gates And Legacy Inspection\n\n```bash\norca orchestration gate-create --task --question [--options ] [--json]\norca orchestration gate-resolve --id --resolution [--json]\norca orchestration gate-list [--task ] [--status ] [--json]\n```\n\nUse `ask` for worker-to-coordinator questions; it creates a `question` message that the coordinator answers with `reply`. Use `gate-create` only for coordinator-managed task DAG decisions, not for answering a worker's `ask`.\n\n`coordinator-start`, `coordinator-stop`, `run`, and `run-stop` are retired scheduler commands. They perform no effects and return the current-skill recovery action. They are not aliases for lightweight Run creation or binding.\n\nRecovery only: `orca orchestration reset --tasks|--messages|--all --json` clears the selected local orchestration database state. Do not run it during active coordination unless explicitly abandoning that state.\n\n## Full Handoffs\n\nFor full ownership transfer, use non-lifecycle terminal/worktree commands and then stop monitoring unless the user asks for supervision.\n\nTreat these as full handoff requests by default: \"hand off\", \"handoff\", \"handover\", \"give this to another agent\", \"give this to another worktree\", \"send this to another agent\", \"another agent\", \"another worktree\", or \"launch another agent to own this.\" Custom model or reasoning effort words such as `gpt-5.5`, `high`, or `xhigh` do not make the handoff supervised.\n\nSupervised orchestration remains available only when the user explicitly asks for supervision or coordination: \"supervise\", \"monitor\", \"wait for worker_done\", \"wait for results\", \"track completion\", \"DAG\", \"decision gate\", \"ask/reply\", or \"coordinate workers.\"\n\nDo not run `orca orchestration task-create`, `orca orchestration dispatch --inject`, or `orca orchestration check --wait` for full handoffs. `task-create` is also forbidden because it records coordinator-owned tracking state; if a task row is needed, the user asked for supervised orchestration. Do not create a `taskId`/`dispatchId`, inject a lifecycle preamble, wait for completion, or read the worker terminal after prompt delivery except to avoid losing the initial prompt.\n\nNew top-level worktree handoff:\n\n```bash\norca worktree create --name --no-parent --agent codex --prompt \"\" --setup run --json\n```\n\nBefore creating a new worktree from an active feature branch, decide and state whether the desired Orca lineage is child or top-level. Use child worktree lineage only when the new work is conceptually stacked under or dependent on the active worktree. For independent repo-wide fixes, standalone feature work, or unrelated follow-up tasks, create a top-level worktree with `--no-parent`.\n\nExisting terminal handoff:\n\n```bash\norca terminal send --terminal --text \"\" --enter --json\n```\n\nCustom Codex model/effort handoff:\n\n`orca worktree create --agent codex --prompt ...` launches the known Codex agent but does not accept Codex-specific `--model` or `-c model_reasoning_effort=...` arguments. When the user asks for a specific Codex model or effort, create the independent worktree first, launch Codex with the requested command in that worktree, wait only for TUI readiness if prompt delivery would otherwise race startup, send the prompt, and stop.\n\nThe two-step custom-argv path cannot enforce a repository's explicit `wait-for-setup` startup policy because the later `terminal create` is not the startup owned by `worktree create`. Use it only when the repository starts agents immediately. If the repository requires `wait-for-setup`, use an agent-first configured launcher that can preserve sequencing, or stop and ask rather than silently bypassing the policy.\n\nNote: when no repo default-terminal configuration supplies a primary terminal, bare create opens a fallback shell before `terminal create` adds the agent. Configured default tabs are materialized instead and may run real commands. Prefer `--agent` whenever custom argv is not required. With the two-step path, target only the agent handle; close a prior terminal only after `terminal list` or `terminal show` confirms it is an unused shell.\n\nUse the exact full `::` worktree id returned by `orca worktree create --json`; a bare repo id cannot target the new worktree.\n\n```bash\norca worktree create --name --no-parent --setup run --json\norca terminal create --worktree id: --title --command 'codex --model gpt-5.5 -c model_reasoning_effort=\"xhigh\"' --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca terminal send --terminal --text \"\" --enter --json\n```\n\nWait only for `tui-idle` when needed to avoid losing the prompt. Do not monitor task completion.\n\n`--no-parent` only controls Orca lineage; it does not choose the Git base. If the work should start from the repo default base, omit `--base-branch` so Orca uses that default, or explicitly pass the repo default base (`origin/main`, `origin/master`, or the `orca repo show --repo --json` value); never base it on the current feature branch unless the user explicitly asks for stacked work or \"branch from current\". Put current-branch context in the prompt instead.\n\n## Worker Terminals\n\nChoose the worker location before creating a terminal. `Fresh worker` means a fresh agent session, not a new git worktree. For parallel work, create one fresh agent terminal per worker in the same required worktree, falling back to the active worktree when none is named. If the task says current worktree only, depends on uncommitted files/artifacts, or must validate/PR the current branch, keep every worker in the active worktree:\n\n```bash\norca terminal create --worktree active --title --command \"codex\" --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca orchestration dispatch --task --to --inject --json\n```\n\nReuse an idle agent in the required worktree only if the prompt allows reuse; otherwise create a fresh terminal there. Create a new worktree only when the user explicitly requests one or a concrete checkout or filesystem conflict makes sharing unsafe or impossible; if the user did not request it, state that conflict before running `worktree create`. Independent tasks, parallel execution, convenience, or a preference for separate checkouts are not isolation requirements.\n\nWhen a new worktree is allowed, use child lineage for isolated work that is stacked under or dependent on the active worktree, and use `--no-parent` when it is not stacked. Decide the Git base separately: `--no-parent` makes the worktree top-level in Orca, while omitted `--base-branch` uses the repo default base.\n\nFor every new worktree, pass `--setup run` so any configured repository setup hook runs. This does not mean waiting for setup before agent launch: preserve the repository's startup policy, whose default starts setup and the agent side by side. Use `--setup skip` or `--setup inherit` only when there is a concrete task-specific reason, and state that reason before creating the worktree. This rule does not rerun setup for current or existing worktrees.\n\n```bash\norca worktree create --name --agent codex --setup run --json\n# or: --agent claude | omp | pi | grok | ...\n# Read from agentTerminalHandle, falling back to startupTerminal.handle.\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca orchestration dispatch --task --to --inject --json\n```\n\nFor new-worktree workers, read the id and `agentTerminalHandle` from `worktree create`, falling back to `startupTerminal.handle` for older runtimes. Use that as the sole worker handle when present; otherwise use `terminal list` to resolve the agent handle. Omit `--repo` only inside an Orca-managed worktree; otherwise pass `--repo `.\n\n**For an allowed new worktree, use agent-first:** `--agent` reveals the new worktree and launches the selected agent **in its first terminal**, without adding a separate fallback shell for that worker. Pass `--setup run`; repo setup and default-terminal settings may add intentional tabs or splits. Do **not** run bare `worktree create` and then `terminal create --command ` for the same worker when agent-first create is available: without configured default tabs, that two-step path leaves a fallback shell + agent pair. Only use it when custom agent argv is required (for example Codex model/effort flags) or when an older CLI rejects `--agent`; if you must, message only the agent handle. Configured default tabs are intentional surfaces, so close a prior terminal only after `terminal list` or `terminal show` confirms it is an unused shell. Do not run `worktree create` when the task must stay in the current worktree.\n\nUse `orca worktree create --prompt ...` or `orca terminal send ...` for full handoffs or untracked/lightweight prompts. Those paths do not attach `taskId`/`dispatchId`; the worker should not send lifecycle messages unless the prompt supplies a live orchestration preamble.\n\nSidebar lineage and orchestration lifecycle are related but not identical. A same-worktree worker may appear as a peer under that worktree in the sidebar while remaining a child dispatch in orchestration state; only an actual child worktree creates visible parent/child worktree lineage.\n\nOther terminal commands coordinators often need:\n\n```bash\norca terminal list [--worktree ] [--include-visual-layouts] [--json]\norca terminal create [--worktree ] [--title ] [--command ] [--json]\norca terminal split --terminal [--direction horizontal|vertical] [--command ] [--json]\norca terminal wait --terminal --for tui-idle --timeout-ms --json\norca terminal read --terminal --json\norca terminal send --terminal --text --enter --json\n```\n\nIf an older CLI rejects `worktree create --agent`, create the worktree normally, then run `orca terminal create --worktree --command \"codex\" --json` or `--command \"claude\"`.\n\nWait for `tui-idle` before dispatching. Always pass `--timeout-ms`; real coding tasks can take 15-60 minutes. During supervision, use rolling `check --wait` windows. If a window returns no matching message, inspect `task-list`, `terminal read`, or `terminal wait --for tui-idle` as a liveness checkpoint; if the terminal is still working or producing activity, keep waiting instead of retrying the task.\n\n## Agent Guidance\n\n- Workers with a valid live preamble must send `worker_done` exactly once from their own terminal with an explicit `--outcome succeeded` or `--outcome failed`:\n `orca orchestration send --type worker_done --subject \"\" --body \"<3-sentence summary: what you did, what you found, what's left>\" --task-id --dispatch-id --outcome succeeded --files-modified \"path/a\" --report-path \"\" --json`\n- A failed outcome is still a terminal report, but Orca records both the Dispatch and Task as failed. Never encode failure only in the subject/body.\n- After sending `worker_done`, end that dispatched turn and idle at the agent prompt. Do not autonomously start more work, poll, or attempt to close the terminal yourself. A direct user instruction takes precedence and starts ordinary user-owned work: follow it without coordinator approval or a fresh Dispatch, never refuse it because of worker/coordinator roles, and do not reuse the settled Dispatch's lifecycle IDs. A coordinator-supervised follow-up still arrives with a fresh preamble + TASK block.\n- For long tasks, send heartbeat/status only when the preamble asks for it, including both IDs:\n `orca orchestration send --type heartbeat --subject \"alive\" --payload '{\"taskId\":\"\",\"dispatchId\":\"\",\"phase\":\"implementing\"}' --json`\n- If blocked before completion, use `ask`; use `escalation` only when ownership is valid and the coordinator must intervene.\n- Treat preambles inherited through terminal history or full handoffs as stale unless the current prompt explicitly keeps that coordinator in the loop.\n- Coordinators must account for every settled worker terminal before waiting again or ending the turn: immediately reuse the exact worker for a new Dispatch, explicitly retain it at the user's request with `worker-retain`, or run `worker-release`. Do not leave a completed worker live merely to inspect output; released workers remain readable through `worker-read`.\n- Coordinators should use `task-list --ready` as external memory, dispatch parallel waves, and avoid dependency chains deeper than 3-4 steps.\n\n## Example\n\n```bash\norca terminal create --worktree active --title login-css-worker --command \"claude\" --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca orchestration task-create --spec \"Fix the login button CSS\" --json\norca orchestration dispatch --task --to --inject --json\norca orchestration check --wait --types worker_done,escalation,question --timeout-ms 900000 --json\n```\n\n## Next Action\n\nCoordinator: confirm `orca status --json`, create or bind a Run, inspect `task-list`/`dispatch-show` if inheriting state, then use the explicit supervised loop (`task-create` -> `worker-start` -> `check --wait`). Use low-level terminal creation plus `dispatch --inject` only when the composed start does not express the needed topology. After every accepted `worker_done`, either transfer the exact terminal to an immediate follow-up Dispatch or run `worker-release` before the next wait.\n\nWorker: if the current prompt contains a live dispatch preamble, do the task, use `ask` for blocking questions, and send `worker_done` once with the required payload. If the preamble is stale or absent, do not send lifecycle messages; inspect state or treat the prompt as an ordinary handoff.\n" // Why: no current guide has bundled reference documents, so --full is byte-identical for now. // oxfmt-ignore diff --git a/src/cli/handler-group-manifest.ts b/src/cli/handler-group-manifest.ts index 0907905e5e3..26ede63649c 100644 --- a/src/cli/handler-group-manifest.ts +++ b/src/cli/handler-group-manifest.ts @@ -108,6 +108,10 @@ export const HANDLER_GROUPS: readonly HandlerGroup[] = [ 'orchestration run-current', 'orchestration run-list', 'orchestration run-show', + 'orchestration capacity-set', + 'orchestration capacity-show', + 'orchestration capacity-enroll', + 'orchestration capacity-withdraw', 'orchestration send', 'orchestration check', 'orchestration reply', diff --git a/src/cli/handlers/orchestration-capacity-cli.test.ts b/src/cli/handlers/orchestration-capacity-cli.test.ts new file mode 100644 index 00000000000..77f5b2f11ea --- /dev/null +++ b/src/cli/handlers/orchestration-capacity-cli.test.ts @@ -0,0 +1,69 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const callMock = vi.fn() + +vi.mock('../format', () => ({ printResult: vi.fn() })) +vi.mock('../selectors', () => ({ getTerminalHandle: vi.fn() })) + +import { ORCHESTRATION_HANDLERS } from './orchestration' + +describe('orchestration capacity CLI contract', () => { + beforeEach(() => callMock.mockReset()) + + it('forwards target configuration with mutation replay identity', async () => { + callMock.mockResolvedValue({ result: { capacity: { runId: 'run_1' } } }) + + await ORCHESTRATION_HANDLERS['orchestration capacity-set']({ + flags: new Map([ + ['target', '5'], + ['run', 'run_1'], + ['from', 'term_coord'], + ['retry-request', 'request_1'] + ]), + client: { call: callMock }, + cwd: '/tmp/repo', + json: true + } as never) + + expect(callMock).toHaveBeenCalledWith( + 'orchestration.capacityConfigure', + { target: 5, run: 'run_1', from: 'term_coord' }, + { orchestrationRequestId: 'request_1' } + ) + }) + + it('marks a worker start as an enrolled capacity claim only when requested', async () => { + callMock + .mockResolvedValueOnce({ + result: { capabilities: ['orchestration.run-capacity.v1'] } + }) + .mockResolvedValueOnce({ + result: { + runId: 'run_1', + taskId: 'task_1', + dispatchId: 'ctx_1', + state: 'ready', + effects: [], + residualResources: [] + } + }) + + await ORCHESTRATION_HANDLERS['orchestration worker-start']({ + flags: new Map([ + ['task', 'task_1'], + ['agent', 'codex'], + ['capacity-slot', true], + ['from', 'term_coord'] + ]), + client: { call: callMock }, + cwd: '/tmp/repo', + json: true + } as never) + + expect(callMock).toHaveBeenNthCalledWith( + 2, + 'orchestration.workerStart', + expect.objectContaining({ task: 'task_1', capacitySlot: true }) + ) + }) +}) diff --git a/src/cli/handlers/orchestration-worker-cli.test.ts b/src/cli/handlers/orchestration-worker-cli.test.ts index d19666556c6..b68e7d21c90 100644 --- a/src/cli/handlers/orchestration-worker-cli.test.ts +++ b/src/cli/handlers/orchestration-worker-cli.test.ts @@ -8,7 +8,10 @@ vi.mock('../selectors', () => ({ getTerminalHandle: vi.fn() })) import { ORCHESTRATION_HANDLERS } from './orchestration' import { printResult } from '../format' -import { ORCHESTRATION_WORKER_LAUNCH_PREFERENCES_RUNTIME_CAPABILITY } from '../../shared/protocol-version' +import { + ORCHESTRATION_RUN_CAPACITY_RUNTIME_CAPABILITY, + ORCHESTRATION_WORKER_LAUNCH_PREFERENCES_RUNTIME_CAPABILITY +} from '../../shared/protocol-version' describe('orchestration worker-start CLI contract', () => { beforeEach(() => { @@ -125,6 +128,56 @@ describe('orchestration worker-start CLI contract', () => { ) }) + it('fails closed when the runtime cannot enforce a requested capacity slot', async () => { + callMock.mockResolvedValueOnce({ result: { capabilities: [] } }) + + await expect( + invokeWorkerStart( + new Map([ + ['task', 'task_1'], + ['agent', 'codex'], + ['capacity-slot', true], + ['from', 'term_coord'] + ]) + ) + ).rejects.toMatchObject({ code: 'incompatible_runtime' }) + expect(callMock).toHaveBeenCalledTimes(1) + expect(callMock).toHaveBeenCalledWith('status.get') + }) + + it('forwards a capacity slot only after the runtime advertises enforcement', async () => { + callMock + .mockResolvedValueOnce({ + result: { capabilities: [ORCHESTRATION_RUN_CAPACITY_RUNTIME_CAPABILITY] } + }) + .mockResolvedValueOnce({ + result: { + runId: 'run_1', + taskId: 'task_1', + dispatchId: 'ctx_1', + state: 'ready', + effects: [], + residualResources: [] + } + }) + + await invokeWorkerStart( + new Map([ + ['task', 'task_1'], + ['agent', 'codex'], + ['capacity-slot', true], + ['from', 'term_coord'] + ]) + ) + + expect(callMock).toHaveBeenNthCalledWith(1, 'status.get') + expect(callMock).toHaveBeenNthCalledWith( + 2, + 'orchestration.workerStart', + expect.objectContaining({ capacitySlot: true }) + ) + }) + it('fails before worker-start when the runtime would strip launch preferences', async () => { callMock.mockResolvedValueOnce({ result: { capabilities: [] } }) diff --git a/src/cli/handlers/orchestration.ts b/src/cli/handlers/orchestration.ts index f1f1c70bc39..4dd82cb5fb2 100644 --- a/src/cli/handlers/orchestration.ts +++ b/src/cli/handlers/orchestration.ts @@ -1,5 +1,6 @@ import type { CommandHandler } from '../dispatch' import { ORCHESTRATION_CHECK_HANDLER } from './orchestration/message-check-handler' +import { ORCHESTRATION_CAPACITY_HANDLERS } from './orchestration/capacity-handlers' import { ORCHESTRATION_DISPATCH_HANDLER, ORCHESTRATION_DISPATCH_INSPECTION_HANDLERS @@ -17,6 +18,7 @@ import { ORCHESTRATION_WORKER_TERMINAL_HANDLERS } from './orchestration/worker-t export const ORCHESTRATION_HANDLERS: Record = { ...ORCHESTRATION_RUN_HANDLERS, + ...ORCHESTRATION_CAPACITY_HANDLERS, ...ORCHESTRATION_SEND_HANDLER, ...ORCHESTRATION_CHECK_HANDLER, ...ORCHESTRATION_INBOX_HANDLERS, diff --git a/src/cli/handlers/orchestration/capacity-handlers.ts b/src/cli/handlers/orchestration/capacity-handlers.ts new file mode 100644 index 00000000000..ad9fbfeed4f --- /dev/null +++ b/src/cli/handlers/orchestration/capacity-handlers.ts @@ -0,0 +1,100 @@ +import type { CommandHandler } from '../../dispatch' +import { printResult } from '../../format' +import { getOptionalStringFlag, getRequiredFiniteNumber, getRequiredStringFlag } from '../../flags' +import { callOrchestrationMutation } from './mutation-request' +import { resolveCoordinatorTerminalHandle } from './terminal-identity' + +type CapacitySnapshot = { + runId: string + targetConcurrency: number + activeCount: number + availableSlots: number + launchableCount: number + launchableTasks: { id: string; display_name?: string | null; task_title?: string | null }[] + eligiblePendingTaskIds: string[] + settledTerminalDebt: { + dispatchId: string + terminalState: string + retainedReason: string | null + }[] +} + +function formatCapacity(snapshot: CapacitySnapshot): string { + const head = + `Run ${snapshot.runId}: target ${snapshot.targetConcurrency}, active ${snapshot.activeCount}, ` + + `open ${snapshot.availableSlots}, launchable ${snapshot.launchableCount}` + const launchable = snapshot.launchableTasks.map((task) => { + const label = task.display_name ?? task.task_title + return label ? `${task.id} ${label}` : task.id + }) + const debt = snapshot.settledTerminalDebt.map( + (worker) => + `${worker.dispatchId} [${worker.terminalState}]${worker.retainedReason ? ` ${worker.retainedReason}` : ''}` + ) + return [ + head, + launchable.length > 0 + ? `Eligible ready tasks:\n${launchable.join('\n')}` + : 'No eligible ready tasks.', + snapshot.eligiblePendingTaskIds.length > 0 + ? `Eligible but blocked/pending: ${snapshot.eligiblePendingTaskIds.join(', ')}` + : '', + debt.length > 0 ? `Settled terminal cleanup:\n${debt.join('\n')}` : '' + ] + .filter(Boolean) + .join('\n') +} + +export const ORCHESTRATION_CAPACITY_HANDLERS: Record = { + 'orchestration capacity-set': async ({ flags, client, cwd, json }) => { + const result = await callOrchestrationMutation<{ capacity: CapacitySnapshot }>( + client, + flags, + 'orchestration.capacityConfigure', + { + target: getRequiredFiniteNumber(flags, 'target'), + run: getOptionalStringFlag(flags, 'run'), + from: await resolveCoordinatorTerminalHandle(flags, cwd, client) + } + ) + printResult(result, json, (value) => formatCapacity(value.capacity)) + }, + 'orchestration capacity-show': async ({ flags, client, cwd, json }) => { + const run = getOptionalStringFlag(flags, 'run') + const result = await client.call<{ capacity: CapacitySnapshot }>('orchestration.capacityShow', { + run, + from: run ? undefined : await resolveCoordinatorTerminalHandle(flags, cwd, client) + }) + printResult(result, json, (value) => formatCapacity(value.capacity)) + }, + 'orchestration capacity-enroll': async ({ flags, client, cwd, json }) => { + const task = getRequiredStringFlag(flags, 'task') + const result = await callOrchestrationMutation<{ task: { id: string } }>( + client, + flags, + 'orchestration.capacityTaskSet', + { + task, + eligible: true, + run: getOptionalStringFlag(flags, 'run'), + from: await resolveCoordinatorTerminalHandle(flags, cwd, client) + } + ) + printResult(result, json, () => `Task ${task} enrolled in the Run capacity pool.`) + }, + 'orchestration capacity-withdraw': async ({ flags, client, cwd, json }) => { + const task = getRequiredStringFlag(flags, 'task') + const result = await callOrchestrationMutation<{ task: { id: string } }>( + client, + flags, + 'orchestration.capacityTaskSet', + { + task, + eligible: false, + run: getOptionalStringFlag(flags, 'run'), + from: await resolveCoordinatorTerminalHandle(flags, cwd, client) + } + ) + printResult(result, json, () => `Task ${task} withdrawn from the Run capacity pool.`) + } +} diff --git a/src/cli/handlers/orchestration/worker-launch-handler.ts b/src/cli/handlers/orchestration/worker-launch-handler.ts index a6161e61ef0..57958374cb2 100644 --- a/src/cli/handlers/orchestration/worker-launch-handler.ts +++ b/src/cli/handlers/orchestration/worker-launch-handler.ts @@ -3,7 +3,10 @@ import { printResult } from '../../format' import { getOptionalStringFlag, getRequiredStringFlag } from '../../flags' import { RuntimeClientError } from '../../runtime-client' import type { RuntimeStatus } from '../../../shared/runtime-types' -import { ORCHESTRATION_WORKER_LAUNCH_PREFERENCES_RUNTIME_CAPABILITY } from '../../../shared/protocol-version' +import { + ORCHESTRATION_RUN_CAPACITY_RUNTIME_CAPABILITY, + ORCHESTRATION_WORKER_LAUNCH_PREFERENCES_RUNTIME_CAPABILITY +} from '../../../shared/protocol-version' import { callOrchestrationMutation } from './mutation-request' import { getOptionalPositiveIntegerValueFlag } from './numeric-flags' import { isDevCliInvocation } from './runtime-compatibility' @@ -13,9 +16,11 @@ export const ORCHESTRATION_WORKER_LAUNCH_HANDLER: Record 'orchestration worker-start': async ({ flags, client, cwd, json }) => { const model = getOptionalStringFlag(flags, 'model') const effort = getOptionalStringFlag(flags, 'effort') - if (model || effort) { + const capacitySlot = flags.has('capacity-slot') + if (model || effort || capacitySlot) { const status = await client.call('status.get') if ( + (model || effort) && !status.result.capabilities?.includes( ORCHESTRATION_WORKER_LAUNCH_PREFERENCES_RUNTIME_CAPABILITY ) @@ -25,6 +30,15 @@ export const ORCHESTRATION_WORKER_LAUNCH_HANDLER: Record 'The connected Orca runtime does not support worker model or effort overrides. Update or restart Orca and try again.' ) } + if ( + capacitySlot && + !status.result.capabilities?.includes(ORCHESTRATION_RUN_CAPACITY_RUNTIME_CAPABILITY) + ) { + throw new RuntimeClientError( + 'incompatible_runtime', + 'The connected Orca runtime cannot enforce Run capacity slots. Update or restart Orca and try again.' + ) + } } const result = await callOrchestrationMutation<{ runId: string @@ -51,6 +65,7 @@ export const ORCHESTRATION_WORKER_LAUNCH_HANDLER: Record effort, terminal: getOptionalStringFlag(flags, 'terminal'), retryOf: getOptionalStringFlag(flags, 'retry-of'), + ...(capacitySlot ? { capacitySlot: true } : {}), timeoutMs: getOptionalPositiveIntegerValueFlag(flags, 'timeout-ms'), run: getOptionalStringFlag(flags, 'run'), from: await resolveCoordinatorTerminalHandle(flags, cwd, client), diff --git a/src/cli/specs/orchestration-worker-specs.ts b/src/cli/specs/orchestration-worker-specs.ts index 31377eb4384..115db846993 100644 --- a/src/cli/specs/orchestration-worker-specs.ts +++ b/src/cli/specs/orchestration-worker-specs.ts @@ -5,10 +5,11 @@ export const ORCHESTRATION_WORKER_COMMAND_SPECS: CommandSpec[] = [ path: ['orchestration', 'worker-start'], summary: 'Start one supervised worker on the Run home or a connected Orca server', usage: - 'orca orchestration worker-start --task [--on ] [--worktree ] (--agent | --terminal ) [--model ] [--effort ] [--name ] [--repo ] [--base-branch ] [--display-name ] [--comment ] [--setup ] [--retry-of ] [--timeout-ms ] [--run ] [--from ] [--retry-request ] [--json]', + 'orca orchestration worker-start --task [--capacity-slot] [--on ] [--worktree ] (--agent | --terminal ) [--model ] [--effort ] [--name ] [--repo ] [--base-branch ] [--display-name ] [--comment ] [--setup ] [--retry-of ] [--timeout-ms ] [--run ] [--from ] [--retry-request ] [--json]', allowedFlags: [ ...GLOBAL_FLAGS, 'task', + 'capacity-slot', 'on', 'worktree', 'name', @@ -35,7 +36,8 @@ export const ORCHESTRATION_WORKER_COMMAND_SPECS: CommandSpec[] = [ '--on selects only the worker server; the Run and this command remain on the current Orca server.', 'Remote current and new-child are invalid; discover an exact remote selector or use new-top-level.', '--retry-of links the replacement attempt but does not inherit placement; repeat the intended --on/worktree and --agent/terminal choices.', - 'The call exits 0 only for ready. Failed or outcome_unknown exits 1 and JSON includes stage/failedStage, setup, effects, residualResources, and recovery commands when needed.' + 'The call exits 0 only for ready. Failed or outcome_unknown exits 1 and JSON includes stage/failedStage, setup, effects, residualResources, and recovery commands when needed.', + '--capacity-slot atomically enforces the Run target and requires this ready Task to be explicitly enrolled.' ] }, { diff --git a/src/cli/specs/orchestration.ts b/src/cli/specs/orchestration.ts index f266e93b10a..4a2c22f82dd 100644 --- a/src/cli/specs/orchestration.ts +++ b/src/cli/specs/orchestration.ts @@ -42,6 +42,34 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [ usage: 'orca orchestration run-show --id [--json]', allowedFlags: [...GLOBAL_FLAGS, 'id'] }, + { + path: ['orchestration', 'capacity-set'], + summary: 'Declare the target concurrent worker count for the bound Run', + usage: + 'orca orchestration capacity-set --target <0-64> [--run ] [--from ] [--retry-request ] [--json]', + allowedFlags: [...GLOBAL_FLAGS, 'target', 'run', 'from', 'retry-request'], + notes: ['A target of 0 disables capacity-slot launches without changing existing workers.'] + }, + { + path: ['orchestration', 'capacity-show'], + summary: 'Show target, active lanes, eligible ready tasks, and settled terminal debt', + usage: 'orca orchestration capacity-show [--run ] [--from ] [--json]', + allowedFlags: [...GLOBAL_FLAGS, 'run', 'from'] + }, + { + path: ['orchestration', 'capacity-enroll'], + summary: 'Explicitly enroll one Task in the Run capacity pool', + usage: + 'orca orchestration capacity-enroll --task [--run ] [--from ] [--retry-request ] [--json]', + allowedFlags: [...GLOBAL_FLAGS, 'task', 'run', 'from', 'retry-request'] + }, + { + path: ['orchestration', 'capacity-withdraw'], + summary: 'Withdraw one Task from the Run capacity pool', + usage: + 'orca orchestration capacity-withdraw --task [--run ] [--from ] [--retry-request ] [--json]', + allowedFlags: [...GLOBAL_FLAGS, 'task', 'run', 'from', 'retry-request'] + }, { path: ['orchestration', 'send'], summary: 'Send an inter-agent message', diff --git a/src/main/runtime/orchestration-capacity-cli-subprocess.test.ts b/src/main/runtime/orchestration-capacity-cli-subprocess.test.ts new file mode 100644 index 00000000000..d28bff8708d --- /dev/null +++ b/src/main/runtime/orchestration-capacity-cli-subprocess.test.ts @@ -0,0 +1,196 @@ +import { existsSync, mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it, vi } from 'vitest' +import { OrcaRuntimeService } from './orca-runtime' +import { OrchestrationDb } from './orchestration/db' +import { OrcaRuntimeRpcServer } from './runtime-rpc' +import { spawnProcess } from '../../shared/child-process/run-process' + +const CLI_PATH = join(process.cwd(), 'out', 'cli', 'index.js') +const describeIfBuilt = existsSync(CLI_PATH) ? describe : describe.skip + +async function runBuiltCli( + userDataPath: string, + args: string[] +): Promise<{ exitCode: number; stdout: string; stderr: string }> { + const child = spawnProcess({ + program: process.execPath, + args: [CLI_PATH, ...args], + env: { + ...process.env, + ORCA_USER_DATA_PATH: userDataPath, + ORCA_TERMINAL_HANDLE: 'term_cli' + }, + stdio: ['ignore', 'pipe', 'pipe'] + }) + if (!child.stdout || !child.stderr) { + throw new Error('Compiled CLI test requires piped stdout and stderr.') + } + const stdout: Buffer[] = [] + const stderr: Buffer[] = [] + child.stdout.on('data', (chunk) => stdout.push(chunk)) + child.stderr.on('data', (chunk) => stderr.push(chunk)) + const exitCode = await new Promise((resolve, reject) => { + child.once('exit', (code) => resolve(code ?? 1)) + child.once('error', reject) + }) + return { + exitCode, + stdout: Buffer.concat(stdout).toString('utf8'), + stderr: Buffer.concat(stderr).toString('utf8') + } +} + +function parseResult(result: { exitCode: number; stdout: string; stderr: string }): T { + expect(result.exitCode, `${result.stderr}\n${result.stdout}`).toBe(0) + return JSON.parse(result.stdout) as T +} + +describeIfBuilt('Run capacity compiled CLI coordinator loop', () => { + it('reopens one slot immediately after settlement and exposes the next enrolled Task', async () => { + const userDataPath = mkdtempSync(join(tmpdir(), 'orca-capacity-cli-')) + const runtime = new OrcaRuntimeService() + const db = new OrchestrationDb(':memory:') + runtime.setOrchestrationDb(db) + vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue('tab_cli:leaf_cli') + vi.spyOn(runtime, 'getLiveTerminalPaneKey').mockReturnValue('tab_cli:leaf_cli') + const server = new OrcaRuntimeRpcServer({ runtime, userDataPath }) + await server.start() + + try { + parseResult( + await runBuiltCli(userDataPath, [ + 'orchestration', + 'run-create', + '--objective', + 'compiled capacity loop', + '--from', + 'term_cli', + '--json' + ]) + ) + parseResult( + await runBuiltCli(userDataPath, [ + 'orchestration', + 'capacity-set', + '--target', + '1', + '--from', + 'term_cli', + '--json' + ]) + ) + + const tasks: string[] = [] + for (const spec of ['first lane', 'backfill lane']) { + const created = parseResult<{ result: { task: { id: string } } }>( + await runBuiltCli(userDataPath, [ + 'orchestration', + 'task-create', + '--spec', + spec, + '--from', + 'term_cli', + '--json' + ]) + ) + tasks.push(created.result.task.id) + parseResult( + await runBuiltCli(userDataPath, [ + 'orchestration', + 'capacity-enroll', + '--task', + created.result.task.id, + '--from', + 'term_cli', + '--json' + ]) + ) + } + + const before = parseResult<{ + result: { capacity: { availableSlots: number; launchableTasks: { id: string }[] } } + }>( + await runBuiltCli(userDataPath, [ + 'orchestration', + 'capacity-show', + '--from', + 'term_cli', + '--json' + ]) + ) + expect(before.result.capacity).toMatchObject({ + availableSlots: 1, + launchableTasks: [{ id: tasks[0] }] + }) + + const started = db.createStartingWorkerDispatch({ + taskId: tasks[0]!, + startOptions: {}, + creator: { kind: 'system' }, + maxDepth: 1, + capacitySlot: true + }) + db.prepareStartingWorkerAuthority({ + dispatchId: started.dispatch.id, + handle: 'term_worker', + paneKey: 'tab_worker:leaf_worker', + processIncarnation: 'pid:worker', + worktreeId: 'repo::worktree', + effects: [], + setupState: 'not_applicable', + terminalOwnership: 'created' + }) + db.markWorkerDispatchReady(started.dispatch.id) + + const full = parseResult<{ + result: { capacity: { activeCount: number; availableSlots: number } } + }>( + await runBuiltCli(userDataPath, [ + 'orchestration', + 'capacity-show', + '--from', + 'term_cli', + '--json' + ]) + ) + expect(full.result.capacity).toMatchObject({ activeCount: 1, availableSlots: 0 }) + + db.settleWorkerReport({ + taskId: tasks[0]!, + dispatchId: started.dispatch.id, + outcome: 'succeeded', + result: '{}' + }) + const reopened = parseResult<{ + result: { + capacity: { + activeCount: number + availableSlots: number + launchableTasks: { id: string }[] + settledTerminalDebt: { dispatchId: string; terminalState: string }[] + } + } + }>( + await runBuiltCli(userDataPath, [ + 'orchestration', + 'capacity-show', + '--from', + 'term_cli', + '--json' + ]) + ) + expect(reopened.result.capacity).toMatchObject({ + activeCount: 0, + availableSlots: 1, + launchableTasks: [{ id: tasks[1] }], + settledTerminalDebt: [{ dispatchId: started.dispatch.id, terminalState: 'reclaimable' }] + }) + } finally { + db.close() + await server.stop() + rmSync(userDataPath, { recursive: true, force: true }) + } + }, 30_000) +}) diff --git a/src/main/runtime/orchestration/db/attach-orchestration-db-methods.ts b/src/main/runtime/orchestration/db/attach-orchestration-db-methods.ts index 69f12d9bbd6..1d935835062 100644 --- a/src/main/runtime/orchestration/db/attach-orchestration-db-methods.ts +++ b/src/main/runtime/orchestration/db/attach-orchestration-db-methods.ts @@ -37,6 +37,7 @@ import { attachRunCoordinatorMailRouting } from './runs/run-coordinator-mail-rou import { attachRunCreate } from './runs/run-create' import { attachRunDelivery } from './runs/run-delivery' import { attachRunLookup } from './runs/run-lookup' +import { attachRunCapacityStore } from './run-capacity/run-capacity-store' import { attachLegacyCoordinatorMailTakeover } from './runs/legacy-coordinator-mail-takeover' import { attachAdoptLegacyRun } from './schema/adopt-legacy-run' import { attachBackfillLegacyQuestionThreads } from './schema/backfill-legacy-question-threads' @@ -81,6 +82,7 @@ export function attachOrchestrationDbMethods(ctor: { prototype: object }): void attachRunCreate(ctor) attachRunBinding(ctor) attachRunLookup(ctor) + attachRunCapacityStore(ctor) attachRunCoordinatorMailRouting(ctor) attachLegacyCoordinatorMailTakeover(ctor) attachRunDelivery(ctor) diff --git a/src/main/runtime/orchestration/db/contract-constants.ts b/src/main/runtime/orchestration/db/contract-constants.ts index 56390138f0a..c970a366d64 100644 --- a/src/main/runtime/orchestration/db/contract-constants.ts +++ b/src/main/runtime/orchestration/db/contract-constants.ts @@ -6,5 +6,5 @@ export const LEGACY_RUN_ID = ORCHESTRATION_LEGACY_RUN_ID export const LEGACY_CONTRACT_VERSION = 0 export const CURRENT_CONTRACT_VERSION = ORCHESTRATION_CONTRACT_VERSION -// Schema versions: v2 'heartbeat'+last_heartbeat_at, v3 delivered_at, v4 task-creator terminal, v5 task_title/display_name, v6 pane identity, v7 lightweight Runs, v8 crash-safe Run deliveries, v9 durable question threads, v10 Dispatch capabilities, v11 durable mutation receipts, v12 composed worker state, v18 post-v6 version-skew repair, v19 adopted legacy Runs and compatibility receipts, v20 legacy question backfill, v21 legacy scheduler-loss provenance, v22 dispatch assignee lookup, v23 worker terminal resource ownership, v24 creator-incarnation authority, v25 active Dispatch handle lookup, v26 indexed mutation receipt capacity, v27 durable federation acknowledgments, v28 durable local mutation caller identity. -export const SCHEMA_VERSION = 30 +// Schema versions: v2 'heartbeat'+last_heartbeat_at, v3 delivered_at, v4 task-creator terminal, v5 task_title/display_name, v6 pane identity, v7 lightweight Runs, v8 crash-safe Run deliveries, v9 durable question threads, v10 Dispatch capabilities, v11 durable mutation receipts, v12 composed worker state, v18 post-v6 version-skew repair, v19 adopted legacy Runs and compatibility receipts, v20 legacy question backfill, v21 legacy scheduler-loss provenance, v22 dispatch assignee lookup, v23 worker terminal resource ownership, v24 creator-incarnation authority, v25 active Dispatch handle lookup, v26 indexed mutation receipt capacity, v27 durable federation acknowledgments, v28 durable local mutation caller identity, v29 termination provenance, v30 dispatch depth, v31 Run capacity targets. +export const SCHEMA_VERSION = 31 diff --git a/src/main/runtime/orchestration/db/dispatch-context/dispatch-context-store.ts b/src/main/runtime/orchestration/db/dispatch-context/dispatch-context-store.ts index cd4083acc0a..f181627f3d8 100644 --- a/src/main/runtime/orchestration/db/dispatch-context/dispatch-context-store.ts +++ b/src/main/runtime/orchestration/db/dispatch-context/dispatch-context-store.ts @@ -72,6 +72,7 @@ export function createDispatchContext( `Terminal ${assigneeHandle} already has an active dispatch (${occupied.id} for task ${occupied.task_id})` ) } + this.assertRunCapacityAvailable(taskId) throw new Error( `Task ${taskId} is ${current?.status ?? 'missing'}; only ready tasks can be dispatched` ) diff --git a/src/main/runtime/orchestration/db/dispatch-row-writer.ts b/src/main/runtime/orchestration/db/dispatch-row-writer.ts index 606081e58a3..c5552bf6b34 100644 --- a/src/main/runtime/orchestration/db/dispatch-row-writer.ts +++ b/src/main/runtime/orchestration/db/dispatch-row-writer.ts @@ -20,6 +20,17 @@ export const DISPATCH_CONTEXT_CLAIM_SQL = `INSERT INTO dispatch_contexts ( SELECT ?, run_id, id, ?, ?, ?, ?, ?, 'dispatched', ?, ?, datetime('now') FROM tasks WHERE id = ? AND status = 'ready' + AND NOT EXISTS ( + SELECT 1 FROM runs + WHERE runs.id = tasks.run_id + AND runs.legacy = 0 + AND runs.target_concurrency > 0 + AND ( + SELECT COUNT(*) FROM dispatch_contexts active + WHERE active.run_id = tasks.run_id + AND active.status IN ('pending', 'dispatched') + ) >= runs.target_concurrency + ) AND NOT EXISTS ( SELECT 1 FROM dispatch_contexts active WHERE active.assignee_handle = ? @@ -44,7 +55,19 @@ WHERE id = ? AND status = 'ready' const STARTING_DISPATCH_CONTEXT_SQL = `INSERT INTO dispatch_contexts ( id, run_id, task_id, contract_version, launch_token_hash, depth, status, dispatched_at - ) VALUES (?, ?, ?, ?, ?, ?, 'pending', datetime('now'))` + ) + SELECT ?, ?, ?, ?, ?, ?, 'pending', datetime('now') + WHERE NOT EXISTS ( + SELECT 1 FROM runs + WHERE runs.id = ? + AND runs.legacy = 0 + AND runs.target_concurrency > 0 + AND ( + SELECT COUNT(*) FROM dispatch_contexts active + WHERE active.run_id = ? + AND active.status IN ('pending', 'dispatched') + ) >= runs.target_concurrency + )` const REMOTE_DISPATCH_ATTACHMENT_SQL = `INSERT INTO remote_dispatch_attachments ( dispatch_id, task_id, home_peer_fingerprint, protocol_version, runtime_epoch, depth @@ -109,14 +132,21 @@ export function insertStartingDispatchContextRow( } ): void { assertStampedDepth(params.depth) - db.prepare(STARTING_DISPATCH_CONTEXT_SQL).run( - params.id, - params.runId, - params.taskId, - params.contractVersion, - params.launchTokenHash, - params.depth - ) + const inserted = db + .prepare(STARTING_DISPATCH_CONTEXT_SQL) + .run( + params.id, + params.runId, + params.taskId, + params.contractVersion, + params.launchTokenHash, + params.depth, + params.runId, + params.runId + ) + if (inserted.changes !== 1) { + throw new Error(`Run capacity changed while starting Task ${params.taskId}.`) + } } /** The worker host's record of a live worker driven by a remote Run home. */ diff --git a/src/main/runtime/orchestration/db/orchestration-db-methods.ts b/src/main/runtime/orchestration/db/orchestration-db-methods.ts index 7f25b209c54..6b3a5bed444 100644 --- a/src/main/runtime/orchestration/db/orchestration-db-methods.ts +++ b/src/main/runtime/orchestration/db/orchestration-db-methods.ts @@ -37,6 +37,7 @@ import type { RunCoordinatorMailRoutingMethods } from './runs/run-coordinator-ma import type { RunCreateMethods } from './runs/run-create' import type { RunDeliveryMethods } from './runs/run-delivery' import type { RunLookupMethods } from './runs/run-lookup' +import type { RunCapacityStoreMethods } from './run-capacity/run-capacity-store' import type { LegacyCoordinatorMailTakeoverMethods } from './runs/legacy-coordinator-mail-takeover' import type { AdoptLegacyRunMethods } from './schema/adopt-legacy-run' import type { BackfillLegacyQuestionThreadsMethods } from './schema/backfill-legacy-question-threads' @@ -80,6 +81,7 @@ export type OrchestrationDbMethods = CreateTablesMethods & RunCreateMethods & RunBindingMethods & RunLookupMethods & + RunCapacityStoreMethods & RunCoordinatorMailRoutingMethods & LegacyCoordinatorMailTakeoverMethods & RunDeliveryMethods & diff --git a/src/main/runtime/orchestration/db/run-capacity/run-capacity-store.ts b/src/main/runtime/orchestration/db/run-capacity/run-capacity-store.ts new file mode 100644 index 00000000000..b5cb46e44ad --- /dev/null +++ b/src/main/runtime/orchestration/db/run-capacity/run-capacity-store.ts @@ -0,0 +1,223 @@ +import { OrchestrationError } from '../../orchestration-error' +import type { TaskRow } from '../../types' +import { + deriveWorkerTerminalListState, + type WorkerDispatchListState, + type WorkerTerminalListState, + type WorkerTerminalResourceRow +} from '../../worker-terminal-ownership' +import type { OrchestrationDb } from '../orchestration-db' + +const MAX_TARGET_CONCURRENCY = 64 + +export type RunCapacitySnapshot = { + runId: string + targetConcurrency: number + activeCount: number + availableSlots: number + launchableCount: number + launchableTasks: TaskRow[] + eligiblePendingTaskIds: string[] + settledTerminalDebt: { + dispatchId: string + terminalState: WorkerTerminalListState + retainedReason: string | null + }[] +} + +export function configureRunCapacity( + this: OrchestrationDb, + runId: string, + targetConcurrency: number +): RunCapacitySnapshot { + if ( + !Number.isInteger(targetConcurrency) || + targetConcurrency < 0 || + targetConcurrency > MAX_TARGET_CONCURRENCY + ) { + throw new OrchestrationError( + 'invalid_argument', + `Target concurrency must be an integer between 0 and ${MAX_TARGET_CONCURRENCY}.` + ) + } + const updated = this.db + .prepare( + "UPDATE runs SET target_concurrency = ?, updated_at = datetime('now') WHERE id = ? AND legacy = 0" + ) + .run(targetConcurrency, runId) + if (updated.changes !== 1) { + throw new OrchestrationError('run_not_found', `Run ${runId} was not found or is inspect-only.`) + } + return this.getRunCapacity(runId) +} + +export function setTaskCapacityEligibility( + this: OrchestrationDb, + taskId: string, + eligible: boolean +): TaskRow { + const updated = this.db + .prepare('UPDATE tasks SET capacity_eligible = ? WHERE id = ?') + .run(eligible ? 1 : 0, taskId) + if (updated.changes !== 1) { + throw new OrchestrationError('task_not_found', `Task ${taskId} was not found.`) + } + return this.getTask(taskId) as TaskRow +} + +export function assertRunCapacityAvailable( + this: OrchestrationDb, + taskId: string, + requireEnrollment = false +): void { + const task = this.getTask(taskId) + if (!task) { + throw new OrchestrationError('task_not_found', `Task ${taskId} was not found.`) + } + if (requireEnrollment && task.capacity_eligible !== 1) { + throw new OrchestrationError( + 'task_not_startable', + `Task ${task.id} is not enrolled in its Run capacity pool.` + ) + } + const run = this.getRunRaw(task.run_id) + if (!run || run.legacy === 1) { + if (!requireEnrollment) { + return + } + throw new OrchestrationError( + 'capacity_not_configured', + `Run ${task.run_id} cannot claim a capacity slot.` + ) + } + if (run.target_concurrency < 1) { + if (!requireEnrollment) { + return + } + throw new OrchestrationError( + 'capacity_not_configured', + `Run ${run.id} has no target concurrency; configure it before claiming a capacity slot.` + ) + } + const row = this.db + .prepare( + `SELECT COUNT(*) AS active_count + FROM dispatch_contexts + WHERE run_id = ? AND status IN ('pending', 'dispatched')` + ) + .get(run.id) as { active_count: number } + if (row.active_count >= run.target_concurrency) { + throw new OrchestrationError( + 'capacity_full', + `Run ${run.id} already has ${row.active_count} active lanes at target concurrency ${run.target_concurrency}.`, + { runId: run.id, activeCount: row.active_count, targetConcurrency: run.target_concurrency } + ) + } +} + +export function assertRunCapacitySlot(this: OrchestrationDb, taskId: string): void { + this.assertRunCapacityAvailable(taskId, true) +} + +export function getRunCapacity(this: OrchestrationDb, runId: string): RunCapacitySnapshot { + const run = this.getRunRaw(runId) + if (!run || run.legacy === 1) { + throw new OrchestrationError('run_not_found', `Run ${runId} was not found or is inspect-only.`) + } + const active = this.db + .prepare( + `SELECT COUNT(*) AS active_count + FROM dispatch_contexts + WHERE run_id = ? AND status IN ('pending', 'dispatched')` + ) + .get(runId) as { active_count: number } + const availableSlots = Math.max(0, run.target_concurrency - active.active_count) + const launchableTasks = this.db + .prepare( + `SELECT tasks.* FROM tasks + WHERE run_id = ? AND capacity_eligible = 1 AND status = 'ready' + AND NOT EXISTS ( + SELECT 1 FROM dispatch_contexts + WHERE dispatch_contexts.task_id = tasks.id + AND dispatch_contexts.status IN ('pending', 'dispatched') + ) + AND NOT EXISTS ( + SELECT 1 FROM decision_gates + WHERE decision_gates.task_id = tasks.id AND decision_gates.status = 'pending' + ) + ORDER BY created_at, rowid + LIMIT ?` + ) + .all(runId, availableSlots) as TaskRow[] + const eligiblePendingTaskIds = ( + this.db + .prepare( + `SELECT id FROM tasks + WHERE run_id = ? AND capacity_eligible = 1 AND status IN ('pending', 'blocked') + ORDER BY created_at, rowid` + ) + .all(runId) as { id: string }[] + ).map((task) => task.id) + const debtRows = this.db + .prepare( + `SELECT r.*, d.id AS dispatch_id, + COALESCE(w.state, 'unsupervised') AS worker_state, + COALESCE(w.agent_terminal_handle, d.assignee_handle) AS agent_terminal_handle + FROM dispatch_contexts d + JOIN worker_terminal_resources r ON r.owner_dispatch_id = d.id + LEFT JOIN worker_dispatches w ON w.dispatch_id = d.id + WHERE d.run_id = ? + AND d.status IN ('completed', 'failed', 'circuit_broken') + AND r.release_state <> 'released' + ORDER BY COALESCE(w.created_at, d.created_at), d.rowid` + ) + .all(runId) as (WorkerTerminalResourceRow & { + dispatch_id: string + worker_state: WorkerDispatchListState + agent_terminal_handle: string | null + })[] + const settledTerminalDebt = debtRows.flatMap((row) => { + const terminalState = deriveWorkerTerminalListState({ + workerState: row.worker_state, + agentTerminalHandle: row.agent_terminal_handle, + resource: row + }) + return terminalState && terminalState !== 'released' + ? [ + { + dispatchId: row.dispatch_id, + terminalState, + retainedReason: row.retained_reason + } + ] + : [] + }) + return { + runId, + targetConcurrency: run.target_concurrency, + activeCount: active.active_count, + availableSlots, + launchableCount: launchableTasks.length, + launchableTasks, + eligiblePendingTaskIds, + settledTerminalDebt + } +} + +export type RunCapacityStoreMethods = { + configureRunCapacity: typeof configureRunCapacity + setTaskCapacityEligibility: typeof setTaskCapacityEligibility + assertRunCapacityAvailable: typeof assertRunCapacityAvailable + assertRunCapacitySlot: typeof assertRunCapacitySlot + getRunCapacity: typeof getRunCapacity +} + +export function attachRunCapacityStore(ctor: { prototype: object }): void { + Object.assign(ctor.prototype, { + configureRunCapacity, + setTaskCapacityEligibility, + assertRunCapacityAvailable, + assertRunCapacitySlot, + getRunCapacity + }) +} diff --git a/src/main/runtime/orchestration/db/schema/create-core-tables-sql.ts b/src/main/runtime/orchestration/db/schema/create-core-tables-sql.ts index 1b3172edf31..cae854e35f9 100644 --- a/src/main/runtime/orchestration/db/schema/create-core-tables-sql.ts +++ b/src/main/runtime/orchestration/db/schema/create-core-tables-sql.ts @@ -10,6 +10,7 @@ CREATE TABLE IF NOT EXISTS runs ( coordinator_pane_key TEXT, consumer_generation INTEGER NOT NULL DEFAULT 0, legacy INTEGER NOT NULL DEFAULT 0, + target_concurrency INTEGER NOT NULL DEFAULT 0 CHECK(target_concurrency BETWEEN 0 AND 64), created_at TEXT NOT NULL DEFAULT (datetime('now')), updated_at TEXT NOT NULL DEFAULT (datetime('now')) ); diff --git a/src/main/runtime/orchestration/db/schema/create-graph-tables-sql.ts b/src/main/runtime/orchestration/db/schema/create-graph-tables-sql.ts index 07a47c3c80c..61ac03ff58e 100644 --- a/src/main/runtime/orchestration/db/schema/create-graph-tables-sql.ts +++ b/src/main/runtime/orchestration/db/schema/create-graph-tables-sql.ts @@ -108,6 +108,7 @@ CREATE TABLE IF NOT EXISTS tasks ( 'pending', 'ready', 'dispatched', 'completed', 'failed', 'blocked' )), + capacity_eligible INTEGER NOT NULL DEFAULT 0 CHECK(capacity_eligible IN (0, 1)), deps TEXT NOT NULL DEFAULT '[]', result TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now')), diff --git a/src/main/runtime/orchestration/db/schema/migrate-v31.ts b/src/main/runtime/orchestration/db/schema/migrate-v31.ts new file mode 100644 index 00000000000..19e1274264a --- /dev/null +++ b/src/main/runtime/orchestration/db/schema/migrate-v31.ts @@ -0,0 +1,21 @@ +import type { OrchestrationDb } from '../orchestration-db' + +export function applySchemaMigrationV31(this: OrchestrationDb, current: number): void { + if (current >= 31) { + return + } + if (!this.hasColumn('runs', 'target_concurrency')) { + this.db.exec( + 'ALTER TABLE runs ADD COLUMN target_concurrency INTEGER NOT NULL DEFAULT 0 CHECK(target_concurrency BETWEEN 0 AND 64)' + ) + } + if (!this.hasColumn('tasks', 'capacity_eligible')) { + this.db.exec( + 'ALTER TABLE tasks ADD COLUMN capacity_eligible INTEGER NOT NULL DEFAULT 0 CHECK(capacity_eligible IN (0, 1))' + ) + } + this.db.exec(` + CREATE INDEX IF NOT EXISTS idx_tasks_capacity_ready + ON tasks(run_id, capacity_eligible, status, created_at); + `) +} diff --git a/src/main/runtime/orchestration/db/schema/migrate.ts b/src/main/runtime/orchestration/db/schema/migrate.ts index b29debf6aa1..71d6923b465 100644 --- a/src/main/runtime/orchestration/db/schema/migrate.ts +++ b/src/main/runtime/orchestration/db/schema/migrate.ts @@ -3,6 +3,7 @@ import { SCHEMA_VERSION } from '../contract-constants' import type { OrchestrationDb } from '../orchestration-db' import { applySchemaMigrationsV13ToV30 } from './migrate-v13-v30' import { applySchemaMigrationsV2ToV12 } from './migrate-v2-v12' +import { applySchemaMigrationV31 } from './migrate-v31' // Why: CREATE TABLE IF NOT EXISTS won't alter existing DBs; migrate in a txn that bumps user_version only on success (atomic all-or-nothing). export function migrate(this: OrchestrationDb): void { @@ -16,6 +17,7 @@ export function migrate(this: OrchestrationDb): void { try { applySchemaMigrationsV2ToV12.call(this, current) applySchemaMigrationsV13ToV30.call(this, current) + applySchemaMigrationV31.call(this, current) this.db.pragma(`user_version = ${SCHEMA_VERSION}`) this.db.exec('COMMIT') } catch (err) { diff --git a/src/main/runtime/orchestration/db/tasks/task-store.ts b/src/main/runtime/orchestration/db/tasks/task-store.ts index 9a0b15259ba..2694d61c712 100644 --- a/src/main/runtime/orchestration/db/tasks/task-store.ts +++ b/src/main/runtime/orchestration/db/tasks/task-store.ts @@ -21,6 +21,7 @@ export function createTask( createdByProcessIncarnation?: string createdByRunGeneration?: number runId?: string + capacityEligible?: boolean } ): TaskRow { const runId = task.runId ?? LEGACY_RUN_ID @@ -49,7 +50,7 @@ export function createTask( `INSERT INTO tasks ( id, run_id, parent_id, created_by_terminal_handle, created_by_pane_key, created_by_process_incarnation, created_by_run_generation, - task_title, display_name, spec, status, deps + task_title, display_name, spec, status, capacity_eligible, deps ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CASE WHEN EXISTS ( @@ -59,7 +60,7 @@ export function createTask( WHERE dependency.id IS NULL OR dependency.run_id <> ? OR dependency.status <> 'completed' - ) THEN 'pending' ELSE 'ready' END, + ) THEN 'pending' ELSE 'ready' END, ?, ? )` ) @@ -76,6 +77,7 @@ export function createTask( task.spec, depsJson, runId, + task.capacityEligible ? 1 : 0, depsJson ) return this.db.prepare('SELECT * FROM tasks WHERE id = ?').get(id) as TaskRow diff --git a/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-start.ts b/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-start.ts index e26369e7fc1..56b43c56584 100644 --- a/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-start.ts +++ b/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-start.ts @@ -30,6 +30,7 @@ export function createStartingWorkerDispatch( /** Who is dispatching, for nesting depth. Required so a new caller must decide. */ creator: DispatchCreator maxDepth: number + capacitySlot?: boolean } ): { dispatch: DispatchContextRow; worker: WorkerDispatchRow } { this.db.exec('BEGIN IMMEDIATE') @@ -85,6 +86,7 @@ export function createStartingWorkerDispatch( `Task ${task.id} is ${task.status}; only a ready Task can start.` ) } + this.assertRunCapacityAvailable(task.id, params.capacitySlot === true) const id = generateId('ctx') if (params.mutationReceipt) { diff --git a/src/main/runtime/orchestration/orchestration-schema-version-skew.ts b/src/main/runtime/orchestration/orchestration-schema-version-skew.ts index 95ecbe6b034..5c3cf0917b7 100644 --- a/src/main/runtime/orchestration/orchestration-schema-version-skew.ts +++ b/src/main/runtime/orchestration/orchestration-schema-version-skew.ts @@ -28,9 +28,13 @@ const POST_V6_COLUMNS = [ const VERSIONED_POST_V6_COLUMNS = [ { version: 27, table: 'federated_dispatches', column: 'to_home_acknowledged_sequence' }, { version: 30, table: 'dispatch_contexts', column: 'depth' }, - { version: 30, table: 'remote_dispatch_attachments', column: 'depth' } + { version: 30, table: 'remote_dispatch_attachments', column: 'depth' }, + { version: 31, table: 'runs', column: 'target_concurrency' }, + { version: 31, table: 'tasks', column: 'capacity_eligible' } ] as const +const VERSIONED_POST_V6_INDEXES = [{ version: 31, index: 'idx_tasks_capacity_ready' }] as const + const POST_V6_INDEXES = [ 'idx_messages_run_sequence', 'idx_messages_delivery_contract', @@ -95,6 +99,9 @@ function hasCompletePostV6Schema(db: Database.Database, storedVersion: number): ({ version, table, column }) => storedVersion < version || hasOrchestrationColumn(db, table, column) ) && + VERSIONED_POST_V6_INDEXES.every( + ({ version, index }) => storedVersion < version || hasOrchestrationIndex(db, index) + ) && POST_V6_INDEXES.every((index) => hasOrchestrationIndex(db, index)) && messagesAllowQuestions(db) && hasConsistentLegacyAdoption(db) diff --git a/src/main/runtime/orchestration/run-capacity-migration.test.ts b/src/main/runtime/orchestration/run-capacity-migration.test.ts new file mode 100644 index 00000000000..e3df3cca2a1 --- /dev/null +++ b/src/main/runtime/orchestration/run-capacity-migration.test.ts @@ -0,0 +1,89 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import Database from '../../sqlite/sync-database' +import { OrchestrationDb } from './db' +import { SCHEMA_VERSION } from './db/contract-constants' +import { resolveOrchestrationMigrationStartVersion } from './orchestration-schema-version-skew' + +describe('Run capacity migration (v31)', () => { + let db: OrchestrationDb | undefined + let tempDir: string | undefined + + afterEach(() => { + db?.close() + if (tempDir) { + rmSync(tempDir, { recursive: true, force: true }) + } + }) + + function createV30Database(): { dbPath: string; runId: string; taskId: string } { + tempDir = mkdtempSync(join(tmpdir(), 'orca-run-capacity-migration-')) + const dbPath = join(tempDir, 'orchestration.db') + const current = new OrchestrationDb(dbPath) + const run = current.createRun({ + objective: 'upgrade capacity defaults', + coordinatorHandle: 'term_coord', + coordinatorPaneKey: 'repo:worktree:tab:leaf' + }) + const task = current.createTask({ spec: 'pre-capacity task', runId: run.id }) + current.close() + + const oldDb = new Database(dbPath) + oldDb.exec('DROP INDEX idx_tasks_capacity_ready') + oldDb.exec('ALTER TABLE tasks DROP COLUMN capacity_eligible') + oldDb.exec('ALTER TABLE runs DROP COLUMN target_concurrency') + oldDb.pragma('user_version = 30') + oldDb.close() + return { dbPath, runId: run.id, taskId: task.id } + } + + it('adds disabled targets and unenrolled Tasks when upgrading v30', () => { + const { dbPath, runId, taskId } = createV30Database() + db = new OrchestrationDb(dbPath) + const sqlite = (db as unknown as { db: Database.Database }).db + + expect(sqlite.pragma('user_version', { simple: true })).toBe(SCHEMA_VERSION) + expect(db.getRun(runId)).toMatchObject({ target_concurrency: 0 }) + expect(db.getTask(taskId)).toMatchObject({ capacity_eligible: 0 }) + expect(db.getRunCapacity(runId)).toMatchObject({ + targetConcurrency: 0, + activeCount: 0, + availableSlots: 0, + launchableCount: 0 + }) + }) + + it('repairs a database that claims v31 without the v31 columns', () => { + const { dbPath } = createV30Database() + const malformed = new Database(dbPath) + malformed.pragma('user_version = 31') + + expect(resolveOrchestrationMigrationStartVersion(malformed, 31, SCHEMA_VERSION)).toBe(6) + malformed.close() + + db = new OrchestrationDb(dbPath) + expect(db.db.pragma('user_version', { simple: true })).toBe(SCHEMA_VERSION) + }) + + it('repairs a database that claims v31 without the capacity-ready index', () => { + tempDir = mkdtempSync(join(tmpdir(), 'orca-run-capacity-index-repair-')) + const dbPath = join(tempDir, 'orchestration.db') + db = new OrchestrationDb(dbPath) + db.close() + db = undefined + + const malformed = new Database(dbPath) + malformed.exec('DROP INDEX idx_tasks_capacity_ready') + expect(resolveOrchestrationMigrationStartVersion(malformed, 31, SCHEMA_VERSION)).toBe(6) + malformed.close() + + db = new OrchestrationDb(dbPath) + expect( + db.db + .prepare("SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = ?") + .get('idx_tasks_capacity_ready') + ).toBeDefined() + }) +}) diff --git a/src/main/runtime/orchestration/run-capacity.test.ts b/src/main/runtime/orchestration/run-capacity.test.ts new file mode 100644 index 00000000000..7fed2892acc --- /dev/null +++ b/src/main/runtime/orchestration/run-capacity.test.ts @@ -0,0 +1,391 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { OrchestrationDb } from './db' +import { createRootDispatch } from './db/root-dispatch-test-fixture' + +describe('orchestration run capacity', () => { + let db: OrchestrationDb | undefined + let tempDirectory: string | undefined + + afterEach(() => { + vi.restoreAllMocks() + db?.close() + if (tempDirectory) { + rmSync(tempDirectory, { recursive: true, force: true }) + } + }) + + function setup(targetConcurrency = 2): { runId: string } { + db = new OrchestrationDb(':memory:') + const run = db.createRun({ + objective: 'keep an explicitly enrolled wave full', + coordinatorHandle: 'term_coordinator', + coordinatorPaneKey: 'repo:worktree:tab:leaf' + }) + db.configureRunCapacity(run.id, targetConcurrency) + return { runId: run.id } + } + + function startAndSettle(taskId: string, terminalHandle: string): string { + const started = db!.createStartingWorkerDispatch({ + taskId, + startOptions: {}, + creator: { kind: 'system' }, + maxDepth: 1, + capacitySlot: true + }) + db!.prepareStartingWorkerAuthority({ + dispatchId: started.dispatch.id, + handle: terminalHandle, + paneKey: `repo:worktree:tab:${terminalHandle}`, + processIncarnation: `pid:${terminalHandle}`, + worktreeId: 'repo::worktree', + effects: [], + setupState: 'not_applicable', + terminalOwnership: 'created' + }) + db!.markWorkerDispatchReady(started.dispatch.id) + db!.settleWorkerReport({ + taskId, + dispatchId: started.dispatch.id, + outcome: 'succeeded', + result: '{}' + }) + return started.dispatch.id + } + + it('exposes only enrolled ready tasks as launchable capacity', () => { + const { runId } = setup(3) + const active = db!.createTask({ spec: 'active', runId, capacityEligible: true }) + createRootDispatch(db!, active.id, 'term_active') + const ready = db!.createTask({ spec: 'ready', runId, capacityEligible: true }) + const unproven = db!.createTask({ spec: 'not enrolled', runId }) + const dependency = db!.createTask({ spec: 'dependency', runId }) + const pending = db!.createTask({ + spec: 'blocked on dependency', + runId, + deps: [dependency.id], + capacityEligible: true + }) + + const snapshot = db!.getRunCapacity(runId) + + expect(snapshot).toMatchObject({ + runId, + targetConcurrency: 3, + activeCount: 1, + availableSlots: 2, + launchableCount: 1 + }) + expect(snapshot.launchableTasks.map((task) => task.id)).toEqual([ready.id]) + expect(snapshot.eligiblePendingTaskIds).toEqual([pending.id]) + expect(snapshot.launchableTasks.map((task) => task.id)).not.toContain(unproven.id) + }) + + it('does not offer a pre-fix split Task that still has an active Dispatch', () => { + const { runId } = setup(2) + const split = db!.createTask({ spec: 'legacy split', runId, capacityEligible: true }) + createRootDispatch(db!, split.id, 'term_active') + db!.db.prepare("UPDATE tasks SET status = 'ready' WHERE id = ?").run(split.id) + + expect(db!.getRunCapacity(runId).launchableTasks).toEqual([]) + }) + + it('does not offer a Task while a decision gate remains pending', () => { + const { runId } = setup(1) + const task = db!.createTask({ spec: 'gated work', runId, capacityEligible: true }) + const gate = db!.createGate({ taskId: task.id, question: 'Proceed?' }) + db!.db.prepare("UPDATE tasks SET status = 'ready' WHERE id = ?").run(task.id) + + expect(db!.getRunCapacity(runId).launchableTasks).toEqual([]) + db!.resolveGate(gate.id, 'yes') + expect(db!.getRunCapacity(runId).launchableTasks.map((candidate) => candidate.id)).toEqual([ + task.id + ]) + }) + + it('bounds the indexed launchable query by the available slots', () => { + const { runId } = setup(3) + for (let index = 0; index < 200; index += 1) { + db!.createTask({ spec: `ready ${index}`, runId, capacityEligible: true }) + } + const prepare = db!.db.prepare.bind(db!.db) + let launchableSql = '' + vi.spyOn(db!.db, 'prepare').mockImplementation((sql) => { + if (sql.includes('SELECT tasks.* FROM tasks')) { + launchableSql = sql + } + return prepare(sql) + }) + + expect(db!.getRunCapacity(runId).launchableTasks).toHaveLength(3) + expect(launchableSql).toContain('LIMIT ?') + const plan = prepare(`EXPLAIN QUERY PLAN ${launchableSql}`).all(runId, 3) as { + detail: string + }[] + expect(plan.some((row) => row.detail.includes('idx_tasks_capacity_ready'))).toBe(true) + }) + + it('atomically refuses a capacity start above the exact target', () => { + const { runId } = setup(1) + const first = db!.createTask({ spec: 'first', runId, capacityEligible: true }) + const second = db!.createTask({ spec: 'second', runId, capacityEligible: true }) + + db!.createStartingWorkerDispatch({ + taskId: first.id, + startOptions: {}, + creator: { kind: 'system' }, + maxDepth: 1, + capacitySlot: true + }) + + expect(() => + db!.createStartingWorkerDispatch({ + taskId: second.id, + startOptions: {}, + creator: { kind: 'system' }, + maxDepth: 1, + capacitySlot: true + }) + ).toThrowError(/target concurrency 1/) + expect(db!.getTask(second.id)?.status).toBe('ready') + expect(() => createRootDispatch(db!, second.id, 'term_bypass')).toThrowError( + /target concurrency 1/ + ) + }) + + it('claims federated capacity at the authoritative Run home', () => { + const { runId } = setup(1) + const first = db!.createTask({ spec: 'first remote', runId, capacityEligible: true }) + const second = db!.createTask({ spec: 'second remote', runId, capacityEligible: true }) + const federation = { + environmentId: 'environment_windows', + environmentName: 'windows', + peerFingerprint: 'windows_peer', + protocolVersion: 3 + } + + db!.createStartingWorkerDispatch({ + taskId: first.id, + startOptions: {}, + creator: { kind: 'system' }, + maxDepth: 1, + capacitySlot: true, + federation + }) + + expect(() => + db!.createStartingWorkerDispatch({ + taskId: second.id, + startOptions: {}, + creator: { kind: 'system' }, + maxDepth: 1, + capacitySlot: true, + federation + }) + ).toThrowError(/target concurrency 1/) + }) + + it('serializes simultaneous low-level Dispatch claims across database connections', () => { + tempDirectory = mkdtempSync(join(tmpdir(), 'orca-run-capacity-race-')) + const databasePath = join(tempDirectory, 'orchestration.db') + db = new OrchestrationDb(databasePath) + const concurrent = new OrchestrationDb(databasePath) + try { + const run = db.createRun({ + objective: 'serialize low-level claims', + coordinatorHandle: 'term_coord', + coordinatorPaneKey: 'repo:worktree:tab:leaf' + }) + db.configureRunCapacity(run.id, 1) + const loser = db.createTask({ spec: 'losing claim', runId: run.id, capacityEligible: true }) + const winner = db.createTask({ spec: 'winning claim', runId: run.id, capacityEligible: true }) + const prepare = db.db.prepare.bind(db.db) + let winnerDispatchId: string | undefined + vi.spyOn(db.db, 'prepare').mockImplementation((sql) => { + if (!winnerDispatchId && sql.includes('INSERT INTO dispatch_contexts')) { + winnerDispatchId = createRootDispatch(concurrent, winner.id, 'term_winner').id + } + return prepare(sql) + }) + + expect(() => createRootDispatch(db!, loser.id, 'term_loser')).toThrowError( + /target concurrency 1/ + ) + expect(winnerDispatchId).toBeDefined() + expect(db.getTask(loser.id)?.status).toBe('ready') + expect(db.getRunCapacity(run.id)).toMatchObject({ activeCount: 1, availableSlots: 0 }) + } finally { + concurrent.close() + } + }) + + it('requires explicit enrollment for capacity starts', () => { + const { runId } = setup(1) + const task = db!.createTask({ spec: 'ordinary ready task', runId }) + + expect(() => + db!.createStartingWorkerDispatch({ + taskId: task.id, + startOptions: {}, + creator: { kind: 'system' }, + maxDepth: 1, + capacitySlot: true + }) + ).toThrowError(/not enrolled/) + }) + + it('reports settled terminal debt separately from live lane capacity', () => { + const { runId } = setup(2) + const task = db!.createTask({ spec: 'settled worker', runId, capacityEligible: true }) + const dispatchId = startAndSettle(task.id, 'term_worker') + + const snapshot = db!.getRunCapacity(runId) + + expect(snapshot.activeCount).toBe(0) + expect(snapshot.availableSlots).toBe(2) + expect(snapshot.settledTerminalDebt).toEqual([ + expect.objectContaining({ dispatchId, terminalState: 'reclaimable' }) + ]) + }) + + it('recomputes slots after simultaneous settlements and dependency promotion', () => { + const { runId } = setup(2) + const first = db!.createTask({ spec: 'first active', runId, capacityEligible: true }) + const second = db!.createTask({ spec: 'second active', runId, capacityEligible: true }) + const dependent = db!.createTask({ + spec: 'newly ready replacement', + runId, + deps: [first.id], + capacityEligible: true + }) + const firstStarted = db!.createStartingWorkerDispatch({ + taskId: first.id, + startOptions: {}, + creator: { kind: 'system' }, + maxDepth: 1, + capacitySlot: true + }) + const secondStarted = db!.createStartingWorkerDispatch({ + taskId: second.id, + startOptions: {}, + creator: { kind: 'system' }, + maxDepth: 1, + capacitySlot: true + }) + for (const [task, started, terminal] of [ + [first, firstStarted, 'term_first'], + [second, secondStarted, 'term_second'] + ] as const) { + db!.prepareStartingWorkerAuthority({ + dispatchId: started.dispatch.id, + handle: terminal, + paneKey: `repo:worktree:tab:${terminal}`, + processIncarnation: `pid:${terminal}`, + worktreeId: 'repo::worktree', + effects: [], + setupState: 'not_applicable', + terminalOwnership: 'created' + }) + db!.markWorkerDispatchReady(started.dispatch.id) + db!.settleWorkerReport({ + taskId: task.id, + dispatchId: started.dispatch.id, + outcome: 'succeeded', + result: '{}' + }) + } + + const snapshot = db!.getRunCapacity(runId) + expect(snapshot.activeCount).toBe(0) + expect(snapshot.availableSlots).toBe(2) + expect(snapshot.launchableTasks.map((task) => task.id)).toEqual([dependent.id]) + }) + + it('excludes failed and blocked enrolled tasks when no eligible work remains', () => { + const { runId } = setup(2) + const failed = db!.createTask({ spec: 'failed', runId, capacityEligible: true }) + const blocked = db!.createTask({ spec: 'blocked', runId, capacityEligible: true }) + db!.updateTaskStatus(failed.id, 'failed') + db!.updateTaskStatus(blocked.id, 'blocked') + + const snapshot = db!.getRunCapacity(runId) + expect(snapshot.launchableCount).toBe(0) + expect(snapshot.eligiblePendingTaskIds).toEqual([blocked.id]) + }) + + it('stops offering an enrolled Task after its retry circuit breaks', () => { + const { runId } = setup(1) + const task = db!.createTask({ + spec: 'repeatedly failing capacity work', + runId, + capacityEligible: true + }) + + for (let attempt = 0; attempt < 3; attempt += 1) { + const dispatch = createRootDispatch(db!, task.id, `term_retry_${attempt}`) + db!.failDispatch(dispatch.id, `attempt ${attempt + 1}`) + } + + expect(db!.getTask(task.id)?.status).toBe('failed') + expect(db!.getRunCapacity(runId)).toMatchObject({ + activeCount: 0, + availableSlots: 1, + launchableCount: 0, + launchableTasks: [] + }) + }) + + it('persists target and enrollment across a database restart', () => { + tempDirectory = mkdtempSync(join(tmpdir(), 'orca-run-capacity-')) + const databasePath = join(tempDirectory, 'orchestration.db') + db = new OrchestrationDb(databasePath) + const run = db.createRun({ + objective: 'restart durable capacity', + coordinatorHandle: 'term_coordinator', + coordinatorPaneKey: 'repo:worktree:tab:leaf' + }) + const task = db.createTask({ + spec: 'durable enrollment', + runId: run.id, + capacityEligible: true + }) + db.configureRunCapacity(run.id, 5) + db.close() + db = new OrchestrationDb(databasePath) + + const snapshot = db.getRunCapacity(run.id) + expect(snapshot.targetConcurrency).toBe(5) + expect(snapshot.launchableTasks.map((candidate) => candidate.id)).toEqual([task.id]) + }) + + it('surfaces user takeover and release-unknown without closing either terminal', () => { + const { runId } = setup(2) + const unknownTask = db!.createTask({ spec: 'unknown release', runId, capacityEligible: true }) + const userTask = db!.createTask({ spec: 'user takeover', runId, capacityEligible: true }) + const unknownDispatch = startAndSettle(unknownTask.id, 'term_unknown') + const userDispatch = startAndSettle(userTask.id, 'term_user') + db!.db + .prepare( + "UPDATE worker_terminal_resources SET release_state = 'unknown' WHERE owner_dispatch_id = ?" + ) + .run(unknownDispatch) + db!.db + .prepare( + "UPDATE worker_terminal_resources SET ownership_state = 'user_owned', release_state = 'retained', retained_reason = 'user_takeover' WHERE owner_dispatch_id = ?" + ) + .run(userDispatch) + + const debt = db!.getRunCapacity(runId).settledTerminalDebt + expect(debt).toEqual([ + expect.objectContaining({ dispatchId: unknownDispatch, terminalState: 'release_unknown' }), + expect.objectContaining({ + dispatchId: userDispatch, + terminalState: 'retained', + retainedReason: 'user_takeover' + }) + ]) + }) +}) diff --git a/src/main/runtime/orchestration/types.ts b/src/main/runtime/orchestration/types.ts index b34f69e3c22..b211afa8e0a 100644 --- a/src/main/runtime/orchestration/types.ts +++ b/src/main/runtime/orchestration/types.ts @@ -48,6 +48,7 @@ export type RunRow = { coordinator_pane_key: string | null consumer_generation: number legacy: number + target_concurrency: number created_at: string updated_at: string } @@ -257,6 +258,7 @@ export type TaskRow = { display_name: string | null spec: string status: TaskStatus + capacity_eligible: number deps: string result: string | null created_at: string diff --git a/src/main/runtime/rpc/methods/orchestration-capacity.test.ts b/src/main/runtime/rpc/methods/orchestration-capacity.test.ts new file mode 100644 index 00000000000..f4d80e086f2 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-capacity.test.ts @@ -0,0 +1,94 @@ +import { afterEach, describe, expect, it } from 'vitest' +import type { RpcContext } from '../core' +import { createOrchestrationRpcHarness } from './orchestration-rpc-test-harness' +import type { OrchestrationDb } from '../../orchestration/db' +import type { OrcaRuntimeService } from '../../orca-runtime' +import { RpcDispatcher } from '../dispatcher' +import { ORCHESTRATION_METHODS } from './orchestration' +import { ORCHESTRATION_CONTRACT_VERSION } from '../../../../shared/protocol-version' + +describe('orchestration capacity RPC', () => { + const h = createOrchestrationRpcHarness() + let db: OrchestrationDb + let runtime: OrcaRuntimeService + let ctx: RpcContext + let runId: string + + afterEach(() => h.cleanup()) + + function setup(): void { + const state = h.setup() + db = state.db + runtime = state.runtime + ctx = state.ctx + runId = state.activeRunId as string + } + + it('declares a target and exposes explicitly enrolled ready work', async () => { + setup() + const task = db.createTask({ spec: 'eligible backfill', runId }) + + await h.call( + 'orchestration.capacityConfigure', + { target: 2, run: runId, from: 'term_coord' }, + ctx + ) + await h.call( + 'orchestration.capacityTaskSet', + { task: task.id, eligible: true, run: runId, from: 'term_coord' }, + ctx + ) + const shown = (await h.call( + 'orchestration.capacityShow', + { run: runId, from: 'term_coord' }, + ctx + )) as { capacity: { targetConcurrency: number; launchableTasks: { id: string }[] } } + + expect(shown.capacity.targetConcurrency).toBe(2) + expect(shown.capacity.launchableTasks.map((candidate) => candidate.id)).toEqual([task.id]) + }) + + it('rejects capacity enrollment outside the bound Run', async () => { + setup() + const foreign = db.createRun({ + objective: 'foreign', + coordinatorHandle: 'term_foreign', + coordinatorPaneKey: 'foreign:worktree:tab:leaf' + }) + const task = db.createTask({ spec: 'foreign task', runId: foreign.id }) + + await expect( + h.call( + 'orchestration.capacityTaskSet', + { task: task.id, eligible: true, run: runId, from: 'term_coord' }, + ctx + ) + ).rejects.toMatchObject({ code: 'task_not_found' }) + }) + + it('replays target configuration without applying the mutation twice', async () => { + setup() + const dispatcher = new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS }) + const request = { + authToken: 'local-token', + method: 'orchestration.capacityConfigure', + params: { target: 2, run: runId, from: 'term_coord' }, + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationRequestId: 'capacity_replay' + } + + const first = await dispatcher.dispatch({ ...request, id: 'rpc_first' }) + db.configureRunCapacity(runId, 3) + const replay = await dispatcher.dispatch({ ...request, id: 'rpc_replay' }) + + expect(first).toMatchObject({ + ok: true, + result: { capacity: { targetConcurrency: 2 }, mutation: { replayed: false } } + }) + expect(replay).toMatchObject({ + ok: true, + result: { capacity: { targetConcurrency: 2 }, mutation: { replayed: true } } + }) + expect(db.getRunCapacity(runId).targetConcurrency).toBe(3) + }) +}) diff --git a/src/main/runtime/rpc/methods/orchestration-capacity.ts b/src/main/runtime/rpc/methods/orchestration-capacity.ts new file mode 100644 index 00000000000..25e83f72846 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-capacity.ts @@ -0,0 +1,81 @@ +import { z } from 'zod' +import { defineMethod, type RpcMethod } from '../core' +import { OptionalString, requiredString } from '../schemas' +import { OrchestrationError } from '../../orchestration/orchestration-error' +import { resolveRunScope } from './orchestration-run-scope' + +const CapacityConfigureParams = z.object({ + target: z.number().int().min(0).max(64), + run: OptionalString, + from: requiredString('Missing coordinator terminal') +}) + +const CapacityTaskParams = z.object({ + task: requiredString('Missing --task'), + eligible: z.boolean(), + run: OptionalString, + from: requiredString('Missing coordinator terminal') +}) + +const CapacityShowParams = z.object({ + run: OptionalString, + from: OptionalString +}) + +export const ORCHESTRATION_CAPACITY_METHODS: RpcMethod[] = [ + defineMethod({ + name: 'orchestration.capacityConfigure', + params: CapacityConfigureParams, + handler: (params, { orchestrationCompatibilityEvidence, runtime, legacyCoordinatorRunId }) => { + const run = resolveRunScope(runtime, { + runId: params.run, + callerTerminalHandle: params.from, + requireCurrentConsumer: true, + legacyCoordinatorRunId, + callerEvidence: orchestrationCompatibilityEvidence + }) + return { capacity: runtime.getOrchestrationDb().configureRunCapacity(run.id, params.target) } + } + }), + defineMethod({ + name: 'orchestration.capacityTaskSet', + params: CapacityTaskParams, + handler: (params, { orchestrationCompatibilityEvidence, runtime, legacyCoordinatorRunId }) => { + const db = runtime.getOrchestrationDb() + const run = resolveRunScope(runtime, { + runId: params.run, + callerTerminalHandle: params.from, + requireCurrentConsumer: true, + legacyCoordinatorRunId, + callerEvidence: orchestrationCompatibilityEvidence + }) + const existing = db.getTask(params.task) + if (!existing || existing.run_id !== run.id) { + throw new OrchestrationError( + 'task_not_found', + `Task ${params.task} was not found in Run ${run.id}.` + ) + } + return { task: db.setTaskCapacityEligibility(params.task, params.eligible) } + } + }), + defineMethod({ + name: 'orchestration.capacityShow', + params: CapacityShowParams, + handler: (params, { orchestrationCompatibilityEvidence, runtime, legacyCoordinatorRunId }) => { + const db = runtime.getOrchestrationDb() + const explicitRun = params.run ? db.getRun(params.run) : undefined + const run = + explicitRun?.legacy === 0 + ? explicitRun + : resolveRunScope(runtime, { + runId: params.run, + callerTerminalHandle: params.from, + requireCurrentConsumer: params.run === undefined, + legacyCoordinatorRunId, + callerEvidence: orchestrationCompatibilityEvidence + }) + return { capacity: db.getRunCapacity(run.id) } + } + }) +] diff --git a/src/main/runtime/rpc/methods/orchestration-federated-worker-start.ts b/src/main/runtime/rpc/methods/orchestration-federated-worker-start.ts index 28c6530d2a0..5f8b0f0a8a1 100644 --- a/src/main/runtime/rpc/methods/orchestration-federated-worker-start.ts +++ b/src/main/runtime/rpc/methods/orchestration-federated-worker-start.ts @@ -118,10 +118,12 @@ export async function startFederatedWorker(args: { ? params.setup ? 'explicit_request' : 'orchestration_default' - : 'existing_worktree' + : 'existing_worktree', + capacitySlot: params.capacitySlot ?? false }, runtimeEpoch: runtime.getRuntimeId(), mutationReceipt: orchestrationMutation, + capacitySlot: params.capacitySlot, federation: { environmentId: server.environmentId, environmentName: server.name, diff --git a/src/main/runtime/rpc/methods/orchestration-worker-start-schema.ts b/src/main/runtime/rpc/methods/orchestration-worker-start-schema.ts index cf081f9b36c..b804c3f35d7 100644 --- a/src/main/runtime/rpc/methods/orchestration-worker-start-schema.ts +++ b/src/main/runtime/rpc/methods/orchestration-worker-start-schema.ts @@ -25,6 +25,7 @@ export const WorkerStartParams = z.object({ model: OptionalWorkerLaunchPreference, effort: OptionalWorkerLaunchPreference, retryOf: OptionalString, + capacitySlot: z.boolean().optional(), timeoutMs: OptionalFiniteNumber, devMode: z.boolean().optional() }) diff --git a/src/main/runtime/rpc/methods/orchestration-workers.ts b/src/main/runtime/rpc/methods/orchestration-workers.ts index 0b8891481cd..a9bc0d8cbc0 100644 --- a/src/main/runtime/rpc/methods/orchestration-workers.ts +++ b/src/main/runtime/rpc/methods/orchestration-workers.ts @@ -117,7 +117,8 @@ export const ORCHESTRATION_WORKER_START_METHODS: RpcMethod[] = [ ? params.setup ? 'explicit_request' : 'orchestration_default' - : 'existing_worktree' + : 'existing_worktree', + capacitySlot: params.capacitySlot ?? false } const started = db.createStartingWorkerDispatch({ creator: resolveDispatchCreator(runtime, params.from), @@ -126,7 +127,8 @@ export const ORCHESTRATION_WORKER_START_METHODS: RpcMethod[] = [ retryOf: params.retryOf, startOptions, runtimeEpoch: runtime.getRuntimeId(), - mutationReceipt: orchestrationMutation + mutationReceipt: orchestrationMutation, + capacitySlot: params.capacitySlot }) const effects: WorkerEffect[] = [] if (resolvedWorktree) { diff --git a/src/main/runtime/rpc/methods/orchestration.ts b/src/main/runtime/rpc/methods/orchestration.ts index 4045ce11409..c6891ebe577 100644 --- a/src/main/runtime/rpc/methods/orchestration.ts +++ b/src/main/runtime/rpc/methods/orchestration.ts @@ -33,6 +33,7 @@ import { resolveRunScope } from './orchestration-run-scope' import { ORCHESTRATION_RUN_METHODS } from './orchestration-runs' import { ORCHESTRATION_WORKER_METHODS } from './orchestration-worker-methods' import { ORCHESTRATION_FEDERATION_METHODS } from './orchestration-federation-methods' +import { ORCHESTRATION_CAPACITY_METHODS } from './orchestration-capacity' import { OrchestrationError } from '../../orchestration/orchestration-error' import type { OrcaRuntimeService } from '../../orca-runtime' import type { RunRow } from '../../orchestration/types' @@ -440,6 +441,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ ...ORCHESTRATION_RUN_METHODS, ...ORCHESTRATION_WORKER_METHODS, ...ORCHESTRATION_FEDERATION_METHODS, + ...ORCHESTRATION_CAPACITY_METHODS, defineMethod({ name: 'orchestration.send', params: SendParams, diff --git a/src/shared/orchestration-rpc-contract.test.ts b/src/shared/orchestration-rpc-contract.test.ts index 90ba326d919..6d351426d35 100644 --- a/src/shared/orchestration-rpc-contract.test.ts +++ b/src/shared/orchestration-rpc-contract.test.ts @@ -13,6 +13,8 @@ describe('orchestration RPC contract', () => { ['orchestration.reply', {}], ['orchestration.taskCreate', {}], ['orchestration.taskUpdate', {}], + ['orchestration.capacityConfigure', {}], + ['orchestration.capacityTaskSet', {}], ['orchestration.dispatch', {}], ['orchestration.workerStart', {}], ['orchestration.workerStop', {}], diff --git a/src/shared/orchestration-rpc-contract.ts b/src/shared/orchestration-rpc-contract.ts index f95fe2f45c4..3b990d271cb 100644 --- a/src/shared/orchestration-rpc-contract.ts +++ b/src/shared/orchestration-rpc-contract.ts @@ -22,6 +22,8 @@ const ORCHESTRATION_MUTATION_METHODS = new Set([ 'orchestration.reply', 'orchestration.taskCreate', 'orchestration.taskUpdate', + 'orchestration.capacityConfigure', + 'orchestration.capacityTaskSet', 'orchestration.dispatch', 'orchestration.workerStart', 'orchestration.workerStop', diff --git a/src/shared/protocol-version.ts b/src/shared/protocol-version.ts index bfd58f181e1..44a1f597bf3 100644 --- a/src/shared/protocol-version.ts +++ b/src/shared/protocol-version.ts @@ -50,6 +50,8 @@ export const ORCHESTRATION_WORKER_STOP_VERDICT_RUNTIME_CAPABILITY = 'orchestration.worker-stop-verdict.v1' as const export const ORCHESTRATION_WORKER_LAUNCH_PREFERENCES_RUNTIME_CAPABILITY = 'orchestration.worker-launch-preferences.v1' as const +export const ORCHESTRATION_RUN_CAPACITY_RUNTIME_CAPABILITY = + 'orchestration.run-capacity.v1' as const export const ORCHESTRATION_FEDERATION_CONTROL_MAIL_PROTOCOL_VERSION = 2 as const export const ORCHESTRATION_FEDERATION_LIFECYCLE_SETTLEMENT_PROTOCOL_VERSION = 3 as const export const ORCHESTRATION_CONTRACT_VERSION = 1 as const @@ -164,6 +166,7 @@ export const RUNTIME_CAPABILITIES = [ ORCHESTRATION_FEDERATION_LIFECYCLE_SETTLEMENT_RUNTIME_CAPABILITY, ORCHESTRATION_WORKER_STOP_VERDICT_RUNTIME_CAPABILITY, ORCHESTRATION_WORKER_LAUNCH_PREFERENCES_RUNTIME_CAPABILITY, + ORCHESTRATION_RUN_CAPACITY_RUNTIME_CAPABILITY, ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY, BROWSER_SCREENCAST_RUNTIME_CAPABILITY, BROWSER_TAB_CREATE_KNOWN_ID_RUNTIME_CAPABILITY,