From efd06382ae115b455fac1bfeccb316d23d2a381d Mon Sep 17 00:00:00 2001 From: OrcaWin <293788423+OrcaWin@users.noreply.github.com> Date: Wed, 29 Jul 2026 04:39:02 -0700 Subject: [PATCH] fix(orchestration): complete legacy worker cutover recovery --- .../orchestration-skill-guidance.test.mjs | 9 +- skill-guides/orchestration.md | 15 +- src/cli/bundled-skill-guides.ts | 2 +- .../orchestration-check-identity.test.ts | 20 + .../orchestration-legacy-read-only.test.ts | 4 +- .../handlers/orchestration-run-cli.test.ts | 22 + src/cli/handlers/orchestration.ts | 192 +--- src/cli/specs/orchestration.ts | 8 +- src/main/index.ts | 2 - src/main/ipc/pty.test.ts | 31 + src/main/ipc/pty.ts | 8 +- src/main/runtime/orca-runtime.test.ts | 757 ++++++++++++++-- src/main/runtime/orca-runtime.ts | 632 +++++++++---- src/main/runtime/orchestration/db.ts | 313 +++++-- .../runtime/orchestration/formatter.test.ts | 8 + src/main/runtime/orchestration/formatter.ts | 9 +- .../orchestration-legacy-storage-db.test.ts | 342 ++++--- ...chestration-legacy-storage-test-fixture.ts | 147 +++ ...on-legacy-worker-terminal-recovery.test.ts | 6 +- ...tration-legacy-worker-terminal-recovery.ts | 15 +- .../orchestration-worker-dispatch-db.test.ts | 10 +- src/main/runtime/rpc/core.ts | 10 + .../dispatcher-feature-interactions.test.ts | 4 + .../runtime/rpc/dispatcher-stream-options.ts | 18 + src/main/runtime/rpc/dispatcher.ts | 107 +-- .../runtime/rpc/methods/orchestration-runs.ts | 10 +- src/main/runtime/rpc/methods/orchestration.ts | 105 ++- .../rpc/orchestration-legacy-authority.ts | 74 +- ...on-legacy-compatibility-dispatcher.test.ts | 193 ++++ .../rpc/orchestration-legacy-compatibility.ts | 47 +- ...hestration-legacy-coordinator-authority.ts | 158 ++++ ...hestration-legacy-coordinator-race.test.ts | 849 ++++++++++++++++++ .../rpc/orchestration-legacy-lifecycle.ts | 2 +- .../orchestration-legacy-process-identity.ts | 33 + .../rpc/orchestration-legacy-question.ts | 7 +- .../rpc/orchestration-mutation-executor.ts | 18 + .../rpc/orchestration-mutation-ledger.test.ts | 17 +- .../rpc/runtime-feature-interaction.ts | 20 + src/main/ssh/ssh-relay-session.ts | 35 + .../ssh/ssh-remote-cli-host-passthrough.ts | 15 + .../ssh/ssh-remote-cli-in-process-result.ts | 40 + src/main/ssh/ssh-remote-orca-cli.test.ts | 1 - src/main/ssh/ssh-remote-orca-cli.ts | 66 +- .../ssh-remote-orchestration-ask-output.ts | 66 ++ .../ssh-remote-orchestration-check-output.ts | 48 + ...remote-orchestration-compatibility.test.ts | 394 ++++++++ .../ssh-remote-orchestration-post-output.ts | 143 +++ .../attach-main-window-services.test.ts | 69 ++ .../window/attach-main-window-services.ts | 32 +- src/preload/api-types.ts | 8 +- src/preload/index.ts | 8 +- src/relay/relay.ts | 96 +- src/renderer/src/hooks/useIpcEvents.test.ts | 70 +- src/renderer/src/hooks/useIpcEvents.ts | 25 +- ...eeping-agent-session-legacy-worker.test.ts | 14 +- .../src/lib/terminal-reveal-identity.test.ts | 65 ++ .../src/lib/terminal-reveal-identity.ts | 23 + src/shared/orchestration-check-output.test.ts | 42 + src/shared/orchestration-check-output.ts | 199 ++++ src/shared/terminal-reveal-identity.ts | 14 + ...ion-legacy-worker-restart-recovery.spec.ts | 605 +++++++------ 61 files changed, 5185 insertions(+), 1117 deletions(-) create mode 100644 src/main/runtime/orchestration/orchestration-legacy-storage-test-fixture.ts create mode 100644 src/main/runtime/rpc/dispatcher-stream-options.ts create mode 100644 src/main/runtime/rpc/orchestration-legacy-coordinator-authority.ts create mode 100644 src/main/runtime/rpc/orchestration-legacy-coordinator-race.test.ts create mode 100644 src/main/runtime/rpc/orchestration-legacy-process-identity.ts create mode 100644 src/main/ssh/ssh-remote-cli-in-process-result.ts create mode 100644 src/main/ssh/ssh-remote-orchestration-ask-output.ts create mode 100644 src/main/ssh/ssh-remote-orchestration-check-output.ts create mode 100644 src/main/ssh/ssh-remote-orchestration-compatibility.test.ts create mode 100644 src/main/ssh/ssh-remote-orchestration-post-output.ts create mode 100644 src/renderer/src/lib/terminal-reveal-identity.test.ts create mode 100644 src/renderer/src/lib/terminal-reveal-identity.ts create mode 100644 src/shared/orchestration-check-output.test.ts create mode 100644 src/shared/orchestration-check-output.ts create mode 100644 src/shared/terminal-reveal-identity.ts diff --git a/config/scripts/orchestration-skill-guidance.test.mjs b/config/scripts/orchestration-skill-guidance.test.mjs index 06b280fbe82..849fe784b46 100644 --- a/config/scripts/orchestration-skill-guidance.test.mjs +++ b/config/scripts/orchestration-skill-guidance.test.mjs @@ -71,8 +71,15 @@ describe('orchestration skill guidance', () => { expect(migration).toContain( 'It must not spawn, write, signal, stop, switch, focus, split, or inject a terminal.' ) - expect(migration).toContain('task-list --run run_legacy_local') + expect(migration).not.toContain('task-list --run run_legacy_local') + expect(migration).toContain('run_legacy_local is an empty audit tombstone') + expect(migration).toContain('Recovered orchestration work from a contract update') + expect(migration).toContain('run-show --id ') + expect(migration).toContain('task-list --run ') expect(migration).toContain('Legacy inspection remains available without consuming mail') + expect(migration).toContain('run-use --id --takeover-legacy') + expect(migration).toContain('Takeover fences only the old coordinator') + expect(migration).toContain('Live legacy workers keep their original Tasks, Dispatches') expect(migration).toContain( 'keep the original worker as the only editor until it reaches a stable handoff point' ) diff --git a/skill-guides/orchestration.md b/skill-guides/orchestration.md index fc3528b4a00..23fbd3fa263 100644 --- a/skill-guides/orchestration.md +++ b/skill-guides/orchestration.md @@ -74,14 +74,27 @@ Legacy inspection remains available without consuming mail: ```bash orca orchestration run-list --json +# run_legacy_local is an empty audit tombstone after adoption. orca orchestration run-show --id run_legacy_local --json -orca orchestration task-list --run run_legacy_local --json +# In run-list, find the ordinary Run whose objective is: +# "Recovered orchestration work from a contract update" +orca orchestration run-show --id --json +orca orchestration task-list --run --json orca orchestration inbox --full --json orca orchestration check --terminal --peek --format --json orca terminal read --terminal --json orca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json ``` +If the original coordinator is unavailable or cannot prove its retained authority, a current coordinator may explicitly take over the adopted Run: + +```bash +orca orchestration run-use --id --takeover-legacy --json +orca orchestration check --run --json +``` + +Takeover fences only the old coordinator, binds the current one, and moves pending worker mail into current Run Delivery. Live legacy workers keep their original Tasks, Dispatches, processes, filesystems, and old prompt commands; their later questions, escalations, and completion reports route to the current coordinator. Do not use takeover while the original coordinator is still actively coordinating, because its later lifecycle mutations are rejected. + Do not launch a replacement editor merely because the desktop app or runtime was updated. If adoption cannot prove continuing authority, keep the original worker as the only editor until it reaches a stable handoff point, then use a new current Dispatch in a conflict-free placement for any remaining work. ## Ownership diff --git a/src/cli/bundled-skill-guides.ts b/src/cli/bundled-skill-guides.ts index da1f18a39e9..3441031f246 100644 --- a/src/cli/bundled-skill-guides.ts +++ b/src/cli/bundled-skill-guides.ts @@ -30,7 +30,7 @@ const ORCA_LINEAR_MARKDOWN = "---\nname: orca-linear\ndescription: >-\n Use Orc const ORCA_PER_WORKSPACE_ENV_MARKDOWN = "---\nname: orca-per-workspace-env\ndescription: >-\n Set up, review, debug, or validate Orca per-workspace environment recipes —\n on-demand, disposable runtimes (cloud sandboxes, VMs, or local) created fresh\n for each workspace. Covers first-time setup (provider prerequisites, the\n reusable base snapshot, the coding-agent auth snapshot, credentials, and\n state), not just the per-workspace lifecycle scripts. Use to stand up\n per-workspace environments, fix an `environmentRecipes` entry in `orca.yaml`, scaffold\n provider lifecycle scripts, or resolve an `orca vm recipe doctor` failure.\n---\n\n# Per-Workspace Environments\n\nHelp a user stand up and maintain a repo-owned per-workspace environment recipe end to end. Each\nworkspace gets its own on-demand, disposable runtime (a cloud sandbox, a VM, or a local one),\ncreated fresh and torn down after.\n\nOrca is a **thin wrapper**: you guide, detect, and scaffold; you never own the user's cloud account,\nbilling, images, or credentials.\n\n- **You DO:** sequence the setup, detect what's detectable (provider CLI present/logged-in? recipe\n present? `doctor` passing?), scaffold provider-templated scripts the user fills in, drive the slow\n snapshot/auth phases with the user, and always show the next action.\n- **You DO NOT:** create accounts, choose plans/regions, invent org/project/scope ids, store or print\n secrets, or run anything that spends money without an explicit user OK.\n\nFirst-time setup has **four phases before the per-workspace recipe runs** — easy to miss, so walk\nthem in order:\n\n1. **Prerequisites** — cloud account, provider CLI, scope/project, plan limits, git token (§2).\n2. **Base snapshot** — reusable image: tools + repo + headless build, snapshotted once (§3).\n3. **Agent-auth snapshot** — boot the base, run interactive device-auth, re-snapshot (§4).\n4. **State** — thread snapshot id / scope / project / port between phases via a state file (§6).\n\nThen the **per-workspace contract** (create/suspend/resume/destroy) runs fast (§8).\n\n**The one branch that shapes everything — connection mode:** **Orca-server** (`create` runs `orca serve`\nin the env and emits a `pairingCode`; §7c/§7f) vs **SSH** (`create` runs no server and emits a\n`connection.type:\"ssh\"` block Orca dials into; §7g/§7h). Settle this first — it changes the `create`\noutput shape and half the templates.\n\n**Quick-start (happy path):** interview the user (connection mode Orca-server vs SSH, provider, agent CLI,\ngit auth — §1.2) + read the provider's CLI docs → scaffold `scripts/orca-vm/` from §7 → run the\nbase-snapshot script, then the auth script (you invoke these by hand; not via `orca.yaml`) → wire\n`environmentRecipes` in `orca.yaml` → `orca vm recipe doctor --json` (free) → then the `--provision`\nself-test loop (§9) until it passes.\n\n---\n\n## 1. Setup workflow\n\nDrive these with the user. **[CHECKPOINT]** steps need explicit confirmation — they spend money, take\na long time, or need the user at the keyboard. Never create an Orca workspace or commit unless asked.\n\n1. **Inspect the repo** for an existing `environmentRecipes` entry, `scripts/orca-vm/`, a state file, or setup\n notes. If a working recipe exists, jump to Doctor (§9) instead of rebuilding.\n2. **Interview the user up front** — gather these choices and confirm them back before scaffolding\n anything. Don't pick for them (§11); don't guess.\n - **Connection mode:** how Orca attaches to the environment — an **Orca server** (the VM runs\n `orca serve` and Orca pairs over its pairing URL; worked example §7f) or **SSH** (Orca connects to\n the host over SSH; §7g). This decides the recipe's connection shape, so settle it first.\n - **Provider:** Vercel Sandbox, Fly, Modal, an existing SSH host, … For non-obvious providers, also\n ask scope/project/region and plan limits (§2). Then **read that provider's CLI/SDK docs** (or\n ` --help`) before scaffolding — you need its exact create/exec/snapshot/remove verbs.\n If a provider advertises `ssh`, verify whether it exposes a real dialable SSH target\n (host/port/user/key or proxy command) or only a provider-mediated interactive shell; Orca SSH mode\n needs the former.\n - **Coding-agent CLI + account:** which agent runs in the VM (`codex`, `claude`, …) and that the user\n has an account for it — it gets logged in during the Phase-3 auth snapshot (§4).\n - **Git auth:** the token source for cloning a private repo (`GH_TOKEN`/`GITHUB_TOKEN` or `gh auth\n token`; §5).\n3. **Check prerequisites (§2)** — detect the provider CLI + auth and confirm the items above are in\n place before any paid step.\n4. **Scaffold scripts + state file** from §7 (worked Vercel example: §7f; SSH host: §7g; Docker SSH:\n §7h; Windows: §7i), filling in the provider's real commands. Make them executable.\n5. **[CHECKPOINT] Build the base snapshot (§3)** — paid, slow.\n6. **[CHECKPOINT] Authenticate the agent (§4)** — interactive; the user follows a URL/code. **You cannot\n drive this step** — you run commands non-interactively, so there's no TTY for `docker exec -it` /\n `ssh -t` to prompt against. The **user** runs the Phase-3 login in their own terminal (or via the\n Claude Code harness bang-prefix — `! `, with the required space after `!`); you scaffold and drive\n the non-interactive phases around it. After kicking it off, **ask the user to report back once the login\n finishes** — you can't observe it completing, and you need that confirmation before resuming the\n non-interactive steps (base/auth commit, doctor, provision).\n7. **Wire the recipe** so `orca.yaml` points create/suspend/resume/destroy at the scripts (§8). The\n workspace composer reads `environmentRecipes` from the project's primary checkout of `orca.yaml`, **not** from\n a feature branch or worktree. So a recipe added only on a branch won't appear as a \"Run on\" option\n until that `orca.yaml` change is committed and merged to the project's primary branch. Tell the user\n this up front: `doctor`/`--provision` validate the scripts from the working copy on any branch, but\n creating a workspace from the recipe in the picker needs it on primary.\n8. **Dry-run doctor** — `orca vm recipe doctor --repo-path --json` (free, static; §9).\n Fix every failure before going live.\n9. **[CHECKPOINT] Live self-test** — get the user's OK once, then run\n `orca vm recipe doctor --provision --json` as a loop: it runs create → validates →\n destroys, and on failure returns a full transcript. Read it, fix the scripts, and re-run yourself until\n it passes (§9). Spends cloud money; the one approval covers the loop.\n10. **[CHECKPOINT] Optional workspace test** — only if asked: create a workspace via the picker, then\n verify sleep/wake/delete.\n\n---\n\n## 2. Phase 1 — Prerequisites\n\nThe user's responsibility; verify what's verifiable, ask for the rest, invent nothing. State which\nitems you verified vs. which the user asserted.\n\n- **Connection mode** (Orca server vs SSH) confirmed with the user — see §1 step 2; it shapes the recipe.\n- **Cloud account + plan** that allows sandboxes/VMs. Ask.\n- **Provider CLI installed + authenticated** — detect (`command -v `), check auth (e.g.\n `vercel whoami`). If missing, point at the provider's docs; don't log them in.\n- **Scope / project / region** the sandboxes live under. Ask; flows into every script via state.\n- **Plan / timeout / RAM caps.** Record them — e.g. Vercel Hobby caps sandbox timeout at **45m**,\n which limits both the base build and per-workspace runtime (see §10).\n- **Git token for private repos** (`GH_TOKEN`/`GITHUB_TOKEN`, or the provider's git auth; can fall back\n to `gh auth token`). See §5.\n- **Coding-agent CLI choice** (`codex`, `claude`…) and that the user has an account — it gets\n authenticated into the VM in Phase 3.\n\n---\n\n## 3. Phase 2 — Base snapshot (the reusable image)\n\nBuild **once**, snapshot, and every workspace boots from it in seconds instead of rebuilding.\nProvisioning + building takes a while (often ~20–30 min), so it runs behind a checkpoint. The script\nshape is §7a; key points:\n\n- Build the **headless Electron main only** (not the renderer) so it fits in plan RAM.\n- Use the VM image's package manager (`apt`/`dnf`/`apk`, per the base distro — not the provider brand).\n- Clone with the git token via `GIT_ASKPASS` (§5).\n- **Trap errors and remove the half-built sandbox** so a crash doesn't leave a paid resource running.\n- Snapshot the stopped sandbox, parse the snapshot id, and write it + scope/project/port/repo to state.\n\n---\n\n## 4. Phase 3 — Agent-auth snapshot (interactive)\n\nThe base snapshot has the agent CLI installed but **not logged in**, and per-workspace VMs are\nephemeral — so authenticate once and bake it into a second snapshot layer. Script shape is §7b:\n\n1. Boot a sandbox from the base `snapshotId` (from state).\n2. Run the agent's login **interactively** (`--interactive --tty`); the user completes the URL/code in\n their browser. On a **headless VM this must be the device-auth flow** (e.g. `codex login --device-auth`),\n **not** plain `codex login`: the default OAuth login starts a loopback callback server on a container\n port the host browser can't reach, so it hangs. Device-auth instead prints a URL + code the user opens\n on the **host**.\n3. Verify login; **refuse to snapshot an unauthenticated VM.** Prefer the status command's **exit code**\n (most agent CLIs exit non-zero when unauthenticated). If you grep instead, agent status often goes to\n **stderr** (e.g. `codex login status` prints \"Logged in using ChatGPT\" there), so **fold stderr first**\n (`... 2>&1 | grep …`) and match the agent's **exact success line** — never `grep -qi 'logged in'`, which\n also matches \"**not** logged in\" and would commit an unauthenticated image.\n4. Re-snapshot, parse the new id, and overwrite `snapshotId` in state to the authenticated image\n (recording `authSourceSnapshotId`). Remove the auth sandbox.\n\n**You can't drive step 2 yourself** (you run commands non-interactively — no TTY). The **user** runs it in\ntheir own terminal, or via the Claude Code harness bang-prefix (`! `, with the required space after\n`!`). You scaffold/boot the sandbox and run steps 3–4, but **you cannot observe the interactive login\nfinishing** — so **ask the user to tell you when it's done** before you verify and re-snapshot.\n\nIf the agent's credentials are short-lived, warn that the snapshot may need periodic re-auth (§10).\n\nFor disposable runtimes, do **not** treat a host agent config directory (for example `~/.codex`) as the\nauth snapshot by bind-mounting or copying it wholesale. Agent homes often contain sqlite state, hook\napproval state, caches, logs, and host-specific env/config. Instead, authenticate/configure the agent\ninside the disposable runtime and snapshot/commit that runtime layer.\n\n---\n\n## 5. Credentials\n\n- **Never** commit secrets or put them in `userData`, recipe JSON, comments, docs, or the state file.\n- **Git token:** read from env (`GH_TOKEN`/`GITHUB_TOKEN`), falling back to `gh auth token`. Pass to the\n VM only via the provider's ephemeral `--env`. Inside the VM, use a `GIT_ASKPASS` helper with\n `x-access-token` (not the token in the clone URL) and `GIT_TERMINAL_PROMPT=0` so a missing token fails\n fast instead of hanging. When you write the helper from inside `bash -lc` under `set -u`, escape the\n positional arg and the token (`\\$1`, `\\$GH_TOKEN`) so they land **literally** and resolve at git-runtime\n — an unescaped `$1` aborts with \"unbound variable\", and a literal `$GH_TOKEN` keeps the real token out of\n the written file. `rm -f` the helper after the clone/fetch.\n- **Provider auth:** rely on the provider CLI's logged-in session, not checked-in keys.\n- **Agent auth:** lives in the authenticated snapshot (Phase 3) — never a file you write or commit.\n- State holds only **non-secret** wiring (snapshot ids, scope, project, port, repo url/ref).\n\n---\n\n## 6. State file\n\nA repo-local JSON file (e.g. `scripts/orca-vm/-state.json`) threads non-secret values between\nphases. Each script resolves values as **env var → state → built-in fallback**, and merges its outputs\nback. Phase 2 writes the base `snapshotId`; Phase 3 overwrites it with the authenticated snapshot;\nper-workspace `create` boots from `snapshotId`.\n\n```json\n{\n \"baseName\": \"orca-base\",\n \"snapshotId\": \"snap_authenticated_image_id\",\n \"authSourceSnapshotId\": \"snap_base_image_id\",\n \"scope\": \"\",\n \"project\": \"\",\n \"port\": 7331,\n \"repoUrl\": \"https://host/org/repo.git\",\n \"repoRef\": \"main\",\n \"projectRoot\": \"/abs/path/on/remote/repo\"\n}\n```\n\n---\n\n## 7. Script templates (provider-agnostic shapes)\n\nScaffold under `scripts/orca-vm/`. These are **shapes** — fill in the provider's real commands. All\nreserve stdout for the final JSON and log progress to stderr. Include a shared `json_value ` /\n`env_value ` reader (env → state → fallback) in each.\n\n**Where each script runs:**\n\n- **Local-side** (`create`/`suspend`/`resume`/`destroy` + the base-snapshot/auth scripts the user\n invokes) runs **on the user's desktop**, so it must run on their OS. macOS/Linux: `#!/usr/bin/env\n bash`, `set -euo pipefail`, quoted paths. **Windows:** a bare `.sh` won't run — scaffold `.ps1`/`.cmd`\n or require WSL/Git-Bash and point `orca.yaml` at the right launcher.\n- **Remote-side** (commands you `exec` *inside* the Linux VM) always runs in the VM's Linux shell, so\n bash is fine there regardless of the user's OS.\n\n### 7a. Base-snapshot (`-base-snapshot.sh`) — Phase 2\n\n```bash\n#!/usr/bin/env bash\nset -euo pipefail\n# resolve base_name/repo_url/repo_ref/project_root/port/scope/project/timeout (env→state→fallback)\n# resolve gh token: GH_TOKEN | GITHUB_TOKEN | `gh auth token`\n# 1. provision a sandbox (timeout/vcpus/published port/snapshot retention); trap: remove on error\n# 2. remote exec (long timeout): install pkgs + gh + corepack/pnpm + agent CLI;\n# clone with GIT_ASKPASS(token); write headless main-only build config;\n# dev setup; pnpm install; build CLI; build headless electron main; smoke-check tools\n# 3. snapshot stopped sandbox; parse snapshot id (fail if unparseable)\n# 4. merge { baseName, snapshotId, projectRoot, repoUrl, repoRef, port, scope, project } into state\n# print only the state JSON to stdout\n```\n\nWorked Vercel commands for this phase are in §7f. You run this script by hand (not via `orca.yaml`),\nafter exporting the first-run inputs the state file doesn't have yet — e.g. provider scope/project, the\nrepo URL/ref, and a git token (`GH_TOKEN`); later runs read them back from state.\n\n### 7b. Auth (`-base-auth.sh`) — Phase 3\n\n```bash\n#!/usr/bin/env bash\nset -euo pipefail\n# read source snapshot from state.snapshotId (fail if absent); auth_name=\"${base_name}-auth\"\n# 1. boot sandbox from source snapshot; trap: remove on error\n# 2. INTERACTIVE/TTY remote exec: agent login — user completes URL/code. Headless VM: MUST use the\n# device-auth flow (e.g. `codex login --device-auth`) — plain OAuth login binds a loopback callback\n# port the host can't reach and hangs. User runs this themselves (you have no interactive TTY); ask\n# them to report back when it's done before continuing.\n# 3. verify login, then refuse to snapshot if not logged in. Prefer the status command's EXIT CODE (most\n# agent CLIs exit non-zero when unauthenticated) over string-matching. If you must grep, fold stderr\n# first (`status 2>&1 | grep …` — many agents print the success line there) and match the agent's exact\n# success line; never `grep -qi 'logged in'`, which also matches \"not logged in\". Codex example: §7f.\n# 4. snapshot; parse new id\n# 5. merge { snapshotId:, authSourceSnapshotId: } into state; remove auth sandbox\n# print only the state JSON to stdout\n```\n\n### 7c. Create (`-create.sh`) — per workspace\n\n```bash\n#!/usr/bin/env bash\nset -euo pipefail\n# read authenticated snapshotId/scope/project/port/repo*/project_root (env→state→fallback)\n# fail clearly if snapshotId is missing (point back to Phases 2–3)\n# name = orca-${ORCA_VM_RECIPE_ID}-${ORCA_VM_INSTANCE_ID} (sanitized, length-capped)\n# 1. boot sandbox from snapshotId with a published port; capture the public URL → pairing address\n# (an externally reachable wss:// URL); trap: remove sandbox on error\n# 2. remote exec: ensure repo at desired commit; rebuild only if commit changed (cache marker)\n# 3. remote exec: start orca serve in the background and read the recipe JSON it writes (see below)\n# 4. print serve's JSON to stdout, optionally enriched with userData:\n# { schemaVersion:1, pairingCode, projectRoot, userData:{ provider, resourceId:name, snapshotId } }\n```\n\n**The exact `orca serve` invocation and its output (verified — do not improvise the flags).** Inside the\nVM, run:\n\n```bash\norca serve \\\n --port \"$PORT\" \\\n --project-root \"$ABS_REPO_PATH_ON_REMOTE\" \\\n --pairing-address \"$EXTERNAL_WSS_URL\" \\\n --recipe-json\n```\n\n**Binary name:** in a VM built from source (the Phase-2 flow), run it as `pnpm exec orca-dev serve …`\nfrom the repo root — `orca-dev` is the in-repo entrypoint and is what the §7f example uses. Plain\n`orca serve …` is the same command when the built CLI is installed on the VM's PATH. The flags/output\nare identical either way.\n\nThere is **no `--host` flag**. `--project-root` must be an absolute directory on the remote. With\n`--recipe-json` the server **stays running** and prints exactly this single object to **stdout**, then\nkeeps serving:\n\n```json\n{ \"schemaVersion\": 1, \"pairingCode\": \"\", \"projectRoot\": \"\" }\n```\n\n`pairingCode` is the pairing URL, already pointing at whatever you passed as `--pairing-address` — so set\n`--pairing-address` to the externally reachable address and **pass `pairingCode` through unchanged; never\nhand-rewrite it**. Because serve runs in the foreground and doesn't exit, redirect its stdout to a file\nand poll until that file parses as JSON (and bail if the process dies — dump its stderr log). Your\n`create` script then prints that JSON (optionally merging `userData`). Concrete pattern: §7f.\n\n### 7d. Suspend / resume / destroy — per workspace\n\n```bash\n#!/usr/bin/env bash\nset -euo pipefail\npayload=\"$(cat)\" # Orca passes lifecycle JSON on stdin\nresource_id=\"$(node -e 'const d=JSON.parse(process.argv[1]); process.stdout.write(d.recipeResult?.userData?.resourceId ?? \"\")' \"$payload\")\"\n[ -n \"$resource_id\" ] || { echo \"No resource id in lifecycle payload\" >&2; exit 1; }\n# suspend: provider suspend \"$resource_id\"\n# resume: provider resume \"$resource_id\"; then RE-EMIT fresh recipe JSON (pairing may change)\n# destroy: provider remove \"$resource_id\" (or set destroy: none in orca.yaml)\n```\n\n### 7e. State file — scaffold with scope/project/repo filled in and snapshot ids empty (§6).\n\n### 7f. Worked example — Vercel Sandbox (all three phases)\n\nA real, working shape (the Vercel surface is a CLI: `vercel sandbox create|exec|snapshot|remove`). Adapt\nnames; verify flags against `vercel sandbox --help` for the user's CLI version before relying on them.\nThese ground §7a (base snapshot) and §7b (auth), which are otherwise generic skeletons.\n\n**Phase 2 — base snapshot (§7a):** provision → install tools + clone + headless build → snapshot.\n\n```bash\n# provision a fresh build sandbox (retain a couple of snapshots); trap-remove on error\nvercel sandbox create --name \"$base\" --runtime node24 --timeout 30m --vcpus 4 --publish-port \"$port\" \\\n --snapshot-expiration 30d --keep-last-snapshots 2 \"${vercel_args[@]}\" >&2\n# remote build (long timeout): install pkgs+gh+pnpm+agent CLI, clone with GIT_ASKPASS (write the helper\n# with LITERAL \\$1/\\$GH_TOKEN so they resolve at git-runtime, not write-time — see §5/§7f create — then\n# `rm -f /tmp/askpass.sh`), write the headless main-only build config (drop the renderer), dev setup,\n# build CLI + headless main, smoke-check\nvercel sandbox exec \"$base\" \"${vercel_args[@]}\" --timeout 25m --env \"GH_TOKEN=$gh_token\" … -- bash -lc '…build…' >&2\n# snapshot the STOPPED sandbox and parse the id from CLI output (fail if unparseable)\nout=\"$(vercel sandbox snapshot \"$base\" --stop --expiration 30d \"${vercel_args[@]}\" 2>&1)\"; printf '%s\\n' \"$out\" >&2\nsnapshot_id=\"$(printf '%s\\n' \"$out\" | sed -nE 's/.*(snap_[A-Za-z0-9]+).*/\\1/p' | tail -1)\"\n# merge { baseName, snapshotId, scope, project, port, repoUrl, repoRef, projectRoot } into state; print state JSON\n```\n\n**Phase 3 — agent-auth snapshot (§7b):** boot the base, log the agent in interactively, re-snapshot.\n(`codex` below is an example — substitute the user's chosen agent's login/status verbs, e.g. `claude`.)\n\n```bash\nvercel sandbox create --name \"$auth\" --snapshot \"$snapshot_id\" --timeout 30m --publish-port \"$port\" \"${vercel_args[@]}\" >&2\n# INTERACTIVE — the USER runs this in their own terminal (you have no interactive TTY) and completes the\n# URL/code on the HOST. --device-auth is MANDATORY on a headless VM: plain `codex login` binds a loopback\n# callback port the host browser can't reach and hangs. Ask the user to report back when login finishes.\nvercel sandbox exec --interactive --tty \"$auth\" \"${vercel_args[@]}\" -- bash -lc 'codex login --device-auth'\n# refuse to snapshot an unauthenticated VM — fold stderr, match codex's exact success line (§4)\nvercel sandbox exec \"$auth\" \"${vercel_args[@]}\" --timeout 30s -- bash -lc 'codex login status 2>&1' | grep -Eqi 'Logged in using ChatGPT|Logged in via device' \\\n || { echo \"agent not logged in; not snapshotting\" >&2; exit 1; }\nout=\"$(vercel sandbox snapshot \"$auth\" --stop --expiration 30d \"${vercel_args[@]}\" 2>&1)\"; printf '%s\\n' \"$out\" >&2\nnew_id=\"$(printf '%s\\n' \"$out\" | sed -nE 's/.*(snap_[A-Za-z0-9]+).*/\\1/p' | tail -1)\"\n# overwrite state.snapshotId = new_id, record authSourceSnapshotId = snapshot_id; remove the auth sandbox\n```\n\n**Per-workspace `create`** (the fast path):\n\n```bash\n#!/usr/bin/env bash\nset -euo pipefail\n# resolve from env→state→fallback: snapshot_id, scope, project, port, repo_url, repo_ref, project_root\nvercel_args=(); [ -n \"$scope\" ] && vercel_args+=(--scope \"$scope\"); [ -n \"$project\" ] && vercel_args+=(--project \"$project\")\n[ -n \"$snapshot_id\" ] || { echo \"snapshotId missing — run Phases 2–3 first\" >&2; exit 1; }\ngh_token=\"${GH_TOKEN:-${GITHUB_TOKEN:-$(command -v gh >/dev/null 2>&1 && gh auth token 2>/dev/null || true)}}\"\nname=\"orca-${ORCA_VM_RECIPE_ID:-vercel-sandbox}-${ORCA_VM_INSTANCE_ID:-$(date +%s)}\" # sanitize+cap to 63 chars\n\n# Arm cleanup BEFORE create so a failing create can't leak a half-built paid sandbox.\ncleanup_on_error() { [ \"$?\" -ne 0 ] && vercel sandbox remove \"$name\" \"${vercel_args[@]}\" >/dev/null 2>&1 || true; }\ntrap cleanup_on_error EXIT\n\n# 1. boot from the authenticated snapshot, publish the serve port\ncreate_output=\"$(vercel sandbox create --name \"$name\" --snapshot \"$snapshot_id\" \\\n --timeout 30m --publish-port \"$port\" \"${vercel_args[@]}\" 2>&1)\"; printf '%s\\n' \"$create_output\" >&2\n# Vercel prints the published https URL; derive the external wss:// pairing address from it\npublic_url=\"$(printf '%s\\n' \"$create_output\" | sed -nE 's#.*(https://[^[:space:]]+\\.vercel\\.run).*#\\1#p' | head -1)\"\n[ -n \"$public_url\" ] || { echo \"no published URL in create output\" >&2; exit 1; }\npairing_ws=\"${public_url/https:\\/\\//wss://}\"\n\n# 2. (remote) ensure the repo is at the right commit; rebuild only if the commit changed (cache marker)\nvercel sandbox exec \"$name\" \"${vercel_args[@]}\" --timeout 20m \\\n --env \"GH_TOKEN=$gh_token\" --env \"ORCA_PROJECT_ROOT=$project_root\" \\\n --env \"ORCA_REPO_URL=$repo_url\" --env \"ORCA_REPO_REF=$repo_ref\" \\\n -- bash -lc 'set -euo pipefail; cd \"$ORCA_PROJECT_ROOT\"; \\\n # Re-establish git auth for the private-repo fetch (why + full rationale: §5); else it hangs on a prompt.\n # Load-bearing escaping: \\$1 and \\$GH_TOKEN must land LITERALLY and resolve at git-runtime. Test after\n # any edit here — reformatting the nested printf/node quoting silently breaks the fetch or leaks the token.\n if [ -n \"${GH_TOKEN:-}\" ]; then \\\n printf \"%s\\n\" \"#!/usr/bin/env bash\" \"case \\\"\\$1\\\" in *Username*) echo x-access-token;; *Password*) echo \\\"\\$GH_TOKEN\\\";; esac\" > /tmp/askpass.sh; \\\n chmod 700 /tmp/askpass.sh; export GIT_ASKPASS=/tmp/askpass.sh GIT_TERMINAL_PROMPT=0; fi; \\\n git fetch origin \"$ORCA_REPO_REF\"; \\\n git checkout -B \"$ORCA_REPO_REF\" FETCH_HEAD; \\\n rm -f /tmp/askpass.sh; \\\n c=\"$(git rev-parse HEAD)\"; [ -f .orca-built ] && [ \"$(cat .orca-built)\" = \"$c\" ] || { \\\n pnpm install --prefer-offline && pnpm run build:cli && \\\n node config/scripts/run-electron-vite-build.mjs --config config/electron-vite.vm-serve.config.ts && \\\n printf \"%s\" \"$c\" > .orca-built; }' >&2\n\n# 3. (remote) start orca serve in the background, writing recipe JSON to a file; poll until it parses\nrecipe_json=\"$(vercel sandbox exec \"$name\" \"${vercel_args[@]}\" --timeout 60s \\\n --env \"ORCA_PORT=$port\" --env \"ORCA_PROJECT_ROOT=$project_root\" --env \"ORCA_PAIRING_ADDRESS=$pairing_ws\" \\\n -- bash -lc 'set -euo pipefail; cd \"$ORCA_PROJECT_ROOT\"; rm -f /tmp/orca-recipe.json /tmp/orca-serve.log; \\\n nohup pnpm exec orca-dev serve --port \"$ORCA_PORT\" --project-root \"$ORCA_PROJECT_ROOT\" \\\n --pairing-address \"$ORCA_PAIRING_ADDRESS\" --recipe-json >/tmp/orca-recipe.json 2>/tmp/orca-serve.log /dev/null 2>&1 && { cat /tmp/orca-recipe.json; exit 0; }; \\\n kill -0 \"$pid\" 2>/dev/null || { cat /tmp/orca-serve.log >&2; exit 1; }; sleep 0.25; \\\n done; cat /tmp/orca-serve.log >&2; echo \"serve recipe JSON timed out\" >&2; exit 1')\"\n\n# 4. print serve's JSON enriched with userData (single object on stdout)\nnode -e 'const p=JSON.parse(process.argv[1]); console.log(JSON.stringify({...p, schemaVersion:1,\n userData:{...p.userData, provider:\"vercel-sandbox\", resourceId:process.argv[2], snapshotId:process.argv[3]}}))' \\\n \"$recipe_json\" \"$name\" \"$snapshot_id\"\ntrap - EXIT\n```\n\n`suspend`/`resume`/`destroy` use `vercel sandbox stop|...|remove \"$resource_id\"` reading\n`userData.resourceId` from stdin (§7d). This is the **Orca-server** connection mode (the recipe emits a\npairing URL). If the user chose **SSH** in the §1 interview, use §7g instead.\n\n### 7g. Worked example — existing SSH host (SSH connection mode)\n\nSSH mode is **fundamentally different from §7c/§7f**, not a relabeling of them:\n\n- **`create` does NOT run `orca serve` and does NOT emit a `pairingCode`.** Orca itself connects to the\n host over its SSH relay, brings up the git + filesystem providers, and imports the repo. The script's\n only job is to make the host ready and **print SSH connection details** Orca will dial.\n- The result uses a `connection` block with `type: \"ssh\"` and a `target`, **not** the flat\n `pairingCode`/`projectRoot` shape. Exact shape (Orca rejects anything else):\n\n```json\n{\n \"schemaVersion\": 1,\n \"connection\": {\n \"type\": \"ssh\",\n \"projectRoot\": \"/abs/path/to/repo/on/host\",\n \"target\": {\n \"label\": \"my-box\",\n \"host\": \"192.0.2.10\",\n \"port\": 22,\n \"username\": \"ubuntu\",\n \"identityFile\": \"~/.ssh/id_ed25519\",\n \"jumpHost\": \"bastion.example.com\",\n \"proxyCommand\": \"cloudflared access ssh --hostname %h\",\n \"relayGracePeriodSeconds\": 0,\n \"portForwards\": []\n }\n }\n}\n```\n\n`label`, `host`, `port`, `username` are required; the rest are optional — omit any you don't need.\n\n**Networking → which `target` fields to set** (how *your desktop* reaches the box — there is no\n`orca serve` URL in SSH mode):\n\n- Public IP / DNS, or a Tailscale/VPN address → `host`; SSH port → `port` (usually 22).\n- Key auth → `identityFile` (add `identitiesOnly: true` if the agent has many keys).\n- Through a bastion → `jumpHost` (a `user@host` ProxyJump) **or** a full `proxyCommand` (e.g. an access\n proxy). Use one, not both.\n- A service port the workspace needs → add entries to `portForwards`.\n- `relayGracePeriodSeconds` (optional): how long Orca keeps the SSH relay alive after the workspace\n detaches before tearing it down; `0` = tear down immediately. Leave it off unless the user wants a\n reconnect grace window.\n\n**Toolchain & agent auth on a persistent (no-snapshot) host — do this ONCE, by hand, before wiring the\nrecipe** (there's no base image to bake; the host *is* the base). Run the §7f Phase-2 install steps and\nthe §7f Phase-3 ` login --device-auth` **directly over SSH on the host** (interactive, e.g.\n`ssh -t user@host ' login --device-auth'`). After that the host stays ready across workspaces.\n\n```bash\n#!/usr/bin/env bash\nset -euo pipefail\n# resolve from env→state→fallback (default unset optionals to \"\"): ssh_username, host,\n# ssh_port (default 22), identity_file, jump_host, proxy_command, project_root, repo_url, repo_ref\n: \"${identity_file:=}\"; : \"${jump_host:=}\"; : \"${proxy_command:=}\" # avoid set -u aborts on optionals\ngh_token=\"${GH_TOKEN:-${GITHUB_TOKEN:-$(command -v gh >/dev/null 2>&1 && gh auth token 2>/dev/null || true)}}\"\nssh_target=\"${ssh_username}@${host}\"\nssh_opts=(-p \"$ssh_port\"); [ -n \"$identity_file\" ] && ssh_opts+=(-i \"$identity_file\")\n# Why: a fresh host's key isn't in known_hosts; a StrictHostKeyChecking prompt would HANG a\n# non-interactive create. Pre-add the key (or set the option) so it can't block.\nssh-keyscan -p \"$ssh_port\" \"$host\" >> \"$HOME/.ssh/known_hosts\" 2>/dev/null || true\n\n# 1. ensure the repo is present and at the right commit on the host (NO orca serve here)\nssh \"${ssh_opts[@]}\" \"$ssh_target\" \\\n \"GH_TOKEN='$gh_token' GIT_TERMINAL_PROMPT=0 bash -lc '\n set -euo pipefail\n [ -d \\\"$project_root/.git\\\" ] || git clone \\\"$repo_url\\\" \\\"$project_root\\\"\n cd \\\"$project_root\\\" && git fetch origin \\\"$repo_ref\\\" && git checkout -B \\\"$repo_ref\\\" FETCH_HEAD\n '\" >&2\n\n# 2. print the SSH connection block (NO pairingCode, NO orca serve). host/port/username tell Orca's\n# relay how to dial in; identityFile/jumpHost/proxyCommand/portForwards are emitted when set.\nnode -e 'const [host,port,user,idf,jh,pc,root]=process.argv.slice(1);\n const target={ label:\"per-workspace-host\", host, port:Number(port), username:user };\n if(idf) target.identityFile=idf; if(jh) target.jumpHost=jh; if(pc) target.proxyCommand=pc;\n // add target.portForwards=[...] here if the workspace needs forwarded service ports\n console.log(JSON.stringify({ schemaVersion:1, connection:{ type:\"ssh\", projectRoot:root, target } }))' \\\n \"$host\" \"$ssh_port\" \"$ssh_username\" \"$identity_file\" \"$jump_host\" \"$proxy_command\" \"$project_root\"\n```\n\n`suspend`/`resume`/`destroy`: on a persistent host there's usually nothing to tear down — set\n`destroy: none` and omit suspend/resume. (Orca still disconnects/reconnects its own SSH relay on\nsleep/wake/delete — that's separate from these scripts.)\n\nIf the SSH host is instead an **ephemeral/snapshot-capable VM** (your hypervisor, or a cloud VM with\nimage support), keep the §7f Phase-2/3 base-image model for provisioning, but still emit the\n`connection.type:\"ssh\"` block above instead of starting `orca serve`.\n\n### 7h. Worked example — local Docker SSH (SSH connection mode)\n\nLocal Docker can model an ephemeral SSH VM without cloud cost: build a base image with `sshd`, tools,\nrepo prerequisites, and the agent CLI; run an **interactive auth container** once; then `docker commit`\nthat container as the authenticated image used by per-workspace `create`.\n\nKey points:\n\n- Publish container SSH to a random localhost port (`-p 127.0.0.1::22`) and emit\n `connection.type:\"ssh\"` with `host:\"127.0.0.1\"`, that port, `username`, `identityFile`, and\n `identitiesOnly:true`.\n- Generate a repo-local SSH key if needed, but gitignore the private/public key files.\n- **Bake SSH host keys into the base image** (`ssh-keygen -A` at **build** time; at runtime only generate\n if absent). Ephemeral containers all present the **same** host key, so `known_hosts` on `127.0.0.1`\n doesn't churn as the published port rotates across workspaces (otherwise every container's freshly\n generated key collides on `localhost` and trips host-key-changed warnings).\n- The auth image is the Docker equivalent of Phase 3: the **user** runs the agent login **inside** the\n container (you can't drive it — you have no interactive TTY), configures proxy env/config, approves\n hooks, and you commit once they report it's done. On a headless container use the **device-auth** flow\n (§4). Verify login before committing — exit code, or fold stderr and match the exact success line (§4).\n- Do not bind-mount or copy the host's full agent home into the image. Let each container have writable\n agent state; only the committed auth image should carry reusable authenticated state.\n- If committing from an interactive shell, force the runtime entrypoint back to `sshd`:\n `docker commit --change='ENTRYPOINT [\"/usr/local/bin/orca-docker-ssh-entrypoint\"]' …`.\n- `destroy` should read `recipeResult.userData.resourceId` and run `docker rm -f \"$resource_id\"`.\n\nValidation before wiring/live use:\n\n```bash\ndocker image inspect \"$auth_image\" --format '{{json .Config.Entrypoint}}'\ndocker run -d --name \"$name\" -p 127.0.0.1::22 -e \"ORCA_SSH_PUBLIC_KEY=$pubkey\" \"$auth_image\"\ndocker ps -a --filter \"name=$name\"\ndocker logs \"$name\"\nssh -i \"$key\" -p \"$port\" -o IdentitiesOnly=yes user@127.0.0.1 'codex --version'\n```\n\nIf the container exits immediately, inspect logs before the cleanup trap removes it; a committed\ninteractive image with `ENTRYPOINT [\"bash\"]` is a common cause.\n\nAlso confirm the **host key is stable** across containers: the SSH `ssh -i … 127.0.0.1` dial should not\ntrigger a host-key-changed warning when a second container reuses the port. If it does, the host keys\nweren't baked into the base image (see the `ssh-keygen -A` point above).\n\n### 7i. Windows local-side scripts\n\nThe local-side scripts run on the user's desktop. On **Windows**, a bare `.sh` won't execute. Either\nrequire WSL/Git-Bash (and point `orca.yaml` at e.g. `bash ./scripts/orca-vm/.sh` via a `.cmd`\nlauncher), or scaffold PowerShell equivalents. Minimal PowerShell shape:\n\n```powershell\n#requires -Version 5\n$ErrorActionPreference = 'Stop'\n# resolve env→state→fallback; run the provider CLI / ssh the same way;\n# capture provider output; build the result object for the chosen mode and write ONE line of JSON to stdout.\n# Orca-server mode: @{ schemaVersion=1; pairingCode=$pairingCode; projectRoot=$projectRoot; userData=@{...} }\n# SSH mode: @{ schemaVersion=1; connection=@{ type=\"ssh\"; projectRoot=$projectRoot;\n# target=@{ label=$label; host=$host; port=$port; username=$user } } } (see §7g/§7h)\n($result | ConvertTo-Json -Compress -Depth 6)\n# progress/errors → Write-Error / the error stream, never stdout.\n```\n\nThe remote-side commands you run *inside* the Linux VM stay bash regardless of the desktop OS.\n\n---\n\n## 8. Per-workspace recipe contract (the fast path)\n\nOnce the authenticated snapshot exists, this runs on every workspace create. Define recipes in\n`orca.yaml`:\n\n```yaml\nenvironmentRecipes:\n - id: cloud-sandbox\n name: Cloud Sandbox\n create: ./scripts/orca-vm/cloud-sandbox-create.sh\n suspend: ./scripts/orca-vm/cloud-sandbox-suspend.sh\n resume: ./scripts/orca-vm/cloud-sandbox-resume.sh\n destroy: ./scripts/orca-vm/cloud-sandbox-destroy.sh\n```\n\n`create` runs **locally from the repo root** and prints **one** JSON object to stdout. Its shape depends\non the connection mode chosen in §1:\n\n**Orca-server mode** — boot the env, start `orca serve` in it, and print serve's result:\n\n```json\n{\n \"schemaVersion\": 1,\n \"pairingCode\": \"orca-pairing-code-or-url\",\n \"projectRoot\": \"/absolute/path/to/repo/on/remote\",\n \"userData\": { \"provider\": \"example\", \"resourceId\": \"provider-resource-id\" }\n}\n```\n\nHere `pairingCode` (from `orca serve --recipe-json`) and `projectRoot` are required; `schemaVersion` (`1`)\nand `userData` are optional.\n\n**SSH mode** — do **not** run `orca serve`; print the `connection.type:\"ssh\"` block instead (full shape +\nworked script in §7g). `pairingCode` is **not** used in SSH mode.\n\nLifecycle hooks (all run locally):\n\n- `create`: required. Prints recipe result JSON.\n- `suspend`: optional. Sleep; reads lifecycle payload on stdin.\n- `resume`: optional. Wake; reads payload on stdin and **prints fresh recipe JSON** (pairing may change).\n- `destroy`: optional unless `destroy: none`. Delete/cleanup; reads payload on stdin.\n\nStart Orca remotely with `orca serve --port \"$PORT\" --project-root \"$ABS_ROOT\" --pairing-address\n\"$EXTERNAL_WSS_URL\" --recipe-json` (exact flags + output in §7c). Set `--pairing-address` to the\nexternally reachable address so the emitted `pairingCode` is reachable; tunneling/port mapping is the\nscript's job.\n\nBackward compatibility: `command`→`create`, `cleanup`→`destroy`, `cleanup: none`→`destroy: none`.\nPrefer the lifecycle names.\n\n---\n\n## 9. Doctor and validation\n\nValidate in two stages — the cheap dry run first, then the live self-test.\n\n### Dry run (free, non-destructive) — always do this first\n\n`orca vm recipe doctor --repo-path --json` validates **static wiring only** — it does\n**not** boot anything. It checks: local-host execution (v1), repo path, recipe id exists,\ncreate/destroy/suspend/resume command paths resolve, suspend/resume are paired, and each script is\nexecutable (POSIX exec bit; skipped on Windows). Fix every failure here before spending any cloud money.\n\n### Live self-test (`--provision`) — diagnose and iterate yourself\n\n`orca vm recipe doctor --repo-path --provision --json` actually runs the recipe end\nto end: it executes `create`, validates the returned recipe JSON, then runs `destroy` to **tear the\nenvironment back down** (so the test leaves nothing running, as long as `destroy` works). It spends real\ncloud money, so get the user's OK **once** before starting — that one approval covers the whole loop\nbelow; do not re-ask before each run.\n\nOn failure, the JSON result includes a `provisionTranscript` with the **complete** captured output of\neach stage so you can self-diagnose without asking the user to relay logs:\n\n```json\n{\n \"ok\": false,\n \"checks\": [ { \"id\": \"recipe.provision\", \"status\": \"fail\", \"message\": \"…\" } ],\n \"provisionTranscript\": {\n \"provision\": { \"exitCode\": 0, \"signal\": null, \"stdout\": \"…\", \"stderr\": \"…\", \"parseError\": \"…\" },\n \"destroy\": { \"exitCode\": 0, \"signal\": null, \"stdout\": \"…\", \"stderr\": \"…\" }\n }\n}\n```\n\n**Run it as a loop:** read `provisionTranscript.provision.stderr` / `.stdout` / `.parseError` (and\n`destroy.*`), fix the script, and re-run `--provision` until `ok` is `true` — iterating on your own\nrather than waiting for the user to paste errors. Common reads: a non-empty `stderr` with `exitCode 0`\nplus a `parseError` means `create` ran but printed something other than the single recipe-result JSON on\nstdout (often a stray `echo` — route it to stderr, see §10); a non-zero `exitCode` is a provider/script\nfailure described in `stderr`. Each stream is redacted and capped (head+tail) — large logs keep both the\nsetup context and the failure.\n\nThe self-test cannot see provider-side truth beyond what the scripts print, so still confirm: state has a\npopulated **authenticated** `snapshotId` (Phases 2–3 done), and `destroy` is implemented/tested (or\nexplicitly `none` — in which case the self-test won't tear down, so clean up manually).\n\nFor SSH recipes, also smoke-test the exact emitted target before declaring success: dial the host/port\nwith the identity/proxy settings, run `pwd`, verify the repo path, check the agent binary, and confirm\n`destroy` removes the provider resource/container. For Docker, inspect the auth image entrypoint and do a\nstartup-only `docker run` before the full clone/install path.\n\n---\n\n## 10. Failure modes\n\n- **Build exceeds plan timeout (e.g. Hobby 45m).** Use enough vCPUs and a timeout covering the build;\n else split work or use a higher plan. The cap also limits per-workspace runtime — surface it.\n- **Build exceeds plan RAM.** Build the **headless main only** (drop the renderer) — the biggest fitter.\n- **Private-repo clone hangs/fails.** Wrong/missing token. Use `GIT_ASKPASS` + `GIT_TERMINAL_PROMPT=0`\n so it fails fast instead of prompting.\n- **`GIT_ASKPASS` helper aborts the clone with \"`$1: unbound variable`\".** The `printf`/heredoc that writes\n the helper inside `bash -lc` under `set -u` expanded `$1`/`$GH_TOKEN` at **write** time. Escape them\n (`\\$1`, `\\$GH_TOKEN`) so they land literally and resolve at git-runtime; this also keeps the real token\n out of the file. `rm -f` the helper afterward (§5, §7f).\n- **Agent verified as \"not logged in\" despite a good login.** `codex login status` (and similar) print\n \"Logged in …\" to **stderr**; an stdout-only `grep` misses it. Prefer the status **exit code**; if you\n grep, fold stderr first (`status 2>&1 | grep …`) and match the exact success line — not `grep -qi\n 'logged in'`, which also matches \"not logged in\".\n- **Headless agent login hangs.** Plain OAuth `login` starts a loopback callback server on a VM/container\n port the host browser can't reach. Use the **device-auth** flow (`login --device-auth`) — it prints a\n URL + code the user opens on the host.\n- **`known_hosts` host-key churn on local Docker.** Each ephemeral container regenerating its SSH host key\n collides on `127.0.0.1` as the published port rotates. Bake host keys into the base image at build time\n (`ssh-keygen -A`; runtime generates only if absent) so all containers share one stable key (§7h).\n- **Snapshot expired/evicted.** If `create` hits an unknown snapshot id, rerun Phases 2–3 and update\n `snapshotId`.\n- **Agent auth didn't persist.** Confirm `snapshotId` points at the **authenticated** snapshot; re-run\n Phase 3. Warn that short-lived tokens may need periodic re-auth.\n- **Agent auth copied from the host breaks.** Do not bind-mount/copy a full host agent home; sqlite\n files can be unwritable or host-specific, hooks may need approval again, and config may reference\n local-only env vars. Authenticate inside the runtime and snapshot/commit that layer.\n- **Docker auth image exits immediately.** Inspect `docker image inspect … .Config.Entrypoint` and\n `docker logs`. If the image was committed from an interactive shell, reset the entrypoint to the SSH\n entrypoint during `docker commit`.\n- **Leaked paid resource.** Every long script must trap errors and remove the sandbox it created.\n- **`create` emits non-JSON on stdout.** A stray `echo` corrupts the result — stdout is for the final\n JSON only; everything else to stderr. The `--provision` self-test surfaces this as `exitCode 0` + a\n `parseError` with the offending stdout in `provisionTranscript` (§9).\n\n---\n\n## 11. Boundaries\n\n- Don't create accounts, choose plans/regions, or invent scope/project/org/image/billing ids.\n- Don't invent or store credentials; no secrets in `userData`, state, comments, docs, or commits.\n- Don't run paid/long phases (base snapshot, auth, live test) without an explicit OK.\n- Don't hide provider errors behind generic messages — preserve actionable stderr.\n- Don't make Orca own provider lifecycle beyond invoking the configured scripts.\n- Don't commit or create an Orca workspace unless asked.\n" // oxfmt-ignore -const ORCHESTRATION_MARKDOWN = "---\nname: orchestration\ndescription: >-\n Use Orca orchestration for structured multi-agent coordination: threaded\n messages, blocking ask/reply flows, task dispatch, worker_done/escalation\n waits, task DAGs, decision gates, coordinator loops, or decomposing work\n across agents. Use `orca-cli` instead for full ownership handoffs, including\n requests phrased as \"hand off\", \"handoff\", \"handover\", \"give this to another\n agent\", or \"another worktree\" when the user did not explicitly ask to\n supervise, monitor, wait for results, or coordinate a DAG. Use `orca-cli` for\n ordinary terminal control, lightweight terminal prompts, shell commands, Orca\n worktree management, reading or waiting on terminals, and automation of the\n browser embedded inside Orca. Use Computer Use for browser windows, webviews,\n Orca app UI, or desktop UI outside Orca's embedded browser.\n---\n\n# Orca Inter-Agent Orchestration\n\nOrchestration is Orca's structured coordination layer for agent messages, task ownership, dispatch state, and worker completion tracking.\n\nUse this skill when coordination state matters. For lightweight terminal prompts or basic worktree/terminal/built-in-browser control, use `orca-cli`.\n\n## Tool Boundary\n\nIf a task says to use Orca orchestration, the coordinator must create or bind a Run, create the Task with `orca orchestration task-create`, then attach the worker with either the preferred `orca orchestration worker-start` composition or the low-level `orca orchestration dispatch --inject` path.\n\nDo not substitute non-Orca subagent tools, generic agent-spawn APIs, or chat-only parallel worker features. Those may create useful workers, but they do not create Orca task/dispatch provenance, injected lifecycle preambles, `worker_done` authority, or decision gates.\n\nBefore claiming a worker was orchestrated, verify the task/dispatch exists:\n\n```bash\norca orchestration task-list --json\norca orchestration dispatch-show --task --json\n```\n\nIf the work was accidentally run outside Orca orchestration, say so plainly. To repair provenance, rerun or revalidate the needed work through a fresh Orca terminal plus injected dispatch; do not retroactively describe the external worker as orchestrated.\n\n## When To Use\n\n- Send/reply/ask between agent terminals with persistent messages.\n- Dispatch structured tasks to workers and wait for `worker_done` or `escalation`.\n- Track task DAGs with dependencies.\n- Run coordinator loops or decision gates.\n\nDo not use orchestration merely because the user says \"hand off\", \"handoff\", \"handover\", \"give this to another agent\", or asks for another worktree/agent/model/effort. Those are full ownership transfers unless the user explicitly asks to supervise, monitor, wait for worker completion/results, coordinate a DAG, use decision gates, or keep a blocking ask/reply loop.\n\n## Preconditions\n\n- `orca status --json` should show a running runtime.\n- `orca` must be on PATH (`orca-ide` on Linux).\n- The orchestration experimental feature must be enabled in Settings > Experimental.\n- `orca orchestration` commands are RPC calls to the running Orca runtime.\n\n## Contract Migration\n\nOrca adopts a live pre-update orchestration assignment into an ordinary Run. Adoption preserves the existing agent process, PTY/session, terminal handle, tab/leaf/pane, worktree or folder workspace, Task, and Dispatch; it never restarts or replaces the worker. The retired scheduler is not revived, and a newly created attempt uses the current grammar.\n\nTreat the authority label on injected or formatted messages as definitive:\n\n- `[LEGACY COMPATIBILITY]` is live and attested. Run only the exact supported command printed with the message, using the same CLI executable and arguments that the original prompt supplied.\n- `[LEGACY RECOVERY REPLAY — MAY HAVE BEEN SEEN]` is one bounded, at-least-once cutover replay. Process it idempotently and acknowledge it only through the exact displayed guidance.\n- `[LEGACY READ-ONLY]` is inspection-only. It has no reply, acknowledgment, or lifecycle action.\n- An unlabeled current message uses the current guide and current grammar.\n\nDatabase provenance, an old-looking terminal, or a legacy Run ID does not prove mutation authority. If the runtime cannot prove liveness, principal ownership, capability, or the exact legacy contract, it degrades to read-only inspection and must not fall back to local execution. Exact recovery may restore the already-live PTY once in its original inactive background tab. It must not spawn, write, signal, stop, switch, focus, split, or inject a terminal. Loss of lifecycle authority does not invalidate the existing assignment, process, or filesystem work.\n\nCompatibility retries have narrow guarantees. A pending ask, a reply, a final Dispatch settlement, and a consuming check have durable recovery identities. A-era heartbeat and escalation calls remain at-least-once across a manual A-to-B retry because identical later signals may be intentional. If an A-era ask may already have been answered, run the exact non-consuming recovery check printed by the runtime first; after its answer is printed and acknowledged, a new invocation with the same question creates a new question. Never guess among multiple identical question threads.\n\nWhen a compatibility or recovery command returns structured next-step arguments, run those exact arguments with the same CLI executable. The arguments intentionally omit the executable name so the guidance works with `orca`, `orca-ide`, `orca-dev`, or another configured Orca CLI command. Do not translate the command from memory, broaden its recipient, or retry it as a current mutation unless the returned guidance explicitly says to.\n\nOn packaged Windows, a legacy ask uses a two-step commit/resume protocol. The initial command durably commits the question, prints its exact `ask --resume ` command, and exits with launcher status `75`; it does not wait for the answer. Run that exact resume command after the launcher or update boundary. Resume is idempotent and read-oriented: it waits for the already-committed question and does not create another one. For a WSL process that received compatibility proof at launch, use the printed executable `orca-ide` WSL resume command so the same distro and packaged launcher authority are preserved; do not substitute a PATH-resolved local CLI. Older WSL processes that never received the hidden launch token remain lifecycle read-only after the update, even while their terminal and filesystem work continue.\n\nLegacy inspection remains available without consuming mail:\n\n```bash\norca orchestration run-list --json\norca orchestration run-show --id run_legacy_local --json\norca orchestration task-list --run run_legacy_local --json\norca orchestration inbox --full --json\norca orchestration check --terminal --peek --format --json\norca terminal read --terminal --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\n```\n\nDo not launch a replacement editor merely because the desktop app or runtime was updated. If adoption cannot prove continuing authority, keep the original worker as the only editor until it reaches a stable handoff point, then use a new current Dispatch in a conflict-free placement for any remaining work.\n\n## Ownership\n\nNew orchestration messages and tasks belong to one explicitly bound Run. A Run is only a durable namespace and coordinator inbox; it never schedules or places workers. Lifecycle authority comes from the active Dispatch, and terminal handles remain routing metadata rather than durable identity. Send `worker_done` and `heartbeat` from the worker's own terminal; Orca routes them to that Dispatch's Run.\n\nClassify inherited context before sending lifecycle messages:\n\n- Coordinated subtask: a live coordinator owns the DAG and waits on this dispatch. Follow the preamble exactly, including `worker_done`, heartbeat/status, `ask`, and `escalation`.\n- Full handoff means ownership transfer, not supervised dispatch. The original actor is not monitoring a DAG, so do not create lifecycle obligations unless the user explicitly asks you to supervise.\n- Classify requests containing \"hand off\", \"handoff\", \"handover\", \"give this to another agent\", \"give this to another worktree\", \"another agent\", or \"another worktree\" as full handoffs by default, even when the user names a custom model or reasoning effort.\n- Use supervised orchestration only when the user explicitly asks you to \"supervise\", \"monitor\", \"wait\", \"track completion\", \"wait for worker_done\", return results, coordinate a DAG, use a decision gate, or manage ask/reply flow.\n- Do not use `orca orchestration dispatch --inject` for full handoffs. It injects a coordinator preamble that tells the worker to send `worker_done`, heartbeat, and `ask` messages, then end its turn under the original terminal's dispatch lifecycle.\n- Do not run `orca orchestration task-create`, `orca orchestration dispatch --inject`, or `orca orchestration check --wait` for full handoffs. Do not peek at terminal output after prompt delivery to monitor progress.\n- A review-only `worker_done` reports findings; it does not authorize coordinator file edits. After a review-only completion, synthesize findings, ask a decision gate if ownership is unclear, and dispatch or hand off fixes unless the user explicitly asked the coordinator to own fixes.\n- If the user's plan names a next owner agent (for example, \"then use opencode to create a PR\"), post-review corrections and PR prep belong to that named owner. The coordinator routes, synthesizes, asks decision gates when needed, and supervises; the named owner edits files and creates the PR.\n\nIf unclear, inspect orchestration state before sending lifecycle messages:\n\n```bash\norca orchestration task-list --json\norca terminal list --json\n# If inherited context includes a task id:\norca orchestration dispatch-show --task --json\n```\n\n## Messaging\n\n```bash\norca orchestration send --subject [--to ] [--from ] [--body ] [--type ] [--priority ] [--thread-id ] [--payload ] [--json]\norca orchestration check [--terminal ] [--ack ] [--peek|--all] [--types ] [--format] [--wait] [--timeout-ms ] [--json]\norca orchestration reply --id --body [--from ] [--json]\norca orchestration ask (--question |--resume ) [--options ] [--timeout-ms ] [--from ] [--json]\norca orchestration inbox [--limit ] [--json]\n```\n\nRules:\n\n- Omit `--from` unless impersonating another terminal; Orca auto-resolves it from the current terminal.\n- A coordinator `check` returns the bound Run's oldest FIFO Delivery (up to 50 messages) and replays that exact batch until `--ack `. Process every message before acknowledging; `check --ack --wait` acknowledges, checks, and waits in one operation.\n- Use `--peek` and `--all` only for read-only history/debugging. Type filters decide when a waiter wakes; the returned actionable Delivery is still the oldest full batch.\n- Use `dispatch:` for coordinator guidance to one supervised worker. Orca routes that stable address locally or through the connected-server relay; do not substitute a remote terminal handle.\n- Terminal handles remain appropriate for low-level pre-Dispatch messaging. Prefer `agentTerminalHandle` from the create response, fall back to `startupTerminal.handle` for older runtimes, then re-resolve with `orca terminal list --worktree ... --json` if missing or stale. Continue with the replacement handle only; never dual-send to old and new handles.\n- `orca orchestration check --peek --format --json` returns locally formatted unread mail without consuming it; it never writes to terminal input or remotely wakes another terminal. Use `orchestration dispatch --inject` to deliver a tracked task, or `terminal send` when an existing agent needs a free-form prompt.\n- While supervising workers manually, use `check --wait --types worker_done,escalation,question --timeout-ms ` instead of sleep/poll loops. Process the whole Delivery, reply to `question` messages with `orca orchestration reply --id --body --json`, then acknowledge and keep waiting.\n- Treat a `check --wait` timeout or `{count:0}` as a checkpoint, not a worker failure. Long coding tasks routinely run 15-60 minutes; keep using rolling waits unless you receive `worker_done`/`escalation`, the terminal exits or disappears, or the user explicitly asks you to stop.\n- Heartbeats and visible terminal activity mean the worker is alive, not done. Do not stop, close, kill, or restart a worker just because it has not produced a completion message yet.\n- Use `ask` when a worker needs a blocking answer from the coordinator; it defaults to the active Dispatch's Run. Timeout or disconnect leaves the question pending, so resume by its original message ID instead of asking again.\n- `check --wait` returns one bounded Delivery, not every future completion. Process every message, acknowledge it, then keep waiting until every expected Dispatch settles.\n- Group addresses include `@all`, `@idle`, `@claude`, `@codex`, `@opencode`, `@gemini`, `@droid`, `@grok`, `@cursor`, and `@worktree:`.\n- Message types include `status`, `dispatch`, `worker_done`, `merge_ready`, `escalation`, `handoff`, `question`, `decision_gate` (legacy/gates), and `heartbeat`.\n- Use group addresses only for messages that are genuinely useful to many terminals, such as `status` broadcasts or intentional fan-out questions. Do not send dispatch lifecycle messages to groups.\n- `worker_done` belongs to the active Dispatch and defaults to its Run mailbox; never target a group.\n- A valid `worker_done` for the active `taskId` + `dispatchId` marks the task and dispatch completed automatically. Do not follow it with `task-update --status completed`; reserve manual updates for explicit recovery or overrides.\n- `heartbeat` is also Dispatch-scoped. Include both IDs and omit `--to` so Orca uses the owning Run; use `status` for broad progress updates.\n\n## Tasks And Dispatch\n\nA Run is the namespace/inbox, a Task is the work item, and a Dispatch assigns one Task attempt to a terminal. Create or bind a Run once before the common loop.\n\n```bash\norca orchestration run-create --objective --json\norca orchestration task-create --spec [--deps ] [--parent ] [--json]\norca orchestration task-list [--status ] [--ready] [--brief] [--json]\norca orchestration task-update --id --status [--result ] [--json]\norca orchestration dispatch --task --to [--from ] [--inject] [--json]\norca orchestration dispatch-show --task [--json]\n```\n\nTask statuses: `pending`, `ready`, `dispatched`, `completed`, `failed`, `blocked`.\n\nDispatch rules:\n\n- `--inject` sends the task spec plus preamble into a recognized agent CLI so it can report `worker_done`.\n- If the target is a bare shell, omit `--inject`, dispatch for tracking if needed, then send the prompt manually with `orca terminal send --terminal --text --enter --json`.\n- After 3 consecutive failures on one task, the dispatch context circuit-breaks and the task is marked failed.\n- Use `task-list --brief --json` for coordinator sweeps; it collapses whitespace and caps each echoed spec at 160 characters (`spec_truncated` marks shortened rows). Omit `--brief` when the full spec is required, or when an older CLI rejects it as an unknown flag.\n\n## Preferred Supervised Worker Loop\n\nUse `worker-start` for the normal supervised path. It composes the existing worktree, terminal, readiness, and dispatch primitives while returning exact created/reused effects. Agents still choose placement and concurrency; Orca does not schedule workers or infer conflicts.\n\nCreate the Run and every independent Task first, then start all independent workers before waiting:\n\n```bash\norca orchestration run-create --objective \"\" --json\norca orchestration task-create --spec \"\" --json\norca orchestration task-create --spec \"\" --json\norca orchestration worker-start --task --worktree current --agent codex --json\norca orchestration worker-start --task --worktree current --agent claude --json\n```\n\n`current` and exact existing worktrees create a fresh agent terminal and do not rerun setup. Reuse an existing agent only with `--terminal `.\n\nFor a new worktree, setup runs by default and agent-first creation reuses the returned startup agent terminal:\n\n```bash\norca orchestration worker-start --task --worktree new-child --name --agent codex --setup run --json\n# Independent/top-level:\norca orchestration worker-start --task --worktree new-top-level --name --agent codex --setup run --json\n```\n\nSetup normally starts alongside the agent. Only a repository explicitly configured with `wait-for-setup` delays agent launch until setup succeeds. Use `--setup skip` or `--setup inherit` only for a concrete reason.\n\nRead the returned receipt before continuing: `ready` plus setup `running` is normal for start-immediately, while wait-for-setup returns setup `succeeded` before accepting task input. A failed or unknown start exits nonzero; inspect its `stage`, `effects`, and `residualResources` instead of guessing or automatically retrying. A wait-for-setup timeout can honestly leave setup `running`, which is not proof of failure.\n\nTo run the worker on another connected Orca server, add `--on `. The Run and Tasks remain authoritative on the current server; later commands route by Dispatch ID, so never repeat `--on`:\n\n```bash\n# Mac Run home -> Windows worker (the reverse is identical from a Windows Run home)\norca orchestration worker-start --task --on windows --worktree new-top-level --repo --name --agent codex --setup run --json\norca orchestration worker-show --dispatch --json\norca orchestration worker-read --dispatch --limit 50 --json\norca orchestration send --to dispatch: --subject \"Follow-up\" --body \"\" --json\n```\n\nRemote `current` and `new-child` are intentionally invalid because those words are ambiguous across servers. Use an exact discovered remote worktree selector or `new-top-level` with an explicit remote repo selector.\n\nThe follow-up is structured inbox mail, not prompt injection. The worker's next\n`orchestration check` receives it even when the Dispatch is on another connected Orca server.\n\n`worker-read` defaults to `--source auto`: Orca returns the exact hook-reported Codex, Claude, OpenClaude, or Grok transcript when it can prove the worker session, otherwise it returns bounded terminal output with `source: \"terminal\"` and a typed `fallbackReason`. Continue with the returned top-level `cursor`; it stays pinned to that exact source. If Orca reports `source_changed`, start a fresh read without the old cursor. Never supply or guess a provider session ID or transcript path.\n\nWait until every expected Dispatch settles, not for a fixed number of batches:\n\n```bash\norca orchestration check --wait --types worker_done,escalation,question --timeout-ms 900000 --json\n# Process every message in the returned Delivery, then atomically ack and continue:\norca orchestration check --ack --wait --types worker_done,escalation,question --timeout-ms 900000 --json\n```\n\nWorkers report exactly once using the IDs and capability injected by Orca; they do not supply Run/server/terminal identity:\n\n```bash\norca orchestration send --type worker_done --subject \"\" --body \"\" --task-id --dispatch-id --outcome succeeded --files-modified \"path/a,path/b\" --json\n# On failure, use --outcome failed; never encode failure only in prose.\n```\n\nA worker question defaults to its owning Run. Timeout leaves it pending:\n\n```bash\norca orchestration ask --question \"\" --options \"yes,no\" --timeout-ms 600000 --json\norca orchestration ask --resume --timeout-ms 600000 --json\n# Coordinator:\norca orchestration reply --id --body \"\" --json\n```\n\nRecovery is conditional, never a fixed destructive sequence:\n\n- `worker-show --dispatch ` says `ready`: keep waiting or read bounded output.\n- It proves `failed` or `stopped`: start a replacement with `worker-start --task --retry-of ` plus an explicit `--on`/`--worktree` and `--agent`/`--terminal` choice. Retry does not silently inherit placement.\n- It remains `outcome_unknown`: either `worker-stop --dispatch ` and inspect again, or explicitly `worker-abandon --dispatch ` while accepting that resources may still be live. Abandon performs no remote, process, or filesystem action.\n- `worker-stop` closes only the exact supervised agent terminal. It never deletes the worktree, setup terminal, configured tabs, or unrelated processes.\n\nLow-level `worktree create`, `terminal create`, and `dispatch --inject` remain valid recipes for custom argv or topology that `worker-start` does not express.\n\n## Gates And Legacy Inspection\n\n```bash\norca orchestration gate-create --task --question [--options ] [--json]\norca orchestration gate-resolve --id --resolution [--json]\norca orchestration gate-list [--task ] [--status ] [--json]\n```\n\nUse `ask` for worker-to-coordinator questions; it creates a `question` message that the coordinator answers with `reply`. Use `gate-create` only for coordinator-managed task DAG decisions, not for answering a worker's `ask`.\n\n`coordinator-start`, `coordinator-stop`, `run`, and `run-stop` are retired scheduler commands. They perform no effects and return the current-skill recovery action. They are not aliases for lightweight Run creation or binding.\n\nRecovery only: `orca orchestration reset --tasks|--messages|--all --json` clears the selected local orchestration database state. Do not run it during active coordination unless explicitly abandoning that state.\n\n## Full Handoffs\n\nFor full ownership transfer, use non-lifecycle terminal/worktree commands and then stop monitoring unless the user asks for supervision.\n\nTreat these as full handoff requests by default: \"hand off\", \"handoff\", \"handover\", \"give this to another agent\", \"give this to another worktree\", \"send this to another agent\", \"another agent\", \"another worktree\", or \"launch another agent to own this.\" Custom model or reasoning effort words such as `gpt-5.5`, `high`, or `xhigh` do not make the handoff supervised.\n\nSupervised orchestration remains available only when the user explicitly asks for supervision or coordination: \"supervise\", \"monitor\", \"wait for worker_done\", \"wait for results\", \"track completion\", \"DAG\", \"decision gate\", \"ask/reply\", or \"coordinate workers.\"\n\nDo not run `orca orchestration task-create`, `orca orchestration dispatch --inject`, or `orca orchestration check --wait` for full handoffs. `task-create` is also forbidden because it records coordinator-owned tracking state; if a task row is needed, the user asked for supervised orchestration. Do not create a `taskId`/`dispatchId`, inject a lifecycle preamble, wait for completion, or read the worker terminal after prompt delivery except to avoid losing the initial prompt.\n\nNew top-level worktree handoff:\n\n```bash\norca worktree create --name --no-parent --agent codex --prompt \"\" --setup run --json\n```\n\nBefore creating a new worktree from an active feature branch, decide and state whether the desired Orca lineage is child or top-level. Use child worktree lineage only when the new work is conceptually stacked under or dependent on the active worktree. For independent repo-wide fixes, standalone feature work, or unrelated follow-up tasks, create a top-level worktree with `--no-parent`.\n\nExisting terminal handoff:\n\n```bash\norca terminal send --terminal --text \"\" --enter --json\n```\n\nCustom Codex model/effort handoff:\n\n`orca worktree create --agent codex --prompt ...` launches the known Codex agent but does not accept Codex-specific `--model` or `-c model_reasoning_effort=...` arguments. When the user asks for a specific Codex model or effort, create the independent worktree first, launch Codex with the requested command in that worktree, wait only for TUI readiness if prompt delivery would otherwise race startup, send the prompt, and stop.\n\nThe two-step custom-argv path cannot enforce a repository's explicit `wait-for-setup` startup policy because the later `terminal create` is not the startup owned by `worktree create`. Use it only when the repository starts agents immediately. If the repository requires `wait-for-setup`, use an agent-first configured launcher that can preserve sequencing, or stop and ask rather than silently bypassing the policy.\n\nNote: when no repo default-terminal configuration supplies a primary terminal, bare create opens a fallback shell before `terminal create` adds the agent. Configured default tabs are materialized instead and may run real commands. Prefer `--agent` whenever custom argv is not required. With the two-step path, target only the agent handle; close a prior terminal only after `terminal list` or `terminal show` confirms it is an unused shell.\n\nUse the exact full `::` worktree id returned by `orca worktree create --json`; a bare repo id cannot target the new worktree.\n\n```bash\norca worktree create --name --no-parent --setup run --json\norca terminal create --worktree id: --title --command 'codex --model gpt-5.5 -c model_reasoning_effort=\"xhigh\"' --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca terminal send --terminal --text \"\" --enter --json\n```\n\nWait only for `tui-idle` when needed to avoid losing the prompt. Do not monitor task completion.\n\n`--no-parent` only controls Orca lineage; it does not choose the Git base. If the work should start from the repo default base, omit `--base-branch` so Orca uses that default, or explicitly pass the repo default base (`origin/main`, `origin/master`, or the `orca repo show --repo --json` value); never base it on the current feature branch unless the user explicitly asks for stacked work or \"branch from current\". Put current-branch context in the prompt instead.\n\n## Worker Terminals\n\nChoose the worker location before creating a terminal. `Fresh worker` means a fresh agent session, not a new git worktree. For parallel work, create one fresh agent terminal per worker in the same required worktree, falling back to the active worktree when none is named. If the task says current worktree only, depends on uncommitted files/artifacts, or must validate/PR the current branch, keep every worker in the active worktree:\n\n```bash\norca terminal create --worktree active --title --command \"codex\" --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca orchestration dispatch --task --to --inject --json\n```\n\nReuse an idle agent in the required worktree only if the prompt allows reuse; otherwise create a fresh terminal there. Create a new worktree only when the user explicitly requests one or a concrete checkout or filesystem conflict makes sharing unsafe or impossible; if the user did not request it, state that conflict before running `worktree create`. Independent tasks, parallel execution, convenience, or a preference for separate checkouts are not isolation requirements.\n\nWhen a new worktree is allowed, use child lineage for isolated work that is stacked under or dependent on the active worktree, and use `--no-parent` when it is not stacked. Decide the Git base separately: `--no-parent` makes the worktree top-level in Orca, while omitted `--base-branch` uses the repo default base.\n\nFor every new worktree, pass `--setup run` so any configured repository setup hook runs. This does not mean waiting for setup before agent launch: preserve the repository's startup policy, whose default starts setup and the agent side by side. Use `--setup skip` or `--setup inherit` only when there is a concrete task-specific reason, and state that reason before creating the worktree. This rule does not rerun setup for current or existing worktrees.\n\n```bash\norca worktree create --name --agent codex --setup run --json\n# or: --agent claude | omp | pi | grok | ...\n# Read from agentTerminalHandle, falling back to startupTerminal.handle.\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca orchestration dispatch --task --to --inject --json\n```\n\nFor new-worktree workers, read the id and `agentTerminalHandle` from `worktree create`, falling back to `startupTerminal.handle` for older runtimes. Use that as the sole worker handle when present; otherwise use `terminal list` to resolve the agent handle. Omit `--repo` only inside an Orca-managed worktree; otherwise pass `--repo `.\n\n**For an allowed new worktree, use agent-first:** `--agent` reveals the new worktree and launches the selected agent **in its first terminal**, without adding a separate fallback shell for that worker. Pass `--setup run`; repo setup and default-terminal settings may add intentional tabs or splits. Do **not** run bare `worktree create` and then `terminal create --command ` for the same worker when agent-first create is available: without configured default tabs, that two-step path leaves a fallback shell + agent pair. Only use it when custom agent argv is required (for example Codex model/effort flags) or when an older CLI rejects `--agent`; if you must, message only the agent handle. Configured default tabs are intentional surfaces, so close a prior terminal only after `terminal list` or `terminal show` confirms it is an unused shell. Do not run `worktree create` when the task must stay in the current worktree.\n\nUse `orca worktree create --prompt ...` or `orca terminal send ...` for full handoffs or untracked/lightweight prompts. Those paths do not attach `taskId`/`dispatchId`; the worker should not send lifecycle messages unless the prompt supplies a live orchestration preamble.\n\nSidebar lineage and orchestration lifecycle are related but not identical. A same-worktree worker may appear as a peer under that worktree in the sidebar while remaining a child dispatch in orchestration state; only an actual child worktree creates visible parent/child worktree lineage.\n\nOther terminal commands coordinators often need:\n\n```bash\norca terminal list [--worktree ] [--json]\norca terminal create [--worktree ] [--title ] [--command ] [--json]\norca terminal split --terminal [--direction horizontal|vertical] [--command ] [--json]\norca terminal wait --terminal --for tui-idle --timeout-ms --json\norca terminal read --terminal --json\norca terminal send --terminal --text --enter --json\n```\n\nIf an older CLI rejects `worktree create --agent`, create the worktree normally, then run `orca terminal create --worktree --command \"codex\" --json` or `--command \"claude\"`.\n\nWait for `tui-idle` before dispatching. Always pass `--timeout-ms`; real coding tasks can take 15-60 minutes. During supervision, use rolling `check --wait` windows. If a window returns no matching message, inspect `task-list`, `terminal read`, or `terminal wait --for tui-idle` as a liveness checkpoint; if the terminal is still working or producing activity, keep waiting instead of retrying the task.\n\n## Agent Guidance\n\n- Workers with a valid live preamble must send `worker_done` exactly once from their own terminal with an explicit `--outcome succeeded` or `--outcome failed`:\n `orca orchestration send --type worker_done --subject \"\" --body \"<3-sentence summary: what you did, what you found, what's left>\" --task-id --dispatch-id --outcome succeeded --files-modified \"path/a\" --report-path \"\" --json`\n- A failed outcome is still a terminal report, but Orca records both the Dispatch and Task as failed. Never encode failure only in the subject/body.\n- After sending `worker_done`, end your turn and idle at the agent prompt. Do not poll or keep calling `orca orchestration check`; the coordinator re-engages you with a fresh preamble + TASK block delivered as new terminal input.\n- For long tasks, send heartbeat/status only when the preamble asks for it, including both IDs:\n `orca orchestration send --type heartbeat --subject \"alive\" --payload '{\"taskId\":\"\",\"dispatchId\":\"\",\"phase\":\"implementing\"}' --json`\n- If blocked before completion, use `ask`; use `escalation` only when ownership is valid and the coordinator must intervene.\n- Treat preambles inherited through terminal history or full handoffs as stale unless the current prompt explicitly keeps that coordinator in the loop.\n- Coordinators should use `task-list --ready` as external memory, dispatch parallel waves, and avoid dependency chains deeper than 3-4 steps.\n\n## Example\n\n```bash\norca terminal create --worktree active --title login-css-worker --command \"claude\" --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca orchestration task-create --spec \"Fix the login button CSS\" --json\norca orchestration dispatch --task --to --inject --json\norca orchestration check --wait --types worker_done,escalation,question --timeout-ms 900000 --json\n```\n\n## Next Action\n\nCoordinator: confirm `orca status --json`, create or bind a Run, inspect `task-list`/`dispatch-show` if inheriting state, then use the explicit supervised loop (`task-create` -> `worker-start` -> `check --wait`). Use low-level terminal creation plus `dispatch --inject` only when the composed start does not express the needed topology.\n\nWorker: if the current prompt contains a live dispatch preamble, do the task, use `ask` for blocking questions, and send `worker_done` once with the required payload. If the preamble is stale or absent, do not send lifecycle messages; inspect state or treat the prompt as an ordinary handoff.\n" +const ORCHESTRATION_MARKDOWN = "---\nname: orchestration\ndescription: >-\n Use Orca orchestration for structured multi-agent coordination: threaded\n messages, blocking ask/reply flows, task dispatch, worker_done/escalation\n waits, task DAGs, decision gates, coordinator loops, or decomposing work\n across agents. Use `orca-cli` instead for full ownership handoffs, including\n requests phrased as \"hand off\", \"handoff\", \"handover\", \"give this to another\n agent\", or \"another worktree\" when the user did not explicitly ask to\n supervise, monitor, wait for results, or coordinate a DAG. Use `orca-cli` for\n ordinary terminal control, lightweight terminal prompts, shell commands, Orca\n worktree management, reading or waiting on terminals, and automation of the\n browser embedded inside Orca. Use Computer Use for browser windows, webviews,\n Orca app UI, or desktop UI outside Orca's embedded browser.\n---\n\n# Orca Inter-Agent Orchestration\n\nOrchestration is Orca's structured coordination layer for agent messages, task ownership, dispatch state, and worker completion tracking.\n\nUse this skill when coordination state matters. For lightweight terminal prompts or basic worktree/terminal/built-in-browser control, use `orca-cli`.\n\n## Tool Boundary\n\nIf a task says to use Orca orchestration, the coordinator must create or bind a Run, create the Task with `orca orchestration task-create`, then attach the worker with either the preferred `orca orchestration worker-start` composition or the low-level `orca orchestration dispatch --inject` path.\n\nDo not substitute non-Orca subagent tools, generic agent-spawn APIs, or chat-only parallel worker features. Those may create useful workers, but they do not create Orca task/dispatch provenance, injected lifecycle preambles, `worker_done` authority, or decision gates.\n\nBefore claiming a worker was orchestrated, verify the task/dispatch exists:\n\n```bash\norca orchestration task-list --json\norca orchestration dispatch-show --task --json\n```\n\nIf the work was accidentally run outside Orca orchestration, say so plainly. To repair provenance, rerun or revalidate the needed work through a fresh Orca terminal plus injected dispatch; do not retroactively describe the external worker as orchestrated.\n\n## When To Use\n\n- Send/reply/ask between agent terminals with persistent messages.\n- Dispatch structured tasks to workers and wait for `worker_done` or `escalation`.\n- Track task DAGs with dependencies.\n- Run coordinator loops or decision gates.\n\nDo not use orchestration merely because the user says \"hand off\", \"handoff\", \"handover\", \"give this to another agent\", or asks for another worktree/agent/model/effort. Those are full ownership transfers unless the user explicitly asks to supervise, monitor, wait for worker completion/results, coordinate a DAG, use decision gates, or keep a blocking ask/reply loop.\n\n## Preconditions\n\n- `orca status --json` should show a running runtime.\n- `orca` must be on PATH (`orca-ide` on Linux).\n- The orchestration experimental feature must be enabled in Settings > Experimental.\n- `orca orchestration` commands are RPC calls to the running Orca runtime.\n\n## Contract Migration\n\nOrca adopts a live pre-update orchestration assignment into an ordinary Run. Adoption preserves the existing agent process, PTY/session, terminal handle, tab/leaf/pane, worktree or folder workspace, Task, and Dispatch; it never restarts or replaces the worker. The retired scheduler is not revived, and a newly created attempt uses the current grammar.\n\nTreat the authority label on injected or formatted messages as definitive:\n\n- `[LEGACY COMPATIBILITY]` is live and attested. Run only the exact supported command printed with the message, using the same CLI executable and arguments that the original prompt supplied.\n- `[LEGACY RECOVERY REPLAY — MAY HAVE BEEN SEEN]` is one bounded, at-least-once cutover replay. Process it idempotently and acknowledge it only through the exact displayed guidance.\n- `[LEGACY READ-ONLY]` is inspection-only. It has no reply, acknowledgment, or lifecycle action.\n- An unlabeled current message uses the current guide and current grammar.\n\nDatabase provenance, an old-looking terminal, or a legacy Run ID does not prove mutation authority. If the runtime cannot prove liveness, principal ownership, capability, or the exact legacy contract, it degrades to read-only inspection and must not fall back to local execution. Exact recovery may restore the already-live PTY once in its original inactive background tab. It must not spawn, write, signal, stop, switch, focus, split, or inject a terminal. Loss of lifecycle authority does not invalidate the existing assignment, process, or filesystem work.\n\nCompatibility retries have narrow guarantees. A pending ask, a reply, a final Dispatch settlement, and a consuming check have durable recovery identities. A-era heartbeat and escalation calls remain at-least-once across a manual A-to-B retry because identical later signals may be intentional. If an A-era ask may already have been answered, run the exact non-consuming recovery check printed by the runtime first; after its answer is printed and acknowledged, a new invocation with the same question creates a new question. Never guess among multiple identical question threads.\n\nWhen a compatibility or recovery command returns structured next-step arguments, run those exact arguments with the same CLI executable. The arguments intentionally omit the executable name so the guidance works with `orca`, `orca-ide`, `orca-dev`, or another configured Orca CLI command. Do not translate the command from memory, broaden its recipient, or retry it as a current mutation unless the returned guidance explicitly says to.\n\nOn packaged Windows, a legacy ask uses a two-step commit/resume protocol. The initial command durably commits the question, prints its exact `ask --resume ` command, and exits with launcher status `75`; it does not wait for the answer. Run that exact resume command after the launcher or update boundary. Resume is idempotent and read-oriented: it waits for the already-committed question and does not create another one. For a WSL process that received compatibility proof at launch, use the printed executable `orca-ide` WSL resume command so the same distro and packaged launcher authority are preserved; do not substitute a PATH-resolved local CLI. Older WSL processes that never received the hidden launch token remain lifecycle read-only after the update, even while their terminal and filesystem work continue.\n\nLegacy inspection remains available without consuming mail:\n\n```bash\norca orchestration run-list --json\n# run_legacy_local is an empty audit tombstone after adoption.\norca orchestration run-show --id run_legacy_local --json\n# In run-list, find the ordinary Run whose objective is:\n# \"Recovered orchestration work from a contract update\"\norca orchestration run-show --id --json\norca orchestration task-list --run --json\norca orchestration inbox --full --json\norca orchestration check --terminal --peek --format --json\norca terminal read --terminal --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\n```\n\nIf the original coordinator is unavailable or cannot prove its retained authority, a current coordinator may explicitly take over the adopted Run:\n\n```bash\norca orchestration run-use --id --takeover-legacy --json\norca orchestration check --run --json\n```\n\nTakeover fences only the old coordinator, binds the current one, and moves pending worker mail into current Run Delivery. Live legacy workers keep their original Tasks, Dispatches, processes, filesystems, and old prompt commands; their later questions, escalations, and completion reports route to the current coordinator. Do not use takeover while the original coordinator is still actively coordinating, because its later lifecycle mutations are rejected.\n\nDo not launch a replacement editor merely because the desktop app or runtime was updated. If adoption cannot prove continuing authority, keep the original worker as the only editor until it reaches a stable handoff point, then use a new current Dispatch in a conflict-free placement for any remaining work.\n\n## Ownership\n\nNew orchestration messages and tasks belong to one explicitly bound Run. A Run is only a durable namespace and coordinator inbox; it never schedules or places workers. Lifecycle authority comes from the active Dispatch, and terminal handles remain routing metadata rather than durable identity. Send `worker_done` and `heartbeat` from the worker's own terminal; Orca routes them to that Dispatch's Run.\n\nClassify inherited context before sending lifecycle messages:\n\n- Coordinated subtask: a live coordinator owns the DAG and waits on this dispatch. Follow the preamble exactly, including `worker_done`, heartbeat/status, `ask`, and `escalation`.\n- Full handoff means ownership transfer, not supervised dispatch. The original actor is not monitoring a DAG, so do not create lifecycle obligations unless the user explicitly asks you to supervise.\n- Classify requests containing \"hand off\", \"handoff\", \"handover\", \"give this to another agent\", \"give this to another worktree\", \"another agent\", or \"another worktree\" as full handoffs by default, even when the user names a custom model or reasoning effort.\n- Use supervised orchestration only when the user explicitly asks you to \"supervise\", \"monitor\", \"wait\", \"track completion\", \"wait for worker_done\", return results, coordinate a DAG, use a decision gate, or manage ask/reply flow.\n- Do not use `orca orchestration dispatch --inject` for full handoffs. It injects a coordinator preamble that tells the worker to send `worker_done`, heartbeat, and `ask` messages, then end its turn under the original terminal's dispatch lifecycle.\n- Do not run `orca orchestration task-create`, `orca orchestration dispatch --inject`, or `orca orchestration check --wait` for full handoffs. Do not peek at terminal output after prompt delivery to monitor progress.\n- A review-only `worker_done` reports findings; it does not authorize coordinator file edits. After a review-only completion, synthesize findings, ask a decision gate if ownership is unclear, and dispatch or hand off fixes unless the user explicitly asked the coordinator to own fixes.\n- If the user's plan names a next owner agent (for example, \"then use opencode to create a PR\"), post-review corrections and PR prep belong to that named owner. The coordinator routes, synthesizes, asks decision gates when needed, and supervises; the named owner edits files and creates the PR.\n\nIf unclear, inspect orchestration state before sending lifecycle messages:\n\n```bash\norca orchestration task-list --json\norca terminal list --json\n# If inherited context includes a task id:\norca orchestration dispatch-show --task --json\n```\n\n## Messaging\n\n```bash\norca orchestration send --subject [--to ] [--from ] [--body ] [--type ] [--priority ] [--thread-id ] [--payload ] [--json]\norca orchestration check [--terminal ] [--ack ] [--peek|--all] [--types ] [--format] [--wait] [--timeout-ms ] [--json]\norca orchestration reply --id --body [--from ] [--json]\norca orchestration ask (--question |--resume ) [--options ] [--timeout-ms ] [--from ] [--json]\norca orchestration inbox [--limit ] [--json]\n```\n\nRules:\n\n- Omit `--from` unless impersonating another terminal; Orca auto-resolves it from the current terminal.\n- A coordinator `check` returns the bound Run's oldest FIFO Delivery (up to 50 messages) and replays that exact batch until `--ack `. Process every message before acknowledging; `check --ack --wait` acknowledges, checks, and waits in one operation.\n- Use `--peek` and `--all` only for read-only history/debugging. Type filters decide when a waiter wakes; the returned actionable Delivery is still the oldest full batch.\n- Use `dispatch:` for coordinator guidance to one supervised worker. Orca routes that stable address locally or through the connected-server relay; do not substitute a remote terminal handle.\n- Terminal handles remain appropriate for low-level pre-Dispatch messaging. Prefer `agentTerminalHandle` from the create response, fall back to `startupTerminal.handle` for older runtimes, then re-resolve with `orca terminal list --worktree ... --json` if missing or stale. Continue with the replacement handle only; never dual-send to old and new handles.\n- `orca orchestration check --peek --format --json` returns locally formatted unread mail without consuming it; it never writes to terminal input or remotely wakes another terminal. Use `orchestration dispatch --inject` to deliver a tracked task, or `terminal send` when an existing agent needs a free-form prompt.\n- While supervising workers manually, use `check --wait --types worker_done,escalation,question --timeout-ms ` instead of sleep/poll loops. Process the whole Delivery, reply to `question` messages with `orca orchestration reply --id --body --json`, then acknowledge and keep waiting.\n- Treat a `check --wait` timeout or `{count:0}` as a checkpoint, not a worker failure. Long coding tasks routinely run 15-60 minutes; keep using rolling waits unless you receive `worker_done`/`escalation`, the terminal exits or disappears, or the user explicitly asks you to stop.\n- Heartbeats and visible terminal activity mean the worker is alive, not done. Do not stop, close, kill, or restart a worker just because it has not produced a completion message yet.\n- Use `ask` when a worker needs a blocking answer from the coordinator; it defaults to the active Dispatch's Run. Timeout or disconnect leaves the question pending, so resume by its original message ID instead of asking again.\n- `check --wait` returns one bounded Delivery, not every future completion. Process every message, acknowledge it, then keep waiting until every expected Dispatch settles.\n- Group addresses include `@all`, `@idle`, `@claude`, `@codex`, `@opencode`, `@gemini`, `@droid`, `@grok`, `@cursor`, and `@worktree:`.\n- Message types include `status`, `dispatch`, `worker_done`, `merge_ready`, `escalation`, `handoff`, `question`, `decision_gate` (legacy/gates), and `heartbeat`.\n- Use group addresses only for messages that are genuinely useful to many terminals, such as `status` broadcasts or intentional fan-out questions. Do not send dispatch lifecycle messages to groups.\n- `worker_done` belongs to the active Dispatch and defaults to its Run mailbox; never target a group.\n- A valid `worker_done` for the active `taskId` + `dispatchId` marks the task and dispatch completed automatically. Do not follow it with `task-update --status completed`; reserve manual updates for explicit recovery or overrides.\n- `heartbeat` is also Dispatch-scoped. Include both IDs and omit `--to` so Orca uses the owning Run; use `status` for broad progress updates.\n\n## Tasks And Dispatch\n\nA Run is the namespace/inbox, a Task is the work item, and a Dispatch assigns one Task attempt to a terminal. Create or bind a Run once before the common loop.\n\n```bash\norca orchestration run-create --objective --json\norca orchestration task-create --spec [--deps ] [--parent ] [--json]\norca orchestration task-list [--status ] [--ready] [--brief] [--json]\norca orchestration task-update --id --status [--result ] [--json]\norca orchestration dispatch --task --to [--from ] [--inject] [--json]\norca orchestration dispatch-show --task [--json]\n```\n\nTask statuses: `pending`, `ready`, `dispatched`, `completed`, `failed`, `blocked`.\n\nDispatch rules:\n\n- `--inject` sends the task spec plus preamble into a recognized agent CLI so it can report `worker_done`.\n- If the target is a bare shell, omit `--inject`, dispatch for tracking if needed, then send the prompt manually with `orca terminal send --terminal --text --enter --json`.\n- After 3 consecutive failures on one task, the dispatch context circuit-breaks and the task is marked failed.\n- Use `task-list --brief --json` for coordinator sweeps; it collapses whitespace and caps each echoed spec at 160 characters (`spec_truncated` marks shortened rows). Omit `--brief` when the full spec is required, or when an older CLI rejects it as an unknown flag.\n\n## Preferred Supervised Worker Loop\n\nUse `worker-start` for the normal supervised path. It composes the existing worktree, terminal, readiness, and dispatch primitives while returning exact created/reused effects. Agents still choose placement and concurrency; Orca does not schedule workers or infer conflicts.\n\nCreate the Run and every independent Task first, then start all independent workers before waiting:\n\n```bash\norca orchestration run-create --objective \"\" --json\norca orchestration task-create --spec \"\" --json\norca orchestration task-create --spec \"\" --json\norca orchestration worker-start --task --worktree current --agent codex --json\norca orchestration worker-start --task --worktree current --agent claude --json\n```\n\n`current` and exact existing worktrees create a fresh agent terminal and do not rerun setup. Reuse an existing agent only with `--terminal `.\n\nFor a new worktree, setup runs by default and agent-first creation reuses the returned startup agent terminal:\n\n```bash\norca orchestration worker-start --task --worktree new-child --name --agent codex --setup run --json\n# Independent/top-level:\norca orchestration worker-start --task --worktree new-top-level --name --agent codex --setup run --json\n```\n\nSetup normally starts alongside the agent. Only a repository explicitly configured with `wait-for-setup` delays agent launch until setup succeeds. Use `--setup skip` or `--setup inherit` only for a concrete reason.\n\nRead the returned receipt before continuing: `ready` plus setup `running` is normal for start-immediately, while wait-for-setup returns setup `succeeded` before accepting task input. A failed or unknown start exits nonzero; inspect its `stage`, `effects`, and `residualResources` instead of guessing or automatically retrying. A wait-for-setup timeout can honestly leave setup `running`, which is not proof of failure.\n\nTo run the worker on another connected Orca server, add `--on `. The Run and Tasks remain authoritative on the current server; later commands route by Dispatch ID, so never repeat `--on`:\n\n```bash\n# Mac Run home -> Windows worker (the reverse is identical from a Windows Run home)\norca orchestration worker-start --task --on windows --worktree new-top-level --repo --name --agent codex --setup run --json\norca orchestration worker-show --dispatch --json\norca orchestration worker-read --dispatch --limit 50 --json\norca orchestration send --to dispatch: --subject \"Follow-up\" --body \"\" --json\n```\n\nRemote `current` and `new-child` are intentionally invalid because those words are ambiguous across servers. Use an exact discovered remote worktree selector or `new-top-level` with an explicit remote repo selector.\n\nThe follow-up is structured inbox mail, not prompt injection. The worker's next\n`orchestration check` receives it even when the Dispatch is on another connected Orca server.\n\n`worker-read` defaults to `--source auto`: Orca returns the exact hook-reported Codex, Claude, OpenClaude, or Grok transcript when it can prove the worker session, otherwise it returns bounded terminal output with `source: \"terminal\"` and a typed `fallbackReason`. Continue with the returned top-level `cursor`; it stays pinned to that exact source. If Orca reports `source_changed`, start a fresh read without the old cursor. Never supply or guess a provider session ID or transcript path.\n\nWait until every expected Dispatch settles, not for a fixed number of batches:\n\n```bash\norca orchestration check --wait --types worker_done,escalation,question --timeout-ms 900000 --json\n# Process every message in the returned Delivery, then atomically ack and continue:\norca orchestration check --ack --wait --types worker_done,escalation,question --timeout-ms 900000 --json\n```\n\nWorkers report exactly once using the IDs and capability injected by Orca; they do not supply Run/server/terminal identity:\n\n```bash\norca orchestration send --type worker_done --subject \"\" --body \"\" --task-id --dispatch-id --outcome succeeded --files-modified \"path/a,path/b\" --json\n# On failure, use --outcome failed; never encode failure only in prose.\n```\n\nA worker question defaults to its owning Run. Timeout leaves it pending:\n\n```bash\norca orchestration ask --question \"\" --options \"yes,no\" --timeout-ms 600000 --json\norca orchestration ask --resume --timeout-ms 600000 --json\n# Coordinator:\norca orchestration reply --id --body \"\" --json\n```\n\nRecovery is conditional, never a fixed destructive sequence:\n\n- `worker-show --dispatch ` says `ready`: keep waiting or read bounded output.\n- It proves `failed` or `stopped`: start a replacement with `worker-start --task --retry-of ` plus an explicit `--on`/`--worktree` and `--agent`/`--terminal` choice. Retry does not silently inherit placement.\n- It remains `outcome_unknown`: either `worker-stop --dispatch ` and inspect again, or explicitly `worker-abandon --dispatch ` while accepting that resources may still be live. Abandon performs no remote, process, or filesystem action.\n- `worker-stop` closes only the exact supervised agent terminal. It never deletes the worktree, setup terminal, configured tabs, or unrelated processes.\n\nLow-level `worktree create`, `terminal create`, and `dispatch --inject` remain valid recipes for custom argv or topology that `worker-start` does not express.\n\n## Gates And Legacy Inspection\n\n```bash\norca orchestration gate-create --task --question [--options ] [--json]\norca orchestration gate-resolve --id --resolution [--json]\norca orchestration gate-list [--task ] [--status ] [--json]\n```\n\nUse `ask` for worker-to-coordinator questions; it creates a `question` message that the coordinator answers with `reply`. Use `gate-create` only for coordinator-managed task DAG decisions, not for answering a worker's `ask`.\n\n`coordinator-start`, `coordinator-stop`, `run`, and `run-stop` are retired scheduler commands. They perform no effects and return the current-skill recovery action. They are not aliases for lightweight Run creation or binding.\n\nRecovery only: `orca orchestration reset --tasks|--messages|--all --json` clears the selected local orchestration database state. Do not run it during active coordination unless explicitly abandoning that state.\n\n## Full Handoffs\n\nFor full ownership transfer, use non-lifecycle terminal/worktree commands and then stop monitoring unless the user asks for supervision.\n\nTreat these as full handoff requests by default: \"hand off\", \"handoff\", \"handover\", \"give this to another agent\", \"give this to another worktree\", \"send this to another agent\", \"another agent\", \"another worktree\", or \"launch another agent to own this.\" Custom model or reasoning effort words such as `gpt-5.5`, `high`, or `xhigh` do not make the handoff supervised.\n\nSupervised orchestration remains available only when the user explicitly asks for supervision or coordination: \"supervise\", \"monitor\", \"wait for worker_done\", \"wait for results\", \"track completion\", \"DAG\", \"decision gate\", \"ask/reply\", or \"coordinate workers.\"\n\nDo not run `orca orchestration task-create`, `orca orchestration dispatch --inject`, or `orca orchestration check --wait` for full handoffs. `task-create` is also forbidden because it records coordinator-owned tracking state; if a task row is needed, the user asked for supervised orchestration. Do not create a `taskId`/`dispatchId`, inject a lifecycle preamble, wait for completion, or read the worker terminal after prompt delivery except to avoid losing the initial prompt.\n\nNew top-level worktree handoff:\n\n```bash\norca worktree create --name --no-parent --agent codex --prompt \"\" --setup run --json\n```\n\nBefore creating a new worktree from an active feature branch, decide and state whether the desired Orca lineage is child or top-level. Use child worktree lineage only when the new work is conceptually stacked under or dependent on the active worktree. For independent repo-wide fixes, standalone feature work, or unrelated follow-up tasks, create a top-level worktree with `--no-parent`.\n\nExisting terminal handoff:\n\n```bash\norca terminal send --terminal --text \"\" --enter --json\n```\n\nCustom Codex model/effort handoff:\n\n`orca worktree create --agent codex --prompt ...` launches the known Codex agent but does not accept Codex-specific `--model` or `-c model_reasoning_effort=...` arguments. When the user asks for a specific Codex model or effort, create the independent worktree first, launch Codex with the requested command in that worktree, wait only for TUI readiness if prompt delivery would otherwise race startup, send the prompt, and stop.\n\nThe two-step custom-argv path cannot enforce a repository's explicit `wait-for-setup` startup policy because the later `terminal create` is not the startup owned by `worktree create`. Use it only when the repository starts agents immediately. If the repository requires `wait-for-setup`, use an agent-first configured launcher that can preserve sequencing, or stop and ask rather than silently bypassing the policy.\n\nNote: when no repo default-terminal configuration supplies a primary terminal, bare create opens a fallback shell before `terminal create` adds the agent. Configured default tabs are materialized instead and may run real commands. Prefer `--agent` whenever custom argv is not required. With the two-step path, target only the agent handle; close a prior terminal only after `terminal list` or `terminal show` confirms it is an unused shell.\n\nUse the exact full `::` worktree id returned by `orca worktree create --json`; a bare repo id cannot target the new worktree.\n\n```bash\norca worktree create --name --no-parent --setup run --json\norca terminal create --worktree id: --title --command 'codex --model gpt-5.5 -c model_reasoning_effort=\"xhigh\"' --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca terminal send --terminal --text \"\" --enter --json\n```\n\nWait only for `tui-idle` when needed to avoid losing the prompt. Do not monitor task completion.\n\n`--no-parent` only controls Orca lineage; it does not choose the Git base. If the work should start from the repo default base, omit `--base-branch` so Orca uses that default, or explicitly pass the repo default base (`origin/main`, `origin/master`, or the `orca repo show --repo --json` value); never base it on the current feature branch unless the user explicitly asks for stacked work or \"branch from current\". Put current-branch context in the prompt instead.\n\n## Worker Terminals\n\nChoose the worker location before creating a terminal. `Fresh worker` means a fresh agent session, not a new git worktree. For parallel work, create one fresh agent terminal per worker in the same required worktree, falling back to the active worktree when none is named. If the task says current worktree only, depends on uncommitted files/artifacts, or must validate/PR the current branch, keep every worker in the active worktree:\n\n```bash\norca terminal create --worktree active --title --command \"codex\" --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca orchestration dispatch --task --to --inject --json\n```\n\nReuse an idle agent in the required worktree only if the prompt allows reuse; otherwise create a fresh terminal there. Create a new worktree only when the user explicitly requests one or a concrete checkout or filesystem conflict makes sharing unsafe or impossible; if the user did not request it, state that conflict before running `worktree create`. Independent tasks, parallel execution, convenience, or a preference for separate checkouts are not isolation requirements.\n\nWhen a new worktree is allowed, use child lineage for isolated work that is stacked under or dependent on the active worktree, and use `--no-parent` when it is not stacked. Decide the Git base separately: `--no-parent` makes the worktree top-level in Orca, while omitted `--base-branch` uses the repo default base.\n\nFor every new worktree, pass `--setup run` so any configured repository setup hook runs. This does not mean waiting for setup before agent launch: preserve the repository's startup policy, whose default starts setup and the agent side by side. Use `--setup skip` or `--setup inherit` only when there is a concrete task-specific reason, and state that reason before creating the worktree. This rule does not rerun setup for current or existing worktrees.\n\n```bash\norca worktree create --name --agent codex --setup run --json\n# or: --agent claude | omp | pi | grok | ...\n# Read from agentTerminalHandle, falling back to startupTerminal.handle.\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca orchestration dispatch --task --to --inject --json\n```\n\nFor new-worktree workers, read the id and `agentTerminalHandle` from `worktree create`, falling back to `startupTerminal.handle` for older runtimes. Use that as the sole worker handle when present; otherwise use `terminal list` to resolve the agent handle. Omit `--repo` only inside an Orca-managed worktree; otherwise pass `--repo `.\n\n**For an allowed new worktree, use agent-first:** `--agent` reveals the new worktree and launches the selected agent **in its first terminal**, without adding a separate fallback shell for that worker. Pass `--setup run`; repo setup and default-terminal settings may add intentional tabs or splits. Do **not** run bare `worktree create` and then `terminal create --command ` for the same worker when agent-first create is available: without configured default tabs, that two-step path leaves a fallback shell + agent pair. Only use it when custom agent argv is required (for example Codex model/effort flags) or when an older CLI rejects `--agent`; if you must, message only the agent handle. Configured default tabs are intentional surfaces, so close a prior terminal only after `terminal list` or `terminal show` confirms it is an unused shell. Do not run `worktree create` when the task must stay in the current worktree.\n\nUse `orca worktree create --prompt ...` or `orca terminal send ...` for full handoffs or untracked/lightweight prompts. Those paths do not attach `taskId`/`dispatchId`; the worker should not send lifecycle messages unless the prompt supplies a live orchestration preamble.\n\nSidebar lineage and orchestration lifecycle are related but not identical. A same-worktree worker may appear as a peer under that worktree in the sidebar while remaining a child dispatch in orchestration state; only an actual child worktree creates visible parent/child worktree lineage.\n\nOther terminal commands coordinators often need:\n\n```bash\norca terminal list [--worktree ] [--json]\norca terminal create [--worktree ] [--title ] [--command ] [--json]\norca terminal split --terminal [--direction horizontal|vertical] [--command ] [--json]\norca terminal wait --terminal --for tui-idle --timeout-ms --json\norca terminal read --terminal --json\norca terminal send --terminal --text --enter --json\n```\n\nIf an older CLI rejects `worktree create --agent`, create the worktree normally, then run `orca terminal create --worktree --command \"codex\" --json` or `--command \"claude\"`.\n\nWait for `tui-idle` before dispatching. Always pass `--timeout-ms`; real coding tasks can take 15-60 minutes. During supervision, use rolling `check --wait` windows. If a window returns no matching message, inspect `task-list`, `terminal read`, or `terminal wait --for tui-idle` as a liveness checkpoint; if the terminal is still working or producing activity, keep waiting instead of retrying the task.\n\n## Agent Guidance\n\n- Workers with a valid live preamble must send `worker_done` exactly once from their own terminal with an explicit `--outcome succeeded` or `--outcome failed`:\n `orca orchestration send --type worker_done --subject \"\" --body \"<3-sentence summary: what you did, what you found, what's left>\" --task-id --dispatch-id --outcome succeeded --files-modified \"path/a\" --report-path \"\" --json`\n- A failed outcome is still a terminal report, but Orca records both the Dispatch and Task as failed. Never encode failure only in the subject/body.\n- After sending `worker_done`, end your turn and idle at the agent prompt. Do not poll or keep calling `orca orchestration check`; the coordinator re-engages you with a fresh preamble + TASK block delivered as new terminal input.\n- For long tasks, send heartbeat/status only when the preamble asks for it, including both IDs:\n `orca orchestration send --type heartbeat --subject \"alive\" --payload '{\"taskId\":\"\",\"dispatchId\":\"\",\"phase\":\"implementing\"}' --json`\n- If blocked before completion, use `ask`; use `escalation` only when ownership is valid and the coordinator must intervene.\n- Treat preambles inherited through terminal history or full handoffs as stale unless the current prompt explicitly keeps that coordinator in the loop.\n- Coordinators should use `task-list --ready` as external memory, dispatch parallel waves, and avoid dependency chains deeper than 3-4 steps.\n\n## Example\n\n```bash\norca terminal create --worktree active --title login-css-worker --command \"claude\" --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca orchestration task-create --spec \"Fix the login button CSS\" --json\norca orchestration dispatch --task --to --inject --json\norca orchestration check --wait --types worker_done,escalation,question --timeout-ms 900000 --json\n```\n\n## Next Action\n\nCoordinator: confirm `orca status --json`, create or bind a Run, inspect `task-list`/`dispatch-show` if inheriting state, then use the explicit supervised loop (`task-create` -> `worker-start` -> `check --wait`). Use low-level terminal creation plus `dispatch --inject` only when the composed start does not express the needed topology.\n\nWorker: if the current prompt contains a live dispatch preamble, do the task, use `ask` for blocking questions, and send `worker_done` once with the required payload. If the preamble is stale or absent, do not send lifecycle messages; inspect state or treat the prompt as an ordinary handoff.\n" // Why: no current guide has bundled reference documents, so --full is byte-identical for now. // oxfmt-ignore diff --git a/src/cli/handlers/orchestration-check-identity.test.ts b/src/cli/handlers/orchestration-check-identity.test.ts index 7c4908dc782..ec0043fd510 100644 --- a/src/cli/handlers/orchestration-check-identity.test.ts +++ b/src/cli/handlers/orchestration-check-identity.test.ts @@ -71,4 +71,24 @@ describe('orchestration check identity', () => { }) ) }) + + it('preserves the pinned legacy --inject check signature', async () => { + process.env.ORCA_TERMINAL_HANDLE = 'term_legacy_worker' + + await invokeCheck( + new Map([ + ['unread', true], + ['inject', true] + ]) + ) + + expect(callMock).toHaveBeenCalledWith( + 'orchestration.check', + expect.objectContaining({ + terminal: 'term_legacy_worker', + unread: true, + inject: true + }) + ) + }) }) diff --git a/src/cli/handlers/orchestration-legacy-read-only.test.ts b/src/cli/handlers/orchestration-legacy-read-only.test.ts index 0bc82699bc1..05cca1869cd 100644 --- a/src/cli/handlers/orchestration-legacy-read-only.test.ts +++ b/src/cli/handlers/orchestration-legacy-read-only.test.ts @@ -179,6 +179,7 @@ describe('legacy orchestration CLI inspection', () => { id: 'msg_current', run_id: 'run_current', from_handle: 'term_current', + to_handle: 'run:run_current', subject: 'current question', type: 'question', body: 'May I continue?', @@ -217,7 +218,8 @@ describe('legacy orchestration CLI inspection', () => { expect(response.result.formatted).not.toContain('RUNTIME_SENTINEL') expect( response.result.formatted.split('\n').filter((line) => line.startsWith('[Reply:')) - ).toEqual(['[Reply: orca orchestration reply --id msg_current --from term_coord --body "..."]']) + ).toEqual(['[Reply: orca orchestration reply --id msg_current --body "..."]']) + expect(response.result.formatted).not.toContain('--from run:run_current') }) it('preserves runtime formatting when every message belongs to a current Run', async () => { diff --git a/src/cli/handlers/orchestration-run-cli.test.ts b/src/cli/handlers/orchestration-run-cli.test.ts index 84f4dc4c659..4ed57bcb169 100644 --- a/src/cli/handlers/orchestration-run-cli.test.ts +++ b/src/cli/handlers/orchestration-run-cli.test.ts @@ -62,6 +62,28 @@ describe('lightweight Run CLI handlers', () => { from: 'term_coord' }) }) + + it('passes explicit legacy takeover only when requested', async () => { + callMock.mockResolvedValue({ + result: { run: { id: 'run_adopted', objective: 'Recovered work' } } + }) + await ORCHESTRATION_HANDLERS['orchestration run-use']({ + flags: new Map([ + ['id', 'run_adopted'], + ['from', 'term_current'], + ['takeover-legacy', true] + ]), + client: { call: callMock }, + cwd: '/tmp/repo', + json: true + } as never) + + expect(callMock).toHaveBeenCalledWith('orchestration.runUse', { + id: 'run_adopted', + from: 'term_current', + takeoverLegacy: true + }) + }) }) describe('orchestration reset CLI handler', () => { diff --git a/src/cli/handlers/orchestration.ts b/src/cli/handlers/orchestration.ts index ece699fe118..0e87bc54d46 100644 --- a/src/cli/handlers/orchestration.ts +++ b/src/cli/handlers/orchestration.ts @@ -21,10 +21,14 @@ import type { } from '../../shared/orchestration-worker-output' import type { NativeChatMessage } from '../../shared/native-chat-types' import type { RuntimeTerminalRead } from '../../shared/runtime-types' +import { orchestrationMigrationData } from '../../shared/orchestration-rpc-contract' import { - ORCHESTRATION_LEGACY_RUN_ID, - orchestrationMigrationData -} from '../../shared/orchestration-rpc-contract' + formatMessageReadOnlyTag, + formatOrchestrationCheckText, + prepareOrchestrationCheckOutput, + type LegacyCompatibilityResult, + type OrchestrationMessageSummary as MessageSummary +} from '../../shared/orchestration-check-output' // Why: 15 s is well under Claude Code's ~2 min Bash-tool silence budget while keeping log volume low. See design doc §3.4. const DEFAULT_KEEPALIVE_INTERVAL_MS = 15_000 @@ -74,93 +78,6 @@ const TASK_STATUS_VALUES = [ 'blocked' ] as const -type MessageSummary = { - id: string - run_id?: string - delivery_contract?: 'legacy_direct' | 'current_delivery' | 'audit_only' - from_handle: string - to_handle?: string - subject?: string - type?: string - body?: string - payload?: string | null - priority?: string - read?: number -} - -function formatMessageReadOnlyTag( - message: MessageSummary, - legacyCompatibilityActive = false -): string { - return isLegacyReadOnlyMessage(message, legacyCompatibilityActive) ? ' [legacy, read-only]' : '' -} - -function isLegacyReadOnlyMessage( - message: MessageSummary, - legacyCompatibilityActive = false -): boolean { - return ( - message.run_id === ORCHESTRATION_LEGACY_RUN_ID || - (message.delivery_contract === 'legacy_direct' && !legacyCompatibilityActive) || - message.delivery_contract === 'audit_only' - ) -} - -function formatMessagePriorityTag(message: MessageSummary): string { - return message.priority === 'urgent' ? ' [URGENT]' : message.priority === 'high' ? ' [HIGH]' : '' -} - -function escapeTerminalControlCharacters(value: string): string { - return [...value] - .map((character) => { - const code = character.charCodeAt(0) - if (character === '\n' || (code >= 0x20 && code < 0x7f) || code > 0x9f) { - return character - } - return `\\x${code.toString(16).padStart(2, '0')}` - }) - .join('') -} - -function formatQuotedMessageField(label: string, value?: string): string { - return `[${label}]\n${escapeTerminalControlCharacters(value ?? '') - .split('\n') - .map((line) => ` ${line}`) - .join('\n')}` -} - -function formatLegacyAwareCheckMessages( - messages: MessageSummary[], - checkedTerminal: string, - legacyCompatibilityActive = false -): string { - return messages - .map((message) => { - const legacyReadOnly = isLegacyReadOnlyMessage(message, legacyCompatibilityActive) - const lines = [ - `${message.id}${formatMessageReadOnlyTag(message, legacyCompatibilityActive)}${formatMessagePriorityTag(message)} [${message.type ?? 'status'}] from=${message.from_handle}`, - formatQuotedMessageField('subject', message.subject) - ] - if (legacyReadOnly) { - lines.push('[Inspection only: reply and acknowledgment are unavailable.]') - } - if (message.body) { - lines.push(formatQuotedMessageField('body', message.body)) - } - if (message.payload) { - lines.push(formatQuotedMessageField('payload', message.payload)) - } - if (!legacyReadOnly) { - const replyFrom = message.to_handle ?? checkedTerminal - lines.push( - `[Reply: orca orchestration reply --id ${message.id} --from ${replyFrom} --body "..."]` - ) - } - return lines.join('\n') - }) - .join('\n\n') -} - type LifecycleSendRejection = { action: 'rejected' code: string @@ -181,23 +98,6 @@ type OrchestrationSendResult = lifecycle?: { action: 'completed' | 'failed' } } -type LegacyCompatibilityResult = { - recovery?: boolean - readOnly?: boolean - ackMessageIds?: string[] - answerAcknowledgement?: { - questionId: string - answerMessageId: string - } - currentDelivery?: { - runId: string - checkCommand: string - ackCommand: string - } - resumeRequired?: boolean - resumeCommand?: string -} - function resolveCompatibilityCliCommand(): 'orca' | 'orca-ide' | 'orca-dev' { const configured = process.env.ORCA_CLI_COMMAND if (configured === 'orca' || configured === 'orca-ide' || configured === 'orca-dev') { @@ -206,19 +106,6 @@ function resolveCompatibilityCliCommand(): 'orca' | 'orca-ide' | 'orca-dev' { return process.platform === 'linux' ? 'orca-ide' : 'orca' } -function formatCurrentDeliveryNotice( - delivery: LegacyCompatibilityResult['currentDelivery'] -): string { - if (!delivery) { - return '' - } - return ( - `\n[CURRENT RUN MAIL WAITING]\n` + - `Read: ${delivery.checkCommand}\n` + - `Then acknowledge the delivery ID it prints: ${delivery.ackCommand}` - ) -} - function resolvePackagedWindowsCompatibilityCommand(): 'orca' | 'orca-ide' | undefined { if (process.env.ORCA_WINDOWS_PACKAGED_CLI_LAUNCHER !== '1') { return undefined @@ -560,7 +447,8 @@ export const ORCHESTRATION_HANDLERS: Record = { run: { id: string; objective: string; consumer_generation: number } }>(client, flags, 'orchestration.runUse', { id: getRequiredStringFlag(flags, 'id'), - from + from, + ...(flags.has('takeover-legacy') ? { takeoverLegacy: true } : {}) }) printResult(result, json, (r) => `Using Run ${r.run.id}: ${r.run.objective}`) }, @@ -715,6 +603,7 @@ export const ORCHESTRATION_HANDLERS: Record = { all: flags.has('all') ? true : undefined, types: getOptionalStringFlag(flags, 'types'), format: flags.has('format') ? true : undefined, + inject: flags.has('inject') ? true : undefined, compatibilityCliCommand: resolveCompatibilityCliCommand(), run: getOptionalStringFlag(flags, 'run'), ack: getOptionalStringFlag(flags, 'ack'), @@ -752,64 +641,11 @@ export const ORCHESTRATION_HANDLERS: Record = { } } } - const compatibilityActive = Boolean( - result.result.legacyCompatibility && !result.result.legacyCompatibility.readOnly - ) - if ( - flags.has('format') && - result.result.messages.some((message) => - isLegacyReadOnlyMessage(message, compatibilityActive) - ) - ) { - // Why: formatted is one opaque batch with untrusted bodies, so selective banner parsing cannot safely remove legacy actions. - result = { - ...result, - result: { - ...result.result, - formatted: formatLegacyAwareCheckMessages( - result.result.messages, - terminal, - compatibilityActive - ) - } - } + result = { + ...result, + result: prepareOrchestrationCheckOutput(result.result, terminal, flags.has('format')) } - printResult(result, json, (r) => { - const legacyHeader = r.legacyCompatibility - ? r.legacyCompatibility.readOnly - ? '[LEGACY READ-ONLY]\n' - : r.legacyCompatibility.recovery - ? '[LEGACY RECOVERY REPLAY — MAY HAVE BEEN SEEN]\n' - : '[LEGACY COMPATIBILITY]\n' - : '' - const deliveryNotice = formatCurrentDeliveryNotice(r.legacyCompatibility?.currentDelivery) - if (r.formatted) { - return `${legacyHeader}${r.formatted}${deliveryNotice}` - } - if (r.count === 0) { - if (r.timedOut) { - return `${legacyHeader}Wait timed out; no messages were consumed.${deliveryNotice}` - } - if (r.cancelled) { - const cancelled = r.connectionLost - ? 'Wait cancelled because the connection closed; no messages were consumed.' - : 'Wait cancelled; no messages were consumed.' - return `${legacyHeader}${cancelled}${deliveryNotice}` - } - return `${legacyHeader}No messages.${deliveryNotice}` - } - const rendered = r.messages - .map( - (m) => - `${m.id}${formatMessageReadOnlyTag( - m, - Boolean(r.legacyCompatibility && !r.legacyCompatibility.readOnly) - )} [${m.type ?? 'status'}] from=${m.from_handle} "${m.subject}"` - ) - .join('\n') - const output = r.deliveryId ? `Delivery ${r.deliveryId}\n${rendered}` : rendered - return `${legacyHeader}${output}${deliveryNotice}` - }) + printResult(result, json, (r) => formatOrchestrationCheckText(r, terminal)) const compatibilityAck = result.result.legacyCompatibility?.ackMessageIds if (compatibilityAck && compatibilityAck.length > 0) { await flushStdout() diff --git a/src/cli/specs/orchestration.ts b/src/cli/specs/orchestration.ts index 226f9dcb7f4..41c306264a5 100644 --- a/src/cli/specs/orchestration.ts +++ b/src/cli/specs/orchestration.ts @@ -16,8 +16,12 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [ { path: ['orchestration', 'run-use'], summary: 'Bind this coordinator terminal to an existing Run', - usage: 'orca orchestration run-use --id [--from ] [--json]', - allowedFlags: [...GLOBAL_FLAGS, 'id', 'from', 'retry-request'] + usage: + 'orca orchestration run-use --id [--from ] [--takeover-legacy] [--json]', + allowedFlags: [...GLOBAL_FLAGS, 'id', 'from', 'takeover-legacy', 'retry-request'], + notes: [ + '--takeover-legacy explicitly replaces an unavailable legacy coordinator while preserving live worker assignments.' + ] }, { path: ['orchestration', 'run-current'], diff --git a/src/main/index.ts b/src/main/index.ts index 2bf30539392..71a5d77a67e 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -2220,8 +2220,6 @@ void app.whenReady().then(async () => { getAgentProviderSessionSnapshot: () => agentHookServer.getStatusSnapshot(), getAgentProviderSessionRowsForPane: (paneKey) => agentHookServer.getStatusSnapshotForPane(paneKey), - getHydratedAgentHookAuthority: () => agentHookServer.getHydratedAuthorityCommitments(), - getCurrentAgentHookAuthority: () => agentHookServer.getCurrentAuthorityObservations(), attestAgentHookCompatibilityAuthority: (candidate) => agentHookServer.attestCompatibilityAuthority(candidate), canRecoverPersistentLocalPtys: () => getDaemonProvider() !== null, diff --git a/src/main/ipc/pty.test.ts b/src/main/ipc/pty.test.ts index 58b841508bd..dad5e721efa 100644 --- a/src/main/ipc/pty.test.ts +++ b/src/main/ipc/pty.test.ts @@ -5374,6 +5374,37 @@ describe('registerPtyHandlers', () => { expect(listProcesses).not.toHaveBeenCalled() }) + it('scopes runtime inventories to the requested provider', async () => { + const localList = vi + .spyOn(getLocalPtyProvider(), 'listProcesses') + .mockResolvedValue([{ id: 'local-pty', title: 'Local', cwd: '/local' }]) + const sshAList = vi.fn(async () => [{ id: 'ssh-a-pty' }]) + const sshBList = vi.fn(async () => { + throw new Error('ssh-b unavailable') + }) + registerSshPtyProvider('ssh-a', { listProcesses: sshAList } as never) + registerSshPtyProvider('ssh-b', { listProcesses: sshBList } as never) + const runtime = { setPtyController: vi.fn() } + handlers.clear() + registerPtyHandlers(mainWindow as never, runtime as never) + const controller = runtime.setPtyController.mock.calls[0]?.[0] as { + listProcesses(connectionId?: string | null): Promise<{ id: string }[]> + } + + await expect(controller.listProcesses(null)).resolves.toEqual([ + { id: 'local-pty', title: 'Local', cwd: '/local' } + ]) + expect(localList).toHaveBeenCalledOnce() + expect(sshAList).not.toHaveBeenCalled() + expect(sshBList).not.toHaveBeenCalled() + + await expect(controller.listProcesses('ssh-a')).resolves.toEqual([{ id: 'ssh-a-pty' }]) + expect(sshAList).toHaveBeenCalledOnce() + expect(sshBList).not.toHaveBeenCalled() + + await expect(controller.listProcesses()).rejects.toThrow('ssh-b unavailable') + }) + it('returns unavailable runtime confirmation for unsupported or missing providers', async () => { registerSshPtyProvider('ssh-1', {} as never) setPtyOwnership('unsupported-pty', 'ssh-1') diff --git a/src/main/ipc/pty.ts b/src/main/ipc/pty.ts index 3a2cc63c6bb..586355402bb 100644 --- a/src/main/ipc/pty.ts +++ b/src/main/ipc/pty.ts @@ -4253,7 +4253,13 @@ export function registerPtyHandlers( return null } }, - listProcesses: async () => { + listProcesses: async (connectionId) => { + if (connectionId === null) { + return localProvider.listProcesses() + } + if (connectionId !== undefined) { + return getProvider(connectionId).listProcesses() + } const providerSessions = await Promise.all([ localProvider.listProcesses(), ...Array.from(sshProviders.values(), (provider) => provider.listProcesses()) diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index 9dd3a30d00d..143b7b0160e 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -16337,6 +16337,17 @@ describe('OrcaRuntimeService', () => { ['pty-shell', 'inc-shell', 'term_shell', 'Shell'] ] as const const runtime = new OrcaRuntimeService({ ...runtimeStore, flushOrThrow: vi.fn() } as never) + const listProcesses = vi.fn(async () => + processes.map(([id, incarnationId, terminalHandle, title]) => ({ + id, + incarnationId, + terminalHandle, + title, + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + })) + ) runtime.setPtyController({ write: (ptyId, data) => { writes.push([ptyId, data]) @@ -16345,16 +16356,7 @@ describe('OrcaRuntimeService', () => { resize, kill: () => true, getForegroundProcess: async () => null, - listProcesses: async () => - processes.map(([id, incarnationId, terminalHandle, title]) => ({ - id, - incarnationId, - terminalHandle, - title, - cwd: TEST_WORKTREE_PATH, - worktreeId: TEST_WORKTREE_ID, - wslDistro: null - })) + listProcesses }) const before = await runtime.listTerminals(`id:${TEST_WORKTREE_ID}`) expect(before.terminals.map((terminal) => terminal.tabId)).toEqual( @@ -16442,6 +16444,14 @@ describe('OrcaRuntimeService', () => { tail: ['legacy output'] }) + const inventoryCount = listProcesses.mock.calls.length + const agentPty = ( + runtime as unknown as { + ptysById: Map + } + ).ptysById.get('pty-agent')! + agentPty.tabId = null + agentPty.paneKey = null const secondClient = await runtime.adoptTerminalOrphans({ worktree: `id:${TEST_WORKTREE_ID}`, expectedTopologyRevision: 0, @@ -16454,6 +16464,22 @@ describe('OrcaRuntimeService', () => { })) }) expect(secondClient).toMatchObject({ adopted: false, topologyRevision: 1 }) + expect(agentPty).toMatchObject({ + tabId: 'tab-agent', + paneKey: makePaneKey('tab-agent', HEADLESS_LEAF_ID) + }) + expect(listProcesses).toHaveBeenCalledTimes(inventoryCount + 1) + expect(listProcesses).toHaveBeenLastCalledWith(null) + expect( + (await runtime.listTerminals()).terminals.find((terminal) => terminal.ptyId === 'pty-agent') + ).toMatchObject({ + handle: 'term_agent', + orphaned: false, + tabId: 'tab-agent', + leafId: HEADLESS_LEAF_ID + }) + expect(listProcesses).toHaveBeenCalledTimes(inventoryCount + 2) + expect(listProcesses).toHaveBeenLastCalledWith(undefined) await expect( runtime.adoptTerminalOrphans({ worktree: `id:${TEST_WORKTREE_ID}`, @@ -16560,6 +16586,7 @@ describe('OrcaRuntimeService', () => { getForegroundProcess: async () => null, hasRendererSerializer: () => true, serializeBuffer, + hasPty: (ptyId) => ptyId === 'pty-legacy', listProcesses: async () => [ { id: 'pty-legacy', @@ -16606,7 +16633,11 @@ describe('OrcaRuntimeService', () => { presentation: 'background', tabId: 'legacy-worker', leafId: workerLeafId, - focus: false + focus: false, + expectedProcessIdentity: { + terminalHandle: 'term_legacy', + incarnationId + } }) expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(workerPaneKey, 'adopted') expect(write).not.toHaveBeenCalled() @@ -16683,6 +16714,7 @@ describe('OrcaRuntimeService', () => { write: vi.fn(() => true), kill: vi.fn(() => true), getForegroundProcess: async () => null, + hasPty: (ptyId) => ptyId === 'pty-reveal-retry', listProcesses: async () => [ { id: 'pty-reveal-retry', @@ -16716,6 +16748,269 @@ describe('OrcaRuntimeService', () => { expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(workerPaneKey, 'adopted') }) + function makePostRevealWorkerRecoveryHarness(hasPty: (ptyId: string) => boolean | null): { + runtime: OrcaRuntimeService + getSession: () => WorkspaceSessionState + workerPaneKey: string + ptyId: string + incarnationId: string + terminalHandle: string + revealTerminalSession: ReturnType + resolveLegacyWorkerTerminalRecovery: ReturnType + } { + const workerPaneKey = `legacy-post-reveal:${HEADLESS_LEAF_ID}` + const ptyId = 'pty-post-reveal' + const incarnationId = '45454545-4545-4545-8545-454545454545' + const terminalHandle = 'term_post_reveal' + const session: WorkspaceSessionState = { + ...getDefaultWorkspaceSession(), + tabsByWorktree: { [TEST_WORKTREE_ID]: [] }, + sleepingAgentSessionsByPaneKey: { + [workerPaneKey]: { + paneKey: workerPaneKey, + tabId: 'legacy-post-reveal', + worktreeId: TEST_WORKTREE_ID, + agent: 'codex', + providerSession: { key: 'session_id', id: 'legacy-post-reveal-session' }, + prompt: 'continue', + state: 'working', + capturedAt: 1, + updatedAt: 1, + origin: 'live' + } + } + } + const { runtimeStore, getSession } = makeRuntimeStoreWithWorkspaceSession(session) + const runtime = new OrcaRuntimeService( + { ...runtimeStore, flushOrThrow: vi.fn() } as never, + undefined, + { canRecoverPersistentLocalPtys: () => true } + ) + runtime.setOrchestrationDb({ + listLegacyWorkerTerminalRecoveryRows: () => [ + { + dispatch_id: 'dispatch-post-reveal', + task_id: 'task-post-reveal', + dispatch_status: 'completed', + contract_version: 0, + assignee_handle: 'term_post_reveal', + assignee_pane_key: workerPaneKey, + process_incarnation: `${ptyId}:${incarnationId}`, + worker_state: 'ready', + worktree_id: TEST_WORKTREE_ID, + agent_terminal_handle: 'term_post_reveal' + } + ] + } as unknown as OrchestrationDb) + runtime.setPtyController({ + write: vi.fn(() => true), + kill: vi.fn(() => true), + getForegroundProcess: async () => null, + hasPty, + listProcesses: async () => [ + { + id: ptyId, + incarnationId, + terminalHandle: 'term_post_reveal', + title: 'Post-reveal worker', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + } + ] + }) + const revealTerminalSession = vi.fn().mockResolvedValue({ tabId: 'legacy-post-reveal' }) + const resolveLegacyWorkerTerminalRecovery = vi.fn() + runtime.setNotifier({ + revealTerminalSession, + resolveLegacyWorkerTerminalRecovery + } as never) + return { + runtime, + getSession, + workerPaneKey, + ptyId, + incarnationId, + terminalHandle, + revealTerminalSession, + resolveLegacyWorkerTerminalRecovery + } + } + + it('keeps a revealed worker fenced until its exact renderer graph is published', async () => { + vi.useFakeTimers() + try { + const harness = makePostRevealWorkerRecoveryHarness(() => true) + const identity = { + worktreeId: TEST_WORKTREE_ID, + tabId: 'legacy-post-reveal', + leafId: HEADLESS_LEAF_ID, + ptyId: harness.ptyId + } + harness.revealTerminalSession.mockResolvedValue({ + tabId: identity.tabId, + identity + }) + + await expect( + harness.runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true }) + ).resolves.toMatchObject({ + adoptedDispatchIds: [], + exitedDispatchIds: [], + deferredDispatchIds: ['dispatch-post-reveal'] + }) + expect(harness.revealTerminalSession).toHaveBeenCalledOnce() + expect( + harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey] + ).toBeDefined() + expect(harness.resolveLegacyWorkerTerminalRecovery).not.toHaveBeenCalled() + + harness.runtime.attachWindow(1) + harness.runtime.syncWindowGraph(1, { + tabs: [ + { + tabId: identity.tabId, + worktreeId: identity.worktreeId, + title: 'Post-reveal worker', + activeLeafId: identity.leafId, + layout: null + } + ], + leaves: [ + { + tabId: identity.tabId, + worktreeId: identity.worktreeId, + leafId: identity.leafId, + paneRuntimeId: 1, + ptyId: identity.ptyId + } + ] + }) + await vi.advanceTimersByTimeAsync(1_000) + + expect(harness.revealTerminalSession).toHaveBeenCalledOnce() + expect( + harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey] + ).toBeUndefined() + expect(harness.resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith( + harness.workerPaneKey, + 'adopted' + ) + expect( + (await harness.runtime.listTerminals()).terminals.filter( + (terminal) => terminal.ptyId === identity.ptyId + ) + ).toEqual([ + expect.objectContaining({ + handle: harness.terminalHandle, + incarnationId: harness.incarnationId, + orphaned: false, + worktreeId: identity.worktreeId, + tabId: identity.tabId, + leafId: identity.leafId + }) + ]) + } finally { + vi.useRealTimers() + } + }) + + it('retires the exact worker when it exits after renderer reveal', async () => { + const hasPty = vi.fn().mockReturnValueOnce(true).mockReturnValueOnce(false) + const harness = makePostRevealWorkerRecoveryHarness(hasPty) + + await expect( + harness.runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true }) + ).resolves.toMatchObject({ + adoptedDispatchIds: [], + exitedDispatchIds: ['dispatch-post-reveal'], + deferredDispatchIds: [] + }) + + expect(hasPty).toHaveBeenCalledTimes(2) + expect(harness.revealTerminalSession).toHaveBeenCalledOnce() + expect(harness.getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([]) + expect( + harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey] + ).toBeUndefined() + expect( + ( + harness.runtime as unknown as { + ptysById: Map + } + ).ptysById.get(harness.ptyId) + ).toMatchObject({ connected: false, incarnationId: '45454545-4545-4545-8545-454545454545' }) + expect(harness.resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith( + harness.workerPaneKey, + 'exited' + ) + }) + + it.each([ + { + name: 'null', + makeHasPty: () => + vi.fn().mockReturnValueOnce(true).mockReturnValueOnce(null).mockReturnValue(true) + }, + { + name: 'throw', + makeHasPty: () => + vi + .fn() + .mockReturnValueOnce(true) + .mockImplementationOnce(() => { + throw new Error('provider unavailable') + }) + .mockReturnValue(true) + } + ])( + 'retries post-reveal $name liveness without duplicating the worker surface', + async ({ makeHasPty }) => { + vi.useFakeTimers() + try { + const hasPty = makeHasPty() + const harness = makePostRevealWorkerRecoveryHarness(hasPty) + + await expect( + harness.runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true }) + ).resolves.toMatchObject({ + adoptedDispatchIds: [], + exitedDispatchIds: [], + deferredDispatchIds: ['dispatch-post-reveal'] + }) + expect(harness.revealTerminalSession).toHaveBeenCalledOnce() + expect(harness.getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([ + expect.objectContaining({ id: 'legacy-post-reveal', ptyId: harness.ptyId }) + ]) + expect( + harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey] + ?.automaticResumeBlockedBy + ).toBe('legacy-orchestration-worker') + + await vi.advanceTimersByTimeAsync(1_000) + + expect(hasPty).toHaveBeenCalledTimes(4) + expect(harness.revealTerminalSession).toHaveBeenCalledOnce() + expect(harness.getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([ + expect.objectContaining({ id: 'legacy-post-reveal', ptyId: harness.ptyId }) + ]) + expect( + harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey] + ).toBeUndefined() + expect(harness.resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith( + harness.workerPaneKey, + 'adopted' + ) + + await vi.advanceTimersByTimeAsync(30_000) + expect(hasPty).toHaveBeenCalledTimes(4) + expect(harness.revealTerminalSession).toHaveBeenCalledOnce() + } finally { + vi.useRealTimers() + } + } + ) + it('keeps the legacy worker resume fence in memory when persistence fails', async () => { const workerPaneKey = `legacy-worker:${HEADLESS_LEAF_ID}` const incarnationId = '99999999-9999-4999-8999-999999999999' @@ -16768,6 +17063,7 @@ describe('OrcaRuntimeService', () => { write: vi.fn(() => true), kill: vi.fn(() => true), getForegroundProcess: async () => null, + hasPty: (ptyId) => ptyId === 'pty-persistence-failure', listProcesses: async () => [ { id: 'pty-persistence-failure', @@ -16804,8 +17100,9 @@ describe('OrcaRuntimeService', () => { warn.mockRestore() }) - it('unblocks provider resume only after authoritative inventory proves the legacy PTY exited', async () => { + it('retires provider resume only after authoritative inventory proves the legacy PTY exited', async () => { const workerPaneKey = `legacy-worker:${HEADLESS_LEAF_ID}` + const secondWorkerPaneKey = `legacy-worker-two:${HEADLESS_SECOND_LEAF_ID}` const incarnationId = '33333333-3333-4333-8333-333333333333' const session: WorkspaceSessionState = { ...getDefaultWorkspaceSession(), @@ -16822,6 +17119,18 @@ describe('OrcaRuntimeService', () => { capturedAt: 1, updatedAt: 1, origin: 'live' + }, + [secondWorkerPaneKey]: { + paneKey: secondWorkerPaneKey, + tabId: 'legacy-worker-two', + worktreeId: TEST_WORKTREE_ID, + agent: 'codex', + providerSession: { key: 'session_id', id: 'legacy-codex-session-two' }, + prompt: 'continue', + state: 'working', + capturedAt: 1, + updatedAt: 1, + origin: 'live' } } } @@ -16844,17 +17153,50 @@ describe('OrcaRuntimeService', () => { worker_state: 'ready', worktree_id: TEST_WORKTREE_ID, agent_terminal_handle: 'term_exited' + }, + { + dispatch_id: 'dispatch-exited-two', + task_id: 'task-exited-two', + dispatch_status: 'completed', + contract_version: 0, + assignee_handle: 'term_exited_two', + assignee_pane_key: secondWorkerPaneKey, + process_incarnation: `pty-exited-two:${incarnationId}`, + worker_state: 'ready', + worktree_id: TEST_WORKTREE_ID, + agent_terminal_handle: 'term_exited_two' } ] } as unknown as OrchestrationDb) + const listProcesses = vi.fn(async (connectionId?: string | null) => { + if (connectionId !== null) { + throw new Error('unrelated SSH inventory must not run') + } + return [ + { + id: 'pty-exited-two', + incarnationId, + terminalHandle: 'term_exited_two', + title: 'Exited worker', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + } + ] + }) runtime.setPtyController({ write: vi.fn(() => true), kill: vi.fn(() => true), getForegroundProcess: async () => null, - listProcesses: async () => [] + hasPty: () => false, + listProcesses }) + const revealTerminalSession = vi.fn() const resolveLegacyWorkerTerminalRecovery = vi.fn() - runtime.setNotifier({ resolveLegacyWorkerTerminalRecovery } as never) + runtime.setNotifier({ + revealTerminalSession, + resolveLegacyWorkerTerminalRecovery + } as never) runtime.prepareLegacyWorkerTerminalRecovery() expect( @@ -16863,16 +17205,27 @@ describe('OrcaRuntimeService', () => { await expect(runtime.reconcileLegacyWorkerTerminals()).resolves.toMatchObject({ adoptedDispatchIds: [], - exitedDispatchIds: ['dispatch-exited'], + exitedDispatchIds: ['dispatch-exited', 'dispatch-exited-two'], deferredDispatchIds: [] }) + expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined() + expect(getSession().sleepingAgentSessionsByPaneKey?.[secondWorkerPaneKey]).toBeUndefined() + expect(listProcesses).toHaveBeenCalledOnce() + expect(listProcesses).toHaveBeenCalledWith(null) + expect(getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([]) + expect(revealTerminalSession).not.toHaveBeenCalled() expect( - getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]?.automaticResumeBlockedBy - ).toBeUndefined() + ( + runtime as unknown as { + ptysById: Map + } + ).ptysById.get('pty-exited-two') + ).toMatchObject({ connected: false }) expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(workerPaneKey, 'exited') + expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(secondWorkerPaneKey, 'exited') }) - it('keeps provider resume fenced when authoritative PTY inventory is unavailable', async () => { + it('retries inventory and unknown liveness without revealing a ghost worker', async () => { const workerPaneKey = `legacy-worker:${HEADLESS_LEAF_ID}` const incarnationId = '55555555-5555-4555-8555-555555555555' const session: WorkspaceSessionState = { @@ -16915,23 +17268,263 @@ describe('OrcaRuntimeService', () => { } ] } as unknown as OrchestrationDb) + const listProcesses = vi + .fn() + .mockRejectedValueOnce(new Error('local provider unavailable')) + .mockResolvedValue([ + { + id: 'pty-inventory-unavailable', + incarnationId, + terminalHandle: 'term_inventory_unavailable', + title: 'Recovered legacy worker', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + } + ]) + const hasPty = vi.fn().mockReturnValueOnce(null).mockReturnValue(true) runtime.setPtyController({ write: vi.fn(() => true), kill: vi.fn(() => true), - getForegroundProcess: async () => null + getForegroundProcess: async () => null, + hasPty, + listProcesses }) + const revealTerminalSession = vi.fn().mockResolvedValue({ tabId: 'legacy-worker' }) const resolveLegacyWorkerTerminalRecovery = vi.fn() - runtime.setNotifier({ resolveLegacyWorkerTerminalRecovery } as never) + runtime.setNotifier({ + revealTerminalSession, + resolveLegacyWorkerTerminalRecovery + } as never) + + vi.useFakeTimers() + try { + await expect( + runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true }) + ).resolves.toMatchObject({ + adoptedDispatchIds: [], + exitedDispatchIds: [], + deferredDispatchIds: ['dispatch-inventory-unavailable'] + }) + expect( + getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]?.automaticResumeBlockedBy + ).toBe('legacy-orchestration-worker') + expect(resolveLegacyWorkerTerminalRecovery).not.toHaveBeenCalled() + expect(listProcesses).toHaveBeenCalledOnce() + expect(getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([]) + expect(revealTerminalSession).not.toHaveBeenCalled() + + await vi.advanceTimersByTimeAsync(1_000) + + expect(listProcesses).toHaveBeenCalledTimes(2) + expect(listProcesses.mock.calls).toEqual([[null], [null]]) + expect(hasPty).toHaveBeenCalledOnce() + expect(getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([]) + expect(revealTerminalSession).not.toHaveBeenCalled() + expect( + getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]?.automaticResumeBlockedBy + ).toBe('legacy-orchestration-worker') + + await vi.advanceTimersByTimeAsync(2_000) + + expect(listProcesses).toHaveBeenCalledTimes(3) + expect(hasPty).toHaveBeenCalledTimes(3) + expect(getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([ + expect.objectContaining({ id: 'legacy-worker', ptyId: 'pty-inventory-unavailable' }) + ]) + expect(revealTerminalSession).toHaveBeenCalledOnce() + expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined() + expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(workerPaneKey, 'adopted') + await vi.advanceTimersByTimeAsync(30_000) + expect(listProcesses).toHaveBeenCalledTimes(3) + expect(revealTerminalSession).toHaveBeenCalledOnce() + } finally { + vi.useRealTimers() + } + }) + + it('cancels a coalesced SSH worker recovery retry when its provider disconnects', async () => { + vi.useFakeTimers() + try { + const runtime = new OrcaRuntimeService(store) + const reconcile = vi.spyOn(runtime, 'reconcileLegacyWorkerTerminals').mockResolvedValue({ + blockedPaneCount: 1, + adoptedDispatchIds: [], + exitedDispatchIds: [], + deferredDispatchIds: [] + }) + const retryInternals = runtime as unknown as { + updateLegacyWorkerTerminalRecoveryRetry: ( + plan: { + candidates: { dispatchId: string; ptyId: string }[] + }, + deferredDispatchIds: ReadonlySet, + options: { connectionId?: string; materializeRenderer?: boolean } + ) => void + } + const plan = { + candidates: [ + { + dispatchId: 'dispatch-ssh-retry', + ptyId: 'ssh:ssh-retry@@pty-worker' + } + ] + } + const deferred = new Set(['dispatch-ssh-retry']) + + retryInternals.updateLegacyWorkerTerminalRecoveryRetry(plan, deferred, { + connectionId: 'ssh-retry', + materializeRenderer: true + }) + retryInternals.updateLegacyWorkerTerminalRecoveryRetry(plan, deferred, { + connectionId: 'ssh-retry', + materializeRenderer: true + }) + runtime.notifySshStateChanged('ssh-retry', { + targetId: 'ssh-retry', + status: 'disconnected', + error: null, + reconnectAttempt: 0 + }) + await vi.advanceTimersByTimeAsync(30_000) + + expect(reconcile).not.toHaveBeenCalled() + } finally { + vi.useRealTimers() + } + }) + + it('keeps live workers fenced without exact controller identity evidence', async () => { + const incarnationId = '56565656-5656-4656-8656-565656565656' + const cases = [ + { + name: 'missing', + leafId: '11111111-1111-4111-8111-111111111111', + terminalHandle: 'term_missing' + }, + { + name: 'ambiguous', + leafId: '22222222-2222-4222-8222-222222222222', + terminalHandle: 'term_ambiguous' + }, + { + name: 'wrong-handle', + leafId: '33333333-3333-4333-8333-333333333333', + terminalHandle: 'term_wrong_handle' + }, + { + name: 'wrong-incarnation', + leafId: '44444444-4444-4444-8444-444444444444', + terminalHandle: 'term_wrong_incarnation' + } + ] as const + const sleepingAgentSessionsByPaneKey = Object.fromEntries( + cases.map(({ name, leafId }) => { + const paneKey = `legacy-${name}:${leafId}` + return [ + paneKey, + { + paneKey, + tabId: `legacy-${name}`, + worktreeId: TEST_WORKTREE_ID, + agent: 'codex', + providerSession: { key: 'session_id', id: `session-${name}` }, + prompt: 'continue', + state: 'working', + capturedAt: 1, + updatedAt: 1, + origin: 'live' + } + ] + }) + ) as WorkspaceSessionState['sleepingAgentSessionsByPaneKey'] + const session: WorkspaceSessionState = { + ...getDefaultWorkspaceSession(), + tabsByWorktree: { [TEST_WORKTREE_ID]: [] }, + sleepingAgentSessionsByPaneKey + } + const { runtimeStore, getSession } = makeRuntimeStoreWithWorkspaceSession(session) + const runtime = new OrcaRuntimeService( + { ...runtimeStore, flushOrThrow: vi.fn() } as never, + undefined, + { canRecoverPersistentLocalPtys: () => true } + ) + runtime.setOrchestrationDb({ + listLegacyWorkerTerminalRecoveryRows: () => + cases.map(({ name, leafId, terminalHandle }) => ({ + dispatch_id: `dispatch-${name}`, + task_id: `task-${name}`, + dispatch_status: 'completed', + contract_version: 0, + assignee_handle: terminalHandle, + assignee_pane_key: `legacy-${name}:${leafId}`, + process_incarnation: `pty-${name}:${incarnationId}`, + worker_state: 'ready', + worktree_id: TEST_WORKTREE_ID, + agent_terminal_handle: terminalHandle + })) + } as unknown as OrchestrationDb) + const listProcesses = vi.fn(async () => [ + { + id: 'pty-missing', + title: 'Missing identity', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID + }, + { + id: 'pty-ambiguous', + incarnationId, + terminalHandle: 'term_ambiguous', + title: 'Ambiguous identity', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID + }, + { + id: 'pty-ambiguous-other', + incarnationId, + terminalHandle: 'term_ambiguous', + title: 'Ambiguous identity duplicate', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID + }, + { + id: 'pty-wrong-handle', + incarnationId, + terminalHandle: 'term_other', + title: 'Wrong handle', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID + }, + { + id: 'pty-wrong-incarnation', + incarnationId: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa', + terminalHandle: 'term_wrong_incarnation', + title: 'Wrong incarnation', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID + } + ]) + runtime.setPtyController({ + write: vi.fn(() => true), + kill: vi.fn(() => true), + getForegroundProcess: async () => null, + hasPty: (candidate) => candidate === 'pty-folder-legacy', + listProcesses + }) await expect(runtime.reconcileLegacyWorkerTerminals()).resolves.toMatchObject({ adoptedDispatchIds: [], exitedDispatchIds: [], - deferredDispatchIds: ['dispatch-inventory-unavailable'] + deferredDispatchIds: cases.map(({ name }) => `dispatch-${name}`) }) - expect( - getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]?.automaticResumeBlockedBy - ).toBe('legacy-orchestration-worker') - expect(resolveLegacyWorkerTerminalRecovery).not.toHaveBeenCalled() + expect(listProcesses).toHaveBeenCalledOnce() + expect(listProcesses).toHaveBeenCalledWith(null) + for (const { name, leafId } of cases) { + expect( + getSession().sleepingAgentSessionsByPaneKey?.[`legacy-${name}:${leafId}`] + ?.automaticResumeBlockedBy + ).toBe('legacy-orchestration-worker') + } }) it('adopts an exact live legacy worker in a folder workspace', async () => { @@ -16986,21 +17579,23 @@ describe('OrcaRuntimeService', () => { } ] } as unknown as OrchestrationDb) + const listProcesses = vi.fn(async () => [ + { + id: 'pty-folder-legacy', + incarnationId, + terminalHandle: 'term_folder', + title: 'Folder worker', + cwd: folderPath, + worktreeId: TEST_FOLDER_WORKSPACE_KEY, + wslDistro: null + } + ]) runtime.setPtyController({ write: vi.fn(() => true), kill: vi.fn(() => true), getForegroundProcess: async () => null, - listProcesses: async () => [ - { - id: 'pty-folder-legacy', - incarnationId, - terminalHandle: 'term_folder', - title: 'Folder worker', - cwd: folderPath, - worktreeId: TEST_FOLDER_WORKSPACE_KEY, - wslDistro: null - } - ] + hasPty: (candidate) => candidate === 'pty-folder-legacy', + listProcesses }) const revealTerminalSession = vi.fn().mockResolvedValue({ tabId: 'legacy-folder-worker' @@ -17022,6 +17617,8 @@ describe('OrcaRuntimeService', () => { }) ) expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined() + expect(listProcesses).toHaveBeenCalledOnce() + expect(listProcesses).toHaveBeenCalledWith(null) expect(revealTerminalSession).toHaveBeenCalledWith(TEST_FOLDER_WORKSPACE_KEY, { ptyId: 'pty-folder-legacy', title: 'Folder worker', @@ -17029,7 +17626,11 @@ describe('OrcaRuntimeService', () => { presentation: 'background', tabId: 'legacy-folder-worker', leafId: HEADLESS_LEAF_ID, - focus: false + focus: false, + expectedProcessIdentity: { + terminalHandle: 'term_folder', + incarnationId + } }) }) @@ -17101,21 +17702,23 @@ describe('OrcaRuntimeService', () => { } ] } as unknown as OrchestrationDb) + const listProcesses = vi.fn(async () => [ + { + id: ptyId, + incarnationId, + terminalHandle: 'term_ssh_folder', + title: 'SSH folder worker', + cwd: folderPath, + worktreeId: TEST_FOLDER_WORKSPACE_KEY, + wslDistro: null + } + ]) runtime.setPtyController({ write: vi.fn(() => true), kill: vi.fn(() => true), getForegroundProcess: async () => null, - listProcesses: async () => [ - { - id: ptyId, - incarnationId, - terminalHandle: 'term_ssh_folder', - title: 'SSH folder worker', - cwd: folderPath, - worktreeId: TEST_FOLDER_WORKSPACE_KEY, - wslDistro: null - } - ] + hasPty: (candidate) => candidate === ptyId, + listProcesses }) const revealTerminalSession = vi.fn().mockResolvedValue({ tabId: 'legacy-ssh-folder-worker' }) runtime.setNotifier({ revealTerminalSession } as never) @@ -17149,6 +17752,8 @@ describe('OrcaRuntimeService', () => { expect(localSession.sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined() expect(getWorkspaceSession).toHaveBeenCalledWith(`ssh:${connectionId}`) expect(setWorkspaceSession).toHaveBeenCalledWith(expect.any(Object), `ssh:${connectionId}`) + expect(listProcesses).toHaveBeenCalledOnce() + expect(listProcesses).toHaveBeenCalledWith(connectionId) expect(sshSession.tabsByWorktree[TEST_FOLDER_WORKSPACE_KEY]).toContainEqual( expect.objectContaining({ id: 'legacy-ssh-folder-worker', @@ -17163,7 +17768,11 @@ describe('OrcaRuntimeService', () => { presentation: 'background', tabId: 'legacy-ssh-folder-worker', leafId: HEADLESS_LEAF_ID, - focus: false + focus: false, + expectedProcessIdentity: { + terminalHandle: 'term_ssh_folder', + incarnationId + } }) }) @@ -17321,6 +17930,7 @@ describe('OrcaRuntimeService', () => { write: vi.fn(() => true), kill: vi.fn(() => true), getForegroundProcess: async () => null, + hasPty: (candidate) => candidate === ptyId, listProcesses }) const revealTerminalSession = vi.fn().mockResolvedValue({ tabId: 'legacy-ssh-worker' }) @@ -17365,12 +17975,29 @@ describe('OrcaRuntimeService', () => { exitedDispatchIds: [], deferredDispatchIds: [] }) + await expect( + runtime.adoptTerminalOrphans({ + worktree: `id:${TEST_WORKTREE_ID}`, + expectedTopologyRevision: 0, + claims: [ + { + terminal: 'term_ssh_legacy', + ptyId, + incarnationId, + tabId: 'legacy-ssh-worker', + leafId: HEADLESS_LEAF_ID + } + ] + }) + ).resolves.toMatchObject({ adopted: false }) + expect(listProcesses).toHaveBeenLastCalledWith(connectionId) } finally { unregisterSshGitProvider(connectionId) } expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined() expect(setWorkspaceSession).toHaveBeenCalledWith(expect.any(Object), `ssh:${connectionId}`) + expect(listProcesses).toHaveBeenCalledTimes(2) expect(revealTerminalSession).toHaveBeenCalledWith(TEST_WORKTREE_ID, { ptyId, title: 'SSH legacy worker', @@ -17378,7 +18005,11 @@ describe('OrcaRuntimeService', () => { presentation: 'background', tabId: 'legacy-ssh-worker', leafId: HEADLESS_LEAF_ID, - focus: false + focus: false, + expectedProcessIdentity: { + terminalHandle: 'term_ssh_legacy', + incarnationId + } }) }) @@ -17445,21 +18076,23 @@ describe('OrcaRuntimeService', () => { } ] } as unknown as OrchestrationDb) + const listProcesses = vi.fn(async () => [ + { + id: 'pty-wsl-legacy', + incarnationId, + terminalHandle: 'term_wsl_legacy', + title: 'WSL legacy worker', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: observedDistro + } + ]) runtime.setPtyController({ write: vi.fn(() => true), kill: vi.fn(() => true), getForegroundProcess: async () => null, - listProcesses: async () => [ - { - id: 'pty-wsl-legacy', - incarnationId, - terminalHandle: 'term_wsl_legacy', - title: 'WSL legacy worker', - cwd: TEST_WORKTREE_PATH, - worktreeId: TEST_WORKTREE_ID, - wslDistro: observedDistro - } - ] + hasPty: (ptyId) => ptyId === 'pty-wsl-legacy', + listProcesses }) const revealTerminalSession = vi.fn().mockResolvedValue({ tabId: 'legacy-wsl-worker' }) runtime.setNotifier({ revealTerminalSession } as never) @@ -17486,6 +18119,8 @@ describe('OrcaRuntimeService', () => { }) expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined() expect(revealTerminalSession).toHaveBeenCalledOnce() + expect(listProcesses).toHaveBeenCalledTimes(2) + expect(listProcesses.mock.calls).toEqual([[null], [null]]) }) it('restores orphan pane and group topology without replacing a newer host-owned tab', async () => { diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index 041617e7153..4c289c48fc1 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -36,10 +36,7 @@ import { type AgentStatusEntry } from '../../shared/agent-status-types' import { indexAgentStatusRowsByPaneKey } from '../agent-hooks/agent-status-pane-index' -import type { - AgentHookAuthorityAttestation, - AgentHookAuthorityEvidence -} from '../agent-hooks/server' +import type { AgentHookAuthorityAttestation } from '../agent-hooks/server' import type { AgentSessionClaimedSpawnResult, AgentSessionExecutionClaim, @@ -110,6 +107,7 @@ import { createSetupCompletionScanner } from './orchestration/setup-completion-signal' import type { RuntimeOrchestrationEnvelope } from '../../shared/runtime-rpc-envelope' +import type { TerminalRevealIdentity } from '../../shared/terminal-reveal-identity' import type { OrchestrationCompatibilityEvidence, OrchestrationCompatibilityHostStamp @@ -1506,7 +1504,7 @@ type RuntimePtyController = { resize?(ptyId: string, cols: number, rows: number): boolean // Why: exact-id mobile polls should not enumerate every local and SSH PTY. hasPty?(ptyId: string): boolean | null - listProcesses?(): Promise + listProcesses?(connectionId?: string | null): Promise serializeBuffer?( ptyId: string, opts?: { scrollbackRows?: number; altScreenForcesZeroRows?: boolean } @@ -1530,6 +1528,17 @@ type RuntimePtyController = { getSize?(ptyId: string): { cols: number; rows: number } | null } +type PtyControllerTerminalIdentity = Readonly<{ + handle: string + incarnationId: string + wslDistro?: string | null +}> + +type PtyControllerInventory = Readonly<{ + livePtyIds: ReadonlySet + terminalIdentityByPtyId: ReadonlyMap +}> + type WorktreeStartupDraftPaste = { agent: TuiAgent content: string @@ -1645,10 +1654,14 @@ type RuntimeNotifier = { splitDirection?: 'horizontal' | 'vertical' splitTelemetrySource?: TerminalPaneSplitSource focus?: boolean + expectedProcessIdentity?: { + terminalHandle: string + incarnationId: string + } } ): - | Promise<{ tabId: string; title?: string | null }> - | { tabId: string; title?: string | null } + | Promise<{ tabId: string; title?: string | null; identity?: TerminalRevealIdentity }> + | { tabId: string; title?: string | null; identity?: TerminalRevealIdentity } | void resolveLegacyWorkerTerminalRecovery?(paneKey: string, resolution: 'adopted' | 'exited'): void splitTerminal( @@ -1733,11 +1746,6 @@ export type OrchestrationCompatibilityTerminalAuthority = { | { kind: 'ssh'; targetId: string } } -export type OrchestrationCompatibilityAuthorityInputs = { - hydratedCommitments: readonly AgentHookAuthorityEvidence[] - currentObservations: readonly AgentHookAuthorityEvidence[] -} - export type LegacyWorkerTerminalRecoveryResult = { blockedPaneCount: number adoptedDispatchIds: string[] @@ -2615,10 +2623,6 @@ export class OrcaRuntimeService { private handles = new Map() private handleByLeafKey = new Map() private handleByPtyId = new Map() - private controllerTerminalIdentityByPtyId = new Map< - string, - { handle: string; incarnationId: string; wslDistro?: string | null } - >() private detachedPreAllocatedLeaves = new Map() private graphSyncCallbacks: (() => void)[] = [] private waitersByHandle = new Map>() @@ -3011,12 +3015,6 @@ export class OrcaRuntimeService { private readonly getAgentProviderSessionRowsForPaneFn: | ((paneKey: string) => AgentStatusIpcPayload[]) | null - private readonly getHydratedAgentHookAuthorityFn: - | (() => readonly AgentHookAuthorityEvidence[]) - | null - private readonly getCurrentAgentHookAuthorityFn: - | (() => readonly AgentHookAuthorityEvidence[]) - | null private readonly attestAgentHookCompatibilityAuthorityFn: | ((candidate: { paneKey: string @@ -3038,7 +3036,17 @@ export class OrcaRuntimeService { >() private sshRelayRecoveryGenerationByTargetId = new Map() private legacyWorkerTerminalRecoveryQueue: Promise = Promise.resolve() + private legacyWorkerTerminalRecoveryRetries = new Map< + string, + { + attempt: number + connectionId?: string + materializeRenderer: boolean + timer: ReturnType | null + } + >() private legacyWorkerTerminalMaterializedPanes = new Set() + private legacyWorkerTerminalIdentityReceiptPanes = new Set() private legacyWorkerRecoveredPtys = new Set() private accountServices: RuntimeAccountServices | null = null private commitMessageAgentEnv: CommitMessageAgentEnvironmentResolvers | null = null @@ -3074,8 +3082,6 @@ export class OrcaRuntimeService { * only carrier of the provider session a transcript is addressed by. */ getAgentProviderSessionSnapshot?: () => AgentStatusIpcPayload[] getAgentProviderSessionRowsForPane?: (paneKey: string) => AgentStatusIpcPayload[] - getHydratedAgentHookAuthority?: () => readonly AgentHookAuthorityEvidence[] - getCurrentAgentHookAuthority?: () => readonly AgentHookAuthorityEvidence[] attestAgentHookCompatibilityAuthority?: (candidate: { paneKey: string launchTokenHash: string @@ -3114,8 +3120,6 @@ export class OrcaRuntimeService { this.getAgentProviderSessionSnapshotFn = deps?.getAgentProviderSessionSnapshot ?? deps?.getAgentStatusSnapshot ?? null this.getAgentProviderSessionRowsForPaneFn = deps?.getAgentProviderSessionRowsForPane ?? null - this.getHydratedAgentHookAuthorityFn = deps?.getHydratedAgentHookAuthority ?? null - this.getCurrentAgentHookAuthorityFn = deps?.getCurrentAgentHookAuthority ?? null this.attestAgentHookCompatibilityAuthorityFn = deps?.attestAgentHookCompatibilityAuthority ?? null this.canRecoverPersistentLocalPtysFn = deps?.canRecoverPersistentLocalPtys ?? (() => true) @@ -3659,6 +3663,70 @@ export class OrcaRuntimeService { return result } + private hasExactTerminalSurfaceIdentity(expected: { + worktreeId: string + tabId: string + leafId: string + ptyId: string + terminalHandle: string + incarnationId: string + }): boolean { + if (this.graphStatus !== 'ready') { + return false + } + const pty = this.ptysById.get(expected.ptyId) + if ( + !pty?.connected || + pty.incarnationId !== expected.incarnationId || + pty.tabId !== expected.tabId || + pty.paneKey !== makePaneKey(expected.tabId, expected.leafId) || + !runtimeWorktreeIdsEqual(pty.worktreeId, expected.worktreeId) || + this.handleByPtyId.get(expected.ptyId) !== expected.terminalHandle + ) { + return false + } + const tab = this.tabs.get(expected.tabId) + const leaf = this.leaves.get(this.getLeafKey(expected.tabId, expected.leafId)) + const ptyLeaves = this.getLeavesForPty(expected.ptyId) + return ( + Boolean(tab && runtimeWorktreeIdsEqual(tab.worktreeId, expected.worktreeId)) && + Boolean( + leaf && + leaf.ptyId === expected.ptyId && + runtimeWorktreeIdsEqual(leaf.worktreeId, expected.worktreeId) + ) && + ptyLeaves.length === 1 && + ptyLeaves[0]?.tabId === expected.tabId && + ptyLeaves[0]?.leafId === expected.leafId + ) + } + + private hasExactPersistedTerminalSurfaceIdentity(expected: { + worktreeId: string + tabId: string + leafId: string + ptyId: string + incarnationId: string + }): boolean { + const session = this.getWorkspaceSessionForWorktree(expected.worktreeId) + const sessionWorktreeId = session + ? resolveTerminalSessionWorktreeId(session, expected.worktreeId) + : null + if (!session || !sessionWorktreeId) { + return false + } + const tab = session.tabsByWorktree[sessionWorktreeId]?.find( + (candidate) => candidate.id === expected.tabId + ) + const paneKey = makePaneKey(expected.tabId, expected.leafId) + return Boolean( + tab && + session.terminalLayoutsByTabId[expected.tabId]?.ptyIdsByLeafId?.[expected.leafId] === + expected.ptyId && + session.terminalPtyIncarnationsByPaneKey?.[paneKey] === expected.incarnationId + ) + } + private persistLegacyWorkerTerminalRecoveryResolution( candidate: LegacyWorkerTerminalRecoveryPlan['candidates'][number], resolution: 'adopted' | 'exited' @@ -3672,23 +3740,8 @@ export class OrcaRuntimeService { if (!record || !runtimeWorktreeIdsEqual(record.worktreeId, candidate.worktreeId)) { return true } - if ( - resolution === 'exited' && - record.automaticResumeBlockedBy !== 'legacy-orchestration-worker' - ) { - return true - } const next = structuredClone(session) - if (resolution === 'adopted') { - delete next.sleepingAgentSessionsByPaneKey?.[candidate.paneKey] - } else { - const unblocked = { ...record } - delete unblocked.automaticResumeBlockedBy - next.sleepingAgentSessionsByPaneKey = { - ...next.sleepingAgentSessionsByPaneKey, - [candidate.paneKey]: unblocked - } - } + delete next.sleepingAgentSessionsByPaneKey?.[candidate.paneKey] try { this.setWorkspaceSessionForWorktree(candidate.worktreeId, next) store.flushOrThrow() @@ -3704,6 +3757,85 @@ export class OrcaRuntimeService { } } + private updateLegacyWorkerTerminalRecoveryRetry( + plan: LegacyWorkerTerminalRecoveryPlan, + deferredDispatchIds: ReadonlySet, + options: { connectionId?: string; materializeRenderer?: boolean } + ): void { + const scopeKey = options.connectionId ? `ssh:${options.connectionId}` : 'local' + const hasDeferredWorker = plan.candidates.some((candidate) => { + const sshPty = parseAppSshPtyId(candidate.ptyId) + const inScope = options.connectionId + ? sshPty?.connectionId === options.connectionId + : sshPty === null + return inScope && deferredDispatchIds.has(candidate.dispatchId) + }) + if (!hasDeferredWorker) { + this.cancelLegacyWorkerTerminalRecoveryRetry(scopeKey) + return + } + const existing = this.legacyWorkerTerminalRecoveryRetries.get(scopeKey) + const retry = existing ?? { + attempt: 0, + ...(options.connectionId ? { connectionId: options.connectionId } : {}), + materializeRenderer: options.materializeRenderer === true, + timer: null + } + retry.materializeRenderer ||= options.materializeRenderer === true + this.legacyWorkerTerminalRecoveryRetries.set(scopeKey, retry) + this.armLegacyWorkerTerminalRecoveryRetry(scopeKey, retry) + } + + private cancelLegacyWorkerTerminalRecoveryRetry(scopeKey: string): void { + const retry = this.legacyWorkerTerminalRecoveryRetries.get(scopeKey) + if (retry?.timer) { + clearTimeout(retry.timer) + } + this.legacyWorkerTerminalRecoveryRetries.delete(scopeKey) + } + + private getLegacyWorkerControllerPtyLiveness( + candidate: LegacyWorkerTerminalRecoveryPlan['candidates'][number] + ): boolean | null { + try { + return this.ptyController?.hasPty?.(candidate.ptyId) ?? null + } catch { + return null + } + } + + private armLegacyWorkerTerminalRecoveryRetry( + scopeKey: string, + retry: { + attempt: number + connectionId?: string + materializeRenderer: boolean + timer: ReturnType | null + } + ): void { + if (retry.timer) { + return + } + const delayMs = Math.min(1_000 * 2 ** retry.attempt, 30_000) + retry.attempt += 1 + retry.timer = setTimeout(() => { + retry.timer = null + void this.reconcileLegacyWorkerTerminals({ + ...(retry.connectionId ? { connectionId: retry.connectionId } : {}), + materializeRenderer: retry.materializeRenderer + }).catch((error) => { + console.warn('[orchestration] worker terminal recovery retry failed', { + scope: scopeKey, + error + }) + if (this.legacyWorkerTerminalRecoveryRetries.get(scopeKey) === retry) { + this.armLegacyWorkerTerminalRecoveryRetry(scopeKey, retry) + } + }) + }, delayMs) + retry.timer.unref?.() + } + private async reconcileLegacyWorkerTerminalsNow(options: { connectionId?: string materializeRenderer?: boolean @@ -3712,142 +3844,251 @@ export class OrcaRuntimeService { const adoptedDispatchIds: string[] = [] const exitedDispatchIds: string[] = [] const deferredDispatchIds = new Set(plan.ambiguousDispatchIds) - for (const candidate of plan.candidates) { - let workspace: TerminalWorkspaceLaunchScope - try { - workspace = await this.resolveTerminalWorkspaceLaunchScope(`id:${candidate.worktreeId}`) - } catch { - deferredDispatchIds.add(candidate.dispatchId) - continue + const recoveryCandidatesByProvider = new Map< + string, + { + connectionId: string | null + entries: { + candidate: (typeof plan.candidates)[number] + workspace: TerminalWorkspaceLaunchScope + resolvedWorkspace: ResolvedWorktree + }[] } - const sshPty = parseAppSshPtyId(candidate.ptyId) - if (workspace.connectionId) { - if ( - options.connectionId !== workspace.connectionId || - sshPty?.connectionId !== workspace.connectionId + >() + for (const candidate of plan.candidates) { + try { + const workspace = await this.resolveTerminalWorkspaceLaunchScope( + `id:${candidate.worktreeId}` + ) + const sshPty = parseAppSshPtyId(candidate.ptyId) + if (workspace.connectionId) { + if ( + options.connectionId !== workspace.connectionId || + sshPty?.connectionId !== workspace.connectionId + ) { + deferredDispatchIds.add(candidate.dispatchId) + continue + } + } else if ( + options.connectionId !== undefined || + sshPty !== null || + !this.canRecoverPersistentLocalPtysFn() ) { deferredDispatchIds.add(candidate.dispatchId) continue } - } else if ( - options.connectionId !== undefined || - sshPty !== null || - !this.canRecoverPersistentLocalPtysFn() - ) { - deferredDispatchIds.add(candidate.dispatchId) - continue - } - const resolvedWorkspace = workspace.folderWorkspace - ? this.folderWorkspaceToResolvedWorktree(workspace.folderWorkspace) - : await this.resolveWorktreeSelector(`id:${workspace.id}`) - const livePtyIds = await this.refreshPtyWorktreeRecordsFromController([resolvedWorkspace]) - if (!livePtyIds) { - deferredDispatchIds.add(candidate.dispatchId) - continue - } - if (!livePtyIds.has(candidate.ptyId)) { - if (this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'exited')) { - this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'exited') - exitedDispatchIds.push(candidate.dispatchId) - } else { - deferredDispatchIds.add(candidate.dispatchId) + const resolvedWorkspace = workspace.folderWorkspace + ? this.folderWorkspaceToResolvedWorktree(workspace.folderWorkspace) + : await this.resolveWorktreeSelector(`id:${workspace.id}`) + const connectionId = workspace.connectionId ?? null + const providerKey = connectionId === null ? 'local' : `ssh:${connectionId}` + const provider = recoveryCandidatesByProvider.get(providerKey) ?? { + connectionId, + entries: [] } - continue - } - const controllerIdentity = this.controllerTerminalIdentityByPtyId.get(candidate.ptyId) - if ( - controllerIdentity?.handle !== candidate.terminalHandle || - controllerIdentity.incarnationId !== candidate.incarnationId - ) { + provider.entries.push({ candidate, workspace, resolvedWorkspace }) + recoveryCandidatesByProvider.set(providerKey, provider) + } catch { deferredDispatchIds.add(candidate.dispatchId) + } + } + for (const provider of recoveryCandidatesByProvider.values()) { + const resolvedWorktrees = [ + ...new Map( + provider.entries.map(({ resolvedWorkspace }) => [resolvedWorkspace.id, resolvedWorkspace]) + ).values() + ] + const inventory = await this.refreshPtyWorktreeRecordsWithControllerInventory( + resolvedWorktrees, + null, + undefined, + provider.connectionId + ) + if (!inventory) { + provider.entries.forEach(({ candidate }) => deferredDispatchIds.add(candidate.dispatchId)) continue } - const session = this.getWorkspaceSessionForWorktree(candidate.worktreeId) - const sessionWorktreeId = session - ? resolveTerminalSessionWorktreeId(session, candidate.worktreeId) - : null - const activeTabId = sessionWorktreeId - ? session?.activeTabIdByWorktree?.[sessionWorktreeId] - : undefined - const activeGroupId = sessionWorktreeId - ? session?.activeGroupIdByWorktree?.[sessionWorktreeId] - : undefined - try { - await this.adoptTerminalOrphans({ - worktree: `id:${candidate.worktreeId}`, - expectedTopologyRevision: this.getTerminalTopologyRevision(candidate.worktreeId), - ...(activeTabId ? { activeTabId } : {}), - ...(activeGroupId ? { activeGroupId } : {}), - claims: [ - { - terminal: candidate.terminalHandle, - ptyId: candidate.ptyId, - incarnationId: candidate.incarnationId, - tabId: candidate.tabId, - leafId: candidate.leafId - } - ] - }) - } catch (error) { - console.warn('[orchestration] legacy worker terminal adoption deferred', { - dispatchId: candidate.dispatchId, - error - }) - deferredDispatchIds.add(candidate.dispatchId) - continue - } - let rendererMaterialized = - options.materializeRenderer !== true || - this.legacyWorkerTerminalMaterializedPanes.has(candidate.paneKey) - const pty = this.ptysById.get(candidate.ptyId) - if ( - options.materializeRenderer && - !rendererMaterialized && - pty && - this.notifier?.revealTerminalSession - ) { - for (let attempt = 0; attempt < 2 && !rendererMaterialized; attempt += 1) { + for (const { candidate, workspace } of provider.entries) { + if (!inventory.livePtyIds.has(candidate.ptyId)) { + if (this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'exited')) { + this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'exited') + exitedDispatchIds.push(candidate.dispatchId) + } else { + deferredDispatchIds.add(candidate.dispatchId) + } + continue + } + const controllerIdentity = inventory.terminalIdentityByPtyId.get(candidate.ptyId) + if ( + controllerIdentity?.handle !== candidate.terminalHandle || + controllerIdentity.incarnationId !== candidate.incarnationId + ) { + deferredDispatchIds.add(candidate.dispatchId) + continue + } + const controllerLiveness = this.getLegacyWorkerControllerPtyLiveness(candidate) + if (controllerLiveness === false) { + this.onPtyExit(candidate.ptyId, 0, candidate.incarnationId) + if (this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'exited')) { + this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'exited') + exitedDispatchIds.push(candidate.dispatchId) + } else { + deferredDispatchIds.add(candidate.dispatchId) + } + continue + } + if (controllerLiveness === null) { + deferredDispatchIds.add(candidate.dispatchId) + continue + } + const session = this.getWorkspaceSessionForWorktree(candidate.worktreeId) + const sessionWorktreeId = session + ? resolveTerminalSessionWorktreeId(session, candidate.worktreeId) + : null + const activeTabId = sessionWorktreeId + ? session?.activeTabIdByWorktree?.[sessionWorktreeId] + : undefined + const activeGroupId = sessionWorktreeId + ? session?.activeGroupIdByWorktree?.[sessionWorktreeId] + : undefined + const exactSurfaceAlreadyPublished = + this.hasExactPersistedTerminalSurfaceIdentity(candidate) && + this.hasExactTerminalSurfaceIdentity(candidate) + if (!exactSurfaceAlreadyPublished) { try { - await this.notifier.revealTerminalSession(candidate.worktreeId, { - ptyId: candidate.ptyId, - title: getLatestPtyTitle(pty) ?? pty.controllerTitle, - activate: false, - presentation: 'background', - tabId: candidate.tabId, - leafId: candidate.leafId, - focus: false - }) - rendererMaterialized = true - this.legacyWorkerTerminalMaterializedPanes.add(candidate.paneKey) + await this.adoptTerminalOrphansFromInventory( + { + worktree: `id:${candidate.worktreeId}`, + expectedTopologyRevision: this.getTerminalTopologyRevision(candidate.worktreeId), + ...(activeTabId ? { activeTabId } : {}), + ...(activeGroupId ? { activeGroupId } : {}), + claims: [ + { + terminal: candidate.terminalHandle, + ptyId: candidate.ptyId, + incarnationId: candidate.incarnationId, + tabId: candidate.tabId, + leafId: candidate.leafId + } + ] + }, + workspace, + inventory + ) } catch (error) { - if (attempt === 0) { - await new Promise((resolve) => setTimeout(resolve, 100)) - continue - } - console.warn('[orchestration] adopted legacy worker was not revealed', { + console.warn('[orchestration] legacy worker terminal adoption deferred', { dispatchId: candidate.dispatchId, error }) + deferredDispatchIds.add(candidate.dispatchId) + continue } } + let rendererMaterialized = + options.materializeRenderer !== true || + this.legacyWorkerTerminalMaterializedPanes.has(candidate.paneKey) + const pty = this.ptysById.get(candidate.ptyId) + if ( + options.materializeRenderer && + !rendererMaterialized && + pty && + this.notifier?.revealTerminalSession + ) { + for (let attempt = 0; attempt < 2 && !rendererMaterialized; attempt += 1) { + try { + const reveal = await this.notifier.revealTerminalSession(candidate.worktreeId, { + ptyId: candidate.ptyId, + title: getLatestPtyTitle(pty) ?? pty.controllerTitle, + activate: false, + presentation: 'background', + tabId: candidate.tabId, + leafId: candidate.leafId, + focus: false, + expectedProcessIdentity: { + terminalHandle: candidate.terminalHandle, + incarnationId: candidate.incarnationId + } + }) + if (reveal?.identity) { + const identity = reveal.identity + if ( + !runtimeWorktreeIdsEqual(identity.worktreeId, candidate.worktreeId) || + identity.tabId !== candidate.tabId || + identity.leafId !== candidate.leafId || + identity.ptyId !== candidate.ptyId + ) { + throw new Error('terminal_reveal_identity_mismatch') + } + this.legacyWorkerTerminalIdentityReceiptPanes.add(candidate.paneKey) + } + rendererMaterialized = true + this.legacyWorkerTerminalMaterializedPanes.add(candidate.paneKey) + } catch (error) { + if (attempt === 0) { + await new Promise((resolve) => setTimeout(resolve, 100)) + continue + } + console.warn('[orchestration] adopted legacy worker was not revealed', { + dispatchId: candidate.dispatchId, + error + }) + } + } + } + if (!rendererMaterialized) { + this.legacyWorkerTerminalMaterializedPanes.delete(candidate.paneKey) + this.legacyWorkerTerminalIdentityReceiptPanes.delete(candidate.paneKey) + deferredDispatchIds.add(candidate.dispatchId) + continue + } + if ( + this.legacyWorkerTerminalIdentityReceiptPanes.has(candidate.paneKey) && + !this.hasExactTerminalSurfaceIdentity({ + worktreeId: candidate.worktreeId, + tabId: candidate.tabId, + leafId: candidate.leafId, + ptyId: candidate.ptyId, + terminalHandle: candidate.terminalHandle, + incarnationId: candidate.incarnationId + }) + ) { + deferredDispatchIds.add(candidate.dispatchId) + continue + } + const postRevealLiveness = this.getLegacyWorkerControllerPtyLiveness(candidate) + if (postRevealLiveness === false) { + this.legacyWorkerTerminalMaterializedPanes.delete(candidate.paneKey) + this.legacyWorkerTerminalIdentityReceiptPanes.delete(candidate.paneKey) + this.onPtyExit(candidate.ptyId, 0, candidate.incarnationId) + if (this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'exited')) { + this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'exited') + exitedDispatchIds.push(candidate.dispatchId) + } else { + deferredDispatchIds.add(candidate.dispatchId) + } + continue + } + if (postRevealLiveness === null) { + deferredDispatchIds.add(candidate.dispatchId) + continue + } + if (!this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'adopted')) { + deferredDispatchIds.add(candidate.dispatchId) + continue + } + this.legacyWorkerRecoveredPtys.add(candidate.ptyId) + this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'adopted') + adoptedDispatchIds.push(candidate.dispatchId) } - if ( - !rendererMaterialized || - !this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'adopted') - ) { - deferredDispatchIds.add(candidate.dispatchId) - continue - } - this.legacyWorkerRecoveredPtys.add(candidate.ptyId) - this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'adopted') - adoptedDispatchIds.push(candidate.dispatchId) } - return { + const result = { blockedPaneCount: plan.blockedPanes.length, adoptedDispatchIds, exitedDispatchIds, deferredDispatchIds: [...deferredDispatchIds] } + this.updateLegacyWorkerTerminalRecoveryRetry(plan, deferredDispatchIds, options) + return result } setAutomationService(service: AutomationService): void { @@ -4213,6 +4454,9 @@ export class OrcaRuntimeService { notifySshStateChanged(targetId: string, state: SshConnectionState): void { this.bumpSshRelayRecoveryGeneration(targetId) this.invalidateSshWorktreeScanCache(targetId) + if (state.status !== 'connected') { + this.cancelLegacyWorkerTerminalRecoveryRetry(`ssh:${targetId}`) + } this.emitClientEvent({ type: 'sshStateChanged', targetId, state: getPublicSshState(state)! }) } @@ -10345,13 +10589,6 @@ export class OrcaRuntimeService { return 'local' } - getOrchestrationCompatibilityAuthorityInputs(): OrchestrationCompatibilityAuthorityInputs { - return { - hydratedCommitments: this.getHydratedAgentHookAuthorityFn?.() ?? [], - currentObservations: this.getCurrentAgentHookAuthorityFn?.() ?? [] - } - } - registerOrchestrationCompatibilitySshAttachment( targetId: string, connectionIncarnation: string @@ -13554,13 +13791,24 @@ export class OrcaRuntimeService { const resolvedWorkspace = workspace.folderWorkspace ? this.folderWorkspaceToResolvedWorktree(workspace.folderWorkspace) : await this.resolveWorktreeSelector(`id:${workspace.id}`) - const livePtyIds = await this.refreshPtyWorktreeRecordsFromController( + const inventory = await this.refreshPtyWorktreeRecordsWithControllerInventory( [resolvedWorkspace], - workspace.id + workspace.id, + undefined, + workspace.connectionId ?? null ) - if (!livePtyIds) { + if (!inventory) { throw new Error('terminal_liveness_unavailable') } + return this.adoptTerminalOrphansFromInventory(request, workspace, inventory) + } + + private async adoptTerminalOrphansFromInventory( + request: RuntimeTerminalOrphanAdoptionRequest, + workspace: TerminalWorkspaceLaunchScope, + inventory: PtyControllerInventory + ): Promise { + const { livePtyIds, terminalIdentityByPtyId } = inventory const store = this.store const session = this.getWorkspaceSessionForWorktree(workspace.id) if (!store?.setWorkspaceSession || !store.flushOrThrow || !session) { @@ -13593,7 +13841,7 @@ export class OrcaRuntimeService { seenPaneKeys.add(paneKey) const live = this.getLivePtyForHandle(claim.terminal) const pty = live?.pty - const controllerIdentity = this.controllerTerminalIdentityByPtyId.get(claim.ptyId) + const controllerIdentity = terminalIdentityByPtyId.get(claim.ptyId) if ( !pty || pty.ptyId !== claim.ptyId || @@ -13681,10 +13929,14 @@ export class OrcaRuntimeService { ) }) if (isExactPersisted && sessionWorktreeId === workspace.id) { + for (const { claim, pty, paneKey } of validated) { + pty.tabId = claim.tabId + pty.paneKey = paneKey + } return { adopted: false, topologyRevision: currentRevision, - snapshot: await this.listMobileSessionTabs(`id:${workspace.id}`) + snapshot: this.getTerminalOrphanAdoptionSnapshot(workspace.id) } } if (currentRevision !== request.expectedTopologyRevision) { @@ -13999,10 +14251,19 @@ export class OrcaRuntimeService { return { adopted: true, topologyRevision: persisted.terminalTopologyRevisionByRepoId?.[repoId] ?? currentRevision + 1, - snapshot: await this.listMobileSessionTabs(`id:${workspace.id}`) + snapshot: this.getTerminalOrphanAdoptionSnapshot(workspace.id) } } + private getTerminalOrphanAdoptionSnapshot(worktreeId: string): RuntimeMobileSessionTabsResult { + this.hydrateHeadlessMobileSessionTabsFromWorkspaceSession(worktreeId, { + allowAttachedWindow: true, + onlyRuntimeOwnedTerminals: true + }) + this.hydrateHeadlessMobileSessionTabsFromWorkspaceSession(worktreeId) + return this.getMobileSessionTabsForWorktree(worktreeId) + } + private buildTerminalVisualLayouts( terminals: RuntimeTerminalSummary[], worktreesById: Map, @@ -26302,17 +26563,34 @@ export class OrcaRuntimeService { targetWorktreeId: string | null = null, deadline?: number ): Promise | null> { + const inventory = await this.refreshPtyWorktreeRecordsWithControllerInventory( + resolvedWorktrees, + targetWorktreeId, + deadline + ) + return inventory ? new Set(inventory.livePtyIds) : null + } + + private async refreshPtyWorktreeRecordsWithControllerInventory( + resolvedWorktrees: ResolvedWorktree[], + targetWorktreeId: string | null = null, + deadline?: number, + connectionId?: string | null + ): Promise { if (targetWorktreeId === FLOATING_TERMINAL_WORKTREE_ID) { const targetedLiveness = this.refreshFloatingWorkspacePtyLiveness() if (targetedLiveness !== null) { - return targetedLiveness + return { + livePtyIds: targetedLiveness, + terminalIdentityByPtyId: new Map() + } } } if (!this.ptyController?.listProcesses) { return null } const sessionsResult = await withTimeoutResult( - this.ptyController.listProcesses(), + this.ptyController.listProcesses(connectionId), deadline === undefined ? PTY_CONTROLLER_LIST_TIMEOUT_MS : Math.max(1, Math.min(PTY_CONTROLLER_LIST_TIMEOUT_MS, deadline - Date.now())) @@ -26322,10 +26600,7 @@ export class OrcaRuntimeService { return null } const sessions = sessionsResult.value - const controllerIdentityByPtyId = new Map< - string, - { handle: string; incarnationId: string; wslDistro?: string | null } - >() + const controllerIdentityByPtyId = new Map() const ptyIdByControllerHandle = new Map() const ambiguousControllerPtyIds = new Set() for (const session of sessions) { @@ -26356,7 +26631,6 @@ export class OrcaRuntimeService { for (const ptyId of ambiguousControllerPtyIds) { controllerIdentityByPtyId.delete(ptyId) } - this.controllerTerminalIdentityByPtyId = controllerIdentityByPtyId const persistedWorktreeIdByPtyId = indexPersistedPtyWorktreeBindings( this.store?.getWorkspaceSession?.() ) @@ -26432,6 +26706,9 @@ export class OrcaRuntimeService { this.refreshPtyForegroundAgent(session.id) } for (const pty of this.ptysById.values()) { + if (connectionId !== undefined && pty.connectionId !== connectionId) { + continue + } if (!allLivePtyIds.has(pty.ptyId) && !this.leafExistsForPty(pty.ptyId)) { if (this.ptyController.hasPty?.(pty.ptyId) === true) { // Why: an SSH spawn can become addressable before an overlapping relay list includes it. @@ -26451,7 +26728,10 @@ export class OrcaRuntimeService { } } this.pruneDisconnectedPtyRecords() - return targetWorktreeId ? selectedLivePtyIds : allLivePtyIds + return { + livePtyIds: targetWorktreeId ? selectedLivePtyIds : allLivePtyIds, + terminalIdentityByPtyId: controllerIdentityByPtyId + } } private refreshFloatingWorkspacePtyLiveness(): Set | null { diff --git a/src/main/runtime/orchestration/db.ts b/src/main/runtime/orchestration/db.ts index 1eea8e43c95..4d819c36265 100644 --- a/src/main/runtime/orchestration/db.ts +++ b/src/main/runtime/orchestration/db.ts @@ -171,10 +171,10 @@ function legacyMessageMatchesQuestion( message: MessageRow, question: string, options: string[], - recipientHandle: string + recipientHandles: readonly string[] ): boolean { if ( - message.to_handle !== recipientHandle || + !recipientHandles.includes(message.to_handle) || normalizeLegacyQuestionText(message.body) !== normalizeLegacyQuestionText(question) ) { return false @@ -1212,6 +1212,13 @@ export class OrchestrationDb { `The ${params.role} compatibility principal is already committed to different proof.` ) } + if (existing.status === 'revoked') { + throw new OrchestrationError( + 'legacy_read_only', + `The ${params.role} compatibility principal has been revoked. No effects were applied.`, + { effectsApplied: false } + ) + } this.db.exec('COMMIT') return { principal: existing, duplicate: true } } @@ -1263,6 +1270,15 @@ export class OrchestrationDb { .all(runId) as LegacyCompatibilityPrincipalRow[] } + getLegacyCoordinatorPrincipal(runId: string): LegacyCompatibilityPrincipalRow | undefined { + return this.db + .prepare( + `SELECT * FROM legacy_compatibility_principals + WHERE run_id = ? AND role = 'coordinator'` + ) + .get(runId) as LegacyCompatibilityPrincipalRow | undefined + } + resolveLegacyCompatibilityPrincipalByIdentity(params: { runId: string role: LegacyPrincipalRole @@ -1362,6 +1378,10 @@ export class OrchestrationDb { } isLegacyCoordinatorHandle(runId: string, terminalHandle: string): boolean { + const principal = this.getLegacyCoordinatorPrincipal(runId) + if (principal?.terminal_handle === terminalHandle) { + return true + } return Boolean( this.db .prepare( @@ -1403,18 +1423,24 @@ export class OrchestrationDb { if (!principal || principal.role !== 'worker' || !principal.dispatch_id) { throw new OrchestrationError('request_mismatch', 'Legacy worker principal was not found.') } + const runAddress = `run:${principal.run_id}` const rows = this.db .prepare( `SELECT * FROM messages - WHERE run_id = ? AND delivery_contract = 'legacy_direct' - AND from_handle = ? AND to_handle = ? AND type = 'worker_done' + WHERE run_id = ? + AND ( + (delivery_contract = 'legacy_direct' AND to_handle = ?) OR + (delivery_contract = 'current_delivery' AND to_handle = ?) + ) + AND from_handle = ? AND type = 'worker_done' AND subject = ? AND body = ? AND payload IS ? ORDER BY sequence` ) .all( principal.run_id, - principal.terminal_handle, params.recipientHandle, + runAddress, + principal.terminal_handle, params.subject, params.body, params.payload @@ -1444,10 +1470,11 @@ export class OrchestrationDb { this.db .prepare( `SELECT 1 FROM messages - WHERE run_id = ? AND delivery_contract = 'current_delivery' AND read = 0 + WHERE run_id = ? AND to_handle = ? + AND delivery_contract = 'current_delivery' AND read = 0 LIMIT 1` ) - .get(runId) + .get(runId, `run:${runId}`) ) } @@ -1832,7 +1859,14 @@ export class OrchestrationDb { runId: string coordinatorHandle: string coordinatorPaneKey: string - allowLegacyCompatibility?: boolean + takeoverLegacy?: boolean + legacyCoordinatorAuthority?: { + runId: string + principalId: string | null + terminalHandle: string + paneKey: string + consumerGeneration: number + } }): RunRow | undefined { this.db.exec('BEGIN IMMEDIATE') try { @@ -1845,30 +1879,95 @@ export class OrchestrationDb { run.coordinator_pane_key !== null && isEquivalentPaneKey(run.coordinator_pane_key, params.coordinatorPaneKey) const adoption = this.getLegacyAdoption() + const adoptedRun = adoption?.adopted_run_id === params.runId + const legacyAuthority = params.legacyCoordinatorAuthority + const legacyPrincipalId = legacyAuthority?.principalId + const legacyPrincipal = legacyPrincipalId + ? this.getLegacyCompatibilityPrincipal(legacyPrincipalId) + : undefined + const provenLegacyBinding = Boolean( + adoptedRun && + legacyAuthority && + legacyAuthority.principalId !== null && + legacyAuthority.runId === params.runId && + legacyAuthority.consumerGeneration === run.consumer_generation && + legacyPrincipal?.run_id === params.runId && + legacyPrincipal.role === 'coordinator' && + legacyPrincipal.status === 'committed' && + legacyPrincipal.terminal_handle === legacyAuthority.terminalHandle && + isEquivalentPaneKey(legacyPrincipal.pane_key, legacyAuthority.paneKey) && + params.coordinatorHandle === legacyAuthority.terminalHandle && + isEquivalentPaneKey(params.coordinatorPaneKey, legacyAuthority.paneKey) + ) + if (legacyAuthority && !provenLegacyBinding) { + throw new OrchestrationError( + 'legacy_read_only', + 'This retained legacy coordinator no longer has lifecycle authority. No effects were applied.', + { effectsApplied: false } + ) + } const activeLegacyAssignment = - adoption?.adopted_run_id === params.runId && + adoptedRun && Boolean( this.db .prepare( `SELECT 1 FROM dispatch_contexts WHERE run_id = ? AND contract_version = ? AND status IN ('pending', 'dispatched') - UNION ALL - SELECT 1 FROM legacy_compatibility_principals - WHERE run_id = ? AND status = 'committed' LIMIT 1` ) - .get(params.runId, LEGACY_CONTRACT_VERSION, params.runId) + .get(params.runId, LEGACY_CONTRACT_VERSION) ) - if (activeLegacyAssignment && !sameBinding && !params.allowLegacyCompatibility) { + const coordinatorPrincipal = adoptedRun + ? this.getLegacyCoordinatorPrincipal(params.runId) + : undefined + const retainedCoordinatorHandle = + coordinatorPrincipal?.terminal_handle ?? + run.coordinator_handle ?? + this.getUniqueLegacyTaskCreatorHandle(params.runId) + const takeoverAlreadyApplied = Boolean( + params.takeoverLegacy && + sameBinding && + run.coordinator_handle === params.coordinatorHandle && + coordinatorPrincipal?.status !== 'committed' + ) + if (params.takeoverLegacy && !adoptedRun) { + throw new OrchestrationError( + 'invalid_argument', + 'Legacy takeover is only available for the automatically adopted Run.' + ) + } + if ( + activeLegacyAssignment && + !sameBinding && + !provenLegacyBinding && + !params.takeoverLegacy + ) { throw new OrchestrationError( 'consumer_fenced', - 'This adopted Run still has a live legacy principal; only its attested coordinator may rebind it.', - { effectsApplied: false } + 'This adopted Run still has live legacy work. Its attested coordinator may rebind it, or a current coordinator may explicitly use run-use --takeover-legacy.', + { + effectsApplied: false, + recoveryCommand: `orca orchestration run-use --id ${params.runId} --takeover-legacy` + } ) } this.unbindOtherRunsForPane(params.coordinatorPaneKey, params.runId) - if (!sameBinding || run.coordinator_handle !== params.coordinatorHandle) { + if ( + (params.takeoverLegacy && !takeoverAlreadyApplied) || + !sameBinding || + run.coordinator_handle !== params.coordinatorHandle + ) { + if (adoptedRun && (params.takeoverLegacy || !activeLegacyAssignment)) { + if ( + coordinatorPrincipal?.status === 'committed' && + (params.takeoverLegacy || + coordinatorPrincipal.terminal_handle !== params.coordinatorHandle || + !isEquivalentPaneKey(coordinatorPrincipal.pane_key, params.coordinatorPaneKey)) + ) { + this.setLegacyCompatibilityPrincipalStatus(coordinatorPrincipal.id, 'revoked') + } + } this.db .prepare( `UPDATE runs @@ -1879,6 +1978,9 @@ export class OrchestrationDb { ) .run(params.coordinatorHandle, params.coordinatorPaneKey, params.runId) this.fenceOutstandingDelivery(params.runId) + if (params.takeoverLegacy) { + this.promoteLegacyCoordinatorMailForTakeover(params.runId, retainedCoordinatorHandle) + } } this.db.exec('COMMIT') } catch (error) { @@ -1953,6 +2055,51 @@ export class OrchestrationDb { .run(runId) } + private promoteLegacyCoordinatorMailForTakeover( + runId: string, + retainedCoordinatorHandle: string | null + ): void { + if (!retainedCoordinatorHandle) { + return + } + this.db + .prepare( + `UPDATE messages + SET to_handle = ?, delivery_contract = 'current_delivery', + read = 0, delivered_at = NULL + WHERE run_id = ? AND delivery_contract = 'legacy_direct' + AND to_handle = ? + AND EXISTS( + SELECT 1 FROM dispatch_contexts d + WHERE d.run_id = messages.run_id + AND d.contract_version = ? + AND ( + messages.from_handle = d.assignee_handle OR + messages.from_handle = 'dispatch:' || d.id + ) + ) + AND ( + read = 0 OR EXISTS( + SELECT 1 FROM question_threads q + WHERE q.message_id = messages.id AND q.status = 'pending' + ) + )` + ) + .run(`run:${runId}`, runId, retainedCoordinatorHandle, LEGACY_CONTRACT_VERSION) + } + + private getUniqueLegacyTaskCreatorHandle(runId: string): string | null { + const rows = this.db + .prepare( + `SELECT DISTINCT created_by_terminal_handle AS handle + FROM tasks + WHERE run_id = ? AND created_by_terminal_handle IS NOT NULL + LIMIT 2` + ) + .all(runId) as { handle: string }[] + return rows.length === 1 ? (rows[0]?.handle ?? null) : null + } + private requireCurrentConsumer(runId: string, consumerGeneration: number): RunRow { const run = this.getRunRaw(runId) if (!run || run.legacy === 1 || run.consumer_generation !== consumerGeneration) { @@ -2269,12 +2416,20 @@ export class OrchestrationDb { let message = params.message.existingId ? this.getMessageById(params.message.existingId) : undefined + const delivery = this.resolveLegacyWorkerCoordinatorDelivery( + principal.run_id, + params.message.to + ) if (params.message.existingId) { + const matchesOriginalLegacyRoute = + message?.delivery_contract === 'legacy_direct' && message.to_handle === params.message.to + const matchesCurrentRoute = + message?.delivery_contract === delivery.contract && message.to_handle === delivery.to if ( !message || message.run_id !== principal.run_id || - message.delivery_contract !== 'legacy_direct' || - message.to_handle !== params.message.to + message.from_handle !== principal.terminal_handle || + (!matchesOriginalLegacyRoute && !matchesCurrentRoute) ) { throw new OrchestrationError( 'request_mismatch', @@ -2284,7 +2439,7 @@ export class OrchestrationDb { } else { message = this.insertMessage({ from: principal.terminal_handle, - to: params.message.to, + to: delivery.to, subject: params.message.subject, body: params.message.body, type: params.message.type, @@ -2292,7 +2447,7 @@ export class OrchestrationDb { payload: params.message.payload, senderPaneKey: principal.pane_key, runId: principal.run_id, - deliveryContract: 'legacy_direct' + deliveryContract: delivery.contract }) } @@ -2300,12 +2455,24 @@ export class OrchestrationDb { if (params.lifecycle.kind === 'heartbeat') { this.recordHeartbeat(dispatchId, params.lifecycle.at) } else if (params.lifecycle.kind === 'worker_report') { - settlement = this.settleWorkerReportInTransaction({ - taskId: params.lifecycle.taskId, - dispatchId, - outcome: params.lifecycle.outcome, - result: params.lifecycle.result - }) + const persistedOutcome = + params.message.existingId && + dispatch.task_id === params.lifecycle.taskId && + dispatch.status === 'completed' + ? 'succeeded' + : params.message.existingId && + dispatch.task_id === params.lifecycle.taskId && + dispatch.status === 'failed' + ? 'failed' + : undefined + settlement = persistedOutcome + ? { action: 'settled', outcome: persistedOutcome, duplicate: true } + : this.settleWorkerReportInTransaction({ + taskId: params.lifecycle.taskId, + dispatchId, + outcome: params.lifecycle.outcome, + result: params.lifecycle.result + }) if (settlement.action === 'rejected') { throw new OrchestrationError(settlement.code, settlement.reason) } @@ -2380,24 +2547,36 @@ export class OrchestrationDb { ) } + const existingQuestionId = + params.existingQuestionId && + !this.db + .prepare( + `SELECT 1 FROM legacy_operation_receipts + WHERE principal_id = ? AND method = 'orchestration.ask' AND effect_id = ? + LIMIT 1` + ) + .get(principal.id, params.existingQuestionId) + ? params.existingQuestionId + : undefined let question: QuestionRow let message: MessageRow - if (params.existingQuestionId) { - const existingQuestion = this.getQuestion(params.existingQuestionId) - const existingMessage = this.getMessageById(params.existingQuestionId) + const delivery = this.resolveLegacyWorkerCoordinatorDelivery( + principal.run_id, + params.recipientHandle + ) + if (existingQuestionId) { + const existingQuestion = this.getQuestion(existingQuestionId) + const existingMessage = this.getMessageById(existingQuestionId) if ( !existingQuestion || !existingMessage || existingQuestion.run_id !== principal.run_id || existingQuestion.dispatch_id !== dispatchId || existingQuestion.status !== 'pending' || - existingMessage.delivery_contract !== 'legacy_direct' || - !legacyMessageMatchesQuestion( - existingMessage, - params.question, - params.options ?? [], - params.recipientHandle - ) + existingMessage.delivery_contract !== delivery.contract || + !legacyMessageMatchesQuestion(existingMessage, params.question, params.options ?? [], [ + delivery.to + ]) ) { throw new OrchestrationError( 'request_mismatch', @@ -2409,7 +2588,7 @@ export class OrchestrationDb { } else { message = this.insertMessage({ from: principal.terminal_handle, - to: params.recipientHandle, + to: delivery.to, subject: 'Question', body: params.question, type: 'question', @@ -2421,7 +2600,7 @@ export class OrchestrationDb { }), senderPaneKey: principal.pane_key, runId: principal.run_id, - deliveryContract: 'legacy_direct' + deliveryContract: delivery.contract }) this.db .prepare('UPDATE messages SET thread_id = ? WHERE id = ?') @@ -2473,21 +2652,37 @@ export class OrchestrationDb { question: QuestionRow message: MessageRow answerAcknowledged: boolean + claimedByOperation: boolean }[] { const principal = this.requireCommittedLegacyPrincipal(params.principalId, 'worker') + const runAddress = `run:${principal.run_id}` const rows = this.db .prepare( - `SELECT q.*, m.id AS source_message_id + `SELECT q.*, m.id AS source_message_id, + EXISTS( + SELECT 1 FROM legacy_operation_receipts lor + WHERE lor.principal_id = ? AND lor.method = 'orchestration.ask' + AND lor.effect_id = q.message_id + ) AS claimed_by_operation FROM question_threads q INNER JOIN messages m ON m.id = q.message_id WHERE q.run_id = ? AND q.dispatch_id = ? - AND m.delivery_contract = 'legacy_direct' - AND m.to_handle = ? + AND ( + (m.delivery_contract = 'legacy_direct' AND m.to_handle = ?) OR + (m.delivery_contract = 'current_delivery' AND m.to_handle = ?) + ) ORDER BY m.sequence LIMIT 501` ) - .all(principal.run_id, principal.dispatch_id, params.recipientHandle) as (QuestionRow & { + .all( + principal.id, + principal.run_id, + principal.dispatch_id, + params.recipientHandle, + runAddress + ) as (QuestionRow & { source_message_id: string + claimed_by_operation: number })[] if (rows.length > 500) { throw new OrchestrationError( @@ -2500,17 +2695,16 @@ export class OrchestrationDb { const message = this.getMessageById(row.source_message_id) return Boolean( message && - legacyMessageMatchesQuestion( - message, - params.question, - params.options ?? [], - params.recipientHandle - ) + legacyMessageMatchesQuestion(message, params.question, params.options ?? [], [ + params.recipientHandle, + runAddress + ]) ) }) .map((row) => ({ question: exposeQuestionTimestamps(row), message: this.getMessageById(row.message_id) as MessageRow, + claimedByOperation: row.claimed_by_operation === 1, answerAcknowledged: row.answer_message_id ? Boolean( this.db @@ -2525,6 +2719,22 @@ export class OrchestrationDb { })) } + private resolveLegacyWorkerCoordinatorDelivery( + runId: string, + retainedCoordinatorHandle: string + ): { to: string; contract: MessageDeliveryContract } { + const run = this.getRunRaw(runId) + const principal = this.getLegacyCoordinatorPrincipal(runId) + const takenOver = + run?.coordinator_handle !== null && + (principal?.status === 'revoked' || + (run?.coordinator_handle !== undefined && + run.coordinator_handle !== retainedCoordinatorHandle)) + return takenOver + ? { to: `run:${runId}`, contract: 'current_delivery' } + : { to: retainedCoordinatorHandle, contract: 'legacy_direct' } + } + commitLegacyReplyOperation(params: { principalId: string operationKey: string @@ -3658,11 +3868,10 @@ export class OrchestrationDb { wd.agent_terminal_handle FROM dispatch_contexts dc INNER JOIN worker_dispatches wd ON wd.dispatch_id = dc.id - WHERE dc.contract_version = ? - AND wd.state IN ('starting', 'ready', 'start_unknown', 'stopping', 'stop_unknown') + WHERE wd.state IN ('starting', 'ready', 'start_unknown', 'stopping', 'stop_unknown') ORDER BY dc.rowid` ) - .all(LEGACY_CONTRACT_VERSION) as LegacyWorkerTerminalRecoveryRow[] + .all() as LegacyWorkerTerminalRecoveryRow[] } getFederatedDispatch(dispatchId: string): FederatedDispatchRow | undefined { diff --git a/src/main/runtime/orchestration/formatter.test.ts b/src/main/runtime/orchestration/formatter.test.ts index 99d5f2f40e9..b4732f04ac2 100644 --- a/src/main/runtime/orchestration/formatter.test.ts +++ b/src/main/runtime/orchestration/formatter.test.ts @@ -77,6 +77,14 @@ describe('formatMessageBanner', () => { ) }) + it('lets the CLI resolve the live sender for Run and Dispatch addresses', () => { + for (const to_handle of ['run:run_test', 'dispatch:dispatch_test']) { + const banner = formatMessageBanner(makeMessage({ to_handle })) + expect(banner).toContain('[Reply: orca orchestration reply --id msg_test1 --body "..."]') + expect(banner).not.toContain(`--from ${to_handle}`) + } + }) + it('marks legacy messages read-only without reply or acknowledgment affordances', () => { const banner = formatMessageBanner( makeMessage({ id: 'msg_legacy', run_id: 'run_legacy_local' }) diff --git a/src/main/runtime/orchestration/formatter.ts b/src/main/runtime/orchestration/formatter.ts index 6bb13ccfbae..269ba29302f 100644 --- a/src/main/runtime/orchestration/formatter.ts +++ b/src/main/runtime/orchestration/formatter.ts @@ -86,10 +86,11 @@ export function formatMessageBanner( } if (authority === 'current') { - // Why: older shells can lack Orca's terminal identity environment. - lines.push( - `[Reply: orca orchestration reply --id ${msg.id} --from ${msg.to_handle} --body "..."]` - ) + const explicitFrom = + msg.to_handle.startsWith('run:') || msg.to_handle.startsWith('dispatch:') + ? '' + : ` --from ${msg.to_handle}` + lines.push(`[Reply: orca orchestration reply --id ${msg.id}${explicitFrom} --body "..."]`) } lines.push(SEPARATOR) diff --git a/src/main/runtime/orchestration/orchestration-legacy-storage-db.test.ts b/src/main/runtime/orchestration/orchestration-legacy-storage-db.test.ts index d5fe2098fe2..4d4cf2facbe 100644 --- a/src/main/runtime/orchestration/orchestration-legacy-storage-db.test.ts +++ b/src/main/runtime/orchestration/orchestration-legacy-storage-db.test.ts @@ -1,6 +1,4 @@ -import { mkdtempSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' +import { rmSync } from 'node:fs' import { afterEach, describe, expect, it } from 'vitest' import Database from '../../sqlite/sync-database' import { @@ -9,21 +7,10 @@ import { LEGACY_RUN_ID, OrchestrationDb } from './db' - -type CutoverFixture = { - dbPath: string - currentRunId: string - unrelatedRunId: string - currentDispatchId: string - legacyTaskId: string - legacyDispatchId: string - legacyGateId: string - legacyMessageIds: string[] - legacyQuestionId: string - legacyDeliveryId: string - rejectionMessageId: string - lookalikeMessageId: string -} +import { + createLegacyStorageCutoverFixture, + type LegacyStorageCutoverFixture +} from './orchestration-legacy-storage-test-fixture' describe('OrchestrationDb legacy contract storage', () => { let db: OrchestrationDb | undefined @@ -38,129 +25,14 @@ describe('OrchestrationDb legacy contract storage', () => { } }) - function createCutoverFixture(): CutoverFixture { - tempDir = mkdtempSync(join(tmpdir(), 'orca-legacy-storage-')) - const dbPath = join(tempDir, 'orchestration.db') - const first = new OrchestrationDb(dbPath) - const currentRun = first.createRun({ - objective: 'Current work', - coordinatorHandle: 'term_current_coord', - coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111' - }) - const currentTask = first.createTask({ spec: 'current', runId: currentRun.id }) - const unrelatedRun = first.createRun({ - objective: 'Unrelated current work', - coordinatorHandle: 'term_unrelated_coord', - coordinatorPaneKey: 'tab_unrelated:55555555-5555-4555-8555-555555555555' - }) - const currentDispatch = first.createDispatchContext( - currentTask.id, - 'term_current_worker', - 'tab_current:22222222-2222-4222-9222-222222222222', - 'current_launch_hash' - ) - first.insertMessage({ - runId: currentRun.id, - from: 'term_current_worker', - to: `run:${currentRun.id}`, - subject: 'current mail' - }) - - const legacyTask = first.createTask({ - spec: 'legacy', - createdByTerminalHandle: 'term_legacy_coord' - }) - first.createDispatchContext( - legacyTask.id, - 'term_legacy_worker', - 'tab_legacy:33333333-3333-4333-8333-333333333333' - ) - const legacyGate = first.createGate({ - taskId: legacyTask.id, - question: 'Retained gate?' - }) - first.resolveGate(legacyGate.id, 'continue') - const retryDispatch = first.createDispatchContext( - legacyTask.id, - 'term_legacy_worker', - 'tab_legacy:33333333-3333-4333-8333-333333333333' - ) - const legacyMessages = [ - first.insertMessage({ - from: 'term_legacy_coord', - to: 'term_legacy_worker', - subject: 'read worker mail' - }), - first.insertMessage({ - from: 'term_legacy_worker', - to: 'term_legacy_coord', - subject: 'read coordinator mail' - }), - first.insertMessage({ - from: 'term_legacy_coord', - to: 'term_legacy_worker', - subject: 'second worker page' - }) - ] - first.markAsRead(legacyMessages.map((message) => message.id)) - const question = first.createQuestion({ - runId: LEGACY_RUN_ID, - dispatchId: retryDispatch.id, - askerHandle: 'term_legacy_worker', - question: 'Retained question?' - }) - const rejection = first.insertMessage({ - from: 'term_legacy_worker', - to: 'term_legacy_coord', - subject: 'Rejected heartbeat', - type: 'heartbeat', - payload: JSON.stringify({ _orcaLifecycleRejection: { code: 'migration', reason: 'cutover' } }) - }) - const lookalike = first.insertMessage({ - from: 'term_legacy_worker', - to: 'term_legacy_coord', - subject: 'Ordinary legacy mail', - payload: JSON.stringify({ '.orcaLifecycleRejection': 'not an audit marker' }) - }) - first.close() - - const raw = new Database(dbPath) - const legacyDeliveryId = 'delivery_legacy_outstanding' - raw - .prepare( - `INSERT INTO deliveries ( - id, run_id, consumer_generation, message_ids, status - ) VALUES (?, ?, 0, ?, 'outstanding')` - ) - .run(legacyDeliveryId, LEGACY_RUN_ID, JSON.stringify([legacyMessages[0].id])) - raw.exec(` - DROP INDEX IF EXISTS idx_messages_delivery_contract; - DROP TABLE legacy_mail_receipts; - DROP TABLE legacy_operation_receipts; - DROP TABLE legacy_compatibility_principals; - DROP TABLE legacy_adoptions; - `) - raw.pragma('user_version = 18') - raw.close() - - return { - dbPath, - currentRunId: currentRun.id, - unrelatedRunId: unrelatedRun.id, - currentDispatchId: currentDispatch.id, - legacyTaskId: legacyTask.id, - legacyDispatchId: retryDispatch.id, - legacyGateId: legacyGate.id, - legacyMessageIds: legacyMessages.map((message) => message.id), - legacyQuestionId: question.message.id, - legacyDeliveryId, - rejectionMessageId: rejection.id, - lookalikeMessageId: lookalike.id - } + function createCutoverFixture(): LegacyStorageCutoverFixture { + const created = createLegacyStorageCutoverFixture() + tempDir = created.tempDir + return created.fixture } function openAdoptedFixture(): { - fixture: CutoverFixture + fixture: LegacyStorageCutoverFixture adoptedRunId: string workerPrincipalId: string coordinatorPrincipalId: string @@ -411,13 +283,19 @@ describe('OrchestrationDb legacy contract storage', () => { coordinatorHandle: 'term_other', coordinatorPaneKey: 'tab_other:55555555-5555-4555-8555-555555555555' }) - ).toThrow(/only its attested coordinator/) + ).toThrow(/attested coordinator may rebind/) const bound = db!.bindRun({ runId: state.adoptedRunId, coordinatorHandle: 'term_legacy_coord', coordinatorPaneKey: 'tab_coord:44444444-4444-4444-8444-444444444444', - allowLegacyCompatibility: true + legacyCoordinatorAuthority: { + runId: state.adoptedRunId, + principalId: state.coordinatorPrincipalId, + terminalHandle: 'term_legacy_coord', + paneKey: 'tab_coord:44444444-4444-4444-8444-444444444444', + consumerGeneration: 0 + } }) expect(bound).toMatchObject({ coordinator_handle: 'term_legacy_coord' }) expect( @@ -429,6 +307,99 @@ describe('OrchestrationDb legacy contract storage', () => { ).toMatchObject({ coordinator_handle: 'term_legacy_coord' }) }) + it.each([ + ['generation', { consumerGeneration: 1 }], + ['principal', { principalId: 'legacy_principal_wrong' }], + ['handle', { terminalHandle: 'term_wrong' }], + ['pane', { paneKey: 'tab_wrong:99999999-9999-4999-8999-999999999999' }] + ] as const)('rejects stale legacy coordinator %s proof inside Run binding', (_label, patch) => { + const state = openAdoptedFixture() + const before = db!.getRun(state.adoptedRunId) + + expect(() => + db!.bindRun({ + runId: state.adoptedRunId, + coordinatorHandle: 'term_legacy_coord', + coordinatorPaneKey: 'tab_coord:44444444-4444-4444-8444-444444444444', + legacyCoordinatorAuthority: { + runId: state.adoptedRunId, + principalId: state.coordinatorPrincipalId, + terminalHandle: 'term_legacy_coord', + paneKey: 'tab_coord:44444444-4444-4444-8444-444444444444', + consumerGeneration: 0, + ...patch + } + }) + ).toThrow(/no longer has lifecycle authority/) + expect(db!.getRun(state.adoptedRunId)).toEqual(before) + }) + + it('revokes the legacy coordinator when a current coordinator takes over after settlement', () => { + const state = openAdoptedFixture() + db!.settleWorkerReport({ + taskId: state.fixture.legacyTaskId, + dispatchId: state.fixture.legacyDispatchId, + outcome: 'succeeded', + result: 'done' + }) + + expect( + db!.bindRun({ + runId: state.adoptedRunId, + coordinatorHandle: 'term_current_coord', + coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111' + }) + ).toMatchObject({ coordinator_handle: 'term_current_coord' }) + expect(db!.getLegacyCompatibilityPrincipal(state.coordinatorPrincipalId)?.status).toBe( + 'revoked' + ) + expect(() => + db!.commitLegacyCompatibilityPrincipal({ + runId: state.adoptedRunId, + role: 'coordinator', + hostScope: 'local:runtime_1', + terminalHandle: 'term_legacy_coord', + paneKey: 'tab_coord:44444444-4444-4444-8444-444444444444', + launchTokenHash: 'coord_launch_hash', + processIncarnation: 'process_coord' + }) + ).toThrow(/revoked/) + }) + + it('promotes only mail addressed to the replaced legacy coordinator', () => { + const state = openAdoptedFixture() + const coordinatorMail = db!.insertMessage({ + runId: state.adoptedRunId, + deliveryContract: 'legacy_direct', + from: 'term_legacy_worker', + to: 'term_legacy_coord', + subject: 'coordinator outcome' + }) + const workerMail = db!.insertMessage({ + runId: state.adoptedRunId, + deliveryContract: 'legacy_direct', + from: 'term_legacy_worker', + to: 'term_other_worker', + subject: 'worker-only guidance' + }) + + db!.bindRun({ + runId: state.adoptedRunId, + coordinatorHandle: 'term_current_coord', + coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111', + takeoverLegacy: true + }) + + expect(db!.getMessageById(coordinatorMail.id)).toMatchObject({ + to_handle: `run:${state.adoptedRunId}`, + delivery_contract: 'current_delivery' + }) + expect(db!.getMessageById(workerMail.id)).toMatchObject({ + to_handle: 'term_other_worker', + delivery_contract: 'legacy_direct' + }) + }) + it('commits legacy messages, lifecycle effects, and invocation receipts exactly once', () => { const state = openAdoptedFixture() const params = { @@ -508,6 +479,65 @@ describe('OrchestrationDb legacy contract storage', () => { expect(db!.getLegacyCompatibilityPrincipal(state.workerPrincipalId)?.status).toBe('settled') }) + it('reconstructs a read pre-takeover completion through its original legacy route', () => { + const state = openAdoptedFixture() + const taskId = db!.getDispatchContextById(state.fixture.legacyDispatchId)!.task_id + const accepted = db!.insertMessage({ + runId: state.adoptedRunId, + deliveryContract: 'legacy_direct', + from: 'term_legacy_worker', + to: 'term_legacy_coord', + subject: 'Completed', + body: 'accepted before takeover', + type: 'worker_done' + }) + db!.markAsRead([accepted.id]) + db!.settleWorkerReport({ + taskId, + dispatchId: state.fixture.legacyDispatchId, + outcome: 'succeeded', + result: 'accepted before takeover' + }) + db!.bindRun({ + runId: state.adoptedRunId, + coordinatorHandle: 'term_current_coord', + coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111', + takeoverLegacy: true + }) + const beforeIds = db!.getInbox(100).map((message) => message.id) + + const reconstructed = db!.commitLegacyLifecycleOperation({ + principalId: state.workerPrincipalId, + operationKey: 'read_completion_after_takeover', + method: 'orchestration.send', + payloadHash: 'read_completion_after_takeover_payload', + message: { + existingId: accepted.id, + to: 'term_legacy_coord', + subject: 'Completed', + body: 'accepted before takeover', + type: 'worker_done' + }, + lifecycle: { + kind: 'worker_report', + taskId, + outcome: 'succeeded', + result: 'accepted before takeover' + } + }) + + expect(reconstructed).toMatchObject({ + duplicate: false, + message: { + id: accepted.id, + to_handle: 'term_legacy_coord', + delivery_contract: 'legacy_direct' + }, + settlement: { action: 'settled', outcome: 'succeeded', duplicate: true } + }) + expect(db!.getInbox(100).map((message) => message.id)).toEqual(beforeIds) + }) + it('rejects cross-cutover completion reconstruction for another recipient', () => { const state = openAdoptedFixture() const taskId = db!.getDispatchContextById(state.fixture.legacyDispatchId)!.task_id @@ -625,6 +655,38 @@ describe('OrchestrationDb legacy contract storage', () => { }) ).toHaveLength(2) + const inherited = db!.createQuestion({ + runId: state.adoptedRunId, + dispatchId: state.fixture.legacyDispatchId, + askerHandle: 'term_legacy_worker', + question: 'Inherited?' + }) + const sqlite = (db as unknown as { db: Database.Database }).db + sqlite + .prepare( + `UPDATE messages + SET from_handle = 'term_legacy_worker', to_handle = 'term_legacy_coord', + delivery_contract = 'legacy_direct' + WHERE id = ?` + ) + .run(inherited.message.id) + const inheritedAsk = { + ...ask, + operationKey: 'ask_inherited_1', + payloadHash: 'ask_inherited_payload_1', + question: 'Inherited?', + options: [], + existingQuestionId: inherited.message.id + } + const adopted = db!.commitLegacyAskOperation(inheritedAsk) + const distinct = db!.commitLegacyAskOperation({ + ...inheritedAsk, + operationKey: 'ask_inherited_2', + payloadHash: 'ask_inherited_payload_2' + }) + expect(adopted.message.id).toBe(inherited.message.id) + expect(distinct.message.id).not.toBe(inherited.message.id) + const reply = { principalId: state.coordinatorPrincipalId, operationKey: 'reply_invocation_1', diff --git a/src/main/runtime/orchestration/orchestration-legacy-storage-test-fixture.ts b/src/main/runtime/orchestration/orchestration-legacy-storage-test-fixture.ts new file mode 100644 index 00000000000..b43f60eda0c --- /dev/null +++ b/src/main/runtime/orchestration/orchestration-legacy-storage-test-fixture.ts @@ -0,0 +1,147 @@ +import { mkdtempSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import Database from '../../sqlite/sync-database' +import { LEGACY_RUN_ID, OrchestrationDb } from './db' + +export type LegacyStorageCutoverFixture = { + dbPath: string + currentRunId: string + unrelatedRunId: string + currentDispatchId: string + legacyTaskId: string + legacyDispatchId: string + legacyGateId: string + legacyMessageIds: string[] + legacyQuestionId: string + legacyDeliveryId: string + rejectionMessageId: string + lookalikeMessageId: string +} + +export function createLegacyStorageCutoverFixture(): { + fixture: LegacyStorageCutoverFixture + tempDir: string +} { + const tempDir = mkdtempSync(join(tmpdir(), 'orca-legacy-storage-')) + const dbPath = join(tempDir, 'orchestration.db') + const first = new OrchestrationDb(dbPath) + const currentRun = first.createRun({ + objective: 'Current work', + coordinatorHandle: 'term_current_coord', + coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111' + }) + const currentTask = first.createTask({ spec: 'current', runId: currentRun.id }) + const unrelatedRun = first.createRun({ + objective: 'Unrelated current work', + coordinatorHandle: 'term_unrelated_coord', + coordinatorPaneKey: 'tab_unrelated:55555555-5555-4555-8555-555555555555' + }) + const currentDispatch = first.createDispatchContext( + currentTask.id, + 'term_current_worker', + 'tab_current:22222222-2222-4222-9222-222222222222', + 'current_launch_hash' + ) + first.insertMessage({ + runId: currentRun.id, + from: 'term_current_worker', + to: `run:${currentRun.id}`, + subject: 'current mail' + }) + + const legacyTask = first.createTask({ + spec: 'legacy', + createdByTerminalHandle: 'term_legacy_coord' + }) + first.createDispatchContext( + legacyTask.id, + 'term_legacy_worker', + 'tab_legacy:33333333-3333-4333-8333-333333333333' + ) + const legacyGate = first.createGate({ + taskId: legacyTask.id, + question: 'Retained gate?' + }) + first.resolveGate(legacyGate.id, 'continue') + const retryDispatch = first.createDispatchContext( + legacyTask.id, + 'term_legacy_worker', + 'tab_legacy:33333333-3333-4333-8333-333333333333' + ) + const legacyMessages = [ + first.insertMessage({ + from: 'term_legacy_coord', + to: 'term_legacy_worker', + subject: 'read worker mail' + }), + first.insertMessage({ + from: 'term_legacy_worker', + to: 'term_legacy_coord', + subject: 'read coordinator mail' + }), + first.insertMessage({ + from: 'term_legacy_coord', + to: 'term_legacy_worker', + subject: 'second worker page' + }) + ] + first.markAsRead(legacyMessages.map((message) => message.id)) + const question = first.createQuestion({ + runId: LEGACY_RUN_ID, + dispatchId: retryDispatch.id, + askerHandle: 'term_legacy_worker', + question: 'Retained question?' + }) + const rejection = first.insertMessage({ + from: 'term_legacy_worker', + to: 'term_legacy_coord', + subject: 'Rejected heartbeat', + type: 'heartbeat', + payload: JSON.stringify({ _orcaLifecycleRejection: { code: 'migration', reason: 'cutover' } }) + }) + const lookalike = first.insertMessage({ + from: 'term_legacy_worker', + to: 'term_legacy_coord', + subject: 'Ordinary legacy mail', + payload: JSON.stringify({ '.orcaLifecycleRejection': 'not an audit marker' }) + }) + first.close() + + const raw = new Database(dbPath) + const legacyDeliveryId = 'delivery_legacy_outstanding' + raw + .prepare( + `INSERT INTO deliveries ( + id, run_id, consumer_generation, message_ids, status + ) VALUES (?, ?, 0, ?, 'outstanding')` + ) + .run(legacyDeliveryId, LEGACY_RUN_ID, JSON.stringify([legacyMessages[0].id])) + raw.exec(` + DROP INDEX IF EXISTS idx_messages_delivery_contract; + DROP TABLE legacy_mail_receipts; + DROP TABLE legacy_operation_receipts; + DROP TABLE legacy_compatibility_principals; + DROP TABLE legacy_adoptions; + `) + raw.pragma('user_version = 18') + raw.close() + + return { + tempDir, + fixture: { + dbPath, + currentRunId: currentRun.id, + unrelatedRunId: unrelatedRun.id, + currentDispatchId: currentDispatch.id, + legacyTaskId: legacyTask.id, + legacyDispatchId: retryDispatch.id, + legacyGateId: legacyGate.id, + legacyMessageIds: legacyMessages.map((message) => message.id), + legacyQuestionId: question.message.id, + legacyDeliveryId, + rejectionMessageId: rejection.id, + lookalikeMessageId: lookalike.id + } + } +} diff --git a/src/main/runtime/orchestration/orchestration-legacy-worker-terminal-recovery.test.ts b/src/main/runtime/orchestration/orchestration-legacy-worker-terminal-recovery.test.ts index 59f559265ae..d425a52f2cb 100644 --- a/src/main/runtime/orchestration/orchestration-legacy-worker-terminal-recovery.test.ts +++ b/src/main/runtime/orchestration/orchestration-legacy-worker-terminal-recovery.test.ts @@ -29,7 +29,8 @@ describe('legacy worker terminal recovery planning', () => { blockedPanes: [ { worktreeId: 'repo::/workspace', - paneKey: `tab-worker:${LEAF_ID}` + paneKey: `tab-worker:${LEAF_ID}`, + contractVersion: 0 } ], candidates: [ @@ -50,7 +51,8 @@ describe('legacy worker terminal recovery planning', () => { blockedPanes: [ { worktreeId: 'repo::/workspace', - paneKey: `tab-worker:${LEAF_ID}` + paneKey: `tab-worker:${LEAF_ID}`, + contractVersion: 0 } ], candidates: [], diff --git a/src/main/runtime/orchestration/orchestration-legacy-worker-terminal-recovery.ts b/src/main/runtime/orchestration/orchestration-legacy-worker-terminal-recovery.ts index b725b2d5900..270fdbff40e 100644 --- a/src/main/runtime/orchestration/orchestration-legacy-worker-terminal-recovery.ts +++ b/src/main/runtime/orchestration/orchestration-legacy-worker-terminal-recovery.ts @@ -4,6 +4,7 @@ import type { LegacyWorkerTerminalRecoveryRow } from './types' export type LegacyWorkerTerminalRecoveryCandidate = { dispatchId: string + contractVersion: number taskId: string worktreeId: string terminalHandle: string @@ -16,7 +17,7 @@ export type LegacyWorkerTerminalRecoveryCandidate = { } export type LegacyWorkerTerminalRecoveryPlan = { - blockedPanes: { worktreeId: string; paneKey: string }[] + blockedPanes: { worktreeId: string; paneKey: string; contractVersion: number }[] candidates: LegacyWorkerTerminalRecoveryCandidate[] ambiguousDispatchIds: string[] } @@ -48,14 +49,21 @@ function countCandidateKeys( export function planLegacyWorkerTerminalRecovery( rows: readonly LegacyWorkerTerminalRecoveryRow[] ): LegacyWorkerTerminalRecoveryPlan { - const blockedPanes = new Map() + const blockedPanes = new Map< + string, + { worktreeId: string; paneKey: string; contractVersion: number } + >() const parsedCandidates: LegacyWorkerTerminalRecoveryCandidate[] = [] for (const row of rows) { const worktreeId = row.worktree_id?.trim() const paneKey = row.assignee_pane_key?.trim() const pane = paneKey ? parsePaneKey(paneKey) : null if (worktreeId && paneKey && pane) { - blockedPanes.set(`${worktreeId}\0${paneKey}`, { worktreeId, paneKey }) + blockedPanes.set(`${worktreeId}\0${paneKey}`, { + worktreeId, + paneKey, + contractVersion: row.contract_version + }) } const terminalHandle = row.assignee_handle?.trim() const workerHandle = row.agent_terminal_handle?.trim() @@ -74,6 +82,7 @@ export function planLegacyWorkerTerminalRecovery( } parsedCandidates.push({ dispatchId: row.dispatch_id, + contractVersion: row.contract_version, taskId: row.task_id, worktreeId, terminalHandle, diff --git a/src/main/runtime/orchestration/orchestration-worker-dispatch-db.test.ts b/src/main/runtime/orchestration/orchestration-worker-dispatch-db.test.ts index 34e9e456f6a..08812067918 100644 --- a/src/main/runtime/orchestration/orchestration-worker-dispatch-db.test.ts +++ b/src/main/runtime/orchestration/orchestration-worker-dispatch-db.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it } from 'vitest' -import { OrchestrationDb } from './db' +import { CURRENT_CONTRACT_VERSION, OrchestrationDb } from './db' describe('OrchestrationDb worker Dispatch state', () => { let db: OrchestrationDb | undefined @@ -44,6 +44,14 @@ describe('OrchestrationDb worker Dispatch state', () => { status: 'dispatched', assignee_handle: 'term_worker' }) + expect(d.listLegacyWorkerTerminalRecoveryRows()).toEqual([ + expect.objectContaining({ + dispatch_id: started.dispatch.id, + contract_version: CURRENT_CONTRACT_VERSION, + worker_state: 'ready', + agent_terminal_handle: 'term_worker' + }) + ]) }) it('commits worker-start mutation acceptance with the starting Dispatch', () => { diff --git a/src/main/runtime/rpc/core.ts b/src/main/runtime/rpc/core.ts index 70d39474377..002771d1b6f 100644 --- a/src/main/runtime/rpc/core.ts +++ b/src/main/runtime/rpc/core.ts @@ -53,6 +53,14 @@ export type RpcRequest = { orchestrationCompatibilityEvidence?: OrchestrationCompatibilityEvidence } +export type LegacyCoordinatorAuthorityProof = Readonly<{ + runId: string + principalId: string | null + terminalHandle: string + paneKey: string + consumerGeneration: number +}> + export type RpcContext = { runtime: OrcaRuntimeService // Why: lets long-poll handlers release immediately on client disconnect instead of running down timeoutMs. See design doc §3.1. @@ -82,6 +90,8 @@ export type RpcContext = { } // Why: only the compatibility authority router can set this trusted scope; user params cannot bypass Run consumer binding. legacyCoordinatorRunId?: string + legacyCoordinatorAuthority?: LegacyCoordinatorAuthorityProof + revalidateLegacyCoordinator?: () => string // Why: federation pins the authenticated saved-environment caller without exposing its token to handlers or storage. authenticatedCallerFingerprint?: string pairing?: PairingRpcContext diff --git a/src/main/runtime/rpc/dispatcher-feature-interactions.test.ts b/src/main/runtime/rpc/dispatcher-feature-interactions.test.ts index 64dc5ee4f7f..e33da2b2661 100644 --- a/src/main/runtime/rpc/dispatcher-feature-interactions.test.ts +++ b/src/main/runtime/rpc/dispatcher-feature-interactions.test.ts @@ -46,6 +46,10 @@ function makeRuntime(ui: PersistedUIState = getDefaultUIState()): OrcaRuntimeSer updateUIState: vi.fn((updates: Partial) => { currentUI = { ...currentUI, ...updates } return currentUI + }), + getOrchestrationDb: () => ({ + getLegacyAdoption: () => undefined, + resolveLegacyWorkerCandidate: () => undefined }) } as unknown as OrcaRuntimeService } diff --git a/src/main/runtime/rpc/dispatcher-stream-options.ts b/src/main/runtime/rpc/dispatcher-stream-options.ts new file mode 100644 index 00000000000..0e492c7ad9a --- /dev/null +++ b/src/main/runtime/rpc/dispatcher-stream-options.ts @@ -0,0 +1,18 @@ +import type { RuntimeCapability } from '../../../shared/protocol-version' +import type { TerminalStreamFrame } from '../../../shared/terminal-stream-protocol' +import type { PairingRpcContext } from './core' + +export type RpcDispatchStreamingOptions = { + connectionId?: string + signal?: AbortSignal + clientId?: string + pairedDeviceId?: string + clientKind?: 'mobile' | 'runtime' + clientCapabilities?: readonly RuntimeCapability[] + pairing?: PairingRpcContext + sendBinary?: (bytes: Uint8Array) => boolean | void + registerBinaryStreamHandler?: ( + streamId: number, + handler: (frame: TerminalStreamFrame) => void + ) => () => void +} diff --git a/src/main/runtime/rpc/dispatcher.ts b/src/main/runtime/rpc/dispatcher.ts index 6b6fb9bb57f..45b2f8755fe 100644 --- a/src/main/runtime/rpc/dispatcher.ts +++ b/src/main/runtime/rpc/dispatcher.ts @@ -10,13 +10,11 @@ import { isStreamingMethod, type RpcAnyMethod, type RpcEnvelopeMeta, - type PairingRpcContext, type RpcRegistry, type RpcRequest, type RpcResponse } from './core' -import type { TerminalStreamFrame } from '../../../shared/terminal-stream-protocol' import type { FeatureInteractionId } from '../../../shared/feature-interactions' import { computerErrorData, @@ -29,20 +27,18 @@ import { import { ALL_RPC_METHODS } from './methods' import { emulatorProbe, emulatorProbeError } from '../../emulator/emulator-probe' import type { OrcaRuntimeService } from '../orca-runtime' -import type { RuntimeCapability } from '../../../shared/protocol-version' import { - OrchestrationMutationExecutor, authenticatedCallerFingerprint, + getOrchestrationMutationExecutor, + type OrchestrationMutationExecutor, type DurableMutationInvocation } from './orchestration-mutation-executor' import { orchestrationMigrationFence } from './orchestration-contract-fence' -import { getRuntimeFeatureInteractionId } from './runtime-feature-interaction' +import { recordRuntimeFeatureInteraction } from './runtime-feature-interaction' import { OrchestrationLegacyCompatibility } from './orchestration-legacy-compatibility' +import type { RpcDispatchStreamingOptions } from './dispatcher-stream-options' -export type DispatcherOptions = { - runtime: OrcaRuntimeService - methods?: readonly RpcAnyMethod[] -} +export type DispatcherOptions = { runtime: OrcaRuntimeService; methods?: readonly RpcAnyMethod[] } export class RpcDispatcher { private readonly runtime: OrcaRuntimeService @@ -53,7 +49,7 @@ export class RpcDispatcher { constructor({ runtime, methods = ALL_RPC_METHODS }: DispatcherOptions) { this.runtime = runtime this.registry = buildRegistry(methods) - this.orchestrationMutations = new OrchestrationMutationExecutor(runtime) + this.orchestrationMutations = getOrchestrationMutationExecutor(runtime) this.legacyOrchestration = new OrchestrationLegacyCompatibility(runtime) } @@ -91,8 +87,7 @@ export class RpcDispatcher { ) } - const isEmulator = request.method.startsWith('emulator.') - if (isEmulator) { + if (request.method.startsWith('emulator.')) { emulatorProbe(`rpc ${request.method}`, request.params) } try { @@ -105,8 +100,13 @@ export class RpcDispatcher { return successResponse(request.id, meta, compatibility.result) } const effectiveParams = compatibility.params ?? parsedParams.value - const invoke = (mutation?: DurableMutationInvocation) => - method.handler(effectiveParams, { + const legacyCoordinator = this.legacyOrchestration.createCoordinatorInvocation( + request, + compatibility.legacyCoordinatorAuthority + ) + const invoke = (mutation?: DurableMutationInvocation) => { + const legacyCoordinatorRunId = legacyCoordinator?.revalidate() + return method.handler(effectiveParams, { runtime: this.runtime, signal: options?.signal, requestId: request.id, @@ -114,13 +114,22 @@ export class RpcDispatcher { authenticatedCallerFingerprint: authenticatedCallerFingerprint(request), recordMutationReceipt: mutation?.recordReceipt, orchestrationMutation: mutation?.identity, - legacyCoordinatorRunId: compatibility.legacyCoordinatorRunId + legacyCoordinatorRunId, + legacyCoordinatorAuthority: legacyCoordinator?.authority, + revalidateLegacyCoordinator: legacyCoordinator?.revalidate }) + } const result = await this.orchestrationMutations.run(request, effectiveParams, invoke) - this.recordRuntimeFeatureInteraction(request.method, result, undefined, request.params) + recordRuntimeFeatureInteraction( + this.runtime, + request.method, + result, + undefined, + request.params + ) return successResponse(request.id, meta, result) } catch (error) { - if (isEmulator) { + if (request.method.startsWith('emulator.')) { emulatorProbeError(`rpc ${request.method}`, error, { params: request.params }) } return this.mapError(request, meta, error) @@ -133,20 +142,7 @@ export class RpcDispatcher { async dispatchStreaming( request: RpcRequest, reply: (response: string) => void, - options?: { - connectionId?: string - signal?: AbortSignal - clientId?: string - pairedDeviceId?: string - clientKind?: 'mobile' | 'runtime' - clientCapabilities?: readonly RuntimeCapability[] - pairing?: PairingRpcContext - sendBinary?: (bytes: Uint8Array) => boolean | void - registerBinaryStreamHandler?: ( - streamId: number, - handler: (frame: TerminalStreamFrame) => void - ) => () => void - } + options?: RpcDispatchStreamingOptions ): Promise { const meta = this.meta() const method = this.registry.get(request.method) @@ -183,8 +179,13 @@ export class RpcDispatcher { return } const effectiveParams = compatibility.params ?? parsedParams.value - const invoke = (mutation?: DurableMutationInvocation) => - method.handler(effectiveParams, { + const legacyCoordinator = this.legacyOrchestration.createCoordinatorInvocation( + request, + compatibility.legacyCoordinatorAuthority + ) + const invoke = (mutation?: DurableMutationInvocation) => { + const legacyCoordinatorRunId = legacyCoordinator?.revalidate() + return method.handler(effectiveParams, { runtime: this.runtime, signal: options?.signal, requestId: request.id, @@ -200,10 +201,19 @@ export class RpcDispatcher { pairing: options?.pairing, sendBinary: options?.sendBinary, registerBinaryStreamHandler: options?.registerBinaryStreamHandler, - legacyCoordinatorRunId: compatibility.legacyCoordinatorRunId + legacyCoordinatorRunId, + legacyCoordinatorAuthority: legacyCoordinator?.authority, + revalidateLegacyCoordinator: legacyCoordinator?.revalidate }) + } const result = await this.orchestrationMutations.run(request, effectiveParams, invoke) - this.recordRuntimeFeatureInteraction(request.method, result, undefined, request.params) + recordRuntimeFeatureInteraction( + this.runtime, + request.method, + result, + undefined, + request.params + ) reply(JSON.stringify(successResponse(request.id, meta, result))) } catch (error) { reply(JSON.stringify(this.mapError(request, meta, error))) @@ -213,7 +223,8 @@ export class RpcDispatcher { const recordedStreamingFeatureInteractions = new Set() const emit = (result: unknown): void => { - this.recordRuntimeFeatureInteraction( + recordRuntimeFeatureInteraction( + this.runtime, request.method, result, recordedStreamingFeatureInteractions, @@ -243,7 +254,8 @@ export class RpcDispatcher { }, emit ) - this.recordRuntimeFeatureInteraction( + recordRuntimeFeatureInteraction( + this.runtime, request.method, result, recordedStreamingFeatureInteractions, @@ -310,25 +322,4 @@ export class RpcDispatcher { private meta(): RpcEnvelopeMeta { return { runtimeId: this.runtime.getRuntimeId() } } - - private recordRuntimeFeatureInteraction( - method: string, - result: unknown, - alreadyRecorded?: Set, - rawParams?: unknown - ): void { - const id = getRuntimeFeatureInteractionId(method, result, rawParams) - if (!id) { - return - } - if (alreadyRecorded?.has(id)) { - return - } - try { - this.runtime.recordFeatureInteraction(id) - alreadyRecorded?.add(id) - } catch { - // Best-effort education state must not break runtime tools. - } - } } diff --git a/src/main/runtime/rpc/methods/orchestration-runs.ts b/src/main/runtime/rpc/methods/orchestration-runs.ts index 3c56a456d2e..924fd6297d5 100644 --- a/src/main/runtime/rpc/methods/orchestration-runs.ts +++ b/src/main/runtime/rpc/methods/orchestration-runs.ts @@ -1,6 +1,6 @@ import { z } from 'zod' import { defineMethod, type RpcMethod } from '../core' -import { OptionalString, requiredString } from '../schemas' +import { OptionalBoolean, OptionalString, requiredString } from '../schemas' import type { OrcaRuntimeService } from '../../orca-runtime' import { OrchestrationError } from '../../orchestration/orchestration-error' @@ -11,7 +11,8 @@ const RunCreateParams = z.object({ const RunUseParams = z.object({ id: requiredString('Missing --id'), - from: requiredString('Missing coordinator terminal') + from: requiredString('Missing coordinator terminal'), + takeoverLegacy: OptionalBoolean }) const RunCurrentParams = z.object({ from: requiredString('Missing coordinator terminal') }) @@ -51,7 +52,7 @@ export const ORCHESTRATION_RUN_METHODS: RpcMethod[] = [ defineMethod({ name: 'orchestration.runUse', params: RunUseParams, - handler: (params, { runtime, legacyCoordinatorRunId }) => { + handler: (params, { runtime, legacyCoordinatorAuthority }) => { const paneKey = requireCallerPane(runtime, params.from) const db = runtime.getOrchestrationDb() const priorRun = db.getCurrentRunForPane(paneKey) @@ -59,7 +60,8 @@ export const ORCHESTRATION_RUN_METHODS: RpcMethod[] = [ runId: params.id, coordinatorHandle: params.from, coordinatorPaneKey: paneKey, - allowLegacyCompatibility: legacyCoordinatorRunId === params.id + takeoverLegacy: params.takeoverLegacy, + legacyCoordinatorAuthority }) if (!run) { throw new OrchestrationError( diff --git a/src/main/runtime/rpc/methods/orchestration.ts b/src/main/runtime/rpc/methods/orchestration.ts index 9954d4c9883..13770d61b25 100644 --- a/src/main/runtime/rpc/methods/orchestration.ts +++ b/src/main/runtime/rpc/methods/orchestration.ts @@ -2,7 +2,12 @@ import { z } from 'zod' import { defineMethod, type RpcMethod } from '../core' import { OptionalFiniteNumber, OptionalString, OptionalBoolean, requiredString } from '../schemas' -import type { MessageType, MessagePriority, TaskStatus } from '../../orchestration/db' +import { + LEGACY_CONTRACT_VERSION, + type MessageType, + type MessagePriority, + type TaskStatus +} from '../../orchestration/db' import { MESSAGE_TYPES } from '../../orchestration/types' import { buildDispatchPreamble } from '../../orchestration/preamble' import { formatMessageBanner } from '../../orchestration/formatter' @@ -379,6 +384,49 @@ function resolveMessageRun( return { run, dispatchId: dispatch?.id ?? dispatchId } } +function legacyWorkerDeliveryContract( + runtime: OrcaRuntimeService, + runId: string | undefined, + recipient: string +): 'legacy_direct' | undefined { + if (!runId) { + return undefined + } + if (!recipient.startsWith('dispatch:')) { + return runtime + .getOrchestrationDb() + .resolveLegacyWorkerCandidate({ runId, terminalHandle: recipient }) + ? 'legacy_direct' + : undefined + } + const dispatch = runtime + .getOrchestrationDb() + .getDispatchContextById(recipient.slice('dispatch:'.length)) + return dispatch?.run_id === runId && + dispatch.contract_version === LEGACY_CONTRACT_VERSION && + (dispatch.status === 'pending' || dispatch.status === 'dispatched') + ? 'legacy_direct' + : undefined +} + +function interruptedAcknowledgedCheck( + runId: string, + acknowledged: string, + reason: 'consumer_fenced' | 'outcome_unknown' | 'waiter_exists' +): Record { + return { + runId, + deliveryId: null, + messages: [], + count: 0, + acknowledged, + timedOut: false, + cancelled: false, + connectionLost: false, + waitInterrupted: reason + } +} + function rejectFederatedExplicitTarget(params: { to?: string; run?: string }): void { if (params.to || params.run) { throw new OrchestrationError( @@ -395,7 +443,10 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ defineMethod({ name: 'orchestration.send', params: SendParams, - handler: async (params, { runtime, orchestrationCapability }) => { + handler: async ( + params, + { runtime, orchestrationCapability, legacyCoordinatorRunId, revalidateLegacyCoordinator } + ) => { const db = runtime.getOrchestrationDb() const from = params.from ?? 'unknown' // Why: caller-supplied pane fields are only compatibility metadata; lifecycle authority uses the runtime-observed pane plus capability. @@ -536,6 +587,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ 'Coordinator-to-worker control mail cannot report worker lifecycle.' ) } + revalidateLegacyCoordinator?.() const relay = db.enqueueFederationRelay({ dispatchId, direction: 'to_worker', @@ -562,6 +614,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ } } // Point-to-point — existing single-recipient behavior + revalidateLegacyCoordinator?.() const msg = db.insertMessage({ from, to, @@ -572,7 +625,12 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ threadId: params.threadId, payload: params.payload, senderPaneKey, - runId: routing.run?.id + runId: routing.run?.id, + deliveryContract: legacyWorkerDeliveryContract( + runtime, + routing.run?.id ?? legacyCoordinatorRunId, + to + ) }) const dispatch = routing.dispatchId ? db.getDispatchContextById(routing.dispatchId) @@ -629,6 +687,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ throw new Error(`No recipients resolved for group address: ${to}`) } + revalidateLegacyCoordinator?.() const threadId = params.threadId ?? `thread_${Date.now()}` const messages = handles.map((handle) => db.insertMessage({ @@ -641,7 +700,12 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ threadId, payload: params.payload, senderPaneKey, - runId: routing.run?.id + runId: routing.run?.id, + deliveryContract: legacyWorkerDeliveryContract( + runtime, + routing.run?.id ?? legacyCoordinatorRunId, + handle + ) }) ) for (const message of messages) { @@ -655,7 +719,16 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ defineMethod({ name: 'orchestration.check', params: CheckParams, - handler: async (params, { runtime, signal, legacyCoordinatorRunId }) => { + handler: async ( + params, + { + runtime, + signal, + legacyCoordinatorRunId, + revalidateLegacyCoordinator, + recordMutationReceipt + } + ) => { const db = runtime.getOrchestrationDb() const handle = params.terminal ?? 'unknown' const typeFilter = parseMessageTypes(params.types) @@ -682,6 +755,11 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ deliveryId: params.ack }) : undefined + if (acknowledged) { + recordMutationReceipt?.( + interruptedAcknowledgedCheck(run.id, acknowledged.delivery.id, 'outcome_unknown') + ) + } if (params.peek || params.all || params.unread === false) { const history = db.getRunMailboxHistory(run.id, 100, typeFilter) const messages = @@ -745,14 +823,28 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ signal, exclusive: true }) + try { + revalidateLegacyCoordinator?.() + } catch (error) { + if (!acknowledged) { + throw error + } + return interruptedAcknowledgedCheck(run.id, acknowledged.delivery.id, 'consumer_fenced') + } const latestRun = db.getRun(run.id) if (!latestRun || latestRun.consumer_generation !== generation) { + if (acknowledged) { + return interruptedAcknowledgedCheck(run.id, acknowledged.delivery.id, 'consumer_fenced') + } throw new OrchestrationError( 'consumer_fenced', 'This mailbox consumer was replaced while waiting.' ) } if (waitResult === 'waiter_exists') { + if (acknowledged) { + return interruptedAcknowledgedCheck(run.id, acknowledged.delivery.id, 'waiter_exists') + } throw new OrchestrationError( 'waiter_exists', `Run ${run.id} already has an active actionable waiter.` @@ -1119,7 +1211,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ defineMethod({ name: 'orchestration.dispatch', params: DispatchParams, - handler: async (params, { runtime, legacyCoordinatorRunId }) => { + handler: async (params, { runtime, legacyCoordinatorRunId, revalidateLegacyCoordinator }) => { const db = runtime.getOrchestrationDb() const task = db.getTask(params.task) if (!task) { @@ -1184,6 +1276,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ ) } + revalidateLegacyCoordinator?.() const ctx = db.createDispatchContext(params.task, to, assigneePaneKey) const dispatchCapability = params.inject ? db.mintDispatchCapability({ diff --git a/src/main/runtime/rpc/orchestration-legacy-authority.ts b/src/main/runtime/rpc/orchestration-legacy-authority.ts index 0654c277e9d..71e9f83acfa 100644 --- a/src/main/runtime/rpc/orchestration-legacy-authority.ts +++ b/src/main/runtime/rpc/orchestration-legacy-authority.ts @@ -1,4 +1,3 @@ -import { parsePaneKey } from '../../../shared/stable-pane-id' import type { OrchestrationCompatibilityEvidence } from '../../../shared/orchestration-compatibility-evidence' import type { DispatchContextRow, @@ -6,9 +5,14 @@ import type { LegacyPrincipalRole } from '../orchestration/types' import { OrchestrationError } from '../orchestration/orchestration-error' -import type { OrcaRuntimeService } from '../orca-runtime' +import type { OrcaRuntimeService, OrchestrationCompatibilityCallerAuthority } from '../orca-runtime' import { LEGACY_CONTRACT_VERSION } from '../orchestration/db' import type { RpcRequest } from './core' +import { + equivalentLegacyPaneKey, + legacyCoordinatorReadOnly, + legacyReadOnlyError +} from './orchestration-legacy-process-identity' export type LegacyPrincipalCandidate = { runId: string @@ -22,8 +26,9 @@ export function resolveAttestedLegacyPrincipal(args: { runtime: OrcaRuntimeService evidence?: OrchestrationCompatibilityEvidence candidate: LegacyPrincipalCandidate + authority?: OrchestrationCompatibilityCallerAuthority }): LegacyCompatibilityPrincipalRow { - const authority = verifyAttestedLegacyCandidate(args) + const authority = args.authority ?? verifyAttestedLegacyCandidate(args) return args.runtime.getOrchestrationDb().commitLegacyCompatibilityPrincipal({ runId: args.candidate.runId, dispatchId: args.candidate.dispatchId, @@ -36,7 +41,7 @@ export function resolveAttestedLegacyPrincipal(args: { }).principal } -function verifyAttestedLegacyCandidate(args: { +export function verifyAttestedLegacyCandidate(args: { runtime: OrcaRuntimeService evidence?: OrchestrationCompatibilityEvidence candidate: LegacyPrincipalCandidate @@ -44,7 +49,7 @@ function verifyAttestedLegacyCandidate(args: { const authority = args.runtime.verifyOrchestrationCompatibilityCaller(args.evidence) if ( !authority || - !equivalentPaneKey(args.candidate.paneKey, authority.paneKey) || + !equivalentLegacyPaneKey(args.candidate.paneKey, authority.paneKey) || args.candidate.terminalHandle !== authority.terminalHandle ) { throw legacyReadOnlyError() @@ -221,37 +226,6 @@ export class LegacyCompatibilityAuthority { }) return principal } - - resolveProvenCoordinatorScope(request: RpcRequest, requestedRunId?: string): string | undefined { - const db = this.runtime.getOrchestrationDb() - const adoption = db.getLegacyAdoption() - const requestedRun = requestedRunId ?? adoption?.adopted_run_id - if (!adoption || requestedRun !== adoption.adopted_run_id) { - return undefined - } - const candidate = db.resolveLegacyCoordinatorCandidate({ - runId: adoption.adopted_run_id, - terminalHandle: request.orchestrationCompatibilityEvidence?.terminalHandle, - paneKey: request.orchestrationCompatibilityEvidence?.paneKey - }) - if (!candidate) { - if (request.orchestrationCompatibilityEvidence) { - throw legacyCoordinatorReadOnly() - } - return undefined - } - verifyAttestedLegacyCandidate({ - runtime: this.runtime, - evidence: request.orchestrationCompatibilityEvidence, - candidate: { - runId: adoption.adopted_run_id, - role: 'coordinator', - terminalHandle: candidate.terminalHandle, - paneKey: candidate.paneKey - } - }) - return adoption.adopted_run_id - } } function candidateFromDispatch(dispatch: DispatchContextRow): LegacyPrincipalCandidate { @@ -281,31 +255,3 @@ function candidateFromPrincipal( paneKey: principal.pane_key } } - -function equivalentPaneKey(a: string | null | undefined, b: string | null | undefined): boolean { - if (!a || !b) { - return false - } - if (a === b) { - return true - } - const aLeaf = parsePaneKey(a)?.leafId - const bLeaf = parsePaneKey(b)?.leafId - return Boolean(aLeaf && bLeaf && aLeaf === bLeaf) -} - -function legacyReadOnlyError(): OrchestrationError { - return new OrchestrationError( - 'legacy_read_only', - 'This retained legacy assignment could not prove authority from its original live process. No effects were applied.', - { effectsApplied: false } - ) -} - -function legacyCoordinatorReadOnly(): OrchestrationError { - return new OrchestrationError( - 'legacy_read_only', - 'This retained legacy coordinator could not prove its original process identity. No effects were applied.', - { effectsApplied: false } - ) -} diff --git a/src/main/runtime/rpc/orchestration-legacy-compatibility-dispatcher.test.ts b/src/main/runtime/rpc/orchestration-legacy-compatibility-dispatcher.test.ts index 0a4f30e6010..0d2bdda36a0 100644 --- a/src/main/runtime/rpc/orchestration-legacy-compatibility-dispatcher.test.ts +++ b/src/main/runtime/rpc/orchestration-legacy-compatibility-dispatcher.test.ts @@ -300,6 +300,82 @@ describe('legacy compatibility through RpcDispatcher', () => { expect(harness.notify).toHaveBeenCalledOnce() }) + it('replays an A-era completion without touching a newer current attempt', async () => { + const harness = createHarness() + const payload = JSON.stringify({ + taskId: harness.taskId, + dispatchId: harness.dispatchId + }) + const completion = harness.db.insertMessage({ + runId: harness.adoptedRunId, + from: WORKER_HANDLE, + to: COORDINATOR_HANDLE, + subject: 'Failed: pinned A treated this as complete', + body: 'persisted result', + type: 'worker_done', + payload, + senderPaneKey: WORKER_PANE, + deliveryContract: 'legacy_direct' + }) + harness.db.settleWorkerReport({ + taskId: harness.taskId, + dispatchId: harness.dispatchId, + outcome: 'succeeded', + result: 'persisted result' + }) + harness.db.commitLegacyCompatibilityPrincipal({ + runId: harness.adoptedRunId, + dispatchId: harness.dispatchId, + role: 'worker', + hostScope: JSON.stringify({ kind: 'local', hostId: 'local' }), + terminalHandle: WORKER_HANDLE, + paneKey: WORKER_PANE, + launchTokenHash: createHash('sha256').update('worker-token').digest('hex'), + processIncarnation: 'process-1' + }) + harness.db.updateTaskStatus(harness.taskId, 'ready') + const currentDispatch = harness.db.createDispatchContext( + harness.taskId, + 'term_current_worker', + 'tab_current_worker:77777777-7777-4777-8777-777777777777', + 'current-launch-hash' + ) + const currentTaskBefore = harness.db.getTask(harness.taskId) + const currentDispatchBefore = harness.db.getDispatchContextById(currentDispatch.id) + const before = counts(harness.db) + + const response = await harness.dispatcher.dispatch( + request( + 'orchestration.send', + { + from: WORKER_HANDLE, + to: COORDINATOR_HANDLE, + subject: completion.subject, + body: completion.body, + type: 'worker_done', + payload + }, + evidence('worker'), + 'reconstruct-pinned-a-completion' + ) + ) + + expect(response).toMatchObject({ + ok: true, + result: { + message: { id: completion.id }, + lifecycle: { action: 'settled', outcome: 'succeeded', duplicate: true } + } + }) + expect(harness.db.getTask(harness.taskId)).toEqual(currentTaskBefore) + expect(harness.db.getDispatchContextById(harness.dispatchId)?.status).toBe('completed') + expect(harness.db.getDispatchContextById(currentDispatch.id)).toEqual(currentDispatchBefore) + expect(counts(harness.db)).toEqual({ + ...before, + legacy_operation_receipts: before.legacy_operation_receipts + 1 + }) + }) + it('rejects a legacy lifecycle recipient outside the adopted Run with zero effects', async () => { const harness = createHarness() const before = counts(harness.db) @@ -367,6 +443,35 @@ describe('legacy compatibility through RpcDispatcher', () => { expect(harness.notify).not.toHaveBeenCalled() }) + it('keeps distinct compatibility ask invocations on distinct questions', async () => { + const harness = createHarness() + const ask = { + from: WORKER_HANDLE, + to: COORDINATOR_HANDLE, + question: 'Same question?', + options: ['yes', 'no'], + timeoutMs: 1 + } + const first = await harness.dispatcher.dispatch( + request('orchestration.ask', ask, evidence('worker'), 'ask-one') + ) + const replay = await harness.dispatcher.dispatch( + request('orchestration.ask', ask, evidence('worker'), 'ask-one') + ) + const second = await harness.dispatcher.dispatch( + request('orchestration.ask', ask, evidence('worker'), 'ask-two') + ) + + expect(first).toMatchObject({ ok: true, result: { legacyCompatibility: { replayed: false } } }) + expect(replay).toMatchObject({ ok: true, result: { legacyCompatibility: { replayed: true } } }) + expect(second).toMatchObject({ ok: true, result: { legacyCompatibility: { replayed: false } } }) + const firstId = (first as { result: { messageId: string } }).result.messageId + const replayId = (replay as { result: { messageId: string } }).result.messageId + const secondId = (second as { result: { messageId: string } }).result.messageId + expect(replayId).toBe(firstId) + expect(secondId).not.toBe(firstId) + }) + it('does not infer an outcome for a current Dispatch when legacy adoption exists', async () => { const harness = createHarness() const run = harness.db.createRun({ @@ -524,6 +629,56 @@ describe('legacy compatibility through RpcDispatcher', () => { expect(harness.notify).not.toHaveBeenCalled() }) + it('advertises only Run-addressed current delivery to a legacy coordinator', async () => { + const harness = createHarness() + harness.db.insertMessage({ + runId: harness.adoptedRunId, + from: 'term_current_worker', + to: WORKER_HANDLE, + subject: 'worker-only current mail', + deliveryContract: 'current_delivery' + }) + const workerOnly = await harness.dispatcher.dispatch( + request( + 'orchestration.check', + { terminal: COORDINATOR_HANDLE }, + evidence('coordinator'), + 'worker-current-mail' + ) + ) + expect(workerOnly).toMatchObject({ + ok: true, + result: { legacyCompatibility: { currentDelivery: undefined } } + }) + + harness.db.insertMessage({ + runId: harness.adoptedRunId, + from: 'term_current_worker', + to: `run:${harness.adoptedRunId}`, + subject: 'Run current mail', + deliveryContract: 'current_delivery' + }) + const runMail = await harness.dispatcher.dispatch( + request( + 'orchestration.check', + { terminal: COORDINATOR_HANDLE }, + evidence('coordinator'), + 'run-current-mail' + ) + ) + expect(runMail).toMatchObject({ + ok: true, + result: { + legacyCompatibility: { + currentDelivery: { + runId: harness.adoptedRunId, + checkCommand: expect.stringContaining(`--run ${harness.adoptedRunId}`) + } + } + } + }) + }) + it.each(['dispatch', 'websocket'] as const)( '%s binds an attested coordinator once and rejects contradictory proof before binding', async (transport) => { @@ -588,4 +743,42 @@ describe('legacy compatibility through RpcDispatcher', () => { expect(harness.db.getRun(harness.adoptedRunId)?.consumer_generation).toBe(0) expect(counts(harness.db)).toEqual(before) }) + + it('rejects stale legacy coordinator proof after current takeover', async () => { + const harness = createHarness() + const principal = harness.db.commitLegacyCompatibilityPrincipal({ + runId: harness.adoptedRunId, + role: 'coordinator', + hostScope: JSON.stringify({ kind: 'local', hostId: 'local' }), + terminalHandle: COORDINATOR_HANDLE, + paneKey: COORDINATOR_PANE, + launchTokenHash: createHash('sha256').update('coordinator-token').digest('hex'), + processIncarnation: 'process-1' + }).principal + harness.db.settleWorkerReport({ + taskId: harness.taskId, + dispatchId: harness.dispatchId, + outcome: 'succeeded', + result: 'done' + }) + harness.db.bindRun({ + runId: harness.adoptedRunId, + coordinatorHandle: 'term_current_coord', + coordinatorPaneKey: 'tab_current:55555555-5555-4555-8555-555555555555' + }) + expect(harness.db.getLegacyCompatibilityPrincipal(principal.id)?.status).toBe('revoked') + const before = counts(harness.db) + + const response = await harness.dispatcher.dispatch( + request( + 'orchestration.taskList', + {}, + evidence('coordinator'), + 'stale-coordinator-after-takeover' + ) + ) + + expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } }) + expect(counts(harness.db)).toEqual(before) + }) }) diff --git a/src/main/runtime/rpc/orchestration-legacy-compatibility.ts b/src/main/runtime/rpc/orchestration-legacy-compatibility.ts index 093e006767a..043bf7567ab 100644 --- a/src/main/runtime/rpc/orchestration-legacy-compatibility.ts +++ b/src/main/runtime/rpc/orchestration-legacy-compatibility.ts @@ -1,4 +1,4 @@ -import type { RpcRequest } from './core' +import type { LegacyCoordinatorAuthorityProof, RpcRequest } from './core' import type { OrcaRuntimeService } from '../orca-runtime' import { LegacyCompatibilityAuthority } from './orchestration-legacy-authority' import { handleLegacyLifecycleSend } from './orchestration-legacy-lifecycle' @@ -10,6 +10,7 @@ import type { LegacyReplyParams, LegacySendParams } from './orchestration-legacy-operation' +import { LegacyCoordinatorAuthority } from './orchestration-legacy-coordinator-authority' const COORDINATOR_PREFLIGHT_METHODS = new Set([ 'orchestration.taskCreate', @@ -19,19 +20,31 @@ const COORDINATOR_PREFLIGHT_METHODS = new Set([ 'orchestration.gateCreate', 'orchestration.gateResolve', 'orchestration.runUse', + 'orchestration.send', 'orchestration.check', 'orchestration.reply' ]) export type LegacyCompatibilityRoute = | { handled: true; result: unknown } - | { handled: false; params?: unknown; legacyCoordinatorRunId?: string } + | { + handled: false + params?: unknown + legacyCoordinatorAuthority?: LegacyCoordinatorAuthorityProof + } + +export type LegacyCoordinatorInvocation = Readonly<{ + authority: LegacyCoordinatorAuthorityProof + revalidate: () => string +}> export class OrchestrationLegacyCompatibility { private readonly authority: LegacyCompatibilityAuthority + private readonly coordinatorAuthority: LegacyCoordinatorAuthority constructor(private readonly runtime: OrcaRuntimeService) { this.authority = new LegacyCompatibilityAuthority(runtime) + this.coordinatorAuthority = new LegacyCoordinatorAuthority(runtime) } async tryHandle( @@ -50,18 +63,40 @@ export class OrchestrationLegacyCompatibility { return { handled: false } } const values = params as Record + if (request.method === 'orchestration.runUse' && values.takeoverLegacy === true) { + return { handled: false } + } const requestedRunId = request.method === 'orchestration.runUse' ? stringValue(values.id) : stringValue(values.run) - const runId = this.authority.resolveProvenCoordinatorScope(request, requestedRunId) - return runId + const authority = this.coordinatorAuthority.resolve(request, requestedRunId) + return authority ? { handled: false, - params: { ...values, run: runId }, - legacyCoordinatorRunId: runId + params: { ...values, run: authority.runId }, + legacyCoordinatorAuthority: authority } : { handled: false } } + revalidateCoordinatorAuthority( + request: RpcRequest, + proof: LegacyCoordinatorAuthorityProof + ): string { + return this.coordinatorAuthority.revalidate(request, proof) + } + + createCoordinatorInvocation( + request: RpcRequest, + authority?: LegacyCoordinatorAuthorityProof + ): LegacyCoordinatorInvocation | undefined { + return authority + ? { + authority, + revalidate: () => this.revalidateCoordinatorAuthority(request, authority) + } + : undefined + } + private async route( request: RpcRequest, params: unknown, diff --git a/src/main/runtime/rpc/orchestration-legacy-coordinator-authority.ts b/src/main/runtime/rpc/orchestration-legacy-coordinator-authority.ts new file mode 100644 index 00000000000..66b46e804bd --- /dev/null +++ b/src/main/runtime/rpc/orchestration-legacy-coordinator-authority.ts @@ -0,0 +1,158 @@ +import type { OrcaRuntimeService, OrchestrationCompatibilityCallerAuthority } from '../orca-runtime' +import type { LegacyCompatibilityPrincipalRow } from '../orchestration/types' +import type { LegacyCoordinatorAuthorityProof, RpcRequest } from './core' +import { + resolveAttestedLegacyPrincipal, + verifyAttestedLegacyCandidate, + type LegacyPrincipalCandidate +} from './orchestration-legacy-authority' +import { + equivalentLegacyPaneKey, + legacyCoordinatorReadOnly +} from './orchestration-legacy-process-identity' + +export class LegacyCoordinatorAuthority { + constructor(private readonly runtime: OrcaRuntimeService) {} + + resolve( + request: RpcRequest, + requestedRunId?: string + ): LegacyCoordinatorAuthorityProof | undefined { + const db = this.runtime.getOrchestrationDb() + const adoption = db.getLegacyAdoption() + const requestedRun = requestedRunId ?? adoption?.adopted_run_id + if (!adoption || requestedRun !== adoption.adopted_run_id) { + return undefined + } + const candidate = db.resolveLegacyCoordinatorCandidate({ + runId: adoption.adopted_run_id, + terminalHandle: request.orchestrationCompatibilityEvidence?.terminalHandle, + paneKey: request.orchestrationCompatibilityEvidence?.paneKey + }) + if (!candidate) { + if (request.orchestrationCompatibilityEvidence) { + throw legacyCoordinatorReadOnly() + } + return undefined + } + const existing = db.getLegacyCoordinatorPrincipal(adoption.adopted_run_id) + const proofCandidate = existing + ? this.candidate(adoption.adopted_run_id, existing) + : this.candidate(adoption.adopted_run_id, candidate) + const attestation = verifyAttestedLegacyCandidate({ + runtime: this.runtime, + evidence: request.orchestrationCompatibilityEvidence, + candidate: proofCandidate + }) + const run = db.getRun(adoption.adopted_run_id) + if (!run || !this.bindingMatches(run, proofCandidate)) { + throw legacyCoordinatorReadOnly() + } + if (existing && !this.principalMatchesAttestation(existing, attestation)) { + throw legacyCoordinatorReadOnly() + } + const principal = existing + ? existing + : request.method === 'orchestration.runUse' + ? resolveAttestedLegacyPrincipal({ + runtime: this.runtime, + evidence: request.orchestrationCompatibilityEvidence, + candidate: proofCandidate, + authority: attestation + }) + : undefined + return { + runId: run.id, + principalId: principal?.id ?? null, + terminalHandle: proofCandidate.terminalHandle, + paneKey: proofCandidate.paneKey, + consumerGeneration: run.consumer_generation + } + } + + revalidate(request: RpcRequest, proof: LegacyCoordinatorAuthorityProof): string { + const db = this.runtime.getOrchestrationDb() + const adoption = db.getLegacyAdoption() + const run = db.getRun(proof.runId) + if ( + adoption?.adopted_run_id !== proof.runId || + !run || + run.consumer_generation !== proof.consumerGeneration + ) { + throw legacyCoordinatorReadOnly() + } + const candidate = this.candidate(proof.runId, proof) + const principal = db.getLegacyCoordinatorPrincipal(proof.runId) + if (proof.principalId) { + if ( + principal?.id !== proof.principalId || + principal.status !== 'committed' || + principal.terminal_handle !== proof.terminalHandle || + !equivalentLegacyPaneKey(principal.pane_key, proof.paneKey) + ) { + throw legacyCoordinatorReadOnly() + } + } else if ( + principal && + (principal.status !== 'committed' || + principal.terminal_handle !== proof.terminalHandle || + !equivalentLegacyPaneKey(principal.pane_key, proof.paneKey)) + ) { + throw legacyCoordinatorReadOnly() + } + const attestation = verifyAttestedLegacyCandidate({ + runtime: this.runtime, + evidence: request.orchestrationCompatibilityEvidence, + candidate + }) + if (principal && !this.principalMatchesAttestation(principal, attestation)) { + throw legacyCoordinatorReadOnly() + } + if (!this.bindingMatches(run, candidate)) { + throw legacyCoordinatorReadOnly() + } + return run.id + } + + private candidate( + runId: string, + identity: { + terminalHandle?: string + terminal_handle?: string + paneKey?: string + pane_key?: string + } + ): LegacyPrincipalCandidate { + return { + runId, + role: 'coordinator', + terminalHandle: identity.terminalHandle ?? (identity.terminal_handle as string), + paneKey: identity.paneKey ?? (identity.pane_key as string) + } + } + + private bindingMatches( + run: { coordinator_handle: string | null; coordinator_pane_key: string | null }, + candidate: LegacyPrincipalCandidate + ): boolean { + return ( + !run.coordinator_pane_key || + (run.coordinator_handle === candidate.terminalHandle && + equivalentLegacyPaneKey(run.coordinator_pane_key, candidate.paneKey)) + ) + } + + private principalMatchesAttestation( + principal: LegacyCompatibilityPrincipalRow, + attestation: OrchestrationCompatibilityCallerAuthority + ): boolean { + return ( + principal.status === 'committed' && + principal.host_scope === JSON.stringify(attestation.hostScope) && + principal.terminal_handle === attestation.terminalHandle && + equivalentLegacyPaneKey(principal.pane_key, attestation.paneKey) && + principal.launch_token_hash === attestation.launchTokenHash && + principal.process_incarnation === attestation.processIncarnation + ) + } +} diff --git a/src/main/runtime/rpc/orchestration-legacy-coordinator-race.test.ts b/src/main/runtime/rpc/orchestration-legacy-coordinator-race.test.ts new file mode 100644 index 00000000000..9ff54a60783 --- /dev/null +++ b/src/main/runtime/rpc/orchestration-legacy-coordinator-race.test.ts @@ -0,0 +1,849 @@ +import { createHash } from 'node:crypto' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { ORCHESTRATION_CONTRACT_VERSION } from '../../../shared/protocol-version' +import type { OrchestrationCompatibilityEvidence } from '../../../shared/orchestration-compatibility-evidence' +import Database from '../../sqlite/sync-database' +import { OrcaRuntimeService } from '../orca-runtime' +import { OrchestrationDb } from '../orchestration/db' +import type { RpcRequest, RpcResponse } from './core' +import { RpcDispatcher } from './dispatcher' +import { ORCHESTRATION_METHODS } from './methods/orchestration' + +const COORDINATOR_HANDLE = 'term_legacy_coord' +const COORDINATOR_PANE = 'tab_coord:44444444-4444-4444-8444-444444444444' +const WORKER_HANDLE = 'term_legacy_worker' +const WORKER_PANE = 'tab_worker:33333333-3333-4333-8333-333333333333' + +type Transport = 'dispatch' | 'websocket' + +type Harness = { + db: OrchestrationDb + dispatcher: RpcDispatcher + runtime: OrcaRuntimeService + adoptedRunId: string + taskId: string + dispatchId: string +} + +const tempDirs: string[] = [] +const databases: OrchestrationDb[] = [] + +afterEach(() => { + databases.splice(0).forEach((database) => database.close()) + tempDirs.splice(0).forEach((dir) => rmSync(dir, { recursive: true, force: true })) +}) + +function createHarness(): Harness { + const dir = mkdtempSync(join(tmpdir(), 'orca-legacy-coordinator-race-')) + tempDirs.push(dir) + const dbPath = join(dir, 'orchestration.db') + const before = new OrchestrationDb(dbPath) + const task = before.createTask({ + spec: 'legacy assignment', + createdByTerminalHandle: COORDINATOR_HANDLE + }) + const dispatch = before.createDispatchContext(task.id, WORKER_HANDLE, WORKER_PANE) + before.close() + + const raw = new Database(dbPath) + raw.exec(` + UPDATE dispatch_contexts SET process_incarnation = 'process-1'; + DROP INDEX IF EXISTS idx_messages_delivery_contract; + DROP TABLE legacy_mail_receipts; + DROP TABLE legacy_operation_receipts; + DROP TABLE legacy_compatibility_principals; + DROP TABLE legacy_adoptions; + `) + raw.pragma('user_version = 18') + raw.close() + + const db = new OrchestrationDb(dbPath) + databases.push(db) + const adoptedRunId = db.getLegacyAdoption()?.adopted_run_id as string + const runtime = new OrcaRuntimeService() + runtime.setOrchestrationDb(db) + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === COORDINATOR_HANDLE ? COORDINATOR_PANE : handle === WORKER_HANDLE ? WORKER_PANE : null + ) + vi.spyOn(runtime, 'verifyOrchestrationCompatibilityCaller').mockImplementation( + (compatibilityEvidence) => { + const coordinator = + compatibilityEvidence?.terminalHandle === COORDINATOR_HANDLE && + compatibilityEvidence.paneKey === COORDINATOR_PANE && + compatibilityEvidence.launchToken === 'coordinator-token' + const worker = + compatibilityEvidence?.terminalHandle === WORKER_HANDLE && + compatibilityEvidence.paneKey === WORKER_PANE && + compatibilityEvidence.launchToken === 'worker-token' + return coordinator || worker + ? { + hostScope: { kind: 'local', hostId: 'local' }, + terminalHandle: coordinator ? COORDINATOR_HANDLE : WORKER_HANDLE, + paneKey: coordinator ? COORDINATOR_PANE : WORKER_PANE, + processIncarnation: 'process-1', + launchTokenHash: createHash('sha256') + .update(coordinator ? 'coordinator-token' : 'worker-token') + .digest('hex') + } + : null + } + ) + return { + db, + dispatcher: new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS }), + runtime, + adoptedRunId, + taskId: task.id, + dispatchId: dispatch.id + } +} + +function evidence( + role: 'coordinator' | 'worker' = 'coordinator' +): OrchestrationCompatibilityEvidence { + return { + terminalHandle: role === 'coordinator' ? COORDINATOR_HANDLE : WORKER_HANDLE, + paneKey: role === 'coordinator' ? COORDINATOR_PANE : WORKER_PANE, + launchToken: role === 'coordinator' ? 'coordinator-token' : 'worker-token' + } +} + +function request( + method: string, + params: unknown, + invocationId: string, + callerEvidence = evidence() +): RpcRequest { + return { + id: `rpc_${invocationId}`, + authToken: 'caller-token', + method, + params, + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationRequestId: invocationId, + compatibilityInvocationId: invocationId, + orchestrationCompatibilityEvidence: callerEvidence + } +} + +async function invoke( + dispatcher: RpcDispatcher, + rpcRequest: RpcRequest, + transport: Transport +): Promise { + if (transport === 'dispatch') { + return await dispatcher.dispatch(rpcRequest) + } + const replies: string[] = [] + await dispatcher.dispatchStreaming(rpcRequest, (reply) => replies.push(reply)) + expect(replies).toHaveLength(1) + return JSON.parse(replies[0]) as RpcResponse +} + +function installTakeoverDuringMutationPreflight(harness: Harness): void { + const originalBegin = harness.db.beginMutationReceipt.bind(harness.db) + vi.spyOn(harness.db, 'beginMutationReceipt').mockImplementation((identity) => { + const begun = originalBegin(identity) + settleLegacyWorkerAndTakeOver(harness, 'settled during takeover') + return begun + }) +} + +function settleLegacyWorkerAndTakeOver(harness: Harness, result: string): void { + harness.db.settleWorkerReport({ + taskId: harness.taskId, + dispatchId: harness.dispatchId, + outcome: 'succeeded', + result + }) + harness.db.bindRun({ + runId: harness.adoptedRunId, + coordinatorHandle: 'term_current_coord', + coordinatorPaneKey: 'tab_current:55555555-5555-4555-8555-555555555555' + }) +} + +function mutationReceiptCount(db: OrchestrationDb): number { + const sqlite = (db as unknown as { db: Database.Database }).db + return ( + sqlite.prepare('SELECT COUNT(*) AS count FROM mutation_receipts').get() as { count: number } + ).count +} + +function messageCount(db: OrchestrationDb): number { + const sqlite = (db as unknown as { db: Database.Database }).db + return (sqlite.prepare('SELECT COUNT(*) AS count FROM messages').get() as { count: number }).count +} + +describe('legacy coordinator takeover races', () => { + it.each(['dispatch', 'websocket'] as const)( + '%s routes a pre-bind coordinator send to the retained worker mailbox', + async (transport) => { + const harness = createHarness() + + const response = await invoke( + harness.dispatcher, + request( + 'orchestration.send', + { from: COORDINATOR_HANDLE, to: WORKER_HANDLE, subject: 'continue safely' }, + `send-direct-${transport}` + ), + transport + ) + + expect(response).toMatchObject({ + ok: true, + result: { + message: { + run_id: harness.adoptedRunId, + to_handle: WORKER_HANDLE, + delivery_contract: 'legacy_direct' + } + } + }) + const workerCheck = await harness.dispatcher.dispatch( + request( + 'orchestration.check', + { terminal: WORKER_HANDLE }, + `worker-check-${transport}`, + evidence('worker') + ) + ) + expect(workerCheck).toMatchObject({ + ok: true, + result: { + messages: [ + { + subject: 'continue safely', + run_id: harness.adoptedRunId, + delivery_contract: 'legacy_direct' + } + ], + legacyCompatibility: { ackMessageIds: [expect.any(String)] } + } + }) + expect( + harness.db.getInbox(100).some((message) => message.run_id === 'run_legacy_local') + ).toBe(false) + } + ) + + it('routes an exact legacy Dispatch recipient to compatibility delivery', async () => { + const harness = createHarness() + + const response = await harness.dispatcher.dispatch( + request( + 'orchestration.send', + { + from: COORDINATOR_HANDLE, + to: `dispatch:${harness.dispatchId}`, + subject: 'dispatch guidance' + }, + 'send-dispatch' + ) + ) + + expect(response).toMatchObject({ + ok: true, + result: { + message: { + run_id: harness.adoptedRunId, + to_handle: `dispatch:${harness.dispatchId}`, + delivery_contract: 'legacy_direct' + } + } + }) + const workerCheck = await harness.dispatcher.dispatch( + request( + 'orchestration.check', + { terminal: WORKER_HANDLE }, + 'worker-check-dispatch', + evidence('worker') + ) + ) + expect(workerCheck).toMatchObject({ + ok: true, + result: { messages: [{ subject: 'dispatch guidance' }] } + }) + }) + + it('partitions a coordinator group send by legacy recipient contract', async () => { + const harness = createHarness() + vi.spyOn(harness.runtime, 'listTerminals').mockResolvedValue({ + terminals: [ + { handle: COORDINATOR_HANDLE }, + { handle: WORKER_HANDLE }, + { handle: 'term_current_worker' } + ], + totalCount: 3, + truncated: false + } as never) + + const response = await harness.dispatcher.dispatch( + request( + 'orchestration.send', + { from: COORDINATOR_HANDLE, to: '@all', subject: 'group guidance' }, + 'send-group' + ) + ) + + expect(response).toMatchObject({ ok: true, result: { recipients: 2 } }) + const messages = harness.db + .getInbox(100) + .filter((message) => message.subject === 'group guidance') + expect(messages).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + run_id: harness.adoptedRunId, + to_handle: WORKER_HANDLE, + delivery_contract: 'legacy_direct' + }), + expect.objectContaining({ + run_id: harness.adoptedRunId, + to_handle: 'term_current_worker', + delivery_contract: 'current_delivery' + }) + ]) + ) + }) + + it.each(['dispatch', 'websocket'] as const)( + '%s rejects a coordinator send after takeover during durable preflight', + async (transport) => { + const harness = createHarness() + installTakeoverDuringMutationPreflight(harness) + const before = messageCount(harness.db) + + const response = await invoke( + harness.dispatcher, + request( + 'orchestration.send', + { from: COORDINATOR_HANDLE, to: WORKER_HANDLE, subject: 'must not send' }, + `send-takeover-${transport}` + ), + transport + ) + + expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } }) + expect(messageCount(harness.db)).toBe(before) + expect(mutationReceiptCount(harness.db)).toBe(0) + } + ) + + it('revalidates a group send after asynchronous terminal discovery', async () => { + const harness = createHarness() + let resolveTerminals: + | ((value: { + terminals: { handle: string }[] + totalCount: number + truncated: boolean + }) => void) + | undefined + let signalListingStarted: (() => void) | undefined + const listingStarted = new Promise((resolve) => { + signalListingStarted = resolve + }) + vi.spyOn(harness.runtime, 'listTerminals').mockImplementation( + () => + new Promise((resolve) => { + resolveTerminals = resolve as typeof resolveTerminals + signalListingStarted?.() + }) as never + ) + const before = messageCount(harness.db) + const pending = harness.dispatcher.dispatch( + request( + 'orchestration.send', + { from: COORDINATOR_HANDLE, to: '@all', subject: 'must remain unsent' }, + 'send-group-takeover' + ) + ) + await listingStarted + settleLegacyWorkerAndTakeOver(harness, 'settled during terminal discovery') + resolveTerminals?.({ + terminals: [{ handle: COORDINATOR_HANDLE }, { handle: WORKER_HANDLE }], + totalCount: 2, + truncated: false + }) + + await expect(pending).resolves.toMatchObject({ + ok: false, + error: { code: 'legacy_read_only' } + }) + expect(messageCount(harness.db)).toBe(before) + expect(mutationReceiptCount(harness.db)).toBe(0) + }) + + it('reports a committed ACK when coordinator takeover interrupts its wait', async () => { + const harness = createHarness() + const incoming = harness.db.insertMessage({ + runId: harness.adoptedRunId, + from: WORKER_HANDLE, + to: `run:${harness.adoptedRunId}`, + subject: 'ready to acknowledge' + }) + const first = await harness.dispatcher.dispatch( + request( + 'orchestration.check', + { terminal: COORDINATOR_HANDLE, run: harness.adoptedRunId }, + 'check-delivery-before-takeover' + ) + ) + expect(first).toMatchObject({ + ok: true, + result: { deliveryId: expect.any(String), messages: [{ id: incoming.id }] } + }) + const deliveryId = (first as { result: { deliveryId: string } }).result.deliveryId + let resolveWait: (() => void) | undefined + let signalWaitStarted: (() => void) | undefined + const waitStarted = new Promise((resolve) => { + signalWaitStarted = resolve + }) + vi.spyOn(harness.runtime, 'waitForMessage').mockImplementation( + () => + new Promise((resolve) => { + resolveWait = () => resolve('notified') + signalWaitStarted?.() + }) + ) + const pending = harness.dispatcher.dispatch( + request( + 'orchestration.check', + { + terminal: COORDINATOR_HANDLE, + run: harness.adoptedRunId, + ack: deliveryId, + wait: true, + timeoutMs: 10_000 + }, + 'check-ack-takeover' + ) + ) + await waitStarted + expect(harness.db.getMessageById(incoming.id)?.read).toBe(1) + settleLegacyWorkerAndTakeOver(harness, 'settled during acknowledged wait') + resolveWait?.() + + await expect(pending).resolves.toMatchObject({ + ok: true, + result: { + acknowledged: deliveryId, + count: 0, + waitInterrupted: 'consumer_fenced', + mutation: { requestId: 'check-ack-takeover', replayed: false } + } + }) + const sqlite = (harness.db as unknown as { db: Database.Database }).db + expect(sqlite.prepare('SELECT state FROM mutation_receipts').get()).toEqual({ + state: 'completed' + }) + }) + + it('keeps concurrent duplicates on the live ACK-and-wait result', async () => { + const harness = createHarness() + harness.db.insertMessage({ + runId: harness.adoptedRunId, + from: WORKER_HANDLE, + to: `run:${harness.adoptedRunId}`, + subject: 'first delivery' + }) + const firstCheck = await harness.dispatcher.dispatch( + request( + 'orchestration.check', + { terminal: COORDINATOR_HANDLE, run: harness.adoptedRunId }, + 'check-first-delivery' + ) + ) + const deliveryId = (firstCheck as { result: { deliveryId: string } }).result.deliveryId + let resolveWait: (() => void) | undefined + let signalWaitStarted: (() => void) | undefined + const waitStarted = new Promise((resolve) => { + signalWaitStarted = resolve + }) + vi.spyOn(harness.runtime, 'waitForMessage').mockImplementation( + () => + new Promise((resolve) => { + resolveWait = () => resolve('notified') + signalWaitStarted?.() + }) + ) + const ackRequest = request( + 'orchestration.check', + { + terminal: COORDINATOR_HANDLE, + run: harness.adoptedRunId, + ack: deliveryId, + wait: true, + timeoutMs: 10_000 + }, + 'check-ack-concurrent' + ) + const original = harness.dispatcher.dispatch(ackRequest) + await waitStarted + const secondLiveDispatcher = new RpcDispatcher({ + runtime: harness.runtime, + methods: ORCHESTRATION_METHODS + }) + const duplicate = secondLiveDispatcher.dispatch({ + ...ackRequest, + id: 'rpc_check-ack-concurrent-duplicate' + }) + let duplicateSettled = false + void duplicate.then(() => { + duplicateSettled = true + }) + await Promise.resolve() + expect(duplicateSettled).toBe(false) + + const arrived = harness.db.insertMessage({ + runId: harness.adoptedRunId, + from: WORKER_HANDLE, + to: `run:${harness.adoptedRunId}`, + subject: 'arrived while waiting' + }) + resolveWait?.() + const [originalResult, duplicateResult] = await Promise.all([original, duplicate]) + expect(originalResult).toMatchObject({ + ok: true, + result: { + messages: [{ id: arrived.id }], + mutation: { requestId: 'check-ack-concurrent', replayed: false } + } + }) + expect(duplicateResult).toMatchObject({ + ok: true, + result: { + messages: [{ id: arrived.id }], + mutation: { requestId: 'check-ack-concurrent', replayed: true } + } + }) + }) + + it.each(['dispatch', 'websocket'] as const)( + '%s rejects run-use after takeover during durable preflight', + async (transport) => { + const harness = createHarness() + installTakeoverDuringMutationPreflight(harness) + + const response = await invoke( + harness.dispatcher, + request( + 'orchestration.runUse', + { id: harness.adoptedRunId, from: COORDINATOR_HANDLE }, + `run-use-takeover-${transport}` + ), + transport + ) + + expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } }) + expect(harness.db.getRun(harness.adoptedRunId)).toMatchObject({ + coordinator_handle: 'term_current_coord', + consumer_generation: 1 + }) + expect(mutationReceiptCount(harness.db)).toBe(0) + expect(harness.db.getLegacyCoordinatorPrincipal(harness.adoptedRunId)?.status).toBe('revoked') + } + ) + + it.each(['dispatch', 'websocket'] as const)( + '%s rejects a mutation after takeover during durable preflight', + async (transport) => { + const harness = createHarness() + const target = harness.db.createTask({ + spec: 'must remain ready', + runId: harness.adoptedRunId + }) + installTakeoverDuringMutationPreflight(harness) + + const response = await invoke( + harness.dispatcher, + request( + 'orchestration.taskUpdate', + { + id: target.id, + status: 'completed', + callerTerminalHandle: COORDINATOR_HANDLE + }, + `task-update-takeover-${transport}` + ), + transport + ) + + expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } }) + expect(harness.db.getTask(target.id)?.status).toBe('ready') + expect(mutationReceiptCount(harness.db)).toBe(0) + } + ) + + it('revalidates after asynchronous agent detection', async () => { + const harness = createHarness() + const target = harness.db.createTask({ + spec: 'must not dispatch after takeover', + runId: harness.adoptedRunId + }) + const targetHandle = 'term_current_worker' + const targetPane = 'tab_current_worker:77777777-7777-4777-8777-777777777777' + vi.spyOn(harness.runtime, 'getTerminalPaneKey').mockImplementation((handle: string) => + handle === COORDINATOR_HANDLE ? COORDINATOR_PANE : handle === targetHandle ? targetPane : null + ) + vi.spyOn(harness.runtime, 'getTerminalProcessIncarnation').mockReturnValue('current-process') + let resolveDetection: ((detected: boolean) => void) | undefined + let signalDetectionStarted: (() => void) | undefined + const detectionStarted = new Promise((resolve) => { + signalDetectionStarted = resolve + }) + vi.spyOn(harness.runtime, 'isTerminalRunningAgent').mockImplementation( + () => + new Promise((resolve) => { + resolveDetection = resolve + signalDetectionStarted?.() + }) + ) + const sendPrompt = vi + .spyOn(harness.runtime, 'sendTerminalAgentPrompt') + .mockResolvedValue({ handle: targetHandle, accepted: true, bytesWritten: 1 }) + + const pending = harness.dispatcher.dispatch( + request( + 'orchestration.dispatch', + { task: target.id, to: targetHandle, from: COORDINATOR_HANDLE, inject: true }, + 'dispatch-detection-takeover' + ) + ) + await detectionStarted + settleLegacyWorkerAndTakeOver(harness, 'settled during detection') + resolveDetection?.(true) + const response = await pending + + expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } }) + expect(harness.db.getTask(target.id)?.status).toBe('ready') + expect(harness.db.getDispatchContext(target.id)).toBeUndefined() + expect(sendPrompt).not.toHaveBeenCalled() + }) + + it('explicitly takes over an active adopted Run without stranding its legacy worker', async () => { + const harness = createHarness() + const currentHandle = 'term_current_coord' + const currentPane = 'tab_current:55555555-5555-4555-8555-555555555555' + vi.mocked(harness.runtime.getTerminalPaneKey).mockImplementation((handle) => + handle === COORDINATOR_HANDLE + ? COORDINATOR_PANE + : handle === WORKER_HANDLE + ? WORKER_PANE + : handle === currentHandle + ? currentPane + : null + ) + + await expect( + harness.dispatcher.dispatch( + request( + 'orchestration.runUse', + { id: harness.adoptedRunId, from: COORDINATOR_HANDLE }, + 'bind-original-coordinator' + ) + ) + ).resolves.toMatchObject({ ok: true }) + const pendingAsk = await harness.dispatcher.dispatch( + request( + 'orchestration.ask', + { + from: WORKER_HANDLE, + to: COORDINATOR_HANDLE, + question: 'Continue with the migration?', + timeoutMs: 0 + }, + 'pending-before-takeover', + evidence('worker') + ) + ) + const questionId = (pendingAsk as { result: { messageId: string } }).result.messageId + const unrelated = harness.db.insertMessage({ + runId: harness.adoptedRunId, + from: WORKER_HANDLE, + to: 'term_peer_worker', + subject: 'peer-only', + deliveryContract: 'legacy_direct' + }) + + const currentRequest = (method: string, params: unknown, invocationId: string): RpcRequest => ({ + ...request(method, params, invocationId), + orchestrationCompatibilityEvidence: undefined + }) + const takeoverParams = { + id: harness.adoptedRunId, + from: currentHandle, + takeoverLegacy: true + } + const takeover = await harness.dispatcher.dispatch( + currentRequest('orchestration.runUse', takeoverParams, 'explicit-takeover') + ) + const repeated = await harness.dispatcher.dispatch( + currentRequest('orchestration.runUse', takeoverParams, 'explicit-takeover-repeat') + ) + + expect(takeover).toMatchObject({ + ok: true, + result: { binding: { consumerGeneration: 2 } } + }) + expect(repeated).toMatchObject({ + ok: true, + result: { binding: { consumerGeneration: 2 } } + }) + expect(harness.db.getDispatchContextById(harness.dispatchId)?.status).toBe('dispatched') + expect(harness.db.getLegacyCoordinatorPrincipal(harness.adoptedRunId)?.status).toBe('revoked') + expect(harness.db.getMessageById(unrelated.id)).toMatchObject({ + to_handle: 'term_peer_worker', + delivery_contract: 'legacy_direct' + }) + + const promoted = await harness.dispatcher.dispatch( + currentRequest( + 'orchestration.check', + { terminal: currentHandle, run: harness.adoptedRunId, format: true }, + 'read-promoted-question' + ) + ) + expect(promoted).toMatchObject({ + ok: true, + result: { + messages: [ + { + id: questionId, + to_handle: `run:${harness.adoptedRunId}`, + delivery_contract: 'current_delivery' + } + ], + formatted: expect.not.stringContaining(`--from run:${harness.adoptedRunId}`) + } + }) + const deliveryId = (promoted as { result: { deliveryId: string } }).result.deliveryId + await expect( + harness.dispatcher.dispatch( + currentRequest( + 'orchestration.reply', + { + id: questionId, + body: 'Yes, continue.', + from: currentHandle, + run: harness.adoptedRunId + }, + 'reply-after-takeover' + ) + ) + ).resolves.toMatchObject({ ok: true, result: { question: { status: 'answered' } } }) + await expect( + harness.dispatcher.dispatch( + request( + 'orchestration.ask', + { from: WORKER_HANDLE, resume: questionId, timeoutMs: 100 }, + 'resume-after-takeover', + evidence('worker') + ) + ) + ).resolves.toMatchObject({ ok: true, result: { answer: 'Yes, continue.' } }) + + const followUp = await harness.dispatcher.dispatch( + currentRequest( + 'orchestration.send', + { + from: currentHandle, + to: `dispatch:${harness.dispatchId}`, + subject: 'Continue from the new coordinator' + }, + 'current-coordinator-to-legacy-worker' + ) + ) + expect(followUp).toMatchObject({ + ok: true, + result: { + message: { + run_id: harness.adoptedRunId, + to_handle: `dispatch:${harness.dispatchId}`, + delivery_contract: 'legacy_direct' + } + } + }) + const workerFollowUp = await harness.dispatcher.dispatch( + request( + 'orchestration.check', + { terminal: WORKER_HANDLE }, + 'legacy-worker-check-after-takeover', + evidence('worker') + ) + ) + expect(workerFollowUp).toMatchObject({ + ok: true, + result: { + messages: [{ subject: 'Continue from the new coordinator' }], + legacyCompatibility: { ackMessageIds: [expect.any(String)] } + } + }) + const [followUpMessageId] = ( + workerFollowUp as { result: { legacyCompatibility: { ackMessageIds: string[] } } } + ).result.legacyCompatibility.ackMessageIds + await expect( + harness.dispatcher.dispatch( + request( + 'orchestration.check', + { + terminal: WORKER_HANDLE, + compatibilityAck: JSON.stringify({ messageIds: [followUpMessageId] }) + }, + 'legacy-worker-ack-after-takeover', + evidence('worker') + ) + ) + ).resolves.toMatchObject({ + ok: true, + result: { + acknowledged: [followUpMessageId], + legacyCompatibility: { acknowledged: true } + } + }) + + await harness.dispatcher.dispatch( + currentRequest( + 'orchestration.check', + { terminal: currentHandle, run: harness.adoptedRunId, ack: deliveryId }, + 'ack-promoted-question' + ) + ) + await expect( + harness.dispatcher.dispatch( + request( + 'orchestration.send', + { + from: WORKER_HANDLE, + to: COORDINATOR_HANDLE, + subject: 'Need current coordinator', + type: 'escalation', + payload: JSON.stringify({ taskId: harness.taskId }) + }, + 'escalation-after-takeover', + evidence('worker') + ) + ) + ).resolves.toMatchObject({ + ok: true, + result: { + message: { + to_handle: `run:${harness.adoptedRunId}`, + delivery_contract: 'current_delivery' + } + } + }) + await expect( + harness.dispatcher.dispatch( + request( + 'orchestration.taskList', + { callerTerminalHandle: COORDINATOR_HANDLE }, + 'old-coordinator-fenced' + ) + ) + ).resolves.toMatchObject({ ok: false, error: { code: 'legacy_read_only' } }) + }) +}) diff --git a/src/main/runtime/rpc/orchestration-legacy-lifecycle.ts b/src/main/runtime/rpc/orchestration-legacy-lifecycle.ts index 9fc558e41cd..9ffe17cca6e 100644 --- a/src/main/runtime/rpc/orchestration-legacy-lifecycle.ts +++ b/src/main/runtime/rpc/orchestration-legacy-lifecycle.ts @@ -104,7 +104,7 @@ export async function handleLegacyLifecycleSend(args: { : { kind: 'message_only' } }) if (!committed.duplicate) { - runtime.notifyMessageArrived(params.to, committed.message.type) + runtime.notifyMessageArrived(committed.message.to_handle, committed.message.type) } return { message: committed.message, diff --git a/src/main/runtime/rpc/orchestration-legacy-process-identity.ts b/src/main/runtime/rpc/orchestration-legacy-process-identity.ts new file mode 100644 index 00000000000..35054c61279 --- /dev/null +++ b/src/main/runtime/rpc/orchestration-legacy-process-identity.ts @@ -0,0 +1,33 @@ +import { parsePaneKey } from '../../../shared/stable-pane-id' +import { OrchestrationError } from '../orchestration/orchestration-error' + +export function equivalentLegacyPaneKey( + a: string | null | undefined, + b: string | null | undefined +): boolean { + if (!a || !b) { + return false + } + if (a === b) { + return true + } + const aLeaf = parsePaneKey(a)?.leafId + const bLeaf = parsePaneKey(b)?.leafId + return Boolean(aLeaf && bLeaf && aLeaf === bLeaf) +} + +export function legacyReadOnlyError(): OrchestrationError { + return new OrchestrationError( + 'legacy_read_only', + 'This retained legacy assignment could not prove authority from its original live process. No effects were applied.', + { effectsApplied: false } + ) +} + +export function legacyCoordinatorReadOnly(): OrchestrationError { + return new OrchestrationError( + 'legacy_read_only', + 'This retained legacy coordinator could not prove its original process identity. No effects were applied.', + { effectsApplied: false } + ) +} diff --git a/src/main/runtime/rpc/orchestration-legacy-question.ts b/src/main/runtime/rpc/orchestration-legacy-question.ts index b9a9da8e0cb..69dec720a3a 100644 --- a/src/main/runtime/rpc/orchestration-legacy-question.ts +++ b/src/main/runtime/rpc/orchestration-legacy-question.ts @@ -65,8 +65,9 @@ export async function handleLegacyAsk(args: { options, recipientHandle: recipient }) - const pending = matches.filter((match) => match.question.status === 'pending') - const lostAnswer = matches.find( + const unclaimed = matches.filter((match) => !match.claimedByOperation) + const pending = unclaimed.filter((match) => match.question.status === 'pending') + const lostAnswer = unclaimed.find( (match) => match.question.status === 'answered' && !match.answerAcknowledged ) if (lostAnswer) { @@ -98,7 +99,7 @@ export async function handleLegacyAsk(args: { questionId = committed.question.message_id duplicate = committed.duplicate || Boolean(existingQuestionId) if (!duplicate) { - runtime.notifyMessageArrived(recipient, committed.message.type) + runtime.notifyMessageArrived(committed.message.to_handle, committed.message.type) } } diff --git a/src/main/runtime/rpc/orchestration-mutation-executor.ts b/src/main/runtime/rpc/orchestration-mutation-executor.ts index 0ec659cc02d..1b4733fcbed 100644 --- a/src/main/runtime/rpc/orchestration-mutation-executor.ts +++ b/src/main/runtime/rpc/orchestration-mutation-executor.ts @@ -55,6 +55,10 @@ export class OrchestrationMutationExecutor { : db.beginMutationReceipt(identity) if (begun.disposition === 'completed') { + const active = this.inFlight.get(key) + if (active) { + return attachMutationReceipt(await active, requestId, true) + } return attachMutationReceipt(JSON.parse(begun.row.receipt ?? 'null'), requestId, true) } if (begun.disposition === 'pending') { @@ -102,6 +106,20 @@ export class OrchestrationMutationExecutor { } } +const executorsByRuntime = new WeakMap() + +export function getOrchestrationMutationExecutor( + runtime: OrcaRuntimeService +): OrchestrationMutationExecutor { + const existing = executorsByRuntime.get(runtime) + if (existing) { + return existing + } + const executor = new OrchestrationMutationExecutor(runtime) + executorsByRuntime.set(runtime, executor) + return executor +} + export function authenticatedCallerFingerprint(request: RpcRequest): string { const callerToken = request.authToken || diff --git a/src/main/runtime/rpc/orchestration-mutation-ledger.test.ts b/src/main/runtime/rpc/orchestration-mutation-ledger.test.ts index 3407211ba6f..8e02d9c8e24 100644 --- a/src/main/runtime/rpc/orchestration-mutation-ledger.test.ts +++ b/src/main/runtime/rpc/orchestration-mutation-ledger.test.ts @@ -252,7 +252,10 @@ describe('durable orchestration mutation ledger', () => { }) it('recovers a lost ask acceptance without creating a second question', async () => { - const db = new OrchestrationDb(':memory:') + const dir = mkdtempSync(join(tmpdir(), 'orca-mutation-ask-recovery-')) + paths.push(dir) + const dbPath = join(dir, 'orchestration.db') + const db = new OrchestrationDb(dbPath) const runtime = new OrcaRuntimeService() runtime.setOrchestrationDb(db) vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue('tab_worker:leaf_worker') @@ -290,7 +293,16 @@ describe('durable orchestration mutation ledger', () => { const first = firstDispatcher.dispatch(askRequest, { signal: controller.signal }) await vi.waitFor(() => expect(db.getInbox(10)).toHaveLength(1)) - const restartedDispatcher = new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS }) + const restartedDb = new OrchestrationDb(dbPath) + const restartedRuntime = new OrcaRuntimeService() + restartedRuntime.setOrchestrationDb(restartedDb) + vi.spyOn(restartedRuntime, 'getTerminalPaneKey').mockReturnValue('tab_worker:leaf_worker') + vi.spyOn(restartedRuntime, 'getTerminalProcessIncarnation').mockReturnValue('runtime:pty:1') + vi.spyOn(restartedRuntime, 'notifyMessageArrived').mockImplementation(() => {}) + const restartedDispatcher = new RpcDispatcher({ + runtime: restartedRuntime, + methods: ORCHESTRATION_METHODS + }) const recovered = await restartedDispatcher.dispatch({ ...askRequest, id: 'rpc_ask_2' }) expect(recovered).toMatchObject({ ok: true, @@ -304,6 +316,7 @@ describe('durable orchestration mutation ledger', () => { controller.abort() await first + restartedDb.close() db.close() }) }) diff --git a/src/main/runtime/rpc/runtime-feature-interaction.ts b/src/main/runtime/rpc/runtime-feature-interaction.ts index 2a25bff1080..23f28001fcd 100644 --- a/src/main/runtime/rpc/runtime-feature-interaction.ts +++ b/src/main/runtime/rpc/runtime-feature-interaction.ts @@ -1,5 +1,6 @@ import type { FeatureInteractionId } from '../../../shared/feature-interactions' import { isBrowserPaneUiRuntimeRpcParams } from '../../../shared/runtime-rpc-feature-interaction-source' +import type { OrcaRuntimeService } from '../orca-runtime' export function getRuntimeFeatureInteractionId( method: string, @@ -37,6 +38,25 @@ export function getRuntimeFeatureInteractionId( return method.startsWith('orchestration.') ? 'agent-orchestration' : null } +export function recordRuntimeFeatureInteraction( + runtime: OrcaRuntimeService, + method: string, + result: unknown, + alreadyRecorded?: Set, + rawParams?: unknown +): void { + const id = getRuntimeFeatureInteractionId(method, result, rawParams) + if (!id || alreadyRecorded?.has(id)) { + return + } + try { + runtime.recordFeatureInteraction(id) + alreadyRecorded?.add(id) + } catch { + // Best-effort education state must not break runtime tools. + } +} + function hasBooleanResult(value: unknown, key: string): boolean { return ( value !== null && typeof value === 'object' && (value as Record)[key] === true diff --git a/src/main/ssh/ssh-relay-session.ts b/src/main/ssh/ssh-relay-session.ts index a782e6fb394..392a422e138 100644 --- a/src/main/ssh/ssh-relay-session.ts +++ b/src/main/ssh/ssh-relay-session.ts @@ -66,6 +66,10 @@ import { import type { Store } from '../persistence' import type { OrcaRuntimeService } from '../runtime/orca-runtime' import { runRemoteOrcaCli } from './ssh-remote-orca-cli' +import { + acknowledgeRemoteOrcaCliPostOutput, + parseRemoteOrcaCliPostOutput +} from './ssh-remote-orchestration-post-output' import { toSshExecutionHostId, type ExecutionHostId } from '../../shared/execution-host' import { isTerminalLeafId, makePaneKey } from '../../shared/stable-pane-id' import { isValidTerminalTabId } from '../../shared/terminal-tab-id' @@ -716,6 +720,37 @@ export class SshRelaySession { this.runtime.releaseOrchestrationCompatibilitySshAttachment(runtimeAuthority.attachmentId) } }) + mux.onRequest('orca.cli.postOutput', async (params) => { + if (!this.runtime) { + throw new Error('Orca runtime is unavailable') + } + const rawEnv = params.env + const env = + rawEnv && typeof rawEnv === 'object' && !Array.isArray(rawEnv) + ? Object.fromEntries( + Object.entries(rawEnv).filter( + (entry): entry is [string, string] => + typeof entry[0] === 'string' && typeof entry[1] === 'string' + ) + ) + : {} + const runtimeAuthority = this.runtime.registerOrchestrationCompatibilitySshAttachment( + this.targetId, + connectionIncarnation + ) + this.activeCompatibilityAttachmentIds.add(runtimeAuthority.attachmentId) + try { + await acknowledgeRemoteOrcaCliPostOutput(this.runtime, { + postOutput: parseRemoteOrcaCliPostOutput(params.postOutput), + env, + runtimeAuthority + }) + return { acknowledged: true } + } finally { + this.activeCompatibilityAttachmentIds.delete(runtimeAuthority.attachmentId) + this.runtime.releaseOrchestrationCompatibilitySshAttachment(runtimeAuthority.attachmentId) + } + }) } // Why: ship plugin/extension source from Orca so agent-event changes don't force a relay redeploy (agent-status-over-ssh.md §4/§8). Best-effort. diff --git a/src/main/ssh/ssh-remote-cli-host-passthrough.ts b/src/main/ssh/ssh-remote-cli-host-passthrough.ts index 0cdbef03211..16510ddf479 100644 --- a/src/main/ssh/ssh-remote-cli-host-passthrough.ts +++ b/src/main/ssh/ssh-remote-cli-host-passthrough.ts @@ -43,8 +43,23 @@ export type RemoteOrcaCliResult = { stdout: string stderr: string exitCode: number + postOutput?: RemoteOrcaCliPostOutput } +export type RemoteOrcaCliPostOutput = + | { + kind: 'legacy_check_ack' + terminal: string + messageIds: string[] + types?: string[] + } + | { + kind: 'legacy_question_ack' + terminal: string + questionId: string + answerMessageId: string + } + export type HostCliPassthroughOptions = { execPath?: string cliEntryPath?: string diff --git a/src/main/ssh/ssh-remote-cli-in-process-result.ts b/src/main/ssh/ssh-remote-cli-in-process-result.ts new file mode 100644 index 00000000000..a35512e3451 --- /dev/null +++ b/src/main/ssh/ssh-remote-cli-in-process-result.ts @@ -0,0 +1,40 @@ +import type { RpcResponse } from '../runtime/rpc/core' +import type { ParsedRemoteCli } from './ssh-remote-cli-argument-error' +import type { RemoteOrcaCliResult } from './ssh-remote-cli-host-passthrough' +import { formatRemoteCli } from './ssh-remote-cli-format' +import { + formatRemoteOrchestrationAsk, + getRemoteCliExitCode +} from './ssh-remote-orchestration-ask-output' +import { formatRemoteOrchestrationCheck } from './ssh-remote-orchestration-check-output' +import { getRemoteCliPostOutput } from './ssh-remote-orchestration-post-output' +import { resolveRemoteCliHandle } from './ssh-remote-cli-args' + +export function formatInProcessRemoteCliResult( + parsed: ParsedRemoteCli, + env: Record, + response: RpcResponse, + json: boolean +): RemoteOrcaCliResult { + const command = parsed.commandPath.join(' ') + const formatted = + command === 'orchestration check' + ? formatRemoteOrchestrationCheck( + response, + json, + resolveRemoteCliHandle(parsed.flags, env, 'terminal'), + parsed.flags.has('format') + ) + : command === 'orchestration ask' + ? formatRemoteOrchestrationAsk(response, json) + : json + ? { stdout: `${JSON.stringify(response, null, 2)}\n`, stderr: '' } + : formatRemoteCli(response) + const postOutput = getRemoteCliPostOutput(parsed, env, response) + return { + stdout: formatted.stdout, + stderr: formatted.stderr, + exitCode: getRemoteCliExitCode(command, response), + ...(postOutput ? { postOutput } : {}) + } +} diff --git a/src/main/ssh/ssh-remote-orca-cli.test.ts b/src/main/ssh/ssh-remote-orca-cli.test.ts index 88534ec688f..f64a9d7bc26 100644 --- a/src/main/ssh/ssh-remote-orca-cli.test.ts +++ b/src/main/ssh/ssh-remote-orca-cli.test.ts @@ -24,7 +24,6 @@ const LEGACY_FALLBACK_OPTIONS: HostCliPassthroughOptions = { userDataPath: '/host/user-data', entryExists: () => false } - type FakeChild = EventEmitter & { stdout: EventEmitter stderr: EventEmitter diff --git a/src/main/ssh/ssh-remote-orca-cli.ts b/src/main/ssh/ssh-remote-orca-cli.ts index 9d13c943981..acd0c94cfaf 100644 --- a/src/main/ssh/ssh-remote-orca-cli.ts +++ b/src/main/ssh/ssh-remote-orca-cli.ts @@ -6,7 +6,6 @@ import { ORCHESTRATION_CONTRACT_VERSION } from '../../shared/protocol-version' import { RpcDispatcher } from '../runtime/rpc/dispatcher' import type { RpcResponse } from '../runtime/rpc/core' import type { OrcaRuntimeService } from '../runtime/orca-runtime' -import { formatRemoteCli } from './ssh-remote-cli-format' import { HostCliUnavailableError, runHostOrcaCliPassthrough, @@ -26,9 +25,9 @@ import { buildRemoteCliError } from './ssh-remote-cli-error-response' import { getRemoteLinearHelp, tryDispatchRemoteLinearCli } from './ssh-remote-linear-cli' import { getRemoteOrchestrationPayload, - hasRemoteLifecycleRejection, resolveRemoteOrchestrationSender } from './ssh-remote-orchestration-send' +import { formatInProcessRemoteCliResult } from './ssh-remote-cli-in-process-result' export type { RemoteOrcaCliRequest, RemoteOrcaCliResult } from './ssh-remote-cli-host-passthrough' @@ -51,6 +50,7 @@ export async function runRemoteOrcaCli( ): Promise { const parsed = parseRemoteCliArgs(request.argv) const json = parsed.flags.has('json') + const command = parsed.commandPath.join(' ') const interactiveMessage = HOST_INTERACTIVE_COMMANDS[parsed.commandPath[0] ?? ''] if (interactiveMessage) { @@ -64,6 +64,17 @@ export async function runRemoteOrcaCli( return { stdout: '', stderr: `${interactiveMessage}\n`, exitCode: 1 } } + if (command === 'orchestration check' || command === 'orchestration ask') { + // Why: compatibility ACKs must wait until relay stdout is observable; a host CLI child can only flush into main's capture pipe. + return await runLegacyRemoteOrcaCli( + runtime, + request, + parsed, + json, + new HostCliUnavailableError('output-ordered orchestration bridge required') + ) + } + let passthroughFailure: HostCliUnavailableError | null = null try { return await runHostOrcaCliPassthrough(request, passthroughOptions) @@ -101,16 +112,7 @@ async function runLegacyRemoteOrcaCli( passthroughFailure.message, request.runtimeAuthority ) - const formatted = json - ? { stdout: `${JSON.stringify(response, null, 2)}\n`, stderr: '' } - : formatRemoteCli(response) - return { - stdout: formatted.stdout, - stderr: formatted.stderr, - // Why: the legacy SSH bridge bypasses the local CLI handler that turns - // a persisted lifecycle rejection into an unsuccessful command. - exitCode: response.ok && !hasRemoteLifecycleRejection(response.result) ? 0 : 1 - } + return formatInProcessRemoteCliResult(parsed, request.env, response, json) } catch (err) { const message = err instanceof Error ? err.message : String(err) const code = @@ -141,13 +143,17 @@ async function dispatchRemoteCli( runtimeAuthority: RemoteOrcaCliRequest['runtimeAuthority'] ): Promise { const command = parsed.commandPath.join(' ') - const compatibilityInvocationId = randomUUID() const inheritedEvidence = readOrchestrationCompatibilityEvidence(env) const orchestrationCompatibilityEvidence = runtimeAuthority ? { ...inheritedEvidence, host: runtimeAuthority } : inheritedEvidence const compatibilityEnvelope: RuntimeOrchestrationEnvelope = { - compatibilityInvocationId, + compatibilityInvocationId: randomUUID(), + orchestrationRequestId: + optionalRemoteCliString(parsed.flags, 'retry-request') ?? + (command === 'orchestration check' || command === 'orchestration ask' + ? randomUUID() + : undefined), orchestrationCompatibilityEvidence } const linearResponse = await tryDispatchRemoteLinearCli(dispatcher, parsed, env, stdin) @@ -201,8 +207,7 @@ async function dispatchRemoteCli( }, { ...compatibilityEnvelope, - orchestrationCapability: optionalRemoteCliString(parsed.flags, 'dispatch-capability'), - orchestrationRequestId: optionalRemoteCliString(parsed.flags, 'retry-request') + orchestrationCapability: optionalRemoteCliString(parsed.flags, 'dispatch-capability') } ) } @@ -215,15 +220,39 @@ async function dispatchRemoteCli( terminalPaneKey: parsed.flags.has('terminal') ? undefined : env.ORCA_PANE_KEY || undefined, - unread: parsed.flags.has('unread') ? true : undefined, + unread: parsed.flags.has('unread') ? true : parsed.flags.has('peek') ? false : undefined, + peek: parsed.flags.has('peek') ? true : undefined, all: parsed.flags.has('all') ? true : undefined, types: optionalRemoteCliString(parsed.flags, 'types'), + format: parsed.flags.has('format') ? true : undefined, inject: parsed.flags.has('inject') ? true : undefined, + compatibilityCliCommand: 'orca', + run: optionalRemoteCliString(parsed.flags, 'run'), + ack: optionalRemoteCliString(parsed.flags, 'ack'), wait: parsed.flags.has('wait') ? true : undefined, timeoutMs: optionalRemoteCliNumber(parsed.flags, 'timeout-ms') }, compatibilityEnvelope ) + case 'orchestration ask': + return await call( + dispatcher, + 'orchestration.ask', + { + to: optionalRemoteCliString(parsed.flags, 'to'), + question: optionalRemoteCliString(parsed.flags, 'question'), + resume: optionalRemoteCliString(parsed.flags, 'resume'), + options: optionalRemoteCliString(parsed.flags, 'options'), + timeoutMs: optionalRemoteCliNumber(parsed.flags, 'timeout-ms'), + from: resolveRemoteCliHandle(parsed.flags, env, 'from'), + run: optionalRemoteCliString(parsed.flags, 'run'), + compatibilityCliCommand: 'orca' + }, + { + ...compatibilityEnvelope, + orchestrationCapability: optionalRemoteCliString(parsed.flags, 'dispatch-capability') + } + ) case 'orchestration reply': return await call( dispatcher, @@ -271,7 +300,8 @@ async function call( ? ORCHESTRATION_CONTRACT_VERSION : undefined, orchestrationRequestId: envelope?.orchestrationRequestId, - compatibilityInvocationId: envelope?.compatibilityInvocationId, + compatibilityInvocationId: + envelope?.orchestrationRequestId ?? envelope?.compatibilityInvocationId, orchestrationCompatibilityEvidence: envelope?.orchestrationCompatibilityEvidence }) } diff --git a/src/main/ssh/ssh-remote-orchestration-ask-output.ts b/src/main/ssh/ssh-remote-orchestration-ask-output.ts new file mode 100644 index 00000000000..d57493db5ce --- /dev/null +++ b/src/main/ssh/ssh-remote-orchestration-ask-output.ts @@ -0,0 +1,66 @@ +import type { RpcResponse } from '../runtime/rpc/core' +import { formatRemoteCli } from './ssh-remote-cli-format' +import { hasRemoteLifecycleRejection } from './ssh-remote-orchestration-send' + +export function formatRemoteOrchestrationAsk( + response: RpcResponse, + json: boolean +): { stdout: string; stderr: string } { + if (!response.ok || !isRecord(response.result)) { + return json + ? { stdout: `${JSON.stringify(response, null, 2)}\n`, stderr: '' } + : formatRemoteCli(response) + } + if (json) { + return { stdout: `${JSON.stringify(response.result)}\n`, stderr: '' } + } + if (isRecord(response.result.legacyCompatibility)) { + const compatibility = response.result.legacyCompatibility + if (compatibility.resumeRequired === true && typeof compatibility.resumeCommand === 'string') { + return { + stdout: + `Question ${String(response.result.messageId ?? 'unknown')} committed.\n` + + `Resume with: ${compatibility.resumeCommand}\n`, + stderr: '' + } + } + } + const answer = typeof response.result.answer === 'string' ? response.result.answer : '' + const thread = typeof response.result.threadId === 'string' ? response.result.threadId : 'unknown' + const timeoutMs = + typeof response.result.timeoutMs === 'number' ? response.result.timeoutMs : undefined + const stderr = response.result.timedOut + ? `ask timeout after ${timeoutMs ?? 0}ms (thread ${thread})\n` + : response.result.cancelled + ? response.result.connectionLost + ? `ask connection closed (question ${String(response.result.messageId ?? 'unknown')})\n` + : `ask cancelled (question ${String(response.result.messageId ?? 'unknown')})\n` + : '' + return { stdout: answer ? `${answer}\n` : '', stderr } +} + +export function getRemoteCliExitCode(command: string, response: RpcResponse): number { + if (!response.ok || hasRemoteLifecycleRejection(response.result)) { + return 1 + } + if ( + command === 'orchestration ask' && + isRecord(response.result) && + isRecord(response.result.legacyCompatibility) && + response.result.legacyCompatibility.resumeRequired === true + ) { + return 75 + } + if ( + command === 'orchestration ask' && + isRecord(response.result) && + (response.result.timedOut === true || response.result.cancelled === true) + ) { + return 1 + } + return 0 +} + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) +} diff --git a/src/main/ssh/ssh-remote-orchestration-check-output.ts b/src/main/ssh/ssh-remote-orchestration-check-output.ts new file mode 100644 index 00000000000..78ec3f4c544 --- /dev/null +++ b/src/main/ssh/ssh-remote-orchestration-check-output.ts @@ -0,0 +1,48 @@ +import { + formatOrchestrationCheckText, + prepareOrchestrationCheckOutput, + type OrchestrationCheckOutput, + type OrchestrationMessageSummary +} from '../../shared/orchestration-check-output' +import type { RpcResponse } from '../runtime/rpc/core' +import { formatRemoteCli } from './ssh-remote-cli-format' + +export function formatRemoteOrchestrationCheck( + response: RpcResponse, + json: boolean, + terminal: string, + formattedRequested: boolean +): { stdout: string; stderr: string } { + if (!response.ok || !isRecord(response.result)) { + return json + ? { stdout: `${JSON.stringify(response, null, 2)}\n`, stderr: '' } + : formatRemoteCli(response) + } + const messages = Array.isArray(response.result.messages) + ? response.result.messages.filter(isMessageSummary) + : [] + const count = typeof response.result.count === 'number' ? response.result.count : messages.length + const result = prepareOrchestrationCheckOutput( + { ...(response.result as OrchestrationCheckOutput), messages, count }, + terminal, + formattedRequested + ) + if (json) { + return { + stdout: `${JSON.stringify({ ...response, result }, null, 2)}\n`, + stderr: '' + } + } + return { + stdout: `${formatOrchestrationCheckText(result, terminal)}\n`, + stderr: '' + } +} + +function isMessageSummary(value: unknown): value is OrchestrationMessageSummary { + return isRecord(value) && typeof value.id === 'string' && typeof value.from_handle === 'string' +} + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) +} diff --git a/src/main/ssh/ssh-remote-orchestration-compatibility.test.ts b/src/main/ssh/ssh-remote-orchestration-compatibility.test.ts new file mode 100644 index 00000000000..9d8787beda7 --- /dev/null +++ b/src/main/ssh/ssh-remote-orchestration-compatibility.test.ts @@ -0,0 +1,394 @@ +import { createHash } from 'node:crypto' +import { describe, expect, it, vi } from 'vitest' + +vi.mock('electron', () => ({ + app: { + isPackaged: false, + getAppPath: () => '/host/app' + } +})) +vi.mock('../persistence', () => ({ + getCanonicalUserDataPath: () => '/host/user-data' +})) + +import { OrchestrationDb } from '../runtime/orchestration/db' +import { OrcaRuntimeService } from '../runtime/orca-runtime' +import type Database from '../sqlite/sync-database' +import type { HostCliPassthroughOptions } from './ssh-remote-cli-host-passthrough' +import { runRemoteOrcaCli } from './ssh-remote-orca-cli' +import { acknowledgeRemoteOrcaCliPostOutput } from './ssh-remote-orchestration-post-output' + +const LEGACY_FALLBACK_OPTIONS: HostCliPassthroughOptions = { + execPath: '/host/electron', + cliEntryPath: '/host/app/out/cli/index.js', + userDataPath: '/host/user-data', + entryExists: () => false +} +const WORKER_HANDLE = 'term_legacy_ssh_worker' +const WORKER_PANE = 'tab_legacy_ssh:33333333-3333-4333-8333-333333333333' +const COORDINATOR_HANDLE = 'term_legacy_ssh_coord' +const COORDINATOR_PANE = 'tab_legacy_coord:44444444-4444-4444-8444-444444444444' +const WORKER_ENV = { + ORCA_TERMINAL_HANDLE: WORKER_HANDLE, + ORCA_PANE_KEY: WORKER_PANE, + ORCA_AGENT_LAUNCH_TOKEN: 'legacy-ssh-token' +} +const COORDINATOR_ENV = { + ORCA_TERMINAL_HANDLE: COORDINATOR_HANDLE, + ORCA_PANE_KEY: COORDINATOR_PANE, + ORCA_AGENT_LAUNCH_TOKEN: 'legacy-ssh-coordinator-token' +} +const RUNTIME_AUTHORITY = { + kind: 'ssh' as const, + targetId: 'saved-target', + connectionIncarnation: 'connection-1', + attachmentId: 'attachment-1' +} + +function createLegacyRuntime() { + const db = new OrchestrationDb(':memory:') + const run = db.createRun({ + objective: 'Adopted legacy SSH work', + coordinatorHandle: COORDINATOR_HANDLE, + coordinatorPaneKey: COORDINATOR_PANE + }) + const task = db.createTask({ + spec: 'legacy SSH assignment', + runId: run.id, + createdByTerminalHandle: COORDINATOR_HANDLE + }) + const dispatch = db.createDispatchContext(task.id, WORKER_HANDLE, WORKER_PANE) + const sqlite = (db as unknown as { db: Database.Database }).db + sqlite + .prepare( + `UPDATE dispatch_contexts + SET contract_version = 0, process_incarnation = 'process-1' + WHERE id = ?` + ) + .run(dispatch.id) + sqlite + .prepare( + `INSERT INTO legacy_adoptions (source_run_id, adopted_run_id, scheduler_state_lost) + VALUES ('run_legacy_local', ?, 1)` + ) + .run(run.id) + + const runtime = new OrcaRuntimeService() + runtime.setOrchestrationDb(db) + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === WORKER_HANDLE ? WORKER_PANE : handle === COORDINATOR_HANDLE ? COORDINATOR_PANE : null + ) + vi.spyOn(runtime, 'verifyOrchestrationCompatibilityCaller').mockImplementation((evidence) => { + const worker = + evidence?.terminalHandle === WORKER_HANDLE && + evidence.paneKey === WORKER_PANE && + evidence.launchToken === WORKER_ENV.ORCA_AGENT_LAUNCH_TOKEN + const coordinator = + evidence?.terminalHandle === COORDINATOR_HANDLE && + evidence.paneKey === COORDINATOR_PANE && + evidence.launchToken === COORDINATOR_ENV.ORCA_AGENT_LAUNCH_TOKEN + if ( + (!worker && !coordinator) || + evidence.host?.kind !== 'ssh' || + evidence.host.targetId !== RUNTIME_AUTHORITY.targetId || + evidence.host.connectionIncarnation !== RUNTIME_AUTHORITY.connectionIncarnation + ) { + return null + } + const identity = worker + ? { + terminalHandle: WORKER_HANDLE, + paneKey: WORKER_PANE, + launchToken: WORKER_ENV.ORCA_AGENT_LAUNCH_TOKEN + } + : { + terminalHandle: COORDINATOR_HANDLE, + paneKey: COORDINATOR_PANE, + launchToken: COORDINATOR_ENV.ORCA_AGENT_LAUNCH_TOKEN + } + return { + hostScope: { kind: 'ssh', targetId: RUNTIME_AUTHORITY.targetId }, + terminalHandle: identity.terminalHandle, + paneKey: identity.paneKey, + processIncarnation: 'process-1', + launchTokenHash: createHash('sha256').update(identity.launchToken).digest('hex') + } + }) + vi.spyOn(runtime, 'notifyMessageArrived').mockImplementation(() => {}) + return { db, runtime, run, dispatch } +} + +describe('legacy SSH orchestration fallback', () => { + it('acknowledges a consuming check only after remote output', async () => { + const { db, runtime, run } = createLegacyRuntime() + const message = db.insertMessage({ + runId: run.id, + deliveryContract: 'legacy_direct', + from: COORDINATOR_HANDLE, + to: WORKER_HANDLE, + subject: 'retained SSH mail' + }) + const request = { + argv: ['orchestration', 'check', '--unread', '--inject', '--json'], + cwd: '/home/alice/repo', + env: WORKER_ENV, + runtimeAuthority: RUNTIME_AUTHORITY + } + + try { + const first = await runRemoteOrcaCli(runtime, request, LEGACY_FALLBACK_OPTIONS) + const replay = await runRemoteOrcaCli(runtime, request, LEGACY_FALLBACK_OPTIONS) + + expect(JSON.parse(first.stdout)).toMatchObject({ + result: { + messages: [{ id: message.id }], + formatted: expect.stringContaining('retained SSH mail') + } + }) + expect(first.postOutput).toEqual({ + kind: 'legacy_check_ack', + terminal: WORKER_HANDLE, + messageIds: [message.id] + }) + expect(JSON.parse(replay.stdout)).toMatchObject({ + result: { messages: [{ id: message.id }] } + }) + expect(db.getMessageById(message.id)?.read).toBe(0) + + await acknowledgeRemoteOrcaCliPostOutput(runtime, { + postOutput: first.postOutput!, + env: WORKER_ENV, + runtimeAuthority: RUNTIME_AUTHORITY + }) + const afterAck = await runRemoteOrcaCli(runtime, request, LEGACY_FALLBACK_OPTIONS) + + expect(JSON.parse(afterAck.stdout)).toMatchObject({ result: { messages: [], count: 0 } }) + expect(afterAck.postOutput).toBeUndefined() + expect(db.getMessageById(message.id)?.read).toBe(1) + } finally { + db.close() + } + }) + + it('keeps peek formatted and non-consuming', async () => { + const { db, runtime, run } = createLegacyRuntime() + const message = db.insertMessage({ + runId: run.id, + deliveryContract: 'legacy_direct', + from: COORDINATOR_HANDLE, + to: WORKER_HANDLE, + subject: 'inspect retained SSH mail' + }) + + try { + const peek = await runRemoteOrcaCli( + runtime, + { + argv: ['orchestration', 'check', '--peek', '--format', '--json'], + cwd: '/home/alice/repo', + env: WORKER_ENV, + runtimeAuthority: RUNTIME_AUTHORITY + }, + LEGACY_FALLBACK_OPTIONS + ) + + const output = JSON.parse(peek.stdout) as { + result: { formatted: string } + } + expect(output).toMatchObject({ + result: { + messages: [{ id: message.id }], + formatted: expect.stringContaining('inspect retained SSH mail'), + legacyCompatibility: { readOnly: true } + } + }) + expect(output.result.formatted).toContain(`${message.id} [legacy, read-only]`) + expect(output.result.formatted).toContain( + 'Inspection only: reply and acknowledgment are unavailable.' + ) + expect(output.result.formatted).not.toContain('orchestration reply') + expect(peek.postOutput).toBeUndefined() + expect(db.getMessageById(message.id)?.read).toBe(0) + } finally { + db.close() + } + }) + + it('reads and acknowledges current Run delivery', async () => { + const { db, runtime, run } = createLegacyRuntime() + const message = db.insertMessage({ + runId: run.id, + from: WORKER_HANDLE, + to: `run:${run.id}`, + subject: 'current Run mail' + }) + const baseRequest = { + cwd: '/home/alice/repo', + env: COORDINATOR_ENV, + runtimeAuthority: RUNTIME_AUTHORITY + } + const spawn = vi.fn() + const hostCliAvailable = { + ...LEGACY_FALLBACK_OPTIONS, + entryExists: () => true, + spawn: spawn as HostCliPassthroughOptions['spawn'] + } + + try { + const checked = await runRemoteOrcaCli( + runtime, + { + ...baseRequest, + argv: ['orchestration', 'check', '--run', run.id] + }, + hostCliAvailable + ) + expect(checked.stdout).toContain('Delivery ') + expect(checked.stdout).toContain(`${message.id} [status] from=${WORKER_HANDLE}`) + expect(spawn).not.toHaveBeenCalled() + + const checkedJson = await runRemoteOrcaCli( + runtime, + { + ...baseRequest, + argv: ['orchestration', 'check', '--run', run.id, '--json'] + }, + hostCliAvailable + ) + const result = JSON.parse(checkedJson.stdout) as { + result: { deliveryId: string; messages: { id: string }[] } + } + expect(result.result.messages).toEqual([expect.objectContaining({ id: message.id })]) + + const acknowledged = await runRemoteOrcaCli( + runtime, + { + ...baseRequest, + argv: [ + 'orchestration', + 'check', + '--run', + run.id, + '--ack', + result.result.deliveryId, + '--json' + ] + }, + hostCliAvailable + ) + + expect(JSON.parse(acknowledged.stdout)).toMatchObject({ + result: { acknowledged: result.result.deliveryId } + }) + expect(db.getMessageById(message.id)?.read).toBe(1) + } finally { + db.close() + } + }) + + it('replays an SSH legacy ask by retry request without creating another question', async () => { + const { db, runtime } = createLegacyRuntime() + const argv = [ + 'orchestration', + 'ask', + '--to', + COORDINATOR_HANDLE, + '--question', + 'Continue?', + '--timeout-ms', + '1', + '--retry-request', + 'ssh-question-1', + '--json' + ] + const request = { + argv, + cwd: '/home/alice/repo', + env: WORKER_ENV, + runtimeAuthority: RUNTIME_AUTHORITY + } + + try { + const first = await runRemoteOrcaCli(runtime, request, LEGACY_FALLBACK_OPTIONS) + const replay = await runRemoteOrcaCli(runtime, request, LEGACY_FALLBACK_OPTIONS) + const sqlite = (db as unknown as { db: Database.Database }).db + + const firstResult = JSON.parse(first.stdout) as { messageId: string; timedOut: boolean } + const replayResult = JSON.parse(replay.stdout) as { messageId: string; timedOut: boolean } + expect(first).toMatchObject({ exitCode: 1 }) + expect(firstResult).toMatchObject({ messageId: expect.any(String), timedOut: true }) + expect(replay).toMatchObject({ exitCode: 1 }) + expect(replayResult).toMatchObject({ messageId: firstResult.messageId, timedOut: true }) + expect( + ( + sqlite.prepare('SELECT COUNT(*) AS count FROM question_threads').get() as { + count: number + } + ).count + ).toBe(1) + } finally { + db.close() + } + }) + + it('resumes and acknowledges a question answer', async () => { + const { db, runtime, run, dispatch } = createLegacyRuntime() + const pending = db.createQuestion({ + runId: run.id, + dispatchId: dispatch.id, + askerHandle: WORKER_HANDLE, + question: 'Proceed?' + }) + const answer = db.answerQuestion({ + messageId: pending.question.message_id, + runId: run.id, + consumerGeneration: run.consumer_generation, + body: 'yes' + }) + const sqlite = (db as unknown as { db: Database.Database }).db + sqlite + .prepare( + `UPDATE messages + SET delivery_contract = 'legacy_direct', read = 0 + WHERE id = ?` + ) + .run(answer.message.id) + + try { + const result = await runRemoteOrcaCli( + runtime, + { + argv: ['orchestration', 'ask', '--resume', pending.question.message_id, '--json'], + cwd: '/home/alice/repo', + env: WORKER_ENV, + runtimeAuthority: RUNTIME_AUTHORITY + }, + LEGACY_FALLBACK_OPTIONS + ) + + expect(result.exitCode).toBe(0) + expect(JSON.parse(result.stdout)).toMatchObject({ + answer: 'yes', + messageId: pending.question.message_id, + timedOut: false + }) + expect(result.postOutput).toEqual({ + kind: 'legacy_question_ack', + terminal: WORKER_HANDLE, + questionId: pending.question.message_id, + answerMessageId: answer.message.id + }) + expect(db.getMessageById(answer.message.id)?.read).toBe(0) + + await acknowledgeRemoteOrcaCliPostOutput(runtime, { + postOutput: result.postOutput!, + env: WORKER_ENV, + runtimeAuthority: RUNTIME_AUTHORITY + }) + + expect(db.getMessageById(answer.message.id)?.read).toBe(1) + } finally { + db.close() + } + }) +}) diff --git a/src/main/ssh/ssh-remote-orchestration-post-output.ts b/src/main/ssh/ssh-remote-orchestration-post-output.ts new file mode 100644 index 00000000000..5d833651350 --- /dev/null +++ b/src/main/ssh/ssh-remote-orchestration-post-output.ts @@ -0,0 +1,143 @@ +import { randomUUID } from 'node:crypto' +import { readOrchestrationCompatibilityEvidence } from '../../shared/orchestration-compatibility-evidence' +import { ORCHESTRATION_CONTRACT_VERSION } from '../../shared/protocol-version' +import type { OrcaRuntimeService } from '../runtime/orca-runtime' +import type { RpcResponse } from '../runtime/rpc/core' +import { RpcDispatcher } from '../runtime/rpc/dispatcher' +import type { + RemoteOrcaCliPostOutput, + RemoteOrcaCliRequest +} from './ssh-remote-cli-host-passthrough' +import { RemoteCliArgumentError, type ParsedRemoteCli } from './ssh-remote-cli-argument-error' +import { optionalRemoteCliString, resolveRemoteCliHandle } from './ssh-remote-cli-args' + +export async function acknowledgeRemoteOrcaCliPostOutput( + runtime: OrcaRuntimeService, + args: { + postOutput: RemoteOrcaCliPostOutput + env: Record + runtimeAuthority?: RemoteOrcaCliRequest['runtimeAuthority'] + } +): Promise { + const inheritedEvidence = readOrchestrationCompatibilityEvidence(args.env) + const orchestrationCompatibilityEvidence = args.runtimeAuthority + ? { ...inheritedEvidence, host: args.runtimeAuthority } + : inheritedEvidence + const params = + args.postOutput.kind === 'legacy_check_ack' + ? { + terminal: args.postOutput.terminal, + compatibilityAck: JSON.stringify({ + messageIds: args.postOutput.messageIds, + types: args.postOutput.types + }) + } + : { + terminal: args.postOutput.terminal, + compatibilityQuestionAck: JSON.stringify({ + questionId: args.postOutput.questionId, + answerMessageId: args.postOutput.answerMessageId + }) + } + const response = await new RpcDispatcher({ runtime }).dispatch({ + id: `remote-cli-post-output-${randomUUID()}`, + authToken: 'remote-cli', + method: 'orchestration.check', + params, + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + compatibilityInvocationId: randomUUID(), + orchestrationCompatibilityEvidence + }) + if (!response.ok) { + throw new Error(response.error.message) + } +} + +export function parseRemoteOrcaCliPostOutput(value: unknown): RemoteOrcaCliPostOutput { + if (!isRecord(value) || typeof value.kind !== 'string' || typeof value.terminal !== 'string') { + throw invalidPostOutput() + } + if ( + value.kind === 'legacy_check_ack' && + Array.isArray(value.messageIds) && + value.messageIds.every((id) => typeof id === 'string') && + (value.types === undefined || + (Array.isArray(value.types) && value.types.every((type) => typeof type === 'string'))) + ) { + return { + kind: value.kind, + terminal: value.terminal, + messageIds: value.messageIds, + ...(value.types === undefined ? {} : { types: value.types }) + } + } + if ( + value.kind === 'legacy_question_ack' && + typeof value.questionId === 'string' && + typeof value.answerMessageId === 'string' + ) { + return { + kind: value.kind, + terminal: value.terminal, + questionId: value.questionId, + answerMessageId: value.answerMessageId + } + } + throw invalidPostOutput() +} + +export function getRemoteCliPostOutput( + parsed: ParsedRemoteCli, + env: Record, + response: RpcResponse +): RemoteOrcaCliPostOutput | undefined { + if (!response.ok || !isRecord(response.result)) { + return undefined + } + const compatibility = response.result.legacyCompatibility + if (!isRecord(compatibility)) { + return undefined + } + const command = parsed.commandPath.join(' ') + if ( + command === 'orchestration check' && + Array.isArray(compatibility.ackMessageIds) && + compatibility.ackMessageIds.length > 0 && + compatibility.ackMessageIds.every((id) => typeof id === 'string') + ) { + const types = optionalRemoteCliString(parsed.flags, 'types') + ?.split(',') + .map((type) => type.trim()) + .filter(Boolean) + return { + kind: 'legacy_check_ack', + terminal: resolveRemoteCliHandle(parsed.flags, env, 'terminal'), + messageIds: compatibility.ackMessageIds, + ...(types ? { types } : {}) + } + } + const acknowledgement = compatibility.answerAcknowledgement + if ( + command === 'orchestration ask' && + response.result.answer !== null && + isRecord(acknowledgement) && + typeof acknowledgement.questionId === 'string' && + typeof acknowledgement.answerMessageId === 'string' + ) { + return { + kind: 'legacy_question_ack', + terminal: resolveRemoteCliHandle(parsed.flags, env, 'from'), + questionId: acknowledgement.questionId, + answerMessageId: acknowledgement.answerMessageId + } + } + return undefined +} + +function invalidPostOutput(): RemoteCliArgumentError { + return new RemoteCliArgumentError('invalid_argument', 'Invalid SSH CLI post-output action.') +} + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) +} diff --git a/src/main/window/attach-main-window-services.test.ts b/src/main/window/attach-main-window-services.test.ts index 833f158fc56..bc1e7ae258d 100644 --- a/src/main/window/attach-main-window-services.test.ts +++ b/src/main/window/attach-main-window-services.test.ts @@ -829,4 +829,73 @@ describe('attachMainWindowServices', () => { await expect(revealPromise).resolves.toEqual({ tabId: 'tab-1', title: 'SSH tmux' }) expect(removeListenerMock).toHaveBeenCalledWith('terminal:tabCreateReply', handler) }) + + it('requires an exact renderer identity receipt for recovered worker reveals', async () => { + const sendMock = vi.fn() + const mainWindow = createMainWindow({ send: sendMock }) + const runtime = createRuntime() + + attachMainWindowServices(mainWindow as never, createStore(), runtime as never) + + const notifier = runtime.setNotifier.mock.calls[0][0] as { + revealTerminalSession: ( + worktreeId: string, + opts: { + ptyId: string + tabId: string + leafId: string + expectedProcessIdentity: { terminalHandle: string; incarnationId: string } + } + ) => Promise + } + const opts = { + ptyId: 'pty-worker', + tabId: 'tab-worker', + leafId: 'leaf-worker', + expectedProcessIdentity: { + terminalHandle: 'term_worker', + incarnationId: 'inc-worker' + } + } + const mismatch = notifier.revealTerminalSession('worktree-1', opts) + const mismatchPayload = sendMock.mock.calls.at(-1)?.[1] + const mismatchHandler = onMock.mock.calls.findLast( + ([channel]) => channel === 'terminal:tabCreateReply' + )?.[1] + mismatchHandler?.( + { sender: mainWindow.webContents }, + { + requestId: mismatchPayload.requestId, + tabId: 'tab-worker', + identity: { + worktreeId: 'worktree-1', + tabId: 'tab-worker', + leafId: 'leaf-worker', + ptyId: 'pty-replacement' + } + } + ) + await expect(mismatch).rejects.toThrow('terminal_reveal_identity_mismatch') + + const exact = notifier.revealTerminalSession('worktree-1', opts) + const exactPayload = sendMock.mock.calls.at(-1)?.[1] + const exactHandler = onMock.mock.calls.findLast( + ([channel]) => channel === 'terminal:tabCreateReply' + )?.[1] + const identity = { + worktreeId: 'worktree-1', + tabId: 'tab-worker', + leafId: 'leaf-worker', + ptyId: 'pty-worker' + } + exactHandler?.( + { sender: mainWindow.webContents }, + { requestId: exactPayload.requestId, tabId: 'tab-worker', identity } + ) + await expect(exact).resolves.toEqual({ + tabId: 'tab-worker', + title: undefined, + identity + }) + }) }) diff --git a/src/main/window/attach-main-window-services.ts b/src/main/window/attach-main-window-services.ts index 2e2186b1d6c..3c8219c4040 100644 --- a/src/main/window/attach-main-window-services.ts +++ b/src/main/window/attach-main-window-services.ts @@ -49,6 +49,7 @@ import type { RuntimeMarkdownSaveTabResult } from '../../shared/mobile-markdown-document' import type { RuntimeMobileSessionTabMove } from '../../shared/runtime-types' +import type { TerminalTabCreateReply } from '../../shared/terminal-reveal-identity' import { isNativeFileDropPayload, type NativeFileDropPayload } from '../../shared/native-file-drop' import { requestMobileMarkdownFromRenderer } from './mobile-markdown-request-relay' import { requestTerminalTabCloseFromRenderer } from './terminal-tab-close-request-relay' @@ -347,14 +348,20 @@ function registerRuntimeWindowLifecycle( revealTerminalSession: (worktreeId, opts) => new Promise((resolve, reject) => { const requestId = randomUUID() + const expectedIdentity = opts.expectedProcessIdentity + ? opts.tabId && opts.leafId + ? { worktreeId, tabId: opts.tabId, leafId: opts.leafId, ptyId: opts.ptyId } + : null + : undefined + if (expectedIdentity === null) { + reject(new Error('terminal_reveal_identity_required')) + return + } const timer = setTimeout(() => { ipcMain.removeListener('terminal:tabCreateReply', handler) reject(new Error('Terminal reveal timed out')) }, 10_000) - const handler = ( - event: Electron.IpcMainEvent, - reply: { requestId: string; tabId?: string; title?: string; error?: string } - ): void => { + const handler = (event: Electron.IpcMainEvent, reply: TerminalTabCreateReply): void => { // Why: requestId is renderer-supplied, so only the targeted main window may satisfy the reveal. if (event.sender !== mainWindow.webContents || reply.requestId !== requestId) { return @@ -365,7 +372,22 @@ function registerRuntimeWindowLifecycle( reject(new Error(reply.error)) return } - resolve({ tabId: reply.tabId!, title: reply.title }) + if ( + expectedIdentity && + (!reply.identity || + reply.identity.worktreeId !== expectedIdentity.worktreeId || + reply.identity.tabId !== expectedIdentity.tabId || + reply.identity.leafId !== expectedIdentity.leafId || + reply.identity.ptyId !== expectedIdentity.ptyId) + ) { + reject(new Error('terminal_reveal_identity_mismatch')) + return + } + resolve({ + tabId: reply.tabId!, + title: reply.title, + ...(reply.identity ? { identity: reply.identity } : {}) + }) } ipcMain.on('terminal:tabCreateReply', handler) send('ui:createTerminal', { diff --git a/src/preload/api-types.ts b/src/preload/api-types.ts index 357701029b5..fc146a22682 100644 --- a/src/preload/api-types.ts +++ b/src/preload/api-types.ts @@ -18,6 +18,7 @@ import type { TerminalTabCloseRequest, TerminalTabCloseResponse } from '../shared/terminal-tab-close' +import type { TerminalTabCreateReply } from '../shared/terminal-reveal-identity' import type { LocalLogTailChangedPayload, LocalLogTailReadArgs, @@ -3149,12 +3150,7 @@ export type PreloadApi = { onRequestTerminalTabMount: ( callback: (data: { worktreeId: string; tabId?: string; ptyId?: string }) => void ) => () => void - replyTerminalCreate: (reply: { - requestId: string - tabId?: string - title?: string - error?: string - }) => void + replyTerminalCreate: (reply: TerminalTabCreateReply) => void onSplitTerminal: ( callback: (data: { tabId: string diff --git a/src/preload/index.ts b/src/preload/index.ts index 75c5c28d53d..827c516324b 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -13,6 +13,7 @@ import type { CliInstallStatus } from '../shared/cli-install-types' import type { AgentHookInstallStatus } from '../shared/agent-hook-types' import type { CodexConfigSyncStatus } from '../shared/codex-config-sync-types' import type { TerminalPaneSplitSource } from '../shared/feature-education-telemetry' +import type { TerminalTabCreateReply } from '../shared/terminal-reveal-identity' import type { ProjectExecutionRuntimeResolution } from '../shared/project-execution-runtime' import type { StartupCommandDelivery } from '../shared/codex-startup-delivery' import type { @@ -3752,12 +3753,7 @@ const api = { ipcRenderer.on('terminal:requestTabMount', listener) return () => ipcRenderer.removeListener('terminal:requestTabMount', listener) }, - replyTerminalCreate: (reply: { - requestId: string - tabId?: string - title?: string - error?: string - }): void => { + replyTerminalCreate: (reply: TerminalTabCreateReply): void => { ipcRenderer.send('terminal:tabCreateReply', reply) }, onSplitTerminal: ( diff --git a/src/relay/relay.ts b/src/relay/relay.ts index 82110b58db2..6daf87b309c 100644 --- a/src/relay/relay.ts +++ b/src/relay/relay.ts @@ -198,6 +198,8 @@ async function runOrcaCliMode(sockPath: string, argv: string[]): Promise { let nextSeq = 1 let highestReceivedSeq = 0 const requestId = 1 + const postOutputRequestId = 2 + let initialExitCode = 0 const sendRequest = (): void => { const env = pickRemoteCliEnv(process.env) @@ -219,6 +221,59 @@ async function runOrcaCliMode(sockPath: string, argv: string[]): Promise { sock.write(frame) } + const finish = (exitCode: number): void => { + sock.destroy() + process.exit(exitCode) + } + + const sendPostOutput = (postOutput: unknown): void => { + sock.write( + encodeJsonRpcFrame( + { + jsonrpc: '2.0', + id: postOutputRequestId, + method: 'orca.cli.postOutput', + params: { postOutput, env: pickRemoteCliEnv(process.env) } + }, + nextSeq++, + highestReceivedSeq + ) + ) + } + + const writeOutput = ( + result: { stdout?: unknown; stderr?: unknown }, + onFlushed: (error?: Error) => void + ): void => { + const writes = [ + [process.stdout, result.stdout], + [process.stderr, result.stderr] + ].filter((entry): entry is [NodeJS.WriteStream, string] => typeof entry[1] === 'string') + if (writes.length === 0) { + onFlushed() + return + } + let pending = writes.length + let completed = false + for (const [stream, output] of writes) { + stream.write(output, (error) => { + if (completed) { + return + } + if (error) { + completed = true + onFlushed(error) + return + } + pending -= 1 + if (pending === 0) { + completed = true + onFlushed() + } + }) + } + } + const decoder = new FrameDecoder((frame: DecodedFrame) => { if (frame.id > highestReceivedSeq) { highestReceivedSeq = frame.id @@ -227,28 +282,41 @@ async function runOrcaCliMode(sockPath: string, argv: string[]): Promise { return } const msg = parseJsonRpcMessage(frame.payload) - if (!('id' in msg) || msg.id !== requestId || !('result' in msg || 'error' in msg)) { + if ( + !('id' in msg) || + (msg.id !== requestId && msg.id !== postOutputRequestId) || + !('result' in msg || 'error' in msg) + ) { return } const response = msg as JsonRpcResponse if (response.error) { process.stderr.write(`${response.error.message}\n`) - sock.destroy() - process.exit(1) + finish(1) + return + } + if (response.id === postOutputRequestId) { + finish(initialExitCode) + return } const result = (response.result ?? {}) as { stdout?: unknown stderr?: unknown exitCode?: unknown + postOutput?: unknown } - if (typeof result.stdout === 'string' && result.stdout.length > 0) { - process.stdout.write(result.stdout) - } - if (typeof result.stderr === 'string' && result.stderr.length > 0) { - process.stderr.write(result.stderr) - } - sock.destroy() - process.exit(typeof result.exitCode === 'number' ? result.exitCode : 0) + initialExitCode = typeof result.exitCode === 'number' ? result.exitCode : 0 + writeOutput(result, (error) => { + if (error) { + finish(1) + return + } + if (result.postOutput === undefined) { + finish(initialExitCode) + return + } + sendPostOutput(result.postOutput) + }) }) const connectTimeout = setTimeout(() => { @@ -443,6 +511,12 @@ async function main(): Promise { timeoutMs: remoteCliRequestTimeoutMs(params) }) }) + dispatcher.onRequest('orca.cli.postOutput', async (params, context) => { + return await dispatcher.requestAnyClient('orca.cli.postOutput', params, { + excludeClientId: context.clientId, + timeoutMs: remoteCliRequestTimeoutMs(params) + }) + }) function configureRelayGraceTime(params: Record): { graceTimeMs: number } { const seconds = Number(params.graceTimeSeconds) diff --git a/src/renderer/src/hooks/useIpcEvents.test.ts b/src/renderer/src/hooks/useIpcEvents.test.ts index 04f3dbe3322..0765019ca83 100644 --- a/src/renderer/src/hooks/useIpcEvents.test.ts +++ b/src/renderer/src/hooks/useIpcEvents.test.ts @@ -1504,7 +1504,6 @@ describe('useIpcEvents updater integration', () => { openAgentTabsInChatByDefault: false } } - vi.doMock('react', async () => { const actual = await vi.importActual('react') return { @@ -1789,7 +1788,11 @@ describe('useIpcEvents updater integration', () => { }) it('surfaces terminal creates without stealing focus unless requested', async () => { - const createTab = vi.fn(() => ({ id: 'tab-new' })) + const createTab = vi.fn( + (_worktreeId: string, _groupId?: string, _tabType?: string, options?: { id?: string }) => ({ + id: options?.id ?? 'tab-new' + }) + ) const setActiveView = vi.fn() const setActiveWorktree = vi.fn() const markWorktreeVisited = vi.fn() @@ -1876,6 +1879,20 @@ describe('useIpcEvents updater integration', () => { activeRuntimeEnvironmentId: undefined as string | undefined } } + updateTabPtyId.mockImplementation((tabId: string, ptyId: string) => { + storeState.ptyIdsByTabId[tabId] = [ + ...new Set([...(storeState.ptyIdsByTabId[tabId] ?? []), ptyId]) + ] + for (const tabs of Object.values(storeState.tabsByWorktree)) { + const tab = tabs.find((candidate) => candidate.id === tabId) + if (tab) { + tab.ptyId = ptyId + } + } + }) + setTabLayout.mockImplementation((tabId: string, layout: unknown) => { + storeState.terminalLayoutsByTabId[tabId] = layout + }) const createTerminalListenerRef: { current: | ((data: { @@ -2280,6 +2297,7 @@ describe('useIpcEvents updater integration', () => { setActiveTabType.mockClear() setActiveTab.mockClear() revealWorktreeInSidebar.mockClear() + dispatchEvent.mockClear() createTerminalListenerRef.current({ worktreeId: 'wt-2', title: 'Runner', @@ -2588,6 +2606,7 @@ describe('useIpcEvents updater integration', () => { setActiveTab.mockClear() revealWorktreeInSidebar.mockClear() replyTerminalCreate.mockClear() + dispatchEvent.mockClear() requestTerminalCreateListenerRef.current({ requestId: 'req-renderer-backed-background', worktreeId: 'wt-2', @@ -2605,6 +2624,12 @@ describe('useIpcEvents updater integration', () => { expect(setActiveTabType).not.toHaveBeenCalled() expect(setActiveTab).not.toHaveBeenCalled() expect(revealWorktreeInSidebar).not.toHaveBeenCalled() + expect(dispatchEvent).toHaveBeenCalledWith( + expect.objectContaining({ + type: 'orca-background-mount-terminal-worktree', + detail: { worktreeId: 'wt-2', tabIds: ['tab-new'] } + }) + ) expect(replyTerminalCreate).toHaveBeenCalledWith({ requestId: 'req-renderer-backed-background', tabId: 'tab-new', @@ -2708,6 +2733,7 @@ describe('useIpcEvents updater integration', () => { setActiveTabType.mockClear() setActiveTab.mockClear() revealWorktreeInSidebar.mockClear() + dispatchEvent.mockClear() createTerminalListenerRef.current({ worktreeId: 'wt-2', ptyId: 'pty-bg-3', @@ -2725,6 +2751,13 @@ describe('useIpcEvents updater integration', () => { expect(setActiveTabType).not.toHaveBeenCalled() expect(setActiveTab).not.toHaveBeenCalled() expect(revealWorktreeInSidebar).not.toHaveBeenCalled() + expect(dispatchEvent).toHaveBeenCalledOnce() + expect(dispatchEvent).toHaveBeenCalledWith( + expect.objectContaining({ + type: 'orca-background-mount-terminal-worktree', + detail: { worktreeId: 'wt-2', tabIds: ['tab-cli-bg-reveal'] } + }) + ) createTab.mockClear() setActiveView.mockClear() @@ -2734,6 +2767,7 @@ describe('useIpcEvents updater integration', () => { revealWorktreeInSidebar.mockClear() focusRuntimeTerminalSurface.mockClear() focusTerminalTabSurface.mockClear() + dispatchEvent.mockClear() createTerminalListenerRef.current({ worktreeId: 'wt-2', ptyId: 'pty-recovery-bg', @@ -2754,6 +2788,13 @@ describe('useIpcEvents updater integration', () => { expect(revealWorktreeInSidebar).not.toHaveBeenCalled() expect(focusRuntimeTerminalSurface).not.toHaveBeenCalled() expect(focusTerminalTabSurface).not.toHaveBeenCalled() + expect(dispatchEvent).toHaveBeenCalledOnce() + expect(dispatchEvent).toHaveBeenCalledWith( + expect.objectContaining({ + type: 'orca-background-mount-terminal-worktree', + detail: { worktreeId: 'wt-2', tabIds: ['tab-recovery-bg'] } + }) + ) storeState.tabsByWorktree = { 'wt-2': [{ id: 'tab-existing', ptyId: 'pty-bg', title: 'Terminal 1' }] @@ -2836,7 +2877,13 @@ describe('useIpcEvents updater integration', () => { expect(replyTerminalCreate).toHaveBeenCalledWith({ requestId: 'req-adopt-pending', tabId: pendingTabId, - title: 'Terminal 3' + title: 'Terminal 3', + identity: { + worktreeId: 'wt-2', + tabId: pendingTabId, + leafId: pendingLeafId, + ptyId: 'serve-cf39bedb-a33a-417c-9ab6-f304dc27a6c0' + } }) storeState.tabsByWorktree = { @@ -2902,7 +2949,13 @@ describe('useIpcEvents updater integration', () => { expect(replyTerminalCreate).toHaveBeenCalledWith({ requestId: 'req-split', tabId: 'tab-existing', - title: 'Terminal 1' + title: 'Terminal 1', + identity: { + worktreeId: 'wt-2', + tabId: 'tab-existing', + leafId: 'leaf-split', + ptyId: 'pty-split' + } }) storeState.terminalLayoutsByTabId = { @@ -5234,7 +5287,7 @@ describe('useIpcEvents agent status snapshot integration', () => { vi.unstubAllGlobals() }) - it('reconciles adopted and exited legacy worker recovery without broad record cleanup', async () => { + it('retires the exact sleeping record after adopted or exited legacy worker recovery', async () => { const clearSleepingAgentSession = vi.fn() const setSleepingAgentAutomaticResumeBlocked = vi.fn() let listener: @@ -5275,11 +5328,8 @@ describe('useIpcEvents agent status snapshot integration', () => { clearSleepingAgentSession.mockClear() listener?.({ paneKey: 'tab-exited:leaf-exited', resolution: 'exited' }) - expect(clearSleepingAgentSession).not.toHaveBeenCalled() - expect(setSleepingAgentAutomaticResumeBlocked).toHaveBeenCalledWith( - 'tab-exited:leaf-exited', - false - ) + expect(clearSleepingAgentSession).toHaveBeenCalledWith('tab-exited:leaf-exited') + expect(setSleepingAgentAutomaticResumeBlocked).not.toHaveBeenCalled() }) it.each([ diff --git a/src/renderer/src/hooks/useIpcEvents.ts b/src/renderer/src/hooks/useIpcEvents.ts index 70b09770345..fffe3174676 100644 --- a/src/renderer/src/hooks/useIpcEvents.ts +++ b/src/renderer/src/hooks/useIpcEvents.ts @@ -101,6 +101,7 @@ import { track } from '@/lib/telemetry' import { singlePaneLayoutSnapshot } from '@/store/slices/terminal-helpers' import { buildWorkspaceSessionPayload } from '@/lib/workspace-session' import { persistWorkspaceSessionByHost } from '@/lib/workspace-session-host-persistence' +import { verifyTerminalRevealIdentity } from '@/lib/terminal-reveal-identity' import { getLinearIssueWorkspaceName } from '../../../shared/workspace-name' import type { RuntimeClientEvent } from '../../../shared/runtime-client-events' import { applyHostWorktreeTerminalSleepState } from '@/components/terminal-pane/pty-shutdown-exit-deferral' @@ -1576,11 +1577,24 @@ export function useIpcEvents(): void { ...(launchAgent ? { launchAgent } : {}) }) } + if (ptyId && terminalPresentation === 'background') { + requestBackgroundTerminalWorktreeMount({ worktreeId, tabIds: [tab.id] }) + } if (requestId) { + const identity = + ptyId && tabId && leafId + ? verifyTerminalRevealIdentity(useAppStore.getState(), { + worktreeId, + tabId, + leafId, + ptyId + }) + : undefined window.api.ui.replyTerminalCreate({ requestId, tabId: tab.id, - title: title ?? tab.title + title: title ?? tab.title, + ...(identity ? { identity } : {}) }) } } catch (err) { @@ -3348,13 +3362,8 @@ export function useIpcEvents(): void { unsubs.push(unsubscribeMigrationUnsupportedClear) } const unsubscribeLegacyWorkerTerminalRecovery = - window.api.agentStatus.onLegacyWorkerTerminalRecovery?.(({ paneKey, resolution }) => { - const store = useAppStore.getState() - if (resolution === 'adopted') { - store.clearSleepingAgentSession(paneKey) - return - } - store.setSleepingAgentAutomaticResumeBlocked(paneKey, false) + window.api.agentStatus.onLegacyWorkerTerminalRecovery?.(({ paneKey }) => { + useAppStore.getState().clearSleepingAgentSession(paneKey) }) if (unsubscribeLegacyWorkerTerminalRecovery) { unsubs.push(unsubscribeLegacyWorkerTerminalRecovery) diff --git a/src/renderer/src/lib/resume-sleeping-agent-session-legacy-worker.test.ts b/src/renderer/src/lib/resume-sleeping-agent-session-legacy-worker.test.ts index 4b5379caf7c..1ec95258f05 100644 --- a/src/renderer/src/lib/resume-sleeping-agent-session-legacy-worker.test.ts +++ b/src/renderer/src/lib/resume-sleeping-agent-session-legacy-worker.test.ts @@ -11,7 +11,7 @@ afterEach(() => { }) describe('legacy worker sleeping-session recovery', () => { - it('keeps the session dormant until exited recovery removes only its resume block', () => { + it('never resumes a proven-exited legacy worker on workspace activation', () => { const record: SleepingAgentSessionRecord = { paneKey: 'tab-legacy:leaf-legacy', tabId: 'tab-legacy', @@ -46,16 +46,8 @@ describe('legacy worker sleeping-session recovery', () => { expect(resumeSleepingAgentSessionsForWorktree('wt-legacy')).toBe(0) expect(useAppStore.getState().sleepingAgentSessionsByPaneKey[record.paneKey]).toBe(record) - useAppStore.getState().setSleepingAgentAutomaticResumeBlocked(record.paneKey, false) - const unblocked = useAppStore.getState().sleepingAgentSessionsByPaneKey[record.paneKey] - expect(unblocked).toMatchObject({ - paneKey: record.paneKey, - providerSession: record.providerSession, - origin: 'live' - }) - expect(unblocked?.automaticResumeBlockedBy).toBeUndefined() - - expect(resumeSleepingAgentSessionsForWorktree('wt-legacy')).toBe(1) + useAppStore.getState().clearSleepingAgentSession(record.paneKey) + expect(resumeSleepingAgentSessionsForWorktree('wt-legacy')).toBe(0) expect(useAppStore.getState().sleepingAgentSessionsByPaneKey[record.paneKey]).toBeUndefined() }) }) diff --git a/src/renderer/src/lib/terminal-reveal-identity.test.ts b/src/renderer/src/lib/terminal-reveal-identity.test.ts new file mode 100644 index 00000000000..48c0585d1a2 --- /dev/null +++ b/src/renderer/src/lib/terminal-reveal-identity.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from 'vitest' + +import { verifyTerminalRevealIdentity } from './terminal-reveal-identity' + +const expected = { + worktreeId: 'worktree-1', + tabId: 'tab-1', + leafId: 'leaf-1', + ptyId: 'pty-1' +} + +const mismatchCases: { + name: string + state: Parameters[0] +}[] = [ + { + name: 'worktree', + state: { + tabsByWorktree: { 'worktree-2': [{ id: 'tab-1' }] }, + terminalLayoutsByTabId: { + 'tab-1': { ptyIdsByLeafId: { 'leaf-1': 'pty-1' } } + } + } + }, + { + name: 'leaf', + state: { + tabsByWorktree: { 'worktree-1': [{ id: 'tab-1' }] }, + terminalLayoutsByTabId: { + 'tab-1': { ptyIdsByLeafId: { 'leaf-2': 'pty-1' } } + } + } + }, + { + name: 'pty', + state: { + tabsByWorktree: { 'worktree-1': [{ id: 'tab-1' }] }, + terminalLayoutsByTabId: { + 'tab-1': { ptyIdsByLeafId: { 'leaf-1': 'pty-2' } } + } + } + } +] + +describe('verifyTerminalRevealIdentity', () => { + it('returns the exact fresh renderer binding', () => { + expect( + verifyTerminalRevealIdentity( + { + tabsByWorktree: { 'worktree-1': [{ id: 'tab-1' }] }, + terminalLayoutsByTabId: { + 'tab-1': { ptyIdsByLeafId: { 'leaf-1': 'pty-1' } } + } + }, + expected + ) + ).toEqual(expected) + }) + + it.each(mismatchCases)('rejects a mismatched $name owner', ({ state }) => { + expect(() => verifyTerminalRevealIdentity(state, expected)).toThrow( + 'terminal_reveal_identity_mismatch' + ) + }) +}) diff --git a/src/renderer/src/lib/terminal-reveal-identity.ts b/src/renderer/src/lib/terminal-reveal-identity.ts new file mode 100644 index 00000000000..d352e96aedb --- /dev/null +++ b/src/renderer/src/lib/terminal-reveal-identity.ts @@ -0,0 +1,23 @@ +import type { TerminalRevealIdentity } from '../../../shared/terminal-reveal-identity' + +type TerminalRevealState = { + tabsByWorktree: Record + terminalLayoutsByTabId: Record< + string, + { ptyIdsByLeafId?: Record } | undefined + > +} + +export function verifyTerminalRevealIdentity( + state: TerminalRevealState, + expected: TerminalRevealIdentity +): TerminalRevealIdentity { + const ownsTab = state.tabsByWorktree[expected.worktreeId]?.some( + (tab) => tab.id === expected.tabId + ) + const boundPtyId = state.terminalLayoutsByTabId[expected.tabId]?.ptyIdsByLeafId?.[expected.leafId] + if (!ownsTab || boundPtyId !== expected.ptyId) { + throw new Error('terminal_reveal_identity_mismatch') + } + return expected +} diff --git a/src/shared/orchestration-check-output.test.ts b/src/shared/orchestration-check-output.test.ts new file mode 100644 index 00000000000..00ffbb79da3 --- /dev/null +++ b/src/shared/orchestration-check-output.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from 'vitest' +import { prepareOrchestrationCheckOutput } from './orchestration-check-output' + +describe('prepareOrchestrationCheckOutput', () => { + it('keeps mixed read-only mail safe and current Run replies executable', () => { + const prepared = prepareOrchestrationCheckOutput( + { + count: 2, + messages: [ + { + id: 'msg_current', + run_id: 'run_adopted', + delivery_contract: 'current_delivery', + from_handle: 'term_worker', + to_handle: 'run:run_adopted', + subject: 'Question' + }, + { + id: 'msg_legacy', + run_id: 'run_legacy_local', + delivery_contract: 'audit_only', + from_handle: 'term_legacy', + to_handle: 'term_coord', + subject: 'Old reply' + } + ], + formatted: '[Reply: unsafe stale formatter output]' + }, + 'term_current_coord', + true + ) + + expect(prepared.formatted).toContain( + '[Reply: orca orchestration reply --id msg_current --body "..."]' + ) + expect(prepared.formatted).not.toContain('--from run:run_adopted') + expect(prepared.formatted).toContain( + '[Inspection only: reply and acknowledgment are unavailable.]' + ) + expect(prepared.formatted).not.toContain('unsafe stale formatter output') + }) +}) diff --git a/src/shared/orchestration-check-output.ts b/src/shared/orchestration-check-output.ts new file mode 100644 index 00000000000..e736b92f4dd --- /dev/null +++ b/src/shared/orchestration-check-output.ts @@ -0,0 +1,199 @@ +import { ORCHESTRATION_LEGACY_RUN_ID } from './orchestration-rpc-contract' + +export type OrchestrationMessageSummary = { + id: string + run_id?: string + delivery_contract?: 'legacy_direct' | 'current_delivery' | 'audit_only' + from_handle: string + to_handle?: string + subject?: string + type?: string + body?: string + payload?: string | null + priority?: string + read?: number +} + +export type LegacyCompatibilityResult = { + recovery?: boolean + readOnly?: boolean + ackMessageIds?: string[] + answerAcknowledgement?: { + questionId: string + answerMessageId: string + } + currentDelivery?: { + runId: string + checkCommand: string + ackCommand: string + } + resumeRequired?: boolean + resumeCommand?: string +} + +export type OrchestrationCheckOutput = { + messages: OrchestrationMessageSummary[] + count: number + formatted?: string + deliveryId?: string | null + timedOut?: boolean + cancelled?: boolean + connectionLost?: boolean + legacyCompatibility?: LegacyCompatibilityResult +} + +export function formatMessageReadOnlyTag( + message: OrchestrationMessageSummary, + legacyCompatibilityActive = false +): string { + return isLegacyReadOnlyMessage(message, legacyCompatibilityActive) ? ' [legacy, read-only]' : '' +} + +export function isLegacyReadOnlyMessage( + message: OrchestrationMessageSummary, + legacyCompatibilityActive = false +): boolean { + return ( + message.run_id === ORCHESTRATION_LEGACY_RUN_ID || + (message.delivery_contract === 'legacy_direct' && !legacyCompatibilityActive) || + message.delivery_contract === 'audit_only' + ) +} + +export function formatOrchestrationCheckText( + result: OrchestrationCheckOutput, + checkedTerminal: string +): string { + const prepared = prepareOrchestrationCheckOutput( + result, + checkedTerminal, + result.formatted !== undefined + ) + const compatibilityActive = Boolean( + prepared.legacyCompatibility && !prepared.legacyCompatibility.readOnly + ) + const legacyHeader = prepared.legacyCompatibility + ? prepared.legacyCompatibility.readOnly + ? '[LEGACY READ-ONLY]\n' + : prepared.legacyCompatibility.recovery + ? '[LEGACY RECOVERY REPLAY — MAY HAVE BEEN SEEN]\n' + : '[LEGACY COMPATIBILITY]\n' + : '' + const deliveryNotice = formatCurrentDeliveryNotice(prepared.legacyCompatibility?.currentDelivery) + if (prepared.formatted) { + return `${legacyHeader}${prepared.formatted}${deliveryNotice}` + } + if (prepared.count === 0) { + if (prepared.timedOut) { + return `${legacyHeader}Wait timed out; no messages were consumed.${deliveryNotice}` + } + if (prepared.cancelled) { + const cancelled = prepared.connectionLost + ? 'Wait cancelled because the connection closed; no messages were consumed.' + : 'Wait cancelled; no messages were consumed.' + return `${legacyHeader}${cancelled}${deliveryNotice}` + } + return `${legacyHeader}No messages.${deliveryNotice}` + } + const rendered = prepared.messages + .map( + (message) => + `${message.id}${formatMessageReadOnlyTag( + message, + compatibilityActive + )} [${message.type ?? 'status'}] from=${message.from_handle} "${message.subject}"` + ) + .join('\n') + const output = prepared.deliveryId ? `Delivery ${prepared.deliveryId}\n${rendered}` : rendered + return `${legacyHeader}${output}${deliveryNotice}` +} + +export function prepareOrchestrationCheckOutput( + result: T, + checkedTerminal: string, + formattedRequested: boolean +): T { + const compatibilityActive = Boolean( + result.legacyCompatibility && !result.legacyCompatibility.readOnly + ) + if ( + !formattedRequested || + !result.messages.some((message) => isLegacyReadOnlyMessage(message, compatibilityActive)) + ) { + return result + } + return { + ...result, + formatted: formatLegacyAwareCheckMessages(result.messages, checkedTerminal, compatibilityActive) + } +} + +function formatMessagePriorityTag(message: OrchestrationMessageSummary): string { + return message.priority === 'urgent' ? ' [URGENT]' : message.priority === 'high' ? ' [HIGH]' : '' +} + +function escapeTerminalControlCharacters(value: string): string { + return [...value] + .map((character) => { + const code = character.charCodeAt(0) + if (character === '\n' || (code >= 0x20 && code < 0x7f) || code > 0x9f) { + return character + } + return `\\x${code.toString(16).padStart(2, '0')}` + }) + .join('') +} + +function formatQuotedMessageField(label: string, value?: string): string { + return `[${label}]\n${escapeTerminalControlCharacters(value ?? '') + .split('\n') + .map((line) => ` ${line}`) + .join('\n')}` +} + +function formatLegacyAwareCheckMessages( + messages: OrchestrationMessageSummary[], + checkedTerminal: string, + legacyCompatibilityActive = false +): string { + return messages + .map((message) => { + const legacyReadOnly = isLegacyReadOnlyMessage(message, legacyCompatibilityActive) + const lines = [ + `${message.id}${formatMessageReadOnlyTag(message, legacyCompatibilityActive)}${formatMessagePriorityTag(message)} [${message.type ?? 'status'}] from=${message.from_handle}`, + formatQuotedMessageField('subject', message.subject) + ] + if (legacyReadOnly) { + lines.push('[Inspection only: reply and acknowledgment are unavailable.]') + } + if (message.body) { + lines.push(formatQuotedMessageField('body', message.body)) + } + if (message.payload) { + lines.push(formatQuotedMessageField('payload', message.payload)) + } + if (!legacyReadOnly) { + const replyTarget = message.to_handle ?? checkedTerminal + const replyFrom = + replyTarget.startsWith('run:') || replyTarget.startsWith('dispatch:') + ? '' + : ` --from ${replyTarget}` + lines.push(`[Reply: orca orchestration reply --id ${message.id}${replyFrom} --body "..."]`) + } + return lines.join('\n') + }) + .join('\n\n') +} + +function formatCurrentDeliveryNotice( + delivery: LegacyCompatibilityResult['currentDelivery'] +): string { + if (!delivery) { + return '' + } + return ( + `\n[CURRENT RUN MAIL WAITING]\n` + + `Read: ${delivery.checkCommand}\n` + + `Then acknowledge the delivery ID it prints: ${delivery.ackCommand}` + ) +} diff --git a/src/shared/terminal-reveal-identity.ts b/src/shared/terminal-reveal-identity.ts new file mode 100644 index 00000000000..514e45e3b6d --- /dev/null +++ b/src/shared/terminal-reveal-identity.ts @@ -0,0 +1,14 @@ +export type TerminalRevealIdentity = { + worktreeId: string + tabId: string + leafId: string + ptyId: string +} + +export type TerminalTabCreateReply = { + requestId: string + tabId?: string + title?: string + identity?: TerminalRevealIdentity + error?: string +} diff --git a/tests/e2e/orchestration-legacy-worker-restart-recovery.spec.ts b/tests/e2e/orchestration-legacy-worker-restart-recovery.spec.ts index 5b5879d8684..b2ed89b1cd7 100644 --- a/tests/e2e/orchestration-legacy-worker-restart-recovery.spec.ts +++ b/tests/e2e/orchestration-legacy-worker-restart-recovery.spec.ts @@ -16,7 +16,10 @@ import { import { waitForActivePaneHookDescriptor, waitForActivePanePtyId } from './helpers/terminal' import { RuntimeClient } from '../../src/cli/runtime-client' import Database from '../../src/main/sqlite/sync-database' -import { LEGACY_CONTRACT_VERSION } from '../../src/main/runtime/orchestration/db' +import { + CURRENT_CONTRACT_VERSION, + LEGACY_CONTRACT_VERSION +} from '../../src/main/runtime/orchestration/db' import { DEFAULT_LOCAL_ORCA_PROFILE_ID } from '../../src/shared/orca-profiles' import type { RuntimeTerminalListResult, RuntimeTerminalRead } from '../../src/shared/runtime-types' @@ -200,6 +203,44 @@ function stripLegacyWorkerRendererBinding( writeFileSync(persistedDataPath(userDataDir), `${JSON.stringify(data, null, 2)}\n`, 'utf8') } +function assertDispatchRemainsCurrent( + userDataDir: string, + input: { + dispatchId: string + terminalHandle: string + paneKey: string + processIncarnation: string + worktreeId: string + } +): void { + const db = new Database(path.join(userDataDir, 'orchestration.db')) + try { + const authority = db + .prepare( + `SELECT dc.status AS dispatch_status, dc.assignee_handle, dc.assignee_pane_key, + dc.process_incarnation, dc.contract_version, dc.capability_hash, + wd.state AS worker_state, wd.worktree_id, wd.agent_terminal_handle + FROM dispatch_contexts dc + INNER JOIN worker_dispatches wd ON wd.dispatch_id = dc.id + WHERE dc.id = ?` + ) + .get(input.dispatchId) + expect(authority).toEqual({ + dispatch_status: 'dispatched', + assignee_handle: input.terminalHandle, + assignee_pane_key: input.paneKey, + process_incarnation: input.processIncarnation, + contract_version: CURRENT_CONTRACT_VERSION, + capability_hash: expect.any(String), + worker_state: 'ready', + worktree_id: input.worktreeId, + agent_terminal_handle: input.terminalHandle + }) + } finally { + db.close() + } +} + function markDispatchAsPreUpdateLegacy( userDataDir: string, input: { @@ -233,9 +274,9 @@ function markDispatchAsPreUpdateLegacy( }) db.prepare( `UPDATE dispatch_contexts - SET contract_version = ?, capability_hash = NULL, capability_revoked_at = NULL, - launch_token_hash = NULL - WHERE id = ?` + SET contract_version = ?, capability_hash = NULL, capability_revoked_at = NULL, + launch_token_hash = NULL + WHERE id = ?` ).run(LEGACY_CONTRACT_VERSION, input.dispatchId) } finally { db.close() @@ -248,285 +289,307 @@ test.afterAll(() => { rmSync(fakeCliDir, { recursive: true, force: true }) }) -test('adopts one live legacy worker after restart without replaying resume', async (// oxlint-disable-next-line no-empty-pattern -- This lifecycle test owns both Electron launches and intentionally opts out of the default app fixture. -{}, testInfo) => { - test.setTimeout(300_000) - const repoPath = existsSync(TEST_REPO_PATH_FILE) - ? readFileSync(TEST_REPO_PATH_FILE, 'utf8').trim() - : '' - test.skip(!repoPath || !existsSync(repoPath), 'Global setup did not produce a seeded test repo') +for (const contractVersion of [LEGACY_CONTRACT_VERSION, CURRENT_CONTRACT_VERSION]) { + const contractLabel = contractVersion === LEGACY_CONTRACT_VERSION ? 'legacy' : 'current' + test(`adopts one live ${contractLabel} worker after restart without replaying resume`, async (// oxlint-disable-next-line no-empty-pattern -- This lifecycle test owns both Electron launches and intentionally opts out of the default app fixture. + {}, testInfo) => { + test.setTimeout(300_000) + rmSync(spawnLedgerPath, { force: true }) + rmSync(interruptionLedgerPath, { force: true }) + const repoPath = existsSync(TEST_REPO_PATH_FILE) + ? readFileSync(TEST_REPO_PATH_FILE, 'utf8').trim() + : '' + test.skip(!repoPath || !existsSync(repoPath), 'Global setup did not produce a seeded test repo') - const session = createRestartSession(testInfo, { - PATH: `${fakeCliDir}${path.delimiter}${process.env.PATH ?? ''}`, - ORCA_E2E_SPAWN_LEDGER: spawnLedgerPath, - ORCA_E2E_INTERRUPTION_LEDGER: interruptionLedgerPath - }) - let firstApp: ElectronApplication | null = null - let secondApp: ElectronApplication | null = null + const session = createRestartSession(testInfo, { + PATH: `${fakeCliDir}${path.delimiter}${process.env.PATH ?? ''}`, + ORCA_E2E_SPAWN_LEDGER: spawnLedgerPath, + ORCA_E2E_INTERRUPTION_LEDGER: interruptionLedgerPath + }) + let firstApp: ElectronApplication | null = null + let secondApp: ElectronApplication | null = null - try { - const first = await session.launch() - firstApp = first.app - const worktreeId = await attachRepoAndOpenTerminal(first.page, repoPath) - await waitForSessionReady(first.page) - await ensureTerminalVisible(first.page) - const coordinatorTabId = await getActiveTabId(first.page) - expect(coordinatorTabId).toBeTruthy() - await waitForActivePanePtyId(first.page) - const coordinatorPane = await waitForActivePaneHookDescriptor(first.page) - const firstClient = new RuntimeClient(session.userDataDir, 30_000, null, null) - const coordinator = await firstClient.call<{ terminal: { handle: string } }>( - 'terminal.resolvePane', - { paneKey: coordinatorPane.paneKey } - ) - const coordinatorTerminal = await firstClient.call<{ - terminal: { worktreeId: string } - }>('terminal.show', { terminal: coordinator.result.terminal.handle }) - await expect - .poll(async () => { - const listed = await firstClient.call<{ worktrees: { id: string }[] }>('worktree.list', {}) - return listed.result.worktrees.some( - (candidate) => candidate.id === coordinatorTerminal.result.terminal.worktreeId - ) - }) - .toBe(true) - const run = await firstClient.call<{ run: { id: string } }>('orchestration.runCreate', { - objective: 'Legacy worker restart recovery', - from: coordinator.result.terminal.handle - }) - const task = await firstClient.call<{ task: { id: string } }>('orchestration.taskCreate', { - spec: 'Respond ACK and remain idle', - run: run.result.run.id, - callerTerminalHandle: coordinator.result.terminal.handle - }) - const started = await firstClient.call<{ - effects: { kind: string; role?: string; id?: string }[] - }>('orchestration.workerStart', { - task: task.result.task.id, - from: coordinator.result.terminal.handle, - agent: 'codex', - timeoutMs: 15_000 - }) - const workerHandle = started.result.effects.find( - (effect) => effect.kind === 'terminal' && effect.role === 'agent' - )?.id - expect(workerHandle).toBeTruthy() - - let worker = ( - await firstClient.call('terminal.list') - ).result.terminals.find((terminal) => terminal.title === 'Codex Ready') - await expect - .poll(async () => { - const listed = await firstClient.call('terminal.list') - worker = listed.result.terminals.find((terminal) => terminal.title === 'Codex Ready') - return worker?.ptyId ?? null - }) - .toBeTruthy() - expect(worker?.incarnationId).toBeTruthy() - const workerPaneKey = `${worker!.tabId}:${worker!.leafId}` - await expect - .poll(async () => { - const read = await firstClient.call<{ terminal: RuntimeTerminalRead }>('terminal.read', { - terminal: worker!.handle, - limit: 200 + try { + const first = await session.launch() + firstApp = first.app + const worktreeId = await attachRepoAndOpenTerminal(first.page, repoPath) + await waitForSessionReady(first.page) + await ensureTerminalVisible(first.page) + const coordinatorTabId = await getActiveTabId(first.page) + expect(coordinatorTabId).toBeTruthy() + await waitForActivePanePtyId(first.page) + const coordinatorPane = await waitForActivePaneHookDescriptor(first.page) + const firstClient = new RuntimeClient(session.userDataDir, 30_000, null, null) + const coordinator = await firstClient.call<{ terminal: { handle: string } }>( + 'terminal.resolvePane', + { paneKey: coordinatorPane.paneKey } + ) + const coordinatorTerminal = await firstClient.call<{ + terminal: { worktreeId: string } + }>('terminal.show', { terminal: coordinator.result.terminal.handle }) + await expect + .poll(async () => { + const listed = await firstClient.call<{ worktrees: { id: string }[] }>( + 'worktree.list', + {} + ) + return listed.result.worktrees.some( + (candidate) => candidate.id === coordinatorTerminal.result.terminal.worktreeId + ) }) - return read.result.terminal.tail.join('\n') + .toBe(true) + const run = await firstClient.call<{ run: { id: string } }>('orchestration.runCreate', { + objective: 'Legacy worker restart recovery', + from: coordinator.result.terminal.handle }) - .toContain('ACK') - const initialWorker = { - ptyId: worker!.ptyId, - incarnationId: worker!.incarnationId, - worktreeId: worker!.worktreeId, - tabId: worker!.tabId, - leafId: worker!.leafId - } - const initialDispatch = await firstClient.call<{ - dispatch: { - id: string - task_id: string - assignee_handle: string - assignee_pane_key: string - process_incarnation: string - } | null - }>('orchestration.dispatchShow', { task: task.result.task.id }) - expect(initialDispatch.result.dispatch).toEqual( - expect.objectContaining({ - task_id: task.result.task.id, - assignee_pane_key: workerPaneKey, - process_incarnation: `${initialWorker.ptyId}:${initialWorker.incarnationId}` + const task = await firstClient.call<{ task: { id: string } }>('orchestration.taskCreate', { + spec: 'Respond ACK and remain idle', + run: run.result.run.id, + callerTerminalHandle: coordinator.result.terminal.handle }) - ) - const dispatchHandle = initialDispatch.result.dispatch!.assignee_handle - await expect.poll(() => readLedger(spawnLedgerPath)).toHaveLength(1) - const [initialSpawn] = readLedger(spawnLedgerPath) - expect(isProcessAlive(initialSpawn.pid)).toBe(true) - expect(readLedger(interruptionLedgerPath)).toEqual([]) + const started = await firstClient.call<{ + effects: { kind: string; role?: string; id?: string }[] + }>('orchestration.workerStart', { + task: task.result.task.id, + from: coordinator.result.terminal.handle, + agent: 'codex', + timeoutMs: 15_000 + }) + const workerHandle = started.result.effects.find( + (effect) => effect.kind === 'terminal' && effect.role === 'agent' + )?.id + expect(workerHandle).toBeTruthy() - const transcriptPath = session.seedCodexResumeRollout(PROVIDER_SESSION_ID, repoPath) - await first.page.evaluate( - ({ paneKey, tabId, worktreeId: workerWorktreeId, terminalHandle, transcript }) => { - window.__store?.getState().setAgentStatus( - paneKey, - { state: 'working', prompt: 'Respond ACK and remain idle', agentType: 'codex' }, - 'Codex Ready', - undefined, - { tabId, worktreeId: workerWorktreeId, terminalHandle }, - { - providerSession: { - key: 'session_id', - id: 'e2e-legacy-orchestration-worker', - transcriptPath: transcript - }, - launchConfig: { - agentCommand: 'codex', - agentArgs: '--dangerously-bypass-approvals-and-sandbox', - agentEnv: {} - } - } - ) - window.__store?.getState().captureAllSleepingAgentSessions('quit') - }, - { - paneKey: workerPaneKey, - tabId: worker!.tabId, + let worker = ( + await firstClient.call('terminal.list') + ).result.terminals.find((terminal) => terminal.title === 'Codex Ready') + await expect + .poll(async () => { + const listed = await firstClient.call('terminal.list') + worker = listed.result.terminals.find((terminal) => terminal.title === 'Codex Ready') + return worker?.ptyId ?? null + }) + .toBeTruthy() + expect(worker?.incarnationId).toBeTruthy() + const workerPaneKey = `${worker!.tabId}:${worker!.leafId}` + await expect + .poll(async () => { + const read = await firstClient.call<{ terminal: RuntimeTerminalRead }>('terminal.read', { + terminal: worker!.handle, + limit: 200 + }) + return read.result.terminal.tail.join('\n') + }) + .toContain('ACK') + const initialWorker = { + ptyId: worker!.ptyId, + incarnationId: worker!.incarnationId, worktreeId: worker!.worktreeId, - terminalHandle: worker!.handle, - transcript: transcriptPath + tabId: worker!.tabId, + leafId: worker!.leafId } - ) - await expect - .poll(() => hasPersistedResumeRecord(session.userDataDir, workerPaneKey), { - timeout: 30_000 - }) - .toBe(true) - - await session.close(firstApp) - firstApp = null - expect(readLedger(spawnLedgerPath)).toEqual([initialSpawn]) - expect(readLedger(interruptionLedgerPath)).toEqual([]) - expect(isProcessAlive(initialSpawn.pid)).toBe(true) - - stripLegacyWorkerRendererBinding(session.userDataDir, { - worktreeId, - coordinatorTabId: coordinatorTabId!, - workerTabId: worker!.tabId, - workerPaneKey - }) - markDispatchAsPreUpdateLegacy(session.userDataDir, { - dispatchId: initialDispatch.result.dispatch!.id, - terminalHandle: dispatchHandle, - paneKey: workerPaneKey, - processIncarnation: `${initialWorker.ptyId}:${initialWorker.incarnationId}`, - worktreeId: initialWorker.worktreeId - }) - - const second = await session.launch() - secondApp = second.app - await waitForSessionReady(second.page) - expect(await waitForActiveWorktree(second.page)).toBe(worktreeId) - const secondClient = new RuntimeClient(session.userDataDir, 30_000, null, null) - let recovered = ( - await secondClient.call('terminal.list') - ).result.terminals.find((terminal) => terminal.ptyId === initialWorker.ptyId) - await expect - .poll(async () => { - const listed = await secondClient.call('terminal.list') - const matches = listed.result.terminals.filter( - (terminal) => terminal.ptyId === initialWorker.ptyId - ) - recovered = matches[0] - return matches - }) - .toEqual([ + const initialDispatch = await firstClient.call<{ + dispatch: { + id: string + task_id: string + assignee_handle: string + assignee_pane_key: string + process_incarnation: string + } | null + }>('orchestration.dispatchShow', { task: task.result.task.id }) + expect(initialDispatch.result.dispatch).toEqual( expect.objectContaining({ - ...initialWorker, - connected: true, - writable: true + task_id: task.result.task.id, + assignee_pane_key: workerPaneKey, + process_incarnation: `${initialWorker.ptyId}:${initialWorker.incarnationId}` }) - ]) + ) + const dispatchHandle = initialDispatch.result.dispatch!.assignee_handle + await expect.poll(() => readLedger(spawnLedgerPath)).toHaveLength(1) + const [initialSpawn] = readLedger(spawnLedgerPath) + expect(isProcessAlive(initialSpawn.pid)).toBe(true) + expect(readLedger(interruptionLedgerPath)).toEqual([]) - const recoveredTab = second.page.locator( - `[data-testid="sortable-tab"][data-tab-id="${initialWorker.tabId}"]` - ) - await expect(recoveredTab).toBeVisible() - await expect(recoveredTab).toHaveCount(1) - await expect(recoveredTab).toHaveAttribute('data-active', 'false') - await expect( - second.page.locator(`[data-testid="sortable-tab"][data-tab-id="${coordinatorTabId!}"]`) - ).toHaveAttribute('data-active', 'true') - await expect - .poll(async () => { - const read = await secondClient.call<{ terminal: RuntimeTerminalRead }>('terminal.read', { - terminal: recovered!.handle, - limit: 200 + const transcriptPath = session.seedCodexResumeRollout(PROVIDER_SESSION_ID, repoPath) + await first.page.evaluate( + ({ paneKey, tabId, worktreeId: workerWorktreeId, terminalHandle, transcript }) => { + window.__store?.getState().setAgentStatus( + paneKey, + { state: 'working', prompt: 'Respond ACK and remain idle', agentType: 'codex' }, + 'Codex Ready', + undefined, + { tabId, worktreeId: workerWorktreeId, terminalHandle }, + { + providerSession: { + key: 'session_id', + id: 'e2e-legacy-orchestration-worker', + transcriptPath: transcript + }, + launchConfig: { + agentCommand: 'codex', + agentArgs: '--dangerously-bypass-approvals-and-sandbox', + agentEnv: {} + } + } + ) + window.__store?.getState().captureAllSleepingAgentSessions('quit') + }, + { + paneKey: workerPaneKey, + tabId: worker!.tabId, + worktreeId: worker!.worktreeId, + terminalHandle: worker!.handle, + transcript: transcriptPath + } + ) + await expect + .poll(() => hasPersistedResumeRecord(session.userDataDir, workerPaneKey), { + timeout: 30_000 }) - return read.result.terminal.tail.join('\n') - }) - .toContain('ACK') + .toBe(true) - const restoredRun = await secondClient.call<{ run: { id: string } }>('orchestration.runShow', { - id: run.result.run.id - }) - expect(restoredRun.result.run.id).toBe(run.result.run.id) - const tasks = await secondClient.call<{ tasks: { id: string }[] }>('orchestration.taskList', { - run: run.result.run.id - }) - expect(tasks.result.tasks).toEqual( - expect.arrayContaining([expect.objectContaining({ id: task.result.task.id })]) - ) - const recoveredDispatch = await secondClient.call<{ - dispatch: { - id: string - task_id: string - assignee_handle: string - assignee_pane_key: string - process_incarnation: string - contract_version: number - } | null - }>('orchestration.dispatchShow', { task: task.result.task.id }) - expect(recoveredDispatch.result.dispatch).toEqual( - expect.objectContaining({ - id: initialDispatch.result.dispatch!.id, - task_id: task.result.task.id, - assignee_handle: dispatchHandle, - assignee_pane_key: workerPaneKey, - process_incarnation: `${initialWorker.ptyId}:${initialWorker.incarnationId}`, - contract_version: LEGACY_CONTRACT_VERSION - }) - ) - await expect - .poll(async () => ({ - renderer: await readRendererRecoveryState(second.page, workerPaneKey, initialWorker.tabId), - persisted: hasPersistedResumeRecord(session.userDataDir, workerPaneKey) - })) - .toEqual({ - renderer: { sleeping: false, resumeClaim: false, pendingStartup: false }, - persisted: false - }) - expect(readLedger(spawnLedgerPath)).toEqual([initialSpawn]) - expect(readLedger(interruptionLedgerPath)).toEqual([]) - expect(isProcessAlive(initialSpawn.pid)).toBe(true) + await session.close(firstApp) + firstApp = null + expect(readLedger(spawnLedgerPath)).toEqual([initialSpawn]) + expect(readLedger(interruptionLedgerPath)).toEqual([]) + expect(isProcessAlive(initialSpawn.pid)).toBe(true) - const otherWorktreeId = await switchToOtherWorktree(second.page, worktreeId) - expect(otherWorktreeId).toBeTruthy() - await switchToWorktree(second.page, worktreeId) - await expect(recoveredTab).toBeVisible() - await expect(recoveredTab).toHaveCount(1) - await expect(recoveredTab).toHaveAttribute('data-active', 'false') - await expect - .poll(async () => readRendererRecoveryState(second.page, workerPaneKey, initialWorker.tabId)) - .toEqual({ sleeping: false, resumeClaim: false, pendingStartup: false }) - expect(readLedger(spawnLedgerPath)).toEqual([initialSpawn]) - expect(readLedger(interruptionLedgerPath)).toEqual([]) - expect(isProcessAlive(initialSpawn.pid)).toBe(true) - await expect(second.page.locator('body')).not.toContainText('Conversation interrupted') - } finally { - if (secondApp) { - await session.close(secondApp).catch(() => undefined) + stripLegacyWorkerRendererBinding(session.userDataDir, { + worktreeId, + coordinatorTabId: coordinatorTabId!, + workerTabId: worker!.tabId, + workerPaneKey + }) + const dispatchIdentity = { + dispatchId: initialDispatch.result.dispatch!.id, + terminalHandle: dispatchHandle, + paneKey: workerPaneKey, + processIncarnation: `${initialWorker.ptyId}:${initialWorker.incarnationId}`, + worktreeId: initialWorker.worktreeId + } + if (contractVersion === LEGACY_CONTRACT_VERSION) { + markDispatchAsPreUpdateLegacy(session.userDataDir, dispatchIdentity) + } else { + assertDispatchRemainsCurrent(session.userDataDir, dispatchIdentity) + } + + const second = await session.launch() + secondApp = second.app + await waitForSessionReady(second.page) + expect(await waitForActiveWorktree(second.page)).toBe(worktreeId) + const secondClient = new RuntimeClient(session.userDataDir, 30_000, null, null) + let recovered = ( + await secondClient.call('terminal.list') + ).result.terminals.find((terminal) => terminal.ptyId === initialWorker.ptyId) + await expect + .poll(async () => { + const listed = await secondClient.call('terminal.list') + const matches = listed.result.terminals.filter( + (terminal) => terminal.ptyId === initialWorker.ptyId + ) + recovered = matches[0] + return matches + }) + .toEqual([ + expect.objectContaining({ + ...initialWorker, + connected: true, + writable: true + }) + ]) + + const recoveredTab = second.page.locator( + `[data-testid="sortable-tab"][data-tab-id="${initialWorker.tabId}"]` + ) + await expect(recoveredTab).toBeVisible() + await expect(recoveredTab).toHaveCount(1) + await expect(recoveredTab).toHaveAttribute('data-active', 'false') + await expect( + second.page.locator(`[data-testid="sortable-tab"][data-tab-id="${coordinatorTabId!}"]`) + ).toHaveAttribute('data-active', 'true') + await expect + .poll(async () => { + const read = await secondClient.call<{ terminal: RuntimeTerminalRead }>('terminal.read', { + terminal: recovered!.handle, + limit: 200 + }) + return read.result.terminal.tail.join('\n') + }) + .toContain('ACK') + + const restoredRun = await secondClient.call<{ run: { id: string } }>( + 'orchestration.runShow', + { + id: run.result.run.id + } + ) + expect(restoredRun.result.run.id).toBe(run.result.run.id) + const tasks = await secondClient.call<{ tasks: { id: string }[] }>('orchestration.taskList', { + run: run.result.run.id + }) + expect(tasks.result.tasks).toEqual( + expect.arrayContaining([expect.objectContaining({ id: task.result.task.id })]) + ) + const recoveredDispatch = await secondClient.call<{ + dispatch: { + id: string + task_id: string + assignee_handle: string + assignee_pane_key: string + process_incarnation: string + contract_version: number + } | null + }>('orchestration.dispatchShow', { task: task.result.task.id }) + expect(recoveredDispatch.result.dispatch).toEqual( + expect.objectContaining({ + id: initialDispatch.result.dispatch!.id, + task_id: task.result.task.id, + assignee_handle: dispatchHandle, + assignee_pane_key: workerPaneKey, + process_incarnation: `${initialWorker.ptyId}:${initialWorker.incarnationId}`, + contract_version: contractVersion + }) + ) + await expect + .poll(async () => ({ + renderer: await readRendererRecoveryState( + second.page, + workerPaneKey, + initialWorker.tabId + ), + persisted: hasPersistedResumeRecord(session.userDataDir, workerPaneKey) + })) + .toEqual({ + renderer: { sleeping: false, resumeClaim: false, pendingStartup: false }, + persisted: false + }) + expect(readLedger(spawnLedgerPath)).toEqual([initialSpawn]) + expect(readLedger(interruptionLedgerPath)).toEqual([]) + expect(isProcessAlive(initialSpawn.pid)).toBe(true) + + const otherWorktreeId = await switchToOtherWorktree(second.page, worktreeId) + expect(otherWorktreeId).toBeTruthy() + await switchToWorktree(second.page, worktreeId) + await expect(recoveredTab).toBeVisible() + await expect(recoveredTab).toHaveCount(1) + await expect(recoveredTab).toHaveAttribute('data-active', 'false') + await expect + .poll(async () => + readRendererRecoveryState(second.page, workerPaneKey, initialWorker.tabId) + ) + .toEqual({ sleeping: false, resumeClaim: false, pendingStartup: false }) + expect(readLedger(spawnLedgerPath)).toEqual([initialSpawn]) + expect(readLedger(interruptionLedgerPath)).toEqual([]) + expect(isProcessAlive(initialSpawn.pid)).toBe(true) + await expect(second.page.locator('body')).not.toContainText('Conversation interrupted') + } finally { + if (secondApp) { + await session.close(secondApp).catch(() => undefined) + } + if (firstApp) { + await session.close(firstApp).catch(() => undefined) + } + await session.dispose() } - if (firstApp) { - await session.close(firstApp).catch(() => undefined) - } - await session.dispose() - } -}) + }) +}