diff --git a/.github/workflows/win-update-e2e.yml b/.github/workflows/win-update-e2e.yml index 21c788256b2..58a27b82d4c 100644 --- a/.github/workflows/win-update-e2e.yml +++ b/.github/workflows/win-update-e2e.yml @@ -128,6 +128,7 @@ jobs: node tools/win-update-e2e/run.mjs ` --from "$env:FROM_EXE" ` --to "$env:TO_EXE" ` + --require-distinct-artifacts ` --expect "$env:EXPECT" ` --soak-seconds "$env:SOAK_SECONDS" 2>&1 | Tee-Object -FilePath $log exit $LASTEXITCODE diff --git a/config/nsis/daemon-host-uninstall.nsh b/config/nsis/daemon-host-uninstall.nsh index dc3a497ce67..fc3ebf4642a 100644 --- a/config/nsis/daemon-host-uninstall.nsh +++ b/config/nsis/daemon-host-uninstall.nsh @@ -13,6 +13,26 @@ ; The image name and the LOCALAPPDATA folder name must stay in sync with ; DAEMON_HOST_EXE_NAME and LOCAL_HOST_ROOT_NAME in ; src/main/daemon/daemon-host-relocation.ts. + +!include "getProcessInfo.nsh" +Var pid + +!macro customCheckAppRunning + ${if} ${isUpdated} + ; Why: an A-era packaged CLI may still hold Orca.exe while its exact native launcher waits. + nsExec::Exec `"$PowerShellPath" -NoProfile -NonInteractive -ExecutionPolicy Bypass -Command "& { $$ErrorActionPreference='Stop'; $$app=[IO.Path]::GetFullPath($$args[0]); $$launcher=[IO.Path]::GetFullPath($$args[1]); $$comparison=[StringComparison]::OrdinalIgnoreCase; $$children=@(Get-CimInstance Win32_Process | Where-Object { $$_.ExecutablePath -and [String]::Equals([IO.Path]::GetFullPath($$_.ExecutablePath), $$app, $$comparison) }); $$launcherProofs=@(); foreach ($$child in $$children) { $$parent=Get-CimInstance Win32_Process -Filter ('ProcessId=' + $$child.ParentProcessId) -ErrorAction SilentlyContinue; if ($$parent -and $$parent.ExecutablePath -and [String]::Equals([IO.Path]::GetFullPath($$parent.ExecutablePath), $$launcher, $$comparison)) { $$launcherProofs += [pscustomobject]@{ ProcessId=[int]$$parent.ProcessId; CreationDate=[string]$$parent.CreationDate } }; Stop-Process -Id $$child.ProcessId -Force -ErrorAction SilentlyContinue }; $$launcherDeadline=[DateTime]::UtcNow.AddSeconds(30); do { $$remaining=@($$launcherProofs | Where-Object { $$proof=$$_; $$current=Get-CimInstance Win32_Process -Filter ('ProcessId=' + $$proof.ProcessId) -ErrorAction SilentlyContinue; $$current -and $$current.ExecutablePath -and [String]::Equals([IO.Path]::GetFullPath($$current.ExecutablePath), $$launcher, $$comparison) -and [String]::Equals([string]$$current.CreationDate, $$proof.CreationDate, [StringComparison]::Ordinal) }); if ($$remaining.Count -eq 0) { break }; Start-Sleep -Milliseconds 100 } while ([DateTime]::UtcNow -lt $$launcherDeadline); if ($$remaining.Count -ne 0) { exit 20 }; $$lockDeadline=[DateTime]::UtcNow.AddSeconds(30); $$released=$$false; do { try { $$stream=[IO.File]::Open($$app, [IO.FileMode]::Open, [IO.FileAccess]::ReadWrite, [IO.FileShare]::None); $$stream.Dispose(); $$released=$$true } catch { Start-Sleep -Milliseconds 100 } } while (-not $$released -and [DateTime]::UtcNow -lt $$lockDeadline); if (-not $$released) { exit 21 } }" "$INSTDIR\Orca.exe" "$INSTDIR\resources\bin\orca.exe"` + Pop $0 + ${if} $0 != 0 + DetailPrint "Orca update release proof failed with status $0." + SetErrorLevel 2 + Quit + ${endIf} + ${else} + !insertmacro IS_POWERSHELL_AVAILABLE + !insertmacro _CHECK_APP_RUNNING + ${endIf} +!macroend + !macro customUnInstall ${ifNot} ${isUpdated} nsExec::Exec 'taskkill /F /IM orca-terminal-daemon.exe' diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index a6f4f2d22f5..7d3bd1717ef 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -5232,7 +5232,7 @@ }, { "id": "orchestration.worker-terminal-delivery", - "title": "Started workers are visible without stealing focus and retain mailbox identity", + "title": "Workers remain visible and observable across workspace entry and app restart", "maturity": "experimental", "protection": "partial", "owner": "orchestration", @@ -5242,7 +5242,9 @@ "worker-start", "terminal creation", "terminal tab materialization", - "workspace re-entry" + "workspace re-entry", + "app restart with retained daemon PTYs", + "legacy update resume fencing" ], "platforms": [ "macos", @@ -5264,25 +5266,58 @@ "daemon", "ssh" ], - "coverageNotes": "Deterministic units cover local worker presentation, reveal-failure warnings, stable-pane Run/Dispatch routing, and the SSH in-process CLI fallback. An isolated macOS Electron journey launches a fake Codex worker through the real RPC path, tolerates spawn-time handle reminting, asserts the inactive tab in the DOM before navigation, checks Run delivery by pane identity, and proves workspace re-entry keeps one worker tab by both original tab ID and visible title. SSH, WSL, remote-runtime, Linux, and Windows remain live-test gaps; federated workers retain explicit background presentation.", + "coverageNotes": "Deterministic units cover authority-aware legacy formatting, exact legacy worker identity planning, local worker presentation, retained-output reads after adoption, reveal-failure warnings, stable-pane Run/Dispatch routing, the SSH in-process CLI fallback, federated non-reveal, and fail-closed distinct Windows package proof. Two isolated macOS Electron journeys launch fake Codex workers through the real RPC path and record append-only spawn/interruption ledgers. They assert immediate inactive presentation, one live agent PID, stable PTY/incarnation/tab/leaf/worktree/Task/Dispatch identity, and no interruption after workspace re-entry; the restart journey additionally removes renderer ownership, marks the Dispatch legacy, retains the daemon process across an app restart, and proves exact background adoption with readable ACK output and no resume replay. Distinct A/B artifacts plus live SSH, WSL, folder, remote-runtime, Linux, and Windows cutover journeys remain explicit gaps.", "motivatingLinks": [ "https://github.com/stablyai/orca/pull/11107#discussion_r3663321387" ], - "invariant": "Starting a worker in the coordinator's current workspace must materialize one inactive terminal tab before worker-start returns, preserve coordinator focus, and remain exactly once after workspace re-entry. An exact existing target workspace must receive a discoverable tab without stealing coordinator focus; if renderer reveal fails, worker-start must expose that the live worker remains background-only. Run and Dispatch checks must resolve through the caller's stable pane identity when a terminal handle is reminted, while a live handle outranks mismatched pane metadata, explicit legacy terminal inspection remains handle-scoped, and remote or headless worker presentation remains background-only.", - "oracle": "Drive Run create, Task create, and worker-start through a production Electron runtime with a deterministic Codex fixture. Require the worker tab to be visible in the DOM with data-active=false while the coordinator tab stays active, send ACK to the Run, read it with a deliberately stale coordinator handle plus its stable pane key, switch workspaces away and back, and require exactly one worker tab by original ID and visible title. Unit tests separately assert local worker-start omits background presentation, reveal failures return an actionable warning without discarding the worker, reminted coordinators and workers retain mailbox routing, live handles outrank mismatched pane metadata, and explicit legacy terminal checks do not inherit the caller pane key locally or through the SSH fallback.", + "invariant": "Starting a worker in the coordinator's current workspace must materialize one inactive terminal tab before worker-start returns, preserve coordinator focus, and remain exactly once after workspace re-entry. After an app update or restart, an exact live legacy worker must fence automatic provider resume, adopt its original PTY into its original background pane, retain readable output, and clear the resume record without spawning, writing, signalling, interrupting, replacing, or focusing the worker. An exact existing target workspace must receive a discoverable tab without stealing coordinator focus; if renderer reveal fails, worker-start must expose that the live worker remains background-only. Run and Dispatch checks must resolve through the caller's stable pane identity when a terminal handle is reminted, while a live handle outranks mismatched pane metadata, explicit legacy terminal inspection remains handle-scoped, and remote or headless worker presentation remains background-only.", + "oracle": "Drive Run create, Task create, and worker-start through production Electron runtimes with a deterministic Codex fixture. Require append-only ledgers with one still-live PID and no interruption, a visible inactive worker tab while the coordinator stays active, Run delivery through stable pane identity, and stable PTY/incarnation, tab, leaf, worktree, Task, and Dispatch across workspace re-entry. In a restart journey, retain the original daemon PTY and PID, remove renderer ownership, retain sleeping-session evidence, mark the Dispatch legacy, relaunch, and require exact inactive tab adoption, readable ACK output, cleared resume state, one spawn, and no resume argv or Conversation interrupted text after another workspace round trip. Unit tests separately assert authority-specific legacy affordances, exact identity and owner matching, retained-output fallback, pane-stable routing, federated non-activation, SSH fallback parity, and distinct packaged installer paths, versions, and hashes.", "commands": [ "pnpm exec vitest run --config config/vitest.config.ts src/cli/handlers/orchestration.test.ts src/cli/handlers/orchestration-check-identity.test.ts src/cli/handlers/orchestration-worker-cli.test.ts src/main/runtime/rpc/methods/orchestration.test.ts src/main/ssh/ssh-remote-orca-cli.test.ts", - "pnpm run test:e2e -- tests/e2e/orchestration-worker-terminal-visibility.spec.ts --workers=1" + "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orchestration/formatter.test.ts src/main/runtime/rpc/methods/orchestration-federation.test.ts tools/win-update-e2e/installer-pair-proof.test.mjs", + "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orchestration/orchestration-legacy-worker-terminal-recovery.test.ts", + "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts", + "pnpm run test:e2e -- tests/e2e/orchestration-worker-terminal-visibility.spec.ts --workers=1", + "pnpm run test:e2e -- tests/e2e/orchestration-legacy-worker-restart-recovery.spec.ts --workers=1" ], "testFiles": [ + "src/main/runtime/orchestration/formatter.test.ts", + "src/main/runtime/orchestration/orchestration-legacy-worker-terminal-recovery.test.ts", + "src/main/runtime/orca-runtime.test.ts", "src/cli/handlers/orchestration.test.ts", "src/cli/handlers/orchestration-check-identity.test.ts", "src/cli/handlers/orchestration-worker-cli.test.ts", "src/main/runtime/rpc/methods/orchestration.test.ts", + "src/main/runtime/rpc/methods/orchestration-federation.test.ts", "src/main/ssh/ssh-remote-orca-cli.test.ts", - "tests/e2e/orchestration-worker-terminal-visibility.spec.ts" + "tests/e2e/orchestration-worker-terminal-visibility.spec.ts", + "tests/e2e/orchestration-legacy-worker-restart-recovery.spec.ts", + "tools/win-update-e2e/installer-pair-proof.test.mjs" ], "assertionRefs": [ + { + "file": "src/main/runtime/orchestration/orchestration-legacy-worker-terminal-recovery.test.ts", + "assertions": [ + "only exact unique terminal, pane, process-incarnation, and worktree evidence becomes recoverable", + "ambiguous, incomplete, or mismatched legacy identities remain fenced and deferred" + ] + }, + { + "file": "src/main/runtime/orca-runtime.test.ts", + "assertions": [ + "one exact live legacy worker is adopted into its original background pane without input or signals", + "automatic provider resume stays fenced until exact adoption is persisted", + "retained renderer output remains readable through the recovered terminal" + ] + }, + { + "file": "src/main/runtime/orchestration/formatter.test.ts", + "assertions": [ + "legacy compatibility and recovery replay show only runtime-supplied supported actions", + "legacy provenance without live authority stays read-only", + "current formatting remains unchanged" + ] + }, { "file": "src/cli/handlers/orchestration-check-identity.test.ts", "assertions": [ @@ -5306,6 +5341,12 @@ "a live handle cannot be retargeted by mismatched pane metadata" ] }, + { + "file": "src/main/runtime/rpc/methods/orchestration-federation.test.ts", + "assertions": [ + "federated worker placement explicitly sets activate=false" + ] + }, { "file": "src/main/ssh/ssh-remote-orca-cli.test.ts", "assertions": [ @@ -5319,7 +5360,25 @@ "worker-start exposes one inactive worker tab before workspace navigation", "the coordinator tab remains active", "ACK delivery reaches a stable coordinator pane through a stale handle", - "workspace re-entry does not duplicate the worker tab under the same or a new tab ID" + "one spawn remains live with no interruption event", + "PTY/incarnation, tab, leaf, worktree, Task, and Dispatch identities remain stable", + "workspace re-entry does not duplicate the worker tab or print Conversation interrupted" + ] + }, + { + "file": "tests/e2e/orchestration-legacy-worker-restart-recovery.spec.ts", + "assertions": [ + "the original daemon PTY, process incarnation, PID, pane, Task, and Dispatch survive app restart", + "the legacy worker tab is restored once in the background with retained ACK output", + "sleeping-session, resume-claim, and pending-startup state are cleared after adoption", + "no second spawn, resume argv, input, signal, interruption, or duplicate tab occurs after workspace re-entry" + ] + }, + { + "file": "tools/win-update-e2e/installer-pair-proof.test.mjs", + "assertions": [ + "distinct package proof records different paths, versions, and SHA-256 hashes", + "identical paths, bytes, versions, or unreadable versions fail closed" ] } ], @@ -5333,43 +5392,66 @@ "durationSeconds": 5.27, "summary": "Five focused files passed with 216 tests, covering visible inactive local worker creation, reveal-failure warnings, stable-pane mailbox routing, live-handle precedence, and SSH fallback parity." }, + { + "date": "2026-07-28", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orchestration/formatter.test.ts src/main/runtime/rpc/methods/orchestration-federation.test.ts tools/win-update-e2e/installer-pair-proof.test.mjs", + "result": "passed", + "durationSeconds": 2.72, + "summary": "Three focused files passed with 39 tests, covering authority-aware legacy affordances, federated non-reveal, and fail-closed distinct packaged installer proof." + }, { "date": "2026-07-28", "runner": "local", "platform": "macos", "command": "pnpm run test:e2e -- tests/e2e/orchestration-worker-terminal-visibility.spec.ts --workers=1", "result": "passed", - "durationSeconds": 11.5, - "summary": "The isolated Electron journey passed with a real Run and Task, deterministic Codex PTY, immediate inactive DOM tab, ACK delivery through a stale coordinator handle, and exactly one worker tab by original ID and management title after workspace re-entry." + "durationSeconds": 8.3, + "summary": "The isolated Electron journey passed with one live fake-agent spawn, no interruption events, stable PTY/incarnation/tab/leaf/worktree/Task/Dispatch identity, immediate inactive presentation, pane-stable ACK delivery, and exactly one tab after workspace re-entry." + }, + { + "date": "2026-07-28", + "runner": "local", + "platform": "macos", + "command": "pnpm run test:e2e -- tests/e2e/orchestration-legacy-worker-restart-recovery.spec.ts --workers=1", + "result": "passed", + "durationSeconds": 12.6, + "summary": "The restart journey retained one daemon PTY and PID, adopted its exact original background pane, preserved readable ACK output and Task/Dispatch authority, cleared all resume state, and recorded no second spawn, resume argv, input, signal, interruption, or duplicate after workspace re-entry." } ], "runtimeBudget": { - "p95Seconds": 45, - "scope": "focused CLI/runtime units plus one isolated Electron worker-start journey" + "p95Seconds": 75, + "scope": "focused CLI/runtime units plus isolated worker-start and app-restart Electron journeys" }, "flakeHistory": { "status": "unknown", - "evidence": "The deterministic units and isolated Electron journey pass locally; CI and soak history are not yet available." + "evidence": "The deterministic units and two isolated Electron journeys pass locally; CI and soak history are not yet available." }, "redGreenEvidence": { "status": "partial", - "evidence": "The focused presentation and stale-handle tests failed against the pre-fix implementation, and the live incident plus Electron topology showed the worker PTY existed without an immediate tab. Saved intentional-break and CI artifacts are still needed." + "evidence": "The focused presentation and stale-handle tests failed against the pre-fix implementation. The restart journey additionally failed first on empty retained output while the original PTY/PID remained live, then passed after the scoped recovered-worker snapshot fallback. The live incident and pre-fix Electron topology showed workspace re-entry replaying provider resume against a worker whose tab binding was missing. Saved intentional-break and CI artifacts are still needed." }, "performanceBudget": { "required": true, - "evidence": "Worker-start reuses the existing one-shot renderer reveal and adds no polling, provider listing, or output work. Check adds one optional pane-key field and reuses the existing Run scan or bounded active-Dispatch lookup; it adds no extra RPC, subprocess, timer, or renderer update. Federated and explicitly background terminals are unchanged." + "evidence": "Worker-start reuses the existing one-shot renderer reveal and adds no polling, provider listing, or output work. Startup recovery performs one bounded controller inventory per legacy candidate, exact owner/identity checks, one background reveal, and a provider/renderer snapshot only when an adopted worker's in-memory tail is empty and terminal.read is explicitly requested. Check adds one optional pane-key field and reuses the existing Run scan or bounded active-Dispatch lookup. Federated and explicitly background terminals are unchanged." }, "promotionCriteria": [ "Collect 100 consecutive passes or 14 days of stable CI history on macOS, Linux, and Windows.", - "Add live SSH and WSL exact-workspace worker-start evidence and a paired remote-runtime control.", + "Run distinct installed A/B artifacts through separate headed and headless paired-runtime cutover journeys, plus a git-independent folder workspace parameter.", + "Add Docker SSH restart/reconnect proof and physical Windows WSL correct-distro/wrong-distro proof.", + "Gate a packaged Windows real-updater journey with a blocked packaged CLI ask, launcher status 75, exact native/WSL resume commands, and one question/reply.", "Attach saved intentional-break artifacts for hidden local presentation and dropped stable-pane delivery." ], "knownGaps": [ - "No live Linux, Windows, SSH, WSL, or paired remote-runtime run is attached.", + "No distinct installed A/B headed or headless paired-runtime cutover is attached; the restart journey relaunches the same build while preserving the daemon and agent.", + "No Docker SSH restart/reconnect, physical Windows WSL distro-authority, or git-independent folder-workspace cutover journey is attached.", + "The Windows workflow now rejects non-distinct package inputs, but it does not yet drive the packaged updater with a blocked packaged CLI ask or prove status 75 and native/WSL resume.", "The Electron journey uses a deterministic fake Codex CLI rather than a real account.", - "The gate does not cover terminal output recovery after app restart; it covers identity, discoverability, focus, and mailbox routing." + "The local presentation journey tolerates its existing terminal-handle remint and proves continuity by PTY/incarnation/tab/leaf; byte-stable handle proof across a real A/B cutover awaits the runtime-authority/RPC implementation.", + "The local Electron restart proof covers retained visible output from the same-build daemon checkpoint, not transcript recovery after a transport cut or distinct A/B runtime replacement." ], - "demotionRule": "Keep experimental or demote if the Electron journey flakes without a product or harness defect, if local worker-start can return before tab materialization without an explicit reveal warning, if focus moves to the worker, if workspace re-entry duplicates the tab, or if pane-stable delivery reads the wrong mailbox." + "demotionRule": "Keep experimental or demote if either Electron journey flakes without a product or harness defect, if local worker-start can return before tab materialization without an explicit reveal warning, if focus moves to the worker, if restart or workspace re-entry spawns/resumes/duplicates/interferes with the worker, if retained output becomes unreadable, or if pane-stable delivery reads the wrong mailbox." }, { "id": "terminal-render.windows-cjk-repaint", diff --git a/config/scripts/orchestration-skill-guidance.test.mjs b/config/scripts/orchestration-skill-guidance.test.mjs index 74d14a9a9a1..849fe784b46 100644 --- a/config/scripts/orchestration-skill-guidance.test.mjs +++ b/config/scripts/orchestration-skill-guidance.test.mjs @@ -51,30 +51,39 @@ describe('orchestration skill guidance', () => { ) }) - it('teaches the hard cutover without reviving a legacy executor', () => { + it('teaches attested adoption without reviving the retired scheduler', () => { const skill = readSkill() const migration = getSection(skill, 'Contract Migration') - expect(migration).toContain('hard cutover') - expect(migration).toContain('effectsApplied') - expect(migration).toContain('skills get orchestration --full') - expect(migration).toContain('Do not retry the rejected command unchanged') - expect(migration).toContain('no longer supervised') - expect(migration).toContain('task-list --run run_legacy_local') - expect(migration).toContain('Read-only inspection never consumes legacy mail') - expect(migration).toContain('does not run a legacy scheduler, translate old writes, or drain') - expect(migration).toContain('does not cancel the prior assignment') - expect(migration).toContain('invalidate its worktree') - expect(migration).toContain('discard filesystem changes') - expect(migration).toContain('leave it as the only editor in that worktree') - expect(migration).toContain('observe it manually, read-only') - expect(migration).toContain('until it reaches a stable handoff point') - expect(migration).toContain('visible activity is a reason to keep observing') expect(migration).toContain( - 'Never launch a replacement editor in the same worktree while the legacy worker may still write there.' + 'adopts a live pre-update orchestration assignment into an ordinary Run' ) - expect(migration).toContain('if remaining work needs new lifecycle supervision') - expect(migration).not.toContain('restart the work using Run -> Task -> `worker-start`') + expect(migration).toContain( + 'preserves the existing agent process, PTY/session, terminal handle, tab/leaf/pane, worktree or folder workspace, Task, and Dispatch' + ) + expect(migration).toContain('never restarts or replaces the worker') + expect(migration).toContain('The retired scheduler is not revived') + expect(migration).toContain('[LEGACY COMPATIBILITY]') + expect(migration).toContain('[LEGACY READ-ONLY]') + expect(migration).toContain( + 'Loss of lifecycle authority does not invalidate the existing assignment, process, or filesystem work.' + ) + expect(migration).toContain( + 'It must not spawn, write, signal, stop, switch, focus, split, or inject a terminal.' + ) + expect(migration).not.toContain('task-list --run run_legacy_local') + expect(migration).toContain('run_legacy_local is an empty audit tombstone') + expect(migration).toContain('Recovered orchestration work from a contract update') + expect(migration).toContain('run-show --id ') + expect(migration).toContain('task-list --run ') + expect(migration).toContain('Legacy inspection remains available without consuming mail') + expect(migration).toContain('run-use --id --takeover-legacy') + expect(migration).toContain('Takeover fences only the old coordinator') + expect(migration).toContain('Live legacy workers keep their original Tasks, Dispatches') + expect(migration).toContain( + 'keep the original worker as the only editor until it reaches a stable handoff point' + ) + expect(migration).toContain('a conflict-free placement for any remaining work') }) it('treats long-running worker waits as liveness checkpoints, not failures', () => { diff --git a/docs/reference/plans/2026-07-28-orchestration-seamless-update-migration.md b/docs/reference/plans/2026-07-28-orchestration-seamless-update-migration.md new file mode 100644 index 00000000000..b648e2601f5 --- /dev/null +++ b/docs/reference/plans/2026-07-28-orchestration-seamless-update-migration.md @@ -0,0 +1,596 @@ +# Preserve active orchestration work across contract updates + +Status: implementation validation + +Date: 2026-07-28 + +## Decision + +Do not block updates, restart active workers, or revive the retired scheduler. + +On first launch after the contract update, atomically adopt the durable +pre-Run graph into one ordinary lightweight Run. Preserve every Task, +Dispatch, message, gate, terminal, PTY, process, worktree, and prompt +identifier. Mark only the pre-update Dispatch attempts as legacy-contract +attempts and accept the exact command shapes already pasted into those +workers through a narrow adapter. + +This is state migration plus a per-Dispatch protocol adapter. It is not a +second scheduler: + +- the migrated Run owns the preserved graph and coordinator inbox; +- legacy attempts keep their original protocol until they settle; +- every new attempt uses the current protocol, even on a preserved Task; and +- no placement, polling, concurrency, or deadline loop is resurrected. + +## User-visible outcome + +- The update installs immediately. +- Existing coordinator and worker terminals keep the same process, PTY, + worktree, prompt, Task ID, Dispatch ID, and history. +- Orca does not inject text, focus a tab, or start a replacement editor. +- The legacy graph appears in an automatically adopted, normal Run instead of + being stranded in `run_legacy_local`. +- An original live worker with retained launch proof can keep using the + `heartbeat`, `worker_done`, `escalation`, `ask`, and `check` commands in its + existing prompt. +- An original live coordinator can keep listing and settling preserved Tasks, + dispatch remaining work with current commands, check direct legacy mail, and + use the pre-cutover `reply` CLI guidance. +- Unprovable, stale, ambiguous, or already-settled rows remain inspectable and + unmistakably read-only. They never advertise Reply or Ack actions. +- A pre-update WSL process whose older launcher did not propagate the hidden + token remains lifecycle read-only. Its assignment, process, terminal, and + filesystem work remain valid and inspectable. + +The retired `orchestration run`, `coordinator-start`, and `coordinator-stop` +scheduler commands remain retired. Their in-memory loop cannot be reconstructed +truthfully. The coordinator agent and all durable work remain accessible; it +explicitly dispatches any ready work that was never assigned before the update. + +## Evidence and constraints + +- Schema v7 preserved old `messages`, `tasks`, `dispatch_contexts`, and + `decision_gates` by assigning them to `run_legacy_local`. +- The exact prompt at `8b154d686` used old `send`, `ask`, and consuming + `check` shapes. It omitted `worker_done --outcome`; a subject equal to + `Failed` or beginning `Failed:` represented failure. Its escalation carried + `taskId` but no `dispatchId`. +- `reply` existed in the old CLI and message formatter guidance, not in that + pinned worker prompt. +- Restored `OrcaRuntime` PTY records currently set `launchToken: null`, and + provider `listProcesses()` does not expose the launch token. Handles and + pane IDs alone therefore cannot authorize compatibility. +- Managed hooks source the owner-only endpoint file on every invocation, so an + old process reaches the new hook server after restart. The hook cache retains + the prior authenticated pane/launch-token association and distinguishes + hydrated rows from events observed by the current runtime. +- Older builds did not propagate `ORCA_AGENT_LAUNCH_TOKEN` through `WSLENV`, and + a running process cannot acquire it retroactively. +- The CLI transport mutation receipt is keyed by rotating runtime + authentication. It cannot deduplicate a legacy mutation across restart. +- Current invalid-capability lifecycle sends deliberately remain as auditable + rejected messages. Compatibility must not silently change that behavior. +- PR #11142 fixes background-tab materialization for newly dispatched workers. + It does not recover a pre-fix worker whose PTY survived an update after its + renderer tab binding was lost; workspace activation can otherwise mistake + that worker for a sleeping provider session and launch `codex resume`. +- The packaged Windows launcher keeps its Electron-as-Node child alive for the + lifetime of a blocking CLI command. A seamless design cannot assume that a + bounded RPC wait releases the installed executable. + +## Non-negotiable invariants + +1. Adoption never spawns, writes, signals, stops, focuses, or injects a + terminal and never mutates the assigned filesystem. It may restore one + persisted background tab/leaf binding for the exact already-live PTY. +2. Every durable row and identifier survives adoption; only its owning Run and + explicit protocol metadata change. +3. Compatibility is write-once per Dispatch. Migration never classifies a + current retry as legacy. +4. A legacy mutation requires one uniquely resolved legacy attempt or + coordinator principal plus current live proof of its original process tree. +5. Caller-supplied handles, pane keys, Task IDs, Dispatch IDs, or remote + attachments never create authority. +6. Current Dispatch behavior remains unchanged, including auditable rejected + lifecycle messages, capabilities, process incarnation, Run Delivery, and + question semantics. +7. Legacy direct mail and current Run Delivery are disjoint, even when their + Dispatches share one migrated Task and Run. +8. A B-era disconnect may replay accepted work; it may not lose mail or create + a second question, reply, completion, or settlement. A cut after an A-era + ask was already answered is explicitly ambiguous because A persisted no + invocation identity; recovery replays its answer through legacy check before + a same-text question is asked again. +9. Different reply bodies for the same source and principal conflict instead + of creating multiple replies. +10. Folder workspaces, SSH, WSL, remote runtimes, macOS, Linux, and Windows + follow the same proof rules. Missing proof degrades to inspection without + invalidating the retained assignment or filesystem work. + +## Storage and atomic adoption + +Raise the schema version and add: + +```text +dispatch_contexts.contract_version INTEGER NOT NULL +dispatch_contexts.launch_token_hash TEXT +messages.delivery_contract TEXT NOT NULL + CHECK(delivery_contract IN ('legacy_direct', 'current_delivery', 'audit_only')) +legacy_adoptions( + source_run_id PRIMARY KEY, + adopted_run_id UNIQUE NOT NULL, + scheduler_state_lost INTEGER NOT NULL, + adopted_at +) +legacy_compatibility_principals( + id PRIMARY KEY, + run_id NOT NULL, + dispatch_id, + role CHECK(role IN ('worker', 'coordinator')), + host_scope NOT NULL, + terminal_handle NOT NULL, + pane_key NOT NULL, + launch_token_hash NOT NULL, + process_incarnation, + status CHECK(status IN ('committed', 'settled', 'revoked')), + UNIQUE(role, run_id, dispatch_id) +) +legacy_operation_receipts( + principal_id, + operation_key, + method, + payload_hash, + effect_id, + response_json, + completed_at, + PRIMARY KEY(principal_id, operation_key) +) +legacy_mail_receipts( + principal_id, + message_id, + acknowledged_at, + PRIMARY KEY(principal_id, message_id) +) +``` + +The schema migration runs in one `BEGIN IMMEDIATE` transaction: + +1. create a persisted random Run ID and `legacy_adoptions` marker if one does + not already exist; +2. create an ordinary `legacy = 0` Run with a recovery objective; +3. move all `run_legacy_local` Tasks, Dispatches, decision gates, messages, + and question threads to it without changing their IDs, sequence, status, or + content; +4. mark pre-existing direct mail `legacy_direct` and hard-cutover rejection + rows `audit_only`; +5. mark a Dispatch legacy only when it predates the contract column and lacks + a current dispatch capability; and +6. fence any impossible outstanding legacy Delivery, move retained Delivery + history to the adopted Run, assert every Task/Dispatch/gate/message/question/ + Delivery Run ID agrees, and leave `run_legacy_local` as an empty read-only + audit tombstone. + +Every new Dispatch and message writes the current contract explicitly. +Repeated startup, a partially initialized fixture, and WAL recovery reuse the +persisted adoption marker. Reset and fresh-database paths cannot synthesize +legacy authority. + +## Durable principals and live authority + +After the hook server and PTY inventory are ready, a one-shot adoption service +may commit compatibility principals. It does not mutate terminal processes or +input. + +Before renderer hydration, legacy Dispatch evidence also fences automatic +provider resume for the exact sleeping pane. Once the relevant local daemon or +SSH relay has authoritative inventory, Orca intersects the durable handle, +pane, PTY ID, PTY incarnation, workspace, and host owner. A unique live match +is adopted into its original tab/leaf as inactive background state while the +coordinator tab remains active. Missing, stale, reused, cross-host, or +ambiguous evidence stays fenced and inspectable. Ordinary provider resume is +unblocked only when authoritative inventory proves that exact PTY exited. + +At hook-server startup, copy the validated hydrated status rows into an +immutable main-process-only startup snapshot before the server listens or +binds the new listener. Later live events update a separate current-runtime +observation map and cannot overwrite the pre-update commitment source. A +current event is preferred but is not required before the worker's first +post-restart lifecycle command when controller inventory proves the exact +original PTY and process incarnation are live. + +For each candidate it resolves the union of: + +- durable Dispatch assignee and pane; +- equivalent-pane aliases; +- remote attachment and host ownership; +- coordinator candidates from active legacy coordinator rows, Task creator + handles, Dispatch coordinator/parent provenance, and direct-message + sender/recipient history; +- the restored terminal inventory and process incarnation; and +- the hydrated, previously authenticated hook pane/launch-token association. + +The candidate commits only when that union identifies exactly one role and +row. The orchestration database stores a SHA-256 token commitment, never the +raw token. The existing owner-only hook status cache is an input to the +one-time commitment, but a hydrated status is not enough to mutate. + +Every compatibility call then requires: + +- a hidden launch token and pane inherited by the calling process; +- the immutable pre-update hook commitment, plus either a matching + current-runtime hook event or exact current controller proof of the original + PTY and process incarnation; +- a live PTY with the committed handle, host scope, and process incarnation; +- the committed principal and exact active legacy Dispatch or adopted Run; and +- an operation-specific unique match. + +The current hook event or exact controller inventory proves that the process +survived the restart; equality with the pre-update commitment proves it is the +same launch. If the hook cache was absent, the provider cannot report the +proof, the token changed, or any candidate is ambiguous, the row remains +read-only. + +For new Dispatches, persist the launch-token hash at dispatch time so later +contract updates do not depend on hook-cache recovery. Never return or log raw +tokens. Add explicit RPC-envelope, structured-log, error, and CLI JSON +redaction tests. + +Host scope is runtime-owned: + +- local: stable local execution-host identity; +- WSL: stable native execution-host identity plus distro; +- SSH: saved-host authority plus remote pane/process identity; and +- federated runtime: authenticated peer fingerprint plus remote attachment. + +`pickRemoteCliEnv`, SSH relay forwarding, WSLENV, and the legacy in-process SSH +fallback must carry the hidden evidence without exposing it in command text or +logs. A local runtime never accepts a raw remote token as local authority. +Pre-update WSL processes without the token cannot use the lifecycle adapter. + +For SSH, the host passthrough and in-process fallback stamp the saved target, +`SshRelaySession` connection incarnation, and attachment provenance from the +authenticated runtime channel; caller environment cannot choose them. WSL +stamps the runtime-owned native host and distro in both the full bridge and +fallback bridge. The verifier intersects those stamps with the committed host +scope before considering token or pane evidence. + +## Mixed-contract Run and mail routing + +The adopted Run is a normal current Run. Tasks belong to it, so a current retry +of a preserved Task naturally routes to the same Run without a cross-Run Task +reference. + +Message delivery is selected by `messages.delivery_contract`, not by Run ID: + +- old and adapter-created direct messages are `legacy_direct`; +- current sends are `current_delivery`; and +- migration rejections and non-actionable history are `audit_only`. + +Current Run Delivery creation, filtering, waiting, and acknowledgment select +only `current_delivery`. Legacy `check` selects only the exact principal's +`legacy_direct` address. History and inbox may show all three with explicit +labels but never attach actions to `audit_only`. Mixed legacy/current attempts +under one Task are covered across send, ask, check, reply, and settlement. + +The original coordinator is automatically associated with the adopted Run +after its principal proves live. `run-use` may bind that same principal without +converting legacy Dispatches. A different coordinator cannot silently take +over while a legacy attempt is active; an explicit recovery takeover is a +separate current-contract action and revokes legacy coordinator mutation +authority before rebinding. + +When that coordinator uses old direct `check`, the adapter also performs a +non-consuming existence check for `current_delivery` mail. It never mixes the +messages into legacy output; it prints the exact current Run check command and, +after that command creates a Delivery, the exact acknowledgment form. This +keeps an unchanged coordinator informed about current retries without +weakening current Delivery semantics. + +## Exact legacy adapter + +The CLI keeps accepting the old flags. It sends a hidden compatibility +envelope; the runtime, not the CLI, decides whether legacy behavior applies. + +Version B generates a random hidden `compatibilityInvocationId` once per CLI +process and reuses it for every transport reconnect by that invocation. +`legacy_operation_receipts.operation_key` is this invocation ID for B calls. +The method and canonical payload hash must also match. + +Attested compatibility calls bypass the rotating-auth generic mutation ledger +after the contract fence routes them to the adapter. Their only dedupe boundary +is the compatibility-principal receipt transaction described here; current +calls continue through the existing generic ledger unchanged. + +Version A had no invocation ID. Cross-cutover reconstruction is therefore +limited to effects with a durable semantic identity: final settlement per +Dispatch, still-pending ask per normalized question/options/recipient, reply +per source/principal/body, and read recovery per message. Heartbeat and +escalation remain at-least-once across an A-to-B manual retry because +collapsing identical commands could suppress a legitimate later event. Tests +and guidance do not promise cross-cutover exactly-once behavior for those +repeatable signals. + +An A-era ask that was already answered before its output was lost is also +ambiguous. B does not guess whether the same text means retry or a new +question. Its answer is in the durable recovery cohort and the error tells the +agent to run the exact legacy check command first. After that answer is printed +and acknowledged, a new B invocation with identical text creates a new +question. If multiple identical A threads exist, ask fails ambiguous and +recovery check drains them by message sequence; no arbitrary thread is chosen. + +### Lifecycle `send` + +- `heartbeat` and `worker_done`: resolve Task + Dispatch from the payload, then + intersect them with the verified worker principal. +- `escalation`: resolve Task + verified worker principal to exactly one active + legacy Dispatch because the old prompt supplied no Dispatch ID. +- Preserve the old direct coordinator recipient after verifying it belongs to + the adopted Run principal. +- Infer missing `worker_done` outcome as `failed` only when the trimmed subject + is `Failed` or begins `Failed:`; otherwise infer `succeeded`. +- Normalize to the current internal settlement model. + +Compatibility validation happens before message insertion. Current lifecycle +handling is not refactored through this prevalidation: its existing +invalid-capability audit message behavior remains locked by regression tests. + +The DB commits the direct message, lifecycle effect, and compatibility receipt +in one transaction. If a pre-update `worker_done` already settled the Dispatch, +the adapter reconstructs the matching semantic receipt from the existing +message and settlement. The persisted A settlement is authoritative: the +pinned A reconciler recorded even a `Failed:` subject as completed, so B +returns that original completed result rather than retroactively changing Task +or dependency state. Only a not-yet-accepted B call applies the corrected +failure inference. A conflicting retry fails. + +### `check` + +Legacy check is at-least-once: + +1. a read RPC returns the exact unacknowledged `legacy_direct` message IDs + without changing them; +2. the CLI prints the complete response; and +3. a hidden ack atomically records receipts and marks those exact rows read. + +A cut after read and before ack replays the same IDs. `--peek` and `--all` +remain read-only. + +Version A could mark a message read before losing its response. On adoption, +every eligible read direct message connected to a still-active legacy attempt +and lacking a post-adoption receipt is inserted into a durable recovery cohort +using `legacy_mail_receipts` with `acknowledged_at = NULL`. Legacy check drains +that stable cohort in bounded sequence-ordered pages before ordinary unread +mail. Each page is labeled +`[LEGACY RECOVERY REPLAY — MAY HAVE BEEN SEEN]`, replays until its post-print +ack, and then advances. The cohort never grows after adoption, so every +eligible row is covered without an unbounded response. Settled-at-cutover and +unrelated historical mail do not enter it. + +### `ask` + +Legacy ask is an idempotent mutation followed by a resumable read: + +1. create or find the direct legacy gate/question by verified Dispatch, + normalized question, options, and recipient; +2. atomically commit its message/thread plus operation receipt and return the + stable question ID; +3. wait in bounded read-only slices using that ID; and +4. after printing the answer, hidden-ack that exact answer. + +On first adoption, existing pending decision-gate threads are indexed into the +same semantic identity. A retry resumes them instead of creating another gate. + +### `reply` + +Legacy reply requires the source thread, the verified coordinator principal, +and a unique question owned by the adopted Run. One transaction inserts the +reply, resolves the exact gate, marks the exact source, and completes the +receipt. + +An existing pre-update reply with the same source, principal, and body is +reconstructed as success. A second body for the same source/principal is a +conflict, not another reply. + +### Coordinator and Task commands + +After automatic binding, unchanged Task listing, Task settlement, dispatch, +and gate inspection use the adopted Run. A new attempt receives a current +preamble, capability, launch-token commitment, and current message contract. +No unrelated Task can be created until the migrated coordinator binding is +proven or explicitly taken over. + +## Windows update behavior + +No compatibility mutation holds a DB transaction while waiting. Windows +legacy `ask` commits the question, prints its stable ID and exact resume +command, then exits before a long wait whenever the installed launcher cannot +prove that the executable is replaceable. + +The native packaged launcher sets an internal +`ORCA_WINDOWS_PACKAGED_CLI_LAUNCHER=1` marker for its Electron-as-Node child. +After the DB commit, legacy `ask` writes and flushes the stable question ID and +the exact resume command, then exits with a documented `75` resume-required +status. The command name comes only from the validated inherited +`ORCA_CLI_COMMAND` packaged enum: `orca` for native packaged terminals or +`orca-ide` for packaged WSL. Arbitrary environment text is rejected rather +than rendered. The launcher propagates status 75. No long wait starts on this +path. + +The candidate installer owns the backward edge: after the desktop has shut +down, it may terminate only remaining processes whose resolved executable path +is the exact installed Orca executable before replacement. This releases an +already-blocked A CLI that cannot know the new marker. Its accepted question +remains durable; retrying the exact old ask under B reconstructs the same +pending question and prints the resume command. If launcher restart +supervision can be proved, it may transparently reconnect instead, but +seamless install cannot depend on that optimization. + +After terminating the exact installed child, the installer waits boundedly for +its exact descendant native launcher to exit and for the installed executable +lock to clear. Replacement fails explicitly if either release cannot be +proved; it never proceeds from a broad process-name match or an assumed delay. + +A non-skipped packaged Windows A-to-B test uses the real launcher and updater, +cuts a blocked ask, verifies post-commit output flush and status 75 propagation +on native and packaged WSL marker-aware paths, including an executable +`orca-ide` WSL resume command. It separately proves the candidate installer +terminates an exact-path marker-unaware A child and observes the exact launcher +exit plus lock release. The test fails if either release is unknown. It then +installs B, resumes the same question, and records one question/reply. This +fallback is product behavior, not test-only guidance. + +## Rendering and authoritative guidance + +Formatting is authority-aware: + +- `[LEGACY COMPATIBILITY]`: live and attested; show only supported commands; +- `[LEGACY RECOVERY REPLAY — MAY HAVE BEEN SEEN]`: one bounded cutover replay; +- `[LEGACY READ-ONLY]`: retained but unactionable; no Reply or Ack hint; and +- current messages: unchanged. + +Update `skill-guides/orchestration.md` and regenerate its bundled source. The +guide explains automatic adoption, exact continuity, the retired scheduler +boundary, proof failures, current retries, and the Windows ask resume fallback. +It must not tell agents to restart a worker whose Dispatch was adopted. + +Formatting cannot use database provenance alone; current liveness and +attestation are runtime facts. Recovery may reveal the exact already-live PTY +once as an inactive background tab. It never creates a PTY or process, writes +input, switches workspaces, focuses, splits, or injects a terminal. + +## Deterministic verification + +### Schema and authority + +- Migrate exact pre-v7, hard-cutover, current, partial, WAL, reset, and fresh + fixtures; preserve IDs and row counts and reuse one adopted Run. +- Assert Task, Dispatch, decision-gate, message, question-thread, and retained + Delivery Run ownership agrees; outstanding legacy Deliveries are fenced. +- Prove current retries in the adopted Run remain current across repeated + migrations. +- Test unique worker/coordinator import from prior hook commitment plus current + live hook, and every forged, stale, wrong-host, wrong-pane, wrong-process, + duplicate-token, duplicate-role, equivalent-pane, and remote-attachment + ambiguity. +- Race a current-runtime hook before adoption and prove the immutable startup + commitment is neither overwritten nor confused with live proof. +- Cover manual coordinators with no `coordinator_runs` row and reject + conflicting candidates from Task, Dispatch, and message provenance. +- Assert all legacy authority failures occur before message/receipt/gate/state + effects while current invalid-capability rejection remains unchanged. +- Assert tokens are absent from RPC logs, errors, CLI output, and orchestration + storage. + +### Exact old behavior and replay + +- Replay the pinned prompt's heartbeat, successful and `Failed:` + `worker_done`, task-only escalation, ask, and consuming check. +- Replay the old formatter's reply command separately. +- Cover every operation once after B and once accepted by A immediately before + the cutover where a durable semantic identity exists, including the bounded + recovery replay for an A check and preservation of A's persisted settlement. +- Prove one B invocation reuses its invocation ID across reconnect, while + separate identical heartbeat/escalation invocations remain distinct. +- Inject real transport cuts after check read, ask commit, reply commit, and + completion commit; prove stable IDs and one receipt/effect. +- Page a recovery cohort larger than one response limit across repeated + disconnects and prove every eligible row appears until ack, then never again. +- Test mixed legacy/current attempts for every send/check/ask/reply route and + prove current Delivery cannot consume legacy direct mail and old coordinator + check only advertises the exact non-consuming current Delivery command. + +### Live two-launch E2E + +The focused renderer/daemon restart regression reconstructs the rc.3-era +orphan artifact in a shared candidate profile: a durable legacy Dispatch, +sleeping provider record, and live daemon PTY with its visual binding removed. +It proves startup restores the exact PID, PTY/incarnation, handle, tab/leaf, +Task, and Dispatch; emits no provider-resume spawn or interruption; preserves +coordinator focus; and remains inert after switching away and back. This +candidate/candidate fixture covers the renderer regression deterministically +but does not replace the distinct installed-A/candidate-B packaged cutover +journey below. + +Use distinct installed A and B artifacts plus an external append-only fake +agent spawn ledger: + +1. launch A with coordinator and worker terminals and record the replaceable A + desktop/main PID, runtime ID, version, and artifact hash separately from the + durable daemon, coordinator, and worker identities; +2. accept one command with durable semantic identity on A—pending ask, reply, + completion, or check—cut its response, close A without stopping the daemon + or fake agent, install B, and prove B owns the relaunched runtime; +3. assert A's desktop/main process and runtime are gone, B has a different + desktop/main PID, runtime ID, version, and artifact hash, while the daemon + plus coordinator/worker PID and start time, PTY/session, terminal handle, + tab/leaf/pane, worktree, Task, and Dispatch remain byte-identical; +4. prove that exact pending ask, reply, completion, or check resumes or + deduplicates on B; do not use heartbeat, escalation, or an already-answered + A ask as an exactly-once oracle; +5. run the remaining exact old lifecycle commands and settle the same + Dispatch; +6. switch away and back and assert no second spawn/tab and no + `Conversation interrupted`; and +7. create a current attempt on preserved state and prove current grammar. + +The append-only spawn ledger and an append-only interruption ledger, daemon +inventory, runtime terminal inventory, and every viewer DOM independently +prove one launch, process, handle, tab, and no `Conversation interrupted`. +Transport fault injection cuts the real CLI connection after check output but +before ack, and after ask, reply, and completion commit. Re-run the byte-exact A +argv under B and assert stable IDs plus exactly one message, gate, reply, +settlement, and receipt. + +Retain #11142's local visible-inactive test and extend its oracle beyond DOM +counting with the ledgers and host inventory. + +Run separate remote journeys: + +- headed paired server A-to-B with a surviving external client; +- headless `orca serve` A-to-B with the same surviving-client oracle; +- both prove the remote PID/PTy continuity, both viewer states, zero + client-local handles/tabs, and wrong-host evidence with zero effects; +- Docker SSH restart/reconnect with a remote proof file binding the process, + pane, and token commitment; +- physical Windows WSL coverage proving the correct distro and rejecting a + wrong-distro token; and +- a git-independent folder workspace parameter through the same adoption + journey. + +The Windows test is a non-skipped PR/release gate. It installs distinct A and +candidate-B packages, drives the real packaged updater while a packaged +`orca.exe` ask is blocked, proves the launcher and child release, verifies A +and B hashes/versions differ, resumes the same question on B, and records +exactly one question and reply. + +## Implementation slices + +1. Schema, atomic Run adoption, per-row contract provenance, and mixed mailbox. +2. Hook-commitment import, live principal verifier, host scopes, and redaction. +3. Atomic compatibility receipts plus lifecycle send/escalation. +4. Check read/ack and cutover recovery replay. +5. Ask create/resume, reply conflict semantics, and coordinator binding. +6. CLI hidden envelope, SSH/WSL forwarding, Windows resume fallback. +7. Authority-aware formatting and the version-matched orchestration guide. +8. Two-launch, remote, platform, and #11142 regression coverage. +9. Focused unit/type/lint/E2E checks and final adversarial review. + +## Definition of done + +- Updates are never blocked. +- Every durable legacy Task is owned by an ordinary visible Run. +- Every provably original live legacy Dispatch can finish through its existing + prompt without changing process or worktree. +- New attempts use the current contract and mailbox in that same Run. +- No authority comes from IDs or handles alone. +- No adoption step starts, stops, writes, signals, focuses, splits, or injects + a terminal. Exact live-PTY recovery may restore one inactive background + renderer tab/leaf binding. +- B-era disconnects replay or deduplicate instead of losing or duplicating + work; A-era repeatable signals and already-answered asks follow the explicit + ambiguity recovery rules. +- Lost scheduler memory is reported precisely without stranding durable state. +- Local/background, remote, SSH, WSL, folder, and Windows guarantees have + deterministic proof. +- The PR contains this plan, focused implementation, and no unrelated changes. diff --git a/native/windows-cli-launcher/OrcaCliLauncher.cs b/native/windows-cli-launcher/OrcaCliLauncher.cs index 9f479b9abea..8f89e303413 100644 --- a/native/windows-cli-launcher/OrcaCliLauncher.cs +++ b/native/windows-cli-launcher/OrcaCliLauncher.cs @@ -50,6 +50,11 @@ internal static class OrcaCliLauncher "ORCA_NODE_REPL_EXTERNAL_MODULE" ); startInfo.EnvironmentVariables["ELECTRON_RUN_AS_NODE"] = "1"; + startInfo.EnvironmentVariables["ORCA_WINDOWS_PACKAGED_CLI_LAUNCHER"] = "1"; + startInfo.EnvironmentVariables["ORCA_CLI_COMMAND"] = + Environment.GetEnvironmentVariable("ORCA_CLI_COMMAND") == "orca-ide" + ? "orca-ide" + : "orca"; using (Process child = Process.Start(startInfo)) { diff --git a/skill-guides/orchestration.md b/skill-guides/orchestration.md index ed11474d97e..05c13c7da24 100644 --- a/skill-guides/orchestration.md +++ b/skill-guides/orchestration.md @@ -53,36 +53,49 @@ Do not use orchestration merely because the user says "hand off", "handoff", "ha ## Contract Migration -Orca uses a hard cutover for orchestration mutations. It does not run a legacy scheduler, translate old writes, or drain pre-upgrade orchestration work. +Orca adopts a live pre-update orchestration assignment into an ordinary Run. Adoption preserves the existing agent process, PTY/session, terminal handle, tab/leaf/pane, worktree or folder workspace, Task, and Dispatch; it never restarts or replaces the worker. The retired scheduler is not revived, and a newly created attempt uses the current grammar. -If a command returns `orchestration_migration_required`, `run_required`, or a lifecycle validation error with `nextCommandArgs`: +Treat the authority label on injected or formatted messages as definitive: -1. Confirm `effectsApplied` is `false`. -2. Using the same CLI executable that returned the error, run the returned arguments: `skills get orchestration --full`. -3. Read the guide completely. Do not retry the rejected command unchanged. -4. Inspect the pre-upgrade Run, terminal, and assigned worktree before deciding whether any work needs replacement. -5. If the legacy worker is still making valid progress, leave it as the only editor in that worktree and observe it manually, read-only, until it reaches a stable handoff point. -6. Only then, if remaining work needs new lifecycle supervision, create or bind a lightweight Run, create a Task for the remaining work, and use `worker-start` in a conflict-free placement. +- `[LEGACY COMPATIBILITY]` is live and attested. Run only the exact supported command printed with the message, using the same CLI executable and arguments that the original prompt supplied. +- `[LEGACY RECOVERY REPLAY — MAY HAVE BEEN SEEN]` is one bounded, at-least-once cutover replay. Process it idempotently and acknowledge it only through the exact displayed guidance. +- `[LEGACY READ-ONLY]` is inspection-only. It has no reply, acknowledgment, or lifecycle action. +- An unlabeled current message uses the current guide and current grammar. -The arguments intentionally omit an executable name so this works with `orca`, `orca-ide`, `orca-dev`, or another configured Orca CLI command. +Database provenance, an old-looking terminal, or a legacy Run ID does not prove mutation authority. If the runtime cannot prove liveness, principal ownership, capability, or the exact legacy contract, it degrades to read-only inspection and must not fall back to local execution. Exact recovery may restore the already-live PTY once in its original inactive background tab. It must not spawn, write, signal, stop, switch, focus, split, or inject a terminal. Loss of lifecycle authority does not invalidate the existing assignment, process, or filesystem work. -The cutover removes lifecycle authority; it does not cancel the prior assignment, invalidate its worktree, discard filesystem changes, or stop the worker process. Pre-upgrade terminals and agents can continue their valid assigned work, but they are no longer supervised by Orca: old heartbeat, question, completion, scheduler, reply, acknowledgment, and mutation calls are rejected before effects. +Compatibility retries have narrow guarantees. A pending ask, a reply, a final Dispatch settlement, and a consuming check have durable recovery identities. A-era heartbeat and escalation calls remain at-least-once across a manual A-to-B retry because identical later signals may be intentional. If an A-era ask may already have been answered, run the exact non-consuming recovery check printed by the runtime first; after its answer is printed and acknowledged, a new invocation with the same question creates a new question. Never guess among multiple identical question threads. -Legacy database rows and terminal output remain available for explicit read-only inspection: +When a compatibility or recovery command returns structured next-step arguments, run those exact arguments with the same CLI executable. The arguments intentionally omit the executable name so the guidance works with `orca`, `orca-ide`, `orca-dev`, or another configured Orca CLI command. Do not translate the command from memory, broaden its recipient, or retry it as a current mutation unless the returned guidance explicitly says to. + +On packaged Windows, a legacy ask uses a two-step commit/resume protocol. The initial command durably commits the question, prints its exact `ask --resume ` command, and exits with launcher status `75`; it does not wait for the answer. Run that exact resume command after the launcher or update boundary. Resume is idempotent and read-oriented: it waits for the already-committed question and does not create another one. For a WSL process that received compatibility proof at launch, use the printed executable `orca-ide` WSL resume command so the same distro and packaged launcher authority are preserved; do not substitute a PATH-resolved local CLI. Older WSL processes that never received the hidden launch token remain lifecycle read-only after the update, even while their terminal and filesystem work continue. + +Legacy inspection remains available without consuming mail: ```bash orca orchestration run-list --json +# run_legacy_local is an empty audit tombstone after adoption. orca orchestration run-show --id run_legacy_local --json -orca orchestration task-list --run run_legacy_local --json +# In run-list, find the ordinary Run whose objective is: +# "Recovered orchestration work from a contract update" +orca orchestration run-show --id --json +orca orchestration task-list --run --json orca orchestration inbox --full --json orca orchestration check --terminal --peek --format --json orca terminal read --terminal --json orca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json ``` -Read-only inspection never consumes legacy mail. A stable handoff point means the worker has become idle, stopped, or completed a coherent edit/test/commit checkpoint; visible activity is a reason to keep observing, not to replace it. Do not prompt the worker to use old lifecycle commands. +If the original coordinator is unavailable or cannot prove its retained authority, a current coordinator may explicitly take over the adopted Run from its own live agent terminal: -Never launch a replacement editor in the same worktree while the legacy worker may still write there. Wait for a stable handoff and preserve its filesystem work; if overlap is truly required, use a separate conflict-free worktree with an explicit plan for transferring existing dirty changes. Do not use actionable `check`, acknowledgment, reply, send, retry, or task updates against the legacy Run. +```bash +orca orchestration run-use --id --takeover-legacy --json +orca orchestration check --run --json +``` + +Takeover fences only the old coordinator, binds the current one, and moves pending worker mail into current Run Delivery. It is bound to the authenticated invoking terminal; `--from` cannot name another coordinator. Live legacy workers keep their original Tasks, Dispatches, processes, filesystems, and old prompt commands; their later questions, escalations, and completion reports route to the current coordinator. Do not use takeover while the original coordinator is still actively coordinating, because its later lifecycle mutations are rejected. + +Do not launch a replacement editor merely because the desktop app or runtime was updated. If adoption cannot prove continuing authority, keep the original worker as the only editor until it reaches a stable handoff point, then use a new current Dispatch in a conflict-free placement for any remaining work. ## Ownership diff --git a/src/cli/bundled-skill-guides.ts b/src/cli/bundled-skill-guides.ts index 93832dea9fe..56aa9c1e6e9 100644 --- a/src/cli/bundled-skill-guides.ts +++ b/src/cli/bundled-skill-guides.ts @@ -30,7 +30,7 @@ const ORCA_LINEAR_MARKDOWN = "---\nname: orca-linear\ndescription: >-\n Use Orc const ORCA_PER_WORKSPACE_ENV_MARKDOWN = "---\nname: orca-per-workspace-env\ndescription: >-\n Set up, review, debug, or validate Orca per-workspace environment recipes —\n on-demand, disposable runtimes (cloud sandboxes, VMs, or local) created fresh\n for each workspace. Covers first-time setup (provider prerequisites, the\n reusable base snapshot, the coding-agent auth snapshot, credentials, and\n state), not just the per-workspace lifecycle scripts. Use to stand up\n per-workspace environments, fix an `environmentRecipes` entry in `orca.yaml`, scaffold\n provider lifecycle scripts, or resolve an `orca vm recipe doctor` failure.\n---\n\n# Per-Workspace Environments\n\nHelp a user stand up and maintain a repo-owned per-workspace environment recipe end to end. Each\nworkspace gets its own on-demand, disposable runtime (a cloud sandbox, a VM, or a local one),\ncreated fresh and torn down after.\n\nOrca is a **thin wrapper**: you guide, detect, and scaffold; you never own the user's cloud account,\nbilling, images, or credentials.\n\n- **You DO:** sequence the setup, detect what's detectable (provider CLI present/logged-in? recipe\n present? `doctor` passing?), scaffold provider-templated scripts the user fills in, drive the slow\n snapshot/auth phases with the user, and always show the next action.\n- **You DO NOT:** create accounts, choose plans/regions, invent org/project/scope ids, store or print\n secrets, or run anything that spends money without an explicit user OK.\n\nFirst-time setup has **four phases before the per-workspace recipe runs** — easy to miss, so walk\nthem in order:\n\n1. **Prerequisites** — cloud account, provider CLI, scope/project, plan limits, git token (§2).\n2. **Base snapshot** — reusable image: tools + repo + headless build, snapshotted once (§3).\n3. **Agent-auth snapshot** — boot the base, run interactive device-auth, re-snapshot (§4).\n4. **State** — thread snapshot id / scope / project / port between phases via a state file (§6).\n\nThen the **per-workspace contract** (create/suspend/resume/destroy) runs fast (§8).\n\n**The one branch that shapes everything — connection mode:** **Orca-server** (`create` runs `orca serve`\nin the env and emits a `pairingCode`; §7c/§7f) vs **SSH** (`create` runs no server and emits a\n`connection.type:\"ssh\"` block Orca dials into; §7g/§7h). Settle this first — it changes the `create`\noutput shape and half the templates.\n\n**Quick-start (happy path):** interview the user (connection mode Orca-server vs SSH, provider, agent CLI,\ngit auth — §1.2) + read the provider's CLI docs → scaffold `scripts/orca-vm/` from §7 → run the\nbase-snapshot script, then the auth script (you invoke these by hand; not via `orca.yaml`) → wire\n`environmentRecipes` in `orca.yaml` → `orca vm recipe doctor --json` (free) → then the `--provision`\nself-test loop (§9) until it passes.\n\n---\n\n## 1. Setup workflow\n\nDrive these with the user. **[CHECKPOINT]** steps need explicit confirmation — they spend money, take\na long time, or need the user at the keyboard. Never create an Orca workspace or commit unless asked.\n\n1. **Inspect the repo** for an existing `environmentRecipes` entry, `scripts/orca-vm/`, a state file, or setup\n notes. If a working recipe exists, jump to Doctor (§9) instead of rebuilding.\n2. **Interview the user up front** — gather these choices and confirm them back before scaffolding\n anything. Don't pick for them (§11); don't guess.\n - **Connection mode:** how Orca attaches to the environment — an **Orca server** (the VM runs\n `orca serve` and Orca pairs over its pairing URL; worked example §7f) or **SSH** (Orca connects to\n the host over SSH; §7g). This decides the recipe's connection shape, so settle it first.\n - **Provider:** Vercel Sandbox, Fly, Modal, an existing SSH host, … For non-obvious providers, also\n ask scope/project/region and plan limits (§2). Then **read that provider's CLI/SDK docs** (or\n ` --help`) before scaffolding — you need its exact create/exec/snapshot/remove verbs.\n If a provider advertises `ssh`, verify whether it exposes a real dialable SSH target\n (host/port/user/key or proxy command) or only a provider-mediated interactive shell; Orca SSH mode\n needs the former.\n - **Coding-agent CLI + account:** which agent runs in the VM (`codex`, `claude`, …) and that the user\n has an account for it — it gets logged in during the Phase-3 auth snapshot (§4).\n - **Git auth:** the token source for cloning a private repo (`GH_TOKEN`/`GITHUB_TOKEN` or `gh auth\n token`; §5).\n3. **Check prerequisites (§2)** — detect the provider CLI + auth and confirm the items above are in\n place before any paid step.\n4. **Scaffold scripts + state file** from §7 (worked Vercel example: §7f; SSH host: §7g; Docker SSH:\n §7h; Windows: §7i), filling in the provider's real commands. Make them executable.\n5. **[CHECKPOINT] Build the base snapshot (§3)** — paid, slow.\n6. **[CHECKPOINT] Authenticate the agent (§4)** — interactive; the user follows a URL/code. **You cannot\n drive this step** — you run commands non-interactively, so there's no TTY for `docker exec -it` /\n `ssh -t` to prompt against. The **user** runs the Phase-3 login in their own terminal (or via the\n Claude Code harness bang-prefix — `! `, with the required space after `!`); you scaffold and drive\n the non-interactive phases around it. After kicking it off, **ask the user to report back once the login\n finishes** — you can't observe it completing, and you need that confirmation before resuming the\n non-interactive steps (base/auth commit, doctor, provision).\n7. **Wire the recipe** so `orca.yaml` points create/suspend/resume/destroy at the scripts (§8). The\n workspace composer reads `environmentRecipes` from the project's primary checkout of `orca.yaml`, **not** from\n a feature branch or worktree. So a recipe added only on a branch won't appear as a \"Run on\" option\n until that `orca.yaml` change is committed and merged to the project's primary branch. Tell the user\n this up front: `doctor`/`--provision` validate the scripts from the working copy on any branch, but\n creating a workspace from the recipe in the picker needs it on primary.\n8. **Dry-run doctor** — `orca vm recipe doctor --repo-path --json` (free, static; §9).\n Fix every failure before going live.\n9. **[CHECKPOINT] Live self-test** — get the user's OK once, then run\n `orca vm recipe doctor --provision --json` as a loop: it runs create → validates →\n destroys, and on failure returns a full transcript. Read it, fix the scripts, and re-run yourself until\n it passes (§9). Spends cloud money; the one approval covers the loop.\n10. **[CHECKPOINT] Optional workspace test** — only if asked: create a workspace via the picker, then\n verify sleep/wake/delete.\n\n---\n\n## 2. Phase 1 — Prerequisites\n\nThe user's responsibility; verify what's verifiable, ask for the rest, invent nothing. State which\nitems you verified vs. which the user asserted.\n\n- **Connection mode** (Orca server vs SSH) confirmed with the user — see §1 step 2; it shapes the recipe.\n- **Cloud account + plan** that allows sandboxes/VMs. Ask.\n- **Provider CLI installed + authenticated** — detect (`command -v `), check auth (e.g.\n `vercel whoami`). If missing, point at the provider's docs; don't log them in.\n- **Scope / project / region** the sandboxes live under. Ask; flows into every script via state.\n- **Plan / timeout / RAM caps.** Record them — e.g. Vercel Hobby caps sandbox timeout at **45m**,\n which limits both the base build and per-workspace runtime (see §10).\n- **Git token for private repos** (`GH_TOKEN`/`GITHUB_TOKEN`, or the provider's git auth; can fall back\n to `gh auth token`). See §5.\n- **Coding-agent CLI choice** (`codex`, `claude`…) and that the user has an account — it gets\n authenticated into the VM in Phase 3.\n\n---\n\n## 3. Phase 2 — Base snapshot (the reusable image)\n\nBuild **once**, snapshot, and every workspace boots from it in seconds instead of rebuilding.\nProvisioning + building takes a while (often ~20–30 min), so it runs behind a checkpoint. The script\nshape is §7a; key points:\n\n- Build the **headless Electron main only** (not the renderer) so it fits in plan RAM.\n- Use the VM image's package manager (`apt`/`dnf`/`apk`, per the base distro — not the provider brand).\n- Clone with the git token via `GIT_ASKPASS` (§5).\n- **Trap errors and remove the half-built sandbox** so a crash doesn't leave a paid resource running.\n- Snapshot the stopped sandbox, parse the snapshot id, and write it + scope/project/port/repo to state.\n\n---\n\n## 4. Phase 3 — Agent-auth snapshot (interactive)\n\nThe base snapshot has the agent CLI installed but **not logged in**, and per-workspace VMs are\nephemeral — so authenticate once and bake it into a second snapshot layer. Script shape is §7b:\n\n1. Boot a sandbox from the base `snapshotId` (from state).\n2. Run the agent's login **interactively** (`--interactive --tty`); the user completes the URL/code in\n their browser. On a **headless VM this must be the device-auth flow** (e.g. `codex login --device-auth`),\n **not** plain `codex login`: the default OAuth login starts a loopback callback server on a container\n port the host browser can't reach, so it hangs. Device-auth instead prints a URL + code the user opens\n on the **host**.\n3. Verify login; **refuse to snapshot an unauthenticated VM.** Prefer the status command's **exit code**\n (most agent CLIs exit non-zero when unauthenticated). If you grep instead, agent status often goes to\n **stderr** (e.g. `codex login status` prints \"Logged in using ChatGPT\" there), so **fold stderr first**\n (`... 2>&1 | grep …`) and match the agent's **exact success line** — never `grep -qi 'logged in'`, which\n also matches \"**not** logged in\" and would commit an unauthenticated image.\n4. Re-snapshot, parse the new id, and overwrite `snapshotId` in state to the authenticated image\n (recording `authSourceSnapshotId`). Remove the auth sandbox.\n\n**You can't drive step 2 yourself** (you run commands non-interactively — no TTY). The **user** runs it in\ntheir own terminal, or via the Claude Code harness bang-prefix (`! `, with the required space after\n`!`). You scaffold/boot the sandbox and run steps 3–4, but **you cannot observe the interactive login\nfinishing** — so **ask the user to tell you when it's done** before you verify and re-snapshot.\n\nIf the agent's credentials are short-lived, warn that the snapshot may need periodic re-auth (§10).\n\nFor disposable runtimes, do **not** treat a host agent config directory (for example `~/.codex`) as the\nauth snapshot by bind-mounting or copying it wholesale. Agent homes often contain sqlite state, hook\napproval state, caches, logs, and host-specific env/config. Instead, authenticate/configure the agent\ninside the disposable runtime and snapshot/commit that runtime layer.\n\n---\n\n## 5. Credentials\n\n- **Never** commit secrets or put them in `userData`, recipe JSON, comments, docs, or the state file.\n- **Git token:** read from env (`GH_TOKEN`/`GITHUB_TOKEN`), falling back to `gh auth token`. Pass to the\n VM only via the provider's ephemeral `--env`. Inside the VM, use a `GIT_ASKPASS` helper with\n `x-access-token` (not the token in the clone URL) and `GIT_TERMINAL_PROMPT=0` so a missing token fails\n fast instead of hanging. When you write the helper from inside `bash -lc` under `set -u`, escape the\n positional arg and the token (`\\$1`, `\\$GH_TOKEN`) so they land **literally** and resolve at git-runtime\n — an unescaped `$1` aborts with \"unbound variable\", and a literal `$GH_TOKEN` keeps the real token out of\n the written file. `rm -f` the helper after the clone/fetch.\n- **Provider auth:** rely on the provider CLI's logged-in session, not checked-in keys.\n- **Agent auth:** lives in the authenticated snapshot (Phase 3) — never a file you write or commit.\n- State holds only **non-secret** wiring (snapshot ids, scope, project, port, repo url/ref).\n\n---\n\n## 6. State file\n\nA repo-local JSON file (e.g. `scripts/orca-vm/-state.json`) threads non-secret values between\nphases. Each script resolves values as **env var → state → built-in fallback**, and merges its outputs\nback. Phase 2 writes the base `snapshotId`; Phase 3 overwrites it with the authenticated snapshot;\nper-workspace `create` boots from `snapshotId`.\n\n```json\n{\n \"baseName\": \"orca-base\",\n \"snapshotId\": \"snap_authenticated_image_id\",\n \"authSourceSnapshotId\": \"snap_base_image_id\",\n \"scope\": \"\",\n \"project\": \"\",\n \"port\": 7331,\n \"repoUrl\": \"https://host/org/repo.git\",\n \"repoRef\": \"main\",\n \"projectRoot\": \"/abs/path/on/remote/repo\"\n}\n```\n\n---\n\n## 7. Script templates (provider-agnostic shapes)\n\nScaffold under `scripts/orca-vm/`. These are **shapes** — fill in the provider's real commands. All\nreserve stdout for the final JSON and log progress to stderr. Include a shared `json_value ` /\n`env_value ` reader (env → state → fallback) in each.\n\n**Where each script runs:**\n\n- **Local-side** (`create`/`suspend`/`resume`/`destroy` + the base-snapshot/auth scripts the user\n invokes) runs **on the user's desktop**, so it must run on their OS. macOS/Linux: `#!/usr/bin/env\n bash`, `set -euo pipefail`, quoted paths. **Windows:** a bare `.sh` won't run — scaffold `.ps1`/`.cmd`\n or require WSL/Git-Bash and point `orca.yaml` at the right launcher.\n- **Remote-side** (commands you `exec` *inside* the Linux VM) always runs in the VM's Linux shell, so\n bash is fine there regardless of the user's OS.\n\n### 7a. Base-snapshot (`-base-snapshot.sh`) — Phase 2\n\n```bash\n#!/usr/bin/env bash\nset -euo pipefail\n# resolve base_name/repo_url/repo_ref/project_root/port/scope/project/timeout (env→state→fallback)\n# resolve gh token: GH_TOKEN | GITHUB_TOKEN | `gh auth token`\n# 1. provision a sandbox (timeout/vcpus/published port/snapshot retention); trap: remove on error\n# 2. remote exec (long timeout): install pkgs + gh + corepack/pnpm + agent CLI;\n# clone with GIT_ASKPASS(token); write headless main-only build config;\n# dev setup; pnpm install; build CLI; build headless electron main; smoke-check tools\n# 3. snapshot stopped sandbox; parse snapshot id (fail if unparseable)\n# 4. merge { baseName, snapshotId, projectRoot, repoUrl, repoRef, port, scope, project } into state\n# print only the state JSON to stdout\n```\n\nWorked Vercel commands for this phase are in §7f. You run this script by hand (not via `orca.yaml`),\nafter exporting the first-run inputs the state file doesn't have yet — e.g. provider scope/project, the\nrepo URL/ref, and a git token (`GH_TOKEN`); later runs read them back from state.\n\n### 7b. Auth (`-base-auth.sh`) — Phase 3\n\n```bash\n#!/usr/bin/env bash\nset -euo pipefail\n# read source snapshot from state.snapshotId (fail if absent); auth_name=\"${base_name}-auth\"\n# 1. boot sandbox from source snapshot; trap: remove on error\n# 2. INTERACTIVE/TTY remote exec: agent login — user completes URL/code. Headless VM: MUST use the\n# device-auth flow (e.g. `codex login --device-auth`) — plain OAuth login binds a loopback callback\n# port the host can't reach and hangs. User runs this themselves (you have no interactive TTY); ask\n# them to report back when it's done before continuing.\n# 3. verify login, then refuse to snapshot if not logged in. Prefer the status command's EXIT CODE (most\n# agent CLIs exit non-zero when unauthenticated) over string-matching. If you must grep, fold stderr\n# first (`status 2>&1 | grep …` — many agents print the success line there) and match the agent's exact\n# success line; never `grep -qi 'logged in'`, which also matches \"not logged in\". Codex example: §7f.\n# 4. snapshot; parse new id\n# 5. merge { snapshotId:, authSourceSnapshotId: } into state; remove auth sandbox\n# print only the state JSON to stdout\n```\n\n### 7c. Create (`-create.sh`) — per workspace\n\n```bash\n#!/usr/bin/env bash\nset -euo pipefail\n# read authenticated snapshotId/scope/project/port/repo*/project_root (env→state→fallback)\n# fail clearly if snapshotId is missing (point back to Phases 2–3)\n# name = orca-${ORCA_VM_RECIPE_ID}-${ORCA_VM_INSTANCE_ID} (sanitized, length-capped)\n# 1. boot sandbox from snapshotId with a published port; capture the public URL → pairing address\n# (an externally reachable wss:// URL); trap: remove sandbox on error\n# 2. remote exec: ensure repo at desired commit; rebuild only if commit changed (cache marker)\n# 3. remote exec: start orca serve in the background and read the recipe JSON it writes (see below)\n# 4. print serve's JSON to stdout, optionally enriched with userData:\n# { schemaVersion:1, pairingCode, projectRoot, userData:{ provider, resourceId:name, snapshotId } }\n```\n\n**The exact `orca serve` invocation and its output (verified — do not improvise the flags).** Inside the\nVM, run:\n\n```bash\norca serve \\\n --port \"$PORT\" \\\n --project-root \"$ABS_REPO_PATH_ON_REMOTE\" \\\n --pairing-address \"$EXTERNAL_WSS_URL\" \\\n --recipe-json\n```\n\n**Binary name:** in a VM built from source (the Phase-2 flow), run it as `pnpm exec orca-dev serve …`\nfrom the repo root — `orca-dev` is the in-repo entrypoint and is what the §7f example uses. Plain\n`orca serve …` is the same command when the built CLI is installed on the VM's PATH. The flags/output\nare identical either way.\n\nThere is **no `--host` flag**. `--project-root` must be an absolute directory on the remote. With\n`--recipe-json` the server **stays running** and prints exactly this single object to **stdout**, then\nkeeps serving:\n\n```json\n{ \"schemaVersion\": 1, \"pairingCode\": \"\", \"projectRoot\": \"\" }\n```\n\n`pairingCode` is the pairing URL, already pointing at whatever you passed as `--pairing-address` — so set\n`--pairing-address` to the externally reachable address and **pass `pairingCode` through unchanged; never\nhand-rewrite it**. Because serve runs in the foreground and doesn't exit, redirect its stdout to a file\nand poll until that file parses as JSON (and bail if the process dies — dump its stderr log). Your\n`create` script then prints that JSON (optionally merging `userData`). Concrete pattern: §7f.\n\n### 7d. Suspend / resume / destroy — per workspace\n\n```bash\n#!/usr/bin/env bash\nset -euo pipefail\npayload=\"$(cat)\" # Orca passes lifecycle JSON on stdin\nresource_id=\"$(node -e 'const d=JSON.parse(process.argv[1]); process.stdout.write(d.recipeResult?.userData?.resourceId ?? \"\")' \"$payload\")\"\n[ -n \"$resource_id\" ] || { echo \"No resource id in lifecycle payload\" >&2; exit 1; }\n# suspend: provider suspend \"$resource_id\"\n# resume: provider resume \"$resource_id\"; then RE-EMIT fresh recipe JSON (pairing may change)\n# destroy: provider remove \"$resource_id\" (or set destroy: none in orca.yaml)\n```\n\n### 7e. State file — scaffold with scope/project/repo filled in and snapshot ids empty (§6).\n\n### 7f. Worked example — Vercel Sandbox (all three phases)\n\nA real, working shape (the Vercel surface is a CLI: `vercel sandbox create|exec|snapshot|remove`). Adapt\nnames; verify flags against `vercel sandbox --help` for the user's CLI version before relying on them.\nThese ground §7a (base snapshot) and §7b (auth), which are otherwise generic skeletons.\n\n**Phase 2 — base snapshot (§7a):** provision → install tools + clone + headless build → snapshot.\n\n```bash\n# provision a fresh build sandbox (retain a couple of snapshots); trap-remove on error\nvercel sandbox create --name \"$base\" --runtime node24 --timeout 30m --vcpus 4 --publish-port \"$port\" \\\n --snapshot-expiration 30d --keep-last-snapshots 2 \"${vercel_args[@]}\" >&2\n# remote build (long timeout): install pkgs+gh+pnpm+agent CLI, clone with GIT_ASKPASS (write the helper\n# with LITERAL \\$1/\\$GH_TOKEN so they resolve at git-runtime, not write-time — see §5/§7f create — then\n# `rm -f /tmp/askpass.sh`), write the headless main-only build config (drop the renderer), dev setup,\n# build CLI + headless main, smoke-check\nvercel sandbox exec \"$base\" \"${vercel_args[@]}\" --timeout 25m --env \"GH_TOKEN=$gh_token\" … -- bash -lc '…build…' >&2\n# snapshot the STOPPED sandbox and parse the id from CLI output (fail if unparseable)\nout=\"$(vercel sandbox snapshot \"$base\" --stop --expiration 30d \"${vercel_args[@]}\" 2>&1)\"; printf '%s\\n' \"$out\" >&2\nsnapshot_id=\"$(printf '%s\\n' \"$out\" | sed -nE 's/.*(snap_[A-Za-z0-9]+).*/\\1/p' | tail -1)\"\n# merge { baseName, snapshotId, scope, project, port, repoUrl, repoRef, projectRoot } into state; print state JSON\n```\n\n**Phase 3 — agent-auth snapshot (§7b):** boot the base, log the agent in interactively, re-snapshot.\n(`codex` below is an example — substitute the user's chosen agent's login/status verbs, e.g. `claude`.)\n\n```bash\nvercel sandbox create --name \"$auth\" --snapshot \"$snapshot_id\" --timeout 30m --publish-port \"$port\" \"${vercel_args[@]}\" >&2\n# INTERACTIVE — the USER runs this in their own terminal (you have no interactive TTY) and completes the\n# URL/code on the HOST. --device-auth is MANDATORY on a headless VM: plain `codex login` binds a loopback\n# callback port the host browser can't reach and hangs. Ask the user to report back when login finishes.\nvercel sandbox exec --interactive --tty \"$auth\" \"${vercel_args[@]}\" -- bash -lc 'codex login --device-auth'\n# refuse to snapshot an unauthenticated VM — fold stderr, match codex's exact success line (§4)\nvercel sandbox exec \"$auth\" \"${vercel_args[@]}\" --timeout 30s -- bash -lc 'codex login status 2>&1' | grep -Eqi 'Logged in using ChatGPT|Logged in via device' \\\n || { echo \"agent not logged in; not snapshotting\" >&2; exit 1; }\nout=\"$(vercel sandbox snapshot \"$auth\" --stop --expiration 30d \"${vercel_args[@]}\" 2>&1)\"; printf '%s\\n' \"$out\" >&2\nnew_id=\"$(printf '%s\\n' \"$out\" | sed -nE 's/.*(snap_[A-Za-z0-9]+).*/\\1/p' | tail -1)\"\n# overwrite state.snapshotId = new_id, record authSourceSnapshotId = snapshot_id; remove the auth sandbox\n```\n\n**Per-workspace `create`** (the fast path):\n\n```bash\n#!/usr/bin/env bash\nset -euo pipefail\n# resolve from env→state→fallback: snapshot_id, scope, project, port, repo_url, repo_ref, project_root\nvercel_args=(); [ -n \"$scope\" ] && vercel_args+=(--scope \"$scope\"); [ -n \"$project\" ] && vercel_args+=(--project \"$project\")\n[ -n \"$snapshot_id\" ] || { echo \"snapshotId missing — run Phases 2–3 first\" >&2; exit 1; }\ngh_token=\"${GH_TOKEN:-${GITHUB_TOKEN:-$(command -v gh >/dev/null 2>&1 && gh auth token 2>/dev/null || true)}}\"\nname=\"orca-${ORCA_VM_RECIPE_ID:-vercel-sandbox}-${ORCA_VM_INSTANCE_ID:-$(date +%s)}\" # sanitize+cap to 63 chars\n\n# Arm cleanup BEFORE create so a failing create can't leak a half-built paid sandbox.\ncleanup_on_error() { [ \"$?\" -ne 0 ] && vercel sandbox remove \"$name\" \"${vercel_args[@]}\" >/dev/null 2>&1 || true; }\ntrap cleanup_on_error EXIT\n\n# 1. boot from the authenticated snapshot, publish the serve port\ncreate_output=\"$(vercel sandbox create --name \"$name\" --snapshot \"$snapshot_id\" \\\n --timeout 30m --publish-port \"$port\" \"${vercel_args[@]}\" 2>&1)\"; printf '%s\\n' \"$create_output\" >&2\n# Vercel prints the published https URL; derive the external wss:// pairing address from it\npublic_url=\"$(printf '%s\\n' \"$create_output\" | sed -nE 's#.*(https://[^[:space:]]+\\.vercel\\.run).*#\\1#p' | head -1)\"\n[ -n \"$public_url\" ] || { echo \"no published URL in create output\" >&2; exit 1; }\npairing_ws=\"${public_url/https:\\/\\//wss://}\"\n\n# 2. (remote) ensure the repo is at the right commit; rebuild only if the commit changed (cache marker)\nvercel sandbox exec \"$name\" \"${vercel_args[@]}\" --timeout 20m \\\n --env \"GH_TOKEN=$gh_token\" --env \"ORCA_PROJECT_ROOT=$project_root\" \\\n --env \"ORCA_REPO_URL=$repo_url\" --env \"ORCA_REPO_REF=$repo_ref\" \\\n -- bash -lc 'set -euo pipefail; cd \"$ORCA_PROJECT_ROOT\"; \\\n # Re-establish git auth for the private-repo fetch (why + full rationale: §5); else it hangs on a prompt.\n # Load-bearing escaping: \\$1 and \\$GH_TOKEN must land LITERALLY and resolve at git-runtime. Test after\n # any edit here — reformatting the nested printf/node quoting silently breaks the fetch or leaks the token.\n if [ -n \"${GH_TOKEN:-}\" ]; then \\\n printf \"%s\\n\" \"#!/usr/bin/env bash\" \"case \\\"\\$1\\\" in *Username*) echo x-access-token;; *Password*) echo \\\"\\$GH_TOKEN\\\";; esac\" > /tmp/askpass.sh; \\\n chmod 700 /tmp/askpass.sh; export GIT_ASKPASS=/tmp/askpass.sh GIT_TERMINAL_PROMPT=0; fi; \\\n git fetch origin \"$ORCA_REPO_REF\"; \\\n git checkout -B \"$ORCA_REPO_REF\" FETCH_HEAD; \\\n rm -f /tmp/askpass.sh; \\\n c=\"$(git rev-parse HEAD)\"; [ -f .orca-built ] && [ \"$(cat .orca-built)\" = \"$c\" ] || { \\\n pnpm install --prefer-offline && pnpm run build:cli && \\\n node config/scripts/run-electron-vite-build.mjs --config config/electron-vite.vm-serve.config.ts && \\\n printf \"%s\" \"$c\" > .orca-built; }' >&2\n\n# 3. (remote) start orca serve in the background, writing recipe JSON to a file; poll until it parses\nrecipe_json=\"$(vercel sandbox exec \"$name\" \"${vercel_args[@]}\" --timeout 60s \\\n --env \"ORCA_PORT=$port\" --env \"ORCA_PROJECT_ROOT=$project_root\" --env \"ORCA_PAIRING_ADDRESS=$pairing_ws\" \\\n -- bash -lc 'set -euo pipefail; cd \"$ORCA_PROJECT_ROOT\"; rm -f /tmp/orca-recipe.json /tmp/orca-serve.log; \\\n nohup pnpm exec orca-dev serve --port \"$ORCA_PORT\" --project-root \"$ORCA_PROJECT_ROOT\" \\\n --pairing-address \"$ORCA_PAIRING_ADDRESS\" --recipe-json >/tmp/orca-recipe.json 2>/tmp/orca-serve.log /dev/null 2>&1 && { cat /tmp/orca-recipe.json; exit 0; }; \\\n kill -0 \"$pid\" 2>/dev/null || { cat /tmp/orca-serve.log >&2; exit 1; }; sleep 0.25; \\\n done; cat /tmp/orca-serve.log >&2; echo \"serve recipe JSON timed out\" >&2; exit 1')\"\n\n# 4. print serve's JSON enriched with userData (single object on stdout)\nnode -e 'const p=JSON.parse(process.argv[1]); console.log(JSON.stringify({...p, schemaVersion:1,\n userData:{...p.userData, provider:\"vercel-sandbox\", resourceId:process.argv[2], snapshotId:process.argv[3]}}))' \\\n \"$recipe_json\" \"$name\" \"$snapshot_id\"\ntrap - EXIT\n```\n\n`suspend`/`resume`/`destroy` use `vercel sandbox stop|...|remove \"$resource_id\"` reading\n`userData.resourceId` from stdin (§7d). This is the **Orca-server** connection mode (the recipe emits a\npairing URL). If the user chose **SSH** in the §1 interview, use §7g instead.\n\n### 7g. Worked example — existing SSH host (SSH connection mode)\n\nSSH mode is **fundamentally different from §7c/§7f**, not a relabeling of them:\n\n- **`create` does NOT run `orca serve` and does NOT emit a `pairingCode`.** Orca itself connects to the\n host over its SSH relay, brings up the git + filesystem providers, and imports the repo. The script's\n only job is to make the host ready and **print SSH connection details** Orca will dial.\n- The result uses a `connection` block with `type: \"ssh\"` and a `target`, **not** the flat\n `pairingCode`/`projectRoot` shape. Exact shape (Orca rejects anything else):\n\n```json\n{\n \"schemaVersion\": 1,\n \"connection\": {\n \"type\": \"ssh\",\n \"projectRoot\": \"/abs/path/to/repo/on/host\",\n \"target\": {\n \"label\": \"my-box\",\n \"host\": \"192.0.2.10\",\n \"port\": 22,\n \"username\": \"ubuntu\",\n \"identityFile\": \"~/.ssh/id_ed25519\",\n \"jumpHost\": \"bastion.example.com\",\n \"proxyCommand\": \"cloudflared access ssh --hostname %h\",\n \"relayGracePeriodSeconds\": 0,\n \"portForwards\": []\n }\n }\n}\n```\n\n`label`, `host`, `port`, `username` are required; the rest are optional — omit any you don't need.\n\n**Networking → which `target` fields to set** (how *your desktop* reaches the box — there is no\n`orca serve` URL in SSH mode):\n\n- Public IP / DNS, or a Tailscale/VPN address → `host`; SSH port → `port` (usually 22).\n- Key auth → `identityFile` (add `identitiesOnly: true` if the agent has many keys).\n- Through a bastion → `jumpHost` (a `user@host` ProxyJump) **or** a full `proxyCommand` (e.g. an access\n proxy). Use one, not both.\n- A service port the workspace needs → add entries to `portForwards`.\n- `relayGracePeriodSeconds` (optional): how long Orca keeps the SSH relay alive after the workspace\n detaches before tearing it down; `0` = tear down immediately. Leave it off unless the user wants a\n reconnect grace window.\n\n**Toolchain & agent auth on a persistent (no-snapshot) host — do this ONCE, by hand, before wiring the\nrecipe** (there's no base image to bake; the host *is* the base). Run the §7f Phase-2 install steps and\nthe §7f Phase-3 ` login --device-auth` **directly over SSH on the host** (interactive, e.g.\n`ssh -t user@host ' login --device-auth'`). After that the host stays ready across workspaces.\n\n```bash\n#!/usr/bin/env bash\nset -euo pipefail\n# resolve from env→state→fallback (default unset optionals to \"\"): ssh_username, host,\n# ssh_port (default 22), identity_file, jump_host, proxy_command, project_root, repo_url, repo_ref\n: \"${identity_file:=}\"; : \"${jump_host:=}\"; : \"${proxy_command:=}\" # avoid set -u aborts on optionals\ngh_token=\"${GH_TOKEN:-${GITHUB_TOKEN:-$(command -v gh >/dev/null 2>&1 && gh auth token 2>/dev/null || true)}}\"\nssh_target=\"${ssh_username}@${host}\"\nssh_opts=(-p \"$ssh_port\"); [ -n \"$identity_file\" ] && ssh_opts+=(-i \"$identity_file\")\n# Why: a fresh host's key isn't in known_hosts; a StrictHostKeyChecking prompt would HANG a\n# non-interactive create. Pre-add the key (or set the option) so it can't block.\nssh-keyscan -p \"$ssh_port\" \"$host\" >> \"$HOME/.ssh/known_hosts\" 2>/dev/null || true\n\n# 1. ensure the repo is present and at the right commit on the host (NO orca serve here)\nssh \"${ssh_opts[@]}\" \"$ssh_target\" \\\n \"GH_TOKEN='$gh_token' GIT_TERMINAL_PROMPT=0 bash -lc '\n set -euo pipefail\n [ -d \\\"$project_root/.git\\\" ] || git clone \\\"$repo_url\\\" \\\"$project_root\\\"\n cd \\\"$project_root\\\" && git fetch origin \\\"$repo_ref\\\" && git checkout -B \\\"$repo_ref\\\" FETCH_HEAD\n '\" >&2\n\n# 2. print the SSH connection block (NO pairingCode, NO orca serve). host/port/username tell Orca's\n# relay how to dial in; identityFile/jumpHost/proxyCommand/portForwards are emitted when set.\nnode -e 'const [host,port,user,idf,jh,pc,root]=process.argv.slice(1);\n const target={ label:\"per-workspace-host\", host, port:Number(port), username:user };\n if(idf) target.identityFile=idf; if(jh) target.jumpHost=jh; if(pc) target.proxyCommand=pc;\n // add target.portForwards=[...] here if the workspace needs forwarded service ports\n console.log(JSON.stringify({ schemaVersion:1, connection:{ type:\"ssh\", projectRoot:root, target } }))' \\\n \"$host\" \"$ssh_port\" \"$ssh_username\" \"$identity_file\" \"$jump_host\" \"$proxy_command\" \"$project_root\"\n```\n\n`suspend`/`resume`/`destroy`: on a persistent host there's usually nothing to tear down — set\n`destroy: none` and omit suspend/resume. (Orca still disconnects/reconnects its own SSH relay on\nsleep/wake/delete — that's separate from these scripts.)\n\nIf the SSH host is instead an **ephemeral/snapshot-capable VM** (your hypervisor, or a cloud VM with\nimage support), keep the §7f Phase-2/3 base-image model for provisioning, but still emit the\n`connection.type:\"ssh\"` block above instead of starting `orca serve`.\n\n### 7h. Worked example — local Docker SSH (SSH connection mode)\n\nLocal Docker can model an ephemeral SSH VM without cloud cost: build a base image with `sshd`, tools,\nrepo prerequisites, and the agent CLI; run an **interactive auth container** once; then `docker commit`\nthat container as the authenticated image used by per-workspace `create`.\n\nKey points:\n\n- Publish container SSH to a random localhost port (`-p 127.0.0.1::22`) and emit\n `connection.type:\"ssh\"` with `host:\"127.0.0.1\"`, that port, `username`, `identityFile`, and\n `identitiesOnly:true`.\n- Generate a repo-local SSH key if needed, but gitignore the private/public key files.\n- **Bake SSH host keys into the base image** (`ssh-keygen -A` at **build** time; at runtime only generate\n if absent). Ephemeral containers all present the **same** host key, so `known_hosts` on `127.0.0.1`\n doesn't churn as the published port rotates across workspaces (otherwise every container's freshly\n generated key collides on `localhost` and trips host-key-changed warnings).\n- The auth image is the Docker equivalent of Phase 3: the **user** runs the agent login **inside** the\n container (you can't drive it — you have no interactive TTY), configures proxy env/config, approves\n hooks, and you commit once they report it's done. On a headless container use the **device-auth** flow\n (§4). Verify login before committing — exit code, or fold stderr and match the exact success line (§4).\n- Do not bind-mount or copy the host's full agent home into the image. Let each container have writable\n agent state; only the committed auth image should carry reusable authenticated state.\n- If committing from an interactive shell, force the runtime entrypoint back to `sshd`:\n `docker commit --change='ENTRYPOINT [\"/usr/local/bin/orca-docker-ssh-entrypoint\"]' …`.\n- `destroy` should read `recipeResult.userData.resourceId` and run `docker rm -f \"$resource_id\"`.\n\nValidation before wiring/live use:\n\n```bash\ndocker image inspect \"$auth_image\" --format '{{json .Config.Entrypoint}}'\ndocker run -d --name \"$name\" -p 127.0.0.1::22 -e \"ORCA_SSH_PUBLIC_KEY=$pubkey\" \"$auth_image\"\ndocker ps -a --filter \"name=$name\"\ndocker logs \"$name\"\nssh -i \"$key\" -p \"$port\" -o IdentitiesOnly=yes user@127.0.0.1 'codex --version'\n```\n\nIf the container exits immediately, inspect logs before the cleanup trap removes it; a committed\ninteractive image with `ENTRYPOINT [\"bash\"]` is a common cause.\n\nAlso confirm the **host key is stable** across containers: the SSH `ssh -i … 127.0.0.1` dial should not\ntrigger a host-key-changed warning when a second container reuses the port. If it does, the host keys\nweren't baked into the base image (see the `ssh-keygen -A` point above).\n\n### 7i. Windows local-side scripts\n\nThe local-side scripts run on the user's desktop. On **Windows**, a bare `.sh` won't execute. Either\nrequire WSL/Git-Bash (and point `orca.yaml` at e.g. `bash ./scripts/orca-vm/.sh` via a `.cmd`\nlauncher), or scaffold PowerShell equivalents. Minimal PowerShell shape:\n\n```powershell\n#requires -Version 5\n$ErrorActionPreference = 'Stop'\n# resolve env→state→fallback; run the provider CLI / ssh the same way;\n# capture provider output; build the result object for the chosen mode and write ONE line of JSON to stdout.\n# Orca-server mode: @{ schemaVersion=1; pairingCode=$pairingCode; projectRoot=$projectRoot; userData=@{...} }\n# SSH mode: @{ schemaVersion=1; connection=@{ type=\"ssh\"; projectRoot=$projectRoot;\n# target=@{ label=$label; host=$host; port=$port; username=$user } } } (see §7g/§7h)\n($result | ConvertTo-Json -Compress -Depth 6)\n# progress/errors → Write-Error / the error stream, never stdout.\n```\n\nThe remote-side commands you run *inside* the Linux VM stay bash regardless of the desktop OS.\n\n---\n\n## 8. Per-workspace recipe contract (the fast path)\n\nOnce the authenticated snapshot exists, this runs on every workspace create. Define recipes in\n`orca.yaml`:\n\n```yaml\nenvironmentRecipes:\n - id: cloud-sandbox\n name: Cloud Sandbox\n create: ./scripts/orca-vm/cloud-sandbox-create.sh\n suspend: ./scripts/orca-vm/cloud-sandbox-suspend.sh\n resume: ./scripts/orca-vm/cloud-sandbox-resume.sh\n destroy: ./scripts/orca-vm/cloud-sandbox-destroy.sh\n```\n\n`create` runs **locally from the repo root** and prints **one** JSON object to stdout. Its shape depends\non the connection mode chosen in §1:\n\n**Orca-server mode** — boot the env, start `orca serve` in it, and print serve's result:\n\n```json\n{\n \"schemaVersion\": 1,\n \"pairingCode\": \"orca-pairing-code-or-url\",\n \"projectRoot\": \"/absolute/path/to/repo/on/remote\",\n \"userData\": { \"provider\": \"example\", \"resourceId\": \"provider-resource-id\" }\n}\n```\n\nHere `pairingCode` (from `orca serve --recipe-json`) and `projectRoot` are required; `schemaVersion` (`1`)\nand `userData` are optional.\n\n**SSH mode** — do **not** run `orca serve`; print the `connection.type:\"ssh\"` block instead (full shape +\nworked script in §7g). `pairingCode` is **not** used in SSH mode.\n\nLifecycle hooks (all run locally):\n\n- `create`: required. Prints recipe result JSON.\n- `suspend`: optional. Sleep; reads lifecycle payload on stdin.\n- `resume`: optional. Wake; reads payload on stdin and **prints fresh recipe JSON** (pairing may change).\n- `destroy`: optional unless `destroy: none`. Delete/cleanup; reads payload on stdin.\n\nStart Orca remotely with `orca serve --port \"$PORT\" --project-root \"$ABS_ROOT\" --pairing-address\n\"$EXTERNAL_WSS_URL\" --recipe-json` (exact flags + output in §7c). Set `--pairing-address` to the\nexternally reachable address so the emitted `pairingCode` is reachable; tunneling/port mapping is the\nscript's job.\n\nBackward compatibility: `command`→`create`, `cleanup`→`destroy`, `cleanup: none`→`destroy: none`.\nPrefer the lifecycle names.\n\n---\n\n## 9. Doctor and validation\n\nValidate in two stages — the cheap dry run first, then the live self-test.\n\n### Dry run (free, non-destructive) — always do this first\n\n`orca vm recipe doctor --repo-path --json` validates **static wiring only** — it does\n**not** boot anything. It checks: local-host execution (v1), repo path, recipe id exists,\ncreate/destroy/suspend/resume command paths resolve, suspend/resume are paired, and each script is\nexecutable (POSIX exec bit; skipped on Windows). Fix every failure here before spending any cloud money.\n\n### Live self-test (`--provision`) — diagnose and iterate yourself\n\n`orca vm recipe doctor --repo-path --provision --json` actually runs the recipe end\nto end: it executes `create`, validates the returned recipe JSON, then runs `destroy` to **tear the\nenvironment back down** (so the test leaves nothing running, as long as `destroy` works). It spends real\ncloud money, so get the user's OK **once** before starting — that one approval covers the whole loop\nbelow; do not re-ask before each run.\n\nOn failure, the JSON result includes a `provisionTranscript` with the **complete** captured output of\neach stage so you can self-diagnose without asking the user to relay logs:\n\n```json\n{\n \"ok\": false,\n \"checks\": [ { \"id\": \"recipe.provision\", \"status\": \"fail\", \"message\": \"…\" } ],\n \"provisionTranscript\": {\n \"provision\": { \"exitCode\": 0, \"signal\": null, \"stdout\": \"…\", \"stderr\": \"…\", \"parseError\": \"…\" },\n \"destroy\": { \"exitCode\": 0, \"signal\": null, \"stdout\": \"…\", \"stderr\": \"…\" }\n }\n}\n```\n\n**Run it as a loop:** read `provisionTranscript.provision.stderr` / `.stdout` / `.parseError` (and\n`destroy.*`), fix the script, and re-run `--provision` until `ok` is `true` — iterating on your own\nrather than waiting for the user to paste errors. Common reads: a non-empty `stderr` with `exitCode 0`\nplus a `parseError` means `create` ran but printed something other than the single recipe-result JSON on\nstdout (often a stray `echo` — route it to stderr, see §10); a non-zero `exitCode` is a provider/script\nfailure described in `stderr`. Each stream is redacted and capped (head+tail) — large logs keep both the\nsetup context and the failure.\n\nThe self-test cannot see provider-side truth beyond what the scripts print, so still confirm: state has a\npopulated **authenticated** `snapshotId` (Phases 2–3 done), and `destroy` is implemented/tested (or\nexplicitly `none` — in which case the self-test won't tear down, so clean up manually).\n\nFor SSH recipes, also smoke-test the exact emitted target before declaring success: dial the host/port\nwith the identity/proxy settings, run `pwd`, verify the repo path, check the agent binary, and confirm\n`destroy` removes the provider resource/container. For Docker, inspect the auth image entrypoint and do a\nstartup-only `docker run` before the full clone/install path.\n\n---\n\n## 10. Failure modes\n\n- **Build exceeds plan timeout (e.g. Hobby 45m).** Use enough vCPUs and a timeout covering the build;\n else split work or use a higher plan. The cap also limits per-workspace runtime — surface it.\n- **Build exceeds plan RAM.** Build the **headless main only** (drop the renderer) — the biggest fitter.\n- **Private-repo clone hangs/fails.** Wrong/missing token. Use `GIT_ASKPASS` + `GIT_TERMINAL_PROMPT=0`\n so it fails fast instead of prompting.\n- **`GIT_ASKPASS` helper aborts the clone with \"`$1: unbound variable`\".** The `printf`/heredoc that writes\n the helper inside `bash -lc` under `set -u` expanded `$1`/`$GH_TOKEN` at **write** time. Escape them\n (`\\$1`, `\\$GH_TOKEN`) so they land literally and resolve at git-runtime; this also keeps the real token\n out of the file. `rm -f` the helper afterward (§5, §7f).\n- **Agent verified as \"not logged in\" despite a good login.** `codex login status` (and similar) print\n \"Logged in …\" to **stderr**; an stdout-only `grep` misses it. Prefer the status **exit code**; if you\n grep, fold stderr first (`status 2>&1 | grep …`) and match the exact success line — not `grep -qi\n 'logged in'`, which also matches \"not logged in\".\n- **Headless agent login hangs.** Plain OAuth `login` starts a loopback callback server on a VM/container\n port the host browser can't reach. Use the **device-auth** flow (`login --device-auth`) — it prints a\n URL + code the user opens on the host.\n- **`known_hosts` host-key churn on local Docker.** Each ephemeral container regenerating its SSH host key\n collides on `127.0.0.1` as the published port rotates. Bake host keys into the base image at build time\n (`ssh-keygen -A`; runtime generates only if absent) so all containers share one stable key (§7h).\n- **Snapshot expired/evicted.** If `create` hits an unknown snapshot id, rerun Phases 2–3 and update\n `snapshotId`.\n- **Agent auth didn't persist.** Confirm `snapshotId` points at the **authenticated** snapshot; re-run\n Phase 3. Warn that short-lived tokens may need periodic re-auth.\n- **Agent auth copied from the host breaks.** Do not bind-mount/copy a full host agent home; sqlite\n files can be unwritable or host-specific, hooks may need approval again, and config may reference\n local-only env vars. Authenticate inside the runtime and snapshot/commit that layer.\n- **Docker auth image exits immediately.** Inspect `docker image inspect … .Config.Entrypoint` and\n `docker logs`. If the image was committed from an interactive shell, reset the entrypoint to the SSH\n entrypoint during `docker commit`.\n- **Leaked paid resource.** Every long script must trap errors and remove the sandbox it created.\n- **`create` emits non-JSON on stdout.** A stray `echo` corrupts the result — stdout is for the final\n JSON only; everything else to stderr. The `--provision` self-test surfaces this as `exitCode 0` + a\n `parseError` with the offending stdout in `provisionTranscript` (§9).\n\n---\n\n## 11. Boundaries\n\n- Don't create accounts, choose plans/regions, or invent scope/project/org/image/billing ids.\n- Don't invent or store credentials; no secrets in `userData`, state, comments, docs, or commits.\n- Don't run paid/long phases (base snapshot, auth, live test) without an explicit OK.\n- Don't hide provider errors behind generic messages — preserve actionable stderr.\n- Don't make Orca own provider lifecycle beyond invoking the configured scripts.\n- Don't commit or create an Orca workspace unless asked.\n" // oxfmt-ignore -const ORCHESTRATION_MARKDOWN = "---\nname: orchestration\ndescription: >-\n Use Orca orchestration for structured multi-agent coordination: threaded\n messages, blocking ask/reply flows, task dispatch, worker_done/escalation\n waits, task DAGs, decision gates, coordinator loops, or decomposing work\n across agents. Use `orca-cli` instead for full ownership handoffs, including\n requests phrased as \"hand off\", \"handoff\", \"handover\", \"give this to another\n agent\", or \"another worktree\" when the user did not explicitly ask to\n supervise, monitor, wait for results, or coordinate a DAG. Use `orca-cli` for\n ordinary terminal control, lightweight terminal prompts, shell commands, Orca\n worktree management, reading or waiting on terminals, and automation of the\n browser embedded inside Orca. Use Computer Use for browser windows, webviews,\n Orca app UI, or desktop UI outside Orca's embedded browser.\n---\n\n# Orca Inter-Agent Orchestration\n\nOrchestration is Orca's structured coordination layer for agent messages, task ownership, dispatch state, and worker completion tracking.\n\nUse this skill when coordination state matters. For lightweight terminal prompts or basic worktree/terminal/built-in-browser control, use `orca-cli`.\n\n## Tool Boundary\n\nIf a task says to use Orca orchestration, the coordinator must create or bind a Run, create the Task with `orca orchestration task-create`, then attach the worker with either the preferred `orca orchestration worker-start` composition or the low-level `orca orchestration dispatch --inject` path.\n\nDo not substitute non-Orca subagent tools, generic agent-spawn APIs, or chat-only parallel worker features. Those may create useful workers, but they do not create Orca task/dispatch provenance, injected lifecycle preambles, `worker_done` authority, or decision gates.\n\nBefore claiming a worker was orchestrated, verify the task/dispatch exists:\n\n```bash\norca orchestration task-list --json\norca orchestration dispatch-show --task --json\n```\n\nIf the work was accidentally run outside Orca orchestration, say so plainly. To repair provenance, rerun or revalidate the needed work through a fresh Orca terminal plus injected dispatch; do not retroactively describe the external worker as orchestrated.\n\n## When To Use\n\n- Send/reply/ask between agent terminals with persistent messages.\n- Dispatch structured tasks to workers and wait for `worker_done` or `escalation`.\n- Track task DAGs with dependencies.\n- Run coordinator loops or decision gates.\n\nDo not use orchestration merely because the user says \"hand off\", \"handoff\", \"handover\", \"give this to another agent\", or asks for another worktree/agent/model/effort. Those are full ownership transfers unless the user explicitly asks to supervise, monitor, wait for worker completion/results, coordinate a DAG, use decision gates, or keep a blocking ask/reply loop.\n\n## Preconditions\n\n- `orca status --json` should show a running runtime.\n- `orca` must be on PATH (`orca-ide` on Linux).\n- The orchestration experimental feature must be enabled in Settings > Experimental.\n- `orca orchestration` commands are RPC calls to the running Orca runtime.\n\n## Contract Migration\n\nOrca uses a hard cutover for orchestration mutations. It does not run a legacy scheduler, translate old writes, or drain pre-upgrade orchestration work.\n\nIf a command returns `orchestration_migration_required`, `run_required`, or a lifecycle validation error with `nextCommandArgs`:\n\n1. Confirm `effectsApplied` is `false`.\n2. Using the same CLI executable that returned the error, run the returned arguments: `skills get orchestration --full`.\n3. Read the guide completely. Do not retry the rejected command unchanged.\n4. Inspect the pre-upgrade Run, terminal, and assigned worktree before deciding whether any work needs replacement.\n5. If the legacy worker is still making valid progress, leave it as the only editor in that worktree and observe it manually, read-only, until it reaches a stable handoff point.\n6. Only then, if remaining work needs new lifecycle supervision, create or bind a lightweight Run, create a Task for the remaining work, and use `worker-start` in a conflict-free placement.\n\nThe arguments intentionally omit an executable name so this works with `orca`, `orca-ide`, `orca-dev`, or another configured Orca CLI command.\n\nThe cutover removes lifecycle authority; it does not cancel the prior assignment, invalidate its worktree, discard filesystem changes, or stop the worker process. Pre-upgrade terminals and agents can continue their valid assigned work, but they are no longer supervised by Orca: old heartbeat, question, completion, scheduler, reply, acknowledgment, and mutation calls are rejected before effects.\n\nLegacy database rows and terminal output remain available for explicit read-only inspection:\n\n```bash\norca orchestration run-list --json\norca orchestration run-show --id run_legacy_local --json\norca orchestration task-list --run run_legacy_local --json\norca orchestration inbox --full --json\norca orchestration check --terminal --peek --format --json\norca terminal read --terminal --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\n```\n\nRead-only inspection never consumes legacy mail. A stable handoff point means the worker has become idle, stopped, or completed a coherent edit/test/commit checkpoint; visible activity is a reason to keep observing, not to replace it. Do not prompt the worker to use old lifecycle commands.\n\nNever launch a replacement editor in the same worktree while the legacy worker may still write there. Wait for a stable handoff and preserve its filesystem work; if overlap is truly required, use a separate conflict-free worktree with an explicit plan for transferring existing dirty changes. Do not use actionable `check`, acknowledgment, reply, send, retry, or task updates against the legacy Run.\n\n## Ownership\n\nNew orchestration messages and tasks belong to one explicitly bound Run. A Run is only a durable namespace and coordinator inbox; it never schedules or places workers. Lifecycle authority comes from the active Dispatch, and terminal handles remain routing metadata rather than durable identity. Send `worker_done` and `heartbeat` from the worker's own terminal; Orca routes them to that Dispatch's Run.\n\nClassify inherited context before sending lifecycle messages:\n\n- Coordinated subtask: a live coordinator owns the DAG and waits on this dispatch. Follow the preamble exactly, including `worker_done`, heartbeat/status, `ask`, and `escalation`.\n- Full handoff means ownership transfer, not supervised dispatch. The original actor is not monitoring a DAG, so do not create lifecycle obligations unless the user explicitly asks you to supervise.\n- Classify requests containing \"hand off\", \"handoff\", \"handover\", \"give this to another agent\", \"give this to another worktree\", \"another agent\", or \"another worktree\" as full handoffs by default, even when the user names a custom model or reasoning effort.\n- Use supervised orchestration only when the user explicitly asks you to \"supervise\", \"monitor\", \"wait\", \"track completion\", \"wait for worker_done\", return results, coordinate a DAG, use a decision gate, or manage ask/reply flow.\n- Do not use `orca orchestration dispatch --inject` for full handoffs. It injects a coordinator preamble that tells the worker to send `worker_done`, heartbeat, and `ask` messages, then end its turn under the original terminal's dispatch lifecycle.\n- Do not run `orca orchestration task-create`, `orca orchestration dispatch --inject`, or `orca orchestration check --wait` for full handoffs. Do not peek at terminal output after prompt delivery to monitor progress.\n- A review-only `worker_done` reports findings; it does not authorize coordinator file edits. After a review-only completion, synthesize findings, ask a decision gate if ownership is unclear, and dispatch or hand off fixes unless the user explicitly asked the coordinator to own fixes.\n- If the user's plan names a next owner agent (for example, \"then use opencode to create a PR\"), post-review corrections and PR prep belong to that named owner. The coordinator routes, synthesizes, asks decision gates when needed, and supervises; the named owner edits files and creates the PR.\n\nIf unclear, inspect orchestration state before sending lifecycle messages:\n\n```bash\norca orchestration task-list --json\norca terminal list --json\n# If inherited context includes a task id:\norca orchestration dispatch-show --task --json\n```\n\n## Messaging\n\n```bash\norca orchestration send --subject [--to ] [--from ] [--body ] [--type ] [--priority ] [--thread-id ] [--payload ] [--json]\norca orchestration check [--terminal ] [--ack ] [--peek|--all] [--types ] [--format] [--wait] [--timeout-ms ] [--json]\norca orchestration reply --id --body [--from ] [--json]\norca orchestration ask (--question |--resume ) [--options ] [--timeout-ms ] [--from ] [--json]\norca orchestration inbox [--limit ] [--json]\n```\n\nRules:\n\n- Omit `--from` unless impersonating another terminal; Orca auto-resolves it from the current terminal.\n- A coordinator `check` returns the bound Run's oldest FIFO Delivery (up to 50 messages) and replays that exact batch until `--ack `. Process every message before acknowledging; `check --ack --wait` acknowledges, checks, and waits in one operation.\n- Use `--peek` and `--all` only for read-only history/debugging. Type filters decide when a waiter wakes; the returned actionable Delivery is still the oldest full batch.\n- Use `dispatch:` for coordinator guidance to one supervised worker. Orca routes that stable address locally or through the connected-server relay; do not substitute a remote terminal handle.\n- Terminal handles remain appropriate for low-level pre-Dispatch messaging. Prefer `agentTerminalHandle` from the create response, fall back to `startupTerminal.handle` for older runtimes, then re-resolve with `orca terminal list --worktree ... --json` if missing or stale. Continue with the replacement handle only; never dual-send to old and new handles.\n- `orca orchestration check --peek --format --json` returns locally formatted unread mail without consuming it; it never writes to terminal input or remotely wakes another terminal. Use `orchestration dispatch --inject` to deliver a tracked task, or `terminal send` when an existing agent needs a free-form prompt.\n- While supervising workers manually, use `check --wait --types worker_done,escalation,question --timeout-ms ` instead of sleep/poll loops. Process the whole Delivery, reply to `question` messages with `orca orchestration reply --id --body --json`, then acknowledge and keep waiting.\n- Treat a `check --wait` timeout or `{count:0}` as a checkpoint, not a worker failure. Long coding tasks routinely run 15-60 minutes; keep using rolling waits unless you receive `worker_done`/`escalation`, the terminal exits or disappears, or the user explicitly asks you to stop.\n- Heartbeats and visible terminal activity mean the worker is alive, not done. Do not stop, close, kill, or restart a worker just because it has not produced a completion message yet.\n- Use `ask` when a worker needs a blocking answer from the coordinator; it defaults to the active Dispatch's Run. Timeout or disconnect leaves the question pending, so resume by its original message ID instead of asking again.\n- `check --wait` returns one bounded Delivery, not every future completion. Process every message, acknowledge it, then keep waiting until every expected Dispatch settles.\n- Group addresses include `@all`, `@idle`, `@claude`, `@codex`, `@opencode`, `@gemini`, `@droid`, `@grok`, `@cursor`, and `@worktree:`.\n- Message types include `status`, `dispatch`, `worker_done`, `merge_ready`, `escalation`, `handoff`, `question`, `decision_gate` (legacy/gates), and `heartbeat`.\n- Use group addresses only for messages that are genuinely useful to many terminals, such as `status` broadcasts or intentional fan-out questions. Do not send dispatch lifecycle messages to groups.\n- `worker_done` belongs to the active Dispatch and defaults to its Run mailbox; never target a group.\n- A valid `worker_done` for the active `taskId` + `dispatchId` marks the task and dispatch completed automatically. Do not follow it with `task-update --status completed`; reserve manual updates for explicit recovery or overrides.\n- `heartbeat` is also Dispatch-scoped. Include both IDs and omit `--to` so Orca uses the owning Run; use `status` for broad progress updates.\n\n## Tasks And Dispatch\n\nA Run is the namespace/inbox, a Task is the work item, and a Dispatch assigns one Task attempt to a terminal. Create or bind a Run once before the common loop.\n\n```bash\norca orchestration run-create --objective --json\norca orchestration task-create --spec [--deps ] [--parent ] [--json]\norca orchestration task-list [--status ] [--ready] [--brief] [--json]\norca orchestration task-update --id --status [--result ] [--json]\norca orchestration dispatch --task --to [--from ] [--inject] [--json]\norca orchestration dispatch-show --task [--json]\n```\n\nTask statuses: `pending`, `ready`, `dispatched`, `completed`, `failed`, `blocked`.\n\nDispatch rules:\n\n- `--inject` sends the task spec plus preamble into a recognized agent CLI so it can report `worker_done`.\n- If the target is a bare shell, omit `--inject`, dispatch for tracking if needed, then send the prompt manually with `orca terminal send --terminal --text --enter --json`.\n- After 3 consecutive failures on one task, the dispatch context circuit-breaks and the task is marked failed.\n- Use `task-list --brief --json` for coordinator sweeps; it collapses whitespace and caps each echoed spec at 160 characters (`spec_truncated` marks shortened rows). Omit `--brief` when the full spec is required, or when an older CLI rejects it as an unknown flag.\n\n## Preferred Supervised Worker Loop\n\nUse `worker-start` for the normal supervised path. It composes the existing worktree, terminal, readiness, and dispatch primitives while returning exact created/reused effects. Agents still choose placement and concurrency; Orca does not schedule workers or infer conflicts.\n\nCreate the Run and every independent Task first, then start all independent workers before waiting:\n\n```bash\norca orchestration run-create --objective \"\" --json\norca orchestration task-create --spec \"\" --json\norca orchestration task-create --spec \"\" --json\norca orchestration worker-start --task --worktree current --agent codex --json\norca orchestration worker-start --task --worktree current --agent claude --json\n```\n\n`current` and exact existing worktrees create a fresh agent terminal and do not rerun setup. Reuse an existing agent only with `--terminal `.\n\nFor a new worktree, setup runs by default and agent-first creation reuses the returned startup agent terminal:\n\n```bash\norca orchestration worker-start --task --worktree new-child --name --agent codex --setup run --json\n# Independent/top-level:\norca orchestration worker-start --task --worktree new-top-level --name --agent codex --setup run --json\n```\n\nSetup normally starts alongside the agent. Only a repository explicitly configured with `wait-for-setup` delays agent launch until setup succeeds. Use `--setup skip` or `--setup inherit` only for a concrete reason.\n\nRead the returned receipt before continuing: `ready` plus setup `running` is normal for start-immediately, while wait-for-setup returns setup `succeeded` before accepting task input. A failed or unknown start exits nonzero; inspect its `stage`, `effects`, and `residualResources` instead of guessing or automatically retrying. A wait-for-setup timeout can honestly leave setup `running`, which is not proof of failure.\n\nTo run the worker on another connected Orca server, add `--on `. The Run and Tasks remain authoritative on the current server; later commands route by Dispatch ID, so never repeat `--on`:\n\n```bash\n# Mac Run home -> Windows worker (the reverse is identical from a Windows Run home)\norca orchestration worker-start --task --on windows --worktree new-top-level --repo --name --agent codex --setup run --json\norca orchestration worker-show --dispatch --json\norca orchestration worker-read --dispatch --limit 50 --json\norca orchestration send --to dispatch: --subject \"Follow-up\" --body \"\" --json\n```\n\nRemote `current` and `new-child` are intentionally invalid because those words are ambiguous across servers. Use an exact discovered remote worktree selector or `new-top-level` with an explicit remote repo selector.\n\nThe follow-up is structured inbox mail, not prompt injection. The worker's next\n`orchestration check` receives it even when the Dispatch is on another connected Orca server.\n\n`worker-read` defaults to `--source auto`: Orca returns the exact hook-reported Codex, Claude, OpenClaude, or Grok transcript when it can prove the worker session, otherwise it returns bounded terminal output with `source: \"terminal\"` and a typed `fallbackReason`. Continue with the returned top-level `cursor`; it stays pinned to that exact source. If Orca reports `source_changed`, start a fresh read without the old cursor. Never supply or guess a provider session ID or transcript path.\n\nWait until every expected Dispatch settles, not for a fixed number of batches:\n\n```bash\norca orchestration check --wait --types worker_done,escalation,question --timeout-ms 900000 --json\n# Process every message in the returned Delivery, then atomically ack and continue:\norca orchestration check --ack --wait --types worker_done,escalation,question --timeout-ms 900000 --json\n```\n\nWorkers report exactly once using the IDs and capability injected by Orca; they do not supply Run/server/terminal identity:\n\n```bash\norca orchestration send --type worker_done --subject \"\" --body \"\" --task-id --dispatch-id --outcome succeeded --files-modified \"path/a,path/b\" --json\n# On failure, use --outcome failed; never encode failure only in prose.\n```\n\nA worker question defaults to its owning Run. Timeout leaves it pending:\n\n```bash\norca orchestration ask --question \"\" --options \"yes,no\" --timeout-ms 600000 --json\norca orchestration ask --resume --timeout-ms 600000 --json\n# Coordinator:\norca orchestration reply --id --body \"\" --json\n```\n\nRecovery is conditional, never a fixed destructive sequence:\n\n- `worker-show --dispatch ` says `ready`: keep waiting or read bounded output.\n- It proves `failed` or `stopped`: start a replacement with `worker-start --task --retry-of ` plus an explicit `--on`/`--worktree` and `--agent`/`--terminal` choice. Retry does not silently inherit placement.\n- It remains `outcome_unknown`: either `worker-stop --dispatch ` and inspect again, or explicitly `worker-abandon --dispatch ` while accepting that resources may still be live. Abandon performs no remote, process, or filesystem action.\n- `worker-stop` closes only the exact supervised agent terminal. It never deletes the worktree, setup terminal, configured tabs, or unrelated processes.\n\nLow-level `worktree create`, `terminal create`, and `dispatch --inject` remain valid recipes for custom argv or topology that `worker-start` does not express.\n\n## Gates And Legacy Inspection\n\n```bash\norca orchestration gate-create --task --question [--options ] [--json]\norca orchestration gate-resolve --id --resolution [--json]\norca orchestration gate-list [--task ] [--status ] [--json]\n```\n\nUse `ask` for worker-to-coordinator questions; it creates a `question` message that the coordinator answers with `reply`. Use `gate-create` only for coordinator-managed task DAG decisions, not for answering a worker's `ask`.\n\n`coordinator-start`, `coordinator-stop`, `run`, and `run-stop` are retired scheduler commands. They perform no effects and return the current-skill recovery action. They are not aliases for lightweight Run creation or binding.\n\nRecovery only: `orca orchestration reset --tasks|--messages|--all --json` clears the selected local orchestration database state. Do not run it during active coordination unless explicitly abandoning that state.\n\n## Full Handoffs\n\nFor full ownership transfer, use non-lifecycle terminal/worktree commands and then stop monitoring unless the user asks for supervision.\n\nTreat these as full handoff requests by default: \"hand off\", \"handoff\", \"handover\", \"give this to another agent\", \"give this to another worktree\", \"send this to another agent\", \"another agent\", \"another worktree\", or \"launch another agent to own this.\" Custom model or reasoning effort words such as `gpt-5.5`, `high`, or `xhigh` do not make the handoff supervised.\n\nSupervised orchestration remains available only when the user explicitly asks for supervision or coordination: \"supervise\", \"monitor\", \"wait for worker_done\", \"wait for results\", \"track completion\", \"DAG\", \"decision gate\", \"ask/reply\", or \"coordinate workers.\"\n\nDo not run `orca orchestration task-create`, `orca orchestration dispatch --inject`, or `orca orchestration check --wait` for full handoffs. `task-create` is also forbidden because it records coordinator-owned tracking state; if a task row is needed, the user asked for supervised orchestration. Do not create a `taskId`/`dispatchId`, inject a lifecycle preamble, wait for completion, or read the worker terminal after prompt delivery except to avoid losing the initial prompt.\n\nNew top-level worktree handoff:\n\n```bash\norca worktree create --name --no-parent --agent codex --prompt \"\" --setup run --json\n```\n\nBefore creating a new worktree from an active feature branch, decide and state whether the desired Orca lineage is child or top-level. Use child worktree lineage only when the new work is conceptually stacked under or dependent on the active worktree. For independent repo-wide fixes, standalone feature work, or unrelated follow-up tasks, create a top-level worktree with `--no-parent`.\n\nExisting terminal handoff:\n\n```bash\norca terminal send --terminal --text \"\" --enter --json\n```\n\nCustom Codex model/effort handoff:\n\n`orca worktree create --agent codex --prompt ...` launches the known Codex agent but does not accept Codex-specific `--model` or `-c model_reasoning_effort=...` arguments. When the user asks for a specific Codex model or effort, create the independent worktree first, launch Codex with the requested command in that worktree, wait only for TUI readiness if prompt delivery would otherwise race startup, send the prompt, and stop.\n\nThe two-step custom-argv path cannot enforce a repository's explicit `wait-for-setup` startup policy because the later `terminal create` is not the startup owned by `worktree create`. Use it only when the repository starts agents immediately. If the repository requires `wait-for-setup`, use an agent-first configured launcher that can preserve sequencing, or stop and ask rather than silently bypassing the policy.\n\nNote: when no repo default-terminal configuration supplies a primary terminal, bare create opens a fallback shell before `terminal create` adds the agent. Configured default tabs are materialized instead and may run real commands. Prefer `--agent` whenever custom argv is not required. With the two-step path, target only the agent handle; close a prior terminal only after `terminal list` or `terminal show` confirms it is an unused shell.\n\nUse the exact full `::` worktree id returned by `orca worktree create --json`; a bare repo id cannot target the new worktree.\n\n```bash\norca worktree create --name --no-parent --setup run --json\norca terminal create --worktree id: --title --command 'codex --model gpt-5.5 -c model_reasoning_effort=\"xhigh\"' --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca terminal send --terminal --text \"\" --enter --json\n```\n\nWait only for `tui-idle` when needed to avoid losing the prompt. Do not monitor task completion.\n\n`--no-parent` only controls Orca lineage; it does not choose the Git base. If the work should start from the repo default base, omit `--base-branch` so Orca uses that default, or explicitly pass the repo default base (`origin/main`, `origin/master`, or the `orca repo show --repo --json` value); never base it on the current feature branch unless the user explicitly asks for stacked work or \"branch from current\". Put current-branch context in the prompt instead.\n\n## Worker Terminals\n\nChoose the worker location before creating a terminal. `Fresh worker` means a fresh agent session, not a new git worktree. For parallel work, create one fresh agent terminal per worker in the same required worktree, falling back to the active worktree when none is named. If the task says current worktree only, depends on uncommitted files/artifacts, or must validate/PR the current branch, keep every worker in the active worktree:\n\n```bash\norca terminal create --worktree active --title --command \"codex\" --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca orchestration dispatch --task --to --inject --json\n```\n\nReuse an idle agent in the required worktree only if the prompt allows reuse; otherwise create a fresh terminal there. Create a new worktree only when the user explicitly requests one or a concrete checkout or filesystem conflict makes sharing unsafe or impossible; if the user did not request it, state that conflict before running `worktree create`. Independent tasks, parallel execution, convenience, or a preference for separate checkouts are not isolation requirements.\n\nWhen a new worktree is allowed, use child lineage for isolated work that is stacked under or dependent on the active worktree, and use `--no-parent` when it is not stacked. Decide the Git base separately: `--no-parent` makes the worktree top-level in Orca, while omitted `--base-branch` uses the repo default base.\n\nFor every new worktree, pass `--setup run` so any configured repository setup hook runs. This does not mean waiting for setup before agent launch: preserve the repository's startup policy, whose default starts setup and the agent side by side. Use `--setup skip` or `--setup inherit` only when there is a concrete task-specific reason, and state that reason before creating the worktree. This rule does not rerun setup for current or existing worktrees.\n\n```bash\norca worktree create --name --agent codex --setup run --json\n# or: --agent claude | omp | pi | grok | ...\n# Read from agentTerminalHandle, falling back to startupTerminal.handle.\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca orchestration dispatch --task --to --inject --json\n```\n\nFor new-worktree workers, read the id and `agentTerminalHandle` from `worktree create`, falling back to `startupTerminal.handle` for older runtimes. Use that as the sole worker handle when present; otherwise use `terminal list` to resolve the agent handle. Omit `--repo` only inside an Orca-managed worktree; otherwise pass `--repo `.\n\n**For an allowed new worktree, use agent-first:** `--agent` reveals the new worktree and launches the selected agent **in its first terminal**, without adding a separate fallback shell for that worker. Pass `--setup run`; repo setup and default-terminal settings may add intentional tabs or splits. Do **not** run bare `worktree create` and then `terminal create --command ` for the same worker when agent-first create is available: without configured default tabs, that two-step path leaves a fallback shell + agent pair. Only use it when custom agent argv is required (for example Codex model/effort flags) or when an older CLI rejects `--agent`; if you must, message only the agent handle. Configured default tabs are intentional surfaces, so close a prior terminal only after `terminal list` or `terminal show` confirms it is an unused shell. Do not run `worktree create` when the task must stay in the current worktree.\n\nUse `orca worktree create --prompt ...` or `orca terminal send ...` for full handoffs or untracked/lightweight prompts. Those paths do not attach `taskId`/`dispatchId`; the worker should not send lifecycle messages unless the prompt supplies a live orchestration preamble.\n\nSidebar lineage and orchestration lifecycle are related but not identical. A same-worktree worker may appear as a peer under that worktree in the sidebar while remaining a child dispatch in orchestration state; only an actual child worktree creates visible parent/child worktree lineage.\n\nOther terminal commands coordinators often need:\n\n```bash\norca terminal list [--worktree ] [--json]\norca terminal create [--worktree ] [--title ] [--command ] [--json]\norca terminal split --terminal [--direction horizontal|vertical] [--command ] [--json]\norca terminal wait --terminal --for tui-idle --timeout-ms --json\norca terminal read --terminal --json\norca terminal send --terminal --text --enter --json\n```\n\nIf an older CLI rejects `worktree create --agent`, create the worktree normally, then run `orca terminal create --worktree --command \"codex\" --json` or `--command \"claude\"`.\n\nWait for `tui-idle` before dispatching. Always pass `--timeout-ms`; real coding tasks can take 15-60 minutes. During supervision, use rolling `check --wait` windows. If a window returns no matching message, inspect `task-list`, `terminal read`, or `terminal wait --for tui-idle` as a liveness checkpoint; if the terminal is still working or producing activity, keep waiting instead of retrying the task.\n\n## Agent Guidance\n\n- Workers with a valid live preamble must send `worker_done` exactly once from their own terminal with an explicit `--outcome succeeded` or `--outcome failed`:\n `orca orchestration send --type worker_done --subject \"\" --body \"<3-sentence summary: what you did, what you found, what's left>\" --task-id --dispatch-id --outcome succeeded --files-modified \"path/a\" --report-path \"\" --json`\n- A failed outcome is still a terminal report, but Orca records both the Dispatch and Task as failed. Never encode failure only in the subject/body.\n- After sending `worker_done`, end your turn and idle at the agent prompt. Do not poll or keep calling `orca orchestration check`; the coordinator re-engages you with a fresh preamble + TASK block delivered as new terminal input.\n- For long tasks, send heartbeat/status only when the preamble asks for it, including both IDs:\n `orca orchestration send --type heartbeat --subject \"alive\" --payload '{\"taskId\":\"\",\"dispatchId\":\"\",\"phase\":\"implementing\"}' --json`\n- If blocked before completion, use `ask`; use `escalation` only when ownership is valid and the coordinator must intervene.\n- Treat preambles inherited through terminal history or full handoffs as stale unless the current prompt explicitly keeps that coordinator in the loop.\n- Coordinators should use `task-list --ready` as external memory, dispatch parallel waves, and avoid dependency chains deeper than 3-4 steps.\n\n## Example\n\n```bash\norca terminal create --worktree active --title login-css-worker --command \"claude\" --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca orchestration task-create --spec \"Fix the login button CSS\" --json\norca orchestration dispatch --task --to --inject --json\norca orchestration check --wait --types worker_done,escalation,question --timeout-ms 900000 --json\n```\n\n## Next Action\n\nCoordinator: confirm `orca status --json`, create or bind a Run, inspect `task-list`/`dispatch-show` if inheriting state, then use the explicit supervised loop (`task-create` -> `worker-start` -> `check --wait`). Use low-level terminal creation plus `dispatch --inject` only when the composed start does not express the needed topology.\n\nWorker: if the current prompt contains a live dispatch preamble, do the task, use `ask` for blocking questions, and send `worker_done` once with the required payload. If the preamble is stale or absent, do not send lifecycle messages; inspect state or treat the prompt as an ordinary handoff.\n" +const ORCHESTRATION_MARKDOWN = "---\nname: orchestration\ndescription: >-\n Use Orca orchestration for structured multi-agent coordination: threaded\n messages, blocking ask/reply flows, task dispatch, worker_done/escalation\n waits, task DAGs, decision gates, coordinator loops, or decomposing work\n across agents. Use `orca-cli` instead for full ownership handoffs, including\n requests phrased as \"hand off\", \"handoff\", \"handover\", \"give this to another\n agent\", or \"another worktree\" when the user did not explicitly ask to\n supervise, monitor, wait for results, or coordinate a DAG. Use `orca-cli` for\n ordinary terminal control, lightweight terminal prompts, shell commands, Orca\n worktree management, reading or waiting on terminals, and automation of the\n browser embedded inside Orca. Use Computer Use for browser windows, webviews,\n Orca app UI, or desktop UI outside Orca's embedded browser.\n---\n\n# Orca Inter-Agent Orchestration\n\nOrchestration is Orca's structured coordination layer for agent messages, task ownership, dispatch state, and worker completion tracking.\n\nUse this skill when coordination state matters. For lightweight terminal prompts or basic worktree/terminal/built-in-browser control, use `orca-cli`.\n\n## Tool Boundary\n\nIf a task says to use Orca orchestration, the coordinator must create or bind a Run, create the Task with `orca orchestration task-create`, then attach the worker with either the preferred `orca orchestration worker-start` composition or the low-level `orca orchestration dispatch --inject` path.\n\nDo not substitute non-Orca subagent tools, generic agent-spawn APIs, or chat-only parallel worker features. Those may create useful workers, but they do not create Orca task/dispatch provenance, injected lifecycle preambles, `worker_done` authority, or decision gates.\n\nBefore claiming a worker was orchestrated, verify the task/dispatch exists:\n\n```bash\norca orchestration task-list --json\norca orchestration dispatch-show --task --json\n```\n\nIf the work was accidentally run outside Orca orchestration, say so plainly. To repair provenance, rerun or revalidate the needed work through a fresh Orca terminal plus injected dispatch; do not retroactively describe the external worker as orchestrated.\n\n## When To Use\n\n- Send/reply/ask between agent terminals with persistent messages.\n- Dispatch structured tasks to workers and wait for `worker_done` or `escalation`.\n- Track task DAGs with dependencies.\n- Run coordinator loops or decision gates.\n\nDo not use orchestration merely because the user says \"hand off\", \"handoff\", \"handover\", \"give this to another agent\", or asks for another worktree/agent/model/effort. Those are full ownership transfers unless the user explicitly asks to supervise, monitor, wait for worker completion/results, coordinate a DAG, use decision gates, or keep a blocking ask/reply loop.\n\n## Preconditions\n\n- `orca status --json` should show a running runtime.\n- `orca` must be on PATH (`orca-ide` on Linux).\n- The orchestration experimental feature must be enabled in Settings > Experimental.\n- `orca orchestration` commands are RPC calls to the running Orca runtime.\n\n## Contract Migration\n\nOrca adopts a live pre-update orchestration assignment into an ordinary Run. Adoption preserves the existing agent process, PTY/session, terminal handle, tab/leaf/pane, worktree or folder workspace, Task, and Dispatch; it never restarts or replaces the worker. The retired scheduler is not revived, and a newly created attempt uses the current grammar.\n\nTreat the authority label on injected or formatted messages as definitive:\n\n- `[LEGACY COMPATIBILITY]` is live and attested. Run only the exact supported command printed with the message, using the same CLI executable and arguments that the original prompt supplied.\n- `[LEGACY RECOVERY REPLAY — MAY HAVE BEEN SEEN]` is one bounded, at-least-once cutover replay. Process it idempotently and acknowledge it only through the exact displayed guidance.\n- `[LEGACY READ-ONLY]` is inspection-only. It has no reply, acknowledgment, or lifecycle action.\n- An unlabeled current message uses the current guide and current grammar.\n\nDatabase provenance, an old-looking terminal, or a legacy Run ID does not prove mutation authority. If the runtime cannot prove liveness, principal ownership, capability, or the exact legacy contract, it degrades to read-only inspection and must not fall back to local execution. Exact recovery may restore the already-live PTY once in its original inactive background tab. It must not spawn, write, signal, stop, switch, focus, split, or inject a terminal. Loss of lifecycle authority does not invalidate the existing assignment, process, or filesystem work.\n\nCompatibility retries have narrow guarantees. A pending ask, a reply, a final Dispatch settlement, and a consuming check have durable recovery identities. A-era heartbeat and escalation calls remain at-least-once across a manual A-to-B retry because identical later signals may be intentional. If an A-era ask may already have been answered, run the exact non-consuming recovery check printed by the runtime first; after its answer is printed and acknowledged, a new invocation with the same question creates a new question. Never guess among multiple identical question threads.\n\nWhen a compatibility or recovery command returns structured next-step arguments, run those exact arguments with the same CLI executable. The arguments intentionally omit the executable name so the guidance works with `orca`, `orca-ide`, `orca-dev`, or another configured Orca CLI command. Do not translate the command from memory, broaden its recipient, or retry it as a current mutation unless the returned guidance explicitly says to.\n\nOn packaged Windows, a legacy ask uses a two-step commit/resume protocol. The initial command durably commits the question, prints its exact `ask --resume ` command, and exits with launcher status `75`; it does not wait for the answer. Run that exact resume command after the launcher or update boundary. Resume is idempotent and read-oriented: it waits for the already-committed question and does not create another one. For a WSL process that received compatibility proof at launch, use the printed executable `orca-ide` WSL resume command so the same distro and packaged launcher authority are preserved; do not substitute a PATH-resolved local CLI. Older WSL processes that never received the hidden launch token remain lifecycle read-only after the update, even while their terminal and filesystem work continue.\n\nLegacy inspection remains available without consuming mail:\n\n```bash\norca orchestration run-list --json\n# run_legacy_local is an empty audit tombstone after adoption.\norca orchestration run-show --id run_legacy_local --json\n# In run-list, find the ordinary Run whose objective is:\n# \"Recovered orchestration work from a contract update\"\norca orchestration run-show --id --json\norca orchestration task-list --run --json\norca orchestration inbox --full --json\norca orchestration check --terminal --peek --format --json\norca terminal read --terminal --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\n```\n\nIf the original coordinator is unavailable or cannot prove its retained authority, a current coordinator may explicitly take over the adopted Run from its own live agent terminal:\n\n```bash\norca orchestration run-use --id --takeover-legacy --json\norca orchestration check --run --json\n```\n\nTakeover fences only the old coordinator, binds the current one, and moves pending worker mail into current Run Delivery. It is bound to the authenticated invoking terminal; `--from` cannot name another coordinator. Live legacy workers keep their original Tasks, Dispatches, processes, filesystems, and old prompt commands; their later questions, escalations, and completion reports route to the current coordinator. Do not use takeover while the original coordinator is still actively coordinating, because its later lifecycle mutations are rejected.\n\nDo not launch a replacement editor merely because the desktop app or runtime was updated. If adoption cannot prove continuing authority, keep the original worker as the only editor until it reaches a stable handoff point, then use a new current Dispatch in a conflict-free placement for any remaining work.\n\n## Ownership\n\nNew orchestration messages and tasks belong to one explicitly bound Run. A Run is only a durable namespace and coordinator inbox; it never schedules or places workers. Lifecycle authority comes from the active Dispatch, and terminal handles remain routing metadata rather than durable identity. Send `worker_done` and `heartbeat` from the worker's own terminal; Orca routes them to that Dispatch's Run.\n\nClassify inherited context before sending lifecycle messages:\n\n- Coordinated subtask: a live coordinator owns the DAG and waits on this dispatch. Follow the preamble exactly, including `worker_done`, heartbeat/status, `ask`, and `escalation`.\n- Full handoff means ownership transfer, not supervised dispatch. The original actor is not monitoring a DAG, so do not create lifecycle obligations unless the user explicitly asks you to supervise.\n- Classify requests containing \"hand off\", \"handoff\", \"handover\", \"give this to another agent\", \"give this to another worktree\", \"another agent\", or \"another worktree\" as full handoffs by default, even when the user names a custom model or reasoning effort.\n- Use supervised orchestration only when the user explicitly asks you to \"supervise\", \"monitor\", \"wait\", \"track completion\", \"wait for worker_done\", return results, coordinate a DAG, use a decision gate, or manage ask/reply flow.\n- Do not use `orca orchestration dispatch --inject` for full handoffs. It injects a coordinator preamble that tells the worker to send `worker_done`, heartbeat, and `ask` messages, then end its turn under the original terminal's dispatch lifecycle.\n- Do not run `orca orchestration task-create`, `orca orchestration dispatch --inject`, or `orca orchestration check --wait` for full handoffs. Do not peek at terminal output after prompt delivery to monitor progress.\n- A review-only `worker_done` reports findings; it does not authorize coordinator file edits. After a review-only completion, synthesize findings, ask a decision gate if ownership is unclear, and dispatch or hand off fixes unless the user explicitly asked the coordinator to own fixes.\n- If the user's plan names a next owner agent (for example, \"then use opencode to create a PR\"), post-review corrections and PR prep belong to that named owner. The coordinator routes, synthesizes, asks decision gates when needed, and supervises; the named owner edits files and creates the PR.\n\nIf unclear, inspect orchestration state before sending lifecycle messages:\n\n```bash\norca orchestration task-list --json\norca terminal list --json\n# If inherited context includes a task id:\norca orchestration dispatch-show --task --json\n```\n\n## Messaging\n\n```bash\norca orchestration send --subject [--to ] [--from ] [--body ] [--type ] [--priority ] [--thread-id ] [--payload ] [--json]\norca orchestration check [--terminal ] [--ack ] [--peek|--all] [--types ] [--format] [--wait] [--timeout-ms ] [--json]\norca orchestration reply --id --body [--from ] [--json]\norca orchestration ask (--question |--resume ) [--options ] [--timeout-ms ] [--from ] [--json]\norca orchestration inbox [--limit ] [--json]\n```\n\nRules:\n\n- Omit `--from` unless impersonating another terminal; Orca auto-resolves it from the current terminal.\n- A coordinator `check` returns the bound Run's oldest FIFO Delivery (up to 50 messages) and replays that exact batch until `--ack `. Process every message before acknowledging; `check --ack --wait` acknowledges, checks, and waits in one operation.\n- Use `--peek` and `--all` only for read-only history/debugging. Type filters decide when a waiter wakes; the returned actionable Delivery is still the oldest full batch.\n- Use `dispatch:` for coordinator guidance to one supervised worker. Orca routes that stable address locally or through the connected-server relay; do not substitute a remote terminal handle.\n- Terminal handles remain appropriate for low-level pre-Dispatch messaging. Prefer `agentTerminalHandle` from the create response, fall back to `startupTerminal.handle` for older runtimes, then re-resolve with `orca terminal list --worktree ... --json` if missing or stale. Continue with the replacement handle only; never dual-send to old and new handles.\n- `orca orchestration check --peek --format --json` returns locally formatted unread mail without consuming it; it never writes to terminal input or remotely wakes another terminal. Use `orchestration dispatch --inject` to deliver a tracked task, or `terminal send` when an existing agent needs a free-form prompt.\n- While supervising workers manually, use `check --wait --types worker_done,escalation,question --timeout-ms ` instead of sleep/poll loops. Process the whole Delivery, reply to `question` messages with `orca orchestration reply --id --body --json`, then acknowledge and keep waiting.\n- Treat a `check --wait` timeout or `{count:0}` as a checkpoint, not a worker failure. Long coding tasks routinely run 15-60 minutes; keep using rolling waits unless you receive `worker_done`/`escalation`, the terminal exits or disappears, or the user explicitly asks you to stop.\n- Heartbeats and visible terminal activity mean the worker is alive, not done. Do not stop, close, kill, or restart a worker just because it has not produced a completion message yet.\n- Use `ask` when a worker needs a blocking answer from the coordinator; it defaults to the active Dispatch's Run. Timeout or disconnect leaves the question pending, so resume by its original message ID instead of asking again.\n- `check --wait` returns one bounded Delivery, not every future completion. Process every message, acknowledge it, then keep waiting until every expected Dispatch settles.\n- Group addresses include `@all`, `@idle`, `@claude`, `@codex`, `@opencode`, `@gemini`, `@droid`, `@grok`, `@cursor`, and `@worktree:`.\n- Message types include `status`, `dispatch`, `worker_done`, `merge_ready`, `escalation`, `handoff`, `question`, `decision_gate` (legacy/gates), and `heartbeat`.\n- Use group addresses only for messages that are genuinely useful to many terminals, such as `status` broadcasts or intentional fan-out questions. Do not send dispatch lifecycle messages to groups.\n- `worker_done` belongs to the active Dispatch and defaults to its Run mailbox; never target a group.\n- A valid `worker_done` for the active `taskId` + `dispatchId` marks the task and dispatch completed automatically. Do not follow it with `task-update --status completed`; reserve manual updates for explicit recovery or overrides.\n- `heartbeat` is also Dispatch-scoped. Include both IDs and omit `--to` so Orca uses the owning Run; use `status` for broad progress updates.\n\n## Tasks And Dispatch\n\nA Run is the namespace/inbox, a Task is the work item, and a Dispatch assigns one Task attempt to a terminal. Create or bind a Run once before the common loop.\n\n```bash\norca orchestration run-create --objective --json\norca orchestration task-create --spec [--deps ] [--parent ] [--json]\norca orchestration task-list [--status ] [--ready] [--brief] [--json]\norca orchestration task-update --id --status [--result ] [--json]\norca orchestration dispatch --task --to [--from ] [--inject] [--json]\norca orchestration dispatch-show --task [--json]\n```\n\nTask statuses: `pending`, `ready`, `dispatched`, `completed`, `failed`, `blocked`.\n\nDispatch rules:\n\n- `--inject` sends the task spec plus preamble into a recognized agent CLI so it can report `worker_done`.\n- If the target is a bare shell, omit `--inject`, dispatch for tracking if needed, then send the prompt manually with `orca terminal send --terminal --text --enter --json`.\n- After 3 consecutive failures on one task, the dispatch context circuit-breaks and the task is marked failed.\n- Use `task-list --brief --json` for coordinator sweeps; it collapses whitespace and caps each echoed spec at 160 characters (`spec_truncated` marks shortened rows). Omit `--brief` when the full spec is required, or when an older CLI rejects it as an unknown flag.\n\n## Preferred Supervised Worker Loop\n\nUse `worker-start` for the normal supervised path. It composes the existing worktree, terminal, readiness, and dispatch primitives while returning exact created/reused effects. Agents still choose placement and concurrency; Orca does not schedule workers or infer conflicts.\n\nCreate the Run and every independent Task first, then start all independent workers before waiting:\n\n```bash\norca orchestration run-create --objective \"\" --json\norca orchestration task-create --spec \"\" --json\norca orchestration task-create --spec \"\" --json\norca orchestration worker-start --task --worktree current --agent codex --json\norca orchestration worker-start --task --worktree current --agent claude --json\n```\n\n`current` and exact existing worktrees create a fresh agent terminal and do not rerun setup. Reuse an existing agent only with `--terminal `.\n\nFor a new worktree, setup runs by default and agent-first creation reuses the returned startup agent terminal:\n\n```bash\norca orchestration worker-start --task --worktree new-child --name --agent codex --setup run --json\n# Independent/top-level:\norca orchestration worker-start --task --worktree new-top-level --name --agent codex --setup run --json\n```\n\nSetup normally starts alongside the agent. Only a repository explicitly configured with `wait-for-setup` delays agent launch until setup succeeds. Use `--setup skip` or `--setup inherit` only for a concrete reason.\n\nRead the returned receipt before continuing: `ready` plus setup `running` is normal for start-immediately, while wait-for-setup returns setup `succeeded` before accepting task input. A failed or unknown start exits nonzero; inspect its `stage`, `effects`, and `residualResources` instead of guessing or automatically retrying. A wait-for-setup timeout can honestly leave setup `running`, which is not proof of failure.\n\nTo run the worker on another connected Orca server, add `--on `. The Run and Tasks remain authoritative on the current server; later commands route by Dispatch ID, so never repeat `--on`:\n\n```bash\n# Mac Run home -> Windows worker (the reverse is identical from a Windows Run home)\norca orchestration worker-start --task --on windows --worktree new-top-level --repo --name --agent codex --setup run --json\norca orchestration worker-show --dispatch --json\norca orchestration worker-read --dispatch --limit 50 --json\norca orchestration send --to dispatch: --subject \"Follow-up\" --body \"\" --json\n```\n\nRemote `current` and `new-child` are intentionally invalid because those words are ambiguous across servers. Use an exact discovered remote worktree selector or `new-top-level` with an explicit remote repo selector.\n\nThe follow-up is structured inbox mail, not prompt injection. The worker's next\n`orchestration check` receives it even when the Dispatch is on another connected Orca server.\n\n`worker-read` defaults to `--source auto`: Orca returns the exact hook-reported Codex, Claude, OpenClaude, or Grok transcript when it can prove the worker session, otherwise it returns bounded terminal output with `source: \"terminal\"` and a typed `fallbackReason`. Continue with the returned top-level `cursor`; it stays pinned to that exact source. If Orca reports `source_changed`, start a fresh read without the old cursor. Never supply or guess a provider session ID or transcript path.\n\nWait until every expected Dispatch settles, not for a fixed number of batches:\n\n```bash\norca orchestration check --wait --types worker_done,escalation,question --timeout-ms 900000 --json\n# Process every message in the returned Delivery, then atomically ack and continue:\norca orchestration check --ack --wait --types worker_done,escalation,question --timeout-ms 900000 --json\n```\n\nWorkers report exactly once using the IDs and capability injected by Orca; they do not supply Run/server/terminal identity:\n\n```bash\norca orchestration send --type worker_done --subject \"\" --body \"\" --task-id --dispatch-id --outcome succeeded --files-modified \"path/a,path/b\" --json\n# On failure, use --outcome failed; never encode failure only in prose.\n```\n\nA worker question defaults to its owning Run. Timeout leaves it pending:\n\n```bash\norca orchestration ask --question \"\" --options \"yes,no\" --timeout-ms 600000 --json\norca orchestration ask --resume --timeout-ms 600000 --json\n# Coordinator:\norca orchestration reply --id --body \"\" --json\n```\n\nRecovery is conditional, never a fixed destructive sequence:\n\n- `worker-show --dispatch ` says `ready`: keep waiting or read bounded output.\n- It proves `failed` or `stopped`: start a replacement with `worker-start --task --retry-of ` plus an explicit `--on`/`--worktree` and `--agent`/`--terminal` choice. Retry does not silently inherit placement.\n- It remains `outcome_unknown`: either `worker-stop --dispatch ` and inspect again, or explicitly `worker-abandon --dispatch ` while accepting that resources may still be live. Abandon performs no remote, process, or filesystem action.\n- `worker-stop` closes only the exact supervised agent terminal. It never deletes the worktree, setup terminal, configured tabs, or unrelated processes.\n\nLow-level `worktree create`, `terminal create`, and `dispatch --inject` remain valid recipes for custom argv or topology that `worker-start` does not express.\n\n## Gates And Legacy Inspection\n\n```bash\norca orchestration gate-create --task --question [--options ] [--json]\norca orchestration gate-resolve --id --resolution [--json]\norca orchestration gate-list [--task ] [--status ] [--json]\n```\n\nUse `ask` for worker-to-coordinator questions; it creates a `question` message that the coordinator answers with `reply`. Use `gate-create` only for coordinator-managed task DAG decisions, not for answering a worker's `ask`.\n\n`coordinator-start`, `coordinator-stop`, `run`, and `run-stop` are retired scheduler commands. They perform no effects and return the current-skill recovery action. They are not aliases for lightweight Run creation or binding.\n\nRecovery only: `orca orchestration reset --tasks|--messages|--all --json` clears the selected local orchestration database state. Do not run it during active coordination unless explicitly abandoning that state.\n\n## Full Handoffs\n\nFor full ownership transfer, use non-lifecycle terminal/worktree commands and then stop monitoring unless the user asks for supervision.\n\nTreat these as full handoff requests by default: \"hand off\", \"handoff\", \"handover\", \"give this to another agent\", \"give this to another worktree\", \"send this to another agent\", \"another agent\", \"another worktree\", or \"launch another agent to own this.\" Custom model or reasoning effort words such as `gpt-5.5`, `high`, or `xhigh` do not make the handoff supervised.\n\nSupervised orchestration remains available only when the user explicitly asks for supervision or coordination: \"supervise\", \"monitor\", \"wait for worker_done\", \"wait for results\", \"track completion\", \"DAG\", \"decision gate\", \"ask/reply\", or \"coordinate workers.\"\n\nDo not run `orca orchestration task-create`, `orca orchestration dispatch --inject`, or `orca orchestration check --wait` for full handoffs. `task-create` is also forbidden because it records coordinator-owned tracking state; if a task row is needed, the user asked for supervised orchestration. Do not create a `taskId`/`dispatchId`, inject a lifecycle preamble, wait for completion, or read the worker terminal after prompt delivery except to avoid losing the initial prompt.\n\nNew top-level worktree handoff:\n\n```bash\norca worktree create --name --no-parent --agent codex --prompt \"\" --setup run --json\n```\n\nBefore creating a new worktree from an active feature branch, decide and state whether the desired Orca lineage is child or top-level. Use child worktree lineage only when the new work is conceptually stacked under or dependent on the active worktree. For independent repo-wide fixes, standalone feature work, or unrelated follow-up tasks, create a top-level worktree with `--no-parent`.\n\nExisting terminal handoff:\n\n```bash\norca terminal send --terminal --text \"\" --enter --json\n```\n\nCustom Codex model/effort handoff:\n\n`orca worktree create --agent codex --prompt ...` launches the known Codex agent but does not accept Codex-specific `--model` or `-c model_reasoning_effort=...` arguments. When the user asks for a specific Codex model or effort, create the independent worktree first, launch Codex with the requested command in that worktree, wait only for TUI readiness if prompt delivery would otherwise race startup, send the prompt, and stop.\n\nThe two-step custom-argv path cannot enforce a repository's explicit `wait-for-setup` startup policy because the later `terminal create` is not the startup owned by `worktree create`. Use it only when the repository starts agents immediately. If the repository requires `wait-for-setup`, use an agent-first configured launcher that can preserve sequencing, or stop and ask rather than silently bypassing the policy.\n\nNote: when no repo default-terminal configuration supplies a primary terminal, bare create opens a fallback shell before `terminal create` adds the agent. Configured default tabs are materialized instead and may run real commands. Prefer `--agent` whenever custom argv is not required. With the two-step path, target only the agent handle; close a prior terminal only after `terminal list` or `terminal show` confirms it is an unused shell.\n\nUse the exact full `::` worktree id returned by `orca worktree create --json`; a bare repo id cannot target the new worktree.\n\n```bash\norca worktree create --name --no-parent --setup run --json\norca terminal create --worktree id: --title --command 'codex --model gpt-5.5 -c model_reasoning_effort=\"xhigh\"' --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca terminal send --terminal --text \"\" --enter --json\n```\n\nWait only for `tui-idle` when needed to avoid losing the prompt. Do not monitor task completion.\n\n`--no-parent` only controls Orca lineage; it does not choose the Git base. If the work should start from the repo default base, omit `--base-branch` so Orca uses that default, or explicitly pass the repo default base (`origin/main`, `origin/master`, or the `orca repo show --repo --json` value); never base it on the current feature branch unless the user explicitly asks for stacked work or \"branch from current\". Put current-branch context in the prompt instead.\n\n## Worker Terminals\n\nChoose the worker location before creating a terminal. `Fresh worker` means a fresh agent session, not a new git worktree. For parallel work, create one fresh agent terminal per worker in the same required worktree, falling back to the active worktree when none is named. If the task says current worktree only, depends on uncommitted files/artifacts, or must validate/PR the current branch, keep every worker in the active worktree:\n\n```bash\norca terminal create --worktree active --title --command \"codex\" --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca orchestration dispatch --task --to --inject --json\n```\n\nReuse an idle agent in the required worktree only if the prompt allows reuse; otherwise create a fresh terminal there. Create a new worktree only when the user explicitly requests one or a concrete checkout or filesystem conflict makes sharing unsafe or impossible; if the user did not request it, state that conflict before running `worktree create`. Independent tasks, parallel execution, convenience, or a preference for separate checkouts are not isolation requirements.\n\nWhen a new worktree is allowed, use child lineage for isolated work that is stacked under or dependent on the active worktree, and use `--no-parent` when it is not stacked. Decide the Git base separately: `--no-parent` makes the worktree top-level in Orca, while omitted `--base-branch` uses the repo default base.\n\nFor every new worktree, pass `--setup run` so any configured repository setup hook runs. This does not mean waiting for setup before agent launch: preserve the repository's startup policy, whose default starts setup and the agent side by side. Use `--setup skip` or `--setup inherit` only when there is a concrete task-specific reason, and state that reason before creating the worktree. This rule does not rerun setup for current or existing worktrees.\n\n```bash\norca worktree create --name --agent codex --setup run --json\n# or: --agent claude | omp | pi | grok | ...\n# Read from agentTerminalHandle, falling back to startupTerminal.handle.\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca orchestration dispatch --task --to --inject --json\n```\n\nFor new-worktree workers, read the id and `agentTerminalHandle` from `worktree create`, falling back to `startupTerminal.handle` for older runtimes. Use that as the sole worker handle when present; otherwise use `terminal list` to resolve the agent handle. Omit `--repo` only inside an Orca-managed worktree; otherwise pass `--repo `.\n\n**For an allowed new worktree, use agent-first:** `--agent` reveals the new worktree and launches the selected agent **in its first terminal**, without adding a separate fallback shell for that worker. Pass `--setup run`; repo setup and default-terminal settings may add intentional tabs or splits. Do **not** run bare `worktree create` and then `terminal create --command ` for the same worker when agent-first create is available: without configured default tabs, that two-step path leaves a fallback shell + agent pair. Only use it when custom agent argv is required (for example Codex model/effort flags) or when an older CLI rejects `--agent`; if you must, message only the agent handle. Configured default tabs are intentional surfaces, so close a prior terminal only after `terminal list` or `terminal show` confirms it is an unused shell. Do not run `worktree create` when the task must stay in the current worktree.\n\nUse `orca worktree create --prompt ...` or `orca terminal send ...` for full handoffs or untracked/lightweight prompts. Those paths do not attach `taskId`/`dispatchId`; the worker should not send lifecycle messages unless the prompt supplies a live orchestration preamble.\n\nSidebar lineage and orchestration lifecycle are related but not identical. A same-worktree worker may appear as a peer under that worktree in the sidebar while remaining a child dispatch in orchestration state; only an actual child worktree creates visible parent/child worktree lineage.\n\nOther terminal commands coordinators often need:\n\n```bash\norca terminal list [--worktree ] [--json]\norca terminal create [--worktree ] [--title ] [--command ] [--json]\norca terminal split --terminal [--direction horizontal|vertical] [--command ] [--json]\norca terminal wait --terminal --for tui-idle --timeout-ms --json\norca terminal read --terminal --json\norca terminal send --terminal --text --enter --json\n```\n\nIf an older CLI rejects `worktree create --agent`, create the worktree normally, then run `orca terminal create --worktree --command \"codex\" --json` or `--command \"claude\"`.\n\nWait for `tui-idle` before dispatching. Always pass `--timeout-ms`; real coding tasks can take 15-60 minutes. During supervision, use rolling `check --wait` windows. If a window returns no matching message, inspect `task-list`, `terminal read`, or `terminal wait --for tui-idle` as a liveness checkpoint; if the terminal is still working or producing activity, keep waiting instead of retrying the task.\n\n## Agent Guidance\n\n- Workers with a valid live preamble must send `worker_done` exactly once from their own terminal with an explicit `--outcome succeeded` or `--outcome failed`:\n `orca orchestration send --type worker_done --subject \"\" --body \"<3-sentence summary: what you did, what you found, what's left>\" --task-id --dispatch-id --outcome succeeded --files-modified \"path/a\" --report-path \"\" --json`\n- A failed outcome is still a terminal report, but Orca records both the Dispatch and Task as failed. Never encode failure only in the subject/body.\n- After sending `worker_done`, end your turn and idle at the agent prompt. Do not poll or keep calling `orca orchestration check`; the coordinator re-engages you with a fresh preamble + TASK block delivered as new terminal input.\n- For long tasks, send heartbeat/status only when the preamble asks for it, including both IDs:\n `orca orchestration send --type heartbeat --subject \"alive\" --payload '{\"taskId\":\"\",\"dispatchId\":\"\",\"phase\":\"implementing\"}' --json`\n- If blocked before completion, use `ask`; use `escalation` only when ownership is valid and the coordinator must intervene.\n- Treat preambles inherited through terminal history or full handoffs as stale unless the current prompt explicitly keeps that coordinator in the loop.\n- Coordinators should use `task-list --ready` as external memory, dispatch parallel waves, and avoid dependency chains deeper than 3-4 steps.\n\n## Example\n\n```bash\norca terminal create --worktree active --title login-css-worker --command \"claude\" --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca orchestration task-create --spec \"Fix the login button CSS\" --json\norca orchestration dispatch --task --to --inject --json\norca orchestration check --wait --types worker_done,escalation,question --timeout-ms 900000 --json\n```\n\n## Next Action\n\nCoordinator: confirm `orca status --json`, create or bind a Run, inspect `task-list`/`dispatch-show` if inheriting state, then use the explicit supervised loop (`task-create` -> `worker-start` -> `check --wait`). Use low-level terminal creation plus `dispatch --inject` only when the composed start does not express the needed topology.\n\nWorker: if the current prompt contains a live dispatch preamble, do the task, use `ask` for blocking questions, and send `worker_done` once with the required payload. If the preamble is stale or absent, do not send lifecycle messages; inspect state or treat the prompt as an ordinary handoff.\n" // Why: no current guide has bundled reference documents, so --full is byte-identical for now. // oxfmt-ignore diff --git a/src/cli/handlers/orchestration-check-identity.test.ts b/src/cli/handlers/orchestration-check-identity.test.ts index 7c4908dc782..ec0043fd510 100644 --- a/src/cli/handlers/orchestration-check-identity.test.ts +++ b/src/cli/handlers/orchestration-check-identity.test.ts @@ -71,4 +71,24 @@ describe('orchestration check identity', () => { }) ) }) + + it('preserves the pinned legacy --inject check signature', async () => { + process.env.ORCA_TERMINAL_HANDLE = 'term_legacy_worker' + + await invokeCheck( + new Map([ + ['unread', true], + ['inject', true] + ]) + ) + + expect(callMock).toHaveBeenCalledWith( + 'orchestration.check', + expect.objectContaining({ + terminal: 'term_legacy_worker', + unread: true, + inject: true + }) + ) + }) }) diff --git a/src/cli/handlers/orchestration-legacy-read-only.test.ts b/src/cli/handlers/orchestration-legacy-read-only.test.ts index 0bc82699bc1..05cca1869cd 100644 --- a/src/cli/handlers/orchestration-legacy-read-only.test.ts +++ b/src/cli/handlers/orchestration-legacy-read-only.test.ts @@ -179,6 +179,7 @@ describe('legacy orchestration CLI inspection', () => { id: 'msg_current', run_id: 'run_current', from_handle: 'term_current', + to_handle: 'run:run_current', subject: 'current question', type: 'question', body: 'May I continue?', @@ -217,7 +218,8 @@ describe('legacy orchestration CLI inspection', () => { expect(response.result.formatted).not.toContain('RUNTIME_SENTINEL') expect( response.result.formatted.split('\n').filter((line) => line.startsWith('[Reply:')) - ).toEqual(['[Reply: orca orchestration reply --id msg_current --from term_coord --body "..."]']) + ).toEqual(['[Reply: orca orchestration reply --id msg_current --body "..."]']) + expect(response.result.formatted).not.toContain('--from run:run_current') }) it('preserves runtime formatting when every message belongs to a current Run', async () => { diff --git a/src/cli/handlers/orchestration-migration.test.ts b/src/cli/handlers/orchestration-migration.test.ts index 8c092dbef83..a1a117b90b3 100644 --- a/src/cli/handlers/orchestration-migration.test.ts +++ b/src/cli/handlers/orchestration-migration.test.ts @@ -1,28 +1,44 @@ -import { describe, expect, it, vi } from 'vitest' +import { afterEach, describe, expect, it, vi } from 'vitest' import { ORCHESTRATION_HANDLERS } from './orchestration' -describe('orchestration CLI migration recovery', () => { - it('redirects worker_done without an outcome before resolving or calling the runtime', async () => { - const call = vi.fn() +const originalPaneKey = process.env.ORCA_PANE_KEY - await expect( - ORCHESTRATION_HANDLERS['orchestration send']({ - flags: new Map([ - ['from', 'term_worker'], - ['subject', 'Done'], - ['type', 'worker_done'] - ]), - client: { call }, - cwd: '/tmp/repo', - json: true - } as never) - ).rejects.toMatchObject({ - code: 'invalid_argument', - data: { - effectsApplied: false, - nextCommandArgs: ['skills', 'get', 'orchestration', '--full'] - } +afterEach(() => { + if (originalPaneKey === undefined) { + delete process.env.ORCA_PANE_KEY + } else { + process.env.ORCA_PANE_KEY = originalPaneKey + } +}) + +describe('orchestration CLI migration recovery', () => { + it('forwards legacy worker_done without inventing a completion outcome', async () => { + process.env.ORCA_PANE_KEY = 'tab-worker:leaf-worker' + const call = vi.fn().mockResolvedValue({ result: { message: { id: 'msg_done' } } }) + + await ORCHESTRATION_HANDLERS['orchestration send']({ + flags: new Map([ + ['from', 'term_worker'], + ['subject', 'Done'], + ['type', 'worker_done'] + ]), + client: { call }, + cwd: '/tmp/repo', + json: true + } as never) + + expect(call).toHaveBeenCalledWith('orchestration.send', { + from: 'term_worker', + to: undefined, + run: undefined, + subject: 'Done', + body: undefined, + type: 'worker_done', + priority: undefined, + threadId: undefined, + payload: undefined, + senderPaneKey: 'tab-worker:leaf-worker', + devMode: false }) - expect(call).not.toHaveBeenCalled() }) }) diff --git a/src/cli/handlers/orchestration-run-cli.test.ts b/src/cli/handlers/orchestration-run-cli.test.ts index 84f4dc4c659..4ed57bcb169 100644 --- a/src/cli/handlers/orchestration-run-cli.test.ts +++ b/src/cli/handlers/orchestration-run-cli.test.ts @@ -62,6 +62,28 @@ describe('lightweight Run CLI handlers', () => { from: 'term_coord' }) }) + + it('passes explicit legacy takeover only when requested', async () => { + callMock.mockResolvedValue({ + result: { run: { id: 'run_adopted', objective: 'Recovered work' } } + }) + await ORCHESTRATION_HANDLERS['orchestration run-use']({ + flags: new Map([ + ['id', 'run_adopted'], + ['from', 'term_current'], + ['takeover-legacy', true] + ]), + client: { call: callMock }, + cwd: '/tmp/repo', + json: true + } as never) + + expect(callMock).toHaveBeenCalledWith('orchestration.runUse', { + id: 'run_adopted', + from: 'term_current', + takeoverLegacy: true + }) + }) }) describe('orchestration reset CLI handler', () => { diff --git a/src/cli/handlers/orchestration-timeout-cli.test.ts b/src/cli/handlers/orchestration-timeout-cli.test.ts index 54282345a6c..ef7caa01713 100644 --- a/src/cli/handlers/orchestration-timeout-cli.test.ts +++ b/src/cli/handlers/orchestration-timeout-cli.test.ts @@ -64,11 +64,13 @@ describe('orchestration timeout flag validation', () => { // Why: unread:false makes pre-peek runtimes fall back to non-consuming all mode. expect(callMock).toHaveBeenCalledWith('orchestration.check', { terminal: 'term_worker', + terminalPaneKey: undefined, unread: false, peek: true, all: undefined, types: undefined, format: undefined, + compatibilityCliCommand: expect.stringMatching(/^orca(?:-ide)?$/), run: undefined, ack: undefined, wait: true, @@ -182,7 +184,9 @@ describe('orchestration timeout flag validation', () => { resume: undefined, options: undefined, timeoutMs: 123, - from: 'term_worker' + from: 'term_worker', + compatibilityCliCommand: expect.stringMatching(/^orca(?:-ide)?$/), + compatibilityWindowsCommand: undefined }, { timeoutMs: 5_123, orchestrationCapability: undefined } ) @@ -209,7 +213,9 @@ describe('orchestration timeout flag validation', () => { resume: 'msg_question', options: undefined, timeoutMs: undefined, - from: 'term_worker' + from: 'term_worker', + compatibilityCliCommand: expect.stringMatching(/^orca(?:-ide)?$/), + compatibilityWindowsCommand: undefined }, { timeoutMs: 605_000, orchestrationCapability: undefined } ) diff --git a/src/cli/handlers/orchestration-timeout.test.ts b/src/cli/handlers/orchestration-timeout.test.ts index 342475c81f8..51f0d75098e 100644 --- a/src/cli/handlers/orchestration-timeout.test.ts +++ b/src/cli/handlers/orchestration-timeout.test.ts @@ -86,11 +86,15 @@ describe('orchestration timeout flag validation', () => { expect(callMock).toHaveBeenCalledWith('orchestration.check', { terminal: 'term_worker', + terminalPaneKey: undefined, unread: false, peek: true, all: undefined, types: undefined, - inject: undefined, + format: undefined, + compatibilityCliCommand: expect.stringMatching(/^orca(?:-ide)?$/), + run: undefined, + ack: undefined, wait: true, timeoutMs: 250 }) @@ -273,12 +277,16 @@ describe('orchestration timeout flag validation', () => { 'orchestration.ask', { to: 'term_coord', + run: undefined, question: 'Proceed?', + resume: undefined, options: undefined, timeoutMs: 123, - from: 'term_worker' + from: 'term_worker', + compatibilityCliCommand: expect.stringMatching(/^orca(?:-ide)?$/), + compatibilityWindowsCommand: undefined }, - { timeoutMs: 5_123 } + { timeoutMs: 5_123, orchestrationCapability: undefined } ) }) }) diff --git a/src/cli/handlers/orchestration-windows-ask-cli.test.ts b/src/cli/handlers/orchestration-windows-ask-cli.test.ts new file mode 100644 index 00000000000..ec7f346c0bd --- /dev/null +++ b/src/cli/handlers/orchestration-windows-ask-cli.test.ts @@ -0,0 +1,110 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const callMock = vi.fn() +const originalCliCommand = process.env.ORCA_CLI_COMMAND +const originalPackagedLauncher = process.env.ORCA_WINDOWS_PACKAGED_CLI_LAUNCHER +const originalTerminalHandle = process.env.ORCA_TERMINAL_HANDLE +const originalExitCode = process.exitCode + +vi.mock('../format', () => ({ printResult: vi.fn() })) +vi.mock('../selectors', () => ({ getTerminalHandle: vi.fn() })) + +import { ORCHESTRATION_HANDLERS } from './orchestration' + +describe('packaged Windows legacy ask protocol', () => { + beforeEach(() => { + callMock.mockReset() + process.env.ORCA_WINDOWS_PACKAGED_CLI_LAUNCHER = '1' + process.env.ORCA_TERMINAL_HANDLE = 'term_worker' + process.exitCode = undefined + }) + + afterEach(() => { + vi.restoreAllMocks() + process.exitCode = originalExitCode + restoreEnv('ORCA_CLI_COMMAND', originalCliCommand) + restoreEnv('ORCA_WINDOWS_PACKAGED_CLI_LAUNCHER', originalPackagedLauncher) + restoreEnv('ORCA_TERMINAL_HANDLE', originalTerminalHandle) + }) + + it.each(['orca', 'orca-ide'] as const)( + 'commits with the %s launcher and exits 75 before resume', + async (command) => { + process.env.ORCA_CLI_COMMAND = command + callMock.mockResolvedValue({ + result: { + answer: null, + messageId: 'msg_question', + threadId: 'msg_question', + timedOut: false, + legacyCompatibility: { + resumeRequired: true, + resumeCommand: `${command} orchestration ask --resume msg_question` + } + } + }) + const log = vi.spyOn(console, 'log').mockImplementation(() => {}) + + await invokeAsk(new Map([['question', 'Proceed?']])) + + expect(callMock).toHaveBeenCalledWith( + 'orchestration.ask', + expect.objectContaining({ + question: 'Proceed?', + resume: undefined, + compatibilityWindowsCommand: command + }), + expect.any(Object) + ) + expect(log.mock.calls.map(([line]) => line)).toEqual([ + 'Question msg_question committed.', + `Resume with: ${command} orchestration ask --resume msg_question` + ]) + expect(process.exitCode).toBe(75) + } + ) + + it('resumes the committed question without another exit-75 handoff', async () => { + process.env.ORCA_CLI_COMMAND = 'orca' + callMock.mockResolvedValue({ + result: { + answer: 'yes', + messageId: 'msg_question', + threadId: 'msg_question', + timedOut: false + } + }) + const log = vi.spyOn(console, 'log').mockImplementation(() => {}) + + await invokeAsk(new Map([['resume', 'msg_question']])) + + expect(callMock).toHaveBeenCalledWith( + 'orchestration.ask', + expect.objectContaining({ + question: undefined, + resume: 'msg_question', + compatibilityWindowsCommand: 'orca' + }), + expect.any(Object) + ) + expect(log).toHaveBeenCalledWith('yes') + expect(process.exitCode).toBeUndefined() + }) +}) + +function invokeAsk(flags: Map): Promise { + return ORCHESTRATION_HANDLERS['orchestration ask']({ + flags, + client: { call: callMock }, + cwd: '/tmp/repo', + json: false + } as never) +} + +function restoreEnv(name: string, value: string | undefined): void { + if (value === undefined) { + delete process.env[name] + } else { + process.env[name] = value + } +} diff --git a/src/cli/handlers/orchestration.test.ts b/src/cli/handlers/orchestration.test.ts index f064a645394..ef7299efb43 100644 --- a/src/cli/handlers/orchestration.test.ts +++ b/src/cli/handlers/orchestration.test.ts @@ -692,6 +692,7 @@ describe('orchestration timeout flag validation', () => { all: undefined, types: undefined, format: undefined, + compatibilityCliCommand: expect.stringMatching(/^orca(?:-ide)?$/), run: undefined, ack: undefined, wait: true, @@ -819,7 +820,9 @@ describe('orchestration timeout flag validation', () => { resume: undefined, options: undefined, timeoutMs: 123, - from: 'term_worker' + from: 'term_worker', + compatibilityCliCommand: expect.stringMatching(/^orca(?:-ide)?$/), + compatibilityWindowsCommand: undefined }, { timeoutMs: 5_123, orchestrationCapability: undefined } ) @@ -848,7 +851,9 @@ describe('orchestration timeout flag validation', () => { resume: 'msg_question', options: undefined, timeoutMs: undefined, - from: 'term_worker' + from: 'term_worker', + compatibilityCliCommand: expect.stringMatching(/^orca(?:-ide)?$/), + compatibilityWindowsCommand: undefined }, { timeoutMs: 605_000, orchestrationCapability: undefined } ) diff --git a/src/cli/handlers/orchestration.ts b/src/cli/handlers/orchestration.ts index 3651be4d3e0..0e87bc54d46 100644 --- a/src/cli/handlers/orchestration.ts +++ b/src/cli/handlers/orchestration.ts @@ -21,11 +21,14 @@ import type { } from '../../shared/orchestration-worker-output' import type { NativeChatMessage } from '../../shared/native-chat-types' import type { RuntimeTerminalRead } from '../../shared/runtime-types' +import { orchestrationMigrationData } from '../../shared/orchestration-rpc-contract' import { - ORCHESTRATION_LEGACY_RUN_ID, - orchestrationMigrationData, - orchestrationSkillRecoveryData -} from '../../shared/orchestration-rpc-contract' + formatMessageReadOnlyTag, + formatOrchestrationCheckText, + prepareOrchestrationCheckOutput, + type LegacyCompatibilityResult, + type OrchestrationMessageSummary as MessageSummary +} from '../../shared/orchestration-check-output' // Why: 15 s is well under Claude Code's ~2 min Bash-tool silence budget while keeping log volume low. See design doc §3.4. const DEFAULT_KEEPALIVE_INTERVAL_MS = 15_000 @@ -75,81 +78,6 @@ const TASK_STATUS_VALUES = [ 'blocked' ] as const -type MessageSummary = { - id: string - run_id?: string - from_handle: string - to_handle?: string - subject?: string - type?: string - body?: string - payload?: string | null - priority?: string - read?: number -} - -function formatMessageReadOnlyTag(message: MessageSummary): string { - return message.run_id === ORCHESTRATION_LEGACY_RUN_ID ? ' [legacy, read-only]' : '' -} - -function isLegacyReadOnlyMessage(message: MessageSummary): boolean { - return message.run_id === ORCHESTRATION_LEGACY_RUN_ID -} - -function formatMessagePriorityTag(message: MessageSummary): string { - return message.priority === 'urgent' ? ' [URGENT]' : message.priority === 'high' ? ' [HIGH]' : '' -} - -function escapeTerminalControlCharacters(value: string): string { - return [...value] - .map((character) => { - const code = character.charCodeAt(0) - if (character === '\n' || (code >= 0x20 && code < 0x7f) || code > 0x9f) { - return character - } - return `\\x${code.toString(16).padStart(2, '0')}` - }) - .join('') -} - -function formatQuotedMessageField(label: string, value?: string): string { - return `[${label}]\n${escapeTerminalControlCharacters(value ?? '') - .split('\n') - .map((line) => ` ${line}`) - .join('\n')}` -} - -function formatLegacyAwareCheckMessages( - messages: MessageSummary[], - checkedTerminal: string -): string { - return messages - .map((message) => { - const legacyReadOnly = isLegacyReadOnlyMessage(message) - const lines = [ - `${message.id}${formatMessageReadOnlyTag(message)}${formatMessagePriorityTag(message)} [${message.type ?? 'status'}] from=${message.from_handle}`, - formatQuotedMessageField('subject', message.subject) - ] - if (legacyReadOnly) { - lines.push('[Inspection only: reply and acknowledgment are unavailable.]') - } - if (message.body) { - lines.push(formatQuotedMessageField('body', message.body)) - } - if (message.payload) { - lines.push(formatQuotedMessageField('payload', message.payload)) - } - if (!legacyReadOnly) { - const replyFrom = message.to_handle ?? checkedTerminal - lines.push( - `[Reply: orca orchestration reply --id ${message.id} --from ${replyFrom} --body "..."]` - ) - } - return lines.join('\n') - }) - .join('\n\n') -} - type LifecycleSendRejection = { action: 'rejected' code: string @@ -170,6 +98,40 @@ type OrchestrationSendResult = lifecycle?: { action: 'completed' | 'failed' } } +function resolveCompatibilityCliCommand(): 'orca' | 'orca-ide' | 'orca-dev' { + const configured = process.env.ORCA_CLI_COMMAND + if (configured === 'orca' || configured === 'orca-ide' || configured === 'orca-dev') { + return configured + } + return process.platform === 'linux' ? 'orca-ide' : 'orca' +} + +function resolvePackagedWindowsCompatibilityCommand(): 'orca' | 'orca-ide' | undefined { + if (process.env.ORCA_WINDOWS_PACKAGED_CLI_LAUNCHER !== '1') { + return undefined + } + const command = process.env.ORCA_CLI_COMMAND + if (command === 'orca' || command === 'orca-ide') { + return command + } + throw new RuntimeClientError( + 'invalid_argument', + 'The packaged Orca launcher did not provide a valid resume command. No question was created.' + ) +} + +async function flushStdout(): Promise { + await new Promise((resolve, reject) => { + process.stdout.write('', (error) => { + if (error) { + reject(error) + } else { + resolve() + } + }) + }) +} + function getOptionalStructuredMessagePayload( flags: Map ): string | undefined { @@ -485,7 +447,8 @@ export const ORCHESTRATION_HANDLERS: Record = { run: { id: string; objective: string; consumer_generation: number } }>(client, flags, 'orchestration.runUse', { id: getRequiredStringFlag(flags, 'id'), - from + from, + ...(flags.has('takeover-legacy') ? { takeoverLegacy: true } : {}) }) printResult(result, json, (r) => `Using Run ${r.run.id}: ${r.run.objective}`) }, @@ -541,13 +504,6 @@ export const ORCHESTRATION_HANDLERS: Record = { rejectLifecycleGroupRecipient(type, to) } const outcome = getOptionalStringFlag(flags, 'outcome') - if (type === 'worker_done' && outcome === undefined && !flags.has('payload')) { - throw new RuntimeClientError( - 'invalid_argument', - 'worker_done requires --outcome succeeded or --outcome failed. No effects were applied.', - orchestrationSkillRecoveryData() - ) - } if (type !== 'worker_done' && outcome !== undefined) { throw new RuntimeClientError( 'invalid_argument', @@ -634,6 +590,7 @@ export const ORCHESTRATION_HANDLERS: Record = { timedOut?: boolean cancelled?: boolean connectionLost?: boolean + legacyCompatibility?: LegacyCompatibilityResult } let result: Awaited>> try { @@ -646,6 +603,8 @@ export const ORCHESTRATION_HANDLERS: Record = { all: flags.has('all') ? true : undefined, types: getOptionalStringFlag(flags, 'types'), format: flags.has('format') ? true : undefined, + inject: flags.has('inject') ? true : undefined, + compatibilityCliCommand: resolveCompatibilityCliCommand(), run: getOptionalStringFlag(flags, 'run'), ack: getOptionalStringFlag(flags, 'ack'), wait: wait ? true : undefined, @@ -682,39 +641,25 @@ export const ORCHESTRATION_HANDLERS: Record = { } } } - if (flags.has('format') && result.result.messages.some(isLegacyReadOnlyMessage)) { - // Why: formatted is one opaque batch with untrusted bodies, so selective banner parsing cannot safely remove legacy actions. - result = { - ...result, - result: { - ...result.result, - formatted: formatLegacyAwareCheckMessages(result.result.messages, terminal) - } - } + result = { + ...result, + result: prepareOrchestrationCheckOutput(result.result, terminal, flags.has('format')) + } + printResult(result, json, (r) => formatOrchestrationCheckText(r, terminal)) + const compatibilityAck = result.result.legacyCompatibility?.ackMessageIds + if (compatibilityAck && compatibilityAck.length > 0) { + await flushStdout() + await client.call('orchestration.check', { + terminal, + compatibilityAck: JSON.stringify({ + messageIds: compatibilityAck, + types: getOptionalStringFlag(flags, 'types') + ?.split(',') + .map((type) => type.trim()) + .filter(Boolean) + }) + }) } - printResult(result, json, (r) => { - if (r.formatted) { - return r.formatted - } - if (r.count === 0) { - if (r.timedOut) { - return 'Wait timed out; no messages were consumed.' - } - if (r.cancelled) { - return r.connectionLost - ? 'Wait cancelled because the connection closed; no messages were consumed.' - : 'Wait cancelled; no messages were consumed.' - } - return 'No messages.' - } - const rendered = r.messages - .map( - (m) => - `${m.id}${formatMessageReadOnlyTag(m)} [${m.type ?? 'status'}] from=${m.from_handle} "${m.subject}"` - ) - .join('\n') - return r.deliveryId ? `Delivery ${r.deliveryId}\n${rendered}` : rendered - }) }, 'orchestration reply': async ({ flags, client, cwd, json }) => { @@ -1034,6 +979,8 @@ export const ORCHESTRATION_HANDLERS: Record = { timeoutMs?: number cancelled?: boolean connectionLost?: boolean + answerMessageId?: string | null + legacyCompatibility?: LegacyCompatibilityResult }>( client, flags, @@ -1045,7 +992,9 @@ export const ORCHESTRATION_HANDLERS: Record = { resume, options: getOptionalStringFlag(flags, 'options'), timeoutMs: parsedTimeoutMs === undefined ? undefined : timeoutMs, - from + from, + compatibilityCliCommand: resolveCompatibilityCliCommand(), + compatibilityWindowsCommand: resolvePackagedWindowsCompatibilityCommand() }, // Why: extend past timeoutMs so the RPC transport's 60s default doesn't abort before the runtime's own timeout resolves. { @@ -1056,9 +1005,25 @@ export const ORCHESTRATION_HANDLERS: Record = { // Why: bypass printResult so --json emits a bare JSON object (no envelope) pipeable via `jq -r .answer`, unlike other verbs. if (json) { console.log(JSON.stringify(result.result)) + } else if (result.result.legacyCompatibility?.resumeRequired) { + console.log(`Question ${result.result.messageId} committed.`) + console.log(`Resume with: ${result.result.legacyCompatibility.resumeCommand}`) } else if (result.result.answer !== null) { console.log(result.result.answer) } + if (result.result.legacyCompatibility?.resumeRequired) { + await flushStdout() + process.exitCode = 75 + return + } + const answerAck = result.result.legacyCompatibility?.answerAcknowledgement + if (answerAck && result.result.answer !== null) { + await flushStdout() + await client.call('orchestration.check', { + terminal: from, + compatibilityQuestionAck: JSON.stringify(answerAck) + }) + } if (result.result.timedOut) { if (!json) { // Why: report the server's effective budget — it clamps large values, so the requested one would overstate the wait. diff --git a/src/cli/runtime-client.test.ts b/src/cli/runtime-client.test.ts index c15e9e1e29c..bb0af016108 100644 --- a/src/cli/runtime-client.test.ts +++ b/src/cli/runtime-client.test.ts @@ -103,20 +103,40 @@ describe.skipIf(process.platform === 'win32')('RuntimeClient', () => { await new Promise((resolve) => server.listen(endpoint, resolve)) writeMetadata(userDataPath, endpoint) + const priorLaunchToken = process.env.ORCA_AGENT_LAUNCH_TOKEN + process.env.ORCA_AGENT_LAUNCH_TOKEN = 'launch-secret' const client = new RuntimeClient(userDataPath, 500) - await client.call( - 'orchestration.send', - { subject: 'hello' }, - { - orchestrationRequestId: 'mutation_explicit' + try { + await client.call( + 'orchestration.send', + { subject: 'hello' }, + { + orchestrationRequestId: 'mutation_explicit' + } + ) + await client.call('orchestration.taskList', {}) + const secondClient = new RuntimeClient(userDataPath, 500) + await secondClient.call('orchestration.taskList', {}) + } finally { + if (priorLaunchToken === undefined) { + delete process.env.ORCA_AGENT_LAUNCH_TOKEN + } else { + process.env.ORCA_AGENT_LAUNCH_TOKEN = priorLaunchToken } - ) - await client.call('orchestration.taskList', {}) + } expect(requests[0]?.method).toBe('status.get') + expect(requests[0]?.compatibilityInvocationId).toBeUndefined() expect(requests[1]?.orchestrationRequestId).toBe('mutation_explicit') expect(requests[1]?.orchestrationContractVersion).toBe(1) + expect(requests[1]?.compatibilityInvocationId).toBe('mutation_explicit') + expect(requests[1]?.orchestrationCompatibilityEvidence).toMatchObject({ + launchToken: 'launch-secret' + }) expect(requests[2]?.orchestrationRequestId).toBeUndefined() + expect(requests[2]?.compatibilityInvocationId).not.toBe(requests[1]?.compatibilityInvocationId) + expect(requests[3]?.method).toBe('orchestration.taskList') + expect(requests[3]?.compatibilityInvocationId).not.toBe(requests[1]?.compatibilityInvocationId) }) it('rejects an old local runtime before sending an orchestration mutation', async () => { diff --git a/src/cli/runtime/client.ts b/src/cli/runtime/client.ts index 2b7df570098..3a50b0c87d1 100644 --- a/src/cli/runtime/client.ts +++ b/src/cli/runtime/client.ts @@ -20,6 +20,8 @@ import { ORCHESTRATION_CONTRACT_VERSION, RUNTIME_PROTOCOL_VERSION } from '../../shared/protocol-version' +import { createOrchestrationCompatibilityEnvelope } from './orchestration-compatibility-envelope' +import { getTimeoutMsParam, isWaitingCheck } from './runtime-request-timeout' // Why: for long-poll methods the caller's method-level // `params.timeoutMs` is the inner waiter budget; we extend the client-side @@ -43,6 +45,9 @@ export class RuntimeClient { private readonly environmentSelector: string | null private remoteCompatChecked = false private orchestrationContractCheck: Promise | null = null + private readonly orchestrationCompatibility = createOrchestrationCompatibilityEnvelope( + process.env + ) // Why: browser commands trigger first-time session init (agent-browser connect + // CDP proxy setup) which can take 15-30s. 60s accommodates cold start without @@ -76,12 +81,20 @@ export class RuntimeClient { const orchestrationRequestId = orchestrationMutation ? (options?.orchestrationRequestId ?? randomUUID()) : undefined + const compatibilityEnvelope = method.startsWith('orchestration.') + ? { + ...this.orchestrationCompatibility, + compatibilityInvocationId: + orchestrationRequestId ?? this.orchestrationCompatibility.compatibilityInvocationId + } + : {} const envelope = { orchestrationCapability: options?.orchestrationCapability, orchestrationContractVersion: method.startsWith('orchestration.') ? ORCHESTRATION_CONTRACT_VERSION : undefined, - orchestrationRequestId + orchestrationRequestId, + ...compatibilityEnvelope } if (this.remotePairing) { if (method !== 'status.get') { @@ -327,19 +340,3 @@ function resolveRemotePairing( function delay(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)) } - -function isWaitingCheck(params: unknown): boolean { - return ( - typeof params === 'object' && - params !== null && - 'wait' in params && - (params as { wait: unknown }).wait === true - ) -} - -function getTimeoutMsParam(params: unknown): unknown { - if (typeof params !== 'object' || params === null || !('timeoutMs' in params)) { - return undefined - } - return (params as { timeoutMs?: unknown }).timeoutMs -} diff --git a/src/cli/runtime/orchestration-compatibility-envelope.ts b/src/cli/runtime/orchestration-compatibility-envelope.ts new file mode 100644 index 00000000000..8172e556dc1 --- /dev/null +++ b/src/cli/runtime/orchestration-compatibility-envelope.ts @@ -0,0 +1,12 @@ +import { randomUUID } from 'node:crypto' +import { readOrchestrationCompatibilityEvidence } from '../../shared/orchestration-compatibility-evidence' +import type { RuntimeOrchestrationEnvelope } from '../../shared/runtime-rpc-envelope' + +export function createOrchestrationCompatibilityEnvelope( + env: NodeJS.ProcessEnv +): RuntimeOrchestrationEnvelope { + return { + compatibilityInvocationId: randomUUID(), + orchestrationCompatibilityEvidence: readOrchestrationCompatibilityEvidence(env) + } +} diff --git a/src/cli/runtime/runtime-request-timeout.ts b/src/cli/runtime/runtime-request-timeout.ts new file mode 100644 index 00000000000..c98447f0937 --- /dev/null +++ b/src/cli/runtime/runtime-request-timeout.ts @@ -0,0 +1,15 @@ +export function isWaitingCheck(params: unknown): boolean { + return ( + typeof params === 'object' && + params !== null && + 'wait' in params && + (params as { wait: unknown }).wait === true + ) +} + +export function getTimeoutMsParam(params: unknown): unknown { + if (typeof params !== 'object' || params === null || !('timeoutMs' in params)) { + return undefined + } + return (params as { timeoutMs?: unknown }).timeoutMs +} diff --git a/src/cli/runtime/transport.ts b/src/cli/runtime/transport.ts index b88f3cf3227..091ca9da0e0 100644 --- a/src/cli/runtime/transport.ts +++ b/src/cli/runtime/transport.ts @@ -188,7 +188,9 @@ export async function sendRequest( params, orchestrationCapability: envelope?.orchestrationCapability, orchestrationContractVersion: envelope?.orchestrationContractVersion, - orchestrationRequestId: envelope?.orchestrationRequestId + orchestrationRequestId: envelope?.orchestrationRequestId, + compatibilityInvocationId: envelope?.compatibilityInvocationId, + orchestrationCompatibilityEvidence: envelope?.orchestrationCompatibilityEvidence })}\n` ) }) diff --git a/src/cli/runtime/types.test.ts b/src/cli/runtime/types.test.ts new file mode 100644 index 00000000000..86422029592 --- /dev/null +++ b/src/cli/runtime/types.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from 'vitest' +import { RuntimeRpcFailureError } from './types' + +describe('RuntimeRpcFailureError compatibility redaction', () => { + it('removes compatibility evidence from CLI JSON errors', () => { + const error = new RuntimeRpcFailureError({ + id: 'request-1', + ok: false, + error: { + code: 'compatibility_rejected', + message: 'Rejected', + data: { + orchestrationCompatibilityEvidence: { + launchToken: 'launch-secret' + } + } + }, + _meta: { runtimeId: 'runtime-1' } + }) + + expect(error.data).toEqual({ + orchestrationCompatibilityEvidence: '[redacted]' + }) + expect(JSON.stringify(error.response)).not.toContain('launch-secret') + }) +}) diff --git a/src/cli/runtime/types.ts b/src/cli/runtime/types.ts index ea347efe09f..a15515acfdb 100644 --- a/src/cli/runtime/types.ts +++ b/src/cli/runtime/types.ts @@ -1,4 +1,5 @@ import type { RuntimeRpcFailure } from '../../shared/runtime-rpc-envelope' +import { redactOrchestrationCompatibilitySecrets } from '../../shared/orchestration-compatibility-evidence' export type { RuntimeRpcFailure, @@ -15,7 +16,7 @@ export class RuntimeClientError extends Error { constructor(code: string, message: string, data?: unknown) { super(message) this.code = code - this.data = data + this.data = redactOrchestrationCompatibilitySecrets(data) } } @@ -25,6 +26,12 @@ export class RuntimeRpcFailureError extends RuntimeClientError { constructor(response: RuntimeRpcFailure) { // Why: all client errors expose recovery through the same inherited channel. super(response.error.code, response.error.message, response.error.data) - this.response = response + this.response = { + ...response, + error: { + ...response.error, + ...(response.error.data === undefined ? {} : { data: this.data }) + } + } } } diff --git a/src/cli/specs/orchestration.ts b/src/cli/specs/orchestration.ts index 226f9dcb7f4..d2bf2d4e2ed 100644 --- a/src/cli/specs/orchestration.ts +++ b/src/cli/specs/orchestration.ts @@ -16,8 +16,12 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [ { path: ['orchestration', 'run-use'], summary: 'Bind this coordinator terminal to an existing Run', - usage: 'orca orchestration run-use --id [--from ] [--json]', - allowedFlags: [...GLOBAL_FLAGS, 'id', 'from', 'retry-request'] + usage: + 'orca orchestration run-use --id [--from ] [--takeover-legacy] [--json]', + allowedFlags: [...GLOBAL_FLAGS, 'id', 'from', 'takeover-legacy', 'retry-request'], + notes: [ + '--takeover-legacy must run in the live coordinator agent terminal it binds; it preserves existing worker assignments.' + ] }, { path: ['orchestration', 'run-current'], diff --git a/src/main/agent-hooks/server-authority-evidence.test.ts b/src/main/agent-hooks/server-authority-evidence.test.ts new file mode 100644 index 00000000000..182eddada02 --- /dev/null +++ b/src/main/agent-hooks/server-authority-evidence.test.ts @@ -0,0 +1,223 @@ +import { createHash } from 'node:crypto' +import { afterEach, describe, expect, it } from 'vitest' +import type { AgentHookEventPayload } from '../../shared/agent-hook-listener' +import { makePaneKey } from '../../shared/stable-pane-id' +import { AgentHookServer } from './server' + +const PANE_KEY = makePaneKey('tab-authority', '11111111-1111-4111-8111-111111111111') +const SECOND_PANE_KEY = makePaneKey('tab-authority-2', '22222222-2222-4222-8222-222222222222') + +describe('AgentHookServer authority evidence', () => { + const servers: AgentHookServer[] = [] + + afterEach(() => { + for (const server of servers) { + server.stop() + } + servers.length = 0 + }) + + it('freezes pre-listen commitments separately from current-runtime observations', async () => { + const server = new AgentHookServer() + servers.push(server) + const hydrated = { + paneKey: PANE_KEY, + launchToken: 'launch-before-restart', + tabId: 'tab-authority', + worktreeId: 'repo::before', + connectionId: 'ssh-target', + payload: { state: 'working', prompt: 'before', agentType: 'codex' }, + receivedAt: 100, + stateStartedAt: 100 + } satisfies AgentHookEventPayload & { receivedAt: number; stateStartedAt: number } + server._getStateForTests().lastStatusByPaneKey.set(PANE_KEY, hydrated) + + await server.start() + const commitments = server.getHydratedAuthorityCommitments() + + expect(commitments).toEqual([ + { + paneKey: PANE_KEY, + launchTokenHash: expect.stringMatching(/^[a-f0-9]{64}$/), + tabId: 'tab-authority', + worktreeId: 'repo::before', + connectionId: 'ssh-target', + observedAt: 100 + } + ]) + expect(Object.isFrozen(commitments)).toBe(true) + expect(Object.isFrozen(commitments[0])).toBe(true) + expect(server.getCurrentAuthorityObservations()).toEqual([]) + expect( + server.attestCompatibilityAuthority({ + paneKey: PANE_KEY, + launchTokenHash: createHash('sha256').update('launch-before-restart').digest('hex'), + connectionId: 'ssh-target', + terminalProvenance: 'restored' + }) + ).toEqual({ paneKey: PANE_KEY, source: 'hydrated_commitment' }) + + server.ingestRemote( + { + paneKey: PANE_KEY, + launchToken: 'launch-after-restart', + tabId: 'tab-authority', + worktreeId: 'repo::after', + payload: { state: 'working', prompt: 'after', agentType: 'codex' } + }, + 'ssh-target' + ) + + expect(server.getHydratedAuthorityCommitments()).toBe(commitments) + expect(JSON.stringify(commitments)).not.toContain('launch-before-restart') + expect(server.getCurrentAuthorityObservations()).toEqual([ + expect.objectContaining({ + paneKey: PANE_KEY, + launchTokenHash: expect.stringMatching(/^[a-f0-9]{64}$/), + connectionId: 'ssh-target', + worktreeId: 'repo::after' + }) + ]) + expect(JSON.stringify(server.getCurrentAuthorityObservations())).not.toContain( + 'launch-after-restart' + ) + expect( + server.attestCompatibilityAuthority({ + paneKey: PANE_KEY, + launchTokenHash: createHash('sha256').update('launch-before-restart').digest('hex'), + connectionId: 'ssh-target', + terminalProvenance: 'restored' + }) + ).toBeNull() + expect( + server.attestCompatibilityAuthority({ + paneKey: PANE_KEY, + launchTokenHash: createHash('sha256').update('launch-before-restart').digest('hex'), + connectionId: 'ssh-target', + terminalProvenance: 'current_runtime' + }) + ).toBeNull() + + expect( + server.attestCompatibilityAuthority({ + paneKey: PANE_KEY, + launchTokenHash: createHash('sha256').update('launch-after-restart').digest('hex'), + connectionId: 'ssh-target', + terminalProvenance: 'restored' + }) + ).toBeNull() + expect( + server.attestCompatibilityAuthority({ + paneKey: PANE_KEY, + launchTokenHash: createHash('sha256').update('launch-after-restart').digest('hex'), + connectionId: 'ssh-target', + terminalProvenance: 'current_runtime' + }) + ).toEqual({ paneKey: PANE_KEY, source: 'current_hook' }) + + server.ingestRemote( + { + paneKey: PANE_KEY, + launchToken: 'launch-before-restart', + tabId: 'tab-authority', + worktreeId: 'repo::current', + payload: { state: 'working', prompt: 'current', agentType: 'codex' } + }, + 'ssh-target' + ) + + expect( + server.attestCompatibilityAuthority({ + paneKey: PANE_KEY, + launchTokenHash: createHash('sha256').update('launch-before-restart').digest('hex'), + connectionId: 'ssh-target', + terminalProvenance: 'restored' + }) + ).toEqual({ paneKey: PANE_KEY, source: 'current_hook' }) + + server.ingestRemote( + { + paneKey: SECOND_PANE_KEY, + launchToken: 'launch-before-restart', + tabId: 'tab-authority-2', + payload: { state: 'working', prompt: 'duplicate', agentType: 'codex' } + }, + 'ssh-target' + ) + + expect( + server.attestCompatibilityAuthority({ + paneKey: PANE_KEY, + launchTokenHash: createHash('sha256').update('launch-before-restart').digest('hex'), + connectionId: 'ssh-target', + terminalProvenance: 'restored' + }) + ).toBeNull() + + server.clearStatusEntriesForConnection('ssh-target') + + expect(server.getHydratedAuthorityCommitments()).toBe(commitments) + expect(server.getCurrentAuthorityObservations()).toEqual([]) + expect( + server.attestCompatibilityAuthority({ + paneKey: PANE_KEY, + launchTokenHash: createHash('sha256').update('launch-before-restart').digest('hex'), + connectionId: 'ssh-target', + terminalProvenance: 'restored' + }) + ).toEqual({ paneKey: PANE_KEY, source: 'hydrated_commitment' }) + + server.clearPaneState(PANE_KEY) + + expect(server.getHydratedAuthorityCommitments()).toBe(commitments) + expect( + server.attestCompatibilityAuthority({ + paneKey: PANE_KEY, + launchTokenHash: createHash('sha256').update('launch-before-restart').digest('hex'), + connectionId: 'ssh-target', + terminalProvenance: 'restored' + }) + ).toBeNull() + }) + + it('preserves hydrated authority when stale alias cleanup does not own the stable pane', async () => { + const server = new AgentHookServer() + servers.push(server) + const launchToken = 'launch-before-restart' + const launchTokenHash = createHash('sha256').update(launchToken).digest('hex') + const hydrated = { + paneKey: PANE_KEY, + launchToken, + tabId: 'tab-authority', + worktreeId: 'repo::before', + connectionId: 'ssh-target', + payload: { state: 'working', prompt: 'before', agentType: 'codex' }, + receivedAt: 100, + stateStartedAt: 100 + } satisfies AgentHookEventPayload & { receivedAt: number; stateStartedAt: number } + server._getStateForTests().lastStatusByPaneKey.set(PANE_KEY, hydrated) + server.registerPaneKeyAlias('tab-authority:0', PANE_KEY, 'old-pty') + await server.start() + server.ingestRemote( + { + paneKey: PANE_KEY, + launchToken, + tabId: 'tab-authority', + worktreeId: 'repo::current', + payload: { state: 'working', prompt: 'current', agentType: 'codex' } + }, + 'ssh-target' + ) + + server.clearPaneKeyAliasesForPty('old-pty', { shouldClearStablePaneKey: () => false }) + + expect( + server.attestCompatibilityAuthority({ + paneKey: PANE_KEY, + launchTokenHash, + connectionId: 'ssh-target', + terminalProvenance: 'restored' + }) + ).toEqual({ paneKey: PANE_KEY, source: 'current_hook' }) + }) +}) diff --git a/src/main/agent-hooks/server.test.ts b/src/main/agent-hooks/server.test.ts index e05e0c17312..2bb905862ad 100644 --- a/src/main/agent-hooks/server.test.ts +++ b/src/main/agent-hooks/server.test.ts @@ -1,6 +1,7 @@ /* eslint-disable max-lines -- Why: this suite exercises the full hook HTTP surface (Claude/Codex/Gemini parsing, transcript chunked scan, paneKey dispatch) and keeping the scenarios co-located avoids fixture drift across files. */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { execFileSync } from 'node:child_process' +import { createHash } from 'node:crypto' import { existsSync, mkdirSync, @@ -57,6 +58,7 @@ const TAB_A_PANE = makePaneKey('tab-A', LEAF_5) type Body = { paneKey: string + launchToken?: string tabId?: string worktreeId?: string env?: string @@ -6254,7 +6256,10 @@ describe('Last-status persistence', () => { try { await postHookEvent( server, - buildBody({ hook_event_name: 'UserPromptSubmit', prompt: 'persist me' }) + buildBody( + { hook_event_name: 'UserPromptSubmit', prompt: 'persist me' }, + { launchToken: 'launch-bearer-must-not-persist' } + ) ) // Synchronous flush via stop() captures the trailing-debounced write. server.flushStatusPersistSync() @@ -6269,11 +6274,148 @@ describe('Last-status persistence', () => { stateStartedAt: expect.any(Number), payload: expect.objectContaining({ state: 'working', prompt: 'persist me' }) }) + expect(file.entries[PANE].launchToken).toBeUndefined() + expect(file.entries[PANE].launchTokenHash).toBe( + createHash('sha256').update('launch-bearer-must-not-persist').digest('hex') + ) + expect(readFileSync(lastStatusPath(), 'utf8')).not.toContain('launch-bearer-must-not-persist') } finally { server.stop() } }) + it('scrubs a legacy persisted launch bearer while retaining its authority commitment', async () => { + mkdirSync(join(userDataPath, 'agent-hooks'), { recursive: true }) + const receivedAt = recentTs() + const launchToken = 'legacy-launch-bearer' + writeFileSync( + lastStatusPath(), + JSON.stringify({ + version: 2, + entries: { + [PANE]: { + paneKey: PANE, + launchToken, + tabId: 'tab-1', + worktreeId: 'wt-1', + connectionId: null, + receivedAt, + stateStartedAt: receivedAt, + payload: { + state: 'working', + prompt: 'legacy worker', + agentType: 'codex' + } + } + } + }), + 'utf8' + ) + + const server = new AgentHookServer() + await server.start({ env: 'production', userDataPath }) + try { + const launchTokenHash = createHash('sha256').update(launchToken).digest('hex') + expect(server.getHydratedAuthorityCommitments()).toEqual([ + expect.objectContaining({ paneKey: PANE, launchTokenHash }) + ]) + expect(server.getStatusSnapshotForPane(PANE)[0]?.launchToken).toBeUndefined() + + const persisted = readFileSync(lastStatusPath(), 'utf8') + expect(persisted).not.toContain(launchToken) + expect(JSON.parse(persisted).entries[PANE]).toMatchObject({ launchTokenHash }) + } finally { + server.stop() + } + + const restartedServer = new AgentHookServer() + await restartedServer.start({ env: 'production', userDataPath }) + try { + expect(restartedServer.getHydratedAuthorityCommitments()).toEqual([ + expect.objectContaining({ + paneKey: PANE, + launchTokenHash: createHash('sha256').update(launchToken).digest('hex') + }) + ]) + expect(restartedServer.getStatusSnapshotForPane(PANE)[0]?.launchToken).toBeUndefined() + } finally { + restartedServer.stop() + } + }) + + it('persists SSH authority through transient clears until explicit retirement', async () => { + mkdirSync(join(userDataPath, 'agent-hooks'), { recursive: true }) + const receivedAt = recentTs() + const launchToken = 'retained-ssh-launch-bearer' + const launchTokenHash = createHash('sha256').update(launchToken).digest('hex') + writeFileSync( + lastStatusPath(), + JSON.stringify({ + version: 2, + entries: { + [PANE]: { + paneKey: PANE, + launchTokenHash, + tabId: 'tab-1', + worktreeId: 'wt-1', + connectionId: 'ssh-target', + receivedAt, + stateStartedAt: receivedAt, + payload: { + state: 'working', + prompt: 'retained SSH worker', + agentType: 'codex' + } + } + } + }), + 'utf8' + ) + + const first = new AgentHookServer() + await first.start({ env: 'production', userDataPath }) + first.clearStatusEntriesForConnection('ssh-target') + first.flushStatusPersistSync() + first.stop() + + const afterClear = JSON.parse(readFileSync(lastStatusPath(), 'utf8')) + expect(afterClear.entries).toEqual({}) + expect(afterClear.authorityCommitments[PANE]).toMatchObject({ + paneKey: PANE, + launchTokenHash, + connectionId: 'ssh-target' + }) + + const restored = new AgentHookServer() + await restored.start({ env: 'production', userDataPath }) + expect( + restored.attestCompatibilityAuthority({ + paneKey: PANE, + launchTokenHash, + connectionId: 'ssh-target', + terminalProvenance: 'restored' + }) + ).toEqual({ paneKey: PANE, source: 'hydrated_commitment' }) + restored.retirePaneAuthority(PANE) + restored.flushStatusPersistSync() + restored.stop() + + const retired = new AgentHookServer() + await retired.start({ env: 'production', userDataPath }) + try { + expect( + retired.attestCompatibilityAuthority({ + paneKey: PANE, + launchTokenHash, + connectionId: 'ssh-target', + terminalProvenance: 'restored' + }) + ).toBeNull() + } finally { + retired.stop() + } + }) + it('persists and hydrates Pi session identity without creating status telemetry', async () => { const firstServer = new AgentHookServer() const firstRendererListener = vi.fn() diff --git a/src/main/agent-hooks/server.ts b/src/main/agent-hooks/server.ts index 8887a285235..812b421da79 100644 --- a/src/main/agent-hooks/server.ts +++ b/src/main/agent-hooks/server.ts @@ -79,6 +79,22 @@ type EnrichedAgentHookEventPayload = AgentHookEventPayload & { stateStartedAt: number } +type PersistedAgentHookEventPayload = Omit< + EnrichedAgentHookEventPayload, + 'launchToken' | 'promptInteractionKey' +> & { + launchTokenHash?: string +} + +type PersistedAgentHookAuthorityCommitment = { + paneKey: string + launchTokenHash: string + connectionId: string | null + tabId?: string + worktreeId?: string + observedAt: number +} + export type AgentHookStatusChangeEntry = { state: AgentStatusState receivedAt: number @@ -92,6 +108,20 @@ export type AgentHookProviderSessionIdentity = { worktreeId?: string } +export type AgentHookAuthorityEvidence = Readonly<{ + paneKey: string + launchTokenHash: string + connectionId: string | null + tabId?: string + worktreeId?: string + observedAt: number +}> + +export type AgentHookAuthorityAttestation = Readonly<{ + paneKey: string + source: 'current_hook' | 'hydrated_commitment' +}> + type StatusChangeListener = (statuses: AgentHookStatusChangeEntry[]) => void type ProviderSessionChangeListener = (providerSessions: AgentHookProviderSessionIdentity[]) => void type PaneStatusClearListener = (clear: AgentStatusClearIpcPayload) => void @@ -128,7 +158,8 @@ export const PANE_KEY_ALIASES_MAX = 1024 type LastStatusFile = { version: number - entries: Record + entries: Record + authorityCommitments?: Record } type AgentPromptSentDedupeEntry = { @@ -251,7 +282,6 @@ function sanitizeHydratedEntry( } return { paneKey, - launchToken: typeof record.launchToken === 'string' ? record.launchToken : undefined, tabId: typeof tabId === 'string' ? tabId : undefined, worktreeId: typeof worktreeId === 'string' ? worktreeId : undefined, connectionId, @@ -268,6 +298,63 @@ function sanitizeHydratedEntry( } } +function readPersistedLaunchTokenHash(rawEntry: unknown): string | null { + if (typeof rawEntry !== 'object' || rawEntry === null) { + return null + } + const record = rawEntry as Record + const launchTokenHash = + typeof record.launchTokenHash === 'string' ? record.launchTokenHash.trim() : '' + if (/^[a-f0-9]{64}$/.test(launchTokenHash)) { + return launchTokenHash + } + const legacyLaunchToken = typeof record.launchToken === 'string' ? record.launchToken.trim() : '' + return legacyLaunchToken ? createHash('sha256').update(legacyLaunchToken).digest('hex') : null +} + +function sanitizePersistedAuthorityCommitment( + paneKey: string, + value: unknown +): AgentHookAuthorityEvidence | null { + if (!isValidPaneKey(paneKey) || typeof value !== 'object' || value === null) { + return null + } + const record = value as Record + const launchTokenHash = + typeof record.launchTokenHash === 'string' ? record.launchTokenHash.trim() : '' + const connectionId = record.connectionId + const observedAt = record.observedAt + if ( + !/^[a-f0-9]{64}$/.test(launchTokenHash) || + (connectionId !== null && typeof connectionId !== 'string') || + typeof observedAt !== 'number' || + !Number.isFinite(observedAt) + ) { + return null + } + return Object.freeze({ + paneKey, + launchTokenHash, + connectionId, + ...(typeof record.tabId === 'string' ? { tabId: record.tabId } : {}), + ...(typeof record.worktreeId === 'string' ? { worktreeId: record.worktreeId } : {}), + observedAt + }) +} + +function authorityCommitmentsMatch( + left: AgentHookAuthorityEvidence, + right: AgentHookAuthorityEvidence +): boolean { + return ( + left.paneKey === right.paneKey && + left.launchTokenHash === right.launchTokenHash && + left.connectionId === right.connectionId && + left.tabId === right.tabId && + left.worktreeId === right.worktreeId + ) +} + function toAgentStatusIpcPayload(entry: EnrichedAgentHookEventPayload): AgentStatusIpcPayload { return { paneKey: entry.paneKey, @@ -479,6 +566,11 @@ export class AgentHookServer { private state: HookListenerState = createHookListenerState() // Why: hydrated rows give UI continuity but aren't evidence of live agent work in this runtime. private runtimeObservedStatusPaneKeys = new Set() + private hydratedAuthorityCommitments: readonly AgentHookAuthorityEvidence[] = Object.freeze([]) + private hydratedLaunchTokenHashByPaneKey = new Map() + private persistedAuthorityCommitmentsByPaneKey = new Map() + private revokedHydratedAuthorityCommitments = new WeakSet() + private currentAuthorityObservations = new Map() private legacyPaneKeyAliases = new Map() private paneKeyAliasPersistenceListener: PaneKeyAliasPersistenceListener | null = null // Why: on-disk last-status cache path; null without a userDataPath (tests), where persistence is a no-op and only in-memory replay applies. @@ -562,6 +654,53 @@ export class AgentHookServer { return entry ? [toAgentStatusIpcPayload(entry as EnrichedAgentHookEventPayload)] : [] } + getHydratedAuthorityCommitments(): readonly AgentHookAuthorityEvidence[] { + return this.hydratedAuthorityCommitments + } + + getCurrentAuthorityObservations(): readonly AgentHookAuthorityEvidence[] { + return Object.freeze( + Array.from(this.currentAuthorityObservations.values(), (entry) => Object.freeze({ ...entry })) + ) + } + + attestCompatibilityAuthority(candidate: { + paneKey: string + launchTokenHash: string + connectionId: string | null + terminalProvenance: 'current_runtime' | 'restored' + }): AgentHookAuthorityAttestation | null { + const paneKey = this.resolvePaneKeyAlias(candidate.paneKey) + const matchesCandidate = (entry: AgentHookAuthorityEvidence): boolean => + entry.launchTokenHash === candidate.launchTokenHash && + entry.connectionId === candidate.connectionId + const commitments = this.hydratedAuthorityCommitments.filter( + (entry) => matchesCandidate(entry) && !this.revokedHydratedAuthorityCommitments.has(entry) + ) + const current = Array.from(this.currentAuthorityObservations.values()) + const observations = current.filter(matchesCandidate) + const paneObservations = current.filter( + (entry) => this.resolvePaneKeyAlias(entry.paneKey) === paneKey + ) + const hasUniqueCurrentObservation = + observations.length === 1 && + paneObservations.length === 1 && + this.resolvePaneKeyAlias(observations[0]!.paneKey) === paneKey + if (candidate.terminalProvenance === 'current_runtime') { + return hasUniqueCurrentObservation ? Object.freeze({ paneKey, source: 'current_hook' }) : null + } + if (commitments.length !== 1 || this.resolvePaneKeyAlias(commitments[0]!.paneKey) !== paneKey) { + return null + } + if (observations.length === 0 && paneObservations.length === 0) { + return Object.freeze({ paneKey, source: 'hydrated_commitment' }) + } + if (!hasUniqueCurrentObservation) { + return null + } + return Object.freeze({ paneKey, source: 'current_hook' }) + } + inferInterrupt(request: AgentInterruptInferenceRequest): boolean { if (!isValidPaneKey(request.paneKey)) { return false @@ -1301,9 +1440,40 @@ export class AgentHookServer { tabId: owner?.tabId }) } + const hydratedLaunchTokenHash = this.hydratedLaunchTokenHashByPaneKey.get(previousOwnerPaneKey) + if (hydratedLaunchTokenHash) { + this.hydratedLaunchTokenHashByPaneKey.delete(previousOwnerPaneKey) + this.hydratedLaunchTokenHashByPaneKey.set(toPaneKey, hydratedLaunchTokenHash) + } + const persistedAuthority = this.persistedAuthorityCommitmentsByPaneKey.get(previousOwnerPaneKey) + if (persistedAuthority) { + const owner = parsePaneKey(toPaneKey) + this.persistedAuthorityCommitmentsByPaneKey.delete(previousOwnerPaneKey) + this.persistedAuthorityCommitmentsByPaneKey.set( + toPaneKey, + Object.freeze({ + ...persistedAuthority, + paneKey: toPaneKey, + ...(owner?.tabId ? { tabId: owner.tabId } : {}) + }) + ) + } if (this.runtimeObservedStatusPaneKeys.delete(previousOwnerPaneKey)) { this.runtimeObservedStatusPaneKeys.add(toPaneKey) } + const authorityObservation = this.currentAuthorityObservations.get(previousOwnerPaneKey) + if (authorityObservation) { + const owner = parsePaneKey(toPaneKey) + this.currentAuthorityObservations.delete(previousOwnerPaneKey) + this.currentAuthorityObservations.set( + toPaneKey, + Object.freeze({ + ...authorityObservation, + paneKey: toPaneKey, + tabId: owner?.tabId + }) + ) + } const promptDedupe = this.promptSentDedupeByPaneKey.get(previousOwnerPaneKey) if (promptDedupe !== undefined) { this.promptSentDedupeByPaneKey.delete(previousOwnerPaneKey) @@ -1321,7 +1491,7 @@ export class AgentHookServer { this.boundPaneKeyAliases() this.closedAgentStatusPaneKeys.delete(toPaneKey) this.notifyPaneKeyAliasPersistenceListener() - if (hadStatus) { + if (hadStatus || persistedAuthority) { this.scheduleStatusPersist() this.notifyStatusChangeListeners() } @@ -1339,6 +1509,7 @@ export class AgentHookServer { aliasChanged = true } } + const authorityChanged = this.revokeHydratedAuthorityForPaneKeys(paneKeys) const hadStatus = [...paneKeys].some((key) => this.state.lastStatusByPaneKey.has(key)) for (const key of paneKeys) { this.markPaneClosedForAgentStatus(key) @@ -1346,12 +1517,13 @@ export class AgentHookServer { this.clearCodexSubagentPoll(key) clearPaneCacheState(this.state, key) this.runtimeObservedStatusPaneKeys.delete(key) + this.currentAuthorityObservations.delete(key) this.promptSentDedupeByPaneKey.delete(key) } if (aliasChanged) { this.notifyPaneKeyAliasPersistenceListener() } - if (hadStatus) { + if (hadStatus || authorityChanged) { this.scheduleStatusPersist() this.notifyStatusChangeListeners() } @@ -1366,11 +1538,19 @@ export class AgentHookServer { const clearedStatusPaneKeys = new Set() for (const [legacyPaneKey, entry] of this.legacyPaneKeyAliases) { if (entry.ptyId === ptyId) { - this.legacyPaneKeyAliases.delete(legacyPaneKey) - clearPaneCacheState(this.state, legacyPaneKey) - this.promptSentDedupeByPaneKey.delete(legacyPaneKey) const shouldClearStablePaneKey = options?.shouldClearStablePaneKey?.(entry.stablePaneKey) ?? true + const revokedPaneKeys = new Set([legacyPaneKey]) + if (shouldClearStablePaneKey) { + revokedPaneKeys.add(entry.stablePaneKey) + } + if (this.revokeHydratedAuthorityForPaneKeys(revokedPaneKeys)) { + statusChanged = true + } + this.legacyPaneKeyAliases.delete(legacyPaneKey) + clearPaneCacheState(this.state, legacyPaneKey) + this.currentAuthorityObservations.delete(legacyPaneKey) + this.promptSentDedupeByPaneKey.delete(legacyPaneKey) if (shouldClearStablePaneKey && this.state.lastStatusByPaneKey.has(entry.stablePaneKey)) { statusChanged = true clearedStatusPaneKeys.add(entry.stablePaneKey) @@ -1379,6 +1559,7 @@ export class AgentHookServer { // Why: hydrated rows live under the stable key; if this PTY dies before ptyPaneKey rebuilds, alias cleanup is the only evictor. clearPaneCacheState(this.state, entry.stablePaneKey) this.runtimeObservedStatusPaneKeys.delete(entry.stablePaneKey) + this.currentAuthorityObservations.delete(entry.stablePaneKey) this.promptSentDedupeByPaneKey.delete(entry.stablePaneKey) } aliasChanged = true @@ -1400,6 +1581,27 @@ export class AgentHookServer { return this.legacyPaneKeyAliases.get(paneKey)?.stablePaneKey ?? paneKey } + private revokeHydratedAuthorityForPaneKeys(paneKeys: ReadonlySet): boolean { + let changed = false + for (const commitment of this.hydratedAuthorityCommitments) { + if ( + paneKeys.has(commitment.paneKey) || + paneKeys.has(this.resolvePaneKeyAlias(commitment.paneKey)) + ) { + this.revokedHydratedAuthorityCommitments.add(commitment) + changed = true + } + } + for (const paneKey of paneKeys) { + const resolvedPaneKey = this.resolvePaneKeyAlias(paneKey) + changed = this.hydratedLaunchTokenHashByPaneKey.delete(paneKey) || changed + changed = this.hydratedLaunchTokenHashByPaneKey.delete(resolvedPaneKey) || changed + changed = this.persistedAuthorityCommitmentsByPaneKey.delete(paneKey) || changed + changed = this.persistedAuthorityCommitmentsByPaneKey.delete(resolvedPaneKey) || changed + } + return changed + } + private normalizeHookBodyPaneKeyAlias(body: unknown): unknown { if (typeof body !== 'object' || body === null) { return body @@ -1602,6 +1804,7 @@ export class AgentHookServer { isReplay: envelope.isReplay === true ? true : undefined, payload: normalizedPayload } + this.recordCurrentAuthorityObservation(event) this.applyNormalizedStatus(event) } @@ -1632,6 +1835,7 @@ export class AgentHookServer { if (this.lastStatusFilePath) { this.hydrateLastStatusFromDisk() } + this.captureHydratedAuthorityCommitments() const handleRequest = async (req: IncomingMessage, res: ServerResponse): Promise => { if (req.method !== 'POST') { res.writeHead(404) @@ -1673,6 +1877,7 @@ export class AgentHookServer { const aliasedBody = this.normalizeHookBodyPaneKeyAlias(body) const normalized = normalizeHookPayload(this.state, source, aliasedBody, this.env) if (normalized && !this.shouldSuppressClosedTabStatus(normalized.paneKey)) { + this.recordCurrentAuthorityObservation(normalized) const enriched = this.applyNormalizedStatus(normalized) this.scheduleAssistantMessageRetry(source, aliasedBody, enriched) this.scheduleCodexSubagentPoll(source, aliasedBody, enriched) @@ -1739,6 +1944,11 @@ export class AgentHookServer { this.lastStatusFilePath = null this.lastWrittenJson = null this.runtimeObservedStatusPaneKeys.clear() + this.hydratedAuthorityCommitments = Object.freeze([]) + this.hydratedLaunchTokenHashByPaneKey.clear() + this.persistedAuthorityCommitmentsByPaneKey.clear() + this.revokedHydratedAuthorityCommitments = new WeakSet() + this.currentAuthorityObservations.clear() this.promptSentDedupeByPaneKey.clear() this.closedAgentStatusTabIds.clear() this.closedAgentStatusPaneKeys.clear() @@ -1750,7 +1960,7 @@ export class AgentHookServer { /** Drop only the status row (user dismissal); do NOT wipe prompt/tool caches since the pane's agent may still be alive. Use clearPaneState for PTY-teardown. */ dropStatusEntry(paneKey: string): void { - if (!this.deleteStatusEntry(paneKey)) { + if (!this.deleteStatusEntry(paneKey, { preserveAuthority: true })) { return } this.scheduleStatusPersist() @@ -1775,7 +1985,7 @@ export class AgentHookServer { if (entry.connectionId !== normalizedConnectionId) { continue } - const deleted = this.deleteStatusEntry(paneKey) + const deleted = this.deleteStatusEntry(paneKey, { preserveAuthority: true }) if (deleted) { statusChanged = true if (deleted.payload.agentType === 'codex') { @@ -1785,6 +1995,11 @@ export class AgentHookServer { } } } + for (const [paneKey, evidence] of this.currentAuthorityObservations) { + if (evidence.connectionId === normalizedConnectionId) { + this.currentAuthorityObservations.delete(paneKey) + } + } if (statusChanged) { // Why: persist/notify once — one disconnect can own many panes. this.scheduleStatusPersist() @@ -1798,7 +2013,10 @@ export class AgentHookServer { }) } - private deleteStatusEntry(paneKey: string): EnrichedAgentHookEventPayload | null { + private deleteStatusEntry( + paneKey: string, + options?: { preserveAuthority?: boolean } + ): EnrichedAgentHookEventPayload | null { const resolvedPaneKey = this.resolvePaneKeyAlias(paneKey) const existing = this.state.lastStatusByPaneKey.get(resolvedPaneKey) as | EnrichedAgentHookEventPayload @@ -1807,9 +2025,14 @@ export class AgentHookServer { return null } this.state.lastStatusByPaneKey.delete(resolvedPaneKey) + if (!options?.preserveAuthority) { + this.hydratedLaunchTokenHashByPaneKey.delete(resolvedPaneKey) + this.persistedAuthorityCommitmentsByPaneKey.delete(resolvedPaneKey) + } this.clearAssistantMessageRetry(resolvedPaneKey) this.clearCodexSubagentPoll(resolvedPaneKey) this.runtimeObservedStatusPaneKeys.delete(resolvedPaneKey) + this.currentAuthorityObservations.delete(resolvedPaneKey) if (existing.payload.state === 'done') { this.promptSentDedupeByPaneKey.delete(resolvedPaneKey) } @@ -1854,6 +2077,11 @@ export class AgentHookServer { paneKeysToClear.add(paneKey) } } + for (const commitment of this.hydratedAuthorityCommitments) { + if (paneCacheKeyMatchesTab(commitment.paneKey, tabId)) { + paneKeysToClear.add(commitment.paneKey) + } + } let aliasChanged = false for (const [legacyPaneKey, entry] of this.legacyPaneKeyAliases) { @@ -1867,6 +2095,7 @@ export class AgentHookServer { aliasChanged = true } } + const authorityChanged = this.revokeHydratedAuthorityForPaneKeys(paneKeysToClear) let statusChanged = false for (const paneKey of paneKeysToClear) { @@ -1877,12 +2106,13 @@ export class AgentHookServer { this.clearCodexSubagentPoll(paneKey) clearPaneCacheState(this.state, paneKey) this.runtimeObservedStatusPaneKeys.delete(paneKey) + this.currentAuthorityObservations.delete(paneKey) this.promptSentDedupeByPaneKey.delete(paneKey) } if (aliasChanged) { this.notifyPaneKeyAliasPersistenceListener() } - if (statusChanged) { + if (statusChanged || authorityChanged) { this.scheduleStatusPersist() this.notifyStatusChangeListeners() } @@ -1890,25 +2120,31 @@ export class AgentHookServer { clearPaneState(paneKey: string): void { const resolvedPaneKey = this.resolvePaneKeyAlias(paneKey) + const paneKeys = new Set([paneKey, resolvedPaneKey]) // Why: only persist when a status entry was actually evicted; dropping prompt/tool caches doesn't change the file. const hadStatus = this.state.lastStatusByPaneKey.has(resolvedPaneKey) this.clearAssistantMessageRetry(resolvedPaneKey) this.clearCodexSubagentPoll(resolvedPaneKey) clearPaneCacheState(this.state, resolvedPaneKey) + this.currentAuthorityObservations.delete(resolvedPaneKey) this.promptSentDedupeByPaneKey.delete(resolvedPaneKey) let clearedAlias = false for (const [legacyPaneKey, stablePaneKey] of this.legacyPaneKeyAliases) { if (stablePaneKey.stablePaneKey === resolvedPaneKey) { this.legacyPaneKeyAliases.delete(legacyPaneKey) + paneKeys.add(legacyPaneKey) + paneKeys.add(stablePaneKey.stablePaneKey) clearPaneCacheState(this.state, legacyPaneKey) + this.currentAuthorityObservations.delete(legacyPaneKey) this.promptSentDedupeByPaneKey.delete(legacyPaneKey) clearedAlias = true } } + const authorityChanged = this.revokeHydratedAuthorityForPaneKeys(paneKeys) if (clearedAlias) { this.notifyPaneKeyAliasPersistenceListener() } - if (hadStatus) { + if (hadStatus || authorityChanged) { this.runtimeObservedStatusPaneKeys.delete(resolvedPaneKey) this.scheduleStatusPersist() this.notifyStatusChangeListeners() @@ -1963,6 +2199,8 @@ export class AgentHookServer { } // Why: keep hydrate idempotent so a future re-start path can't merge prior-session state. this.state.lastStatusByPaneKey.clear() + this.hydratedLaunchTokenHashByPaneKey.clear() + this.persistedAuthorityCommitmentsByPaneKey.clear() let raw: string try { raw = readFileSync(this.lastStatusFilePath, 'utf8') @@ -2001,6 +2239,7 @@ export class AgentHookServer { let hydrated = 0 let dropped = 0 let prunedLegacyClaudeSubagents = 0 + let scrubbedLegacyLaunchTokens = 0 // Why: drop entries older than HYDRATE_MAX_AGE_MS to bound disk growth (one Date.now() for a consistent cutoff). const ttlCutoff = Date.now() - HYDRATE_MAX_AGE_MS for (const [paneKey, rawEntry] of Object.entries(entries)) { @@ -2011,6 +2250,21 @@ export class AgentHookServer { : { ...(rawEntry as Record), paneKey: resolvedPaneKey } const entry = sanitizeHydratedEntry(resolvedPaneKey, rawResolvedEntry) if (entry && entry.receivedAt >= ttlCutoff) { + const launchTokenHash = readPersistedLaunchTokenHash(rawResolvedEntry) + if (launchTokenHash) { + this.hydratedLaunchTokenHashByPaneKey.set(resolvedPaneKey, launchTokenHash) + const evidence = this.toAuthorityEvidence(entry, launchTokenHash) + if (evidence) { + this.persistedAuthorityCommitmentsByPaneKey.set(resolvedPaneKey, evidence) + } + } + if ( + typeof rawResolvedEntry === 'object' && + rawResolvedEntry !== null && + typeof (rawResolvedEntry as Record).launchToken === 'string' + ) { + scrubbedLegacyLaunchTokens += 1 + } const hydratedPayload = dropHydratedIdleClaudeSubagents(entry.payload) if (hydratedPayload !== entry.payload) { prunedLegacyClaudeSubagents += @@ -2041,13 +2295,30 @@ export class AgentHookServer { dropped += 1 } } + for (const [paneKey, rawCommitment] of Object.entries(file.authorityCommitments ?? {})) { + const resolvedPaneKey = this.resolvePaneKeyAlias(paneKey) + const commitment = sanitizePersistedAuthorityCommitment(resolvedPaneKey, rawCommitment) + if (!commitment || commitment.observedAt < ttlCutoff) { + dropped += 1 + continue + } + const existing = this.persistedAuthorityCommitmentsByPaneKey.get(resolvedPaneKey) + if (existing && !authorityCommitmentsMatch(existing, commitment)) { + this.persistedAuthorityCommitmentsByPaneKey.delete(resolvedPaneKey) + this.hydratedLaunchTokenHashByPaneKey.delete(resolvedPaneKey) + dropped += 1 + continue + } + this.persistedAuthorityCommitmentsByPaneKey.set(resolvedPaneKey, commitment) + this.hydratedLaunchTokenHashByPaneKey.set(resolvedPaneKey, commitment.launchTokenHash) + } if (dropped > 0) { console.warn( `[agent-hooks] last-status hydrate dropped ${dropped} entries (kept ${hydrated})` ) } - if (dropped > 0 || prunedLegacyClaudeSubagents > 0) { - // Why: persist load-time pruning once so legacy idle rows aren't re-parsed every launch. + if (dropped > 0 || prunedLegacyClaudeSubagents > 0 || scrubbedLegacyLaunchTokens > 0) { + // Why: persist load-time pruning and bearer scrubbing once. this.runStatusPersist() } else if (hydrated > 0) { // Why: prime dedup from raw bytes (not re-serialized) only when hydration was lossless. @@ -2055,17 +2326,92 @@ export class AgentHookServer { } } + private captureHydratedAuthorityCommitments(): void { + this.revokedHydratedAuthorityCommitments = new WeakSet() + for (const entry of this.state.lastStatusByPaneKey.values()) { + const evidence = this.toAuthorityEvidence( + entry as EnrichedAgentHookEventPayload, + this.hydratedLaunchTokenHashByPaneKey.get(entry.paneKey) + ) + if (evidence && !this.persistedAuthorityCommitmentsByPaneKey.has(entry.paneKey)) { + this.persistedAuthorityCommitmentsByPaneKey.set(entry.paneKey, evidence) + } + } + this.hydratedAuthorityCommitments = Object.freeze( + Array.from(this.persistedAuthorityCommitmentsByPaneKey.values()) + ) + } + + private recordCurrentAuthorityObservation(payload: AgentHookEventPayload): void { + const evidence = this.toAuthorityEvidence(payload) + if (evidence) { + this.currentAuthorityObservations.set(evidence.paneKey, evidence) + this.persistedAuthorityCommitmentsByPaneKey.set(evidence.paneKey, evidence) + this.hydratedLaunchTokenHashByPaneKey.set(evidence.paneKey, evidence.launchTokenHash) + } + } + + private toAuthorityEvidence( + payload: AgentHookEventPayload | EnrichedAgentHookEventPayload, + launchTokenHashOverride?: string + ): AgentHookAuthorityEvidence | null { + const launchToken = payload.launchToken?.trim() + const launchTokenHash = + launchTokenHashOverride ?? + (launchToken ? createHash('sha256').update(launchToken).digest('hex') : null) + if (!launchTokenHash) { + return null + } + return Object.freeze({ + paneKey: payload.paneKey, + launchTokenHash, + connectionId: payload.connectionId, + ...(payload.tabId ? { tabId: payload.tabId } : {}), + ...(payload.worktreeId ? { worktreeId: payload.worktreeId } : {}), + observedAt: 'receivedAt' in payload ? payload.receivedAt : Date.now() + }) + } + private serializeStatusFile(): string { - const entries: Record = {} + const entries: Record = {} + const authorityCommitments: Record = {} + const conflictedCommitments = new Set() + for (const [paneKey, commitment] of this.persistedAuthorityCommitmentsByPaneKey) { + authorityCommitments[paneKey] = { ...commitment } + } for (const [paneKey, payload] of this.state.lastStatusByPaneKey) { // Why: never persist invalid keys (matches the hydrate-path invariant). if (!isValidPaneKey(paneKey)) { continue } - const { promptInteractionKey: _promptInteractionKey, ...persistedPayload } = payload - entries[paneKey] = persistedPayload as EnrichedAgentHookEventPayload + const { + promptInteractionKey: _promptInteractionKey, + launchToken, + ...persistedPayload + } = payload as EnrichedAgentHookEventPayload + const launchTokenHash = launchToken?.trim() + ? createHash('sha256').update(launchToken.trim()).digest('hex') + : this.hydratedLaunchTokenHashByPaneKey.get(paneKey) + entries[paneKey] = { + ...persistedPayload, + ...(launchTokenHash ? { launchTokenHash } : {}) + } + const commitment = this.toAuthorityEvidence(payload, launchTokenHash) + if (commitment && !conflictedCommitments.has(paneKey)) { + const existing = authorityCommitments[paneKey] + if (existing && !authorityCommitmentsMatch(existing, commitment)) { + delete authorityCommitments[paneKey] + conflictedCommitments.add(paneKey) + } else { + authorityCommitments[paneKey] = { ...commitment } + } + } + } + const file: LastStatusFile = { + version: LAST_STATUS_FILE_VERSION, + entries, + authorityCommitments } - const file: LastStatusFile = { version: LAST_STATUS_FILE_VERSION, entries } return JSON.stringify(file) } diff --git a/src/main/cli/windows-launcher-asset.test.ts b/src/main/cli/windows-launcher-asset.test.ts index 53c7434d63f..b615e88c88d 100644 --- a/src/main/cli/windows-launcher-asset.test.ts +++ b/src/main/cli/windows-launcher-asset.test.ts @@ -11,4 +11,41 @@ describe('packaged Windows CLI launcher asset', () => { expect(launcher).toContain('orca.cmd cannot safely forward orchestration message bodies') expect(launcher).not.toContain('"%ELECTRON%" "%CLI%" %*') }) + + it('marks the packaged child and propagates its exact exit status', () => { + const sourcePath = join(process.cwd(), 'native', 'windows-cli-launcher', 'OrcaCliLauncher.cs') + const source = readFileSync(sourcePath, 'utf8') + + expect(source).toContain( + 'startInfo.EnvironmentVariables["ORCA_WINDOWS_PACKAGED_CLI_LAUNCHER"] = "1";' + ) + expect(source).toContain('Environment.GetEnvironmentVariable("ORCA_CLI_COMMAND") == "orca-ide"') + expect(source).toContain('? "orca-ide"') + expect(source).toContain(': "orca";') + expect(source).toContain('child.WaitForExit();') + expect(source).toContain('return child.ExitCode;') + }) + + it('requires exact updater process and executable release proof', () => { + const includePath = join(process.cwd(), 'config', 'nsis', 'daemon-host-uninstall.nsh') + const source = readFileSync(includePath, 'utf8') + + expect(source).toContain('!macro customCheckAppRunning') + expect(source).toContain('!include "getProcessInfo.nsh"') + expect(source).toContain('Var pid') + expect(source).toContain('[String]::Equals') + expect(source).toContain('"$INSTDIR\\Orca.exe" "$INSTDIR\\resources\\bin\\orca.exe"') + expect(source).toContain('CreationDate=[string]$$parent.CreationDate') + expect(source).toContain("'ProcessId=' + $$proof.ProcessId") + expect(source).toContain('[IO.FileShare]::None') + expect(source).toContain('$$released=$$false') + expect(source).toContain('$$released=$$true') + expect(source).not.toMatch(/(? ({ + ...(await importOriginal()), + getDefaultWslDistro: () => 'Ubuntu' +})) + import { TerminalHost } from './terminal-host' import type { SubprocessHandle } from './session' import { resolveWslSessionContext } from './wsl-session-context' @@ -60,6 +67,40 @@ describe('TerminalHost WSL context', () => { } }) + it('persists the resolved default distro across daemon attaches', async () => { + const spawnSubprocess = vi.fn(() => createSubprocess()) + host = new TerminalHost({ spawnSubprocess }) + const platform = Object.getOwnPropertyDescriptor(process, 'platform') + Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' }) + try { + const created = await host.createOrAttach({ + sessionId: 'session-default-wsl', + cols: 80, + rows: 24, + cwd: 'C:\\Users\\jin\\repo', + shellOverride: 'wsl.exe', + terminalWindowsWslDistro: null, + streamClient: { onData: vi.fn(), onExit: vi.fn() } + }) + const attached = await host.createOrAttach({ + sessionId: 'session-default-wsl', + cols: 80, + rows: 24, + shellOverride: 'wsl.exe', + terminalWindowsWslDistro: 'Debian', + streamClient: { onData: vi.fn(), onExit: vi.fn() } + }) + + expect(created.wslDistro).toBe('Ubuntu') + expect(attached.wslDistro).toBe('Ubuntu') + expect(spawnSubprocess).toHaveBeenCalledOnce() + } finally { + if (platform) { + Object.defineProperty(process, 'platform', platform) + } + } + }) + it('returns authoritative null when a native session is attached with a WSL preference', async () => { const spawnSubprocess = vi.fn(() => createSubprocess()) host = new TerminalHost({ spawnSubprocess }) @@ -118,6 +159,13 @@ describe('TerminalHost WSL context', () => { terminalWindowsWslDistro: ' Ubuntu ' }) ).toEqual({ distro: 'Ubuntu', treatPosixCwdAsWsl: true }) + expect( + resolveWslSessionContext({ + cwd: 'C:\\Users\\jin\\repo', + shellOverride: 'wsl.exe', + terminalWindowsWslDistro: null + }) + ).toEqual({ distro: 'Ubuntu', treatPosixCwdAsWsl: true }) } finally { if (platform) { Object.defineProperty(process, 'platform', platform) diff --git a/src/main/daemon/wsl-session-context.ts b/src/main/daemon/wsl-session-context.ts index 582e8a66a05..fa69089f55b 100644 --- a/src/main/daemon/wsl-session-context.ts +++ b/src/main/daemon/wsl-session-context.ts @@ -1,6 +1,6 @@ import { splitWorktreeIdForFilesystem } from '../../shared/worktree-id' import { isWslShellName } from '../../shared/local-windows-terminal-runtime' -import { parseWslPath } from '../wsl' +import { getDefaultWslDistro, parseWslPath } from '../wsl' import { parsePtySessionId } from './pty-session-id' import { parseWslUncPath } from '../../shared/wsl-paths' @@ -41,7 +41,8 @@ export function resolveWslSessionContext(args: { return ( (args.sessionId ? getWslContextFromSessionId(args.sessionId) : undefined) ?? (isWslShellName(args.shellOverride) - ? getWslContextFromPreferredDistro(args.terminalWindowsWslDistro) + ? (getWslContextFromPreferredDistro(args.terminalWindowsWslDistro) ?? + getWslContextFromPreferredDistro(getDefaultWslDistro())) : undefined) ) } diff --git a/src/main/index.ts b/src/main/index.ts index 95c214cf886..0fab1442377 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -30,7 +30,12 @@ import { getSshPtyProvider, registerHeadlessPtyRuntime } from './ipc/pty' -import { initDaemonPtyProvider, disconnectDaemon, shutdownDaemon } from './daemon/daemon-init' +import { + initDaemonPtyProvider, + disconnectDaemon, + getDaemonProvider, + shutdownDaemon +} from './daemon/daemon-init' import { closeAllWatchers } from './ipc/filesystem-watcher' import { disposeWorktreeBaseDirectoryWatchers } from './ipc/worktree-base-directory-watcher' import { registerCoreHandlers } from './ipc/register-core-handlers' @@ -132,6 +137,7 @@ import { import { maybeRedirectAppImageCliLaunch } from './startup/appimage-cli-redirect' import { maybeRedirectPackagedCliEntryLaunch } from './startup/packaged-cli-entry-redirect' import { startFirstWindowStartupServices } from './startup/first-window-startup-services' +import { recoverLegacyWorkerTerminalsForRendererStartup } from './startup/legacy-worker-renderer-recovery' import { createWslCliReconciliationStartupBarrier } from './startup/wsl-cli-reconciliation-startup-barrier' import { getDevInstanceIdentity } from './startup/dev-instance-identity' import { hydrateShellPath, mergePathSegments } from './startup/hydrate-shell-path' @@ -724,10 +730,24 @@ if (hasSingleInstanceLock) { } ipcMain.handle('app:awaitFirstWindowStartupServices', async () => { - // Why: restored WSL terminals get a bounded chance to receive launcher repairs before window rendering proceeds. await Promise.all([firstWindowStartupServicesReady, managedWslCliStartupBarrierReady]) }) +ipcMain.handle('app:recoverLegacyWorkerTerminalsForRendererStartup', () => + recoverLegacyWorkerTerminalsForRendererStartup({ + firstWindowStartupServicesReady, + managedWslCliStartupBarrierReady, + localPtyProviderStartupReady, + reconcile: async () => { + await runtime?.refreshRestoredOrchestrationAuthority() + return runtime?.reconcileLegacyWorkerTerminals({ materializeRenderer: true }) + }, + onDeferredRecoveryError: (error) => { + console.warn('[orchestration] legacy worker provider-ready recovery failed', error) + } + }) +) + // Why: the renderer pulls this once its ui:openSettings listener attaches, so a Settings request queued before mount isn't lost. ipcMain.handle('ui:consumePendingOpenSettings', (event) => pendingOpenSettings.matches(event.sender.id, { consume: true }) @@ -2207,6 +2227,11 @@ void app.whenReady().then(async () => { getAgentProviderSessionSnapshot: () => agentHookServer.getStatusSnapshot(), getAgentProviderSessionRowsForPane: (paneKey) => agentHookServer.getStatusSnapshotForPane(paneKey), + attestAgentHookCompatibilityAuthority: (candidate) => + agentHookServer.attestCompatibilityAuthority(candidate), + retireAgentHookCompatibilityAuthority: (paneKey) => + agentHookServer.retirePaneAuthority(paneKey), + canRecoverPersistentLocalPtys: () => getDaemonProvider() !== null, // Why: source codex-home here (runs in window AND serve) so aiVault.listSessions includes managed-Codex sessions; registerCoreHandlers is window-only. getAdditionalAiVaultCodexHomePaths: () => codexRuntimeHome ? codexRuntimeHome.getHostCodexHomePathsForSessionDiscovery() : [], @@ -2222,6 +2247,7 @@ void app.whenReady().then(async () => { orchestrationEnvironmentTransport }) runtime = runtimeService + runtimeService.prepareLegacyWorkerTerminalRecovery() publishProviderSessionChanges(agentHookServer.getProviderSessionIdentities()) browserManager.setBrowserGuestStateChangedListener((worktreeId) => { runtimeService.notifyMobileSessionTabsChanged(worktreeId) @@ -2664,6 +2690,8 @@ void app.whenReady().then(async () => { store, prepareCodexSessionResumeForLaunch ) + await runtime.refreshRestoredOrchestrationAuthority() + await runtime.reconcileLegacyWorkerTerminals() // Why: headless servers can't mount panes; use offscreen WebContents, gated on a real display so browser.headless.v1 stays honest. if (headlessBrowserDisplayAvailable) { runtime.setOffscreenBrowserBackend(new OffscreenBrowserBackend(browserManager)) diff --git a/src/main/ipc/pty.test.ts b/src/main/ipc/pty.test.ts index 190a1288be4..b04550084cc 100644 --- a/src/main/ipc/pty.test.ts +++ b/src/main/ipc/pty.test.ts @@ -3599,6 +3599,64 @@ describe('registerPtyHandlers', () => { }) }) + it('resolves default WSL authority before daemon host env and spawn metadata', async () => { + await withWin32Platform(async () => { + _setWslCachesForTests({ available: true, distros: ['Ubuntu'] }) + const daemonSpawn = setupDaemonAdapter() + const runtime = { + setPtyController: vi.fn(), + registerPty: vi.fn(), + onPtySpawned: vi.fn(), + onPtyExit: vi.fn(), + onPtyData: vi.fn(), + preparePtyExecutionContext: vi.fn().mockReturnValue(true), + getOrchestrationCompatibilityHostId: vi.fn(() => 'compat-host') + } + const settings = { + terminalWindowsShell: 'wsl.exe', + terminalWindowsWslDistro: null, + terminalWindowsPowerShellImplementation: 'auto' + } + handlers.clear() + registerPtyHandlers( + mainWindow as never, + runtime as never, + undefined, + (() => settings) as never + ) + const controller = runtime.setPtyController.mock.calls[0]?.[0] as { + spawn(args: { + cols: number + rows: number + cwd?: string + worktreeId?: string + env?: Record + }): Promise<{ id: string }> + } + + await controller.spawn({ + cols: 80, + rows: 24, + cwd: 'C:\\repo', + worktreeId: 'repo-1::C:\\repo', + env: {} + }) + + const spawnOptions = daemonSpawn.mock.calls.at(-1)?.[0] as DaemonSpawnCall + expect(spawnOptions.terminalWindowsWslDistro).toBe('Ubuntu') + expect(spawnOptions.env).toMatchObject({ + ORCA_ORCHESTRATION_COMPATIBILITY_HOST_KIND: 'wsl', + ORCA_ORCHESTRATION_COMPATIBILITY_HOST_ID: 'compat-host', + ORCA_ORCHESTRATION_COMPATIBILITY_HOST_INCARNATION: 'Ubuntu' + }) + expect(runtime.preparePtyExecutionContext).toHaveBeenCalledWith( + expect.any(String), + 'Ubuntu', + expect.objectContaining({ resetIncarnation: true }) + ) + }) + }) + it('distinguishes an attached native context from an older daemon fallback', async () => { await withWin32Platform(async () => { _setWslCachesForTests({ available: true, distros: ['Ubuntu'] }) @@ -5374,6 +5432,37 @@ describe('registerPtyHandlers', () => { expect(listProcesses).not.toHaveBeenCalled() }) + it('scopes runtime inventories to the requested provider', async () => { + const localList = vi + .spyOn(getLocalPtyProvider(), 'listProcesses') + .mockResolvedValue([{ id: 'local-pty', title: 'Local', cwd: '/local' }]) + const sshAList = vi.fn(async () => [{ id: 'ssh-a-pty' }]) + const sshBList = vi.fn(async () => { + throw new Error('ssh-b unavailable') + }) + registerSshPtyProvider('ssh-a', { listProcesses: sshAList } as never) + registerSshPtyProvider('ssh-b', { listProcesses: sshBList } as never) + const runtime = { setPtyController: vi.fn() } + handlers.clear() + registerPtyHandlers(mainWindow as never, runtime as never) + const controller = runtime.setPtyController.mock.calls[0]?.[0] as { + listProcesses(connectionId?: string | null): Promise<{ id: string }[]> + } + + await expect(controller.listProcesses(null)).resolves.toEqual([ + { id: 'local-pty', title: 'Local', cwd: '/local' } + ]) + expect(localList).toHaveBeenCalledOnce() + expect(sshAList).not.toHaveBeenCalled() + expect(sshBList).not.toHaveBeenCalled() + + await expect(controller.listProcesses('ssh-a')).resolves.toEqual([{ id: 'ssh-a-pty' }]) + expect(sshAList).toHaveBeenCalledOnce() + expect(sshBList).not.toHaveBeenCalled() + + await expect(controller.listProcesses()).rejects.toThrow('ssh-b unavailable') + }) + it('returns unavailable runtime confirmation for unsupported or missing providers', async () => { registerSshPtyProvider('ssh-1', {} as never) setPtyOwnership('unsupported-pty', 'ssh-1') diff --git a/src/main/ipc/pty.ts b/src/main/ipc/pty.ts index 5a82ff3dfa0..3d3c4811747 100644 --- a/src/main/ipc/pty.ts +++ b/src/main/ipc/pty.ts @@ -140,7 +140,7 @@ import { } from '../agent-hooks/migration-unsupported-pty-state' import { parseWslPath } from '../wsl' import { mergePersistedWindowsPath } from '../pty/windows-environment-path' -import { addOrcaWslInteropEnv } from '../pty/wsl-orca-env' +import { addOrcaWslInteropEnv, stampWslOrchestrationCompatibilityHost } from '../pty/wsl-orca-env' import { PtyProducerFlowController } from './pty-producer-flow-control' import { beginTerminalInstall } from './watcher-removal-gate' import { @@ -1787,6 +1787,11 @@ export function registerPtyHandlers( if (preAllocatedHandle) { env.ORCA_TERMINAL_HANDLE = preAllocatedHandle } + stampWslOrchestrationCompatibilityHost( + env, + runtime?.getOrchestrationCompatibilityHostId?.(), + ctx?.isWsl === true ? ctx.wslDistro : null + ) if (ctx?.isWsl === true) { addOrcaWslInteropEnv(env) } @@ -3362,10 +3367,35 @@ export function registerPtyHandlers( }) : { shellOverride: undefined, terminalWindowsWslDistro: null } const daemonShellOverride = terminalRuntimeOptions.shellOverride + const isDaemonHostSpawn = + !args.connectionId && + !(provider instanceof LocalPtyProvider) && + !routesFreshSpawnsToLocalProvider(provider) + const callerRequestedSessionId = args.sessionId?.trim() + const requestedSessionId = + callerRequestedSessionId ?? + (isDaemonHostSpawn && args.agentSessionCreateOperationId + ? ptySessionIdForAgentCreateOperation(args.worktreeId, args.agentSessionCreateOperationId) + : undefined) + const sessionId = + requestedSessionId ?? (isDaemonHostSpawn ? mintPtySessionId(args.worktreeId) : undefined) + const effectiveSessionRelayId = + sessionId !== undefined ? getRelayPtyId(args.connectionId, sessionId) : undefined + const effectiveSessionAppId = + sessionId !== undefined ? getAppPtyId(args.connectionId, sessionId) : undefined + const isMintedSessionId = callerRequestedSessionId === undefined && isDaemonHostSpawn + const expectedWslDistro = !args.connectionId + ? (resolveWslSessionContext({ + cwd, + sessionId, + shellOverride: terminalRuntimeOptions.shellOverride, + terminalWindowsWslDistro: terminalRuntimeOptions.terminalWindowsWslDistro + })?.distro ?? null) + : null const codexSelectionTarget = getCodexSelectionTargetForPty( daemonShellOverride, cwd, - terminalRuntimeOptions.terminalWindowsWslDistro ?? null + expectedWslDistro ) const codexResumePreparation = prepareCodexResumeHome({ connectionId: args.connectionId, @@ -3393,31 +3423,6 @@ export function registerPtyHandlers( ) } - const isDaemonHostSpawn = - !args.connectionId && - !(provider instanceof LocalPtyProvider) && - !routesFreshSpawnsToLocalProvider(provider) - const callerRequestedSessionId = args.sessionId?.trim() - const requestedSessionId = - callerRequestedSessionId ?? - (isDaemonHostSpawn && args.agentSessionCreateOperationId - ? ptySessionIdForAgentCreateOperation(args.worktreeId, args.agentSessionCreateOperationId) - : undefined) - const sessionId = - requestedSessionId ?? (isDaemonHostSpawn ? mintPtySessionId(args.worktreeId) : undefined) - const effectiveSessionRelayId = - sessionId !== undefined ? getRelayPtyId(args.connectionId, sessionId) : undefined - const effectiveSessionAppId = - sessionId !== undefined ? getAppPtyId(args.connectionId, sessionId) : undefined - const isMintedSessionId = callerRequestedSessionId === undefined && isDaemonHostSpawn - const expectedWslDistro = !args.connectionId - ? (resolveWslSessionContext({ - cwd, - sessionId, - shellOverride: terminalRuntimeOptions.shellOverride, - terminalWindowsWslDistro: terminalRuntimeOptions.terminalWindowsWslDistro - })?.distro ?? null) - : null const shouldPersistHostSessionBinding = args.persistHostSessionBinding === true let hostSessionBinding: { store: NonNullable @@ -3492,11 +3497,16 @@ export function registerPtyHandlers( launchAgent: isTuiAgent(args.launchAgent) ? args.launchAgent : undefined, shellPath: daemonShellOverride ?? process.env.COMSPEC, isWsl: shouldSkipCodexHomeEnvForWindowsShell(daemonShellOverride, cwd), - wslDistro: codexSelectionTarget.runtime === 'wsl' ? codexSelectionTarget.wslDistro : null, + wslDistro: codexSelectionTarget.runtime === 'wsl' ? expectedWslDistro : null, agentStatusHooksEnabled: isAgentStatusHooksEnabled(getSettings?.()), networkProxySettings: getSettings?.(), deferGitConfigGuardToDaemon: provider.supportsGitCredentialGuardHost?.(sessionId) === true }) + stampWslOrchestrationCompatibilityHost( + env, + runtime?.getOrchestrationCompatibilityHostId?.(), + codexSelectionTarget.runtime === 'wsl' ? expectedWslDistro : null + ) promoteAgentTeamsShimPath(env, requestedAgentTeamsPath) } @@ -3596,8 +3606,7 @@ export function registerPtyHandlers( } if (process.platform === 'win32' && !args.connectionId) { spawnOptions.shellOverride = terminalRuntimeOptions.shellOverride - spawnOptions.terminalWindowsWslDistro = - terminalRuntimeOptions.terminalWindowsWslDistro ?? null + spawnOptions.terminalWindowsWslDistro = expectedWslDistro spawnOptions.terminalWindowsPowerShellImplementation = getSettings ? (getSettings()?.terminalWindowsPowerShellImplementation ?? 'auto') : undefined @@ -4243,7 +4252,13 @@ export function registerPtyHandlers( return null } }, - listProcesses: async () => { + listProcesses: async (connectionId) => { + if (connectionId === null) { + return localProvider.listProcesses() + } + if (connectionId !== undefined) { + return getProvider(connectionId).listProcesses() + } const providerSessions = await Promise.all([ localProvider.listProcesses(), ...Array.from(sshProviders.values(), (provider) => provider.listProcesses()) @@ -4450,24 +4465,6 @@ export function registerPtyHandlers( }) : { shellOverride: args.shellOverride, terminalWindowsWslDistro: null } const initialShellOverride = terminalRuntimeOptions.shellOverride - const initialSelectionTarget = getCodexSelectionTargetForPty( - initialShellOverride, - cwd, - terminalRuntimeOptions.terminalWindowsWslDistro ?? null - ) - const claudeAuth = - isClaudeLaunch && prepareClaudeAuth ? await prepareClaudeAuth(initialSelectionTarget) : null - spawnTiming.mark('auth') - if (isClaudeLaunch && isClaudeAuthSwitchInProgress()) { - throw new Error('A Claude account switch is in progress. Try again after it finishes.') - } - if (claudeAuth?.stripAuthEnv && hasClaudeAuthEnvConflict(args.env)) { - throw new Error( - 'This Claude launch defines explicit Anthropic auth environment variables. Remove those overrides before using a managed Claude account.' - ) - } - // Why: the daemon-backed provider skips LocalPtyProvider's buildSpawnEnv, so assemble the same host-local env here for parity. - // Safety: skip entirely for SSH — every injection is a loopback secret or a local path that leaks or misleads on the remote host. const isDaemonHostSpawn = !args.connectionId && !(provider instanceof LocalPtyProvider) && @@ -4494,6 +4491,24 @@ export function registerPtyHandlers( terminalWindowsWslDistro: terminalRuntimeOptions.terminalWindowsWslDistro })?.distro ?? null) : null + const initialSelectionTarget = getCodexSelectionTargetForPty( + initialShellOverride, + cwd, + expectedWslDistro + ) + const claudeAuth = + isClaudeLaunch && prepareClaudeAuth ? await prepareClaudeAuth(initialSelectionTarget) : null + spawnTiming.mark('auth') + if (isClaudeLaunch && isClaudeAuthSwitchInProgress()) { + throw new Error('A Claude account switch is in progress. Try again after it finishes.') + } + if (claudeAuth?.stripAuthEnv && hasClaudeAuthEnvConflict(args.env)) { + throw new Error( + 'This Claude launch defines explicit Anthropic auth environment variables. Remove those overrides before using a managed Claude account.' + ) + } + // Why: the daemon-backed provider skips LocalPtyProvider's buildSpawnEnv, so assemble the same host-local env here for parity. + // Safety: skip entirely for SSH — every injection is a loopback secret or a local path that leaks or misleads on the remote host. const startupTerminalColorQueryReplyColors = getStartupTerminalColorQueryReplyColors(args) // Why: forward pane env to SSH only when the relay hook path is enabled, or a newer relay could emit statuses this build can't route. const sshSourceEnv = stripRemotePaneEnvWhenHooksDisabled(args.connectionId, args.env) @@ -4603,7 +4618,7 @@ export function registerPtyHandlers( const codexSelectionTarget = getCodexSelectionTargetForPty( effectiveShellOverride, cwd, - terminalRuntimeOptions.terminalWindowsWslDistro ?? null + expectedWslDistro ) const codexResumePreparation = prepareCodexResumeHome({ connectionId: args.connectionId, @@ -4669,13 +4684,17 @@ export function registerPtyHandlers( launchAgent: isTuiAgent(args.launchAgent) ? args.launchAgent : undefined, shellPath: effectiveShellOverride ?? process.env.COMSPEC, isWsl: shouldSkipCodexHomeEnvForWindowsShell(effectiveShellOverride, cwd), - wslDistro: - codexSelectionTarget.runtime === 'wsl' ? codexSelectionTarget.wslDistro : null, + wslDistro: codexSelectionTarget.runtime === 'wsl' ? expectedWslDistro : null, agentStatusHooksEnabled: isAgentStatusHooksEnabled(getSettings?.()), networkProxySettings: getSettings?.(), deferGitConfigGuardToDaemon: provider.supportsGitCredentialGuardHost?.(effectiveSessionId) === true }) + stampWslOrchestrationCompatibilityHost( + env, + runtime?.getOrchestrationCompatibilityHostId?.(), + codexSelectionTarget.runtime === 'wsl' ? expectedWslDistro : null + ) promoteAgentTeamsShimPath(env, requestedAgentTeamsPath) } catch (err) { // Why: buildPtyHostEnv has fs side-effects (Pi/OMP install); clear per-PTY state on throw, but only minted ids — caller ids may name existing PTYs. @@ -4762,8 +4781,7 @@ export function registerPtyHandlers( } if (process.platform === 'win32' && !args.connectionId) { // Why: the renderer models PowerShell as one shell family; thread the implementation choice so both PTY paths resolve the same executable. - spawnOptions.terminalWindowsWslDistro = - terminalRuntimeOptions.terminalWindowsWslDistro ?? null + spawnOptions.terminalWindowsWslDistro = expectedWslDistro spawnOptions.terminalWindowsPowerShellImplementation = getSettings ? (getSettings()?.terminalWindowsPowerShellImplementation ?? 'auto') : undefined diff --git a/src/main/persistence.test.ts b/src/main/persistence.test.ts index 9c0b0d47e9e..0730f3a98ba 100644 --- a/src/main/persistence.test.ts +++ b/src/main/persistence.test.ts @@ -3691,6 +3691,21 @@ describe('Store', () => { expect(hostSession.lastVisitedAtByWorktreeId?.['r1::/path/wt1']).toBeUndefined() }) + it('lists only persisted workspace-session host partitions', async () => { + const store = await createStore() + expect(store.getWorkspaceSessionHostIds()).toEqual(['local']) + + store.getWorkspaceSession('ssh:not-persisted') + store.setWorkspaceSession(getDefaultWorkspaceSession(), 'ssh:ssh-a') + store.setWorkspaceSession(getDefaultWorkspaceSession(), 'runtime:environment-a') + + expect(store.getWorkspaceSessionHostIds()).toEqual([ + 'local', + 'ssh:ssh-a', + 'runtime:environment-a' + ]) + }) + it('removeProject removes the derived project host setup compatibility record', async () => { const store = await createStore() store.addRepo(makeRepo({ id: 'r1' })) diff --git a/src/main/persistence.ts b/src/main/persistence.ts index d0f27346e83..37108fbbce9 100644 --- a/src/main/persistence.ts +++ b/src/main/persistence.ts @@ -5769,6 +5769,17 @@ export class Store { return this.state.workspaceSessionsByHostId?.[resolved] ?? getDefaultWorkspaceSession() } + getWorkspaceSessionHostIds(): ExecutionHostId[] { + const hostIds = new Set([LOCAL_EXECUTION_HOST_ID]) + for (const key of Object.keys(this.state.workspaceSessionsByHostId ?? {})) { + const hostId = normalizeExecutionHostId(key) + if (hostId) { + hostIds.add(hostId) + } + } + return [...hostIds] + } + readTerminalScrollbackSnapshot(ref: string): string | null { return readTerminalScrollbackSnapshotSync(ref, this.terminalScrollbackSnapshotStorage) } diff --git a/src/main/providers/local-pty-provider.test.ts b/src/main/providers/local-pty-provider.test.ts index 632cfe90591..5f90d22b64e 100644 --- a/src/main/providers/local-pty-provider.test.ts +++ b/src/main/providers/local-pty-provider.test.ts @@ -96,6 +96,7 @@ vi.mock('../wsl', () => ({ toLinuxPath: (path: string) => path.replace(/^C:\\/i, '/mnt/c/').replace(/\\/g, '/'), toWindowsWslPath: (path: string, distro: string) => `\\\\wsl.localhost\\${distro}${path.replace(/\//g, '\\')}`, + getDefaultWslDistro: () => 'Ubuntu', isWslAvailable: () => true, // Why: WSL worktree validation now asks the distro; these tests use WSL UNC // cwds that are meant to exist, so report them present without spawning wsl.exe. @@ -880,6 +881,45 @@ describe('LocalPtyProvider', () => { expect(spawnCall[2].env.HISTFILE).toContain('terminal-history-wsl/Debian') }) + it('resolves and persists the default distro for Windows cwd WSL terminals', async () => { + Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' }) + const buildSpawnEnv = vi.fn( + ( + _id: string, + env: Record, + _ctx?: { isWsl?: boolean; wslDistro?: string | null } + ) => env + ) + provider.configure({ buildSpawnEnv }) + + const result = await provider.spawn({ + cols: 80, + rows: 24, + worktreeId: 'repo-1::C:\\Users\\jin\\repo', + cwd: 'C:\\Users\\jin\\repo', + shellOverride: 'wsl.exe', + terminalWindowsWslDistro: null + }) + + const spawnCall = spawnMock.mock.calls.at(-1)! + expect(spawnCall[1]).toEqual([ + '-d', + 'Ubuntu', + '--', + 'sh', + '-c', + expect.stringContaining("cd '/mnt/c/Users/jin/repo'") + ]) + expect(buildSpawnEnv.mock.calls[0]?.[2]).toMatchObject({ + isWsl: true, + wslDistro: 'Ubuntu' + }) + expect(result.wslDistro).toBe('Ubuntu') + expect( + (await provider.listProcesses()).find((entry) => entry.id === result.id)?.wslDistro + ).toBe('Ubuntu') + }) + it('repro: keeps explicit PowerShell 7 selection when the pwsh probe is cold-false', async () => { Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' }) const pwshAvailable = vi.fn(() => false) diff --git a/src/main/providers/local-pty-provider.ts b/src/main/providers/local-pty-provider.ts index 5e29ee31eb7..c3ad6f36103 100644 --- a/src/main/providers/local-pty-provider.ts +++ b/src/main/providers/local-pty-provider.ts @@ -11,7 +11,7 @@ import { buildWindowsPowerShellSpawnAttempts } from './windows-shell-fallback-ch import { resolveProcessCwd } from './process-cwd' import { existsSync } from 'node:fs' import * as pty from 'node-pty' -import { parseWslPath, isWslAvailable } from '../wsl' +import { getDefaultWslDistro, parseWslPath, isWslAvailable } from '../wsl' import { splitWorktreeIdForFilesystem } from '../../shared/worktree-id' import { injectHistoryEnv, @@ -547,6 +547,10 @@ export class LocalPtyProvider implements IPtyProvider { process.platform === 'win32' ? getWslContextFromPreferredDistro(args.terminalWindowsWslDistro) : undefined + let launchWslContext = + wslInfo !== null + ? getWslContextFromPreferredDistro(wslInfo.distro) + : (worktreeWslContext ?? preferredWslContext) let shellPath: string let shellArgs: string[] @@ -572,6 +576,9 @@ export class LocalPtyProvider implements IPtyProvider { process.env.COMSPEC || 'powershell.exe' const shellFamily = worktreeWslContext ? 'wsl.exe' : requestedShellFamily + if (!launchWslContext && pathWin32.basename(shellFamily).toLowerCase() === 'wsl.exe') { + launchWslContext = getWslContextFromPreferredDistro(getDefaultWslDistro()) + } const normalizedShellFamily = pathWin32.basename(shellFamily).toLowerCase() const resolvedGitBashPath = resolveWindowsGitBashShellPath(shellFamily) // Why: normalize setting-value and path forms to the PowerShell family so the resolver can fall back to inbox powershell.exe. @@ -606,7 +613,7 @@ export class LocalPtyProvider implements IPtyProvider { shellPath, cwd, defaultCwd, - wslContext: worktreeWslContext ?? preferredWslContext, + wslContext: launchWslContext, startupCommand: args.command }) const primaryAttempt = windowsFallbackAttempts[0] @@ -621,7 +628,7 @@ export class LocalPtyProvider implements IPtyProvider { shellPath, cwd, defaultCwd, - worktreeWslContext ?? preferredWslContext, + launchWslContext, args.command ) shellArgs = resolved.shellArgs @@ -672,8 +679,7 @@ export class LocalPtyProvider implements IPtyProvider { } const isWslShell = Boolean(wslInfo) || pathWin32.basename(shellPath).toLowerCase() === 'wsl.exe' - const launchWslDistro = - wslInfo?.distro ?? worktreeWslContext?.distro ?? preferredWslContext?.distro ?? null + const launchWslDistro = isWslShell ? (launchWslContext?.distro ?? null) : null const finalEnv = this.opts.buildSpawnEnv ? this.opts.buildSpawnEnv(id, spawnEnv, { command: args.command, @@ -796,7 +802,7 @@ export class LocalPtyProvider implements IPtyProvider { let historyResult: ReturnType | null = null if (historyEnabled) { historyResult = injectHistoryEnv(finalEnv, worktreeId, effectiveShellPath, cwd, { - wslDistro: preferredWslContext?.distro ?? worktreeWslContext?.distro ?? null + wslDistro: launchWslDistro }) logHistoryInjection(worktreeId, historyResult) } diff --git a/src/main/pty/wsl-orca-env.test.ts b/src/main/pty/wsl-orca-env.test.ts index 1d2831c1855..e4b1c9d06a6 100644 --- a/src/main/pty/wsl-orca-env.test.ts +++ b/src/main/pty/wsl-orca-env.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from 'vitest' -import { addOrcaWslInteropEnv, addWorktreeSetupWslInteropEnv } from './wsl-orca-env' +import { + addOrcaWslInteropEnv, + addWorktreeSetupWslInteropEnv, + stampWslOrchestrationCompatibilityHost +} from './wsl-orca-env' describe('addOrcaWslInteropEnv', () => { it('marks the Orca terminal handle for Windows to WSL env import', () => { @@ -29,10 +33,14 @@ describe('addOrcaWslInteropEnv', () => { ORCA_PANE_KEY: 'tab-1:leaf-1', ORCA_TAB_ID: 'tab-1', ORCA_WORKTREE_ID: 'repo::\\\\wsl.localhost\\Ubuntu\\home\\jin\\repo', + ORCA_AGENT_LAUNCH_TOKEN: 'launch-secret', ORCA_AGENT_HOOK_PORT: '4567', ORCA_AGENT_HOOK_TOKEN: 'token', ORCA_AGENT_HOOK_ENV: 'dev', - ORCA_AGENT_HOOK_VERSION: '1' + ORCA_AGENT_HOOK_VERSION: '1', + ORCA_ORCHESTRATION_COMPATIBILITY_HOST_KIND: 'wsl', + ORCA_ORCHESTRATION_COMPATIBILITY_HOST_ID: 'local', + ORCA_ORCHESTRATION_COMPATIBILITY_HOST_INCARNATION: 'Ubuntu' } addOrcaWslInteropEnv(env) @@ -44,10 +52,44 @@ describe('addOrcaWslInteropEnv', () => { expect(env.WSLENV).toContain('ORCA_PANE_KEY/u') expect(env.WSLENV).toContain('ORCA_TAB_ID/u') expect(env.WSLENV).toContain('ORCA_WORKTREE_ID/u') + expect(env.WSLENV).toContain('ORCA_AGENT_LAUNCH_TOKEN/u') expect(env.WSLENV).toContain('ORCA_AGENT_HOOK_PORT/u') expect(env.WSLENV).toContain('ORCA_AGENT_HOOK_TOKEN/u') expect(env.WSLENV).toContain('ORCA_AGENT_HOOK_ENV/u') expect(env.WSLENV).toContain('ORCA_AGENT_HOOK_VERSION/u') + expect(env.WSLENV).toContain('ORCA_ORCHESTRATION_COMPATIBILITY_HOST_KIND/u') + expect(env.WSLENV).toContain('ORCA_ORCHESTRATION_COMPATIBILITY_HOST_ID/u') + expect(env.WSLENV).toContain('ORCA_ORCHESTRATION_COMPATIBILITY_HOST_INCARNATION/u') + }) + + it('overwrites caller host evidence with native runtime WSL authority', () => { + const env = { + ORCA_ORCHESTRATION_COMPATIBILITY_HOST_KIND: 'ssh', + ORCA_ORCHESTRATION_COMPATIBILITY_HOST_ID: 'caller-host', + ORCA_ORCHESTRATION_COMPATIBILITY_HOST_INCARNATION: 'caller-incarnation', + ORCA_ORCHESTRATION_COMPATIBILITY_ATTACHMENT: 'caller-attachment' + } + + stampWslOrchestrationCompatibilityHost(env, 'local', 'Ubuntu') + + expect(env).toEqual({ + ORCA_ORCHESTRATION_COMPATIBILITY_HOST_KIND: 'wsl', + ORCA_ORCHESTRATION_COMPATIBILITY_HOST_ID: 'local', + ORCA_ORCHESTRATION_COMPATIBILITY_HOST_INCARNATION: 'Ubuntu' + }) + }) + + it('clears inherited host evidence outside a runtime-owned WSL scope', () => { + const env = { + ORCA_ORCHESTRATION_COMPATIBILITY_HOST_KIND: 'ssh', + ORCA_ORCHESTRATION_COMPATIBILITY_HOST_ID: 'caller-host', + ORCA_ORCHESTRATION_COMPATIBILITY_HOST_INCARNATION: 'caller-incarnation', + ORCA_ORCHESTRATION_COMPATIBILITY_ATTACHMENT: 'caller-attachment' + } + + stampWslOrchestrationCompatibilityHost(env, 'local', null) + + expect(env).toEqual({}) }) it('path-translates a Windows hook endpoint but passes a guest-side one untouched', () => { diff --git a/src/main/pty/wsl-orca-env.ts b/src/main/pty/wsl-orca-env.ts index f4f72953f34..bc4ec7aee8f 100644 --- a/src/main/pty/wsl-orca-env.ts +++ b/src/main/pty/wsl-orca-env.ts @@ -1,3 +1,10 @@ +import { + ORCHESTRATION_COMPATIBILITY_ATTACHMENT_ENV, + ORCHESTRATION_COMPATIBILITY_HOST_ID_ENV, + ORCHESTRATION_COMPATIBILITY_HOST_INCARNATION_ENV, + ORCHESTRATION_COMPATIBILITY_HOST_KIND_ENV +} from '../../shared/orchestration-compatibility-evidence' + const WSLENV_ENTRY_SEPARATOR = ':' function parseWslenvEntries(value: string | undefined): string[] { @@ -70,6 +77,9 @@ export function addOrcaWslInteropEnv(env: Record): void { 'ORCA_TAB_ID/u', 'ORCA_WORKTREE_ID/u', 'ORCA_AGENT_LAUNCH_TOKEN/u', + 'ORCA_ORCHESTRATION_COMPATIBILITY_HOST_KIND/u', + 'ORCA_ORCHESTRATION_COMPATIBILITY_HOST_ID/u', + 'ORCA_ORCHESTRATION_COMPATIBILITY_HOST_INCARNATION/u', 'ORCA_AGENT_HOOK_PORT/u', 'ORCA_AGENT_HOOK_TOKEN/u', 'ORCA_AGENT_HOOK_ENV/u', @@ -84,3 +94,22 @@ export function addOrcaWslInteropEnv(env: Record): void { ] applyWslenvPassthrough(env, passthroughEntries) } + +export function stampWslOrchestrationCompatibilityHost( + env: Record, + hostId: string | null | undefined, + distro: string | null | undefined +): void { + delete env[ORCHESTRATION_COMPATIBILITY_HOST_KIND_ENV] + delete env[ORCHESTRATION_COMPATIBILITY_HOST_ID_ENV] + delete env[ORCHESTRATION_COMPATIBILITY_HOST_INCARNATION_ENV] + delete env[ORCHESTRATION_COMPATIBILITY_ATTACHMENT_ENV] + const normalizedHostId = hostId?.trim() + const normalizedDistro = distro?.trim() + if (!normalizedHostId || !normalizedDistro) { + return + } + env[ORCHESTRATION_COMPATIBILITY_HOST_KIND_ENV] = 'wsl' + env[ORCHESTRATION_COMPATIBILITY_HOST_ID_ENV] = normalizedHostId + env[ORCHESTRATION_COMPATIBILITY_HOST_INCARNATION_ENV] = normalizedDistro +} diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index cbf0b08ba4f..b44dfc5c5ed 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -3,7 +3,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type * as GitUsernameModule from '../git/git-username' import { performance } from 'node:perf_hooks' import { EventEmitter } from 'node:events' -import { randomUUID } from 'node:crypto' +import { createHash, randomUUID } from 'node:crypto' import { execFileSync } from 'node:child_process' import { mkdirSync } from 'node:fs' import { lstat, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' @@ -55,7 +55,7 @@ import { shouldRunSetupForCreate } from '../hooks' import { getBaseRefDefault, getBranchConflictKind } from '../git/repo' -import type { OrchestrationDb } from './orchestration/db' +import { OrchestrationDb } from './orchestration/db' import type { MessagePriority, MessageRow, MessageType } from './orchestration/types' import { appendNormalizedToTailBuffer, @@ -930,6 +930,10 @@ const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0- const HEADLESS_LEAF_ID = '11111111-1111-4111-8111-111111111111' const HEADLESS_SECOND_LEAF_ID = '22222222-2222-4222-8222-222222222222' const HEADLESS_THIRD_LEAF_ID = '33333333-3333-4333-8333-333333333333' +const RESTORED_AUTHORITY_TOKEN = 'restored-authority-secret' +const RESTORED_AUTHORITY_TOKEN_HASH = createHash('sha256') + .update(RESTORED_AUTHORITY_TOKEN) + .digest('hex') function isOriginMainBaseRefProbe(args: string[]): boolean { return ( @@ -11132,6 +11136,542 @@ describe('OrcaRuntimeService', () => { }) }) + it('retires inherited launch authority when the agent command exits', async () => { + const spawn = vi.fn().mockResolvedValue({ id: 'pty-authority', incarnationId: 'process-1' }) + const retireAuthority = vi.fn() + const runtime = new OrcaRuntimeService(store, undefined, { + attestAgentHookCompatibilityAuthority: (candidate) => ({ + paneKey: candidate.paneKey, + source: 'current_hook' + }), + retireAgentHookCompatibilityAuthority: retireAuthority + }) + runtime.setPtyController({ + spawn, + write: () => true, + kill: () => true, + getForegroundProcess: async () => null + }) + runtime.setNotifier({ + worktreesChanged: vi.fn(), + reposChanged: vi.fn(), + activateWorktree: vi.fn(), + createTerminal: vi.fn(), + revealTerminalSession: vi.fn().mockResolvedValue({ tabId: 'tab-authority' }), + splitTerminal: vi.fn(), + renameTerminal: vi.fn(), + focusTerminal: vi.fn(), + closeTerminal: vi.fn(), + sleepWorktree: vi.fn(), + terminalFitOverrideChanged: vi.fn(), + terminalDriverChanged: vi.fn() + }) + runtime.attachWindow(1) + runtime.syncWindowGraph(1, { tabs: [], leaves: [] }) + + const terminal = await runtime.createTerminal(`path:${TEST_WORKTREE_PATH}`, { + command: 'codex', + launchConfig: { agentCommand: 'codex', agentArgs: '', agentEnv: {} } + }) + const spawnEnv = + (spawn.mock.calls[0]?.[0] as { env?: Record } | undefined)?.env ?? {} + const evidence = { + terminalHandle: terminal.handle, + paneKey: spawnEnv.ORCA_PANE_KEY, + launchToken: spawnEnv.ORCA_AGENT_LAUNCH_TOKEN + } + + expect(runtime.verifyOrchestrationCompatibilityCaller(evidence)).not.toBeNull() + expect( + runtime.getAgentStatusLaunchConfigForPaneKey(spawnEnv.ORCA_PANE_KEY, { + launchToken: spawnEnv.ORCA_AGENT_LAUNCH_TOKEN + }) + ).toBeDefined() + + runtime.onPtyData('pty-authority', '\x1b]133;D;0\x07', 100) + + expect(retireAuthority).toHaveBeenCalledWith(spawnEnv.ORCA_PANE_KEY) + expect(runtime.verifyOrchestrationCompatibilityCaller(evidence)).toBeNull() + expect( + runtime.getAgentStatusLaunchConfigForPaneKey(spawnEnv.ORCA_PANE_KEY, { + launchToken: spawnEnv.ORCA_AGENT_LAUNCH_TOKEN + }) + ).toBeUndefined() + }) + + it('retires only receipted restored PTY authority on command completion and exit', () => { + const retireAuthority = vi.fn() + const runtime = new OrcaRuntimeService(store, undefined, { + retireAgentHookCompatibilityAuthority: retireAuthority + }) + const internals = runtime as unknown as { + recordPtyWorktree: (ptyId: string, worktreeId: string, state: Record) => void + restoredOrchestrationAuthorityByPtyId: Map> + } + const firstPane = '11111111-1111-4111-8111-111111111111:22222222-2222-4222-8222-222222222222' + const secondPane = '33333333-3333-4333-8333-333333333333:44444444-4444-4444-8444-444444444444' + internals.recordPtyWorktree('pty-restored-command', TEST_WORKTREE_ID, { + connected: true, + tabId: '11111111-1111-4111-8111-111111111111', + paneKey: firstPane, + incarnationId: 'restored-command' + }) + internals.recordPtyWorktree('pty-restored-exit', TEST_WORKTREE_ID, { + connected: true, + tabId: '33333333-3333-4333-8333-333333333333', + paneKey: secondPane, + incarnationId: 'restored-exit' + }) + internals.recordPtyWorktree('pty-ordinary-shell', TEST_WORKTREE_ID, { + connected: true, + tabId: '55555555-5555-4555-8555-555555555555', + paneKey: '55555555-5555-4555-8555-555555555555:66666666-6666-4666-8666-666666666666', + incarnationId: 'ordinary-shell' + }) + internals.restoredOrchestrationAuthorityByPtyId.set('pty-restored-command', { + ptyId: 'pty-restored-command', + worktreeId: TEST_WORKTREE_ID, + terminalHandle: 'term-restored-command', + paneKey: firstPane, + processIncarnation: 'pty-restored-command:restored-command', + hostScope: { kind: 'local', hostId: 'local' } + }) + internals.restoredOrchestrationAuthorityByPtyId.set('pty-restored-exit', { + ptyId: 'pty-restored-exit', + worktreeId: TEST_WORKTREE_ID, + terminalHandle: 'term-restored-exit', + paneKey: secondPane, + processIncarnation: 'pty-restored-exit:restored-exit', + hostScope: { kind: 'local', hostId: 'local' } + }) + + runtime.emitDaemonPtyTransientFact('pty-restored-command', { + kind: 'command-finished', + exitCode: 0 + }) + runtime.onPtyExit('pty-restored-exit', 0, 'restored-exit') + runtime.onPtyExit('pty-ordinary-shell', 0, 'ordinary-shell') + + expect(retireAuthority).toHaveBeenCalledWith(firstPane) + expect(retireAuthority).toHaveBeenCalledWith(secondPane) + expect(retireAuthority).toHaveBeenCalledTimes(2) + }) + + it('restores a retained coordinator handle after a late controller inventory', async () => { + const paneKey = makePaneKey('host-tab', HEADLESS_LEAF_ID) + const incarnationId = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' + const session = makeWorkspaceSessionWithHeadlessTerminal({ + terminalPtyIncarnationsByPaneKey: { [paneKey]: incarnationId } + }) + const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession(session) + const runtime = new OrcaRuntimeService(runtimeStore as never, undefined, { + canRecoverPersistentLocalPtys: () => true, + attestAgentHookCompatibilityAuthority: ({ paneKey: candidate, launchTokenHash }) => + candidate === paneKey && launchTokenHash === RESTORED_AUTHORITY_TOKEN_HASH + ? { paneKey: candidate, source: 'hydrated_commitment' } + : null + }) + const controllerHandle = 'term_retained_coordinator' + const listProcesses = vi + .fn() + .mockRejectedValueOnce(new Error('provider starting')) + .mockResolvedValue([ + { + id: 'persisted-pty', + incarnationId, + terminalHandle: controllerHandle, + title: 'Coordinator', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + } + ]) + runtime.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null, + listProcesses + }) + + await expect(runtime.refreshRestoredOrchestrationAuthority()).rejects.toThrow( + 'terminal_liveness_unavailable' + ) + runtime.attachWindow(TEST_WINDOW_ID) + runtime.syncWindowGraph(TEST_WINDOW_ID, { + tabs: [ + { + tabId: 'host-tab', + worktreeId: TEST_WORKTREE_ID, + title: 'Coordinator', + activeLeafId: HEADLESS_LEAF_ID, + layout: null + } + ], + leaves: [ + { + tabId: 'host-tab', + worktreeId: TEST_WORKTREE_ID, + leafId: HEADLESS_LEAF_ID, + paneRuntimeId: 1, + ptyId: 'persisted-pty' + } + ] + }) + const syntheticHandle = runtime.getAgentStatusTerminalHandleForPaneKey(paneKey) + expect(syntheticHandle).toMatch(/^term_/) + expect(syntheticHandle).not.toBe(controllerHandle) + + await expect(runtime.refreshRestoredOrchestrationAuthority()).resolves.toBeUndefined() + + expect(runtime.getAgentStatusTerminalHandleForPaneKey(paneKey)).toBe(controllerHandle) + expect( + runtime.verifyOrchestrationCompatibilityCaller({ + terminalHandle: controllerHandle, + paneKey, + launchToken: RESTORED_AUTHORITY_TOKEN + }) + ).toMatchObject({ + terminalHandle: controllerHandle, + paneKey, + processIncarnation: `persisted-pty:${incarnationId}` + }) + }) + + it('forgets synthetic handles when disconnected PTY records are pruned', () => { + const runtime = new OrcaRuntimeService(store) + const internals = runtime as unknown as { + recordPtyWorktree: ( + ptyId: string, + worktreeId: string, + state: Record + ) => unknown + issuePtyHandle: (pty: unknown) => string + dropDisconnectedPtyRecord: (ptyId: string) => void + syntheticTerminalHandles: Set + } + const pty = internals.recordPtyWorktree('pty-pruned', TEST_WORKTREE_ID, { + connected: false + }) + const handle = internals.issuePtyHandle(pty) + expect(internals.syntheticTerminalHandles.has(handle)).toBe(true) + + internals.dropDisconnectedPtyRecord('pty-pruned') + + expect(internals.syntheticTerminalHandles.has(handle)).toBe(false) + }) + + it('drops an out-of-order aggregate inventory after a newer SSH inventory', async () => { + const targetId = 'ssh-1' + const ptyId = `ssh:${targetId}@@persisted-pty` + const paneKey = makePaneKey('host-tab', HEADLESS_LEAF_ID) + const oldIncarnation = 'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb' + const newIncarnation = 'cccccccc-cccc-4ccc-8ccc-cccccccccccc' + const session = makeWorkspaceSessionWithHeadlessTerminal({ + tabsByWorktree: { + [TEST_WORKTREE_ID]: [ + { + id: 'host-tab', + ptyId, + worktreeId: TEST_WORKTREE_ID, + title: 'Persisted Terminal', + customTitle: null, + color: null, + sortOrder: 0, + createdAt: 1 + } + ] + }, + terminalLayoutsByTabId: { + 'host-tab': makeHeadlessTerminalLayout({ [HEADLESS_LEAF_ID]: ptyId }) + }, + terminalPtyIncarnationsByPaneKey: { [paneKey]: oldIncarnation } + }) + const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession(session) + const runtime = new OrcaRuntimeService(runtimeStore as never, undefined, { + attestAgentHookCompatibilityAuthority: ({ paneKey: candidate, launchTokenHash }) => + candidate === paneKey && launchTokenHash === RESTORED_AUTHORITY_TOKEN_HASH + ? { paneKey: candidate, source: 'hydrated_commitment' } + : null + }) + const oldInventory = deferred< + { + id: string + incarnationId: string + terminalHandle: string + worktreeId: string + cwd: string + title: string + wslDistro: null + }[] + >() + const newInventory = + deferred ? T : never>() + const listProcesses = vi + .fn() + .mockImplementationOnce(() => oldInventory.promise) + .mockImplementationOnce(() => newInventory.promise) + runtime.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null, + listProcesses + }) + const internals = runtime as unknown as { + refreshPtyWorktreeRecordsWithControllerInventory: ( + worktrees: [], + targetWorktreeId: string | null, + deadline: number | undefined, + connectionId: string | null | undefined + ) => Promise + ptysById: Map + restoredOrchestrationAuthorityByPtyId: Map + } + const host = runtime.registerOrchestrationCompatibilitySshAttachment( + targetId, + 'connection-incarnation' + ) + + const staleRefresh = internals.refreshPtyWorktreeRecordsWithControllerInventory( + [], + null, + undefined, + undefined + ) + const currentRefresh = internals.refreshPtyWorktreeRecordsWithControllerInventory( + [], + null, + undefined, + targetId + ) + newInventory.resolve([ + { + id: ptyId, + incarnationId: newIncarnation, + terminalHandle: 'term_new_process', + worktreeId: TEST_WORKTREE_ID, + cwd: TEST_WORKTREE_PATH, + title: 'Replacement', + wslDistro: null + } + ]) + await expect(currentRefresh).resolves.not.toBeNull() + oldInventory.resolve([ + { + id: ptyId, + incarnationId: oldIncarnation, + terminalHandle: 'term_old_process', + worktreeId: TEST_WORKTREE_ID, + cwd: TEST_WORKTREE_PATH, + title: 'Retained coordinator', + wslDistro: null + } + ]) + await expect(staleRefresh).resolves.toBeNull() + + expect(internals.ptysById.get(ptyId)?.incarnationId).toBe(newIncarnation) + expect(internals.restoredOrchestrationAuthorityByPtyId.has(ptyId)).toBe(false) + expect( + runtime.verifyOrchestrationCompatibilityCaller({ + terminalHandle: 'term_old_process', + paneKey, + launchToken: RESTORED_AUTHORITY_TOKEN, + host + }) + ).toBeNull() + }) + + it('keeps restored receipts outside a targeted worktree scan', async () => { + const secondWorktreeId = `${TEST_REPO_ID}::/tmp/worktree-b` + const runtime = new OrcaRuntimeService(store) + runtime.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null, + listProcesses: async () => [ + { + id: 'pty-moved', + incarnationId: 'inc-moved', + terminalHandle: 'term_moved', + title: 'Moved', + cwd: '/tmp/worktree-b', + worktreeId: secondWorktreeId, + wslDistro: null + }, + { + id: 'pty-second', + incarnationId: 'inc-second', + terminalHandle: 'term_second', + title: 'Second', + cwd: '/tmp/worktree-b', + worktreeId: secondWorktreeId, + wslDistro: null + } + ] + }) + const receipts = ( + runtime as unknown as { + restoredOrchestrationAuthorityByPtyId: Map> + } + ).restoredOrchestrationAuthorityByPtyId + receipts.set('pty-moved', { + ptyId: 'pty-moved', + worktreeId: TEST_WORKTREE_ID, + terminalHandle: 'term_moved', + paneKey: makePaneKey('moved-tab', HEADLESS_LEAF_ID), + processIncarnation: 'pty-moved:inc-moved', + hostScope: { kind: 'local', hostId: 'local' } + }) + receipts.set('pty-second', { + ptyId: 'pty-second', + worktreeId: secondWorktreeId, + terminalHandle: 'term_second', + paneKey: makePaneKey('second-tab', HEADLESS_LEAF_ID), + processIncarnation: 'pty-second:inc-second', + hostScope: { kind: 'local', hostId: 'local' } + }) + + await runtime.listTerminals(`id:${TEST_WORKTREE_ID}`) + + expect(receipts.has('pty-moved')).toBe(false) + expect(receipts.has('pty-second')).toBe(true) + }) + + it('preserves SSH dispatch authority commitment across transient relay loss', async () => { + const targetId = 'ssh-1' + const ptyId = `ssh:${targetId}@@pty-retained` + const tabId = 'ssh-worker' + const paneKey = makePaneKey(tabId, HEADLESS_LEAF_ID) + const incarnationId = 'dddddddd-dddd-4ddd-8ddd-dddddddddddd' + const sshSession = makeWorkspaceSessionWithHeadlessTerminal({ + activeTabId: tabId, + activeTabIdByWorktree: { [TEST_WORKTREE_ID]: tabId }, + tabsByWorktree: { + [TEST_WORKTREE_ID]: [ + { + id: tabId, + ptyId, + worktreeId: TEST_WORKTREE_ID, + title: 'SSH worker', + customTitle: null, + color: null, + sortOrder: 0, + createdAt: 1 + } + ] + }, + terminalLayoutsByTabId: { + [tabId]: makeHeadlessTerminalLayout({ [HEADLESS_LEAF_ID]: ptyId }) + }, + terminalPtyIncarnationsByPaneKey: { [paneKey]: incarnationId } + }) + const retireAuthority = vi.fn() + const failDispatch = vi.fn() + const runtime = new OrcaRuntimeService( + { + ...store, + getWorkspaceSession: (hostId?: string | null) => + hostId === `ssh:${targetId}` ? sshSession : getDefaultWorkspaceSession() + }, + undefined, + { + attestAgentHookCompatibilityAuthority: ({ + paneKey: candidate, + launchTokenHash, + connectionId + }) => + candidate === paneKey && + launchTokenHash === RESTORED_AUTHORITY_TOKEN_HASH && + connectionId === targetId + ? { paneKey: candidate, source: 'hydrated_commitment' } + : null, + retireAgentHookCompatibilityAuthority: retireAuthority + } + ) + runtime.setOrchestrationDb({ + getActiveDispatchForTerminal: (handle: string) => + handle === 'term_ssh_retained' + ? { id: 'dispatch-ssh', task_id: 'task-ssh', status: 'dispatched' } + : undefined, + failDispatch, + getActiveCoordinatorRun: () => undefined + } as unknown as OrchestrationDb) + runtime.attachWindow(TEST_WINDOW_ID) + runtime.syncWindowGraph(TEST_WINDOW_ID, { + tabs: [ + { + tabId, + worktreeId: TEST_WORKTREE_ID, + title: 'SSH worker', + activeLeafId: HEADLESS_LEAF_ID, + layout: null + } + ], + leaves: [ + { + tabId, + worktreeId: TEST_WORKTREE_ID, + leafId: HEADLESS_LEAF_ID, + paneRuntimeId: 1, + ptyId + } + ] + }) + const listProcesses = vi.fn(async () => [ + { + id: ptyId, + incarnationId, + terminalHandle: 'term_ssh_retained', + title: 'SSH worker', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + } + ]) + runtime.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null, + listProcesses + }) + const refreshInventory = (connectionId: string | undefined) => + ( + runtime as unknown as { + refreshPtyWorktreeRecordsWithControllerInventory: ( + worktrees: [], + targetWorktreeId: string | null, + deadline: number | undefined, + connectionId: string | undefined + ) => Promise + } + ).refreshPtyWorktreeRecordsWithControllerInventory([], null, undefined, connectionId) + const host = runtime.registerOrchestrationCompatibilitySshAttachment( + targetId, + 'connection-incarnation' + ) + const evidence = { + terminalHandle: 'term_ssh_retained', + paneKey, + launchToken: RESTORED_AUTHORITY_TOKEN, + host + } as const + + await expect(refreshInventory(targetId)).resolves.not.toBeNull() + expect(runtime.verifyOrchestrationCompatibilityCaller(evidence)).not.toBeNull() + await expect(refreshInventory(undefined)).resolves.not.toBeNull() + expect(runtime.verifyOrchestrationCompatibilityCaller(evidence)).not.toBeNull() + + runtime.onPtyExit(ptyId, -1, incarnationId) + + expect(runtime.verifyOrchestrationCompatibilityCaller(evidence)).toBeNull() + expect(retireAuthority).not.toHaveBeenCalled() + expect(failDispatch).not.toHaveBeenCalled() + + await expect(refreshInventory(targetId)).resolves.not.toBeNull() + + expect(runtime.verifyOrchestrationCompatibilityCaller(evidence)).not.toBeNull() + expect(listProcesses).toHaveBeenCalledTimes(3) + }) + it('passes cached view colors to background agent spawns for source-owned startup replies', async () => { setTerminalViewAttributes({ foreground: [0xff, 0xff, 0xff], @@ -16352,6 +16892,17 @@ describe('OrcaRuntimeService', () => { ['pty-shell', 'inc-shell', 'term_shell', 'Shell'] ] as const const runtime = new OrcaRuntimeService({ ...runtimeStore, flushOrThrow: vi.fn() } as never) + const listProcesses = vi.fn(async () => + processes.map(([id, incarnationId, terminalHandle, title]) => ({ + id, + incarnationId, + terminalHandle, + title, + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + })) + ) runtime.setPtyController({ write: (ptyId, data) => { writes.push([ptyId, data]) @@ -16360,16 +16911,7 @@ describe('OrcaRuntimeService', () => { resize, kill: () => true, getForegroundProcess: async () => null, - listProcesses: async () => - processes.map(([id, incarnationId, terminalHandle, title]) => ({ - id, - incarnationId, - terminalHandle, - title, - cwd: TEST_WORKTREE_PATH, - worktreeId: TEST_WORKTREE_ID, - wslDistro: null - })) + listProcesses }) const before = await runtime.listTerminals(`id:${TEST_WORKTREE_ID}`) expect(before.terminals.map((terminal) => terminal.tabId)).toEqual( @@ -16457,6 +16999,14 @@ describe('OrcaRuntimeService', () => { tail: ['legacy output'] }) + const inventoryCount = listProcesses.mock.calls.length + const agentPty = ( + runtime as unknown as { + ptysById: Map + } + ).ptysById.get('pty-agent')! + agentPty.tabId = null + agentPty.paneKey = null const secondClient = await runtime.adoptTerminalOrphans({ worktree: `id:${TEST_WORKTREE_ID}`, expectedTopologyRevision: 0, @@ -16469,6 +17019,22 @@ describe('OrcaRuntimeService', () => { })) }) expect(secondClient).toMatchObject({ adopted: false, topologyRevision: 1 }) + expect(agentPty).toMatchObject({ + tabId: 'tab-agent', + paneKey: makePaneKey('tab-agent', HEADLESS_LEAF_ID) + }) + expect(listProcesses).toHaveBeenCalledTimes(inventoryCount + 1) + expect(listProcesses).toHaveBeenLastCalledWith(null) + expect( + (await runtime.listTerminals()).terminals.find((terminal) => terminal.ptyId === 'pty-agent') + ).toMatchObject({ + handle: 'term_agent', + orphaned: false, + tabId: 'tab-agent', + leafId: HEADLESS_LEAF_ID + }) + expect(listProcesses).toHaveBeenCalledTimes(inventoryCount + 2) + expect(listProcesses).toHaveBeenLastCalledWith(undefined) await expect( runtime.adoptTerminalOrphans({ worktree: `id:${TEST_WORKTREE_ID}`, @@ -16486,6 +17052,2053 @@ describe('OrcaRuntimeService', () => { ).rejects.toThrow('terminal_orphan_competing_owner') }) + function publishLegacyWorkerReveal( + runtime: OrcaRuntimeService, + identity: { worktreeId: string; tabId: string; leafId: string; ptyId: string }, + title = 'Recovered legacy worker' + ): { tabId: string; identity: typeof identity } { + runtime.attachWindow(TEST_WINDOW_ID) + runtime.syncWindowGraph(TEST_WINDOW_ID, { + tabs: [ + { + tabId: identity.tabId, + worktreeId: identity.worktreeId, + title, + activeLeafId: identity.leafId, + layout: null + } + ], + leaves: [ + { + tabId: identity.tabId, + worktreeId: identity.worktreeId, + leafId: identity.leafId, + paneRuntimeId: 1, + ptyId: identity.ptyId + } + ] + }) + return { tabId: identity.tabId, identity } + } + + it('fences provider resume and reveals one exact live legacy worker without stealing focus', async () => { + const workerLeafId = HEADLESS_LEAF_ID + const coordinatorLeafId = HEADLESS_SECOND_LEAF_ID + const workerPaneKey = `legacy-worker:${workerLeafId}` + const incarnationId = '22222222-2222-4222-8222-222222222222' + const session: WorkspaceSessionState = { + ...getDefaultWorkspaceSession(), + activeWorktreeId: TEST_WORKTREE_ID, + activeTabId: 'coordinator', + activeTabIdByWorktree: { [TEST_WORKTREE_ID]: 'coordinator' }, + activeGroupIdByWorktree: { [TEST_WORKTREE_ID]: 'coordinator-group' }, + tabsByWorktree: { + [TEST_WORKTREE_ID]: [ + { + id: 'coordinator', + ptyId: 'pty-coordinator', + worktreeId: TEST_WORKTREE_ID, + title: 'Coordinator', + customTitle: null, + color: null, + sortOrder: 0, + createdAt: 1 + } + ] + }, + terminalLayoutsByTabId: { + coordinator: makeHeadlessTerminalLayout({ + [coordinatorLeafId]: 'pty-coordinator' + }) + }, + tabGroups: { + [TEST_WORKTREE_ID]: [ + { + id: 'coordinator-group', + worktreeId: TEST_WORKTREE_ID, + activeTabId: 'coordinator', + tabOrder: ['coordinator'] + } + ] + }, + sleepingAgentSessionsByPaneKey: { + [workerPaneKey]: { + paneKey: workerPaneKey, + tabId: 'legacy-worker', + worktreeId: TEST_WORKTREE_ID, + agent: 'codex', + providerSession: { key: 'session_id', id: 'legacy-codex-session' }, + prompt: 'continue', + state: 'working', + capturedAt: 1, + updatedAt: 1, + origin: 'live' + } + } + } + const { runtimeStore, getSession } = makeRuntimeStoreWithWorkspaceSession(session) + const flushOrThrow = vi.fn() + const runtime = new OrcaRuntimeService({ ...runtimeStore, flushOrThrow } as never, undefined, { + canRecoverPersistentLocalPtys: () => true, + attestAgentHookCompatibilityAuthority: ({ paneKey, launchTokenHash }) => + paneKey === workerPaneKey && launchTokenHash === RESTORED_AUTHORITY_TOKEN_HASH + ? { paneKey, source: 'hydrated_commitment' } + : null + }) + runtime.setOrchestrationDb({ + getActiveDispatchForTerminal: () => undefined, + listLegacyWorkerTerminalRecoveryRows: () => [ + { + dispatch_id: 'dispatch-legacy', + task_id: 'task-legacy', + dispatch_status: 'completed', + contract_version: 0, + assignee_handle: 'term_legacy', + assignee_pane_key: workerPaneKey, + process_incarnation: `pty-legacy:${incarnationId}`, + worker_state: 'ready', + worktree_id: TEST_WORKTREE_ID, + agent_terminal_handle: 'term_legacy' + } + ] + } as unknown as OrchestrationDb) + const write = vi.fn(() => true) + const kill = vi.fn(() => true) + const serializeProviderBuffer = vi.fn().mockResolvedValue({ + data: '', + scrollbackAnsi: 'OpenAI Codex\r\nACK\r\n', + cols: 80, + rows: 24, + seq: 100, + source: 'headless' as const, + alternateScreen: false + }) + runtime.setPtyController({ + write, + kill, + getForegroundProcess: async () => null, + hasRendererSerializer: () => false, + serializeProviderBuffer, + hasPty: (ptyId) => ptyId === 'pty-legacy', + listProcesses: async () => [ + { + id: 'pty-legacy', + incarnationId, + terminalHandle: 'term_legacy', + title: 'Legacy worker', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + } + ] + }) + const revealTerminalSession = vi.fn().mockImplementation(() => + publishLegacyWorkerReveal( + runtime, + { + worktreeId: TEST_WORKTREE_ID, + tabId: 'legacy-worker', + leafId: workerLeafId, + ptyId: 'pty-legacy' + }, + 'Legacy worker' + ) + ) + const resolveLegacyWorkerTerminalRecovery = vi.fn() + runtime.setNotifier({ + revealTerminalSession, + resolveLegacyWorkerTerminalRecovery + } as never) + + runtime.prepareLegacyWorkerTerminalRecovery() + expect( + getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]?.automaticResumeBlockedBy + ).toBe('legacy-orchestration-worker') + + const recovered = await runtime.reconcileLegacyWorkerTerminals({ + materializeRenderer: true + }) + + expect(recovered).toMatchObject({ + adoptedDispatchIds: ['dispatch-legacy'], + exitedDispatchIds: [], + deferredDispatchIds: [] + }) + expect(getSession().activeTabIdByWorktree?.[TEST_WORKTREE_ID]).toBe('coordinator') + expect(getSession().tabGroups?.[TEST_WORKTREE_ID]?.[0]).toMatchObject({ + activeTabId: 'coordinator', + tabOrder: ['coordinator', 'legacy-worker'] + }) + expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined() + expect(revealTerminalSession).toHaveBeenCalledWith(TEST_WORKTREE_ID, { + ptyId: 'pty-legacy', + title: 'Legacy worker', + activate: false, + presentation: 'background', + tabId: 'legacy-worker', + leafId: workerLeafId, + focus: false, + expectedProcessIdentity: { + terminalHandle: 'term_legacy', + incarnationId + } + }) + expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(workerPaneKey, 'adopted') + expect(write).not.toHaveBeenCalled() + expect(kill).not.toHaveBeenCalled() + const [terminal] = (await runtime.listTerminals()).terminals + await expect(runtime.readTerminal(terminal.handle)).resolves.toMatchObject({ + tail: ['OpenAI Codex', 'ACK'] + }) + expect(serializeProviderBuffer).toHaveBeenCalledWith('pty-legacy', { + scrollbackRows: 120 + }) + expect( + runtime.verifyOrchestrationCompatibilityCaller({ + terminalHandle: 'term_legacy', + paneKey: workerPaneKey, + launchToken: RESTORED_AUTHORITY_TOKEN + }) + ).toMatchObject({ + terminalHandle: 'term_legacy', + paneKey: workerPaneKey, + processIncarnation: `pty-legacy:${incarnationId}` + }) + + await runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true }) + expect(revealTerminalSession).toHaveBeenCalledTimes(1) + expect(getSession().terminalTopologyRevisionByRepoId?.[TEST_REPO_ID]).toBe(1) + expect(flushOrThrow).toHaveBeenCalled() + + serializeProviderBuffer.mockClear() + runtime.onPtyExit('pty-legacy', 0, incarnationId) + runtime.onPtySpawned('pty-legacy', 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa', { + awaitsRegistration: false + }) + const replacementHandle = runtime.createPreAllocatedTerminalHandle() + runtime.registerPreAllocatedHandleForPty('pty-legacy', replacementHandle) + await expect(runtime.readTerminal(replacementHandle)).resolves.toMatchObject({ tail: [] }) + expect(serializeProviderBuffer).not.toHaveBeenCalled() + }) + + it('retries renderer reveal before clearing an adopted legacy worker resume fence', async () => { + const workerPaneKey = `legacy-worker:${HEADLESS_LEAF_ID}` + const incarnationId = '44444444-4444-4444-8444-444444444444' + const session: WorkspaceSessionState = { + ...getDefaultWorkspaceSession(), + tabsByWorktree: { [TEST_WORKTREE_ID]: [] }, + sleepingAgentSessionsByPaneKey: { + [workerPaneKey]: { + paneKey: workerPaneKey, + tabId: 'legacy-worker', + worktreeId: TEST_WORKTREE_ID, + agent: 'codex', + providerSession: { key: 'session_id', id: 'legacy-codex-session' }, + prompt: 'continue', + state: 'working', + capturedAt: 1, + updatedAt: 1, + origin: 'live' + } + } + } + const { runtimeStore, getSession } = makeRuntimeStoreWithWorkspaceSession(session) + const runtime = new OrcaRuntimeService( + { ...runtimeStore, flushOrThrow: vi.fn() } as never, + undefined, + { canRecoverPersistentLocalPtys: () => true } + ) + runtime.setOrchestrationDb({ + getActiveDispatchForTerminal: () => undefined, + listLegacyWorkerTerminalRecoveryRows: () => [ + { + dispatch_id: 'dispatch-reveal-retry', + task_id: 'task-reveal-retry', + dispatch_status: 'completed', + contract_version: 0, + assignee_handle: 'term_reveal_retry', + assignee_pane_key: workerPaneKey, + process_incarnation: `pty-reveal-retry:${incarnationId}`, + worker_state: 'ready', + worktree_id: TEST_WORKTREE_ID, + agent_terminal_handle: 'term_reveal_retry' + } + ] + } as unknown as OrchestrationDb) + runtime.setPtyController({ + write: vi.fn(() => true), + kill: vi.fn(() => true), + getForegroundProcess: async () => null, + hasPty: (ptyId) => ptyId === 'pty-reveal-retry', + listProcesses: async () => [ + { + id: 'pty-reveal-retry', + incarnationId, + terminalHandle: 'term_reveal_retry', + title: 'Legacy worker', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + } + ] + }) + const revealTerminalSession = vi + .fn() + .mockRejectedValueOnce(new Error('renderer unavailable')) + .mockImplementationOnce(() => + publishLegacyWorkerReveal(runtime, { + worktreeId: TEST_WORKTREE_ID, + tabId: 'legacy-worker', + leafId: HEADLESS_LEAF_ID, + ptyId: 'pty-reveal-retry' + }) + ) + const resolveLegacyWorkerTerminalRecovery = vi.fn() + runtime.setNotifier({ + revealTerminalSession, + resolveLegacyWorkerTerminalRecovery + } as never) + + await expect( + runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true }) + ).resolves.toMatchObject({ + adoptedDispatchIds: ['dispatch-reveal-retry'], + deferredDispatchIds: [] + }) + expect(revealTerminalSession).toHaveBeenCalledTimes(2) + expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined() + expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(workerPaneKey, 'adopted') + }) + + function makePostRevealWorkerRecoveryHarness( + hasPty: (ptyId: string) => boolean | null, + listProcesses?: () => Promise< + { + id: string + incarnationId: string + terminalHandle: string + title: string + cwd: string + worktreeId: string + wslDistro: null + }[] + > + ): { + runtime: OrcaRuntimeService + getSession: () => WorkspaceSessionState + workerPaneKey: string + ptyId: string + incarnationId: string + terminalHandle: string + kill: ReturnType + revealTerminalSession: ReturnType + resolveLegacyWorkerTerminalRecovery: ReturnType + } { + const workerPaneKey = `legacy-post-reveal:${HEADLESS_LEAF_ID}` + const ptyId = 'pty-post-reveal' + const incarnationId = '45454545-4545-4545-8545-454545454545' + const terminalHandle = 'term_post_reveal' + const session: WorkspaceSessionState = { + ...getDefaultWorkspaceSession(), + tabsByWorktree: { [TEST_WORKTREE_ID]: [] }, + sleepingAgentSessionsByPaneKey: { + [workerPaneKey]: { + paneKey: workerPaneKey, + tabId: 'legacy-post-reveal', + worktreeId: TEST_WORKTREE_ID, + agent: 'codex', + providerSession: { key: 'session_id', id: 'legacy-post-reveal-session' }, + prompt: 'continue', + state: 'working', + capturedAt: 1, + updatedAt: 1, + origin: 'live' + } + } + } + const { runtimeStore, getSession } = makeRuntimeStoreWithWorkspaceSession(session) + const runtime = new OrcaRuntimeService( + { ...runtimeStore, flushOrThrow: vi.fn() } as never, + undefined, + { canRecoverPersistentLocalPtys: () => true } + ) + runtime.setOrchestrationDb({ + getActiveDispatchForTerminal: () => undefined, + listLegacyWorkerTerminalRecoveryRows: () => [ + { + dispatch_id: 'dispatch-post-reveal', + task_id: 'task-post-reveal', + dispatch_status: 'completed', + contract_version: 0, + assignee_handle: 'term_post_reveal', + assignee_pane_key: workerPaneKey, + process_incarnation: `${ptyId}:${incarnationId}`, + worker_state: 'ready', + worktree_id: TEST_WORKTREE_ID, + agent_terminal_handle: 'term_post_reveal' + } + ] + } as unknown as OrchestrationDb) + const kill = vi.fn(() => true) + runtime.setPtyController({ + write: vi.fn(() => true), + kill, + getForegroundProcess: async () => null, + hasPty, + listProcesses: + listProcesses ?? + (async () => [ + { + id: ptyId, + incarnationId, + terminalHandle: 'term_post_reveal', + title: 'Post-reveal worker', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + } + ]) + }) + const revealTerminalSession = vi.fn().mockResolvedValue({ + tabId: 'legacy-post-reveal', + identity: { + worktreeId: TEST_WORKTREE_ID, + tabId: 'legacy-post-reveal', + leafId: HEADLESS_LEAF_ID, + ptyId + } + }) + const resolveLegacyWorkerTerminalRecovery = vi.fn() + runtime.setNotifier({ + revealTerminalSession, + resolveLegacyWorkerTerminalRecovery + } as never) + return { + runtime, + getSession, + workerPaneKey, + ptyId, + incarnationId, + terminalHandle, + kill, + revealTerminalSession, + resolveLegacyWorkerTerminalRecovery + } + } + + it('keeps a revealed worker fenced until its exact renderer graph is published', async () => { + vi.useFakeTimers() + try { + const harness = makePostRevealWorkerRecoveryHarness(() => true) + const identity = { + worktreeId: TEST_WORKTREE_ID, + tabId: 'legacy-post-reveal', + leafId: HEADLESS_LEAF_ID, + ptyId: harness.ptyId + } + harness.revealTerminalSession.mockResolvedValue({ + tabId: identity.tabId, + identity + }) + + await expect( + harness.runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true }) + ).resolves.toMatchObject({ + adoptedDispatchIds: [], + exitedDispatchIds: [], + deferredDispatchIds: ['dispatch-post-reveal'] + }) + expect(harness.revealTerminalSession).toHaveBeenCalledOnce() + expect( + harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey] + ).toBeDefined() + expect(harness.resolveLegacyWorkerTerminalRecovery).not.toHaveBeenCalled() + + harness.runtime.attachWindow(1) + harness.runtime.syncWindowGraph(1, { + tabs: [ + { + tabId: identity.tabId, + worktreeId: identity.worktreeId, + title: 'Post-reveal worker', + activeLeafId: identity.leafId, + layout: null + } + ], + leaves: [ + { + tabId: identity.tabId, + worktreeId: identity.worktreeId, + leafId: identity.leafId, + paneRuntimeId: 1, + ptyId: identity.ptyId + } + ] + }) + await vi.advanceTimersByTimeAsync(2_000) + + expect(harness.revealTerminalSession).toHaveBeenCalledOnce() + expect( + harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey] + ).toBeUndefined() + expect(harness.resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith( + harness.workerPaneKey, + 'adopted' + ) + expect( + (await harness.runtime.listTerminals()).terminals.filter( + (terminal) => terminal.ptyId === identity.ptyId + ) + ).toEqual([ + expect.objectContaining({ + handle: harness.terminalHandle, + incarnationId: harness.incarnationId, + orphaned: false, + worktreeId: identity.worktreeId, + tabId: identity.tabId, + leafId: identity.leafId + }) + ]) + } finally { + vi.useRealTimers() + } + }) + + it('retires the exact worker when it exits after renderer reveal', async () => { + const liveProcess = { + id: 'pty-post-reveal', + incarnationId: '45454545-4545-4545-8545-454545454545', + terminalHandle: 'term_post_reveal', + title: 'Post-reveal worker', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + } as const + const listProcesses = vi + .fn() + .mockResolvedValueOnce([liveProcess]) + .mockResolvedValueOnce([liveProcess]) + .mockResolvedValueOnce([]) + const harness = makePostRevealWorkerRecoveryHarness(() => false, listProcesses) + harness.revealTerminalSession.mockImplementation(() => + publishLegacyWorkerReveal(harness.runtime, { + worktreeId: TEST_WORKTREE_ID, + tabId: 'legacy-post-reveal', + leafId: HEADLESS_LEAF_ID, + ptyId: harness.ptyId + }) + ) + + await expect( + harness.runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true }) + ).resolves.toMatchObject({ + adoptedDispatchIds: [], + exitedDispatchIds: ['dispatch-post-reveal'], + deferredDispatchIds: [] + }) + + expect(listProcesses).toHaveBeenCalledTimes(3) + expect(harness.revealTerminalSession).toHaveBeenCalledOnce() + expect(harness.getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([]) + expect( + harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey] + ).toBeUndefined() + expect( + ( + harness.runtime as unknown as { + ptysById: Map + } + ).ptysById.get(harness.ptyId) + ).toMatchObject({ connected: false, incarnationId: '45454545-4545-4545-8545-454545454545' }) + expect(harness.resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith( + harness.workerPaneKey, + 'exited' + ) + }) + + it('re-reveals a recovered worker after the renderer graph epoch changes', async () => { + vi.useFakeTimers() + try { + const harness = makePostRevealWorkerRecoveryHarness(() => false) + harness.runtime.attachWindow(TEST_WINDOW_ID) + harness.revealTerminalSession + .mockResolvedValueOnce({ + tabId: 'legacy-post-reveal', + identity: { + worktreeId: TEST_WORKTREE_ID, + tabId: 'legacy-post-reveal', + leafId: HEADLESS_LEAF_ID, + ptyId: harness.ptyId + } + }) + .mockImplementationOnce(() => + publishLegacyWorkerReveal(harness.runtime, { + worktreeId: TEST_WORKTREE_ID, + tabId: 'legacy-post-reveal', + leafId: HEADLESS_LEAF_ID, + ptyId: harness.ptyId + }) + ) + + await expect( + harness.runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true }) + ).resolves.toMatchObject({ + adoptedDispatchIds: [], + deferredDispatchIds: ['dispatch-post-reveal'] + }) + harness.runtime.syncWindowGraph(TEST_WINDOW_ID, { tabs: [], leaves: [] }) + harness.runtime.markRendererReloading(TEST_WINDOW_ID) + await expect( + harness.runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true }) + ).resolves.toMatchObject({ + adoptedDispatchIds: ['dispatch-post-reveal'], + deferredDispatchIds: [] + }) + + expect(harness.revealTerminalSession).toHaveBeenCalledTimes(2) + expect( + harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey] + ).toBeUndefined() + expect(harness.resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith( + harness.workerPaneKey, + 'adopted' + ) + } finally { + vi.useRealTimers() + } + }) + + it('keeps recovery fenced when the renderer omits the exact reveal identity', async () => { + const harness = makePostRevealWorkerRecoveryHarness(() => false) + harness.revealTerminalSession.mockResolvedValue({ tabId: 'legacy-post-reveal' }) + + await expect( + harness.runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true }) + ).resolves.toMatchObject({ + adoptedDispatchIds: [], + exitedDispatchIds: [], + deferredDispatchIds: ['dispatch-post-reveal'] + }) + + expect(harness.revealTerminalSession).toHaveBeenCalledTimes(2) + expect( + harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey] + ).toBeDefined() + expect(harness.resolveLegacyWorkerTerminalRecovery).not.toHaveBeenCalled() + }) + + it('reconciles a same-id process replacement before headless adoption', async () => { + const exactProcess = { + id: 'pty-post-reveal', + incarnationId: '45454545-4545-4545-8545-454545454545', + terminalHandle: 'term_post_reveal', + title: 'Post-reveal worker', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + } as const + const replacement = { + ...exactProcess, + incarnationId: '56565656-5656-4656-8656-565656565656', + terminalHandle: 'term_replacement' + } + const listProcesses = vi + .fn() + .mockResolvedValueOnce([exactProcess]) + .mockResolvedValueOnce([replacement]) + const harness = makePostRevealWorkerRecoveryHarness(() => false, listProcesses) + + await expect(harness.runtime.reconcileLegacyWorkerTerminals()).resolves.toMatchObject({ + adoptedDispatchIds: [], + exitedDispatchIds: ['dispatch-post-reveal'], + deferredDispatchIds: [] + }) + + expect(harness.getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([]) + expect( + harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey] + ).toBeUndefined() + expect(harness.resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith( + harness.workerPaneKey, + 'rolled_back', + harness.ptyId + ) + expect(harness.resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith( + harness.workerPaneKey, + 'exited' + ) + }) + + it('reconciles a same-id process replacement after renderer materialization', async () => { + const exactProcess = { + id: 'pty-post-reveal', + incarnationId: '45454545-4545-4545-8545-454545454545', + terminalHandle: 'term_post_reveal', + title: 'Post-reveal worker', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + } as const + const replacement = { + ...exactProcess, + incarnationId: '67676767-6767-4767-8767-676767676767', + terminalHandle: 'term_replacement' + } + const listProcesses = vi + .fn() + .mockResolvedValueOnce([exactProcess]) + .mockResolvedValueOnce([exactProcess]) + .mockResolvedValueOnce([replacement]) + const harness = makePostRevealWorkerRecoveryHarness(() => false, listProcesses) + harness.revealTerminalSession.mockImplementation(() => + publishLegacyWorkerReveal(harness.runtime, { + worktreeId: TEST_WORKTREE_ID, + tabId: 'legacy-post-reveal', + leafId: HEADLESS_LEAF_ID, + ptyId: harness.ptyId + }) + ) + + await expect( + harness.runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true }) + ).resolves.toMatchObject({ + adoptedDispatchIds: [], + exitedDispatchIds: ['dispatch-post-reveal'], + deferredDispatchIds: [] + }) + + expect( + harness.getSession().sleepingAgentSessionsByPaneKey?.[harness.workerPaneKey] + ).toBeUndefined() + expect(harness.getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([]) + expect(harness.getSession().terminalLayoutsByTabId['legacy-post-reveal']).toBeUndefined() + const runtimeState = harness.runtime as unknown as { + tabs: Map + leaves: Map + ptysById: Map + } + expect(runtimeState.tabs.has('legacy-post-reveal')).toBe(false) + expect([...runtimeState.leaves.keys()].some((key) => key.includes('legacy-post-reveal'))).toBe( + false + ) + expect(runtimeState.ptysById.get(harness.ptyId)).toMatchObject({ + connected: true, + incarnationId: replacement.incarnationId + }) + expect(harness.kill).not.toHaveBeenCalled() + expect(harness.resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith( + harness.workerPaneKey, + 'rolled_back', + harness.ptyId + ) + expect(harness.resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith( + harness.workerPaneKey, + 'exited' + ) + }) + + it('keeps the legacy worker resume fence in memory when persistence fails', async () => { + const workerPaneKey = `legacy-worker:${HEADLESS_LEAF_ID}` + const incarnationId = '99999999-9999-4999-8999-999999999999' + const session: WorkspaceSessionState = { + ...getDefaultWorkspaceSession(), + tabsByWorktree: { [TEST_WORKTREE_ID]: [] }, + sleepingAgentSessionsByPaneKey: { + [workerPaneKey]: { + paneKey: workerPaneKey, + tabId: 'legacy-worker', + worktreeId: TEST_WORKTREE_ID, + agent: 'codex', + providerSession: { key: 'session_id', id: 'legacy-codex-session' }, + prompt: 'continue', + state: 'working', + capturedAt: 1, + updatedAt: 1, + origin: 'live' + } + } + } + const { runtimeStore, getSession } = makeRuntimeStoreWithWorkspaceSession(session) + let flushCount = 0 + const flushOrThrow = vi.fn(() => { + flushCount += 1 + if (flushCount === 1 || flushCount === 3) { + throw new Error('disk unavailable') + } + }) + const runtime = new OrcaRuntimeService({ ...runtimeStore, flushOrThrow } as never, undefined, { + canRecoverPersistentLocalPtys: () => true + }) + runtime.setOrchestrationDb({ + listLegacyWorkerTerminalRecoveryRows: () => [ + { + dispatch_id: 'dispatch-persistence-failure', + task_id: 'task-persistence-failure', + dispatch_status: 'completed', + contract_version: 0, + assignee_handle: 'term_persistence_failure', + assignee_pane_key: workerPaneKey, + process_incarnation: `pty-persistence-failure:${incarnationId}`, + worker_state: 'ready', + worktree_id: TEST_WORKTREE_ID, + agent_terminal_handle: 'term_persistence_failure' + } + ] + } as unknown as OrchestrationDb) + runtime.setPtyController({ + write: vi.fn(() => true), + kill: vi.fn(() => true), + getForegroundProcess: async () => null, + hasPty: (ptyId) => ptyId === 'pty-persistence-failure', + listProcesses: async () => [ + { + id: 'pty-persistence-failure', + incarnationId, + terminalHandle: 'term_persistence_failure', + title: 'Legacy worker', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + } + ] + }) + const revealTerminalSession = vi.fn().mockImplementation(() => + publishLegacyWorkerReveal(runtime, { + worktreeId: TEST_WORKTREE_ID, + tabId: 'legacy-worker', + leafId: HEADLESS_LEAF_ID, + ptyId: 'pty-persistence-failure' + }) + ) + const resolveLegacyWorkerTerminalRecovery = vi.fn() + runtime.setNotifier({ + revealTerminalSession, + resolveLegacyWorkerTerminalRecovery + } as never) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + + await expect( + runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true }) + ).resolves.toMatchObject({ + adoptedDispatchIds: [], + exitedDispatchIds: [], + deferredDispatchIds: ['dispatch-persistence-failure'] + }) + expect(flushOrThrow).toHaveBeenCalledTimes(3) + expect(revealTerminalSession).toHaveBeenCalledOnce() + expect( + getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]?.automaticResumeBlockedBy + ).toBe('legacy-orchestration-worker') + expect(resolveLegacyWorkerTerminalRecovery).not.toHaveBeenCalled() + warn.mockRestore() + }) + + it('requeues an active Task before clearing recovery for an authoritatively missing worker', async () => { + const workerPaneKey = `legacy-missing:${HEADLESS_LEAF_ID}` + const incarnationId = '32323232-3232-4232-8232-323232323232' + const session: WorkspaceSessionState = { + ...getDefaultWorkspaceSession(), + tabsByWorktree: { [TEST_WORKTREE_ID]: [] }, + sleepingAgentSessionsByPaneKey: { + [workerPaneKey]: { + paneKey: workerPaneKey, + tabId: 'legacy-missing', + worktreeId: TEST_WORKTREE_ID, + agent: 'codex', + providerSession: { key: 'session_id', id: 'legacy-missing-session' }, + prompt: 'continue', + state: 'working', + capturedAt: 1, + updatedAt: 1, + origin: 'live' + } + } + } + const { runtimeStore, getSession } = makeRuntimeStoreWithWorkspaceSession(session) + const runtime = new OrcaRuntimeService( + { ...runtimeStore, flushOrThrow: vi.fn() } as never, + undefined, + { canRecoverPersistentLocalPtys: () => true } + ) + const db = new OrchestrationDb(':memory:') + try { + const task = db.createTask({ spec: 'continue after missing worker recovery' }) + const started = db.createStartingWorkerDispatch({ + taskId: task.id, + startOptions: { topology: 'current', agent: 'codex' } + }) + db.prepareStartingWorkerAuthority({ + dispatchId: started.dispatch.id, + handle: 'term_missing_worker', + paneKey: workerPaneKey, + processIncarnation: `pty-missing-worker:${incarnationId}`, + worktreeId: TEST_WORKTREE_ID, + setupState: 'not_applicable', + effects: [] + }) + db.markWorkerDispatchReady(started.dispatch.id) + runtime.setOrchestrationDb(db) + runtime.setPtyController({ + write: vi.fn(() => true), + kill: vi.fn(() => true), + getForegroundProcess: async () => null, + hasPty: () => false, + listProcesses: async () => [] + }) + const resolveLegacyWorkerTerminalRecovery = vi.fn() + runtime.setNotifier({ resolveLegacyWorkerTerminalRecovery } as never) + + await expect(runtime.reconcileLegacyWorkerTerminals()).resolves.toMatchObject({ + adoptedDispatchIds: [], + exitedDispatchIds: [started.dispatch.id], + deferredDispatchIds: [] + }) + + expect(db.getDispatchContextById(started.dispatch.id)).toMatchObject({ + status: 'failed', + failure_count: 1 + }) + expect(db.getWorkerDispatch(started.dispatch.id)?.state).toBe('abandoned') + expect(db.getTask(task.id)?.status).toBe('ready') + expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined() + expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith( + workerPaneKey, + 'rolled_back', + 'pty-missing-worker' + ) + expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(workerPaneKey, 'exited') + } finally { + db.close() + } + }) + + it('retries missing-worker recovery when clearing its persisted resume fence fails', async () => { + const workerPaneKey = `legacy-missing-retry:${HEADLESS_LEAF_ID}` + const incarnationId = '34343434-3434-4434-8434-343434343434' + const session: WorkspaceSessionState = { + ...getDefaultWorkspaceSession(), + tabsByWorktree: { [TEST_WORKTREE_ID]: [] }, + sleepingAgentSessionsByPaneKey: { + [workerPaneKey]: { + paneKey: workerPaneKey, + tabId: 'legacy-missing-retry', + worktreeId: TEST_WORKTREE_ID, + agent: 'codex', + providerSession: { key: 'session_id', id: 'legacy-missing-retry-session' }, + prompt: 'continue', + state: 'working', + capturedAt: 1, + updatedAt: 1, + origin: 'live' + } + } + } + const { runtimeStore, getSession } = makeRuntimeStoreWithWorkspaceSession(session) + const flushOrThrow = vi + .fn() + .mockImplementationOnce(() => undefined) + .mockImplementationOnce(() => { + throw new Error('disk unavailable') + }) + .mockImplementation(() => undefined) + const runtime = new OrcaRuntimeService({ ...runtimeStore, flushOrThrow } as never, undefined, { + canRecoverPersistentLocalPtys: () => true + }) + const db = new OrchestrationDb(':memory:') + try { + const task = db.createTask({ spec: 'retry missing worker recovery' }) + const started = db.createStartingWorkerDispatch({ + taskId: task.id, + startOptions: { topology: 'current', agent: 'codex' } + }) + db.prepareStartingWorkerAuthority({ + dispatchId: started.dispatch.id, + handle: 'term_missing_retry', + paneKey: workerPaneKey, + processIncarnation: `pty-missing-retry:${incarnationId}`, + worktreeId: TEST_WORKTREE_ID, + setupState: 'not_applicable', + effects: [] + }) + db.markWorkerDispatchReady(started.dispatch.id) + runtime.setOrchestrationDb(db) + runtime.setPtyController({ + write: vi.fn(() => true), + kill: vi.fn(() => true), + getForegroundProcess: async () => null, + hasPty: () => false, + listProcesses: async () => [] + }) + const resolveLegacyWorkerTerminalRecovery = vi.fn() + runtime.setNotifier({ resolveLegacyWorkerTerminalRecovery } as never) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + + await expect(runtime.reconcileLegacyWorkerTerminals()).resolves.toMatchObject({ + exitedDispatchIds: [], + deferredDispatchIds: [started.dispatch.id] + }) + expect(db.getDispatchContextById(started.dispatch.id)?.status).toBe('dispatched') + expect(db.getWorkerDispatch(started.dispatch.id)?.state).toBe('ready') + expect( + getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]?.automaticResumeBlockedBy + ).toBe('legacy-orchestration-worker') + + await expect(runtime.reconcileLegacyWorkerTerminals()).resolves.toMatchObject({ + exitedDispatchIds: [started.dispatch.id], + deferredDispatchIds: [] + }) + expect(db.getWorkerDispatch(started.dispatch.id)?.state).toBe('abandoned') + expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined() + expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith( + workerPaneKey, + 'rolled_back', + 'pty-missing-retry' + ) + expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(workerPaneKey, 'exited') + warn.mockRestore() + } finally { + db.close() + } + }) + + it('retires provider resume only after authoritative inventory proves the legacy PTY exited', async () => { + const workerPaneKey = `legacy-worker:${HEADLESS_LEAF_ID}` + const secondWorkerPaneKey = `legacy-worker-two:${HEADLESS_SECOND_LEAF_ID}` + const incarnationId = '33333333-3333-4333-8333-333333333333' + const session: WorkspaceSessionState = { + ...getDefaultWorkspaceSession(), + tabsByWorktree: { [TEST_WORKTREE_ID]: [] }, + sleepingAgentSessionsByPaneKey: { + [workerPaneKey]: { + paneKey: workerPaneKey, + tabId: 'legacy-worker', + worktreeId: TEST_WORKTREE_ID, + agent: 'codex', + providerSession: { key: 'session_id', id: 'legacy-codex-session' }, + prompt: 'continue', + state: 'working', + capturedAt: 1, + updatedAt: 1, + origin: 'live' + }, + [secondWorkerPaneKey]: { + paneKey: secondWorkerPaneKey, + tabId: 'legacy-worker-two', + worktreeId: TEST_WORKTREE_ID, + agent: 'codex', + providerSession: { key: 'session_id', id: 'legacy-codex-session-two' }, + prompt: 'continue', + state: 'working', + capturedAt: 1, + updatedAt: 1, + origin: 'live' + } + } + } + const { runtimeStore, getSession } = makeRuntimeStoreWithWorkspaceSession(session) + const runtime = new OrcaRuntimeService( + { ...runtimeStore, flushOrThrow: vi.fn() } as never, + undefined, + { canRecoverPersistentLocalPtys: () => true } + ) + runtime.setOrchestrationDb({ + listLegacyWorkerTerminalRecoveryRows: () => [ + { + dispatch_id: 'dispatch-exited', + task_id: 'task-exited', + dispatch_status: 'completed', + contract_version: 0, + assignee_handle: 'term_exited', + assignee_pane_key: workerPaneKey, + process_incarnation: `pty-exited:${incarnationId}`, + worker_state: 'ready', + worktree_id: TEST_WORKTREE_ID, + agent_terminal_handle: 'term_exited' + }, + { + dispatch_id: 'dispatch-exited-two', + task_id: 'task-exited-two', + dispatch_status: 'completed', + contract_version: 0, + assignee_handle: 'term_exited_two', + assignee_pane_key: secondWorkerPaneKey, + process_incarnation: `pty-exited-two:${incarnationId}`, + worker_state: 'ready', + worktree_id: TEST_WORKTREE_ID, + agent_terminal_handle: 'term_exited_two' + } + ] + } as unknown as OrchestrationDb) + const listProcesses = vi.fn(async (connectionId?: string | null) => { + if (connectionId !== null) { + throw new Error('unrelated SSH inventory must not run') + } + return [ + { + id: 'pty-exited-two', + incarnationId, + terminalHandle: 'term_exited_two', + title: 'Exited worker', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + } + ] + }) + runtime.setPtyController({ + write: vi.fn(() => true), + kill: vi.fn(() => true), + getForegroundProcess: async () => null, + hasPty: () => false, + listProcesses + }) + const revealTerminalSession = vi.fn() + const resolveLegacyWorkerTerminalRecovery = vi.fn() + runtime.setNotifier({ + revealTerminalSession, + resolveLegacyWorkerTerminalRecovery + } as never) + + runtime.prepareLegacyWorkerTerminalRecovery() + expect( + getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]?.automaticResumeBlockedBy + ).toBe('legacy-orchestration-worker') + + await expect(runtime.reconcileLegacyWorkerTerminals()).resolves.toMatchObject({ + adoptedDispatchIds: ['dispatch-exited-two'], + exitedDispatchIds: ['dispatch-exited'], + deferredDispatchIds: [] + }) + expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined() + expect(getSession().sleepingAgentSessionsByPaneKey?.[secondWorkerPaneKey]).toBeUndefined() + expect(listProcesses).toHaveBeenCalledTimes(3) + expect(listProcesses).toHaveBeenCalledWith(null) + expect(getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([ + expect.objectContaining({ id: 'legacy-worker-two', ptyId: 'pty-exited-two' }) + ]) + expect(revealTerminalSession).not.toHaveBeenCalled() + expect( + ( + runtime as unknown as { + ptysById: Map + } + ).ptysById.get('pty-exited-two') + ).toMatchObject({ connected: true }) + expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(workerPaneKey, 'exited') + expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(secondWorkerPaneKey, 'adopted') + }) + + it('retries inventory and unknown liveness without revealing a ghost worker', async () => { + const workerPaneKey = `legacy-worker:${HEADLESS_LEAF_ID}` + const incarnationId = '55555555-5555-4555-8555-555555555555' + const session: WorkspaceSessionState = { + ...getDefaultWorkspaceSession(), + tabsByWorktree: { [TEST_WORKTREE_ID]: [] }, + sleepingAgentSessionsByPaneKey: { + [workerPaneKey]: { + paneKey: workerPaneKey, + tabId: 'legacy-worker', + worktreeId: TEST_WORKTREE_ID, + agent: 'codex', + providerSession: { key: 'session_id', id: 'legacy-codex-session' }, + prompt: 'continue', + state: 'working', + capturedAt: 1, + updatedAt: 1, + origin: 'live' + } + } + } + const { runtimeStore, getSession } = makeRuntimeStoreWithWorkspaceSession(session) + const runtime = new OrcaRuntimeService( + { ...runtimeStore, flushOrThrow: vi.fn() } as never, + undefined, + { canRecoverPersistentLocalPtys: () => true } + ) + runtime.setOrchestrationDb({ + listLegacyWorkerTerminalRecoveryRows: () => [ + { + dispatch_id: 'dispatch-inventory-unavailable', + task_id: 'task-inventory-unavailable', + dispatch_status: 'completed', + contract_version: 0, + assignee_handle: 'term_inventory_unavailable', + assignee_pane_key: workerPaneKey, + process_incarnation: `pty-inventory-unavailable:${incarnationId}`, + worker_state: 'ready', + worktree_id: TEST_WORKTREE_ID, + agent_terminal_handle: 'term_inventory_unavailable' + } + ] + } as unknown as OrchestrationDb) + const listProcesses = vi + .fn() + .mockRejectedValueOnce(new Error('local provider unavailable')) + .mockResolvedValue([ + { + id: 'pty-inventory-unavailable', + incarnationId, + terminalHandle: 'term_inventory_unavailable', + title: 'Recovered legacy worker', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + } + ]) + const hasPty = vi.fn().mockReturnValueOnce(null).mockReturnValue(true) + runtime.setPtyController({ + write: vi.fn(() => true), + kill: vi.fn(() => true), + getForegroundProcess: async () => null, + hasPty, + listProcesses + }) + const revealTerminalSession = vi.fn().mockImplementation(() => + publishLegacyWorkerReveal(runtime, { + worktreeId: TEST_WORKTREE_ID, + tabId: 'legacy-worker', + leafId: HEADLESS_LEAF_ID, + ptyId: 'pty-inventory-unavailable' + }) + ) + const resolveLegacyWorkerTerminalRecovery = vi.fn() + runtime.setNotifier({ + revealTerminalSession, + resolveLegacyWorkerTerminalRecovery + } as never) + + vi.useFakeTimers() + try { + await expect( + runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true }) + ).resolves.toMatchObject({ + adoptedDispatchIds: [], + exitedDispatchIds: [], + deferredDispatchIds: ['dispatch-inventory-unavailable'] + }) + expect( + getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]?.automaticResumeBlockedBy + ).toBe('legacy-orchestration-worker') + expect(resolveLegacyWorkerTerminalRecovery).not.toHaveBeenCalled() + expect(listProcesses).toHaveBeenCalledOnce() + expect(getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([]) + expect(revealTerminalSession).not.toHaveBeenCalled() + + await vi.advanceTimersByTimeAsync(1_000) + + expect(listProcesses).toHaveBeenCalledTimes(4) + expect(listProcesses.mock.calls).toEqual([[null], [null], [null], [null]]) + expect(hasPty).not.toHaveBeenCalled() + expect(getSession().tabsByWorktree[TEST_WORKTREE_ID]).toEqual([ + expect.objectContaining({ id: 'legacy-worker', ptyId: 'pty-inventory-unavailable' }) + ]) + expect(revealTerminalSession).toHaveBeenCalledOnce() + expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined() + expect(resolveLegacyWorkerTerminalRecovery).toHaveBeenCalledWith(workerPaneKey, 'adopted') + await vi.advanceTimersByTimeAsync(30_000) + expect(listProcesses).toHaveBeenCalledTimes(4) + expect(revealTerminalSession).toHaveBeenCalledOnce() + } finally { + vi.useRealTimers() + } + }) + + it('cancels a coalesced SSH worker recovery retry when its provider disconnects', async () => { + vi.useFakeTimers() + try { + const runtime = new OrcaRuntimeService(store) + const reconcile = vi.spyOn(runtime, 'reconcileLegacyWorkerTerminals').mockResolvedValue({ + blockedPaneCount: 1, + adoptedDispatchIds: [], + exitedDispatchIds: [], + deferredDispatchIds: [] + }) + const retryInternals = runtime as unknown as { + updateLegacyWorkerTerminalRecoveryRetry: ( + plan: { + candidates: { dispatchId: string; ptyId: string }[] + }, + deferredDispatchIds: ReadonlySet, + options: { connectionId?: string; materializeRenderer?: boolean } + ) => void + } + const plan = { + candidates: [ + { + dispatchId: 'dispatch-ssh-retry', + ptyId: 'ssh:ssh-retry@@pty-worker' + } + ] + } + const deferred = new Set(['dispatch-ssh-retry']) + + retryInternals.updateLegacyWorkerTerminalRecoveryRetry(plan, deferred, { + connectionId: 'ssh-retry', + materializeRenderer: true + }) + retryInternals.updateLegacyWorkerTerminalRecoveryRetry(plan, deferred, { + connectionId: 'ssh-retry', + materializeRenderer: true + }) + runtime.notifySshStateChanged('ssh-retry', { + targetId: 'ssh-retry', + status: 'disconnected', + error: null, + reconnectAttempt: 0 + }) + await vi.advanceTimersByTimeAsync(30_000) + + expect(reconcile).not.toHaveBeenCalled() + } finally { + vi.useRealTimers() + } + }) + + it('keeps live workers fenced without exact controller identity evidence', async () => { + const incarnationId = '56565656-5656-4656-8656-565656565656' + const cases = [ + { + name: 'missing', + leafId: '11111111-1111-4111-8111-111111111111', + terminalHandle: 'term_missing' + }, + { + name: 'ambiguous', + leafId: '22222222-2222-4222-8222-222222222222', + terminalHandle: 'term_ambiguous' + }, + { + name: 'wrong-handle', + leafId: '33333333-3333-4333-8333-333333333333', + terminalHandle: 'term_wrong_handle' + }, + { + name: 'wrong-incarnation', + leafId: '44444444-4444-4444-8444-444444444444', + terminalHandle: 'term_wrong_incarnation' + } + ] as const + const sleepingAgentSessionsByPaneKey = Object.fromEntries( + cases.map(({ name, leafId }) => { + const paneKey = `legacy-${name}:${leafId}` + return [ + paneKey, + { + paneKey, + tabId: `legacy-${name}`, + worktreeId: TEST_WORKTREE_ID, + agent: 'codex', + providerSession: { key: 'session_id', id: `session-${name}` }, + prompt: 'continue', + state: 'working', + capturedAt: 1, + updatedAt: 1, + origin: 'live' + } + ] + }) + ) as WorkspaceSessionState['sleepingAgentSessionsByPaneKey'] + const session: WorkspaceSessionState = { + ...getDefaultWorkspaceSession(), + tabsByWorktree: { [TEST_WORKTREE_ID]: [] }, + sleepingAgentSessionsByPaneKey + } + const { runtimeStore, getSession } = makeRuntimeStoreWithWorkspaceSession(session) + const runtime = new OrcaRuntimeService( + { ...runtimeStore, flushOrThrow: vi.fn() } as never, + undefined, + { canRecoverPersistentLocalPtys: () => true } + ) + runtime.setOrchestrationDb({ + listLegacyWorkerTerminalRecoveryRows: () => + cases.map(({ name, leafId, terminalHandle }) => ({ + dispatch_id: `dispatch-${name}`, + task_id: `task-${name}`, + dispatch_status: 'completed', + contract_version: 0, + assignee_handle: terminalHandle, + assignee_pane_key: `legacy-${name}:${leafId}`, + process_incarnation: `pty-${name}:${incarnationId}`, + worker_state: 'ready', + worktree_id: TEST_WORKTREE_ID, + agent_terminal_handle: terminalHandle + })) + } as unknown as OrchestrationDb) + const listProcesses = vi.fn(async () => [ + { + id: 'pty-missing', + title: 'Missing identity', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID + }, + { + id: 'pty-ambiguous', + incarnationId, + terminalHandle: 'term_ambiguous', + title: 'Ambiguous identity', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID + }, + { + id: 'pty-ambiguous-other', + incarnationId, + terminalHandle: 'term_ambiguous', + title: 'Ambiguous identity duplicate', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID + }, + { + id: 'pty-wrong-handle', + incarnationId, + terminalHandle: 'term_other', + title: 'Wrong handle', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID + }, + { + id: 'pty-wrong-incarnation', + incarnationId: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa', + terminalHandle: 'term_wrong_incarnation', + title: 'Wrong incarnation', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID + } + ]) + runtime.setPtyController({ + write: vi.fn(() => true), + kill: vi.fn(() => true), + getForegroundProcess: async () => null, + hasPty: (candidate) => candidate === 'pty-folder-legacy', + listProcesses + }) + + await expect(runtime.reconcileLegacyWorkerTerminals()).resolves.toMatchObject({ + adoptedDispatchIds: [], + exitedDispatchIds: ['dispatch-wrong-handle', 'dispatch-wrong-incarnation'], + deferredDispatchIds: ['dispatch-missing', 'dispatch-ambiguous'] + }) + expect(listProcesses).toHaveBeenCalledOnce() + expect(listProcesses).toHaveBeenCalledWith(null) + for (const { name, leafId } of cases.slice(0, 2)) { + expect( + getSession().sleepingAgentSessionsByPaneKey?.[`legacy-${name}:${leafId}`] + ?.automaticResumeBlockedBy + ).toBe('legacy-orchestration-worker') + } + for (const { name, leafId } of cases.slice(2)) { + expect( + getSession().sleepingAgentSessionsByPaneKey?.[`legacy-${name}:${leafId}`] + ).toBeUndefined() + } + }) + + it('adopts an exact live legacy worker in a folder workspace', async () => { + const workerPaneKey = `legacy-folder-worker:${HEADLESS_LEAF_ID}` + const incarnationId = '66666666-6666-4666-8666-666666666666' + const folderPath = await mkdtemp(join(tmpdir(), 'orca-legacy-worker-folder-')) + const session: WorkspaceSessionState = { + ...getDefaultWorkspaceSession(), + activeWorktreeId: TEST_FOLDER_WORKSPACE_KEY, + tabsByWorktree: { [TEST_FOLDER_WORKSPACE_KEY]: [] }, + sleepingAgentSessionsByPaneKey: { + [workerPaneKey]: { + paneKey: workerPaneKey, + tabId: 'legacy-folder-worker', + worktreeId: TEST_FOLDER_WORKSPACE_KEY, + agent: 'codex', + providerSession: { key: 'session_id', id: 'legacy-folder-codex-session' }, + prompt: 'continue', + state: 'working', + capturedAt: 1, + updatedAt: 1, + origin: 'live' + } + } + } + const { runtimeStore, getSession } = makeRuntimeStoreWithWorkspaceSession(session) + const folderWorkspace = makeFolderWorkspace({ folderPath }) + const projectGroup = makeFolderProjectGroup({ parentPath: folderPath }) + const runtime = new OrcaRuntimeService( + { + ...runtimeStore, + getFolderWorkspaces: () => [folderWorkspace], + getProjectGroups: () => [projectGroup], + flushOrThrow: vi.fn() + } as never, + undefined, + { canRecoverPersistentLocalPtys: () => true } + ) + runtime.setOrchestrationDb({ + listLegacyWorkerTerminalRecoveryRows: () => [ + { + dispatch_id: 'dispatch-folder', + task_id: 'task-folder', + dispatch_status: 'completed', + contract_version: 0, + assignee_handle: 'term_folder', + assignee_pane_key: workerPaneKey, + process_incarnation: `pty-folder-legacy:${incarnationId}`, + worker_state: 'ready', + worktree_id: TEST_FOLDER_WORKSPACE_KEY, + agent_terminal_handle: 'term_folder' + } + ] + } as unknown as OrchestrationDb) + const listProcesses = vi.fn(async () => [ + { + id: 'pty-folder-legacy', + incarnationId, + terminalHandle: 'term_folder', + title: 'Folder worker', + cwd: folderPath, + worktreeId: TEST_FOLDER_WORKSPACE_KEY, + wslDistro: null + } + ]) + runtime.setPtyController({ + write: vi.fn(() => true), + kill: vi.fn(() => true), + getForegroundProcess: async () => null, + hasPty: (candidate) => candidate === 'pty-folder-legacy', + listProcesses + }) + const revealTerminalSession = vi.fn().mockImplementation(() => + publishLegacyWorkerReveal(runtime, { + worktreeId: TEST_FOLDER_WORKSPACE_KEY, + tabId: 'legacy-folder-worker', + leafId: HEADLESS_LEAF_ID, + ptyId: 'pty-folder-legacy' + }) + ) + runtime.setNotifier({ revealTerminalSession } as never) + + await expect( + runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true }) + ).resolves.toMatchObject({ + adoptedDispatchIds: ['dispatch-folder'], + exitedDispatchIds: [], + deferredDispatchIds: [] + }) + expect(getSession().tabsByWorktree[TEST_FOLDER_WORKSPACE_KEY]).toContainEqual( + expect.objectContaining({ + id: 'legacy-folder-worker', + ptyId: 'pty-folder-legacy', + worktreeId: TEST_FOLDER_WORKSPACE_KEY + }) + ) + expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined() + expect(listProcesses).toHaveBeenCalledTimes(3) + expect(listProcesses).toHaveBeenCalledWith(null) + expect(revealTerminalSession).toHaveBeenCalledWith(TEST_FOLDER_WORKSPACE_KEY, { + ptyId: 'pty-folder-legacy', + title: 'Folder worker', + activate: false, + presentation: 'background', + tabId: 'legacy-folder-worker', + leafId: HEADLESS_LEAF_ID, + focus: false, + expectedProcessIdentity: { + terminalHandle: 'term_folder', + incarnationId + } + }) + }) + + it('adopts an SSH folder legacy worker through its SSH workspace-session partition', async () => { + const connectionId = 'ssh-folder' + const ptyId = `ssh:${connectionId}@@pty-folder-legacy` + const workerPaneKey = `legacy-ssh-folder-worker:${HEADLESS_LEAF_ID}` + const incarnationId = '67676767-6767-4767-8767-676767676767' + const folderPath = '/srv/platform' + const sshInitialSession: WorkspaceSessionState = { + ...getDefaultWorkspaceSession(), + activeWorktreeId: TEST_FOLDER_WORKSPACE_KEY, + tabsByWorktree: { [TEST_FOLDER_WORKSPACE_KEY]: [] }, + sleepingAgentSessionsByPaneKey: { + [workerPaneKey]: { + paneKey: workerPaneKey, + tabId: 'legacy-ssh-folder-worker', + worktreeId: TEST_FOLDER_WORKSPACE_KEY, + agent: 'codex', + providerSession: { key: 'session_id', id: 'legacy-ssh-folder-codex-session' }, + prompt: 'continue', + state: 'working', + capturedAt: 1, + updatedAt: 1, + origin: 'live', + connectionId + } + } + } + const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession(sshInitialSession) + const localSession = getDefaultWorkspaceSession() + let sshSession = sshInitialSession + const getWorkspaceSession = vi.fn((hostId?: string | null) => + hostId === `ssh:${connectionId}` ? sshSession : localSession + ) + const setWorkspaceSession = vi.fn((next: WorkspaceSessionState, hostId?: string | null) => { + if (hostId !== `ssh:${connectionId}`) { + throw new Error(`unexpected workspace-session host ${hostId ?? 'default'}`) + } + sshSession = next + }) + const folderWorkspace = makeFolderWorkspace({ folderPath, connectionId }) + const projectGroup = makeFolderProjectGroup({ parentPath: folderPath }) + const runtime = new OrcaRuntimeService( + { + ...runtimeStore, + getFolderWorkspaces: () => [folderWorkspace], + getProjectGroups: () => [projectGroup], + getWorkspaceSession, + setWorkspaceSession, + flushOrThrow: vi.fn() + } as never, + undefined, + { canRecoverPersistentLocalPtys: () => true } + ) + runtime.setOrchestrationDb({ + listLegacyWorkerTerminalRecoveryRows: () => [ + { + dispatch_id: 'dispatch-ssh-folder', + task_id: 'task-ssh-folder', + dispatch_status: 'completed', + contract_version: 0, + assignee_handle: 'term_ssh_folder', + assignee_pane_key: workerPaneKey, + process_incarnation: `${ptyId}:${incarnationId}`, + worker_state: 'ready', + worktree_id: TEST_FOLDER_WORKSPACE_KEY, + agent_terminal_handle: 'term_ssh_folder' + } + ] + } as unknown as OrchestrationDb) + const listProcesses = vi.fn(async () => [ + { + id: ptyId, + incarnationId, + terminalHandle: 'term_ssh_folder', + title: 'SSH folder worker', + cwd: folderPath, + worktreeId: TEST_FOLDER_WORKSPACE_KEY, + wslDistro: null + } + ]) + runtime.setPtyController({ + write: vi.fn(() => true), + kill: vi.fn(() => true), + getForegroundProcess: async () => null, + hasPty: (candidate) => candidate === ptyId, + listProcesses + }) + const revealTerminalSession = vi.fn().mockImplementation(() => + publishLegacyWorkerReveal(runtime, { + worktreeId: TEST_FOLDER_WORKSPACE_KEY, + tabId: 'legacy-ssh-folder-worker', + leafId: HEADLESS_LEAF_ID, + ptyId + }) + ) + runtime.setNotifier({ revealTerminalSession } as never) + registerSshFilesystemProvider(connectionId, { + stat: vi.fn(async () => ({ size: 0, type: 'directory', mtime: 1 })) + } as never) + + try { + expect(runtime.prepareLegacyWorkerTerminalRecovery()).toMatchObject({ + blockedPanes: [expect.objectContaining({ paneKey: workerPaneKey })] + }) + expect( + sshSession.sleepingAgentSessionsByPaneKey?.[workerPaneKey]?.automaticResumeBlockedBy + ).toBe('legacy-orchestration-worker') + expect(localSession.sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined() + await expect( + runtime.reconcileLegacyWorkerTerminals({ + connectionId, + materializeRenderer: true + }) + ).resolves.toMatchObject({ + adoptedDispatchIds: ['dispatch-ssh-folder'], + exitedDispatchIds: [], + deferredDispatchIds: [] + }) + } finally { + unregisterSshFilesystemProvider(connectionId) + } + + expect(sshSession.sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined() + expect(localSession.sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined() + expect(getWorkspaceSession).toHaveBeenCalledWith(`ssh:${connectionId}`) + expect(setWorkspaceSession).toHaveBeenCalledWith(expect.any(Object), `ssh:${connectionId}`) + expect(listProcesses).toHaveBeenCalledTimes(3) + expect(listProcesses).toHaveBeenCalledWith(connectionId) + expect(sshSession.tabsByWorktree[TEST_FOLDER_WORKSPACE_KEY]).toContainEqual( + expect.objectContaining({ + id: 'legacy-ssh-folder-worker', + ptyId, + worktreeId: TEST_FOLDER_WORKSPACE_KEY + }) + ) + expect(revealTerminalSession).toHaveBeenCalledWith(TEST_FOLDER_WORKSPACE_KEY, { + ptyId, + title: 'SSH folder worker', + activate: false, + presentation: 'background', + tabId: 'legacy-ssh-folder-worker', + leafId: HEADLESS_LEAF_ID, + focus: false, + expectedProcessIdentity: { + terminalHandle: 'term_ssh_folder', + incarnationId + } + }) + }) + + it('fences an unresolved folder legacy worker in its exact retained session partition', () => { + const connectionId = 'ssh-unresolved-folder' + const worktreeId = 'folder:missing-folder' + const workerPaneKey = `legacy-unresolved-folder-worker:${HEADLESS_LEAF_ID}` + const remoteInitialSession: WorkspaceSessionState = { + ...getDefaultWorkspaceSession(), + tabsByWorktree: { [worktreeId]: [] }, + sleepingAgentSessionsByPaneKey: { + [workerPaneKey]: { + paneKey: workerPaneKey, + tabId: 'legacy-unresolved-folder-worker', + worktreeId, + agent: 'codex', + providerSession: { key: 'session_id', id: 'legacy-unresolved-folder-session' }, + prompt: 'continue', + state: 'working', + capturedAt: 1, + updatedAt: 1, + origin: 'live', + connectionId + } + } + } + const localSession = getDefaultWorkspaceSession() + let remoteSession = remoteInitialSession + const getWorkspaceSession = vi.fn((hostId?: string | null) => + hostId === `ssh:${connectionId}` ? remoteSession : localSession + ) + const setWorkspaceSession = vi.fn((next: WorkspaceSessionState, hostId?: string | null) => { + if (hostId !== `ssh:${connectionId}`) { + throw new Error(`unexpected workspace-session host ${hostId ?? 'default'}`) + } + remoteSession = next + }) + const runtime = new OrcaRuntimeService({ + ...store, + getFolderWorkspaces: () => [], + getWorkspaceSession, + getWorkspaceSessionHostIds: () => ['local', `ssh:${connectionId}`], + setWorkspaceSession, + flushOrThrow: vi.fn() + } as never) + runtime.setOrchestrationDb({ + listLegacyWorkerTerminalRecoveryRows: () => [ + { + dispatch_id: 'dispatch-unresolved-folder', + task_id: 'task-unresolved-folder', + dispatch_status: 'completed', + contract_version: 0, + assignee_handle: 'term_unresolved_folder', + assignee_pane_key: workerPaneKey, + process_incarnation: 'pty-unresolved-folder:68686868-6868-4868-8868-686868686868', + worker_state: 'ready', + worktree_id: worktreeId, + agent_terminal_handle: 'term_unresolved_folder' + } + ] + } as unknown as OrchestrationDb) + + expect(runtime.prepareLegacyWorkerTerminalRecovery()).toMatchObject({ + blockedPanes: [expect.objectContaining({ paneKey: workerPaneKey, worktreeId })] + }) + expect( + remoteSession.sleepingAgentSessionsByPaneKey?.[workerPaneKey]?.automaticResumeBlockedBy + ).toBe('legacy-orchestration-worker') + expect(localSession.sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined() + expect(setWorkspaceSession).toHaveBeenCalledOnce() + expect(setWorkspaceSession).toHaveBeenCalledWith(expect.any(Object), `ssh:${connectionId}`) + }) + + it('adopts an SSH legacy worker only after its matching relay is ready', async () => { + const connectionId = 'ssh-legacy-worker' + const ptyId = `ssh:${connectionId}@@pty-legacy-worker` + const workerPaneKey = `legacy-ssh-worker:${HEADLESS_LEAF_ID}` + const incarnationId = '77777777-7777-4777-8777-777777777777' + const session: WorkspaceSessionState = { + ...getDefaultWorkspaceSession(), + tabsByWorktree: { [TEST_WORKTREE_ID]: [] }, + sleepingAgentSessionsByPaneKey: { + [workerPaneKey]: { + paneKey: workerPaneKey, + tabId: 'legacy-ssh-worker', + worktreeId: TEST_WORKTREE_ID, + agent: 'codex', + providerSession: { key: 'session_id', id: 'legacy-ssh-codex-session' }, + prompt: 'continue', + state: 'working', + capturedAt: 1, + updatedAt: 1, + origin: 'live', + connectionId + } + } + } + const { runtimeStore } = makeRuntimeStoreWithWorkspaceSession(session) + const localSession = getDefaultWorkspaceSession() + let sshSession = session + const getWorkspaceSession = vi.fn((hostId?: string | null) => + hostId === `ssh:${connectionId}` ? sshSession : localSession + ) + const setWorkspaceSession = vi.fn((next: WorkspaceSessionState, hostId?: string | null) => { + if (hostId !== `ssh:${connectionId}`) { + throw new Error(`unexpected workspace-session host ${hostId ?? 'default'}`) + } + sshSession = next + }) + const getSession = (): WorkspaceSessionState => sshSession + const remoteRepo = { + ...store.getRepos()[0], + connectionId + } + const listProcesses = vi.fn(async () => [ + { + id: ptyId, + incarnationId, + terminalHandle: 'term_ssh_legacy', + title: 'SSH legacy worker', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: null + } + ]) + const runtime = new OrcaRuntimeService( + { + ...runtimeStore, + getRepos: () => [remoteRepo], + getRepo: (id: string) => (id === TEST_REPO_ID ? remoteRepo : undefined), + getWorkspaceSession, + setWorkspaceSession, + flushOrThrow: vi.fn() + } as never, + undefined, + { canRecoverPersistentLocalPtys: () => true } + ) + runtime.setOrchestrationDb({ + listLegacyWorkerTerminalRecoveryRows: () => [ + { + dispatch_id: 'dispatch-ssh', + task_id: 'task-ssh', + dispatch_status: 'completed', + contract_version: 0, + assignee_handle: 'term_ssh_legacy', + assignee_pane_key: workerPaneKey, + process_incarnation: `${ptyId}:${incarnationId}`, + worker_state: 'ready', + worktree_id: TEST_WORKTREE_ID, + agent_terminal_handle: 'term_ssh_legacy' + } + ] + } as unknown as OrchestrationDb) + runtime.setPtyController({ + write: vi.fn(() => true), + kill: vi.fn(() => true), + getForegroundProcess: async () => null, + hasPty: (candidate) => candidate === ptyId, + listProcesses + }) + const revealTerminalSession = vi.fn().mockImplementation(() => + publishLegacyWorkerReveal(runtime, { + worktreeId: TEST_WORKTREE_ID, + tabId: 'legacy-ssh-worker', + leafId: HEADLESS_LEAF_ID, + ptyId + }) + ) + runtime.setNotifier({ revealTerminalSession } as never) + registerSshGitProvider(connectionId, { + listWorktrees: vi.fn(async () => [ + { + path: TEST_WORKTREE_PATH, + head: 'abc', + branch: 'main', + isBare: false, + isMainWorktree: false + } + ]) + } as never) + + try { + await expect( + runtime.reconcileLegacyWorkerTerminals({ + connectionId: 'ssh-wrong-host', + materializeRenderer: true + }) + ).resolves.toMatchObject({ + adoptedDispatchIds: [], + exitedDispatchIds: [], + deferredDispatchIds: ['dispatch-ssh'] + }) + expect(listProcesses).not.toHaveBeenCalled() + expect( + getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]?.automaticResumeBlockedBy + ).toBe('legacy-orchestration-worker') + expect(localSession.sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined() + expect(getWorkspaceSession).toHaveBeenCalledWith(`ssh:${connectionId}`) + + await expect( + runtime.reconcileLegacyWorkerTerminals({ + connectionId, + materializeRenderer: true + }) + ).resolves.toMatchObject({ + adoptedDispatchIds: ['dispatch-ssh'], + exitedDispatchIds: [], + deferredDispatchIds: [] + }) + expect(listProcesses).toHaveBeenLastCalledWith(connectionId) + } finally { + unregisterSshGitProvider(connectionId) + } + + expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined() + expect(setWorkspaceSession).toHaveBeenCalledWith(expect.any(Object), `ssh:${connectionId}`) + expect(listProcesses).toHaveBeenCalledTimes(3) + expect(revealTerminalSession).toHaveBeenCalledWith(TEST_WORKTREE_ID, { + ptyId, + title: 'SSH legacy worker', + activate: false, + presentation: 'background', + tabId: 'legacy-ssh-worker', + leafId: HEADLESS_LEAF_ID, + focus: false, + expectedProcessIdentity: { + terminalHandle: 'term_ssh_legacy', + incarnationId + } + }) + }) + + it('refuses a cross-distro WSL worker and adopts it after exact host ownership matches', async () => { + setPlatform('win32') + const workerPaneKey = `legacy-wsl-worker:${HEADLESS_LEAF_ID}` + const incarnationId = '88888888-8888-4888-8888-888888888888' + let observedDistro = 'Debian' + const session: WorkspaceSessionState = { + ...getDefaultWorkspaceSession(), + tabsByWorktree: { [TEST_WORKTREE_ID]: [] }, + sleepingAgentSessionsByPaneKey: { + [workerPaneKey]: { + paneKey: workerPaneKey, + tabId: 'legacy-wsl-worker', + worktreeId: TEST_WORKTREE_ID, + agent: 'codex', + providerSession: { key: 'session_id', id: 'legacy-wsl-codex-session' }, + prompt: 'continue', + state: 'working', + capturedAt: 1, + updatedAt: 1, + origin: 'live' + } + } + } + const { runtimeStore, getSession } = makeRuntimeStoreWithWorkspaceSession(session) + const runtime = new OrcaRuntimeService( + { + ...runtimeStore, + getProjects: () => [ + { + id: 'project-wsl', + displayName: 'repo', + badgeColor: 'blue', + sourceRepoIds: [TEST_REPO_ID], + localWindowsRuntimePreference: { kind: 'wsl', distro: 'Ubuntu' }, + createdAt: 0, + updatedAt: 0 + } + ], + getSettings: () => ({ + ...store.getSettings(), + localWindowsRuntimeDefault: { kind: 'windows-host' } + }), + flushOrThrow: vi.fn() + } as never, + undefined, + { canRecoverPersistentLocalPtys: () => true } + ) + runtime.setOrchestrationDb({ + listLegacyWorkerTerminalRecoveryRows: () => [ + { + dispatch_id: 'dispatch-wsl', + task_id: 'task-wsl', + dispatch_status: 'completed', + contract_version: 0, + assignee_handle: 'term_wsl_legacy', + assignee_pane_key: workerPaneKey, + process_incarnation: `pty-wsl-legacy:${incarnationId}`, + worker_state: 'ready', + worktree_id: TEST_WORKTREE_ID, + agent_terminal_handle: 'term_wsl_legacy' + } + ] + } as unknown as OrchestrationDb) + const listProcesses = vi.fn(async () => [ + { + id: 'pty-wsl-legacy', + incarnationId, + terminalHandle: 'term_wsl_legacy', + title: 'WSL legacy worker', + cwd: TEST_WORKTREE_PATH, + worktreeId: TEST_WORKTREE_ID, + wslDistro: observedDistro + } + ]) + runtime.setPtyController({ + write: vi.fn(() => true), + kill: vi.fn(() => true), + getForegroundProcess: async () => null, + hasPty: (ptyId) => ptyId === 'pty-wsl-legacy', + listProcesses + }) + const revealTerminalSession = vi.fn().mockImplementation(() => + publishLegacyWorkerReveal(runtime, { + worktreeId: TEST_WORKTREE_ID, + tabId: 'legacy-wsl-worker', + leafId: HEADLESS_LEAF_ID, + ptyId: 'pty-wsl-legacy' + }) + ) + runtime.setNotifier({ revealTerminalSession } as never) + + await expect( + runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true }) + ).resolves.toMatchObject({ + adoptedDispatchIds: [], + exitedDispatchIds: [], + deferredDispatchIds: ['dispatch-wsl'] + }) + expect( + getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]?.automaticResumeBlockedBy + ).toBe('legacy-orchestration-worker') + expect(revealTerminalSession).not.toHaveBeenCalled() + + observedDistro = 'Ubuntu' + await expect( + runtime.reconcileLegacyWorkerTerminals({ materializeRenderer: true }) + ).resolves.toMatchObject({ + adoptedDispatchIds: ['dispatch-wsl'], + exitedDispatchIds: [], + deferredDispatchIds: [] + }) + expect(getSession().sleepingAgentSessionsByPaneKey?.[workerPaneKey]).toBeUndefined() + expect(revealTerminalSession).toHaveBeenCalledOnce() + expect(listProcesses).toHaveBeenCalledTimes(5) + expect(listProcesses.mock.calls).toEqual([[null], [null], [null], [null], [null]]) + }) + it('restores orphan pane and group topology without replacing a newer host-owned tab', async () => { const session: WorkspaceSessionState = { ...makeWorkspaceSessionWithHeadlessTerminal({ @@ -23369,6 +25982,9 @@ describe('OrcaRuntimeService', () => { }) const events: RuntimeMobileSessionTabsResult[] = [] runtime.onMobileSessionTabsChanged((snapshot) => events.push(snapshot)) + const reconcile = vi + .spyOn(runtime, 'reconcileLegacyWorkerTerminals') + .mockReturnValue(new Promise(() => undefined)) runtime.notifySshRelayReady('ssh-1') await vi.waitFor(() => @@ -23389,6 +26005,10 @@ describe('OrcaRuntimeService', () => { terminal: expect.any(String) }) ]) + expect(reconcile).toHaveBeenCalledWith({ + connectionId: 'ssh-1', + materializeRenderer: false + }) }) it('uses only a recent expired SSH lease as a bounded pane-recovery tombstone', async () => { diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index 792eb43eb07..c41d88c83a5 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -37,6 +37,7 @@ import { type AgentStatusEntry } from '../../shared/agent-status-types' import { indexAgentStatusRowsByPaneKey } from '../agent-hooks/agent-status-pane-index' +import type { AgentHookAuthorityAttestation } from '../agent-hooks/server' import type { AgentSessionClaimedSpawnResult, AgentSessionExecutionClaim, @@ -98,11 +99,21 @@ import { resolveWorktreeCreateBase } from '../worktree-create-base' import { resolveWorktreeAddBaseRef } from '../../shared/worktree-base-ref' import { OrchestrationDb } from './orchestration/db' import { OrchestrationError } from './orchestration/orchestration-error' +import { + planLegacyWorkerTerminalRecovery, + type LegacyWorkerTerminalRecoveryPlan +} from './orchestration/orchestration-legacy-worker-terminal-recovery' import { buildObservedSetupCommand, createSetupCompletionScanner } from './orchestration/setup-completion-signal' import type { RuntimeOrchestrationEnvelope } from '../../shared/runtime-rpc-envelope' +import { ORCHESTRATION_MESSAGE_WAIT_DEFAULT_TIMEOUT_MS } from '../../shared/orchestration-message-wait-timeout' +import type { TerminalRevealIdentity } from '../../shared/terminal-reveal-identity' +import type { + OrchestrationCompatibilityEvidence, + OrchestrationCompatibilityHostStamp +} from '../../shared/orchestration-compatibility-evidence' import { isOrchestrationMutation, orchestrationMigrationData @@ -1017,6 +1028,7 @@ type RuntimeStore = { removeWorkspaceLineage?: Store['removeWorkspaceLineage'] getGitHubCache: Store['getGitHubCache'] getWorkspaceSession?: Store['getWorkspaceSession'] + getWorkspaceSessionHostIds?: Store['getWorkspaceSessionHostIds'] setWorkspaceSession?: Store['setWorkspaceSession'] flushOrThrow?: Store['flushOrThrow'] persistPtyBinding?: Store['persistPtyBinding'] @@ -1495,7 +1507,7 @@ type RuntimePtyController = { resize?(ptyId: string, cols: number, rows: number): boolean // Why: exact-id mobile polls should not enumerate every local and SSH PTY. hasPty?(ptyId: string): boolean | null - listProcesses?(): Promise + listProcesses?(connectionId?: string | null): Promise serializeBuffer?( ptyId: string, opts?: { scrollbackRows?: number; altScreenForcesZeroRows?: boolean } @@ -1519,6 +1531,17 @@ type RuntimePtyController = { getSize?(ptyId: string): { cols: number; rows: number } | null } +type PtyControllerTerminalIdentity = Readonly<{ + handle: string + incarnationId: string + wslDistro?: string | null +}> + +type PtyControllerInventory = Readonly<{ + livePtyIds: ReadonlySet + terminalIdentityByPtyId: ReadonlyMap +}> + type WorktreeStartupDraftPaste = { agent: TuiAgent content: string @@ -1633,11 +1656,21 @@ type RuntimeNotifier = { splitFromLeafId?: string splitDirection?: 'horizontal' | 'vertical' splitTelemetrySource?: TerminalPaneSplitSource + focus?: boolean + expectedProcessIdentity?: { + terminalHandle: string + incarnationId: string + } } ): - | Promise<{ tabId: string; title?: string | null }> - | { tabId: string; title?: string | null } + | Promise<{ tabId: string; title?: string | null; identity?: TerminalRevealIdentity }> + | { tabId: string; title?: string | null; identity?: TerminalRevealIdentity } | void + resolveLegacyWorkerTerminalRecovery?( + paneKey: string, + resolution: 'adopted' | 'exited' | 'rolled_back', + ptyId?: string + ): void splitTerminal( tabId: string, paneRuntimeId: number, @@ -1707,6 +1740,49 @@ type TerminalHandleRecord = { ptyGeneration: number } +export type OrchestrationCompatibilityTerminalAuthority = { + runtimeId: string + terminalHandle: string + ptyId: string + worktreeId: string + processIncarnation: string | null + paneKey: string | null + launchTokenHash: string | null + hostScope: + | { kind: 'local'; hostId: 'local' } + | { kind: 'wsl'; hostId: 'local'; distro: string } + | { kind: 'ssh'; targetId: string } +} + +export type LegacyWorkerTerminalRecoveryResult = { + blockedPaneCount: number + adoptedDispatchIds: string[] + exitedDispatchIds: string[] + deferredDispatchIds: string[] +} + +export type OrchestrationCompatibilityCallerAuthority = Readonly<{ + hostScope: OrchestrationCompatibilityTerminalAuthority['hostScope'] + paneKey: string + terminalHandle: string + processIncarnation: string + launchTokenHash: string +}> + +type RestoredOrchestrationAuthorityReceipt = Readonly<{ + ptyId: string + worktreeId: string + terminalHandle: string + paneKey: string + processIncarnation: string + hostScope: OrchestrationCompatibilityTerminalAuthority['hostScope'] +}> + +type OrchestrationCompatibilitySshAttachmentAuthority = Extract< + OrchestrationCompatibilityHostStamp, + { kind: 'ssh' } +> + type TerminalWaiter = { handle: string condition: RuntimeTerminalWaitCondition @@ -2564,10 +2640,7 @@ export class OrcaRuntimeService { private handles = new Map() private handleByLeafKey = new Map() private handleByPtyId = new Map() - private controllerTerminalIdentityByPtyId = new Map< - string, - { handle: string; incarnationId: string; wslDistro?: string | null } - >() + private syntheticTerminalHandles = new Set() private detachedPreAllocatedLeaves = new Map() private graphSyncCallbacks: (() => void)[] = [] private waitersByHandle = new Map>() @@ -2960,6 +3033,16 @@ export class OrcaRuntimeService { private readonly getAgentProviderSessionRowsForPaneFn: | ((paneKey: string) => AgentStatusIpcPayload[]) | null + private readonly attestAgentHookCompatibilityAuthorityFn: + | ((candidate: { + paneKey: string + launchTokenHash: string + connectionId: string | null + terminalProvenance: 'current_runtime' | 'restored' + }) => AgentHookAuthorityAttestation | null) + | null + private readonly retireAgentHookCompatibilityAuthorityFn: ((paneKey: string) => void) | null + private readonly canRecoverPersistentLocalPtysFn: () => boolean private readonly buildAgentHookPtyEnv: (() => Record) | null private readonly getDesktopWindowStatusFn: () => RuntimeDesktopWindowStatus private readonly prepareAiVaultSessionResumeFn: @@ -2967,7 +3050,30 @@ export class OrcaRuntimeService { | null private readonly agentSessionClaimSigner: AgentSessionClaimSigner private readonly agentSessionCreateOperations = new Map() + private readonly orchestrationCompatibilitySshAttachments = new Map< + string, + OrchestrationCompatibilitySshAttachmentAuthority + >() private sshRelayRecoveryGenerationByTargetId = new Map() + private legacyWorkerTerminalRecoveryQueue: Promise = Promise.resolve() + private legacyWorkerTerminalRecoveryRetries = new Map< + string, + { + attempt: number + connectionId?: string + materializeRenderer: boolean + timer: ReturnType | null + } + >() + private legacyWorkerTerminalReceiptEpochByPane = new Map() + private legacyWorkerRecoveredPtys = new Set() + private restoredOrchestrationAuthorityByPtyId = new Map< + string, + RestoredOrchestrationAuthorityReceipt + >() + private ptyControllerInventorySequence = 0 + private ptyControllerAggregateInventoryGeneration = 0 + private ptyControllerInventoryGenerationByProvider = new Map() private accountServices: RuntimeAccountServices | null = null private commitMessageAgentEnv: CommitMessageAgentEnvironmentResolvers | null = null private automationService: AutomationService | null = null @@ -3002,6 +3108,14 @@ export class OrcaRuntimeService { * only carrier of the provider session a transcript is addressed by. */ getAgentProviderSessionSnapshot?: () => AgentStatusIpcPayload[] getAgentProviderSessionRowsForPane?: (paneKey: string) => AgentStatusIpcPayload[] + attestAgentHookCompatibilityAuthority?: (candidate: { + paneKey: string + launchTokenHash: string + connectionId: string | null + terminalProvenance: 'current_runtime' | 'restored' + }) => AgentHookAuthorityAttestation | null + retireAgentHookCompatibilityAuthority?: (paneKey: string) => void + canRecoverPersistentLocalPtys?: () => boolean // Why: codex-home paths for the Agent Session History scan must be sourced // here, not via the window-only registerCoreHandlers path — that path never // runs under `orca serve`, so remote/SSH hosts would silently drop @@ -3034,6 +3148,11 @@ export class OrcaRuntimeService { this.getAgentProviderSessionSnapshotFn = deps?.getAgentProviderSessionSnapshot ?? deps?.getAgentStatusSnapshot ?? null this.getAgentProviderSessionRowsForPaneFn = deps?.getAgentProviderSessionRowsForPane ?? null + this.attestAgentHookCompatibilityAuthorityFn = + deps?.attestAgentHookCompatibilityAuthority ?? null + this.retireAgentHookCompatibilityAuthorityFn = + deps?.retireAgentHookCompatibilityAuthority ?? null + this.canRecoverPersistentLocalPtysFn = deps?.canRecoverPersistentLocalPtys ?? (() => true) // Why: configure the shared AiVault scan cache from a serve-mode-reachable // seam so the aiVault.listSessions RPC includes managed-Codex + WSL sessions // even on headless `orca serve` hosts where registerCoreHandlers never runs. @@ -3478,6 +3597,704 @@ export class OrcaRuntimeService { this._orchestrationDb = db } + private getLegacyWorkerTerminalRecoveryPlan(): LegacyWorkerTerminalRecoveryPlan { + try { + return planLegacyWorkerTerminalRecovery( + this.getOrchestrationDb().listLegacyWorkerTerminalRecoveryRows() + ) + } catch (error) { + console.warn('[orchestration] failed to plan legacy worker terminal recovery', error) + return { blockedPanes: [], candidates: [], ambiguousDispatchIds: [] } + } + } + + prepareLegacyWorkerTerminalRecovery(): LegacyWorkerTerminalRecoveryPlan { + const plan = this.getLegacyWorkerTerminalRecoveryPlan() + const store = this.store + if (!store?.getWorkspaceSession || !store.setWorkspaceSession || !store.flushOrThrow) { + return plan + } + const sessions = new Map< + ExecutionHostId, + { current: WorkspaceSessionState; next: WorkspaceSessionState } + >() + const changedHostIds = new Set() + for (const blocked of plan.blockedPanes) { + let hostIds: ExecutionHostId[] + try { + hostIds = [this.getWorkspaceSessionHostIdForWorktree(blocked.worktreeId)] + } catch (error) { + console.warn('[orchestration] legacy worker resume fence owner is unavailable', { + worktreeId: blocked.worktreeId, + error + }) + hostIds = store.getWorkspaceSessionHostIds?.() ?? [LOCAL_EXECUTION_HOST_ID] + } + for (const hostId of hostIds) { + let state = sessions.get(hostId) + if (!state) { + const current = store.getWorkspaceSession(hostId) + if (!current) { + continue + } + state = { current, next: structuredClone(current) } + sessions.set(hostId, state) + } + const record = state.next.sleepingAgentSessionsByPaneKey?.[blocked.paneKey] + if ( + !record || + !runtimeWorktreeIdsEqual(record.worktreeId, blocked.worktreeId) || + record.automaticResumeBlockedBy === 'legacy-orchestration-worker' + ) { + continue + } + state.next.sleepingAgentSessionsByPaneKey = { + ...state.next.sleepingAgentSessionsByPaneKey, + [blocked.paneKey]: { + ...record, + automaticResumeBlockedBy: 'legacy-orchestration-worker' + } + } + changedHostIds.add(hostId) + } + } + const changed = [...sessions].filter(([hostId]) => changedHostIds.has(hostId)) + if (changed.length === 0) { + return plan + } + try { + for (const [hostId, state] of changed) { + store.setWorkspaceSession(state.next, hostId) + } + store.flushOrThrow() + } catch (error) { + console.warn('[orchestration] failed to persist legacy worker resume fence', error) + } + return plan + } + + async reconcileLegacyWorkerTerminals( + options: { connectionId?: string; materializeRenderer?: boolean } = {} + ): Promise { + let resolveResult!: (result: LegacyWorkerTerminalRecoveryResult) => void + let rejectResult!: (error: unknown) => void + const result = new Promise((resolve, reject) => { + resolveResult = resolve + rejectResult = reject + }) + const run = this.legacyWorkerTerminalRecoveryQueue.then(async () => { + try { + resolveResult(await this.reconcileLegacyWorkerTerminalsNow(options)) + } catch (error) { + rejectResult(error) + } + }) + this.legacyWorkerTerminalRecoveryQueue = run.catch(() => undefined) + return result + } + + async refreshRestoredOrchestrationAuthority(connectionId: string | null = null): Promise { + if (connectionId === null && !this.canRecoverPersistentLocalPtysFn()) { + return + } + const inventory = await this.refreshPtyWorktreeRecordsWithControllerInventory( + [...(await this.getResolvedWorktreeMap()).values()], + null, + undefined, + connectionId + ) + if (!inventory) { + throw new Error('terminal_liveness_unavailable') + } + } + + private hasExactTerminalSurfaceIdentity(expected: { + worktreeId: string + tabId: string + leafId: string + ptyId: string + terminalHandle: string + incarnationId: string + }): boolean { + if (this.graphStatus !== 'ready') { + return false + } + const pty = this.ptysById.get(expected.ptyId) + if ( + !pty?.connected || + pty.incarnationId !== expected.incarnationId || + pty.tabId !== expected.tabId || + pty.paneKey !== makePaneKey(expected.tabId, expected.leafId) || + !runtimeWorktreeIdsEqual(pty.worktreeId, expected.worktreeId) || + this.handleByPtyId.get(expected.ptyId) !== expected.terminalHandle + ) { + return false + } + const tab = this.tabs.get(expected.tabId) + const leaf = this.leaves.get(this.getLeafKey(expected.tabId, expected.leafId)) + const ptyLeaves = this.getLeavesForPty(expected.ptyId) + return ( + Boolean(tab && runtimeWorktreeIdsEqual(tab.worktreeId, expected.worktreeId)) && + Boolean( + leaf && + leaf.ptyId === expected.ptyId && + runtimeWorktreeIdsEqual(leaf.worktreeId, expected.worktreeId) + ) && + ptyLeaves.length === 1 && + ptyLeaves[0]?.tabId === expected.tabId && + ptyLeaves[0]?.leafId === expected.leafId + ) + } + + private hasExactPersistedTerminalSurfaceIdentity(expected: { + worktreeId: string + tabId: string + leafId: string + ptyId: string + incarnationId: string + }): boolean { + const session = this.getWorkspaceSessionForWorktree(expected.worktreeId) + const sessionWorktreeId = session + ? resolveTerminalSessionWorktreeId(session, expected.worktreeId) + : null + if (!session || !sessionWorktreeId) { + return false + } + const tab = session.tabsByWorktree[sessionWorktreeId]?.find( + (candidate) => candidate.id === expected.tabId + ) + const paneKey = makePaneKey(expected.tabId, expected.leafId) + return Boolean( + tab && + session.terminalLayoutsByTabId[expected.tabId]?.ptyIdsByLeafId?.[expected.leafId] === + expected.ptyId && + session.terminalPtyIncarnationsByPaneKey?.[paneKey] === expected.incarnationId + ) + } + + private persistLegacyWorkerTerminalRecoveryResolution( + candidate: LegacyWorkerTerminalRecoveryPlan['candidates'][number], + resolution: 'adopted' | 'exited' + ): boolean { + const store = this.store + const session = this.getWorkspaceSessionForWorktree(candidate.worktreeId) + if (!store?.setWorkspaceSession || !store.flushOrThrow || !session) { + return false + } + const record = session.sleepingAgentSessionsByPaneKey?.[candidate.paneKey] + if (!record || !runtimeWorktreeIdsEqual(record.worktreeId, candidate.worktreeId)) { + return true + } + const next = structuredClone(session) + delete next.sleepingAgentSessionsByPaneKey?.[candidate.paneKey] + try { + this.setWorkspaceSessionForWorktree(candidate.worktreeId, next) + store.flushOrThrow() + return true + } catch (error) { + this.setWorkspaceSessionForWorktree(candidate.worktreeId, session) + console.warn('[orchestration] failed to persist legacy worker recovery resolution', { + dispatchId: candidate.dispatchId, + resolution, + error + }) + return false + } + } + + private reconcileMissingLegacyWorkerTerminal( + candidate: LegacyWorkerTerminalRecoveryPlan['candidates'][number] + ): boolean { + if (candidate.dispatchStatus !== 'pending' && candidate.dispatchStatus !== 'dispatched') { + return true + } + try { + this.getOrchestrationDb().reconcileMissingWorkerTerminal( + candidate.dispatchId, + 'The assigned worker terminal is no longer live after orchestration recovery.' + ) + return true + } catch (error) { + console.warn('[orchestration] failed to reconcile missing worker terminal', { + dispatchId: candidate.dispatchId, + error + }) + return false + } + } + + private resolveExitedLegacyWorkerTerminal( + candidate: LegacyWorkerTerminalRecoveryPlan['candidates'][number] + ): boolean { + if ( + !this.rollbackLegacyWorkerTerminalSurface(candidate) || + !this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'exited') || + !this.reconcileMissingLegacyWorkerTerminal(candidate) + ) { + return false + } + this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'exited') + return true + } + + private rollbackLegacyWorkerTerminalSurface( + candidate: LegacyWorkerTerminalRecoveryPlan['candidates'][number] + ): boolean { + const store = this.store + const session = this.getWorkspaceSessionForWorktree(candidate.worktreeId) + if (store?.setWorkspaceSession && store.flushOrThrow && session) { + const retired = retireTerminalSurfaceFromPersistence(session, { + worktreeId: candidate.worktreeId, + parentTabId: candidate.tabId, + leafId: candidate.leafId, + ptyId: candidate.ptyId, + incarnationId: candidate.incarnationId + }) + if (retired !== session) { + try { + this.setWorkspaceSessionForWorktree(candidate.worktreeId, retired) + store.flushOrThrow() + } catch (error) { + this.setWorkspaceSessionForWorktree(candidate.worktreeId, session) + console.warn('[orchestration] failed to persist legacy worker surface rollback', { + dispatchId: candidate.dispatchId, + error + }) + return false + } + } + } + + const snapshot = this.mobileSessionTabsByWorktree.get(candidate.worktreeId) + if (snapshot) { + const retired = retireTerminalSurfacesFromSnapshot({ + snapshot, + ptyId: candidate.ptyId, + exactSurfaces: [{ parentTabId: candidate.tabId, leafId: candidate.leafId }], + exactOnly: true + }) + if (retired) { + this.mobileSessionTabsByWorktree.set(candidate.worktreeId, retired.snapshot) + this.notifyMobileSessionTabsChanged(candidate.worktreeId) + } + } + + const leafKey = this.getLeafKey(candidate.tabId, candidate.leafId) + const leaf = this.leaves.get(leafKey) + const pty = this.ptysById.get(candidate.ptyId) + if ( + leaf?.ptyId === candidate.ptyId && + runtimeWorktreeIdsEqual(leaf.worktreeId, candidate.worktreeId) + ) { + this.leaves.delete(leafKey) + const surfaceHandle = this.handleByLeafKey.get(leafKey) + this.handleByLeafKey.delete(leafKey) + const handleRecord = surfaceHandle ? this.handles.get(surfaceHandle) : undefined + if ( + surfaceHandle && + handleRecord?.tabId === candidate.tabId && + handleRecord.leafId === candidate.leafId && + handleRecord.ptyId === candidate.ptyId + ) { + this.handles.delete(surfaceHandle) + } + this.rebuildLeafPtyIndex() + if (![...this.leaves.values()].some((entry) => entry.tabId === candidate.tabId)) { + this.tabs.delete(candidate.tabId) + } + } + if (pty?.tabId === candidate.tabId) { + pty.tabId = null + pty.paneKey = null + } + this.notifier?.resolveLegacyWorkerTerminalRecovery?.( + candidate.paneKey, + 'rolled_back', + candidate.ptyId + ) + return true + } + + private resolveReplacedLegacyWorkerTerminal( + candidate: LegacyWorkerTerminalRecoveryPlan['candidates'][number] + ): boolean { + return this.resolveExitedLegacyWorkerTerminal(candidate) + } + + private updateLegacyWorkerTerminalRecoveryRetry( + plan: LegacyWorkerTerminalRecoveryPlan, + deferredDispatchIds: ReadonlySet, + options: { connectionId?: string; materializeRenderer?: boolean } + ): void { + const scopeKey = options.connectionId ? `ssh:${options.connectionId}` : 'local' + const hasDeferredWorker = plan.candidates.some((candidate) => { + const sshPty = parseAppSshPtyId(candidate.ptyId) + const inScope = options.connectionId + ? sshPty?.connectionId === options.connectionId + : sshPty === null + return inScope && deferredDispatchIds.has(candidate.dispatchId) + }) + if (!hasDeferredWorker) { + this.cancelLegacyWorkerTerminalRecoveryRetry(scopeKey) + return + } + const existing = this.legacyWorkerTerminalRecoveryRetries.get(scopeKey) + const retry = existing ?? { + attempt: 0, + ...(options.connectionId ? { connectionId: options.connectionId } : {}), + materializeRenderer: options.materializeRenderer === true, + timer: null + } + retry.materializeRenderer ||= options.materializeRenderer === true + this.legacyWorkerTerminalRecoveryRetries.set(scopeKey, retry) + this.armLegacyWorkerTerminalRecoveryRetry(scopeKey, retry) + } + + private cancelLegacyWorkerTerminalRecoveryRetry(scopeKey: string): void { + const retry = this.legacyWorkerTerminalRecoveryRetries.get(scopeKey) + if (retry?.timer) { + clearTimeout(retry.timer) + } + this.legacyWorkerTerminalRecoveryRetries.delete(scopeKey) + } + + private armLegacyWorkerTerminalRecoveryRetry( + scopeKey: string, + retry: { + attempt: number + connectionId?: string + materializeRenderer: boolean + timer: ReturnType | null + } + ): void { + if (retry.timer) { + return + } + const delayMs = Math.min(1_000 * 2 ** retry.attempt, 30_000) + retry.attempt += 1 + retry.timer = setTimeout(() => { + retry.timer = null + void this.reconcileLegacyWorkerTerminals({ + ...(retry.connectionId ? { connectionId: retry.connectionId } : {}), + materializeRenderer: retry.materializeRenderer + }).catch((error) => { + console.warn('[orchestration] worker terminal recovery retry failed', { + scope: scopeKey, + error + }) + if (this.legacyWorkerTerminalRecoveryRetries.get(scopeKey) === retry) { + this.armLegacyWorkerTerminalRecoveryRetry(scopeKey, retry) + } + }) + }, delayMs) + retry.timer.unref?.() + } + + private async reconcileLegacyWorkerTerminalsNow(options: { + connectionId?: string + materializeRenderer?: boolean + }): Promise { + const plan = this.prepareLegacyWorkerTerminalRecovery() + const adoptedDispatchIds: string[] = [] + const exitedDispatchIds: string[] = [] + const deferredDispatchIds = new Set(plan.ambiguousDispatchIds) + const recoveryCandidatesByProvider = new Map< + string, + { + connectionId: string | null + entries: { + candidate: (typeof plan.candidates)[number] + workspace: TerminalWorkspaceLaunchScope + resolvedWorkspace: ResolvedWorktree + }[] + } + >() + for (const candidate of plan.candidates) { + try { + const workspace = await this.resolveTerminalWorkspaceLaunchScope( + `id:${candidate.worktreeId}` + ) + const sshPty = parseAppSshPtyId(candidate.ptyId) + if (workspace.connectionId) { + if ( + options.connectionId !== workspace.connectionId || + sshPty?.connectionId !== workspace.connectionId + ) { + deferredDispatchIds.add(candidate.dispatchId) + continue + } + } else if ( + options.connectionId !== undefined || + sshPty !== null || + !this.canRecoverPersistentLocalPtysFn() + ) { + deferredDispatchIds.add(candidate.dispatchId) + continue + } + const resolvedWorkspace = workspace.folderWorkspace + ? this.folderWorkspaceToResolvedWorktree(workspace.folderWorkspace) + : await this.resolveWorktreeSelector(`id:${workspace.id}`) + const connectionId = workspace.connectionId ?? null + const providerKey = connectionId === null ? 'local' : `ssh:${connectionId}` + const provider = recoveryCandidatesByProvider.get(providerKey) ?? { + connectionId, + entries: [] + } + provider.entries.push({ candidate, workspace, resolvedWorkspace }) + recoveryCandidatesByProvider.set(providerKey, provider) + } catch { + deferredDispatchIds.add(candidate.dispatchId) + } + } + for (const provider of recoveryCandidatesByProvider.values()) { + const resolvedWorktrees = [ + ...new Map( + provider.entries.map(({ resolvedWorkspace }) => [resolvedWorkspace.id, resolvedWorkspace]) + ).values() + ] + const inventory = await this.refreshPtyWorktreeRecordsWithControllerInventory( + resolvedWorktrees, + null, + undefined, + provider.connectionId + ) + if (!inventory) { + provider.entries.forEach(({ candidate }) => deferredDispatchIds.add(candidate.dispatchId)) + continue + } + for (const { candidate, workspace } of provider.entries) { + if (!inventory.livePtyIds.has(candidate.ptyId)) { + if (this.resolveExitedLegacyWorkerTerminal(candidate)) { + exitedDispatchIds.push(candidate.dispatchId) + } else { + deferredDispatchIds.add(candidate.dispatchId) + } + continue + } + const controllerIdentity = inventory.terminalIdentityByPtyId.get(candidate.ptyId) + if (!controllerIdentity) { + deferredDispatchIds.add(candidate.dispatchId) + continue + } + if ( + controllerIdentity.handle !== candidate.terminalHandle || + controllerIdentity.incarnationId !== candidate.incarnationId + ) { + if (this.resolveReplacedLegacyWorkerTerminal(candidate)) { + exitedDispatchIds.push(candidate.dispatchId) + } else { + deferredDispatchIds.add(candidate.dispatchId) + } + continue + } + const preAdoptionInventory = await this.refreshPtyWorktreeRecordsWithControllerInventory( + resolvedWorktrees, + null, + undefined, + provider.connectionId + ) + if (!preAdoptionInventory) { + deferredDispatchIds.add(candidate.dispatchId) + continue + } + if (!preAdoptionInventory.livePtyIds.has(candidate.ptyId)) { + if (this.resolveExitedLegacyWorkerTerminal(candidate)) { + exitedDispatchIds.push(candidate.dispatchId) + } else { + deferredDispatchIds.add(candidate.dispatchId) + } + continue + } + const preAdoptionIdentity = preAdoptionInventory.terminalIdentityByPtyId.get( + candidate.ptyId + ) + if (!preAdoptionIdentity) { + deferredDispatchIds.add(candidate.dispatchId) + continue + } + if ( + preAdoptionIdentity.handle !== candidate.terminalHandle || + preAdoptionIdentity.incarnationId !== candidate.incarnationId + ) { + if (this.resolveReplacedLegacyWorkerTerminal(candidate)) { + exitedDispatchIds.push(candidate.dispatchId) + } else { + deferredDispatchIds.add(candidate.dispatchId) + } + continue + } + const session = this.getWorkspaceSessionForWorktree(candidate.worktreeId) + const sessionWorktreeId = session + ? resolveTerminalSessionWorktreeId(session, candidate.worktreeId) + : null + const activeTabId = sessionWorktreeId + ? session?.activeTabIdByWorktree?.[sessionWorktreeId] + : undefined + const activeGroupId = sessionWorktreeId + ? session?.activeGroupIdByWorktree?.[sessionWorktreeId] + : undefined + const exactSurfaceAlreadyPublished = + this.hasExactPersistedTerminalSurfaceIdentity(candidate) && + this.hasExactTerminalSurfaceIdentity(candidate) + if (!exactSurfaceAlreadyPublished) { + try { + await this.adoptTerminalOrphansFromInventory( + { + worktree: `id:${candidate.worktreeId}`, + expectedTopologyRevision: this.getTerminalTopologyRevision(candidate.worktreeId), + ...(activeTabId ? { activeTabId } : {}), + ...(activeGroupId ? { activeGroupId } : {}), + claims: [ + { + terminal: candidate.terminalHandle, + ptyId: candidate.ptyId, + incarnationId: candidate.incarnationId, + tabId: candidate.tabId, + leafId: candidate.leafId + } + ] + }, + workspace, + preAdoptionInventory + ) + } catch (error) { + console.warn('[orchestration] legacy worker terminal adoption deferred', { + dispatchId: candidate.dispatchId, + error + }) + deferredDispatchIds.add(candidate.dispatchId) + continue + } + } + let rendererMaterialized = + options.materializeRenderer !== true || + this.legacyWorkerTerminalReceiptEpochByPane.get(candidate.paneKey) === + this.rendererGraphEpoch + const pty = this.ptysById.get(candidate.ptyId) + if ( + options.materializeRenderer && + !rendererMaterialized && + pty && + this.notifier?.revealTerminalSession + ) { + for (let attempt = 0; attempt < 2 && !rendererMaterialized; attempt += 1) { + try { + const reveal = await this.notifier.revealTerminalSession(candidate.worktreeId, { + ptyId: candidate.ptyId, + title: getLatestPtyTitle(pty) ?? pty.controllerTitle, + activate: false, + presentation: 'background', + tabId: candidate.tabId, + leafId: candidate.leafId, + focus: false, + expectedProcessIdentity: { + terminalHandle: candidate.terminalHandle, + incarnationId: candidate.incarnationId + } + }) + const identity = reveal?.identity + if ( + !identity || + !runtimeWorktreeIdsEqual(identity.worktreeId, candidate.worktreeId) || + identity.tabId !== candidate.tabId || + identity.leafId !== candidate.leafId || + identity.ptyId !== candidate.ptyId + ) { + throw new Error('terminal_reveal_identity_mismatch') + } + rendererMaterialized = true + this.legacyWorkerTerminalReceiptEpochByPane.set( + candidate.paneKey, + this.rendererGraphEpoch + ) + } catch (error) { + if (attempt === 0) { + await new Promise((resolve) => setTimeout(resolve, 100)) + continue + } + console.warn('[orchestration] adopted legacy worker was not revealed', { + dispatchId: candidate.dispatchId, + error + }) + } + } + } + if (!rendererMaterialized) { + this.legacyWorkerTerminalReceiptEpochByPane.delete(candidate.paneKey) + deferredDispatchIds.add(candidate.dispatchId) + continue + } + if ( + options.materializeRenderer === true && + !this.hasExactTerminalSurfaceIdentity({ + worktreeId: candidate.worktreeId, + tabId: candidate.tabId, + leafId: candidate.leafId, + ptyId: candidate.ptyId, + terminalHandle: candidate.terminalHandle, + incarnationId: candidate.incarnationId + }) + ) { + deferredDispatchIds.add(candidate.dispatchId) + continue + } + const finalInventory = await this.refreshPtyWorktreeRecordsWithControllerInventory( + resolvedWorktrees, + null, + undefined, + provider.connectionId + ) + if (!finalInventory) { + deferredDispatchIds.add(candidate.dispatchId) + continue + } + if (!finalInventory.livePtyIds.has(candidate.ptyId)) { + this.legacyWorkerTerminalReceiptEpochByPane.delete(candidate.paneKey) + this.onPtyExit(candidate.ptyId, 0, candidate.incarnationId) + if (this.resolveExitedLegacyWorkerTerminal(candidate)) { + exitedDispatchIds.push(candidate.dispatchId) + } else { + deferredDispatchIds.add(candidate.dispatchId) + } + continue + } + const finalIdentity = finalInventory.terminalIdentityByPtyId.get(candidate.ptyId) + if (!finalIdentity) { + this.legacyWorkerTerminalReceiptEpochByPane.delete(candidate.paneKey) + deferredDispatchIds.add(candidate.dispatchId) + continue + } + if ( + finalIdentity.handle !== candidate.terminalHandle || + finalIdentity.incarnationId !== candidate.incarnationId + ) { + this.legacyWorkerTerminalReceiptEpochByPane.delete(candidate.paneKey) + if (this.resolveReplacedLegacyWorkerTerminal(candidate)) { + exitedDispatchIds.push(candidate.dispatchId) + } else { + deferredDispatchIds.add(candidate.dispatchId) + } + continue + } + if (!this.persistLegacyWorkerTerminalRecoveryResolution(candidate, 'adopted')) { + deferredDispatchIds.add(candidate.dispatchId) + continue + } + this.legacyWorkerRecoveredPtys.add(candidate.ptyId) + this.notifier?.resolveLegacyWorkerTerminalRecovery?.(candidate.paneKey, 'adopted') + adoptedDispatchIds.push(candidate.dispatchId) + } + } + const result = { + blockedPaneCount: plan.blockedPanes.length, + adoptedDispatchIds, + exitedDispatchIds, + deferredDispatchIds: [...deferredDispatchIds] + } + this.updateLegacyWorkerTerminalRecoveryRetry(plan, deferredDispatchIds, options) + return result + } + setAutomationService(service: AutomationService): void { this.automationService = service } @@ -3621,8 +4438,20 @@ export class OrcaRuntimeService { } private getWorkspaceSessionHostIdForWorktree(worktreeId: string): ExecutionHostId { - const repo = this.store?.getRepo?.(getRepoIdFromWorktreeId(worktreeId)) - return repo ? getRepoExecutionHostId(repo) : 'local' + const scope = parseWorkspaceKey(worktreeId) + if (scope?.type === 'folder') { + const workspace = this.store + ?.getFolderWorkspaces?.() + .find((entry) => entry.id === scope.folderWorkspaceId) + if (!workspace) { + throw new Error('folder_workspace_not_found') + } + const connectionId = this.resolveFolderWorkspaceConnectionId(workspace) + return connectionId ? toSshExecutionHostId(connectionId) : LOCAL_EXECUTION_HOST_ID + } + const resolvedWorktreeId = scope?.type === 'worktree' ? scope.worktreeId : worktreeId + const repo = this.store?.getRepo?.(getRepoIdFromWorktreeId(resolvedWorktreeId)) + return repo ? getRepoExecutionHostId(repo) : LOCAL_EXECUTION_HOST_ID } private getWorkspaceSessionForWorktree(worktreeId: string): WorkspaceSessionState | null { @@ -3829,20 +4658,48 @@ export class OrcaRuntimeService { notifySshStateChanged(targetId: string, state: SshConnectionState): void { this.bumpSshRelayRecoveryGeneration(targetId) this.invalidateSshWorktreeScanCache(targetId) + if (state.status !== 'connected') { + this.cancelLegacyWorkerTerminalRecoveryRetry(`ssh:${targetId}`) + } this.emitClientEvent({ type: 'sshStateChanged', targetId, state: getPublicSshState(state)! }) } notifySshRelayReady(targetId: string): void { const generation = this.bumpSshRelayRecoveryGeneration(targetId) - void this.publishRecoveredSshMobileSessionTabs(targetId, generation).catch((error) => { - if (this.sshRelayRecoveryGenerationByTargetId.get(targetId) !== generation) { - return + const publish = async (): Promise => { + try { + await this.publishRecoveredSshMobileSessionTabs(targetId, generation) + } catch (error) { + if (this.sshRelayRecoveryGenerationByTargetId.get(targetId) === generation) { + console.warn('[runtime] failed to publish recovered SSH session tabs', { + targetId, + error + }) + } } - console.warn('[runtime] failed to publish recovered SSH session tabs', { - targetId, - error + } + const initialPublication = publish() + void initialPublication + void this.refreshRestoredOrchestrationAuthority(targetId) + .then(() => + this.reconcileLegacyWorkerTerminals({ + connectionId: targetId, + materializeRenderer: this.notifier !== null + }) + ) + .then(async () => { + await initialPublication + await publish() + }) + .catch((error) => { + if (this.sshRelayRecoveryGenerationByTargetId.get(targetId) !== generation) { + return + } + console.warn('[orchestration] legacy worker reconcile failed on relay ready', { + targetId, + error + }) }) - }) } private bumpSshRelayRecoveryGeneration(targetId: string): number { @@ -7454,7 +8311,8 @@ export class OrcaRuntimeService { private adoptControllerTerminalHandle( ptyId: string, handle: string | undefined, - incarnationId?: string + incarnationId?: string, + options: { exactRestoredSurface?: boolean } = {} ): void { const trimmed = handle?.trim() if (!trimmed || !trimmed.startsWith('term_')) { @@ -7475,7 +8333,13 @@ export class OrcaRuntimeService { } } if (this.isTerminalHandleAdoptionBlocked(ptyId, trimmed)) { - return + if ( + !options.exactRestoredSurface || + !this.replaceSyntheticTerminalHandlesForRestoredPty(ptyId, trimmed) || + this.isTerminalHandleAdoptionBlocked(ptyId, trimmed) + ) { + return + } } // Why: after an app/runtime restart, the live PTY child still has its // original ORCA_TERMINAL_HANDLE, but the runtime's in-memory map is gone. @@ -7489,6 +8353,7 @@ export class OrcaRuntimeService { if (record.ptyId === ptyId) { invalidated.add(handle) this.handles.delete(handle) + this.syntheticTerminalHandles.delete(handle) this.rejectWaitersForHandle(handle, 'terminal_handle_stale') } } @@ -7499,6 +8364,45 @@ export class OrcaRuntimeService { } } + private replaceSyntheticTerminalHandlesForRestoredPty( + ptyId: string, + controllerHandle: string + ): boolean { + const boundHandles = new Set() + const directHandle = this.handleByPtyId.get(ptyId) + if (directHandle) { + boundHandles.add(directHandle) + } + for (const [handle, record] of this.handles) { + if (record.ptyId === ptyId) { + boundHandles.add(handle) + } else if (handle === controllerHandle) { + return false + } + } + for (const [otherPtyId, handle] of this.handleByPtyId) { + if (otherPtyId !== ptyId && handle === controllerHandle) { + return false + } + } + for (const leaf of this.getLeavesForPty(ptyId)) { + const handle = this.handleByLeafKey.get(this.getLeafKey(leaf.tabId, leaf.leafId)) + if (handle) { + boundHandles.add(handle) + } + } + if ( + boundHandles.size === 0 || + [...boundHandles].some( + (handle) => handle === controllerHandle || !this.syntheticTerminalHandles.has(handle) + ) + ) { + return false + } + this.invalidateAllHandlesForPty(ptyId) + return true + } + // Why: adoption is best-effort restart recovery and must be first-wins. // Re-keying a pty that already has a handle this session would strand // waiters registered under the old handle, and provider-reported values @@ -8114,6 +9018,7 @@ export class OrcaRuntimeService { this.recordTerminalSideEffectFact(ptyId, { kind: 'bell' }) return case 'command-finished': + this.retirePtyAgentLaunchAuthority(ptyId) this.recordTerminalSideEffectFact(ptyId, { kind: 'command-finished', exitCode: fact.exitCode @@ -8355,17 +9260,17 @@ export class OrcaRuntimeService { onAgentExited: () => { this.recordTerminalSideEffectFact(ptyId, { kind: 'agent-exited' }) }, - // Why: bell/command-finished/pr-link/2031 facts exist only for the - // pty:sideEffect channel. Headless serve has no consumer, so skip the - // per-chunk bell walk and 133/URL/2031 scans entirely. + onCommandFinished: (exitCode: number | null) => { + this.retirePtyAgentLaunchAuthority(ptyId) + this.recordTerminalSideEffectFact(ptyId, { kind: 'command-finished', exitCode }) + }, + // Why: headless serve still scans command completion to retire agent + // launch authority; other transient facts remain desktop-only. ...(this.terminalSideEffectConsumerAvailable ? { onBell: () => { this.recordTerminalSideEffectFact(ptyId, { kind: 'bell' }) }, - onCommandFinished: (exitCode: number | null) => { - this.recordTerminalSideEffectFact(ptyId, { kind: 'command-finished', exitCode }) - }, onPrLink: (link: TerminalGitHubPRLink) => { this.recordTerminalSideEffectFact(ptyId, { kind: 'pr-link', link }) }, @@ -8724,6 +9629,7 @@ export class OrcaRuntimeService { private advancePtyLifecycleGeneration(ptyId: string): void { this.ptyLifecycleGenerationById.set(ptyId, this.nextPtyLifecycleGeneration++) + this.legacyWorkerRecoveredPtys.delete(ptyId) // Why: a provider response belongs to the process generation that issued // it; a respawn must neither reuse its frame nor join its in-flight call. this.providerBufferAcquisitionsByPtyId.delete(ptyId) @@ -9632,11 +10538,20 @@ export class OrcaRuntimeService { return read } const blankFallback = shouldFallbackToVisibleTerminalSnapshot(read, opts) + const recoveredWorkerFallback = + read.tail.length === 0 && this.legacyWorkerRecoveredPtys.has(ptyId) + if (recoveredWorkerFallback) { + const providerLines = await this.readProviderTerminalTailLines(ptyId, opts.limit) + if (providerLines.length > 0) { + return buildVisibleSnapshotReadFallback(read, providerLines, opts.limit) + } + } const knownAlternateScreen = this.isTerminalAlternateScreen(ptyId) const providerModeUnknown = this.providerSnapshotPreferredPtys.has(ptyId) && !this.providerModeTrackersByPtyId.has(ptyId) if ( !blankFallback && + !recoveredWorkerFallback && !providerModeUnknown && !knownAlternateScreen && !this.headlessTerminals.has(ptyId) @@ -9644,7 +10559,12 @@ export class OrcaRuntimeService { return read } const visibleState = await this.readVisibleTerminalState(ptyId) - if (!blankFallback && !knownAlternateScreen && !visibleState?.isAlternateScreen) { + if ( + !blankFallback && + !recoveredWorkerFallback && + !knownAlternateScreen && + !visibleState?.isAlternateScreen + ) { return read } let lines = visibleState?.lines ?? [] @@ -9657,6 +10577,31 @@ export class OrcaRuntimeService { return buildVisibleSnapshotReadFallback(read, lines, opts.limit) } + private async readProviderTerminalTailLines( + ptyId: string, + limit: number | undefined + ): Promise { + const lineLimit = terminalReadLimit(limit, DEFAULT_TERMINAL_READ_LIMIT) + const snapshot = await this.serializeProviderTerminalBuffer(ptyId, { + scrollbackRows: lineLimit + }) + const data = snapshot ? `${snapshot.scrollbackAnsi ?? ''}${snapshot.data}` : '' + if (!snapshot || data.length === 0) { + return [] + } + const emulator = new HeadlessEmulator({ + cols: snapshot.cols, + rows: snapshot.rows, + scrollback: lineLimit + }) + try { + await emulator.write(data) + return visibleNonBlankTerminalLines(emulator.getBufferTailLines(lineLimit)) + } finally { + emulator.dispose() + } + } + private async visibleSnapshotPreview(ptyId: string, preview: string): Promise { const knownAlternateScreen = this.isTerminalAlternateScreen(ptyId) const providerModeUnknown = @@ -9926,6 +10871,226 @@ export class OrcaRuntimeService { return { ptyId: leaf.ptyId } } + getOrchestrationCompatibilityHostId(): 'local' { + return 'local' + } + + registerOrchestrationCompatibilitySshAttachment( + targetId: string, + connectionIncarnation: string + ): OrchestrationCompatibilitySshAttachmentAuthority { + const authority = Object.freeze({ + kind: 'ssh' as const, + targetId, + connectionIncarnation, + attachmentId: randomUUID() + }) + this.orchestrationCompatibilitySshAttachments.set(authority.attachmentId, authority) + return authority + } + + releaseOrchestrationCompatibilitySshAttachment(attachmentId: string): void { + this.orchestrationCompatibilitySshAttachments.delete(attachmentId) + } + + verifyOrchestrationCompatibilityCaller( + evidence: OrchestrationCompatibilityEvidence | null | undefined + ): OrchestrationCompatibilityCallerAuthority | null { + const terminalHandle = + typeof evidence?.terminalHandle === 'string' ? evidence.terminalHandle.trim() : '' + const claimedPaneKey = typeof evidence?.paneKey === 'string' ? evidence.paneKey.trim() : '' + const launchToken = typeof evidence?.launchToken === 'string' ? evidence.launchToken.trim() : '' + const host = evidence?.host + if (!terminalHandle || !claimedPaneKey || !launchToken) { + return null + } + const terminal = this.getOrchestrationDispatchAuthority(terminalHandle) + if ( + !terminal?.processIncarnation || + !terminal.paneKey || + !this.orchestrationCompatibilityHostMatches(terminal.hostScope, host) + ) { + return null + } + const launchTokenHash = createHash('sha256').update(launchToken).digest('hex') + let terminalProvenance: 'current_runtime' | 'restored' + if (terminal.launchTokenHash) { + if (launchTokenHash !== terminal.launchTokenHash) { + return null + } + terminalProvenance = 'current_runtime' + } else { + const receipt = this.restoredOrchestrationAuthorityByPtyId.get(terminal.ptyId) + if ( + !receipt || + receipt.ptyId !== terminal.ptyId || + receipt.worktreeId !== terminal.worktreeId || + receipt.terminalHandle !== terminal.terminalHandle || + receipt.paneKey !== terminal.paneKey || + receipt.processIncarnation !== terminal.processIncarnation || + !this.orchestrationCompatibilityHostScopesEqual(receipt.hostScope, terminal.hostScope) + ) { + return null + } + terminalProvenance = 'restored' + } + const attestation = this.attestAgentHookCompatibilityAuthorityFn?.({ + paneKey: claimedPaneKey, + launchTokenHash, + connectionId: terminal.hostScope.kind === 'ssh' ? terminal.hostScope.targetId : null, + terminalProvenance + }) + if (!attestation || attestation.paneKey !== terminal.paneKey) { + return null + } + return Object.freeze({ + hostScope: Object.freeze({ ...terminal.hostScope }), + paneKey: attestation.paneKey, + terminalHandle, + processIncarnation: terminal.processIncarnation, + launchTokenHash + }) + } + + private orchestrationCompatibilityHostMatches( + hostScope: OrchestrationCompatibilityTerminalAuthority['hostScope'], + host: OrchestrationCompatibilityHostStamp | undefined + ): boolean { + if (hostScope.kind === 'local') { + return host === undefined + } + if (hostScope.kind === 'wsl') { + return ( + host?.kind === 'wsl' && host.hostId === hostScope.hostId && host.distro === hostScope.distro + ) + } + if (host?.kind !== 'ssh' || host.targetId !== hostScope.targetId) { + return false + } + const authority = this.orchestrationCompatibilitySshAttachments.get(host.attachmentId) + return ( + authority?.targetId === host.targetId && + authority.connectionIncarnation === host.connectionIncarnation + ) + } + + private orchestrationCompatibilityHostScopesEqual( + left: OrchestrationCompatibilityTerminalAuthority['hostScope'], + right: OrchestrationCompatibilityTerminalAuthority['hostScope'] + ): boolean { + if (left.kind !== right.kind) { + return false + } + if (left.kind === 'local' && right.kind === 'local') { + return left.hostId === right.hostId + } + if (left.kind === 'wsl' && right.kind === 'wsl') { + return left.hostId === right.hostId && left.distro === right.distro + } + return left.kind === 'ssh' && right.kind === 'ssh' && left.targetId === right.targetId + } + + private getOrchestrationCompatibilityHostScope( + pty: RuntimePtyWorktreeRecord + ): OrchestrationCompatibilityTerminalAuthority['hostScope'] | null { + if (pty.connectionId) { + return { kind: 'ssh', targetId: pty.connectionId } + } + if (pty.isWsl || pty.wslDistro) { + return pty.wslDistro ? { kind: 'wsl', hostId: 'local', distro: pty.wslDistro } : null + } + return { kind: 'local', hostId: 'local' } + } + + private rememberRestoredOrchestrationAuthority( + pty: RuntimePtyWorktreeRecord, + terminalHandle: string, + incarnationId: string + ): void { + const paneKey = pty.paneKey + const hostScope = this.getOrchestrationCompatibilityHostScope(pty) + if (!paneKey || !parsePaneKey(paneKey) || !hostScope) { + this.restoredOrchestrationAuthorityByPtyId.delete(pty.ptyId) + return + } + this.restoredOrchestrationAuthorityByPtyId.set( + pty.ptyId, + Object.freeze({ + ptyId: pty.ptyId, + worktreeId: pty.worktreeId, + terminalHandle, + paneKey, + processIncarnation: `${pty.ptyId}:${incarnationId}`, + hostScope: Object.freeze({ ...hostScope }) + }) + ) + } + + getOrchestrationDispatchAuthority( + terminalHandle: string + ): OrchestrationCompatibilityTerminalAuthority | null { + let ptyId: string | null + try { + ptyId = + this.getLivePtyForHandle(terminalHandle)?.pty.ptyId ?? + this.resolveLiveLeafForHandle(terminalHandle)?.ptyId ?? + null + } catch { + return null + } + if (!ptyId) { + return null + } + const pty = this.ptysById.get(ptyId) + if (!pty?.connected) { + return null + } + const hostScope = this.getOrchestrationCompatibilityHostScope(pty) + if (!hostScope) { + return null + } + return { + runtimeId: this.runtimeId, + terminalHandle, + ptyId, + worktreeId: pty.worktreeId, + processIncarnation: this.getTerminalProcessIncarnation(terminalHandle), + paneKey: pty.paneKey, + launchTokenHash: pty.launchToken + ? createHash('sha256').update(pty.launchToken).digest('hex') + : null, + hostScope + } + } + + private retirePtyAgentLaunchAuthority(ptyId: string): void { + const pty = this.ptysById.get(ptyId) + if (!pty) { + return + } + const receipt = this.restoredOrchestrationAuthorityByPtyId.get(ptyId) + if (!pty.launchToken && !receipt) { + return + } + this.restoredOrchestrationAuthorityByPtyId.delete(ptyId) + pty.launchToken = null + const paneKeys = new Set() + if (pty.paneKey && parsePaneKey(pty.paneKey)) { + paneKeys.add(pty.paneKey) + } + if (receipt?.paneKey && parsePaneKey(receipt.paneKey)) { + paneKeys.add(receipt.paneKey) + } + for (const leaf of this.getLeavesForPty(ptyId)) { + if (isValidTerminalTabId(leaf.tabId) && isTerminalLeafId(leaf.leafId)) { + paneKeys.add(makePaneKey(leaf.tabId, leaf.leafId)) + } + } + for (const paneKey of paneKeys) { + this.retireAgentHookCompatibilityAuthorityFn?.(paneKey) + } + } + async resolveTerminalCwd(handle: string): Promise { const ptyId = this.resolveLeafForHandle(handle)?.ptyId if (!ptyId) { @@ -11059,6 +12224,13 @@ export class OrcaRuntimeService { if (exitIncarnationId && pty?.incarnationId && exitIncarnationId !== pty.incarnationId) { return } + const preservesAbnormalSshSurface = + this.isSshOwnedPtyId(ptyId) && pty?.connectionId != null && exitCode < 0 + if (preservesAbnormalSshSurface) { + this.restoredOrchestrationAuthorityByPtyId.delete(ptyId) + } else { + this.retirePtyAgentLaunchAuthority(ptyId) + } const incarnationId = exitIncarnationId ?? pty?.incarnationId ?? @@ -11184,8 +12356,6 @@ export class OrcaRuntimeService { this.resolvePtyExitWaiters(pty, ptyId) this.pruneDisconnectedPtyTranscript(pty) } - const preservesAbnormalSshSurface = - this.isSshOwnedPtyId(ptyId) && pty?.connectionId != null && exitCode < 0 if (preservesIntentionalHandlelessSurface || preservesAbnormalSshSurface) { // Why: relay loss is recoverable; keep the HUB-owned pane addressable through the bounded reconnect grace. this.touchMobileSessionSnapshotsForPty(ptyId, { immediate: true }) @@ -11201,7 +12371,9 @@ export class OrcaRuntimeService { leaf.writable = false leaf.lastExitCode = exitCode this.resolveExitWaiters(leaf) - this.failActiveDispatchOnExit(leaf, exitCode) + if (!preservesAbnormalSshSurface) { + this.failActiveDispatchOnExit(leaf, exitCode) + } } this.pruneDisconnectedPtyRecords() } @@ -12993,7 +14165,7 @@ export class OrcaRuntimeService { private getTerminalTopologyRevision(worktreeId: string): number { const repoId = getRepoIdFromWorktreeId(worktreeId) return ( - this.store?.getWorkspaceSession?.()?.terminalTopologyRevisionByRepoId?.[repoId] ?? + this.getWorkspaceSessionForWorktree(worktreeId)?.terminalTopologyRevisionByRepoId?.[repoId] ?? this.terminalTopologyRevisionByRepoId.get(repoId) ?? 0 ) @@ -13005,36 +14177,49 @@ export class OrcaRuntimeService { if (request.claims.length === 0) { throw new Error('terminal_orphan_claims_required') } - const worktree = await this.resolveWorktreeSelector(request.worktree) - const livePtyIds = await this.refreshPtyWorktreeRecordsFromController([worktree], worktree.id) - if (!livePtyIds) { + const workspace = await this.resolveTerminalWorkspaceLaunchScope(request.worktree) + const resolvedWorkspace = workspace.folderWorkspace + ? this.folderWorkspaceToResolvedWorktree(workspace.folderWorkspace) + : await this.resolveWorktreeSelector(`id:${workspace.id}`) + const inventory = await this.refreshPtyWorktreeRecordsWithControllerInventory( + [resolvedWorkspace], + workspace.id, + undefined, + workspace.connectionId ?? null + ) + if (!inventory) { throw new Error('terminal_liveness_unavailable') } + return this.adoptTerminalOrphansFromInventory(request, workspace, inventory) + } + + private async adoptTerminalOrphansFromInventory( + request: RuntimeTerminalOrphanAdoptionRequest, + workspace: TerminalWorkspaceLaunchScope, + inventory: PtyControllerInventory + ): Promise { + const { livePtyIds, terminalIdentityByPtyId } = inventory const store = this.store - const session = store?.getWorkspaceSession?.() + const session = this.getWorkspaceSessionForWorktree(workspace.id) if (!store?.setWorkspaceSession || !store.flushOrThrow || !session) { throw new Error('workspace_session_unavailable') } - const sessionWorktreeId = resolveTerminalSessionWorktreeId(session, worktree.id) + const sessionWorktreeId = resolveTerminalSessionWorktreeId(session, workspace.id) if (!sessionWorktreeId) { throw new Error('terminal_orphan_competing_owner') } - const repoId = getRepoIdFromWorktreeId(worktree.id) - const worktreeRepo = store.getRepo(repoId) - if (!worktreeRepo) { - throw new Error('terminal_orphan_owner_mismatch') - } - const worktreeConnectionId = worktreeRepo.connectionId ?? null + const repoId = getRepoIdFromWorktreeId(workspace.id) + const worktreeConnectionId = workspace.connectionId let worktreeWslDistro: string | null = null - if (!worktreeConnectionId) { + if (!worktreeConnectionId && workspace.repo) { try { worktreeWslDistro = - getLocalProjectWorktreeGitOptions(this.requireStore(), worktreeRepo).wslDistro ?? null + getLocalProjectWorktreeGitOptions(this.requireStore(), workspace.repo).wslDistro ?? null } catch { throw new Error('terminal_orphan_owner_mismatch') } } - const currentRevision = this.getTerminalTopologyRevision(worktree.id) + const currentRevision = this.getTerminalTopologyRevision(workspace.id) const seenPtyIds = new Set() const seenPaneKeys = new Set() const validated = request.claims.map((claim) => { @@ -13046,7 +14231,7 @@ export class OrcaRuntimeService { seenPaneKeys.add(paneKey) const live = this.getLivePtyForHandle(claim.terminal) const pty = live?.pty - const controllerIdentity = this.controllerTerminalIdentityByPtyId.get(claim.ptyId) + const controllerIdentity = terminalIdentityByPtyId.get(claim.ptyId) if ( !pty || pty.ptyId !== claim.ptyId || @@ -13060,7 +14245,7 @@ export class OrcaRuntimeService { throw new Error('terminal_orphan_stale') } if ( - !runtimeWorktreeIdsEqual(pty.worktreeId, worktree.id) || + !runtimeWorktreeIdsEqual(pty.worktreeId, workspace.id) || !terminalOrphanExecutionOwnersEqual( { connectionId: worktreeConnectionId, wslDistro: worktreeWslDistro }, { @@ -13079,7 +14264,7 @@ export class OrcaRuntimeService { if ( visualOwners.some( (owner) => - !runtimeWorktreeIdsEqual(owner.worktreeId, worktree.id) || + !runtimeWorktreeIdsEqual(owner.worktreeId, workspace.id) || owner.tabId !== claim.tabId || owner.leafId !== claim.leafId ) @@ -13128,16 +14313,20 @@ export class OrcaRuntimeService { const binding = persistedBinding(claim.ptyId) return ( binding !== null && - runtimeWorktreeIdsEqual(binding.worktreeId, worktree.id) && + runtimeWorktreeIdsEqual(binding.worktreeId, workspace.id) && binding.paneKey === paneKey && session.terminalPtyIncarnationsByPaneKey?.[paneKey] === claim.incarnationId ) }) - if (isExactPersisted && sessionWorktreeId === worktree.id) { + if (isExactPersisted && sessionWorktreeId === workspace.id) { + for (const { claim, pty, paneKey } of validated) { + pty.tabId = claim.tabId + pty.paneKey = paneKey + } return { adopted: false, topologyRevision: currentRevision, - snapshot: await this.listMobileSessionTabs(`id:${worktree.id}`) + snapshot: this.getTerminalOrphanAdoptionSnapshot(workspace.id) } } if (currentRevision !== request.expectedTopologyRevision) { @@ -13217,7 +14406,7 @@ export class OrcaRuntimeService { const existingBinding = persistedBinding(claim.ptyId) if ( existingBinding && - (!runtimeWorktreeIdsEqual(existingBinding.worktreeId, worktree.id) || + (!runtimeWorktreeIdsEqual(existingBinding.worktreeId, workspace.id) || existingBinding.paneKey !== paneKey) ) { throw new Error('terminal_orphan_competing_owner') @@ -13231,14 +14420,14 @@ export class OrcaRuntimeService { if ( graphOwner && (graphOwner.ptyId !== claim.ptyId || - !runtimeWorktreeIdsEqual(graphOwner.worktreeId, worktree.id)) + !runtimeWorktreeIdsEqual(graphOwner.worktreeId, workspace.id)) ) { throw new Error('terminal_orphan_surface_occupied') } if ( Object.entries(session.tabsByWorktree).some( ([ownerWorktreeId, tabs]) => - !runtimeWorktreeIdsEqual(ownerWorktreeId, worktree.id) && + !runtimeWorktreeIdsEqual(ownerWorktreeId, workspace.id) && tabs.some((tab) => tab.id === claim.tabId) ) ) { @@ -13256,7 +14445,7 @@ export class OrcaRuntimeService { ) if ( surfaceOwner && - (snapshot.worktree !== worktree.id || surfaceOwner.ptyId !== claim.ptyId) + (snapshot.worktree !== workspace.id || surfaceOwner.ptyId !== claim.ptyId) ) { throw new Error('terminal_orphan_surface_occupied') } @@ -13266,7 +14455,7 @@ export class OrcaRuntimeService { ) if ( owner && - (snapshot.worktree !== worktree.id || + (snapshot.worktree !== workspace.id || owner.parentTabId !== claim.tabId || owner.leafId !== claim.leafId) ) { @@ -13276,8 +14465,8 @@ export class OrcaRuntimeService { } const next = structuredClone(session) - canonicalizeTerminalSessionWorktreeId(next, sessionWorktreeId, worktree.id) - const existingTabs = next.tabsByWorktree[worktree.id] ?? [] + canonicalizeTerminalSessionWorktreeId(next, sessionWorktreeId, workspace.id) + const existingTabs = next.tabsByWorktree[workspace.id] ?? [] const tabsById = new Map(existingTabs.map((tab) => [tab.id, tab])) for (const { claim, pty, paneKey } of validated) { let tab = tabsById.get(claim.tabId) @@ -13287,7 +14476,7 @@ export class OrcaRuntimeService { tab = { id: claim.tabId, ptyId: claim.ptyId, - worktreeId: worktree.id, + worktreeId: workspace.id, title, defaultTitle: title, customTitle: null, @@ -13341,12 +14530,12 @@ export class OrcaRuntimeService { } } const adoptedTabIds = [...new Set(validated.map(({ claim }) => claim.tabId))] - next.tabsByWorktree[worktree.id] = [...tabsById.values()] + next.tabsByWorktree[workspace.id] = [...tabsById.values()] const activeTabId = request.activeTabId && tabsById.has(request.activeTabId) ? request.activeTabId : (adoptedTabIds[0] ?? null) - const existingGroups = next.tabGroups?.[worktree.id] ?? [] + const existingGroups = next.tabGroups?.[workspace.id] ?? [] const targetGroupId = (request.activeGroupId && existingGroups.some((group) => group.id === request.activeGroupId) ? request.activeGroupId @@ -13355,7 +14544,7 @@ export class OrcaRuntimeService { randomUUID() const proposedGroups = topologyGroups.map((group) => ({ ...group, - worktreeId: worktree.id + worktreeId: workspace.id })) const groups = existingGroups.length === 0 && proposedGroups.length > 0 @@ -13390,14 +14579,14 @@ export class OrcaRuntimeService { (proposed) => !existingGroups.some((group) => group.id === proposed.id) ) ) - : [{ id: targetGroupId, worktreeId: worktree.id, activeTabId, tabOrder: adoptedTabIds }] + : [{ id: targetGroupId, worktreeId: workspace.id, activeTabId, tabOrder: adoptedTabIds }] const retainedGroups = groups.filter((group) => group.tabOrder.length > 0) next.tabGroups = { ...next.tabGroups, - [worktree.id]: retainedGroups + [workspace.id]: retainedGroups } const mergedGroupLayout = mergeTerminalOrphanGroupLayout({ - existingLayout: next.tabGroupLayouts?.[worktree.id], + existingLayout: next.tabGroupLayouts?.[workspace.id], existingGroupIds: existingGroups.map((group) => group.id), proposedLayout: request.topology?.groupLayout, proposedGroupIds: proposedGroups.map((group) => group.id), @@ -13406,7 +14595,7 @@ export class OrcaRuntimeService { if (mergedGroupLayout) { next.tabGroupLayouts = { ...next.tabGroupLayouts, - [worktree.id]: mergedGroupLayout + [workspace.id]: mergedGroupLayout } } const activeGroup = @@ -13425,37 +14614,46 @@ export class OrcaRuntimeService { : activeGroup.activeTabId next.activeTabIdByWorktree = { ...next.activeTabIdByWorktree, - ...(convergedActiveTabId ? { [worktree.id]: convergedActiveTabId } : {}) + ...(convergedActiveTabId ? { [workspace.id]: convergedActiveTabId } : {}) } next.activeGroupIdByWorktree = { ...next.activeGroupIdByWorktree, - [worktree.id]: activeGroup.id + [workspace.id]: activeGroup.id } - const persisted = advanceTerminalTopologyRevision(next, worktree.id) + const persisted = advanceTerminalTopologyRevision(next, workspace.id) try { - store.setWorkspaceSession(persisted) + this.setWorkspaceSessionForWorktree(workspace.id, persisted) store.flushOrThrow() } catch (error) { - store.setWorkspaceSession(session) + this.setWorkspaceSessionForWorktree(workspace.id, session) throw error } for (const { claim, pty, paneKey } of validated) { pty.tabId = claim.tabId pty.paneKey = paneKey } - this.hydrateHeadlessMobileSessionTabsFromWorkspaceSession(worktree.id, { + this.hydrateHeadlessMobileSessionTabsFromWorkspaceSession(workspace.id, { force: true, allowAttachedWindow: true, onlyRuntimeOwnedTerminals: true }) - this.notifyMobileSessionTabsChanged(worktree.id) + this.notifyMobileSessionTabsChanged(workspace.id) return { adopted: true, topologyRevision: persisted.terminalTopologyRevisionByRepoId?.[repoId] ?? currentRevision + 1, - snapshot: await this.listMobileSessionTabs(`id:${worktree.id}`) + snapshot: this.getTerminalOrphanAdoptionSnapshot(workspace.id) } } + private getTerminalOrphanAdoptionSnapshot(worktreeId: string): RuntimeMobileSessionTabsResult { + this.hydrateHeadlessMobileSessionTabsFromWorkspaceSession(worktreeId, { + allowAttachedWindow: true, + onlyRuntimeOwnedTerminals: true + }) + this.hydrateHeadlessMobileSessionTabsFromWorkspaceSession(worktreeId) + return this.getMobileSessionTabsForWorktree(worktreeId) + } + private buildTerminalVisualLayouts( terminals: RuntimeTerminalSummary[], worktreesById: Map, @@ -25767,17 +26965,42 @@ export class OrcaRuntimeService { targetWorktreeId: string | null = null, deadline?: number ): Promise | null> { + const inventory = await this.refreshPtyWorktreeRecordsWithControllerInventory( + resolvedWorktrees, + targetWorktreeId, + deadline + ) + return inventory ? new Set(inventory.livePtyIds) : null + } + + private async refreshPtyWorktreeRecordsWithControllerInventory( + resolvedWorktrees: ResolvedWorktree[], + targetWorktreeId: string | null = null, + deadline?: number, + connectionId?: string | null + ): Promise { if (targetWorktreeId === FLOATING_TERMINAL_WORKTREE_ID) { const targetedLiveness = this.refreshFloatingWorkspacePtyLiveness() if (targetedLiveness !== null) { - return targetedLiveness + return { + livePtyIds: targetedLiveness, + terminalIdentityByPtyId: new Map() + } } } if (!this.ptyController?.listProcesses) { return null } + const inventoryGeneration = this.ptyControllerInventorySequence + 1 + this.ptyControllerInventorySequence = inventoryGeneration + const providerKey = typeof connectionId === 'string' ? `ssh:${connectionId}` : 'local' + if (connectionId === undefined) { + this.ptyControllerAggregateInventoryGeneration = inventoryGeneration + } else { + this.ptyControllerInventoryGenerationByProvider.set(providerKey, inventoryGeneration) + } const sessionsResult = await withTimeoutResult( - this.ptyController.listProcesses(), + this.ptyController.listProcesses(connectionId), deadline === undefined ? PTY_CONTROLLER_LIST_TIMEOUT_MS : Math.max(1, Math.min(PTY_CONTROLLER_LIST_TIMEOUT_MS, deadline - Date.now())) @@ -25786,11 +27009,20 @@ export class OrcaRuntimeService { // Why: a transient controller failure is not evidence that retained PTYs exited. return null } + const isCurrentInventory = + connectionId === undefined + ? this.ptyControllerAggregateInventoryGeneration === inventoryGeneration && + ![...this.ptyControllerInventoryGenerationByProvider.values()].some( + (generation) => generation > inventoryGeneration + ) + : this.ptyControllerInventoryGenerationByProvider.get(providerKey) === + inventoryGeneration && + this.ptyControllerAggregateInventoryGeneration <= inventoryGeneration + if (!isCurrentInventory) { + return null + } const sessions = sessionsResult.value - const controllerIdentityByPtyId = new Map< - string, - { handle: string; incarnationId: string; wslDistro?: string | null } - >() + const controllerIdentityByPtyId = new Map() const ptyIdByControllerHandle = new Map() const ambiguousControllerPtyIds = new Set() for (const session of sessions) { @@ -25821,23 +27053,37 @@ export class OrcaRuntimeService { for (const ptyId of ambiguousControllerPtyIds) { controllerIdentityByPtyId.delete(ptyId) } - this.controllerTerminalIdentityByPtyId = controllerIdentityByPtyId - const persistedWorktreeIdByPtyId = indexPersistedPtyWorktreeBindings( - this.store?.getWorkspaceSession?.() - ) - const persistedSurfaceByPtyId = indexPersistedPtySurfaceBindings( - this.store?.getWorkspaceSession?.() - ) + const persistedIndexesByHostId = new Map< + ExecutionHostId, + { + worktreeIdByPtyId: ReadonlyMap + surfaceByPtyId: ReturnType + } + >() + const getPersistedIndexes = (hostId: ExecutionHostId) => { + const existing = persistedIndexesByHostId.get(hostId) + if (existing) { + return existing + } + const persistedSession = this.store?.getWorkspaceSession?.(hostId) + const indexes = { + worktreeIdByPtyId: indexPersistedPtyWorktreeBindings(persistedSession), + surfaceByPtyId: indexPersistedPtySurfaceBindings(persistedSession) + } + persistedIndexesByHostId.set(hostId, indexes) + return indexes + } const allLivePtyIds = new Set(sessions.map((session) => session.id)) const selectedLivePtyIds = new Set() for (const session of sessions) { - const controllerIdentity = controllerIdentityByPtyId.get(session.id) - this.adoptControllerTerminalHandle( - session.id, - controllerIdentity?.handle ?? session.terminalHandle, - controllerIdentity?.incarnationId ?? session.incarnationId + const sessionConnectionId = + parseAppSshPtyId(session.id)?.connectionId ?? + (typeof connectionId === 'string' ? connectionId : null) + const persistedIndexes = getPersistedIndexes( + sessionConnectionId ? toSshExecutionHostId(sessionConnectionId) : LOCAL_EXECUTION_HOST_ID ) - const persistedWorktreeId = persistedWorktreeIdByPtyId.get(session.id) + const controllerIdentity = controllerIdentityByPtyId.get(session.id) + const persistedWorktreeId = persistedIndexes.worktreeIdByPtyId.get(session.id) const providerWorktree = resolvedWorktrees.find( (worktree) => session.worktreeId && runtimeWorktreeIdsEqual(worktree.id, session.worktreeId) ) @@ -25862,6 +27108,19 @@ export class OrcaRuntimeService { persistedWorktree?.id ?? inferredWorktreeId ?? findResolvedWorktreeIdForPath(resolvedWorktrees, session.cwd)) + const persistedSurface = persistedIndexes.surfaceByPtyId.get(session.id) + const restoresExactSurface = + persistedSurface && + session.incarnationId && + persistedSurface.incarnationId === session.incarnationId && + Boolean(worktreeId) && + runtimeWorktreeIdsEqual(persistedSurface.worktreeId, worktreeId as string) + this.adoptControllerTerminalHandle( + session.id, + controllerIdentity?.handle ?? session.terminalHandle, + controllerIdentity?.incarnationId ?? session.incarnationId, + { exactRestoredSurface: Boolean(restoresExactSurface && controllerIdentity) } + ) if ( !targetWorktreeId || (worktreeId && runtimeWorktreeIdsEqual(worktreeId, targetWorktreeId)) @@ -25872,15 +27131,14 @@ export class OrcaRuntimeService { targetWorktreeId && (!worktreeId || !runtimeWorktreeIdsEqual(worktreeId, targetWorktreeId)) ) { + const receipt = this.restoredOrchestrationAuthorityByPtyId.get(session.id) + if (receipt && runtimeWorktreeIdsEqual(receipt.worktreeId, targetWorktreeId)) { + this.restoredOrchestrationAuthorityByPtyId.delete(session.id) + } continue } + this.restoredOrchestrationAuthorityByPtyId.delete(session.id) if (worktreeId) { - const persistedSurface = persistedSurfaceByPtyId.get(session.id) - const restoresExactSurface = - persistedSurface && - session.incarnationId && - persistedSurface.incarnationId === session.incarnationId && - runtimeWorktreeIdsEqual(persistedSurface.worktreeId, worktreeId) const pty = this.recordPtyWorktree(session.id, worktreeId, { connected: true, ...(session.incarnationId ? { incarnationId: session.incarnationId } : {}), @@ -25891,12 +27149,35 @@ export class OrcaRuntimeService { ? { tabId: persistedSurface.tabId, paneKey: persistedSurface.paneKey } : {}) }) + if (restoresExactSurface && controllerIdentity) { + this.rememberRestoredOrchestrationAuthority( + pty, + controllerIdentity.handle, + controllerIdentity.incarnationId + ) + } else { + this.restoredOrchestrationAuthorityByPtyId.delete(session.id) + } pty.controllerTitle = session.title?.trim() || null } // Why: fire-and-forget so this listing hot path doesn't serialize a relay round-trip per session and a throw can't abort the sweep below. this.refreshPtyForegroundAgent(session.id) } + for (const [ptyId, receipt] of this.restoredOrchestrationAuthorityByPtyId) { + const inScope = + connectionId === undefined || + (connectionId === null && receipt.hostScope.kind !== 'ssh') || + (typeof connectionId === 'string' && + receipt.hostScope.kind === 'ssh' && + receipt.hostScope.targetId === connectionId) + if (inScope && !allLivePtyIds.has(ptyId)) { + this.restoredOrchestrationAuthorityByPtyId.delete(ptyId) + } + } for (const pty of this.ptysById.values()) { + if (connectionId !== undefined && pty.connectionId !== connectionId) { + continue + } if (!allLivePtyIds.has(pty.ptyId) && !this.leafExistsForPty(pty.ptyId)) { if (this.ptyController.hasPty?.(pty.ptyId) === true) { // Why: an SSH spawn can become addressable before an overlapping relay list includes it. @@ -25916,7 +27197,10 @@ export class OrcaRuntimeService { } } this.pruneDisconnectedPtyRecords() - return targetWorktreeId ? selectedLivePtyIds : allLivePtyIds + return { + livePtyIds: targetWorktreeId ? selectedLivePtyIds : allLivePtyIds, + terminalIdentityByPtyId: controllerIdentityByPtyId + } } private refreshFloatingWorkspacePtyLiveness(): Set | null { @@ -26060,6 +27344,7 @@ export class OrcaRuntimeService { // Why: pruning can remove a PTY without onPtyExit firing; release this leader's agent team so it doesn't leak. this.claudeAgentTeams.removeTeamForLeaderHandle(handle) this.handleByPtyId.delete(ptyId) + this.syntheticTerminalHandles.delete(handle) const record = this.handles.get(handle) if (record?.tabId.startsWith('pty:')) { this.handles.delete(handle) @@ -27514,7 +28799,7 @@ export class OrcaRuntimeService { resolve('waiter_exists') return } - const timeoutMs = options?.timeoutMs ?? MESSAGE_WAIT_DEFAULT_TIMEOUT_MS + const timeoutMs = options?.timeoutMs ?? ORCHESTRATION_MESSAGE_WAIT_DEFAULT_TIMEOUT_MS const waiter: MessageWaiter = { handle, @@ -27710,7 +28995,11 @@ export class OrcaRuntimeService { } } - const handle = this.adoptPreAllocatedHandle(leaf) ?? `term_${randomUUID()}` + const preAllocatedHandle = this.adoptPreAllocatedHandle(leaf) + const handle = preAllocatedHandle ?? `term_${randomUUID()}` + if (!preAllocatedHandle) { + this.syntheticTerminalHandles.add(handle) + } if (this.handles.has(handle)) { return handle } @@ -27767,6 +29056,9 @@ export class OrcaRuntimeService { } const handle = existingHandle ?? `term_${randomUUID()}` + if (!existingHandle) { + this.syntheticTerminalHandles.add(handle) + } const syntheticId = `pty:${pty.ptyId}` this.handles.set(handle, { handle, @@ -27808,6 +29100,7 @@ export class OrcaRuntimeService { } this.handleByLeafKey.delete(leafKey) this.handles.delete(handle) + this.syntheticTerminalHandles.delete(handle) this.rejectWaitersForHandle(handle, 'terminal_handle_stale') } @@ -32542,7 +33835,6 @@ async function assertTerminalInputWithinLimitWithYield(text: string | undefined) const TUI_IDLE_DEFAULT_TIMEOUT_MS = 5 * 60 * 1000 const TUI_IDLE_POLL_INTERVAL_MS = 2000 const TUI_IDLE_QUIESCENCE_MS = 3000 -const MESSAGE_WAIT_DEFAULT_TIMEOUT_MS = 2 * 60 * 1000 const EXPLICIT_IDLE_TITLE_RE = /(^|\s)(ready|idle|done)(\s|$|[.!?])/i const CLAUDE_IDLE_PREFIX = '\u2733' const GEMINI_IDLE_PREFIX = '\u25c7' diff --git a/src/main/runtime/orchestration-compatibility-authority.test.ts b/src/main/runtime/orchestration-compatibility-authority.test.ts new file mode 100644 index 00000000000..082d6e698ae --- /dev/null +++ b/src/main/runtime/orchestration-compatibility-authority.test.ts @@ -0,0 +1,350 @@ +import { createHash } from 'node:crypto' +import { describe, expect, it, vi } from 'vitest' +import { AgentHookServer } from '../agent-hooks/server' +import { OrcaRuntimeService } from './orca-runtime' + +const PANE_KEY = '11111111-1111-4111-8111-111111111111:22222222-2222-4222-8222-222222222222' +const TOKEN = 'launch-secret' +const TOKEN_HASH = createHash('sha256').update(TOKEN).digest('hex') + +type TerminalAuthorityResolver = { + getOrchestrationDispatchAuthority: (terminalHandle: string) => unknown + restoredOrchestrationAuthorityByPtyId: Map> +} + +function createRuntime( + hostScope: + | { kind: 'local'; hostId: 'local' } + | { kind: 'wsl'; hostId: 'local'; distro: string } + | { kind: 'ssh'; targetId: string }, + launchTokenHash: string | null = TOKEN_HASH +) { + const runtime = new OrcaRuntimeService(null, undefined, { + attestAgentHookCompatibilityAuthority: ({ paneKey, launchTokenHash, connectionId }) => + paneKey === PANE_KEY && + launchTokenHash === TOKEN_HASH && + connectionId === (hostScope.kind === 'ssh' ? hostScope.targetId : null) + ? { paneKey, source: 'hydrated_commitment' } + : null + }) + const resolveTerminal = vi.fn(() => ({ + runtimeId: 'runtime-1', + terminalHandle: 'term-1', + ptyId: 'pty-1', + worktreeId: 'repo-1::/worktree', + processIncarnation: 'incarnation-1', + paneKey: PANE_KEY, + launchTokenHash, + hostScope + })) + ;(runtime as unknown as TerminalAuthorityResolver).getOrchestrationDispatchAuthority = + resolveTerminal + if (launchTokenHash === null) { + ;(runtime as unknown as TerminalAuthorityResolver).restoredOrchestrationAuthorityByPtyId.set( + 'pty-1', + { + ptyId: 'pty-1', + worktreeId: 'repo-1::/worktree', + terminalHandle: 'term-1', + paneKey: PANE_KEY, + processIncarnation: 'incarnation-1', + hostScope + } + ) + } + return runtime +} + +describe('orchestration compatibility runtime authority', () => { + it('returns only attested local identity and its token hash', () => { + const runtime = createRuntime({ kind: 'local', hostId: 'local' }) + + const authority = runtime.verifyOrchestrationCompatibilityCaller({ + terminalHandle: 'term-1', + paneKey: PANE_KEY, + launchToken: TOKEN + }) + + expect(authority).toEqual({ + hostScope: { kind: 'local', hostId: 'local' }, + paneKey: PANE_KEY, + terminalHandle: 'term-1', + processIncarnation: 'incarnation-1', + launchTokenHash: TOKEN_HASH + }) + expect(JSON.stringify(authority)).not.toContain(TOKEN) + expect( + runtime.verifyOrchestrationCompatibilityCaller({ + terminalHandle: 'term-1', + paneKey: PANE_KEY, + launchToken: 'wrong' + }) + ).toBeNull() + expect( + runtime.verifyOrchestrationCompatibilityCaller({ + terminalHandle: 'term-1', + paneKey: PANE_KEY, + launchToken: TOKEN_HASH + }) + ).toBeNull() + }) + + it('keeps local and WSL authority stable across app runtime generations', () => { + const firstLocal = createRuntime({ kind: 'local', hostId: 'local' }) + const secondLocal = createRuntime({ kind: 'local', hostId: 'local' }) + const wsl = createRuntime({ kind: 'wsl', hostId: 'local', distro: 'Ubuntu' }) + const localEvidence = { + terminalHandle: 'term-1', + paneKey: PANE_KEY, + launchToken: TOKEN + } + const wslEvidence = { + ...localEvidence, + host: { kind: 'wsl', hostId: 'local', distro: 'Ubuntu' } + } as const + + expect(firstLocal.verifyOrchestrationCompatibilityCaller(localEvidence)?.hostScope).toEqual( + secondLocal.verifyOrchestrationCompatibilityCaller(localEvidence)?.hostScope + ) + expect(wsl.verifyOrchestrationCompatibilityCaller(wslEvidence)?.hostScope).toEqual({ + kind: 'wsl', + hostId: 'local', + distro: 'Ubuntu' + }) + expect(wsl.verifyOrchestrationCompatibilityCaller(localEvidence)).toBeNull() + expect( + wsl.verifyOrchestrationCompatibilityCaller({ + ...wslEvidence, + host: { kind: 'wsl', hostId: 'runtime-before-restart', distro: 'Ubuntu' } + }) + ).toBeNull() + }) + + it('requires a live exact terminal even when the hook proof is hydrated', () => { + const runtime = createRuntime({ kind: 'local', hostId: 'local' }) + ;(runtime as unknown as TerminalAuthorityResolver).getOrchestrationDispatchAuthority = () => + null + + expect( + runtime.verifyOrchestrationCompatibilityCaller({ + terminalHandle: 'term-1', + paneKey: PANE_KEY, + launchToken: TOKEN + }) + ).toBeNull() + }) + + it('uses the hydrated hook commitment for a restored exact PTY', () => { + const restored = createRuntime({ kind: 'local', hostId: 'local' }, null) + const uncommitted = new OrcaRuntimeService() + ;(uncommitted as unknown as TerminalAuthorityResolver).getOrchestrationDispatchAuthority = + () => ({ + runtimeId: 'runtime-1', + terminalHandle: 'term-1', + ptyId: 'pty-1', + worktreeId: 'repo-1::/worktree', + processIncarnation: 'incarnation-1', + paneKey: PANE_KEY, + launchTokenHash: null, + hostScope: { kind: 'local', hostId: 'local' } + }) + const evidence = { + terminalHandle: 'term-1', + paneKey: PANE_KEY, + launchToken: TOKEN + } + + expect(restored.verifyOrchestrationCompatibilityCaller(evidence)).toMatchObject({ + processIncarnation: 'incarnation-1', + launchTokenHash: TOKEN_HASH + }) + expect(uncommitted.verifyOrchestrationCompatibilityCaller(evidence)).toBeNull() + expect( + restored.verifyOrchestrationCompatibilityCaller({ + ...evidence, + launchToken: 'wrong' + }) + ).toBeNull() + }) + + it.each([ + ['PTY', { ptyId: 'pty-other' }], + ['worktree', { worktreeId: 'repo-1::/other' }], + ['terminal handle', { terminalHandle: 'term-other' }], + [ + 'pane', + { paneKey: '33333333-3333-4333-8333-333333333333:44444444-4444-4444-8444-444444444444' } + ], + ['process incarnation', { processIncarnation: 'incarnation-other' }] + ])('rejects a restored receipt with mismatched %s identity', (_field, mismatch) => { + const runtime = createRuntime({ kind: 'local', hostId: 'local' }, null) + const internals = runtime as unknown as TerminalAuthorityResolver + const receipt = internals.restoredOrchestrationAuthorityByPtyId.get('pty-1')! + internals.restoredOrchestrationAuthorityByPtyId.set('pty-1', { ...receipt, ...mismatch }) + + expect( + runtime.verifyOrchestrationCompatibilityCaller({ + terminalHandle: 'term-1', + paneKey: PANE_KEY, + launchToken: TOKEN + }) + ).toBeNull() + }) + + it('rejects restored receipts from a different WSL distro or SSH target', () => { + const wsl = createRuntime({ kind: 'wsl', hostId: 'local', distro: 'Ubuntu' }, null) + const wslInternals = wsl as unknown as TerminalAuthorityResolver + const wslReceipt = wslInternals.restoredOrchestrationAuthorityByPtyId.get('pty-1')! + wslInternals.restoredOrchestrationAuthorityByPtyId.set('pty-1', { + ...wslReceipt, + hostScope: { kind: 'wsl', hostId: 'local', distro: 'Debian' } + }) + expect( + wsl.verifyOrchestrationCompatibilityCaller({ + terminalHandle: 'term-1', + paneKey: PANE_KEY, + launchToken: TOKEN, + host: { kind: 'wsl', hostId: 'local', distro: 'Ubuntu' } + }) + ).toBeNull() + + const ssh = createRuntime({ kind: 'ssh', targetId: 'saved-target' }, null) + const sshInternals = ssh as unknown as TerminalAuthorityResolver + const sshReceipt = sshInternals.restoredOrchestrationAuthorityByPtyId.get('pty-1')! + sshInternals.restoredOrchestrationAuthorityByPtyId.set('pty-1', { + ...sshReceipt, + hostScope: { kind: 'ssh', targetId: 'other-target' } + }) + const host = ssh.registerOrchestrationCompatibilitySshAttachment('saved-target', 'connection-1') + expect( + ssh.verifyOrchestrationCompatibilityCaller({ + terminalHandle: 'term-1', + paneKey: PANE_KEY, + launchToken: TOKEN, + host + }) + ).toBeNull() + }) + + it('does not fall back to a restored receipt when a fresh launch token mismatches', () => { + const runtime = createRuntime({ kind: 'local', hostId: 'local' }) + ;(runtime as unknown as TerminalAuthorityResolver).restoredOrchestrationAuthorityByPtyId.set( + 'pty-1', + { + ptyId: 'pty-1', + worktreeId: 'repo-1::/worktree', + terminalHandle: 'term-1', + paneKey: PANE_KEY, + processIncarnation: 'incarnation-1', + hostScope: { kind: 'local', hostId: 'local' } + } + ) + + expect( + runtime.verifyOrchestrationCompatibilityCaller({ + terminalHandle: 'term-1', + paneKey: PANE_KEY, + launchToken: 'wrong' + }) + ).toBeNull() + }) + + it('attests current coordinators immediately without upgrading restored legacy callers', () => { + const server = new AgentHookServer() + server.ingestRemote( + { + paneKey: PANE_KEY, + launchToken: TOKEN, + payload: { state: 'working', prompt: 'coordinate', agentType: 'codex' } + }, + 'saved-target' + ) + const createIntegratedRuntime = (launchTokenHash: string | null): OrcaRuntimeService => { + const runtime = new OrcaRuntimeService(null, undefined, { + attestAgentHookCompatibilityAuthority: (candidate) => + server.attestCompatibilityAuthority(candidate) + }) + ;(runtime as unknown as TerminalAuthorityResolver).getOrchestrationDispatchAuthority = vi.fn( + () => ({ + runtimeId: 'runtime-1', + terminalHandle: 'term-1', + ptyId: 'pty-1', + worktreeId: 'repo-1::/worktree', + processIncarnation: 'incarnation-1', + paneKey: PANE_KEY, + launchTokenHash, + hostScope: { kind: 'ssh', targetId: 'saved-target' } + }) + ) + if (launchTokenHash === null) { + ;( + runtime as unknown as TerminalAuthorityResolver + ).restoredOrchestrationAuthorityByPtyId.set('pty-1', { + ptyId: 'pty-1', + worktreeId: 'repo-1::/worktree', + terminalHandle: 'term-1', + paneKey: PANE_KEY, + processIncarnation: 'incarnation-1', + hostScope: { kind: 'ssh', targetId: 'saved-target' } + }) + } + return runtime + } + const evidenceFor = (runtime: OrcaRuntimeService, launchToken = TOKEN) => ({ + terminalHandle: 'term-1', + paneKey: PANE_KEY, + launchToken, + host: runtime.registerOrchestrationCompatibilitySshAttachment('saved-target', 'connection-1') + }) + + const current = createIntegratedRuntime(TOKEN_HASH) + expect(current.verifyOrchestrationCompatibilityCaller(evidenceFor(current))).not.toBeNull() + + const restored = createIntegratedRuntime(null) + expect(restored.verifyOrchestrationCompatibilityCaller(evidenceFor(restored))).toBeNull() + + server.ingestRemote( + { + paneKey: '33333333-3333-4333-8333-333333333333:44444444-4444-4444-8444-444444444444', + launchToken: TOKEN, + payload: { state: 'working', prompt: 'duplicate', agentType: 'codex' } + }, + 'saved-target' + ) + expect(current.verifyOrchestrationCompatibilityCaller(evidenceFor(current))).toBeNull() + + const mismatchedToken = 'different-launch-secret' + const mismatched = createIntegratedRuntime( + createHash('sha256').update(mismatchedToken).digest('hex') + ) + expect( + mismatched.verifyOrchestrationCompatibilityCaller(evidenceFor(mismatched, mismatchedToken)) + ).toBeNull() + }) + + it('accepts only a live runtime-issued SSH attachment', () => { + const runtime = createRuntime({ kind: 'ssh', targetId: 'saved-target' }) + const host = runtime.registerOrchestrationCompatibilitySshAttachment( + 'saved-target', + 'connection-1' + ) + const evidence = { + terminalHandle: 'term-1', + paneKey: PANE_KEY, + launchToken: TOKEN, + host + } as const + + expect(runtime.verifyOrchestrationCompatibilityCaller(evidence)).not.toBeNull() + expect( + runtime.verifyOrchestrationCompatibilityCaller({ + ...evidence, + host: { ...host, attachmentId: 'caller-chosen' } + }) + ).toBeNull() + + runtime.releaseOrchestrationCompatibilitySshAttachment(host.attachmentId) + + expect(runtime.verifyOrchestrationCompatibilityCaller(evidence)).toBeNull() + }) +}) diff --git a/src/main/runtime/orchestration/db.test.ts b/src/main/runtime/orchestration/db.test.ts index 73e0468d011..4c89c2c10a3 100644 --- a/src/main/runtime/orchestration/db.test.ts +++ b/src/main/runtime/orchestration/db.test.ts @@ -942,7 +942,7 @@ describe('OrchestrationDb', () => { // v1 data preserved expect(d.getMessageById('msg_v1')?.subject).toBe('pre-migration') - expect(d.getMessageById('msg_v1')?.run_id).toBe(LEGACY_RUN_ID) + expect(d.getMessageById('msg_v1')?.run_id).toBe(d.getLegacyAdoption()?.adopted_run_id) expect(d.getRun(LEGACY_RUN_ID)).toMatchObject({ legacy: 1 }) }) diff --git a/src/main/runtime/orchestration/db.ts b/src/main/runtime/orchestration/db.ts index dfcdc81f567..a80ed4ac780 100644 --- a/src/main/runtime/orchestration/db.ts +++ b/src/main/runtime/orchestration/db.ts @@ -5,6 +5,7 @@ import Database from '../../sqlite/sync-database' import type { MessageType, MessagePriority, + MessageDeliveryContract, TaskStatus, DispatchStatus, GateStatus, @@ -19,12 +20,18 @@ import type { RunRow, DeliveryRow, DeliveryStatus, + LegacyAdoptionRow, + LegacyCompatibilityPrincipalRow, + LegacyPrincipalRole, + LegacyOperationReceiptRow, + LegacyMailReceiptRow, QuestionRow, QuestionStatus, MutationReceiptRow, MutationState, WorkerDispatchRow, WorkerDispatchState, + LegacyWorkerTerminalRecoveryRow, FederatedDispatchRow, RemoteDispatchAttachmentRow, FederationRelayDirection, @@ -35,6 +42,7 @@ import { ORCHESTRATION_LEGACY_RUN_ID } from '../../../shared/orchestration-rpc-c import { parsePaneKey } from '../../../shared/stable-pane-id' import { OrchestrationError } from './orchestration-error' import { resolveOrchestrationMigrationStartVersion } from './orchestration-schema-version-skew' +import { ORCHESTRATION_CONTRACT_VERSION } from '../../../shared/protocol-version' // Why: leaf UUID is the remint-stable pane identity (tab half changes on break-out); exact match covers legacy/unparseable keys. function isEquivalentPaneKey(a: string, b: string): boolean { @@ -49,6 +57,7 @@ function isEquivalentPaneKey(a: string, b: string): boolean { export type { MessageType, MessagePriority, + MessageDeliveryContract, TaskStatus, DispatchStatus, GateStatus, @@ -63,6 +72,11 @@ export type { RunRow, DeliveryRow, DeliveryStatus, + LegacyAdoptionRow, + LegacyCompatibilityPrincipalRow, + LegacyPrincipalRole, + LegacyOperationReceiptRow, + LegacyMailReceiptRow, QuestionRow, QuestionStatus, MutationReceiptRow, @@ -95,6 +109,25 @@ function addLifecycleRejectionMarker(payload: string | null, code: string, reaso }) } +function hasLifecycleRejectionMarker(payload: string | null): boolean { + try { + const value: unknown = JSON.parse(payload ?? 'null') + if (!value || typeof value !== 'object' || Array.isArray(value)) { + return false + } + const marker = (value as Record)._orcaLifecycleRejection + return Boolean( + marker && + typeof marker === 'object' && + !Array.isArray(marker) && + typeof (marker as Record).code === 'string' && + typeof (marker as Record).reason === 'string' + ) + } catch { + return false + } +} + const SQLITE_UTC_TIMESTAMP_RE = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:\.\d+)?$/ function exposeUtcTimestamp(timestamp: string | null): string | null { @@ -142,10 +175,46 @@ function exposeQuestionTimestamps(question: QuestionRow): QuestionRow { } } +function normalizeLegacyQuestionText(value: string): string { + return value.replace(/\r\n/g, '\n').trim() +} + +function normalizeLegacyQuestionOptions(options: unknown): string { + if (!Array.isArray(options) || !options.every((option) => typeof option === 'string')) { + return '[]' + } + return JSON.stringify(options.map((option) => option.trim())) +} + +function legacyMessageMatchesQuestion( + message: MessageRow, + question: string, + options: string[], + recipientHandles: readonly string[] +): boolean { + if ( + !recipientHandles.includes(message.to_handle) || + normalizeLegacyQuestionText(message.body) !== normalizeLegacyQuestionText(question) + ) { + return false + } + try { + const payload = JSON.parse(message.payload ?? '{}') as { options?: unknown } + return ( + normalizeLegacyQuestionOptions(payload.options) === normalizeLegacyQuestionOptions(options) + ) + } catch { + return false + } +} + export const LEGACY_RUN_ID = ORCHESTRATION_LEGACY_RUN_ID -// Schema versions: v2 'heartbeat'+last_heartbeat_at, v3 delivered_at, v4 task-creator terminal, v5 task_title/display_name, v6 pane identity, v7 lightweight Runs, v8 crash-safe Run deliveries, v9 durable question threads, v10 Dispatch capabilities, v11 durable mutation receipts, v12 composed worker state, v18 post-v6 version-skew repair. -const SCHEMA_VERSION = 18 +export const LEGACY_CONTRACT_VERSION = 0 +export const CURRENT_CONTRACT_VERSION = ORCHESTRATION_CONTRACT_VERSION + +// Schema versions: v2 'heartbeat'+last_heartbeat_at, v3 delivered_at, v4 task-creator terminal, v5 task_title/display_name, v6 pane identity, v7 lightweight Runs, v8 crash-safe Run deliveries, v9 durable question threads, v10 Dispatch capabilities, v11 durable mutation receipts, v12 composed worker state, v18 post-v6 version-skew repair, v19 adopted legacy Runs and compatibility receipts, v20 legacy question backfill, v21 legacy scheduler-loss provenance. +const SCHEMA_VERSION = 21 function hardenOrchestrationDatabaseFiles(dbPath: string | ':memory:'): void { if (dbPath === ':memory:' || process.platform === 'win32') { @@ -197,6 +266,8 @@ export class OrchestrationDb { CREATE TABLE IF NOT EXISTS messages ( id TEXT NOT NULL, run_id TEXT NOT NULL DEFAULT '${LEGACY_RUN_ID}', + delivery_contract TEXT NOT NULL DEFAULT 'current_delivery' + CHECK(delivery_contract IN ('legacy_direct', 'current_delivery', 'audit_only')), from_handle TEXT NOT NULL, to_handle TEXT NOT NULL, subject TEXT NOT NULL, @@ -367,6 +438,8 @@ export class OrchestrationDb { id TEXT PRIMARY KEY, run_id TEXT NOT NULL DEFAULT '${LEGACY_RUN_ID}', task_id TEXT NOT NULL, + contract_version INTEGER NOT NULL DEFAULT ${CURRENT_CONTRACT_VERSION}, + launch_token_hash TEXT, assignee_handle TEXT, assignee_pane_key TEXT, capability_hash TEXT, @@ -409,7 +482,8 @@ export class OrchestrationDb { coordinator_handle TEXT NOT NULL, poll_interval_ms INTEGER NOT NULL DEFAULT 2000, created_at TEXT NOT NULL DEFAULT (datetime('now')), - completed_at TEXT + completed_at TEXT, + scheduler_lost_at TEXT ); `) this.createUndeliveredInboxIndexIfPossible() @@ -427,7 +501,7 @@ export class OrchestrationDb { return } - this.db.exec('BEGIN') + this.db.exec('BEGIN IMMEDIATE') try { // v1 → v2: SQLite can't ALTER a CHECK, so rebuild messages to allow 'heartbeat'; fold in v3's delivered_at to skip a second rebuild. if (current < 2) { @@ -752,6 +826,15 @@ export class OrchestrationDb { 'ALTER TABLE remote_dispatch_attachments ADD COLUMN protocol_version INTEGER NOT NULL DEFAULT 1' ) } + if (current < 19) { + this.migrateLegacyContractStorage() + } + if (current < 20) { + this.backfillLegacyQuestionThreads() + } + if (current < 21) { + this.migrateLegacySchedulerLossProvenance() + } this.createUndeliveredInboxIndexIfPossible() this.db.pragma(`user_version = ${SCHEMA_VERSION}`) @@ -762,6 +845,374 @@ export class OrchestrationDb { } } + private migrateLegacyContractStorage(): void { + if (!this.hasColumn('dispatch_contexts', 'contract_version')) { + this.db.exec( + `ALTER TABLE dispatch_contexts + ADD COLUMN contract_version INTEGER NOT NULL DEFAULT ${CURRENT_CONTRACT_VERSION}` + ) + } + if (!this.hasColumn('dispatch_contexts', 'launch_token_hash')) { + this.db.exec('ALTER TABLE dispatch_contexts ADD COLUMN launch_token_hash TEXT') + } + if (!this.hasColumn('messages', 'delivery_contract')) { + this.db.exec( + `ALTER TABLE messages + ADD COLUMN delivery_contract TEXT NOT NULL DEFAULT 'current_delivery' + CHECK(delivery_contract IN ('legacy_direct', 'current_delivery', 'audit_only'))` + ) + } + this.db.exec(` + CREATE INDEX IF NOT EXISTS idx_messages_delivery_contract + ON messages(run_id, delivery_contract, to_handle, read, sequence); + + CREATE TABLE IF NOT EXISTS legacy_adoptions ( + source_run_id TEXT PRIMARY KEY, + adopted_run_id TEXT UNIQUE NOT NULL, + scheduler_state_lost INTEGER NOT NULL, + adopted_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + + CREATE TABLE IF NOT EXISTS legacy_compatibility_principals ( + id TEXT PRIMARY KEY, + run_id TEXT NOT NULL, + dispatch_id TEXT, + role TEXT NOT NULL CHECK(role IN ('worker', 'coordinator')), + host_scope TEXT NOT NULL, + terminal_handle TEXT NOT NULL, + pane_key TEXT NOT NULL, + launch_token_hash TEXT NOT NULL, + process_incarnation TEXT, + status TEXT NOT NULL + CHECK(status IN ('committed', 'settled', 'revoked')), + CHECK( + (role = 'worker' AND dispatch_id IS NOT NULL) OR + (role = 'coordinator' AND dispatch_id IS NULL) + ), + UNIQUE(role, run_id, dispatch_id) + ); + + CREATE UNIQUE INDEX IF NOT EXISTS idx_legacy_principal_coordinator + ON legacy_compatibility_principals(run_id) + WHERE role = 'coordinator'; + CREATE UNIQUE INDEX IF NOT EXISTS idx_legacy_principal_dispatch + ON legacy_compatibility_principals(dispatch_id) + WHERE role = 'worker'; + + CREATE TABLE IF NOT EXISTS legacy_operation_receipts ( + principal_id TEXT NOT NULL, + operation_key TEXT NOT NULL, + method TEXT NOT NULL, + payload_hash TEXT NOT NULL, + effect_id TEXT NOT NULL, + response_json TEXT NOT NULL, + completed_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY(principal_id, operation_key) + ); + + CREATE TABLE IF NOT EXISTS legacy_mail_receipts ( + principal_id TEXT NOT NULL, + message_id TEXT NOT NULL, + acknowledged_at TEXT, + PRIMARY KEY(principal_id, message_id) + ); + `) + + this.db + .prepare( + `UPDATE dispatch_contexts + SET contract_version = ? + WHERE run_id = ? AND capability_hash IS NULL` + ) + .run(LEGACY_CONTRACT_VERSION, LEGACY_RUN_ID) + this.classifyLegacyMessageContracts(LEGACY_RUN_ID, false) + this.ensureLegacySchedulerLossColumn() + this.adoptLegacyRunIfNeeded() + } + + private classifyLegacyMessageContracts(runId: string, adoptedOnly: boolean): void { + const contractFilter = adoptedOnly + ? " AND delivery_contract IN ('legacy_direct', 'audit_only')" + : '' + this.db + .prepare( + `UPDATE messages SET delivery_contract = 'legacy_direct' + WHERE run_id = ?${contractFilter}` + ) + .run(runId) + const rows = this.db + .prepare(`SELECT id, payload FROM messages WHERE run_id = ?${contractFilter}`) + .all(runId) as { id: string; payload: string | null }[] + const markAuditOnly = this.db.prepare( + "UPDATE messages SET delivery_contract = 'audit_only' WHERE id = ? AND run_id = ?" + ) + for (const row of rows) { + if (hasLifecycleRejectionMarker(row.payload)) { + markAuditOnly.run(row.id, runId) + } + } + } + + private migrateLegacySchedulerLossProvenance(): void { + this.ensureLegacySchedulerLossColumn() + this.adoptLegacyRunIfNeeded() + const adoption = this.getLegacyAdoption() + if (adoption) { + this.classifyLegacyMessageContracts(adoption.adopted_run_id, true) + } + } + + private ensureLegacySchedulerLossColumn(): void { + if (!this.hasColumn('coordinator_runs', 'scheduler_lost_at')) { + this.db.exec('ALTER TABLE coordinator_runs ADD COLUMN scheduler_lost_at TEXT') + } + } + + private backfillLegacyQuestionThreads(): void { + const messages = this.db + .prepare( + `SELECT id, run_id, from_handle, to_handle, payload, created_at, sequence + FROM messages + WHERE type = 'decision_gate' + AND delivery_contract IN ('legacy_direct', 'current_delivery') + ORDER BY sequence` + ) + .all() as { + id: string + run_id: string + from_handle: string + to_handle: string + payload: string | null + created_at: string + sequence: number + }[] + const getDispatch = this.db.prepare( + 'SELECT id, run_id, task_id FROM dispatch_contexts WHERE id = ? AND contract_version = ?' + ) + const getDispatchesForLegacyQuestion = this.db.prepare( + `SELECT id, run_id, task_id + FROM dispatch_contexts + WHERE contract_version = ? AND assignee_handle = ? + AND (? IS NULL OR task_id = ?) + AND created_at <= ? + AND (completed_at IS NULL OR completed_at >= ?) + ORDER BY rowid + LIMIT 2` + ) + const getAnswer = this.db.prepare( + `SELECT id, body, created_at + FROM messages + WHERE run_id = ? + AND thread_id = ? + AND delivery_contract IN ('legacy_direct', 'current_delivery') + AND from_handle = ? + AND to_handle IN (?, ?) + AND sequence > ? + ORDER BY sequence + LIMIT 1` + ) + const insert = this.db.prepare( + `INSERT OR IGNORE INTO question_threads ( + message_id, run_id, dispatch_id, asker_handle, status, + answer_message_id, answer_body, created_at, answered_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` + ) + for (const message of messages) { + let payload: { taskId?: unknown; dispatchId?: unknown } + try { + payload = JSON.parse(message.payload ?? '{}') as { + taskId?: unknown + dispatchId?: unknown + } + } catch { + continue + } + const inferredDispatches = + typeof payload.dispatchId === 'string' + ? [] + : (getDispatchesForLegacyQuestion.all( + LEGACY_CONTRACT_VERSION, + message.from_handle, + typeof payload.taskId === 'string' ? payload.taskId : null, + typeof payload.taskId === 'string' ? payload.taskId : null, + message.created_at, + message.created_at + ) as { id: string; run_id: string; task_id: string }[]) + const dispatch = + typeof payload.dispatchId === 'string' + ? (getDispatch.get(payload.dispatchId, LEGACY_CONTRACT_VERSION) as + | { id: string; run_id: string; task_id: string } + | undefined) + : inferredDispatches.length === 1 + ? inferredDispatches[0] + : undefined + if ( + !dispatch || + (typeof payload.taskId === 'string' && payload.taskId !== dispatch.task_id) || + (message.run_id !== LEGACY_RUN_ID && message.run_id !== dispatch.run_id) + ) { + continue + } + const answer = getAnswer.get( + message.run_id, + message.id, + message.to_handle, + message.from_handle, + `dispatch:${dispatch.id}`, + message.sequence + ) as { id: string; body: string; created_at: string } | undefined + insert.run( + message.id, + dispatch.run_id, + dispatch.id, + message.from_handle, + answer ? 'answered' : 'pending', + answer?.id ?? null, + answer?.body ?? null, + message.created_at, + answer?.created_at ?? null + ) + } + const adoption = this.getLegacyAdoption() + const coordinator = adoption + ? this.getLegacyCoordinatorPrincipal(adoption.adopted_run_id) + : undefined + if (adoption && coordinator?.status === 'revoked') { + this.promoteLegacyCoordinatorMailForTakeover( + adoption.adopted_run_id, + coordinator.terminal_handle + ) + } + } + + private adoptLegacyRunIfNeeded(): void { + const existing = this.db + .prepare('SELECT * FROM legacy_adoptions WHERE source_run_id = ?') + .get(LEGACY_RUN_ID) as LegacyAdoptionRow | undefined + const hasGraph = this.db + .prepare( + `SELECT 1 + WHERE EXISTS(SELECT 1 FROM tasks WHERE run_id = ?) + OR EXISTS(SELECT 1 FROM dispatch_contexts WHERE run_id = ?) + OR EXISTS(SELECT 1 FROM decision_gates WHERE run_id = ?) + OR EXISTS(SELECT 1 FROM messages WHERE run_id = ?) + OR EXISTS(SELECT 1 FROM question_threads WHERE run_id = ?) + OR EXISTS(SELECT 1 FROM deliveries WHERE run_id = ?)` + ) + .get(LEGACY_RUN_ID, LEGACY_RUN_ID, LEGACY_RUN_ID, LEGACY_RUN_ID, LEGACY_RUN_ID, LEGACY_RUN_ID) + if (!existing && !hasGraph) { + return + } + + const adoptedRunId = existing?.adopted_run_id ?? generateId('run') + this.db + .prepare( + `INSERT OR IGNORE INTO runs ( + id, objective, home_database, consumer_generation, legacy + ) VALUES (?, ?, 'this_database', 0, 0)` + ) + .run(adoptedRunId, 'Recovered orchestration work from a contract update') + this.db + .prepare( + `INSERT OR IGNORE INTO legacy_adoptions ( + source_run_id, adopted_run_id, scheduler_state_lost + ) VALUES (?, ?, 1)` + ) + .run(LEGACY_RUN_ID, adoptedRunId) + this.db + .prepare( + `UPDATE coordinator_runs + SET status = 'failed', + completed_at = COALESCE( + completed_at, + (SELECT adopted_at FROM legacy_adoptions WHERE source_run_id = ?) + ), + scheduler_lost_at = ( + SELECT adopted_at FROM legacy_adoptions WHERE source_run_id = ? + ) + WHERE status = 'running' + AND julianday(created_at) <= julianday(( + SELECT adopted_at FROM legacy_adoptions WHERE source_run_id = ? + ))` + ) + .run(LEGACY_RUN_ID, LEGACY_RUN_ID, LEGACY_RUN_ID) + + this.db + .prepare( + `UPDATE deliveries SET status = 'fenced' + WHERE run_id = ? AND status = 'outstanding'` + ) + .run(LEGACY_RUN_ID) + for (const table of [ + 'tasks', + 'dispatch_contexts', + 'decision_gates', + 'messages', + 'question_threads', + 'deliveries' + ]) { + this.db + .prepare(`UPDATE ${table} SET run_id = ? WHERE run_id = ?`) + .run(adoptedRunId, LEGACY_RUN_ID) + } + this.db + .prepare( + `UPDATE runs + SET objective = 'Legacy orchestration state (adopted; inspect only)', + coordinator_handle = NULL, coordinator_pane_key = NULL, + updated_at = datetime('now') + WHERE id = ?` + ) + .run(LEGACY_RUN_ID) + + const mismatch = this.db + .prepare( + `WITH migration_runs(run_id) AS (VALUES (?), (?)) + SELECT 1 + WHERE EXISTS( + SELECT 1 FROM dispatch_contexts d + INNER JOIN tasks t ON t.id = d.task_id + WHERE d.run_id <> t.run_id + AND ( + d.run_id IN (SELECT run_id FROM migration_runs) + OR t.run_id IN (SELECT run_id FROM migration_runs) + ) + ) + OR EXISTS( + SELECT 1 FROM decision_gates g + INNER JOIN tasks t ON t.id = g.task_id + WHERE g.run_id <> t.run_id + AND ( + g.run_id IN (SELECT run_id FROM migration_runs) + OR t.run_id IN (SELECT run_id FROM migration_runs) + ) + ) + OR EXISTS( + SELECT 1 FROM question_threads q + INNER JOIN dispatch_contexts d ON d.id = q.dispatch_id + WHERE q.run_id <> d.run_id + AND ( + q.run_id IN (SELECT run_id FROM migration_runs) + OR d.run_id IN (SELECT run_id FROM migration_runs) + ) + ) + OR EXISTS( + SELECT 1 FROM deliveries d + INNER JOIN json_each(d.message_ids) ids + INNER JOIN messages m ON m.id = ids.value + WHERE d.run_id <> m.run_id + AND ( + d.run_id IN (SELECT run_id FROM migration_runs) + OR m.run_id IN (SELECT run_id FROM migration_runs) + ) + )` + ) + .get(LEGACY_RUN_ID, adoptedRunId) + if (mismatch) { + throw new Error('Legacy orchestration adoption produced inconsistent Run ownership.') + } + } + private hasColumn(table: string, column: string): boolean { const rows = this.db.pragma(`table_info(${table})`) as { name: string }[] return rows.some((r) => r.name === column) @@ -881,6 +1332,720 @@ export class OrchestrationDb { .get(callerFingerprint, requestId) as MutationReceiptRow | undefined } + // ── Legacy adoption and compatibility principals ── + + getLegacyAdoption(): LegacyAdoptionRow | undefined { + return this.db + .prepare('SELECT * FROM legacy_adoptions WHERE source_run_id = ?') + .get(LEGACY_RUN_ID) as LegacyAdoptionRow | undefined + } + + commitLegacyCompatibilityPrincipal(params: { + runId: string + dispatchId?: string + role: LegacyPrincipalRole + hostScope: string + terminalHandle: string + paneKey: string + launchTokenHash: string + processIncarnation?: string + }): { principal: LegacyCompatibilityPrincipalRow; duplicate: boolean } { + this.db.exec('BEGIN IMMEDIATE') + try { + const adoption = this.getLegacyAdoption() + if (!adoption || adoption.adopted_run_id !== params.runId) { + throw new OrchestrationError( + 'request_mismatch', + `Run ${params.runId} is not the adopted legacy Run.` + ) + } + const dispatchId = params.role === 'worker' ? (params.dispatchId ?? null) : null + let initialStatus: 'committed' | 'settled' = 'committed' + if (params.role === 'worker') { + const dispatch = dispatchId ? this.getDispatchContextById(dispatchId) : undefined + if ( + !dispatch || + dispatch.run_id !== params.runId || + dispatch.contract_version !== LEGACY_CONTRACT_VERSION + ) { + throw new OrchestrationError( + 'request_mismatch', + `Dispatch ${dispatchId ?? '(missing)'} is not a legacy attempt in this Run.` + ) + } + initialStatus = ['pending', 'dispatched'].includes(dispatch.status) + ? 'committed' + : 'settled' + } else if (params.dispatchId) { + throw new OrchestrationError( + 'request_mismatch', + 'A coordinator compatibility principal cannot name a Dispatch.' + ) + } + + const existing = this.db + .prepare( + `SELECT * FROM legacy_compatibility_principals + WHERE role = ? AND run_id = ? AND dispatch_id IS ?` + ) + .get(params.role, params.runId, dispatchId) as LegacyCompatibilityPrincipalRow | undefined + if (existing) { + const same = + existing.host_scope === params.hostScope && + existing.terminal_handle === params.terminalHandle && + existing.pane_key === params.paneKey && + existing.launch_token_hash === params.launchTokenHash && + existing.process_incarnation === (params.processIncarnation ?? null) + if (!same) { + throw new OrchestrationError( + 'request_mismatch', + `The ${params.role} compatibility principal is already committed to different proof.` + ) + } + if (existing.status === 'revoked') { + throw new OrchestrationError( + 'legacy_read_only', + `The ${params.role} compatibility principal has been revoked. No effects were applied.`, + { effectsApplied: false } + ) + } + this.db.exec('COMMIT') + return { principal: existing, duplicate: true } + } + if ( + params.role === 'coordinator' && + !this.resolveLegacyCoordinatorCandidate({ + runId: params.runId, + terminalHandle: params.terminalHandle, + paneKey: params.paneKey + }) + ) { + throw new OrchestrationError( + 'legacy_read_only', + 'This retained legacy coordinator no longer has lifecycle authority. No effects were applied.', + { effectsApplied: false } + ) + } + + const id = generateId('legacy_principal') + this.db + .prepare( + `INSERT INTO legacy_compatibility_principals ( + id, run_id, dispatch_id, role, host_scope, terminal_handle, + pane_key, launch_token_hash, process_incarnation, status + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ) + .run( + id, + params.runId, + dispatchId, + params.role, + params.hostScope, + params.terminalHandle, + params.paneKey, + params.launchTokenHash, + params.processIncarnation ?? null, + initialStatus + ) + const principal = this.getLegacyCompatibilityPrincipal(id) as LegacyCompatibilityPrincipalRow + if (principal.status === 'committed') { + this.initializeLegacyRecoveryCohort(principal) + } + this.db.exec('COMMIT') + return { principal, duplicate: false } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + } + + getLegacyCompatibilityPrincipal(id: string): LegacyCompatibilityPrincipalRow | undefined { + return this.db.prepare('SELECT * FROM legacy_compatibility_principals WHERE id = ?').get(id) as + | LegacyCompatibilityPrincipalRow + | undefined + } + + listLegacyCompatibilityPrincipals(runId: string): LegacyCompatibilityPrincipalRow[] { + return this.db + .prepare( + `SELECT * FROM legacy_compatibility_principals + WHERE run_id = ? ORDER BY rowid` + ) + .all(runId) as LegacyCompatibilityPrincipalRow[] + } + + getLegacyCoordinatorPrincipal(runId: string): LegacyCompatibilityPrincipalRow | undefined { + return this.db + .prepare( + `SELECT * FROM legacy_compatibility_principals + WHERE run_id = ? AND role = 'coordinator'` + ) + .get(runId) as LegacyCompatibilityPrincipalRow | undefined + } + + resolveLegacyCompatibilityPrincipalByIdentity(params: { + runId: string + role: LegacyPrincipalRole + terminalHandle?: string + paneKey?: string + }): LegacyCompatibilityPrincipalRow | undefined { + if (!params.terminalHandle && !params.paneKey) { + return undefined + } + const rows = ( + this.db + .prepare( + `SELECT * FROM legacy_compatibility_principals + WHERE run_id = ? AND role = ? AND status IN ('committed', 'settled') + ORDER BY rowid` + ) + .all(params.runId, params.role) as LegacyCompatibilityPrincipalRow[] + ).filter((principal) => + params.paneKey + ? isEquivalentPaneKey(principal.pane_key, params.paneKey) + : principal.terminal_handle === params.terminalHandle + ) + if (rows.length > 1) { + throw new OrchestrationError( + 'operation_unknown', + 'Multiple legacy principals match this process identity.' + ) + } + return rows[0] + } + + resolveLegacyWorkerCandidate(params: { + runId?: string + terminalHandle?: string + paneKey?: string + dispatchId?: string + taskId?: string + }): { dispatch: DispatchContextRow } | undefined { + if (!params.runId || (!params.terminalHandle && !params.paneKey)) { + return undefined + } + const rows = ( + params.dispatchId + ? [this.getDispatchContextById(params.dispatchId)].filter( + (row): row is DispatchContextRow => row !== undefined + ) + : (this.db + .prepare( + `SELECT * FROM dispatch_contexts + WHERE run_id = ? AND contract_version = ? + AND status IN ('pending', 'dispatched') + ORDER BY rowid` + ) + .all(params.runId, LEGACY_CONTRACT_VERSION) as DispatchContextRow[]) + ).filter( + (dispatch) => + dispatch.run_id === params.runId && + dispatch.contract_version === LEGACY_CONTRACT_VERSION && + (!params.taskId || dispatch.task_id === params.taskId) && + (params.paneKey + ? Boolean( + dispatch.assignee_pane_key && + isEquivalentPaneKey(dispatch.assignee_pane_key, params.paneKey) + ) + : dispatch.assignee_handle === params.terminalHandle) + ) + if (rows.length > 1) { + throw new OrchestrationError( + 'operation_unknown', + 'Multiple active legacy Dispatches match this process identity.' + ) + } + if (params.dispatchId && rows.length === 0) { + const target = this.getDispatchContextById(params.dispatchId) + if (target?.contract_version === LEGACY_CONTRACT_VERSION) { + throw new OrchestrationError( + 'legacy_read_only', + `Dispatch ${params.dispatchId} is retained but this process cannot prove ownership.` + ) + } + } + return rows[0] ? { dispatch: rows[0] } : undefined + } + + resolveLegacyCoordinatorCandidate(params: { + runId: string + terminalHandle?: string + paneKey?: string + }): { terminalHandle: string; paneKey: string } | undefined { + if (!params.terminalHandle || !params.paneKey) { + return undefined + } + const run = this.getRunRaw(params.runId) + const principal = this.getLegacyCoordinatorPrincipal(params.runId) + if (principal) { + if ( + principal.status !== 'committed' || + principal.terminal_handle !== params.terminalHandle || + !isEquivalentPaneKey(principal.pane_key, params.paneKey) || + (run?.coordinator_pane_key !== null && + (run?.coordinator_handle !== principal.terminal_handle || + !isEquivalentPaneKey(run.coordinator_pane_key, principal.pane_key))) + ) { + return undefined + } + return { terminalHandle: params.terminalHandle, paneKey: params.paneKey } + } + // Why: the first current binding durably fences uncommitted legacy processes. + if ( + !run || + run.coordinator_pane_key !== null || + this.getUniqueLegacyCoordinatorHandle(params.runId) !== params.terminalHandle + ) { + return undefined + } + return { terminalHandle: params.terminalHandle, paneKey: params.paneKey } + } + + isLegacyCoordinatorHandle(runId: string, terminalHandle: string): boolean { + const principal = this.getLegacyCoordinatorPrincipal(runId) + if (principal) { + return principal.terminal_handle === terminalHandle + } + return this.getUniqueLegacyCoordinatorHandle(runId) === terminalHandle + } + + findLegacyWorkerCompletion(params: { + principalId: string + taskId: string + recipientHandle: string + subject: string + body: string + payload: string | null + }): MessageRow | undefined { + const principal = this.getLegacyCompatibilityPrincipal(params.principalId) + if (!principal || principal.role !== 'worker' || !principal.dispatch_id) { + throw new OrchestrationError('request_mismatch', 'Legacy worker principal was not found.') + } + const runAddress = `run:${principal.run_id}` + const rows = this.db + .prepare( + `SELECT * FROM messages + WHERE run_id = ? + AND ( + (delivery_contract = 'legacy_direct' AND to_handle = ?) OR + (delivery_contract = 'current_delivery' AND to_handle = ?) + ) + AND from_handle = ? AND type = 'worker_done' + AND subject = ? AND body = ? AND payload IS ? + ORDER BY sequence` + ) + .all( + principal.run_id, + params.recipientHandle, + runAddress, + principal.terminal_handle, + params.subject, + params.body, + params.payload + ) as MessageRow[] + const matches = rows.filter((message) => { + try { + const payload = JSON.parse(message.payload ?? '{}') as { + taskId?: unknown + dispatchId?: unknown + } + return payload.taskId === params.taskId && payload.dispatchId === principal.dispatch_id + } catch { + return false + } + }) + if (matches.length > 1) { + throw new OrchestrationError( + 'operation_unknown', + 'Multiple matching legacy worker completions exist.' + ) + } + return matches[0] ? exposeMessageTimestamps(matches[0]) : undefined + } + + hasPendingCurrentDelivery(runId: string): boolean { + return Boolean( + this.db + .prepare( + `SELECT 1 FROM messages + WHERE run_id = ? AND to_handle = ? + AND delivery_contract = 'current_delivery' AND read = 0 + LIMIT 1` + ) + .get(runId, `run:${runId}`) + ) + } + + setLegacyCompatibilityPrincipalStatus( + id: string, + status: 'settled' | 'revoked' + ): LegacyCompatibilityPrincipalRow | undefined { + this.db + .prepare( + `UPDATE legacy_compatibility_principals + SET status = ? + WHERE id = ? AND status = 'committed'` + ) + .run(status, id) + return this.getLegacyCompatibilityPrincipal(id) + } + + getLegacyOperationReceipt( + principalId: string, + operationKey: string + ): LegacyOperationReceiptRow | undefined { + return this.db + .prepare( + `SELECT * FROM legacy_operation_receipts + WHERE principal_id = ? AND operation_key = ?` + ) + .get(principalId, operationKey) as LegacyOperationReceiptRow | undefined + } + + private requireCommittedLegacyPrincipal( + principalId: string, + role?: LegacyPrincipalRole + ): LegacyCompatibilityPrincipalRow { + const principal = this.getLegacyCompatibilityPrincipal(principalId) + if (!principal || principal.status !== 'committed' || (role && principal.role !== role)) { + throw new OrchestrationError( + 'request_mismatch', + `Legacy compatibility principal ${principalId} is not committed for this operation.` + ) + } + return principal + } + + private requireLegacyMailPrincipal( + principalId: string, + role?: LegacyPrincipalRole + ): LegacyCompatibilityPrincipalRow { + const principal = this.getLegacyCompatibilityPrincipal(principalId) + if ( + !principal || + !['committed', 'settled'].includes(principal.status) || + (role && principal.role !== role) + ) { + throw new OrchestrationError( + 'request_mismatch', + `Legacy compatibility principal ${principalId} cannot access retained mail.` + ) + } + return principal + } + + private initializeLegacyRecoveryCohort(principal: LegacyCompatibilityPrincipalRow): void { + if (principal.role === 'worker') { + this.db + .prepare( + `INSERT OR IGNORE INTO legacy_mail_receipts ( + principal_id, message_id, acknowledged_at + ) + SELECT ?, m.id, NULL + FROM messages m + INNER JOIN dispatch_contexts d ON d.id = ? + WHERE m.run_id = ? AND m.delivery_contract = 'legacy_direct' AND m.read = 1 + AND d.status IN ('pending', 'dispatched') + AND m.created_at >= d.created_at + AND (m.to_handle = ? OR m.to_handle = ?)` + ) + .run( + principal.id, + principal.dispatch_id, + principal.run_id, + principal.terminal_handle, + `dispatch:${principal.dispatch_id}` + ) + return + } + this.db + .prepare( + `INSERT OR IGNORE INTO legacy_mail_receipts ( + principal_id, message_id, acknowledged_at + ) + SELECT ?, m.id, NULL + FROM messages m + WHERE m.run_id = ? AND m.delivery_contract = 'legacy_direct' AND m.read = 1 + AND m.to_handle = ? + AND EXISTS( + SELECT 1 FROM dispatch_contexts d + WHERE d.run_id = m.run_id + AND d.contract_version = ? + AND d.status IN ('pending', 'dispatched') + AND m.created_at >= d.created_at + AND (m.from_handle = d.assignee_handle OR m.from_handle = 'dispatch:' || d.id) + )` + ) + .run(principal.id, principal.run_id, principal.terminal_handle, LEGACY_CONTRACT_VERSION) + } + + getLegacyMailPage(params: { principalId: string; limit?: number; types?: MessageType[] }): { + messages: MessageRow[] + recovery: boolean + } { + const principal = this.requireLegacyMailPrincipal(params.principalId) + const limit = Math.min(Math.max(params.limit ?? 50, 1), 50) + const addressSql = + principal.role === 'worker' ? '(m.to_handle = ? OR m.to_handle = ?)' : 'm.to_handle = ?' + const addressParams = + principal.role === 'worker' + ? [principal.terminal_handle, `dispatch:${principal.dispatch_id}`] + : [principal.terminal_handle] + const typeSql = + params.types && params.types.length > 0 + ? `AND m.type IN (${params.types.map(() => '?').join(',')})` + : '' + const typeParams = params.types ?? [] + const recovery = this.db + .prepare( + `SELECT m.* + FROM legacy_mail_receipts r + INNER JOIN messages m ON m.id = r.message_id + WHERE r.principal_id = ? AND r.acknowledged_at IS NULL + AND m.run_id = ? AND m.delivery_contract = 'legacy_direct' + AND ${addressSql} + ${typeSql} + ORDER BY m.sequence ASC LIMIT ?` + ) + .all( + params.principalId, + principal.run_id, + ...addressParams, + ...typeParams, + limit + ) as MessageRow[] + if (recovery.length > 0) { + return { messages: exposeMessageListTimestamps(recovery), recovery: true } + } + + const unread = this.db + .prepare( + `SELECT m.* + FROM messages m + LEFT JOIN legacy_mail_receipts r + ON r.principal_id = ? AND r.message_id = m.id + WHERE m.run_id = ? AND m.delivery_contract = 'legacy_direct' + AND m.read = 0 AND r.message_id IS NULL AND ${addressSql} + ${typeSql} + ORDER BY m.sequence ASC LIMIT ?` + ) + .all( + params.principalId, + principal.run_id, + ...addressParams, + ...typeParams, + limit + ) as MessageRow[] + return { messages: exposeMessageListTimestamps(unread), recovery: false } + } + + getLegacyMailHistory(params: { principalId: string; limit?: number; types?: MessageType[] }): { + messages: MessageRow[] + recovery: false + } { + const principal = this.requireLegacyMailPrincipal(params.principalId) + const limit = Math.min(Math.max(params.limit ?? 100, 1), 100) + const addressSql = + principal.role === 'worker' ? '(to_handle = ? OR to_handle = ?)' : 'to_handle = ?' + const addressParams = + principal.role === 'worker' + ? [principal.terminal_handle, `dispatch:${principal.dispatch_id}`] + : [principal.terminal_handle] + const typeSql = + params.types && params.types.length > 0 + ? `AND type IN (${params.types.map(() => '?').join(',')})` + : '' + const messages = this.db + .prepare( + `SELECT * FROM messages + WHERE run_id = ? AND delivery_contract = 'legacy_direct' + AND ${addressSql} ${typeSql} + ORDER BY sequence ASC LIMIT ?` + ) + .all(principal.run_id, ...addressParams, ...(params.types ?? []), limit) as MessageRow[] + return { messages: exposeMessageListTimestamps(messages), recovery: false } + } + + acknowledgeLegacyMail(params: { + principalId: string + messageIds: string[] + types?: MessageType[] + }): { + receipts: LegacyMailReceiptRow[] + duplicate: boolean + } { + if (params.messageIds.length === 0) { + return { receipts: [], duplicate: true } + } + this.db.exec('BEGIN IMMEDIATE') + try { + const principal = this.requireLegacyMailPrincipal(params.principalId) + const uniqueIds = [...new Set(params.messageIds)] + const placeholders = uniqueIds.map(() => '?').join(',') + const prior = this.db + .prepare( + `SELECT COUNT(*) AS count FROM legacy_mail_receipts + WHERE principal_id = ? AND message_id IN (${placeholders}) + AND acknowledged_at IS NOT NULL` + ) + .get(params.principalId, ...uniqueIds) as { count: number } + if (prior.count !== uniqueIds.length) { + const actionable = this.getLegacyMailPage({ + principalId: params.principalId, + limit: uniqueIds.length, + types: params.types + }).messages + if ( + actionable.length !== uniqueIds.length || + actionable.some((message, index) => message.id !== uniqueIds[index]) + ) { + throw new OrchestrationError( + 'request_mismatch', + 'Legacy mail acknowledgment does not match the current replay page.' + ) + } + } + const rows = this.db + .prepare( + `SELECT * FROM messages + WHERE id IN (${placeholders}) AND run_id = ? + AND delivery_contract = 'legacy_direct'` + ) + .all(...uniqueIds, principal.run_id) as MessageRow[] + const validIds = new Set( + rows + .filter( + (message) => + message.to_handle === principal.terminal_handle || + (principal.role === 'worker' && + message.to_handle === `dispatch:${principal.dispatch_id}`) + ) + .map((message) => message.id) + ) + if (validIds.size !== uniqueIds.length || uniqueIds.some((id) => !validIds.has(id))) { + throw new OrchestrationError( + 'request_mismatch', + 'Legacy mail acknowledgment contains a message outside this principal inbox.' + ) + } + + this.db + .prepare( + `UPDATE messages + SET read = 1, delivered_at = COALESCE(delivered_at, datetime('now')) + WHERE id IN (${placeholders})` + ) + .run(...uniqueIds) + const insert = this.db.prepare( + `INSERT INTO legacy_mail_receipts ( + principal_id, message_id, acknowledged_at + ) VALUES (?, ?, datetime('now')) + ON CONFLICT(principal_id, message_id) + DO UPDATE SET acknowledged_at = COALESCE( + legacy_mail_receipts.acknowledged_at, excluded.acknowledged_at + )` + ) + for (const messageId of uniqueIds) { + insert.run(params.principalId, messageId) + } + const receipts = this.db + .prepare( + `SELECT * FROM legacy_mail_receipts + WHERE principal_id = ? AND message_id IN (${placeholders}) + ORDER BY message_id` + ) + .all(params.principalId, ...uniqueIds) as LegacyMailReceiptRow[] + this.db.exec('COMMIT') + return { receipts, duplicate: prior.count === uniqueIds.length } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + } + + acknowledgeLegacyQuestionAnswer(params: { + principalId: string + questionId: string + answerMessageId: string + }): { receipt: LegacyMailReceiptRow; duplicate: boolean } { + this.db.exec('BEGIN IMMEDIATE') + try { + const principal = this.requireLegacyMailPrincipal(params.principalId, 'worker') + const question = this.getQuestionRaw(params.questionId) + const source = this.getMessageById(params.questionId) + const answer = this.getMessageById(params.answerMessageId) + const dispatch = principal.dispatch_id + ? this.getDispatchContextById(principal.dispatch_id) + : undefined + const exactLegacyAnswer = + answer?.delivery_contract === 'legacy_direct' && + (answer.to_handle === principal.terminal_handle || + answer.to_handle === `dispatch:${principal.dispatch_id}`) + const adoption = this.getLegacyAdoption() + const exactTakenOverAnswer = + adoption?.adopted_run_id === principal.run_id && + dispatch?.run_id === principal.run_id && + dispatch.contract_version === LEGACY_CONTRACT_VERSION && + source?.run_id === principal.run_id && + source.from_handle === principal.terminal_handle && + source.to_handle === `run:${principal.run_id}` && + source.delivery_contract === 'current_delivery' && + answer?.run_id === principal.run_id && + answer?.delivery_contract === 'current_delivery' && + answer.from_handle === `run:${principal.run_id}` && + answer.to_handle === `dispatch:${principal.dispatch_id}` && + answer.thread_id === question?.message_id + if ( + !question || + !answer || + question.run_id !== principal.run_id || + question.dispatch_id !== principal.dispatch_id || + question.answer_message_id !== params.answerMessageId || + (!exactLegacyAnswer && !exactTakenOverAnswer) + ) { + throw new OrchestrationError( + 'request_mismatch', + 'Legacy answer acknowledgment does not match this principal question.' + ) + } + const existing = this.db + .prepare( + `SELECT * FROM legacy_mail_receipts + WHERE principal_id = ? AND message_id = ?` + ) + .get(params.principalId, params.answerMessageId) as LegacyMailReceiptRow | undefined + this.db + .prepare( + `UPDATE messages + SET read = 1, delivered_at = COALESCE(delivered_at, datetime('now')) + WHERE id = ?` + ) + .run(params.answerMessageId) + this.db + .prepare( + `INSERT INTO legacy_mail_receipts ( + principal_id, message_id, acknowledged_at + ) VALUES (?, ?, datetime('now')) + ON CONFLICT(principal_id, message_id) + DO UPDATE SET acknowledged_at = COALESCE( + legacy_mail_receipts.acknowledged_at, excluded.acknowledged_at + )` + ) + .run(params.principalId, params.answerMessageId) + const receipt = this.db + .prepare( + `SELECT * FROM legacy_mail_receipts + WHERE principal_id = ? AND message_id = ?` + ) + .get(params.principalId, params.answerMessageId) as LegacyMailReceiptRow + this.db.exec('COMMIT') + return { receipt, duplicate: Boolean(existing?.acknowledged_at) } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + } + // ── Runs ── createRun(params: { @@ -912,6 +2077,14 @@ export class OrchestrationDb { runId: string coordinatorHandle: string coordinatorPaneKey: string + takeoverLegacy?: boolean + legacyCoordinatorAuthority?: { + runId: string + principalId: string | null + terminalHandle: string + paneKey: string + consumerGeneration: number + } }): RunRow | undefined { this.db.exec('BEGIN IMMEDIATE') try { @@ -923,8 +2096,104 @@ export class OrchestrationDb { const sameBinding = run.coordinator_pane_key !== null && isEquivalentPaneKey(run.coordinator_pane_key, params.coordinatorPaneKey) + const adoption = this.getLegacyAdoption() + const adoptedRun = adoption?.adopted_run_id === params.runId + const legacyAuthority = params.legacyCoordinatorAuthority + const legacyPrincipalId = legacyAuthority?.principalId + const legacyPrincipal = legacyPrincipalId + ? this.getLegacyCompatibilityPrincipal(legacyPrincipalId) + : undefined + const provenLegacyBinding = Boolean( + adoptedRun && + legacyAuthority && + legacyAuthority.principalId !== null && + legacyAuthority.runId === params.runId && + legacyAuthority.consumerGeneration === run.consumer_generation && + legacyPrincipal?.run_id === params.runId && + legacyPrincipal.role === 'coordinator' && + legacyPrincipal.status === 'committed' && + legacyPrincipal.terminal_handle === legacyAuthority.terminalHandle && + isEquivalentPaneKey(legacyPrincipal.pane_key, legacyAuthority.paneKey) && + params.coordinatorHandle === legacyAuthority.terminalHandle && + isEquivalentPaneKey(params.coordinatorPaneKey, legacyAuthority.paneKey) + ) + if (legacyAuthority && !provenLegacyBinding) { + throw new OrchestrationError( + 'legacy_read_only', + 'This retained legacy coordinator no longer has lifecycle authority. No effects were applied.', + { effectsApplied: false } + ) + } + const activeLegacyAssignment = + adoptedRun && + Boolean( + this.db + .prepare( + `SELECT 1 FROM dispatch_contexts + WHERE run_id = ? AND contract_version = ? + AND status IN ('pending', 'dispatched') + LIMIT 1` + ) + .get(params.runId, LEGACY_CONTRACT_VERSION) + ) + const coordinatorPrincipal = adoptedRun + ? this.getLegacyCoordinatorPrincipal(params.runId) + : undefined + const retainedCoordinatorHandle = + coordinatorPrincipal?.terminal_handle ?? + run.coordinator_handle ?? + this.getUniqueLegacyCoordinatorHandle(params.runId) + const takeoverAlreadyApplied = Boolean( + params.takeoverLegacy && + sameBinding && + run.coordinator_handle === params.coordinatorHandle && + coordinatorPrincipal?.status !== 'committed' + ) + const replacesLegacyCoordinator = Boolean( + adoptedRun && + !provenLegacyBinding && + retainedCoordinatorHandle && + (params.takeoverLegacy || + retainedCoordinatorHandle !== params.coordinatorHandle || + !sameBinding) + ) + if (params.takeoverLegacy && !adoptedRun) { + throw new OrchestrationError( + 'invalid_argument', + 'Legacy takeover is only available for the automatically adopted Run.' + ) + } + if ( + activeLegacyAssignment && + !sameBinding && + !provenLegacyBinding && + !params.takeoverLegacy + ) { + throw new OrchestrationError( + 'consumer_fenced', + 'This adopted Run still has live legacy work. Its attested coordinator may rebind it, or a current coordinator may explicitly use run-use --takeover-legacy.', + { + effectsApplied: false, + recoveryCommand: `orca orchestration run-use --id ${params.runId} --takeover-legacy` + } + ) + } this.unbindOtherRunsForPane(params.coordinatorPaneKey, params.runId) - if (!sameBinding || run.coordinator_handle !== params.coordinatorHandle) { + if ( + (params.takeoverLegacy && !takeoverAlreadyApplied) || + !sameBinding || + run.coordinator_handle !== params.coordinatorHandle + ) { + if (adoptedRun && (params.takeoverLegacy || !activeLegacyAssignment)) { + if ( + coordinatorPrincipal?.status === 'committed' && + (params.takeoverLegacy || + coordinatorPrincipal.terminal_handle !== params.coordinatorHandle || + !isEquivalentPaneKey(coordinatorPrincipal.pane_key, params.coordinatorPaneKey)) + ) { + this.setLegacyCompatibilityPrincipalStatus(coordinatorPrincipal.id, 'revoked') + } + } this.db .prepare( `UPDATE runs @@ -935,6 +2204,9 @@ export class OrchestrationDb { ) .run(params.coordinatorHandle, params.coordinatorPaneKey, params.runId) this.fenceOutstandingDelivery(params.runId) + if (params.takeoverLegacy || replacesLegacyCoordinator) { + this.promoteLegacyCoordinatorMailForTakeover(params.runId, retainedCoordinatorHandle) + } } this.db.exec('COMMIT') } catch (error) { @@ -1009,6 +2281,158 @@ export class OrchestrationDb { .run(runId) } + private promoteLegacyCoordinatorMailForTakeover( + runId: string, + retainedCoordinatorHandle: string | null + ): void { + if (!retainedCoordinatorHandle) { + return + } + this.db + .prepare( + `UPDATE messages + SET to_handle = ?, delivery_contract = 'current_delivery', + read = 0, delivered_at = NULL + WHERE run_id = ? AND delivery_contract = 'legacy_direct' + AND to_handle = ? + AND EXISTS( + SELECT 1 FROM dispatch_contexts d + WHERE d.run_id = messages.run_id + AND d.contract_version = ? + AND ( + messages.from_handle = d.assignee_handle OR + messages.from_handle = 'dispatch:' || d.id + ) + ) + AND ( + read = 0 OR EXISTS( + SELECT 1 FROM question_threads q + WHERE q.message_id = messages.id AND q.status = 'pending' + ) OR EXISTS( + SELECT 1 + FROM legacy_mail_receipts r + INNER JOIN legacy_compatibility_principals p + ON p.id = r.principal_id + WHERE r.message_id = messages.id + AND r.acknowledged_at IS NULL + AND p.run_id = messages.run_id + AND p.role = 'coordinator' + AND p.terminal_handle = ? + ) OR ( + read = 1 + AND NOT EXISTS( + SELECT 1 FROM legacy_compatibility_principals p + WHERE p.run_id = messages.run_id AND p.role = 'coordinator' + ) + AND EXISTS( + SELECT 1 FROM dispatch_contexts d + WHERE d.run_id = messages.run_id + AND d.contract_version = ? + AND d.status IN ('pending', 'dispatched') + AND messages.created_at >= d.created_at + AND ( + messages.from_handle = d.assignee_handle OR + messages.from_handle = 'dispatch:' || d.id + ) + ) + ) + )` + ) + .run( + `run:${runId}`, + runId, + retainedCoordinatorHandle, + LEGACY_CONTRACT_VERSION, + retainedCoordinatorHandle, + LEGACY_CONTRACT_VERSION + ) + } + + private getUniqueLegacyCoordinatorHandle(runId: string): string | null { + const adoption = this.getLegacyAdoption() + if (!adoption || adoption.adopted_run_id !== runId) { + return null + } + const workerHandles = new Set( + ( + this.db + .prepare( + `SELECT DISTINCT assignee_handle AS handle + FROM dispatch_contexts + WHERE run_id = ? AND contract_version = ? + AND assignee_handle IS NOT NULL + UNION + SELECT DISTINCT terminal_handle AS handle + FROM legacy_compatibility_principals + WHERE run_id = ? AND role = 'worker' + AND status IN ('committed', 'settled')` + ) + .all(runId, LEGACY_CONTRACT_VERSION, runId) as { handle: string }[] + ).map((row) => row.handle) + ) + const durableRows = this.db + .prepare( + `SELECT coordinator_handle AS handle + FROM coordinator_runs + WHERE scheduler_lost_at = ? + UNION + SELECT created_by_terminal_handle AS handle + FROM tasks t + WHERE t.run_id = ? AND t.created_by_terminal_handle IS NOT NULL + AND t.created_at <= ? + AND EXISTS( + SELECT 1 FROM dispatch_contexts d + WHERE d.task_id = t.id AND d.run_id = t.run_id + AND d.contract_version = ? + )` + ) + .all(adoption.adopted_at, runId, adoption.adopted_at, LEGACY_CONTRACT_VERSION) as { + handle: string + }[] + if (durableRows.some((row) => workerHandles.has(row.handle))) { + return null + } + const candidates = new Set(durableRows.map((row) => row.handle)) + const mailRows = this.db + .prepare( + `SELECT m.to_handle AS handle + FROM messages m + INNER JOIN dispatch_contexts d + ON d.run_id = m.run_id AND d.contract_version = ? + AND (m.from_handle = d.assignee_handle OR m.from_handle = 'dispatch:' || d.id) + WHERE m.run_id = ? AND m.delivery_contract = 'legacy_direct' + AND m.created_at <= ? + UNION + SELECT m.from_handle AS handle + FROM messages m + INNER JOIN dispatch_contexts d + ON d.run_id = m.run_id AND d.contract_version = ? + AND (m.to_handle = d.assignee_handle OR m.to_handle = 'dispatch:' || d.id) + WHERE m.run_id = ? AND m.delivery_contract = 'legacy_direct' + AND m.created_at <= ?` + ) + .all( + LEGACY_CONTRACT_VERSION, + runId, + adoption.adopted_at, + LEGACY_CONTRACT_VERSION, + runId, + adoption.adopted_at + ) as { + handle: string + }[] + for (const row of mailRows) { + if ( + !workerHandles.has(row.handle) && + !row.handle.startsWith('dispatch:') && + !row.handle.startsWith('run:') + ) { + candidates.add(row.handle) + } + } + return candidates.size === 1 ? ([...candidates][0] ?? null) : null + } + private requireCurrentConsumer(runId: string, consumerGeneration: number): RunRow { const run = this.getRunRaw(runId) if (!run || run.legacy === 1 || run.consumer_generation !== consumerGeneration) { @@ -1072,6 +2496,7 @@ export class OrchestrationDb { .prepare( `SELECT 1 FROM messages WHERE run_id = ? AND to_handle = ? AND read = 0 + AND delivery_contract = 'current_delivery' AND type IN (${placeholders}) LIMIT 1` ) .get(params.runId, address, ...params.wakeTypes) @@ -1086,6 +2511,7 @@ export class OrchestrationDb { .prepare( `SELECT * FROM messages WHERE run_id = ? AND to_handle = ? AND read = 0 + AND delivery_contract = 'current_delivery' ORDER BY sequence ASC LIMIT ?` ) .all(params.runId, address, limit) as MessageRow[] @@ -1203,17 +2629,23 @@ export class OrchestrationDb { payload?: string senderPaneKey?: string runId?: string + deliveryContract?: MessageDeliveryContract }): MessageRow { const runId = msg.runId ?? LEGACY_RUN_ID + const deliveryContract = msg.deliveryContract ?? 'current_delivery' this.requireRun(runId) const id = msg.id ?? generateId('msg') const stmt = this.db.prepare(` - INSERT INTO messages (id, run_id, from_handle, to_handle, subject, body, type, priority, thread_id, payload, sender_pane_key) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO messages ( + id, run_id, delivery_contract, from_handle, to_handle, subject, body, + type, priority, thread_id, payload, sender_pane_key + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `) stmt.run( id, runId, + deliveryContract, msg.from, msg.to, msg.subject, @@ -1229,20 +2661,593 @@ export class OrchestrationDb { ) } + commitLegacyLifecycleOperation(params: { + principalId: string + operationKey: string + method: string + payloadHash: string + message: { + existingId?: string + to: string + subject: string + body?: string + type: MessageType + priority?: MessagePriority + payload?: string + } + lifecycle: + | { kind: 'message_only' } + | { kind: 'heartbeat'; at: string } + | { + kind: 'worker_report' + taskId: string + outcome: WorkerReportOutcome + result: string + } + }): { + receipt: LegacyOperationReceiptRow + message: MessageRow + settlement?: WorkerReportSettlement + duplicate: boolean + } { + this.db.exec('BEGIN IMMEDIATE') + try { + const principal = this.getLegacyCompatibilityPrincipal(params.principalId) + if ( + !principal || + principal.role !== 'worker' || + !['committed', 'settled'].includes(principal.status) + ) { + throw new OrchestrationError( + 'request_mismatch', + `Legacy compatibility principal ${params.principalId} cannot send lifecycle work.` + ) + } + const dispatchId = principal.dispatch_id as string + const existingReceipt = this.requireMatchingLegacyOperationReceipt(params) + if (existingReceipt) { + const response = JSON.parse(existingReceipt.response_json) as { + messageId: string + settlement?: WorkerReportSettlement + } + const message = this.getMessageById(response.messageId) + if (!message) { + throw new OrchestrationError( + 'operation_unknown', + `Legacy operation ${params.operationKey} lost its recorded message.` + ) + } + this.db.exec('COMMIT') + return { + receipt: existingReceipt, + message, + settlement: response.settlement, + duplicate: true + } + } + + const dispatch = this.getDispatchContextById(dispatchId) + if ( + !dispatch || + dispatch.run_id !== principal.run_id || + dispatch.contract_version !== LEGACY_CONTRACT_VERSION + ) { + throw new OrchestrationError( + 'dispatch_inactive', + `Dispatch ${dispatchId} is not this principal's legacy attempt.` + ) + } + if ( + (principal.status === 'settled' || !['pending', 'dispatched'].includes(dispatch.status)) && + (!params.message.existingId || params.lifecycle.kind !== 'worker_report') + ) { + throw new OrchestrationError( + 'dispatch_inactive', + `Dispatch ${dispatchId} is settled and only matching completion reconstruction is allowed.` + ) + } + let message = params.message.existingId + ? this.getMessageById(params.message.existingId) + : undefined + const delivery = this.resolveLegacyWorkerCoordinatorDelivery( + principal.run_id, + params.message.to + ) + if (params.message.existingId) { + const matchesOriginalLegacyRoute = + message?.delivery_contract === 'legacy_direct' && message.to_handle === params.message.to + const matchesCurrentRoute = + message?.delivery_contract === delivery.contract && message.to_handle === delivery.to + if ( + !message || + message.run_id !== principal.run_id || + message.from_handle !== principal.terminal_handle || + (!matchesOriginalLegacyRoute && !matchesCurrentRoute) + ) { + throw new OrchestrationError( + 'request_mismatch', + `Existing legacy message ${params.message.existingId} does not match this principal.` + ) + } + } else { + message = this.insertMessage({ + from: principal.terminal_handle, + to: delivery.to, + subject: params.message.subject, + body: params.message.body, + type: params.message.type, + priority: params.message.priority, + payload: params.message.payload, + senderPaneKey: principal.pane_key, + runId: principal.run_id, + deliveryContract: delivery.contract + }) + } + + let settlement: WorkerReportSettlement | undefined + if (params.lifecycle.kind === 'heartbeat') { + this.recordHeartbeat(dispatchId, params.lifecycle.at) + } else if (params.lifecycle.kind === 'worker_report') { + const persistedOutcome = + params.message.existingId && + dispatch.task_id === params.lifecycle.taskId && + dispatch.status === 'completed' + ? 'succeeded' + : params.message.existingId && + dispatch.task_id === params.lifecycle.taskId && + dispatch.status === 'failed' + ? 'failed' + : undefined + settlement = persistedOutcome + ? { action: 'settled', outcome: persistedOutcome, duplicate: true } + : this.settleWorkerReportInTransaction({ + taskId: params.lifecycle.taskId, + dispatchId, + outcome: params.lifecycle.outcome, + result: params.lifecycle.result + }) + if (settlement.action === 'rejected') { + throw new OrchestrationError(settlement.code, settlement.reason) + } + this.db + .prepare( + `UPDATE legacy_compatibility_principals + SET status = 'settled' WHERE id = ? AND status = 'committed'` + ) + .run(principal.id) + } + const responseJson = JSON.stringify({ messageId: message.id, settlement }) + const receipt = this.insertLegacyOperationReceipt({ + principalId: principal.id, + operationKey: params.operationKey, + method: params.method, + payloadHash: params.payloadHash, + effectId: message.id, + responseJson + }) + this.db.exec('COMMIT') + return { receipt, message, settlement, duplicate: false } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + } + + commitLegacyAskOperation(params: { + principalId: string + operationKey: string + method: string + payloadHash: string + question: string + options?: string[] + recipientHandle: string + existingQuestionId?: string + }): { + receipt: LegacyOperationReceiptRow + question: QuestionRow + message: MessageRow + duplicate: boolean + } { + this.db.exec('BEGIN IMMEDIATE') + try { + const principal = this.requireCommittedLegacyPrincipal(params.principalId, 'worker') + const receipt = this.requireMatchingLegacyOperationReceipt(params) + if (receipt) { + const response = JSON.parse(receipt.response_json) as { questionId: string } + const question = this.getQuestion(response.questionId) + const message = this.getMessageById(response.questionId) + if (!question || !message) { + throw new OrchestrationError( + 'operation_unknown', + `Legacy ask ${params.operationKey} lost its durable question.` + ) + } + this.db.exec('COMMIT') + return { receipt, question, message, duplicate: true } + } + + const dispatchId = principal.dispatch_id as string + const dispatch = this.getDispatchContextById(dispatchId) + if ( + !dispatch || + dispatch.run_id !== principal.run_id || + dispatch.contract_version !== LEGACY_CONTRACT_VERSION || + !['pending', 'dispatched'].includes(dispatch.status) + ) { + throw new OrchestrationError( + 'dispatch_inactive', + `Dispatch ${dispatchId} is not an active legacy attempt.` + ) + } + + const existingQuestionId = + params.existingQuestionId && + !this.db + .prepare( + `SELECT 1 FROM legacy_operation_receipts + WHERE principal_id = ? AND method = 'orchestration.ask' AND effect_id = ? + LIMIT 1` + ) + .get(principal.id, params.existingQuestionId) + ? params.existingQuestionId + : undefined + let question: QuestionRow + let message: MessageRow + const delivery = this.resolveLegacyWorkerCoordinatorDelivery( + principal.run_id, + params.recipientHandle + ) + if (existingQuestionId) { + const existingQuestion = this.getQuestion(existingQuestionId) + const existingMessage = this.getMessageById(existingQuestionId) + if ( + !existingQuestion || + !existingMessage || + existingQuestion.run_id !== principal.run_id || + existingQuestion.dispatch_id !== dispatchId || + existingQuestion.status !== 'pending' || + existingMessage.delivery_contract !== delivery.contract || + !legacyMessageMatchesQuestion(existingMessage, params.question, params.options ?? [], [ + delivery.to + ]) + ) { + throw new OrchestrationError( + 'request_mismatch', + `Question ${params.existingQuestionId} is not a pending ask for this principal.` + ) + } + question = existingQuestion + message = existingMessage + } else { + message = this.insertMessage({ + from: principal.terminal_handle, + to: delivery.to, + subject: 'Question', + body: params.question, + type: delivery.contract === 'legacy_direct' ? 'decision_gate' : 'question', + payload: JSON.stringify({ + taskId: dispatch.task_id, + dispatchId, + question: params.question, + options: params.options ?? [] + }), + senderPaneKey: principal.pane_key, + runId: principal.run_id, + deliveryContract: delivery.contract + }) + this.db + .prepare('UPDATE messages SET thread_id = ? WHERE id = ?') + .run(message.id, message.id) + this.db + .prepare( + `INSERT INTO question_threads ( + message_id, run_id, dispatch_id, asker_handle + ) VALUES (?, ?, ?, ?)` + ) + .run(message.id, principal.run_id, dispatchId, principal.terminal_handle) + question = this.getQuestion(message.id) as QuestionRow + message = this.getMessageById(message.id) as MessageRow + } + + const committedReceipt = this.insertLegacyOperationReceipt({ + principalId: principal.id, + operationKey: params.operationKey, + method: params.method, + payloadHash: params.payloadHash, + effectId: question.message_id, + responseJson: JSON.stringify({ questionId: question.message_id }) + }) + this.db.exec('COMMIT') + return { receipt: committedReceipt, question, message, duplicate: false } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + } + + findPendingLegacyQuestions(params: { + principalId: string + question: string + options?: string[] + recipientHandle: string + }): { question: QuestionRow; message: MessageRow }[] { + return this.findLegacyQuestionsBySemanticIdentity(params) + .filter((row) => row.question.status === 'pending') + .map(({ question, message }) => ({ question, message })) + } + + findLegacyQuestionsBySemanticIdentity(params: { + principalId: string + question: string + options?: string[] + recipientHandle: string + }): { + question: QuestionRow + message: MessageRow + answerAcknowledged: boolean + claimedByOperation: boolean + }[] { + const principal = this.requireCommittedLegacyPrincipal(params.principalId, 'worker') + const runAddress = `run:${principal.run_id}` + const rows = this.db + .prepare( + `SELECT q.*, m.id AS source_message_id, + EXISTS( + SELECT 1 FROM legacy_operation_receipts lor + WHERE lor.principal_id = ? AND lor.method = 'orchestration.ask' + AND lor.effect_id = q.message_id + ) AS claimed_by_operation + FROM question_threads q + INNER JOIN messages m ON m.id = q.message_id + WHERE q.run_id = ? AND q.dispatch_id = ? + AND ( + (m.delivery_contract = 'legacy_direct' AND m.to_handle = ?) OR + (m.delivery_contract = 'current_delivery' AND m.to_handle = ?) + ) + ORDER BY m.sequence + LIMIT 501` + ) + .all( + principal.id, + principal.run_id, + principal.dispatch_id, + params.recipientHandle, + runAddress + ) as (QuestionRow & { + source_message_id: string + claimed_by_operation: number + })[] + if (rows.length > 500) { + throw new OrchestrationError( + 'operation_unknown', + 'Legacy ask identity is too ambiguous to reconstruct safely.' + ) + } + return rows + .filter((row) => { + const message = this.getMessageById(row.source_message_id) + return Boolean( + message && + legacyMessageMatchesQuestion(message, params.question, params.options ?? [], [ + params.recipientHandle, + runAddress + ]) + ) + }) + .map((row) => ({ + question: exposeQuestionTimestamps(row), + message: this.getMessageById(row.message_id) as MessageRow, + claimedByOperation: row.claimed_by_operation === 1, + answerAcknowledged: row.answer_message_id + ? Boolean( + this.db + .prepare( + `SELECT 1 FROM legacy_mail_receipts + WHERE principal_id = ? AND message_id = ? + AND acknowledged_at IS NOT NULL` + ) + .get(principal.id, row.answer_message_id) + ) + : false + })) + } + + private resolveLegacyWorkerCoordinatorDelivery( + runId: string, + retainedCoordinatorHandle: string + ): { to: string; contract: MessageDeliveryContract } { + const run = this.getRunRaw(runId) + const principal = this.getLegacyCoordinatorPrincipal(runId) + const takenOver = run?.coordinator_handle !== null && principal?.status !== 'committed' + return takenOver + ? { to: `run:${runId}`, contract: 'current_delivery' } + : { to: retainedCoordinatorHandle, contract: 'legacy_direct' } + } + + commitLegacyReplyOperation(params: { + principalId: string + operationKey: string + method: string + payloadHash: string + questionId: string + body: string + }): { + receipt: LegacyOperationReceiptRow + question: QuestionRow + message: MessageRow + duplicate: boolean + } { + this.db.exec('BEGIN IMMEDIATE') + try { + const principal = this.requireCommittedLegacyPrincipal(params.principalId, 'coordinator') + const receipt = this.requireMatchingLegacyOperationReceipt(params) + if (receipt) { + const response = JSON.parse(receipt.response_json) as { + questionId: string + messageId: string + } + const question = this.getQuestion(response.questionId) + const message = this.getMessageById(response.messageId) + if (!question || !message) { + throw new OrchestrationError( + 'operation_unknown', + `Legacy reply ${params.operationKey} lost its durable effect.` + ) + } + this.db.exec('COMMIT') + return { receipt, question, message, duplicate: true } + } + + const question = this.getQuestionRaw(params.questionId) + const sourceMessage = this.getMessageById(params.questionId) + const dispatch = question ? this.getDispatchContextById(question.dispatch_id) : undefined + if ( + !question || + !sourceMessage || + !dispatch || + question.run_id !== principal.run_id || + sourceMessage.delivery_contract !== 'legacy_direct' || + dispatch.run_id !== principal.run_id || + dispatch.contract_version !== LEGACY_CONTRACT_VERSION || + question.status === 'closed' + ) { + throw new OrchestrationError( + 'question_not_found', + `Question ${params.questionId} is not actionable in the adopted Run.` + ) + } + let message: MessageRow + if (question.status === 'answered') { + if (question.answer_body !== params.body || !question.answer_message_id) { + throw new OrchestrationError( + 'answer_conflict', + `Question ${params.questionId} already has a different answer.` + ) + } + message = this.getMessageById(question.answer_message_id) as MessageRow + if ( + !message || + message.run_id !== principal.run_id || + message.delivery_contract !== 'legacy_direct' + ) { + throw new OrchestrationError( + 'operation_unknown', + `Question ${params.questionId} lost its recorded answer message.` + ) + } + } else { + message = this.insertMessage({ + from: principal.terminal_handle, + to: question.asker_handle, + subject: 'Re: Question', + body: params.body, + threadId: question.message_id, + runId: principal.run_id, + deliveryContract: 'legacy_direct' + }) + this.markAsRead([question.message_id]) + this.db + .prepare( + `UPDATE question_threads + SET status = 'answered', answer_message_id = ?, answer_body = ?, + answered_at = datetime('now') + WHERE message_id = ? AND status = 'pending'` + ) + .run(message.id, params.body, question.message_id) + } + + const answered = this.getQuestion(params.questionId) as QuestionRow + const committedReceipt = this.insertLegacyOperationReceipt({ + principalId: principal.id, + operationKey: params.operationKey, + method: params.method, + payloadHash: params.payloadHash, + effectId: message.id, + responseJson: JSON.stringify({ + questionId: answered.message_id, + messageId: message.id + }) + }) + this.db.exec('COMMIT') + return { + receipt: committedReceipt, + question: answered, + message, + duplicate: question.status === 'answered' + } + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + } + + private requireMatchingLegacyOperationReceipt(params: { + principalId: string + operationKey: string + method: string + payloadHash: string + }): LegacyOperationReceiptRow | undefined { + const receipt = this.getLegacyOperationReceipt(params.principalId, params.operationKey) + if ( + receipt && + (receipt.method !== params.method || receipt.payload_hash !== params.payloadHash) + ) { + throw new OrchestrationError( + 'request_mismatch', + `Legacy operation ${params.operationKey} was already used with different input.` + ) + } + return receipt + } + + private insertLegacyOperationReceipt(params: { + principalId: string + operationKey: string + method: string + payloadHash: string + effectId: string + responseJson: string + }): LegacyOperationReceiptRow { + this.db + .prepare( + `INSERT INTO legacy_operation_receipts ( + principal_id, operation_key, method, payload_hash, effect_id, response_json + ) VALUES (?, ?, ?, ?, ?, ?)` + ) + .run( + params.principalId, + params.operationKey, + params.method, + params.payloadHash, + params.effectId, + params.responseJson + ) + return this.getLegacyOperationReceipt( + params.principalId, + params.operationKey + ) as LegacyOperationReceiptRow + } + getUnreadMessages(toHandle: string, types?: MessageType[]): MessageRow[] { if (types && types.length > 0) { const placeholders = types.map(() => '?').join(',') return exposeMessageListTimestamps( this.db .prepare( - `SELECT * FROM messages WHERE to_handle = ? AND read = 0 AND type IN (${placeholders}) ORDER BY sequence` + `SELECT * FROM messages + WHERE to_handle = ? AND read = 0 AND delivery_contract = 'current_delivery' + AND type IN (${placeholders}) ORDER BY sequence` ) .all(toHandle, ...types) as MessageRow[] ) } return exposeMessageListTimestamps( this.db - .prepare('SELECT * FROM messages WHERE to_handle = ? AND read = 0 ORDER BY sequence') + .prepare( + `SELECT * FROM messages + WHERE to_handle = ? AND read = 0 AND delivery_contract = 'current_delivery' + ORDER BY sequence` + ) .all(toHandle) as MessageRow[] ) } @@ -1278,7 +3283,10 @@ export class OrchestrationDb { return exposeMessageListTimestamps( this.db .prepare( - `SELECT * FROM messages WHERE to_handle = ? AND read = 0 AND delivered_at IS NULL AND type IN (${placeholders}) ORDER BY sequence` + `SELECT * FROM messages + WHERE to_handle = ? AND read = 0 AND delivered_at IS NULL + AND delivery_contract = 'current_delivery' + AND type IN (${placeholders}) ORDER BY sequence` ) .all(toHandle, ...types) as MessageRow[] ) @@ -1286,7 +3294,10 @@ export class OrchestrationDb { return exposeMessageListTimestamps( this.db .prepare( - 'SELECT * FROM messages WHERE to_handle = ? AND read = 0 AND delivered_at IS NULL ORDER BY sequence' + `SELECT * FROM messages + WHERE to_handle = ? AND read = 0 AND delivered_at IS NULL + AND delivery_contract = 'current_delivery' + ORDER BY sequence` ) .all(toHandle) as MessageRow[] ) @@ -1704,6 +3715,7 @@ export class OrchestrationDb { createStartingWorkerDispatch(params: { taskId: string startOptions: unknown + launchTokenHash?: string retryOf?: string runtimeEpoch?: string federation?: { @@ -1789,10 +3801,10 @@ export class OrchestrationDb { this.db .prepare( `INSERT INTO dispatch_contexts ( - id, run_id, task_id, status, dispatched_at - ) VALUES (?, ?, ?, 'pending', datetime('now'))` + id, run_id, task_id, contract_version, launch_token_hash, status, dispatched_at + ) VALUES (?, ?, ?, ?, ?, 'pending', datetime('now'))` ) - .run(id, task.run_id, task.id) + .run(id, task.run_id, task.id, CURRENT_CONTRACT_VERSION, params.launchTokenHash ?? null) this.db .prepare( `INSERT INTO worker_dispatches ( @@ -1907,6 +3919,7 @@ export class OrchestrationDb { handle: string paneKey: string processIncarnation: string + launchTokenHash?: string worktreeId: string effects: unknown[] setupState: string @@ -1919,6 +3932,16 @@ export class OrchestrationDb { `Dispatch ${params.dispatchId} is not starting.` ) } + if ( + dispatch.launch_token_hash && + params.launchTokenHash && + dispatch.launch_token_hash !== params.launchTokenHash + ) { + throw new OrchestrationError( + 'request_mismatch', + `Dispatch ${params.dispatchId} already has a different launch-token commitment.` + ) + } const capability = `dcap_${randomBytes(32).toString('base64url')}` this.db.exec('BEGIN IMMEDIATE') try { @@ -1926,7 +3949,8 @@ export class OrchestrationDb { .prepare( `UPDATE dispatch_contexts SET assignee_handle = ?, assignee_pane_key = ?, process_incarnation = ?, - capability_hash = ?, capability_revoked_at = NULL + capability_hash = ?, launch_token_hash = COALESCE(launch_token_hash, ?), + capability_revoked_at = NULL WHERE id = ? AND status = 'pending'` ) .run( @@ -1934,6 +3958,7 @@ export class OrchestrationDb { params.paneKey, params.processIncarnation, hashDispatchCapability(capability), + params.launchTokenHash ?? null, params.dispatchId ) this.db @@ -2163,6 +4188,75 @@ export class OrchestrationDb { .get(dispatchId) as WorkerDispatchRow | undefined } + listLegacyWorkerTerminalRecoveryRows(): LegacyWorkerTerminalRecoveryRow[] { + return this.db + .prepare( + `SELECT dc.id AS dispatch_id, dc.task_id, dc.status AS dispatch_status, + dc.contract_version, dc.assignee_handle, dc.assignee_pane_key, + dc.process_incarnation, wd.state AS worker_state, wd.worktree_id, + wd.agent_terminal_handle + FROM dispatch_contexts dc + INNER JOIN worker_dispatches wd ON wd.dispatch_id = dc.id + WHERE wd.state IN ('starting', 'ready', 'start_unknown', 'stopping', 'stop_unknown') + ORDER BY dc.rowid` + ) + .all() as LegacyWorkerTerminalRecoveryRow[] + } + + reconcileMissingWorkerTerminal(dispatchId: string, reason: string): WorkerDispatchRow { + this.db.exec('BEGIN IMMEDIATE') + try { + const dispatch = this.getDispatchContextById(dispatchId) + const worker = this.getWorkerDispatch(dispatchId) + if (!dispatch || !worker) { + throw new OrchestrationError('dispatch_not_found', `Dispatch ${dispatchId} was not found.`) + } + if (['succeeded', 'failed', 'stopped', 'abandoned'].includes(worker.state)) { + this.db.exec('COMMIT') + return worker + } + + const activeDispatch = dispatch.status === 'pending' || dispatch.status === 'dispatched' + const stopWasPending = worker.state === 'stopping' || worker.state === 'stop_unknown' + if (activeDispatch) { + const failureCount = dispatch.failure_count + 1 + const dispatchStatus: DispatchStatus = failureCount >= 3 ? 'circuit_broken' : 'failed' + this.db + .prepare( + `UPDATE dispatch_contexts + SET status = ?, failure_count = ?, last_failure = ?, + completed_at = datetime('now'), + capability_revoked_at = COALESCE(capability_revoked_at, datetime('now')) + WHERE id = ? AND status IN ('pending', 'dispatched')` + ) + .run(dispatchStatus, failureCount, reason, dispatchId) + if (!stopWasPending) { + const taskStatus: TaskStatus = dispatchStatus === 'circuit_broken' ? 'failed' : 'ready' + this.db + .prepare( + `UPDATE tasks + SET status = ?, completed_at = CASE WHEN ? = 'failed' THEN datetime('now') ELSE NULL END + WHERE id = ? AND status IN ('dispatched', 'blocked')` + ) + .run(taskStatus, taskStatus, dispatch.task_id) + } + this.closeQuestionsForDispatch(dispatchId) + } + this.db + .prepare( + `UPDATE worker_dispatches + SET state = ?, stage = 'terminal_missing', last_error = ?, updated_at = datetime('now') + WHERE dispatch_id = ?` + ) + .run(stopWasPending ? 'stopped' : 'abandoned', reason, dispatchId) + this.db.exec('COMMIT') + return this.getWorkerDispatch(dispatchId) as WorkerDispatchRow + } catch (error) { + this.db.exec('ROLLBACK') + throw error + } + } + getFederatedDispatch(dispatchId: string): FederatedDispatchRow | undefined { return this.db .prepare('SELECT * FROM federated_dispatches WHERE dispatch_id = ?') @@ -3197,7 +5291,8 @@ export class OrchestrationDb { taskId: string, assigneeHandle: string, // Why: pane key is the remint-stable identity behind the handle — lets worker_done ownership survive handle reissue. - assigneePaneKey?: string + assigneePaneKey?: string, + launchTokenHash?: string ): DispatchContextRow { const task = this.getTask(taskId) if (!task) { @@ -3225,10 +5320,21 @@ export class OrchestrationDb { const id = generateId('ctx') this.db .prepare( - `INSERT INTO dispatch_contexts (id, run_id, task_id, assignee_handle, assignee_pane_key, status, failure_count, dispatched_at) - VALUES (?, ?, ?, ?, ?, 'dispatched', ?, datetime('now'))` + `INSERT INTO dispatch_contexts ( + id, run_id, task_id, contract_version, launch_token_hash, + assignee_handle, assignee_pane_key, status, failure_count, dispatched_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, 'dispatched', ?, datetime('now'))` + ) + .run( + id, + task.run_id, + taskId, + CURRENT_CONTRACT_VERSION, + launchTokenHash ?? null, + assigneeHandle, + assigneePaneKey ?? null, + priorFailures ) - .run(id, task.run_id, taskId, assigneeHandle, assigneePaneKey ?? null, priorFailures) this.hasAnyDispatchContextsCache = true this.db.prepare("UPDATE tasks SET status = 'dispatched' WHERE id = ?").run(taskId) @@ -3250,6 +5356,33 @@ export class OrchestrationDb { | undefined } + commitDispatchLaunchTokenHash(dispatchId: string, launchTokenHash: string): DispatchContextRow { + const dispatch = this.getDispatchContextById(dispatchId) + if (!dispatch) { + throw new OrchestrationError('dispatch_not_found', `Dispatch ${dispatchId} was not found.`) + } + if (dispatch.contract_version !== CURRENT_CONTRACT_VERSION) { + throw new OrchestrationError( + 'request_mismatch', + `Dispatch ${dispatchId} does not use the current contract.` + ) + } + if (dispatch.launch_token_hash && dispatch.launch_token_hash !== launchTokenHash) { + throw new OrchestrationError( + 'request_mismatch', + `Dispatch ${dispatchId} already has a different launch-token commitment.` + ) + } + this.db + .prepare( + `UPDATE dispatch_contexts + SET launch_token_hash = COALESCE(launch_token_hash, ?) + WHERE id = ?` + ) + .run(launchTokenHash, dispatchId) + return this.getDispatchContextById(dispatchId) as DispatchContextRow + } + mintDispatchCapability(params: { dispatchId: string paneKey: string @@ -3743,6 +5876,10 @@ export class OrchestrationDb { DELETE FROM remote_questions; DELETE FROM question_threads; DELETE FROM deliveries; + DELETE FROM legacy_mail_receipts; + DELETE FROM legacy_operation_receipts; + DELETE FROM legacy_compatibility_principals; + DELETE FROM legacy_adoptions; DELETE FROM federation_relay_items; DELETE FROM remote_dispatch_attachments; DELETE FROM federated_dispatches; @@ -3763,6 +5900,10 @@ export class OrchestrationDb { DELETE FROM decision_gates; DELETE FROM remote_questions; DELETE FROM question_threads; + DELETE FROM legacy_mail_receipts; + DELETE FROM legacy_operation_receipts; + DELETE FROM legacy_compatibility_principals; + DELETE FROM legacy_adoptions; DELETE FROM federation_relay_items; DELETE FROM remote_dispatch_attachments; DELETE FROM federated_dispatches; @@ -3776,6 +5917,7 @@ export class OrchestrationDb { resetMessages(): void { // Why: relay rows carry contiguous cross-server cursors, not just inbox history. this.runResetTransaction(` + DELETE FROM legacy_mail_receipts; DELETE FROM question_threads; DELETE FROM deliveries; DELETE FROM messages; diff --git a/src/main/runtime/orchestration/formatter.test.ts b/src/main/runtime/orchestration/formatter.test.ts index 54f13a477a4..b4732f04ac2 100644 --- a/src/main/runtime/orchestration/formatter.test.ts +++ b/src/main/runtime/orchestration/formatter.test.ts @@ -77,6 +77,14 @@ describe('formatMessageBanner', () => { ) }) + it('lets the CLI resolve the live sender for Run and Dispatch addresses', () => { + for (const to_handle of ['run:run_test', 'dispatch:dispatch_test']) { + const banner = formatMessageBanner(makeMessage({ to_handle })) + expect(banner).toContain('[Reply: orca orchestration reply --id msg_test1 --body "..."]') + expect(banner).not.toContain(`--from ${to_handle}`) + } + }) + it('marks legacy messages read-only without reply or acknowledgment affordances', () => { const banner = formatMessageBanner( makeMessage({ id: 'msg_legacy', run_id: 'run_legacy_local' }) @@ -88,6 +96,63 @@ describe('formatMessageBanner', () => { expect(banner).not.toContain('orchestration reply') }) + it('renders only attested actions for legacy compatibility authority', () => { + const banner = formatMessageBanner(makeMessage({ id: 'msg_legacy' }), { + authority: 'legacy_compatibility', + supportedActionHints: [ + 'orca orchestration reply --id msg_legacy --from term_coord --body "..."' + ] + }) + + expect(banner).toContain('[LEGACY COMPATIBILITY]') + expect(banner).toContain( + '[Supported action: orca orchestration reply --id msg_legacy --from term_coord --body "..."]' + ) + expect(banner).not.toContain('[Reply:') + expect(banner).not.toContain('acknowledgment') + }) + + it('warns that a bounded legacy recovery replay may already have been seen', () => { + const banner = formatMessageBanner(makeMessage(), { + authority: 'legacy_recovery_replay', + supportedActionHints: ['orca orchestration check --ack delivery_legacy'] + }) + + expect(banner).toContain('[LEGACY RECOVERY REPLAY — MAY HAVE BEEN SEEN]') + expect(banner).toContain('bounded recovery replay may already have been seen') + expect(banner).toContain('[Supported action: orca orchestration check --ack delivery_legacy]') + expect(banner).not.toContain('[Reply:') + }) + + it('does not infer live compatibility from legacy database provenance', () => { + const banner = formatMessageBanner(makeMessage({ run_id: 'run_legacy_local' }), { + supportedActionHints: ['orca orchestration check --ack delivery_legacy'] + }) + + expect(banner).toContain('[LEGACY READ-ONLY]') + expect(banner).not.toContain('Supported action:') + expect(banner).not.toContain('check --ack') + }) + + it('keeps adopted legacy and audit messages read-only without runtime authority', () => { + for (const deliveryContract of ['legacy_direct', 'audit_only'] as const) { + const banner = formatMessageBanner( + makeMessage({ run_id: 'run_adopted', delivery_contract: deliveryContract }) + ) + + expect(banner).toContain('[LEGACY READ-ONLY]') + expect(banner).not.toContain('[Reply:') + } + }) + + it('keeps current formatting unchanged when authority is explicit', () => { + const message = makeMessage({ id: 'msg_current' }) + + expect(formatMessageBanner(message, { authority: 'current' })).toBe( + formatMessageBanner(message) + ) + }) + it('ends with a separator line', () => { const banner = formatMessageBanner(makeMessage()) const lines = banner.split('\n') diff --git a/src/main/runtime/orchestration/formatter.ts b/src/main/runtime/orchestration/formatter.ts index e7ca26558bf..269ba29302f 100644 --- a/src/main/runtime/orchestration/formatter.ts +++ b/src/main/runtime/orchestration/formatter.ts @@ -4,19 +4,77 @@ import { ORCHESTRATION_LEGACY_RUN_ID } from '../../../shared/orchestration-rpc-c const BANNER_WIDTH = 60 const SEPARATOR = '─'.repeat(BANNER_WIDTH) -export function formatMessageBanner(msg: MessageRow): string { +export type MessageFormattingAuthority = + | 'current' + | 'legacy_compatibility' + | 'legacy_recovery_replay' + | 'legacy_read_only' + +export type MessageFormattingOptions = { + authority?: MessageFormattingAuthority + supportedActionHints?: readonly string[] +} + +function resolveAuthority( + msg: MessageRow, + authority: MessageFormattingAuthority | undefined +): MessageFormattingAuthority { + if (authority) { + return authority + } + return msg.run_id === ORCHESTRATION_LEGACY_RUN_ID || + msg.delivery_contract === 'legacy_direct' || + msg.delivery_contract === 'audit_only' + ? 'legacy_read_only' + : 'current' +} + +function appendLegacyGuidance( + lines: string[], + authority: MessageFormattingAuthority, + supportedActionHints: readonly string[] +): void { + if (authority === 'legacy_read_only') { + lines.push('[Inspection only: reply and acknowledgment are unavailable.]') + return + } + if (authority === 'legacy_compatibility') { + lines.push('[Use only the supported legacy action shown below.]') + } else if (authority === 'legacy_recovery_replay') { + lines.push( + '[This bounded recovery replay may already have been seen. Use only the action shown below.]' + ) + } + for (const hint of supportedActionHints) { + lines.push(`[Supported action: ${hint}]`) + } +} + +export function formatMessageBanner(msg: MessageRow, options: MessageFormattingOptions): string +export function formatMessageBanner(msg: MessageRow): string +export function formatMessageBanner( + msg: MessageRow, + options: MessageFormattingOptions = {} +): string { const priorityTag = msg.priority === 'urgent' ? ' [URGENT]' : msg.priority === 'high' ? ' [HIGH]' : '' - const legacyReadOnly = msg.run_id === ORCHESTRATION_LEGACY_RUN_ID - const authorityTag = legacyReadOnly ? ' [LEGACY READ-ONLY]' : '' + const authority = resolveAuthority(msg, options.authority) + const authorityTag = + authority === 'legacy_compatibility' + ? ' [LEGACY COMPATIBILITY]' + : authority === 'legacy_recovery_replay' + ? ' [LEGACY RECOVERY REPLAY — MAY HAVE BEEN SEEN]' + : authority === 'legacy_read_only' + ? ' [LEGACY READ-ONLY]' + : '' const senderName = msg.from_handle.toUpperCase() const header = `──── From: ${senderName} (${msg.from_handle})${priorityTag}${authorityTag} (${msg.type}) ────` const lines: string[] = [header] lines.push(`Subject: ${msg.subject}`) - if (legacyReadOnly) { - lines.push('[Inspection only: reply and acknowledgment are unavailable.]') + if (authority !== 'current') { + appendLegacyGuidance(lines, authority, options.supportedActionHints ?? []) } if (msg.body) { @@ -27,11 +85,12 @@ export function formatMessageBanner(msg: MessageRow): string { lines.push(`[Payload: ${msg.payload}]`) } - if (!legacyReadOnly) { - // Why: older shells can lack Orca's terminal identity environment. - lines.push( - `[Reply: orca orchestration reply --id ${msg.id} --from ${msg.to_handle} --body "..."]` - ) + if (authority === 'current') { + const explicitFrom = + msg.to_handle.startsWith('run:') || msg.to_handle.startsWith('dispatch:') + ? '' + : ` --from ${msg.to_handle}` + lines.push(`[Reply: orca orchestration reply --id ${msg.id}${explicitFrom} --body "..."]`) } lines.push(SEPARATOR) diff --git a/src/main/runtime/orchestration/orchestration-legacy-coordinator-authority-db.test.ts b/src/main/runtime/orchestration/orchestration-legacy-coordinator-authority-db.test.ts new file mode 100644 index 00000000000..e7212965b39 --- /dev/null +++ b/src/main/runtime/orchestration/orchestration-legacy-coordinator-authority-db.test.ts @@ -0,0 +1,381 @@ +import { rmSync } from 'node:fs' +import { afterEach, describe, expect, it } from 'vitest' +import Database from '../../sqlite/sync-database' +import { LEGACY_RUN_ID, OrchestrationDb } from './db' +import { + createLegacyStorageCutoverFixture, + type LegacyStorageCutoverFixture +} from './orchestration-legacy-storage-test-fixture' + +describe('OrchestrationDb legacy coordinator authority', () => { + let db: OrchestrationDb | undefined + let tempDir: string | undefined + + afterEach(() => { + db?.close() + db = undefined + if (tempDir) { + rmSync(tempDir, { recursive: true, force: true }) + tempDir = undefined + } + }) + + function createCutoverFixture(): LegacyStorageCutoverFixture { + const created = createLegacyStorageCutoverFixture() + tempDir = created.tempDir + return created.fixture + } + + function openAdoptedFixture(): { + fixture: LegacyStorageCutoverFixture + adoptedRunId: string + workerPrincipalId: string + } { + const fixture = createCutoverFixture() + db = new OrchestrationDb(fixture.dbPath) + const adoptedRunId = db.getLegacyAdoption()?.adopted_run_id as string + const worker = db.commitLegacyCompatibilityPrincipal({ + runId: adoptedRunId, + dispatchId: fixture.legacyDispatchId, + role: 'worker', + hostScope: 'local:runtime_1', + terminalHandle: 'term_legacy_worker', + paneKey: 'tab_legacy:33333333-3333-4333-8333-333333333333', + launchTokenHash: 'legacy_launch_hash', + processIncarnation: 'process_1' + }) + return { + fixture, + adoptedRunId, + workerPrincipalId: worker.principal.id + } + } + + it('records exact scheduler loss without admitting naturally settled coordinators', () => { + const fixture = createCutoverFixture() + const raw = new Database(fixture.dbPath) + raw.exec(` + UPDATE tasks + SET created_by_terminal_handle = 'term_manual_coord' + WHERE run_id = '${LEGACY_RUN_ID}'; + UPDATE messages + SET from_handle = CASE + WHEN from_handle = 'term_legacy_coord' THEN 'term_manual_coord' + ELSE from_handle + END, + to_handle = CASE + WHEN to_handle = 'term_legacy_coord' THEN 'term_manual_coord' + ELSE to_handle + END + WHERE run_id = '${LEGACY_RUN_ID}'; + INSERT INTO coordinator_runs ( + id, spec, status, coordinator_handle, created_at + ) VALUES ( + 'coordinator_manual', 'manual legacy coordinator', 'running', + 'term_manual_coord', datetime('now', '-1 minute') + ); + INSERT INTO coordinator_runs ( + id, spec, status, coordinator_handle, created_at, completed_at + ) VALUES ( + 'coordinator_natural', 'naturally completed coordinator', 'completed', + 'term_natural_coord', datetime('now', '-2 minutes'), + '2099-01-01T00:00:00.000Z' + ); + `) + raw.close() + + db = new OrchestrationDb(fixture.dbPath) + const adoptedRunId = db.getLegacyAdoption()?.adopted_run_id as string + expect(db.getActiveCoordinatorRun()).toBeUndefined() + const adoption = db.getLegacyAdoption()! + const migratedCoordinator = db.getCoordinatorRun('coordinator_manual')! + expect(migratedCoordinator).toMatchObject({ + status: 'failed', + completed_at: adoption.adopted_at, + scheduler_lost_at: adoption.adopted_at + }) + expect(db.getCoordinatorRun('coordinator_natural')).toMatchObject({ + status: 'completed', + completed_at: '2099-01-01T00:00:00.000Z', + scheduler_lost_at: null + }) + expect( + db.resolveLegacyCoordinatorCandidate({ + runId: adoptedRunId, + terminalHandle: 'term_manual_coord', + paneKey: 'tab_manual:44444444-4444-4444-8444-444444444444' + }) + ).toMatchObject({ terminalHandle: 'term_manual_coord' }) + expect(db.isLegacyCoordinatorHandle(adoptedRunId, 'term_natural_coord')).toBe(false) + + db.close() + db = new OrchestrationDb(fixture.dbPath) + expect(db.getLegacyAdoption()).toEqual(adoption) + expect(db.getCoordinatorRun('coordinator_manual')).toEqual(migratedCoordinator) + expect( + db.resolveLegacyCoordinatorCandidate({ + runId: adoptedRunId, + terminalHandle: 'term_manual_coord', + paneKey: 'tab_manual:44444444-4444-4444-8444-444444444444' + }) + ).toMatchObject({ terminalHandle: 'term_manual_coord' }) + }) + + it('fails closed when multiple pre-cutover schedulers lose authority', () => { + const fixture = createCutoverFixture() + const raw = new Database(fixture.dbPath) + raw.exec(` + INSERT INTO coordinator_runs ( + id, spec, status, coordinator_handle, created_at + ) VALUES + ('coordinator_first', 'first', 'running', 'term_first_coord', datetime('now', '-2 minutes')), + ('coordinator_second', 'second', 'running', 'term_second_coord', datetime('now', '-1 minute')); + `) + raw.close() + + db = new OrchestrationDb(fixture.dbPath) + const adoption = db.getLegacyAdoption()! + const adoptedRunId = adoption.adopted_run_id + expect(db.getActiveCoordinatorRun()).toBeUndefined() + expect(db.getCoordinatorRun('coordinator_first')).toMatchObject({ + status: 'failed', + scheduler_lost_at: adoption.adopted_at + }) + expect(db.getCoordinatorRun('coordinator_second')).toMatchObject({ + status: 'failed', + scheduler_lost_at: adoption.adopted_at + }) + expect( + db.resolveLegacyCoordinatorCandidate({ + runId: adoptedRunId, + terminalHandle: 'term_legacy_coord', + paneKey: 'tab_manual:44444444-4444-4444-8444-444444444444' + }) + ).toBeUndefined() + }) + + it('does not infer a coordinator from worker-to-worker legacy mail', () => { + const fixture = createCutoverFixture() + const raw = new Database(fixture.dbPath) + raw.exec(` + INSERT INTO tasks ( + id, run_id, spec, status, created_by_terminal_handle + ) VALUES ( + 'task_second_worker', '${LEGACY_RUN_ID}', 'second worker', + 'dispatched', 'term_legacy_coord' + ); + INSERT INTO dispatch_contexts ( + id, run_id, task_id, contract_version, assignee_handle, + assignee_pane_key, status, dispatched_at + ) VALUES ( + 'dispatch_second_worker', '${LEGACY_RUN_ID}', 'task_second_worker', 0, + 'term_second_worker', 'tab_second:55555555-5555-4555-8555-555555555555', + 'dispatched', datetime('now') + ); + INSERT INTO messages ( + id, run_id, from_handle, to_handle, subject + ) VALUES ( + 'message_worker_to_worker', '${LEGACY_RUN_ID}', 'term_legacy_worker', + 'term_second_worker', 'peer coordination' + ); + `) + raw.close() + + db = new OrchestrationDb(fixture.dbPath) + const adoptedRunId = db.getLegacyAdoption()?.adopted_run_id as string + expect(db.isLegacyCoordinatorHandle(adoptedRunId, 'term_second_worker')).toBe(false) + expect(db.isLegacyCoordinatorHandle(adoptedRunId, 'term_legacy_worker')).toBe(false) + expect(db.isLegacyCoordinatorHandle(adoptedRunId, 'term_legacy_coord')).toBe(true) + }) + + it('promotes only the unacknowledged coordinator recovery cohort during takeover', () => { + const fixture = createCutoverFixture() + db = new OrchestrationDb(fixture.dbPath) + const adoptedRunId = db.getLegacyAdoption()?.adopted_run_id as string + const unacknowledged = db.insertMessage({ + runId: adoptedRunId, + deliveryContract: 'legacy_direct', + from: 'term_legacy_worker', + to: 'term_legacy_coord', + subject: 'second recovered coordinator outcome' + }) + db.markAsRead([unacknowledged.id]) + const coordinator = db.commitLegacyCompatibilityPrincipal({ + runId: adoptedRunId, + role: 'coordinator', + hostScope: 'local:runtime_1', + terminalHandle: 'term_legacy_coord', + paneKey: 'tab_coord:44444444-4444-4444-8444-444444444444', + launchTokenHash: 'coord_launch_hash', + processIncarnation: 'process_coord' + }).principal + const firstPage = db.getLegacyMailPage({ principalId: coordinator.id, limit: 1 }) + expect(firstPage.recovery).toBe(true) + db.acknowledgeLegacyMail({ + principalId: coordinator.id, + messageIds: firstPage.messages.map((message) => message.id) + }) + const acknowledgedId = firstPage.messages[0].id + + db.bindRun({ + runId: adoptedRunId, + coordinatorHandle: 'term_current_coord', + coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111', + takeoverLegacy: true + }) + + expect(db.getMessageById(acknowledgedId)).toMatchObject({ + to_handle: 'term_legacy_coord', + delivery_contract: 'legacy_direct', + read: 1 + }) + expect(db.getMessageById(unacknowledged.id)).toMatchObject({ + to_handle: `run:${adoptedRunId}`, + delivery_contract: 'current_delivery', + read: 0 + }) + }) + + it('promotes a pre-cutover read recovery message when takeover precedes attestation', () => { + const fixture = createCutoverFixture() + db = new OrchestrationDb(fixture.dbPath) + const adoptedRunId = db.getLegacyAdoption()?.adopted_run_id as string + const recoveryMessageId = fixture.legacyMessageIds[1] + expect(db.getLegacyCoordinatorPrincipal(adoptedRunId)).toBeUndefined() + expect(db.getMessageById(recoveryMessageId)).toMatchObject({ + to_handle: 'term_legacy_coord', + delivery_contract: 'legacy_direct', + read: 1 + }) + + const run = db.bindRun({ + runId: adoptedRunId, + coordinatorHandle: 'term_current_coord', + coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111', + takeoverLegacy: true + })! + expect(db.getMessageById(recoveryMessageId)).toMatchObject({ + to_handle: `run:${adoptedRunId}`, + delivery_contract: 'current_delivery', + read: 0 + }) + + const first = db.getOrCreateRunDelivery({ + runId: adoptedRunId, + consumerGeneration: run.consumer_generation + })! + const replay = db.getOrCreateRunDelivery({ + runId: adoptedRunId, + consumerGeneration: run.consumer_generation + })! + expect(first.messages.map((message) => message.id)).toContain(recoveryMessageId) + expect(replay).toMatchObject({ replayed: true, delivery: { id: first.delivery.id } }) + + db.acknowledgeRunDelivery({ + runId: adoptedRunId, + consumerGeneration: run.consumer_generation, + deliveryId: first.delivery.id + }) + expect( + db + .getOrCreateRunDelivery({ + runId: adoptedRunId, + consumerGeneration: run.consumer_generation + }) + ?.messages.map((message) => message.id) ?? [] + ).not.toContain(recoveryMessageId) + }) + + it('promotes retained mail on an ordinary post-settlement coordinator replacement', () => { + const state = openAdoptedFixture() + const coordinator = db!.commitLegacyCompatibilityPrincipal({ + runId: state.adoptedRunId, + role: 'coordinator', + hostScope: 'local:runtime_1', + terminalHandle: 'term_legacy_coord', + paneKey: 'tab_coord:44444444-4444-4444-8444-444444444444', + launchTokenHash: 'coord_launch_hash', + processIncarnation: 'process_coord' + }).principal + const completion = db!.insertMessage({ + runId: state.adoptedRunId, + deliveryContract: 'legacy_direct', + from: 'term_legacy_worker', + to: 'term_legacy_coord', + subject: 'Completed before ordinary run-use', + type: 'worker_done' + }) + db!.settleWorkerReport({ + taskId: state.fixture.legacyTaskId, + dispatchId: state.fixture.legacyDispatchId, + outcome: 'succeeded', + result: 'done' + }) + + const run = db!.bindRun({ + runId: state.adoptedRunId, + coordinatorHandle: 'term_current_coord', + coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111' + })! + + expect(db!.getLegacyCompatibilityPrincipal(coordinator.id)?.status).toBe('revoked') + expect(db!.getMessageById(completion.id)).toMatchObject({ + to_handle: `run:${state.adoptedRunId}`, + delivery_contract: 'current_delivery', + read: 0 + }) + expect( + db! + .getOrCreateRunDelivery({ + runId: state.adoptedRunId, + consumerGeneration: run.consumer_generation + }) + ?.messages.map((message) => message.id) + ).toContain(completion.id) + }) + + it('acknowledges the exact current Run answer to a legacy ask after takeover', () => { + const state = openAdoptedFixture() + const ask = db!.commitLegacyAskOperation({ + principalId: state.workerPrincipalId, + operationKey: 'ask_before_takeover', + method: 'orchestration.ask', + payloadHash: 'ask_before_takeover_payload', + question: 'Continue after takeover?', + recipientHandle: 'term_legacy_coord' + }) + const run = db!.bindRun({ + runId: state.adoptedRunId, + coordinatorHandle: 'term_current_coord', + coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111', + takeoverLegacy: true + })! + const answered = db!.answerQuestion({ + messageId: ask.question.message_id, + runId: state.adoptedRunId, + consumerGeneration: run.consumer_generation, + body: 'continue' + }) + expect(answered.message).toMatchObject({ + from_handle: `run:${state.adoptedRunId}`, + to_handle: `dispatch:${state.fixture.legacyDispatchId}`, + delivery_contract: 'current_delivery' + }) + + const acknowledged = db!.acknowledgeLegacyQuestionAnswer({ + principalId: state.workerPrincipalId, + questionId: ask.question.message_id, + answerMessageId: answered.message.id + }) + + expect(acknowledged.duplicate).toBe(false) + expect(db!.getMessageById(answered.message.id)?.read).toBe(1) + expect( + db!.acknowledgeLegacyQuestionAnswer({ + principalId: state.workerPrincipalId, + questionId: ask.question.message_id, + answerMessageId: answered.message.id + }).duplicate + ).toBe(true) + }) +}) diff --git a/src/main/runtime/orchestration/orchestration-legacy-question-migration-db.test.ts b/src/main/runtime/orchestration/orchestration-legacy-question-migration-db.test.ts new file mode 100644 index 00000000000..6fc60695b3b --- /dev/null +++ b/src/main/runtime/orchestration/orchestration-legacy-question-migration-db.test.ts @@ -0,0 +1,255 @@ +import { rmSync } from 'node:fs' +import { afterEach, describe, expect, it } from 'vitest' +import Database from '../../sqlite/sync-database' +import { LEGACY_RUN_ID, OrchestrationDb } from './db' +import { createLegacyStorageCutoverFixture } from './orchestration-legacy-storage-test-fixture' + +describe('OrchestrationDb legacy question migration', () => { + let db: OrchestrationDb | undefined + let tempDir: string | undefined + + afterEach(() => { + db?.close() + if (tempDir) { + rmSync(tempDir, { recursive: true, force: true }) + } + }) + + function prepareTakenOverSchema19Question(readAndAcknowledge: boolean): { + dbPath: string + adoptedRunId: string + questionId: string + } { + const created = createLegacyStorageCutoverFixture() + tempDir = created.tempDir + db = new OrchestrationDb(created.fixture.dbPath) + const adoptedRunId = db.getLegacyAdoption()?.adopted_run_id as string + const questionId = readAndAcknowledge ? 'msg_schema19_acknowledged' : 'msg_schema19_unread' + db.insertMessage({ + id: questionId, + runId: adoptedRunId, + deliveryContract: 'legacy_direct', + from: 'term_legacy_worker', + to: 'term_legacy_coord', + subject: 'Question', + body: 'Continue after takeover?', + type: 'decision_gate', + threadId: questionId, + payload: JSON.stringify({ + taskId: created.fixture.legacyTaskId, + dispatchId: created.fixture.legacyDispatchId + }) + }) + if (readAndAcknowledge) { + db.markAsRead([questionId]) + } + const coordinator = db.commitLegacyCompatibilityPrincipal({ + runId: adoptedRunId, + role: 'coordinator', + hostScope: 'local:runtime_1', + terminalHandle: 'term_legacy_coord', + paneKey: 'tab_coord:44444444-4444-4444-8444-444444444444', + launchTokenHash: 'coord_launch_hash', + processIncarnation: 'process_coord' + }).principal + if (readAndAcknowledge) { + const recovery = db.getLegacyMailPage({ principalId: coordinator.id }) + expect(recovery.messages.map((message) => message.id)).toContain(questionId) + db.acknowledgeLegacyMail({ + principalId: coordinator.id, + messageIds: recovery.messages.map((message) => message.id) + }) + } + db.bindRun({ + runId: adoptedRunId, + coordinatorHandle: 'term_current_coord', + coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111', + takeoverLegacy: true + }) + expect(db.getMessageById(questionId)).toMatchObject( + readAndAcknowledge + ? { delivery_contract: 'legacy_direct', read: 1 } + : { delivery_contract: 'current_delivery', read: 0 } + ) + db.close() + db = undefined + + const raw = new Database(created.fixture.dbPath) + raw.exec('DROP TABLE question_threads') + raw.pragma('user_version = 19') + raw.close() + return { dbPath: created.fixture.dbPath, adoptedRunId, questionId } + } + + it('backfills a pre-question-thread ask and its accepted answer', () => { + const created = createLegacyStorageCutoverFixture() + tempDir = created.tempDir + db = new OrchestrationDb(created.fixture.dbPath) + const adoptedRunId = db.getLegacyAdoption()?.adopted_run_id as string + db.close() + db = undefined + + const raw = new Database(created.fixture.dbPath) + raw.exec(` + DROP TABLE question_threads; + INSERT INTO messages ( + id, run_id, delivery_contract, from_handle, to_handle, + subject, body, type, thread_id, payload + ) VALUES ( + 'msg_cutover', '${adoptedRunId}', 'legacy_direct', + 'term_legacy_worker', 'term_legacy_coord', 'Question', + 'Continue?', 'decision_gate', 'msg_cutover', + '{"taskId":"${created.fixture.legacyTaskId}","dispatchId":"${created.fixture.legacyDispatchId}"}' + ); + INSERT INTO messages ( + id, run_id, delivery_contract, from_handle, to_handle, + subject, body, thread_id + ) VALUES + ( + 'msg_cross_run_decoy', 'run_decoy', 'legacy_direct', + 'term_legacy_coord', 'term_legacy_worker', + 'Re: Question', 'cross-run decoy', 'msg_cutover' + ), + ( + 'msg_self_sent_decoy', '${adoptedRunId}', 'legacy_direct', + 'term_legacy_worker', 'term_legacy_worker', + 'Re: Question', 'self-sent decoy', 'msg_cutover' + ), + ( + 'msg_cutover_answer', '${adoptedRunId}', 'legacy_direct', + 'term_legacy_coord', 'term_legacy_worker', + 'Re: Question', 'yes', 'msg_cutover' + ); + `) + raw.pragma('user_version = 19') + raw.close() + + db = new OrchestrationDb(created.fixture.dbPath) + expect(db.getQuestion('msg_cutover')).toMatchObject({ + run_id: adoptedRunId, + dispatch_id: created.fixture.legacyDispatchId, + asker_handle: 'term_legacy_worker', + status: 'answered', + answer_message_id: 'msg_cutover_answer', + answer_body: 'yes' + }) + }) + + it('leaves a question pending when only cross-run and self-sent decoys exist', () => { + const created = createLegacyStorageCutoverFixture() + tempDir = created.tempDir + db = new OrchestrationDb(created.fixture.dbPath) + const adoptedRunId = db.getLegacyAdoption()?.adopted_run_id as string + db.close() + db = undefined + + const raw = new Database(created.fixture.dbPath) + raw.exec(` + DROP TABLE question_threads; + INSERT INTO messages ( + id, run_id, delivery_contract, from_handle, to_handle, + subject, body, type, thread_id, payload + ) VALUES ( + 'msg_pending_cutover', '${adoptedRunId}', 'legacy_direct', + 'term_legacy_worker', 'term_legacy_coord', 'Question', + 'Continue?', 'decision_gate', 'msg_pending_cutover', + '{"taskId":"${created.fixture.legacyTaskId}","dispatchId":"${created.fixture.legacyDispatchId}"}' + ); + INSERT INTO messages ( + id, run_id, delivery_contract, from_handle, to_handle, + subject, body, thread_id + ) VALUES + ( + 'msg_pending_cross_run_decoy', 'run_decoy', 'legacy_direct', + 'term_legacy_coord', 'term_legacy_worker', + 'Re: Question', 'cross-run decoy', 'msg_pending_cutover' + ), + ( + 'msg_pending_self_sent_decoy', '${adoptedRunId}', 'legacy_direct', + 'term_legacy_worker', 'term_legacy_worker', + 'Re: Question', 'self-sent decoy', 'msg_pending_cutover' + ); + `) + raw.pragma('user_version = 19') + raw.close() + + db = new OrchestrationDb(created.fixture.dbPath) + expect(db.getQuestion('msg_pending_cutover')).toMatchObject({ + run_id: adoptedRunId, + dispatch_id: created.fixture.legacyDispatchId, + asker_handle: 'term_legacy_worker', + status: 'pending', + answer_message_id: null, + answer_body: null + }) + }) + + it('uses task provenance to disambiguate same-handle same-second asks', () => { + const created = createLegacyStorageCutoverFixture() + tempDir = created.tempDir + const raw = new Database(created.fixture.dbPath) + raw.exec(` + INSERT INTO tasks ( + id, run_id, spec, status, created_by_terminal_handle, created_at + ) VALUES + ('task_question_target', '${LEGACY_RUN_ID}', 'target', 'dispatched', + 'term_legacy_coord', '2026-01-01 00:00:00'), + ('task_question_other', '${LEGACY_RUN_ID}', 'other', 'dispatched', + 'term_legacy_coord', '2026-01-01 00:00:00'); + INSERT INTO dispatch_contexts ( + id, run_id, task_id, contract_version, assignee_handle, + assignee_pane_key, status, dispatched_at, created_at + ) VALUES + ('dispatch_question_target', '${LEGACY_RUN_ID}', 'task_question_target', 0, + 'term_legacy_worker', 'tab_target:11111111-1111-4111-8111-111111111111', + 'dispatched', '2026-01-01 00:00:00', '2026-01-01 00:00:00'), + ('dispatch_question_other', '${LEGACY_RUN_ID}', 'task_question_other', 0, + 'term_legacy_worker', 'tab_other:22222222-2222-4222-8222-222222222222', + 'dispatched', '2026-01-01 00:00:00', '2026-01-01 00:00:00'); + INSERT INTO messages ( + id, run_id, delivery_contract, from_handle, to_handle, + subject, body, type, payload, created_at + ) VALUES ( + 'msg_question_task_only', '${LEGACY_RUN_ID}', 'legacy_direct', + 'term_legacy_worker', 'term_legacy_coord', 'Question', 'Continue?', + 'decision_gate', '{"taskId":"task_question_target","question":"Continue?","options":[]}', + '2026-01-01 00:00:00' + ); + `) + raw.close() + + db = new OrchestrationDb(created.fixture.dbPath) + expect(db.getQuestion('msg_question_task_only')).toMatchObject({ + dispatch_id: 'dispatch_question_target', + asker_handle: 'term_legacy_worker', + status: 'pending' + }) + }) + + it.each([ + ['promoted unread', false], + ['acknowledged recovery', true] + ] as const)('backfills and routes a %s v19 question after takeover', (_label, acknowledged) => { + const state = prepareTakenOverSchema19Question(acknowledged) + + db = new OrchestrationDb(state.dbPath) + expect(db.getQuestion(state.questionId)).toMatchObject({ + run_id: state.adoptedRunId, + status: 'pending' + }) + expect(db.getMessageById(state.questionId)).toMatchObject({ + to_handle: `run:${state.adoptedRunId}`, + delivery_contract: 'current_delivery', + read: 0 + }) + const run = db.getRun(state.adoptedRunId)! + expect( + db + .getOrCreateRunDelivery({ + runId: state.adoptedRunId, + consumerGeneration: run.consumer_generation + }) + ?.messages.map((message) => message.id) + ).toContain(state.questionId) + }) +}) diff --git a/src/main/runtime/orchestration/orchestration-legacy-storage-db.test.ts b/src/main/runtime/orchestration/orchestration-legacy-storage-db.test.ts new file mode 100644 index 00000000000..91f8fcacf39 --- /dev/null +++ b/src/main/runtime/orchestration/orchestration-legacy-storage-db.test.ts @@ -0,0 +1,751 @@ +import { rmSync } from 'node:fs' +import { afterEach, describe, expect, it } from 'vitest' +import Database from '../../sqlite/sync-database' +import { + CURRENT_CONTRACT_VERSION, + LEGACY_CONTRACT_VERSION, + LEGACY_RUN_ID, + OrchestrationDb +} from './db' +import { + createLegacyStorageCutoverFixture, + type LegacyStorageCutoverFixture +} from './orchestration-legacy-storage-test-fixture' + +describe('OrchestrationDb legacy contract storage', () => { + let db: OrchestrationDb | undefined + let tempDir: string | undefined + + afterEach(() => { + db?.close() + db = undefined + if (tempDir) { + rmSync(tempDir, { recursive: true, force: true }) + tempDir = undefined + } + }) + + function createCutoverFixture(): LegacyStorageCutoverFixture { + const created = createLegacyStorageCutoverFixture() + tempDir = created.tempDir + return created.fixture + } + + function openAdoptedFixture(): { + fixture: LegacyStorageCutoverFixture + adoptedRunId: string + workerPrincipalId: string + coordinatorPrincipalId: string + } { + const fixture = createCutoverFixture() + db = new OrchestrationDb(fixture.dbPath) + const adoptedRunId = db.getLegacyAdoption()?.adopted_run_id as string + const worker = db.commitLegacyCompatibilityPrincipal({ + runId: adoptedRunId, + dispatchId: fixture.legacyDispatchId, + role: 'worker', + hostScope: 'local:runtime_1', + terminalHandle: 'term_legacy_worker', + paneKey: 'tab_legacy:33333333-3333-4333-8333-333333333333', + launchTokenHash: 'legacy_launch_hash', + processIncarnation: 'process_1' + }) + const coordinator = db.commitLegacyCompatibilityPrincipal({ + runId: adoptedRunId, + role: 'coordinator', + hostScope: 'local:runtime_1', + terminalHandle: 'term_legacy_coord', + paneKey: 'tab_coord:44444444-4444-4444-8444-444444444444', + launchTokenHash: 'coord_launch_hash', + processIncarnation: 'process_coord' + }) + return { + fixture, + adoptedRunId, + workerPrincipalId: worker.principal.id, + coordinatorPrincipalId: coordinator.principal.id + } + } + + it('atomically rehomes the full graph, fences legacy Delivery, and preserves current rows', () => { + const fixture = createCutoverFixture() + db = new OrchestrationDb(fixture.dbPath) + const adoption = db.getLegacyAdoption() + const adoptedRunId = adoption?.adopted_run_id as string + const sqlite = (db as unknown as { db: Database.Database }).db + + expect(adoption).toMatchObject({ + source_run_id: LEGACY_RUN_ID, + scheduler_state_lost: 1 + }) + expect(db.getRun(adoptedRunId)).toMatchObject({ legacy: 0, consumer_generation: 0 }) + expect(db.listTasks({ runId: LEGACY_RUN_ID })).toEqual([]) + expect(db.getDispatchContextById(fixture.legacyDispatchId)).toMatchObject({ + run_id: adoptedRunId, + contract_version: LEGACY_CONTRACT_VERSION, + launch_token_hash: null + }) + expect(db.getGate(fixture.legacyGateId)).toMatchObject({ run_id: adoptedRunId }) + expect(db.getQuestion(fixture.legacyQuestionId)).toMatchObject({ run_id: adoptedRunId }) + expect(db.getMessageById(fixture.legacyMessageIds[0])).toMatchObject({ + run_id: adoptedRunId, + delivery_contract: 'legacy_direct' + }) + expect(db.getMessageById(fixture.rejectionMessageId)).toMatchObject({ + run_id: adoptedRunId, + delivery_contract: 'audit_only' + }) + expect(db.getMessageById(fixture.lookalikeMessageId)).toMatchObject({ + run_id: adoptedRunId, + delivery_contract: 'legacy_direct' + }) + for (const messageId of fixture.malformedRejectionMessageIds) { + expect(db.getMessageById(messageId)).toMatchObject({ + run_id: adoptedRunId, + delivery_contract: 'legacy_direct' + }) + } + expect( + sqlite.prepare('SELECT * FROM deliveries WHERE id = ?').get(fixture.legacyDeliveryId) + ).toMatchObject({ run_id: adoptedRunId, status: 'fenced' }) + expect(db.getDispatchContextById(fixture.currentDispatchId)).toMatchObject({ + run_id: fixture.currentRunId, + contract_version: CURRENT_CONTRACT_VERSION, + launch_token_hash: 'current_launch_hash' + }) + + db.close() + const partial = new Database(fixture.dbPath) + partial + .prepare('UPDATE tasks SET run_id = ? WHERE id = ?') + .run(LEGACY_RUN_ID, fixture.legacyTaskId) + partial.pragma('user_version = 19') + partial.close() + db = new OrchestrationDb(fixture.dbPath) + expect(db.getLegacyAdoption()?.adopted_run_id).toBe(adoptedRunId) + expect(db.getTask(fixture.legacyTaskId)?.run_id).toBe(adoptedRunId) + expect(db.getDispatchContextById(fixture.currentDispatchId)?.contract_version).toBe( + CURRENT_CONTRACT_VERSION + ) + expect(db.listTasks({ runId: LEGACY_RUN_ID })).toEqual([]) + }) + + it('ignores unrelated cross-Run anomalies while validating adopted rows', () => { + const fixture = createCutoverFixture() + const raw = new Database(fixture.dbPath) + raw + .prepare('UPDATE dispatch_contexts SET run_id = ? WHERE id = ?') + .run(fixture.unrelatedRunId, fixture.currentDispatchId) + raw.close() + + db = new OrchestrationDb(fixture.dbPath) + + expect(db.getLegacyAdoption()).toBeDefined() + expect(db.getDispatchContextById(fixture.currentDispatchId)?.run_id).toBe( + fixture.unrelatedRunId + ) + }) + + it('does not synthesize an adopted Run or compatibility authority for a fresh database', () => { + db = new OrchestrationDb(':memory:') + + expect(db.getLegacyAdoption()).toBeUndefined() + expect(db.listLegacyCompatibilityPrincipals(LEGACY_RUN_ID)).toEqual([]) + expect(db.listRuns()).toEqual([expect.objectContaining({ id: LEGACY_RUN_ID, legacy: 1 })]) + }) + + it('keeps current Delivery disjoint from adopted direct and audit-only mail', () => { + const state = openAdoptedFixture() + const run = db!.getRun(state.adoptedRunId) as NonNullable> + db!.insertMessage({ + runId: state.adoptedRunId, + from: 'current_worker', + to: `run:${state.adoptedRunId}`, + subject: 'current retry mail' + }) + + const delivery = db!.getOrCreateRunDelivery({ + runId: state.adoptedRunId, + consumerGeneration: run.consumer_generation + }) + + expect(delivery?.messages.map((message) => message.subject)).toEqual(['current retry mail']) + }) + + it('drains a durable recovery cohort in bounded replaying pages before unread mail', () => { + const state = openAdoptedFixture() + const first = db!.getLegacyMailPage({ principalId: state.workerPrincipalId, limit: 1 }) + const replay = db!.getLegacyMailPage({ principalId: state.workerPrincipalId, limit: 1 }) + + expect(first.recovery).toBe(true) + expect(replay.messages.map((message) => message.id)).toEqual( + first.messages.map((message) => message.id) + ) + expect(() => + db!.acknowledgeLegacyMail({ + principalId: state.workerPrincipalId, + messageIds: [state.fixture.legacyMessageIds[2]] + }) + ).toThrow(/current replay page/) + db!.acknowledgeLegacyMail({ + principalId: state.workerPrincipalId, + messageIds: first.messages.map((message) => message.id) + }) + const second = db!.getLegacyMailPage({ principalId: state.workerPrincipalId, limit: 1 }) + expect(second.recovery).toBe(true) + expect(second.messages[0].id).not.toBe(first.messages[0].id) + db!.acknowledgeLegacyMail({ + principalId: state.workerPrincipalId, + messageIds: second.messages.map((message) => message.id) + }) + expect(db!.getLegacyMailPage({ principalId: state.workerPrincipalId, limit: 1 }).recovery).toBe( + false + ) + }) + + it('returns complete addressed legacy history without changing read state', () => { + const state = openAdoptedFixture() + const unread = db!.insertMessage({ + runId: state.adoptedRunId, + deliveryContract: 'legacy_direct', + from: 'term_legacy_coord', + to: 'term_legacy_worker', + subject: 'unread history' + }) + db!.insertMessage({ + runId: state.adoptedRunId, + deliveryContract: 'legacy_direct', + from: 'other', + to: 'other', + subject: 'not addressed' + }) + + const history = db!.getLegacyMailHistory({ principalId: state.workerPrincipalId }) + + expect(history.recovery).toBe(false) + expect(history.messages.map((message) => message.subject)).toEqual([ + 'read worker mail', + 'second worker page', + 'unread history' + ]) + expect(db!.getMessageById(unread.id)?.read).toBe(0) + }) + + it('resolves legacy principals and completion evidence only within exact assignments', () => { + const state = openAdoptedFixture() + const taskId = db!.getDispatchContextById(state.fixture.legacyDispatchId)!.task_id + const payload = JSON.stringify({ taskId, dispatchId: state.fixture.legacyDispatchId }) + const completion = db!.insertMessage({ + runId: state.adoptedRunId, + deliveryContract: 'legacy_direct', + from: 'term_legacy_worker', + to: 'term_legacy_coord', + subject: 'Completed', + body: 'done', + type: 'worker_done', + payload + }) + + expect( + db!.resolveLegacyWorkerCandidate({ + runId: state.adoptedRunId, + terminalHandle: 'term_legacy_worker', + paneKey: 'tab_reminted:33333333-3333-4333-8333-333333333333', + taskId + }) + ).toMatchObject({ dispatch: { id: state.fixture.legacyDispatchId } }) + expect( + db!.resolveLegacyWorkerCandidate({ + runId: state.adoptedRunId, + terminalHandle: 'term_legacy_worker', + paneKey: 'tab_wrong:99999999-9999-4999-8999-999999999999', + taskId + }) + ).toBeUndefined() + expect( + db!.resolveLegacyCoordinatorCandidate({ + runId: state.adoptedRunId, + terminalHandle: 'term_legacy_coord', + paneKey: 'tab_coord:44444444-4444-4444-8444-444444444444' + }) + ).toMatchObject({ terminalHandle: 'term_legacy_coord' }) + expect( + db!.findLegacyWorkerCompletion({ + principalId: state.workerPrincipalId, + taskId, + recipientHandle: 'term_legacy_coord', + subject: 'Completed', + body: 'done', + payload + }) + ).toMatchObject({ id: completion.id }) + }) + + it('prevents an unproven coordinator from taking over an active adopted Run', () => { + const state = openAdoptedFixture() + expect(() => + db!.bindRun({ + runId: state.adoptedRunId, + coordinatorHandle: 'term_other', + coordinatorPaneKey: 'tab_other:55555555-5555-4555-8555-555555555555' + }) + ).toThrow(/attested coordinator may rebind/) + + const bound = db!.bindRun({ + runId: state.adoptedRunId, + coordinatorHandle: 'term_legacy_coord', + coordinatorPaneKey: 'tab_coord:44444444-4444-4444-8444-444444444444', + legacyCoordinatorAuthority: { + runId: state.adoptedRunId, + principalId: state.coordinatorPrincipalId, + terminalHandle: 'term_legacy_coord', + paneKey: 'tab_coord:44444444-4444-4444-8444-444444444444', + consumerGeneration: 0 + } + }) + expect(bound).toMatchObject({ coordinator_handle: 'term_legacy_coord' }) + expect( + db!.bindRun({ + runId: state.adoptedRunId, + coordinatorHandle: 'term_legacy_coord', + coordinatorPaneKey: 'tab_coord:44444444-4444-4444-8444-444444444444' + }) + ).toMatchObject({ coordinator_handle: 'term_legacy_coord' }) + }) + + it.each([ + ['generation', { consumerGeneration: 1 }], + ['principal', { principalId: 'legacy_principal_wrong' }], + ['handle', { terminalHandle: 'term_wrong' }], + ['pane', { paneKey: 'tab_wrong:99999999-9999-4999-8999-999999999999' }] + ] as const)('rejects stale legacy coordinator %s proof inside Run binding', (_label, patch) => { + const state = openAdoptedFixture() + const before = db!.getRun(state.adoptedRunId) + + expect(() => + db!.bindRun({ + runId: state.adoptedRunId, + coordinatorHandle: 'term_legacy_coord', + coordinatorPaneKey: 'tab_coord:44444444-4444-4444-8444-444444444444', + legacyCoordinatorAuthority: { + runId: state.adoptedRunId, + principalId: state.coordinatorPrincipalId, + terminalHandle: 'term_legacy_coord', + paneKey: 'tab_coord:44444444-4444-4444-8444-444444444444', + consumerGeneration: 0, + ...patch + } + }) + ).toThrow(/no longer has lifecycle authority/) + expect(db!.getRun(state.adoptedRunId)).toEqual(before) + }) + + it('revokes the legacy coordinator when a current coordinator takes over after settlement', () => { + const state = openAdoptedFixture() + db!.settleWorkerReport({ + taskId: state.fixture.legacyTaskId, + dispatchId: state.fixture.legacyDispatchId, + outcome: 'succeeded', + result: 'done' + }) + + expect( + db!.bindRun({ + runId: state.adoptedRunId, + coordinatorHandle: 'term_current_coord', + coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111' + }) + ).toMatchObject({ coordinator_handle: 'term_current_coord' }) + expect(db!.getLegacyCompatibilityPrincipal(state.coordinatorPrincipalId)?.status).toBe( + 'revoked' + ) + expect(() => + db!.commitLegacyCompatibilityPrincipal({ + runId: state.adoptedRunId, + role: 'coordinator', + hostScope: 'local:runtime_1', + terminalHandle: 'term_legacy_coord', + paneKey: 'tab_coord:44444444-4444-4444-8444-444444444444', + launchTokenHash: 'coord_launch_hash', + processIncarnation: 'process_coord' + }) + ).toThrow(/revoked/) + }) + + it('promotes only mail addressed to the replaced legacy coordinator', () => { + const state = openAdoptedFixture() + const coordinatorMail = db!.insertMessage({ + runId: state.adoptedRunId, + deliveryContract: 'legacy_direct', + from: 'term_legacy_worker', + to: 'term_legacy_coord', + subject: 'coordinator outcome' + }) + const workerMail = db!.insertMessage({ + runId: state.adoptedRunId, + deliveryContract: 'legacy_direct', + from: 'term_legacy_worker', + to: 'term_other_worker', + subject: 'worker-only guidance' + }) + + db!.bindRun({ + runId: state.adoptedRunId, + coordinatorHandle: 'term_current_coord', + coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111', + takeoverLegacy: true + }) + + expect(db!.getMessageById(coordinatorMail.id)).toMatchObject({ + to_handle: `run:${state.adoptedRunId}`, + delivery_contract: 'current_delivery' + }) + expect(db!.getMessageById(workerMail.id)).toMatchObject({ + to_handle: 'term_other_worker', + delivery_contract: 'legacy_direct' + }) + }) + + it('commits legacy messages, lifecycle effects, and invocation receipts exactly once', () => { + const state = openAdoptedFixture() + const params = { + principalId: state.workerPrincipalId, + operationKey: 'invocation_1', + method: 'orchestration.send', + payloadHash: 'payload_1', + message: { + to: 'term_legacy_coord', + subject: 'alive', + type: 'heartbeat' as const + }, + lifecycle: { kind: 'heartbeat' as const, at: '2026-07-28T12:00:00.000Z' } + } + + const first = db!.commitLegacyLifecycleOperation(params) + const replay = db!.commitLegacyLifecycleOperation(params) + + expect(replay).toMatchObject({ + duplicate: true, + message: { id: first.message.id }, + receipt: { effect_id: first.message.id } + }) + expect(db!.getDispatchContextById(state.fixture.legacyDispatchId)?.last_heartbeat_at).toBe( + '2026-07-28T12:00:00.000Z' + ) + expect(db!.getUnreadMessages('term_legacy_coord')).toEqual([]) + expect(db!.getUndeliveredUnreadMessages('term_legacy_coord')).toEqual([]) + expect(() => + db!.commitLegacyLifecycleOperation({ ...params, payloadHash: 'different' }) + ).toThrow(/different input/) + }) + + it('reconstructs a matching pre-receipt settlement without changing its persisted outcome', () => { + const state = openAdoptedFixture() + const accepted = db!.insertMessage({ + runId: state.adoptedRunId, + deliveryContract: 'legacy_direct', + from: 'term_legacy_worker', + to: 'term_legacy_coord', + subject: 'Completed', + type: 'worker_done' + }) + expect( + db!.settleWorkerReport({ + taskId: db!.getDispatchContextById(state.fixture.legacyDispatchId)!.task_id, + dispatchId: state.fixture.legacyDispatchId, + outcome: 'succeeded', + result: 'accepted by A' + }) + ).toMatchObject({ action: 'settled', duplicate: false }) + + const reconstructed = db!.commitLegacyLifecycleOperation({ + principalId: state.workerPrincipalId, + operationKey: `settlement:${state.fixture.legacyDispatchId}`, + method: 'orchestration.send', + payloadHash: 'settlement_payload', + message: { + existingId: accepted.id, + to: 'term_legacy_coord', + subject: 'Completed', + type: 'worker_done' + }, + lifecycle: { + kind: 'worker_report', + taskId: db!.getDispatchContextById(state.fixture.legacyDispatchId)!.task_id, + outcome: 'succeeded', + result: 'accepted by A' + } + }) + + expect(reconstructed).toMatchObject({ + duplicate: false, + message: { id: accepted.id }, + settlement: { action: 'settled', outcome: 'succeeded', duplicate: true } + }) + expect(db!.getLegacyCompatibilityPrincipal(state.workerPrincipalId)?.status).toBe('settled') + }) + + it('reconstructs a read pre-takeover completion through its original legacy route', () => { + const state = openAdoptedFixture() + const taskId = db!.getDispatchContextById(state.fixture.legacyDispatchId)!.task_id + const accepted = db!.insertMessage({ + runId: state.adoptedRunId, + deliveryContract: 'legacy_direct', + from: 'term_legacy_worker', + to: 'term_legacy_coord', + subject: 'Completed', + body: 'accepted before takeover', + type: 'worker_done' + }) + db!.markAsRead([accepted.id]) + db!.settleWorkerReport({ + taskId, + dispatchId: state.fixture.legacyDispatchId, + outcome: 'succeeded', + result: 'accepted before takeover' + }) + db!.bindRun({ + runId: state.adoptedRunId, + coordinatorHandle: 'term_current_coord', + coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111', + takeoverLegacy: true + }) + const beforeIds = db!.getInbox(100).map((message) => message.id) + + const reconstructed = db!.commitLegacyLifecycleOperation({ + principalId: state.workerPrincipalId, + operationKey: 'read_completion_after_takeover', + method: 'orchestration.send', + payloadHash: 'read_completion_after_takeover_payload', + message: { + existingId: accepted.id, + to: 'term_legacy_coord', + subject: 'Completed', + body: 'accepted before takeover', + type: 'worker_done' + }, + lifecycle: { + kind: 'worker_report', + taskId, + outcome: 'succeeded', + result: 'accepted before takeover' + } + }) + + expect(reconstructed).toMatchObject({ + duplicate: false, + message: { + id: accepted.id, + to_handle: 'term_legacy_coord', + delivery_contract: 'legacy_direct' + }, + settlement: { action: 'settled', outcome: 'succeeded', duplicate: true } + }) + expect(db!.getInbox(100).map((message) => message.id)).toEqual(beforeIds) + }) + + it('rejects cross-cutover completion reconstruction for another recipient', () => { + const state = openAdoptedFixture() + const taskId = db!.getDispatchContextById(state.fixture.legacyDispatchId)!.task_id + const payload = JSON.stringify({ taskId, dispatchId: state.fixture.legacyDispatchId }) + const foreign = db!.insertMessage({ + runId: state.adoptedRunId, + deliveryContract: 'legacy_direct', + from: 'term_legacy_worker', + to: 'term_other_coord', + subject: 'Completed', + body: 'accepted elsewhere', + type: 'worker_done', + payload + }) + db!.settleWorkerReport({ + taskId, + dispatchId: state.fixture.legacyDispatchId, + outcome: 'succeeded', + result: 'accepted elsewhere' + }) + const beforeIds = db!.getInbox(100).map((message) => message.id) + + expect( + db!.findLegacyWorkerCompletion({ + principalId: state.workerPrincipalId, + taskId, + recipientHandle: 'term_legacy_coord', + subject: 'Completed', + body: 'accepted elsewhere', + payload + }) + ).toBeUndefined() + expect(() => + db!.commitLegacyLifecycleOperation({ + principalId: state.workerPrincipalId, + operationKey: 'wrong_recipient_retry', + method: 'orchestration.send', + payloadHash: 'wrong_recipient_retry_payload', + message: { + to: 'term_legacy_coord', + subject: 'Completed', + body: 'accepted elsewhere', + type: 'worker_done', + payload + }, + lifecycle: { + kind: 'worker_report', + taskId, + outcome: 'succeeded', + result: 'accepted elsewhere' + } + }) + ).toThrow(/settled/) + expect(() => + db!.commitLegacyLifecycleOperation({ + principalId: state.workerPrincipalId, + operationKey: 'wrong_recipient_reconstruction', + method: 'orchestration.send', + payloadHash: 'wrong_recipient_payload', + message: { + existingId: foreign.id, + to: 'term_legacy_coord', + subject: 'Completed', + body: 'accepted elsewhere', + type: 'worker_done', + payload + }, + lifecycle: { + kind: 'worker_report', + taskId, + outcome: 'succeeded', + result: 'accepted elsewhere' + } + }) + ).toThrow(/does not match this principal/) + expect(db!.getInbox(100).map((message) => message.id)).toEqual(beforeIds) + expect( + db!.getLegacyOperationReceipt(state.workerPrincipalId, 'wrong_recipient_reconstruction') + ).toBeUndefined() + expect( + db!.getLegacyOperationReceipt(state.workerPrincipalId, 'wrong_recipient_retry') + ).toBeUndefined() + expect(db!.getTask(taskId)).toMatchObject({ + status: 'completed', + result: 'accepted elsewhere' + }) + }) + + it('uses invocation identity for repeated asks and atomically conflicts divergent replies', () => { + const state = openAdoptedFixture() + const ask = { + principalId: state.workerPrincipalId, + operationKey: 'ask_invocation_1', + method: 'orchestration.ask', + payloadHash: 'ask_payload', + question: 'Same text?', + options: ['yes', 'no'], + recipientHandle: 'term_legacy_coord' + } + const first = db!.commitLegacyAskOperation(ask) + const replay = db!.commitLegacyAskOperation(ask) + const repeated = db!.commitLegacyAskOperation({ + ...ask, + operationKey: 'ask_invocation_2' + }) + + expect(replay).toMatchObject({ duplicate: true, question: { message_id: first.message.id } }) + expect(repeated.message.id).not.toBe(first.message.id) + expect(db!.findPendingLegacyQuestions(ask)).toHaveLength(2) + expect( + db!.findPendingLegacyQuestions({ + ...ask, + question: ' Same text?\r\n', + options: ['yes ', ' no'] + }) + ).toHaveLength(2) + + const inherited = db!.createQuestion({ + runId: state.adoptedRunId, + dispatchId: state.fixture.legacyDispatchId, + askerHandle: 'term_legacy_worker', + question: 'Inherited?' + }) + const sqlite = (db as unknown as { db: Database.Database }).db + sqlite + .prepare( + `UPDATE messages + SET from_handle = 'term_legacy_worker', to_handle = 'term_legacy_coord', + delivery_contract = 'legacy_direct' + WHERE id = ?` + ) + .run(inherited.message.id) + const inheritedAsk = { + ...ask, + operationKey: 'ask_inherited_1', + payloadHash: 'ask_inherited_payload_1', + question: 'Inherited?', + options: [], + existingQuestionId: inherited.message.id + } + const adopted = db!.commitLegacyAskOperation(inheritedAsk) + const distinct = db!.commitLegacyAskOperation({ + ...inheritedAsk, + operationKey: 'ask_inherited_2', + payloadHash: 'ask_inherited_payload_2' + }) + expect(adopted.message.id).toBe(inherited.message.id) + expect(distinct.message.id).not.toBe(inherited.message.id) + + const reply = { + principalId: state.coordinatorPrincipalId, + operationKey: 'reply_invocation_1', + method: 'orchestration.reply', + payloadHash: 'reply_payload', + questionId: first.message.id, + body: 'yes' + } + const answered = db!.commitLegacyReplyOperation(reply) + const answerReplay = db!.commitLegacyReplyOperation(reply) + expect(answerReplay).toMatchObject({ + duplicate: true, + message: { id: answered.message.id } + }) + expect( + db! + .findLegacyQuestionsBySemanticIdentity(ask) + .find((row) => row.question.message_id === first.message.id) + ).toMatchObject({ question: { status: 'answered' }, answerAcknowledged: false }) + db!.acknowledgeLegacyQuestionAnswer({ + principalId: state.workerPrincipalId, + questionId: first.message.id, + answerMessageId: answered.message.id + }) + expect( + db! + .findLegacyQuestionsBySemanticIdentity(ask) + .find((row) => row.question.message_id === first.message.id) + ).toMatchObject({ answerAcknowledged: true }) + expect(() => + db!.commitLegacyReplyOperation({ + ...reply, + operationKey: 'reply_invocation_2', + payloadHash: 'different_reply', + body: 'no' + }) + ).toThrow(/different answer/) + + const currentTask = db!.createTask({ runId: state.adoptedRunId, spec: 'current retry' }) + const currentDispatch = db!.createDispatchContext(currentTask.id, 'term_current_retry') + const currentQuestion = db!.createQuestion({ + runId: state.adoptedRunId, + dispatchId: currentDispatch.id, + askerHandle: 'term_current_retry', + question: 'Current question?' + }) + expect(() => + db!.commitLegacyReplyOperation({ + ...reply, + operationKey: 'reply_current_question', + payloadHash: 'current_question', + questionId: currentQuestion.message.id + }) + ).toThrow(/not actionable/) + }) +}) diff --git a/src/main/runtime/orchestration/orchestration-legacy-storage-test-fixture.ts b/src/main/runtime/orchestration/orchestration-legacy-storage-test-fixture.ts new file mode 100644 index 00000000000..d28498e9c2f --- /dev/null +++ b/src/main/runtime/orchestration/orchestration-legacy-storage-test-fixture.ts @@ -0,0 +1,206 @@ +import { mkdtempSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import Database from '../../sqlite/sync-database' +import { LEGACY_RUN_ID, OrchestrationDb } from './db' + +export type LegacyStorageCutoverFixture = { + dbPath: string + currentRunId: string + unrelatedRunId: string + currentDispatchId: string + legacyTaskId: string + legacyDispatchId: string + legacyGateId: string + legacyMessageIds: string[] + legacyQuestionId: string + legacyDeliveryId: string + rejectionMessageId: string + lookalikeMessageId: string + malformedRejectionMessageIds: string[] +} + +export function createLegacyStorageCutoverFixture(): { + fixture: LegacyStorageCutoverFixture + tempDir: string +} { + const tempDir = mkdtempSync(join(tmpdir(), 'orca-legacy-storage-')) + const dbPath = join(tempDir, 'orchestration.db') + const first = new OrchestrationDb(dbPath) + const currentRun = first.createRun({ + objective: 'Current work', + coordinatorHandle: 'term_current_coord', + coordinatorPaneKey: 'tab_current:11111111-1111-4111-8111-111111111111' + }) + const currentTask = first.createTask({ spec: 'current', runId: currentRun.id }) + const unrelatedRun = first.createRun({ + objective: 'Unrelated current work', + coordinatorHandle: 'term_unrelated_coord', + coordinatorPaneKey: 'tab_unrelated:55555555-5555-4555-8555-555555555555' + }) + const currentDispatch = first.createDispatchContext( + currentTask.id, + 'term_current_worker', + 'tab_current:22222222-2222-4222-9222-222222222222', + 'current_launch_hash' + ) + first.insertMessage({ + runId: currentRun.id, + from: 'term_current_worker', + to: `run:${currentRun.id}`, + subject: 'current mail' + }) + + const legacyTask = first.createTask({ + spec: 'legacy', + createdByTerminalHandle: 'term_legacy_coord' + }) + first.createDispatchContext( + legacyTask.id, + 'term_legacy_worker', + 'tab_legacy:33333333-3333-4333-8333-333333333333' + ) + const legacyGate = first.createGate({ + taskId: legacyTask.id, + question: 'Retained gate?' + }) + first.resolveGate(legacyGate.id, 'continue') + const retryDispatch = first.createDispatchContext( + legacyTask.id, + 'term_legacy_worker', + 'tab_legacy:33333333-3333-4333-8333-333333333333' + ) + const legacyMessages = [ + first.insertMessage({ + from: 'term_legacy_coord', + to: 'term_legacy_worker', + subject: 'read worker mail' + }), + first.insertMessage({ + from: 'term_legacy_worker', + to: 'term_legacy_coord', + subject: 'read coordinator mail' + }), + first.insertMessage({ + from: 'term_legacy_coord', + to: 'term_legacy_worker', + subject: 'second worker page' + }) + ] + first.markAsRead(legacyMessages.map((message) => message.id)) + const question = first.createQuestion({ + runId: LEGACY_RUN_ID, + dispatchId: retryDispatch.id, + askerHandle: 'term_legacy_worker', + question: 'Retained question?' + }) + const rejection = first.insertMessage({ + from: 'term_legacy_worker', + to: 'term_legacy_coord', + subject: 'Rejected heartbeat', + type: 'heartbeat', + payload: JSON.stringify({ _orcaLifecycleRejection: { code: 'migration', reason: 'cutover' } }) + }) + const lookalike = first.insertMessage({ + from: 'term_legacy_worker', + to: 'term_legacy_coord', + subject: 'Ordinary legacy mail', + payload: JSON.stringify({ + userData: { _orcaLifecycleRejection: { code: 'not-a-top-level-audit-marker' } } + }) + }) + const malformedRejections = [ + first.insertMessage({ + from: 'term_legacy_worker', + to: 'term_legacy_coord', + subject: 'Invalid JSON marker', + payload: '{"_orcaLifecycleRejection":' + }), + first.insertMessage({ + from: 'term_legacy_worker', + to: 'term_legacy_coord', + subject: 'Array marker', + payload: JSON.stringify({ _orcaLifecycleRejection: [] }) + }), + first.insertMessage({ + from: 'term_legacy_worker', + to: 'term_legacy_coord', + subject: 'String marker', + payload: JSON.stringify({ _orcaLifecycleRejection: 'migration' }) + }), + first.insertMessage({ + from: 'term_legacy_worker', + to: 'term_legacy_coord', + subject: 'Incomplete marker', + payload: JSON.stringify({ _orcaLifecycleRejection: { code: 'migration' } }) + }), + first.insertMessage({ + from: 'term_legacy_worker', + to: 'term_legacy_coord', + subject: 'Non-string marker fields', + payload: JSON.stringify({ _orcaLifecycleRejection: { code: 19, reason: false } }) + }), + first.insertMessage({ + from: 'term_legacy_worker', + to: 'term_legacy_coord', + subject: 'Array root', + payload: JSON.stringify([ + { _orcaLifecycleRejection: { code: 'migration', reason: 'nested' } } + ]) + }), + first.insertMessage({ + from: 'term_legacy_worker', + to: 'term_legacy_coord', + subject: 'String root', + payload: JSON.stringify('_orcaLifecycleRejection') + }) + ] + first.close() + + const raw = new Database(dbPath) + const legacyDeliveryId = 'delivery_legacy_outstanding' + raw + .prepare( + `INSERT INTO deliveries ( + id, run_id, consumer_generation, message_ids, status + ) VALUES (?, ?, 0, ?, 'outstanding')` + ) + .run(legacyDeliveryId, LEGACY_RUN_ID, JSON.stringify([legacyMessages[0].id])) + raw + .prepare("UPDATE messages SET delivery_contract = 'legacy_direct' WHERE id = ?") + .run(rejection.id) + const seedAuditOnly = raw.prepare( + "UPDATE messages SET delivery_contract = 'audit_only' WHERE id = ?" + ) + for (const message of [lookalike, ...malformedRejections]) { + seedAuditOnly.run(message.id) + } + raw.exec(` + DROP INDEX IF EXISTS idx_messages_delivery_contract; + DROP TABLE legacy_mail_receipts; + DROP TABLE legacy_operation_receipts; + DROP TABLE legacy_compatibility_principals; + DROP TABLE legacy_adoptions; + `) + raw.pragma('user_version = 18') + raw.close() + + return { + tempDir, + fixture: { + dbPath, + currentRunId: currentRun.id, + unrelatedRunId: unrelatedRun.id, + currentDispatchId: currentDispatch.id, + legacyTaskId: legacyTask.id, + legacyDispatchId: retryDispatch.id, + legacyGateId: legacyGate.id, + legacyMessageIds: legacyMessages.map((message) => message.id), + legacyQuestionId: question.message.id, + legacyDeliveryId, + rejectionMessageId: rejection.id, + lookalikeMessageId: lookalike.id, + malformedRejectionMessageIds: malformedRejections.map((message) => message.id) + } + } +} diff --git a/src/main/runtime/orchestration/orchestration-legacy-worker-terminal-recovery.test.ts b/src/main/runtime/orchestration/orchestration-legacy-worker-terminal-recovery.test.ts new file mode 100644 index 00000000000..d425a52f2cb --- /dev/null +++ b/src/main/runtime/orchestration/orchestration-legacy-worker-terminal-recovery.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it } from 'vitest' +import type { LegacyWorkerTerminalRecoveryRow } from './types' +import { planLegacyWorkerTerminalRecovery } from './orchestration-legacy-worker-terminal-recovery' + +const LEAF_ID = '11111111-1111-4111-8111-111111111111' +const INCARNATION_ID = '22222222-2222-4222-8222-222222222222' + +function recoveryRow( + overrides: Partial = {} +): LegacyWorkerTerminalRecoveryRow { + return { + dispatch_id: 'dispatch-1', + task_id: 'task-1', + dispatch_status: 'completed', + contract_version: 0, + assignee_handle: 'term-worker', + assignee_pane_key: `tab-worker:${LEAF_ID}`, + process_incarnation: `ssh:ssh-1@@pty-worker:${INCARNATION_ID}`, + worker_state: 'ready', + worktree_id: 'repo::/workspace', + agent_terminal_handle: 'term-worker', + ...overrides + } +} + +describe('legacy worker terminal recovery planning', () => { + it('retains completed Dispatches when the worker process row is still live', () => { + expect(planLegacyWorkerTerminalRecovery([recoveryRow()])).toEqual({ + blockedPanes: [ + { + worktreeId: 'repo::/workspace', + paneKey: `tab-worker:${LEAF_ID}`, + contractVersion: 0 + } + ], + candidates: [ + expect.objectContaining({ + dispatchId: 'dispatch-1', + ptyId: 'ssh:ssh-1@@pty-worker', + incarnationId: INCARNATION_ID + }) + ], + ambiguousDispatchIds: [] + }) + }) + + it('blocks resume but refuses recovery when durable handles disagree', () => { + expect( + planLegacyWorkerTerminalRecovery([recoveryRow({ agent_terminal_handle: 'term-replacement' })]) + ).toEqual({ + blockedPanes: [ + { + worktreeId: 'repo::/workspace', + paneKey: `tab-worker:${LEAF_ID}`, + contractVersion: 0 + } + ], + candidates: [], + ambiguousDispatchIds: [] + }) + }) + + it('fails closed when two Dispatches claim one terminal identity', () => { + const plan = planLegacyWorkerTerminalRecovery([ + recoveryRow(), + recoveryRow({ dispatch_id: 'dispatch-2', task_id: 'task-2' }) + ]) + + expect(plan.candidates).toEqual([]) + expect(plan.ambiguousDispatchIds).toEqual(['dispatch-1', 'dispatch-2']) + expect(plan.blockedPanes).toHaveLength(1) + }) + + it('does not trust malformed pane or process identities', () => { + const plan = planLegacyWorkerTerminalRecovery([ + recoveryRow({ + assignee_pane_key: 'tab-worker:1', + process_incarnation: 'runtime:pty:generation' + }) + ]) + + expect(plan).toEqual({ + blockedPanes: [], + candidates: [], + ambiguousDispatchIds: [] + }) + }) +}) diff --git a/src/main/runtime/orchestration/orchestration-legacy-worker-terminal-recovery.ts b/src/main/runtime/orchestration/orchestration-legacy-worker-terminal-recovery.ts new file mode 100644 index 00000000000..c994101aeda --- /dev/null +++ b/src/main/runtime/orchestration/orchestration-legacy-worker-terminal-recovery.ts @@ -0,0 +1,119 @@ +import { isPtyIncarnationId, type PtyIncarnationId } from '../../../shared/pty-incarnation' +import { parsePaneKey } from '../../../shared/stable-pane-id' +import type { LegacyWorkerTerminalRecoveryRow } from './types' + +export type LegacyWorkerTerminalRecoveryCandidate = { + dispatchId: string + dispatchStatus: LegacyWorkerTerminalRecoveryRow['dispatch_status'] + contractVersion: number + taskId: string + worktreeId: string + terminalHandle: string + paneKey: string + tabId: string + leafId: string + processIncarnation: string + ptyId: string + incarnationId: PtyIncarnationId +} + +export type LegacyWorkerTerminalRecoveryPlan = { + blockedPanes: { worktreeId: string; paneKey: string; contractVersion: number }[] + candidates: LegacyWorkerTerminalRecoveryCandidate[] + ambiguousDispatchIds: string[] +} + +function parseProcessIncarnation( + value: string +): { ptyId: string; incarnationId: PtyIncarnationId } | null { + const separator = value.lastIndexOf(':') + if (separator <= 0) { + return null + } + const ptyId = value.slice(0, separator) + const incarnationId = value.slice(separator + 1) + return ptyId && isPtyIncarnationId(incarnationId) ? { ptyId, incarnationId } : null +} + +function countCandidateKeys( + candidates: readonly LegacyWorkerTerminalRecoveryCandidate[], + select: (candidate: LegacyWorkerTerminalRecoveryCandidate) => string +): Map { + const counts = new Map() + for (const candidate of candidates) { + const key = select(candidate) + counts.set(key, (counts.get(key) ?? 0) + 1) + } + return counts +} + +export function planLegacyWorkerTerminalRecovery( + rows: readonly LegacyWorkerTerminalRecoveryRow[] +): LegacyWorkerTerminalRecoveryPlan { + const blockedPanes = new Map< + string, + { worktreeId: string; paneKey: string; contractVersion: number } + >() + const parsedCandidates: LegacyWorkerTerminalRecoveryCandidate[] = [] + for (const row of rows) { + const worktreeId = row.worktree_id?.trim() + const paneKey = row.assignee_pane_key?.trim() + const pane = paneKey ? parsePaneKey(paneKey) : null + if (worktreeId && paneKey && pane) { + blockedPanes.set(`${worktreeId}\0${paneKey}`, { + worktreeId, + paneKey, + contractVersion: row.contract_version + }) + } + const terminalHandle = row.assignee_handle?.trim() + const workerHandle = row.agent_terminal_handle?.trim() + const processIncarnation = row.process_incarnation?.trim() + const process = processIncarnation ? parseProcessIncarnation(processIncarnation) : null + if ( + !worktreeId || + !paneKey || + !pane || + !terminalHandle || + terminalHandle !== workerHandle || + !processIncarnation || + !process + ) { + continue + } + parsedCandidates.push({ + dispatchId: row.dispatch_id, + dispatchStatus: row.dispatch_status, + contractVersion: row.contract_version, + taskId: row.task_id, + worktreeId, + terminalHandle, + paneKey, + tabId: pane.tabId, + leafId: pane.leafId, + processIncarnation, + ptyId: process.ptyId, + incarnationId: process.incarnationId + }) + } + + const identityCounts = [ + countCandidateKeys(parsedCandidates, (candidate) => candidate.terminalHandle), + countCandidateKeys(parsedCandidates, (candidate) => candidate.paneKey), + countCandidateKeys(parsedCandidates, (candidate) => candidate.processIncarnation) + ] + const ambiguousDispatchIds = new Set() + const candidates = parsedCandidates.filter((candidate) => { + const keys = [candidate.terminalHandle, candidate.paneKey, candidate.processIncarnation] + const ambiguous = keys.some((key, index) => (identityCounts[index]?.get(key) ?? 0) !== 1) + if (ambiguous) { + ambiguousDispatchIds.add(candidate.dispatchId) + } + return !ambiguous + }) + return { + blockedPanes: [...blockedPanes.values()], + candidates, + ambiguousDispatchIds: [...ambiguousDispatchIds] + } +} diff --git a/src/main/runtime/orchestration/orchestration-schema-version-skew.ts b/src/main/runtime/orchestration/orchestration-schema-version-skew.ts index 9939974b4e9..1a28d5e4d89 100644 --- a/src/main/runtime/orchestration/orchestration-schema-version-skew.ts +++ b/src/main/runtime/orchestration/orchestration-schema-version-skew.ts @@ -2,8 +2,12 @@ import type Database from '../../sqlite/sync-database' const POST_V6_COLUMNS = [ ['messages', 'run_id'], + ['messages', 'delivery_contract'], + ['coordinator_runs', 'scheduler_lost_at'], ['tasks', 'run_id'], ['dispatch_contexts', 'run_id'], + ['dispatch_contexts', 'contract_version'], + ['dispatch_contexts', 'launch_token_hash'], ['dispatch_contexts', 'capability_hash'], ['dispatch_contexts', 'process_incarnation'], ['dispatch_contexts', 'capability_revoked_at'], @@ -14,11 +18,16 @@ const POST_V6_COLUMNS = [ ['remote_dispatch_attachments', 'to_worker_imported_sequence'], ['remote_dispatch_attachments', 'protocol_version'], ['federation_relay_items', 'dispatch_id'], - ['remote_questions', 'message_id'] + ['remote_questions', 'message_id'], + ['legacy_adoptions', 'source_run_id'], + ['legacy_compatibility_principals', 'id'], + ['legacy_operation_receipts', 'principal_id'], + ['legacy_mail_receipts', 'principal_id'] ] as const const POST_V6_INDEXES = [ 'idx_messages_run_sequence', + 'idx_messages_delivery_contract', 'idx_tasks_run_status', 'idx_dispatch_run_status', 'idx_gates_run_status', @@ -46,11 +55,39 @@ function messagesAllowQuestions(db: Database.Database): boolean { return !!row && row.sql.includes("'question'") } +function hasConsistentLegacyAdoption(db: Database.Database): boolean { + const sourceRunId = 'run_legacy_local' + const sourceGraph = db + .prepare( + `SELECT 1 + WHERE EXISTS(SELECT 1 FROM tasks WHERE run_id = ?) + OR EXISTS(SELECT 1 FROM dispatch_contexts WHERE run_id = ?) + OR EXISTS(SELECT 1 FROM decision_gates WHERE run_id = ?) + OR EXISTS(SELECT 1 FROM messages WHERE run_id = ?) + OR EXISTS(SELECT 1 FROM question_threads WHERE run_id = ?) + OR EXISTS(SELECT 1 FROM deliveries WHERE run_id = ?)` + ) + .get(sourceRunId, sourceRunId, sourceRunId, sourceRunId, sourceRunId, sourceRunId) + const adoption = db + .prepare('SELECT adopted_run_id FROM legacy_adoptions WHERE source_run_id = ?') + .get(sourceRunId) as { adopted_run_id: string } | undefined + if (sourceGraph) { + return false + } + if (adoption) { + return Boolean( + db.prepare('SELECT 1 FROM runs WHERE id = ? AND legacy = 0').get(adoption.adopted_run_id) + ) + } + return true +} + function hasCompletePostV6Schema(db: Database.Database): boolean { return ( POST_V6_COLUMNS.every(([table, column]) => hasOrchestrationColumn(db, table, column)) && POST_V6_INDEXES.every((index) => hasOrchestrationIndex(db, index)) && - messagesAllowQuestions(db) + messagesAllowQuestions(db) && + hasConsistentLegacyAdoption(db) ) } @@ -59,9 +96,12 @@ export function resolveOrchestrationMigrationStartVersion( storedVersion: number, schemaVersion: number ): number { - if (storedVersion >= schemaVersion || hasCompletePostV6Schema(db)) { + if (storedVersion > schemaVersion) { + return storedVersion + } + if (hasCompletePostV6Schema(db)) { return storedVersion } // Why: version-skewed pre-Run databases can claim the post-v6 range while retaining v6 tables. - return Math.min(storedVersion, 6) + return Math.min(storedVersion, schemaVersion, 6) } diff --git a/src/main/runtime/orchestration/orchestration-version-skew-migration.test.ts b/src/main/runtime/orchestration/orchestration-version-skew-migration.test.ts index f8a97aa1ed0..763301bde22 100644 --- a/src/main/runtime/orchestration/orchestration-version-skew-migration.test.ts +++ b/src/main/runtime/orchestration/orchestration-version-skew-migration.test.ts @@ -3,7 +3,8 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, describe, expect, it } from 'vitest' import Database from '../../sqlite/sync-database' -import { LEGACY_RUN_ID, OrchestrationDb } from './db' +import { LEGACY_CONTRACT_VERSION, LEGACY_RUN_ID, OrchestrationDb } from './db' +import { resolveOrchestrationMigrationStartVersion } from './orchestration-schema-version-skew' describe('OrchestrationDb version-skew migration', () => { let db: OrchestrationDb | undefined @@ -11,12 +12,14 @@ describe('OrchestrationDb version-skew migration', () => { afterEach(() => { db?.close() + db = undefined if (tempDir) { rmSync(tempDir, { recursive: true, force: true }) + tempDir = undefined } }) - function createLegacySchemaClaimingVersion17(): string { + function createLegacySchemaClaimingVersion(claimedVersion = 17): string { tempDir = mkdtempSync(join(tmpdir(), 'orca-db-version-skew-')) const dbPath = join(tempDir, 'orchestration.db') const raw = new Database(dbPath) @@ -126,20 +129,28 @@ describe('OrchestrationDb version-skew migration', () => { 'gate_legacy', 'task_legacy', 'retained gate' ); `) - raw.pragma('user_version = 17') + raw.pragma(`user_version = ${claimedVersion}`) raw.close() return dbPath } it('repairs retained v6 rows when the database already claims v17', () => { - const dbPath = createLegacySchemaClaimingVersion17() + const dbPath = createLegacySchemaClaimingVersion() db = new OrchestrationDb(dbPath) + const adoptedRunId = db.getLegacyAdoption()?.adopted_run_id + expect(adoptedRunId).toBeTruthy() expect(db.getRun(LEGACY_RUN_ID)).toMatchObject({ legacy: 1 }) - expect(db.getMessageById('msg_legacy')).toMatchObject({ run_id: LEGACY_RUN_ID }) - expect(db.getTask('task_legacy')).toMatchObject({ run_id: LEGACY_RUN_ID }) - expect(db.getDispatchContextById('ctx_legacy')).toMatchObject({ run_id: LEGACY_RUN_ID }) - expect(db.getGate('gate_legacy')).toMatchObject({ run_id: LEGACY_RUN_ID }) + expect(db.getMessageById('msg_legacy')).toMatchObject({ + run_id: adoptedRunId, + delivery_contract: 'legacy_direct' + }) + expect(db.getTask('task_legacy')).toMatchObject({ run_id: adoptedRunId }) + expect(db.getDispatchContextById('ctx_legacy')).toMatchObject({ + run_id: adoptedRunId, + contract_version: LEGACY_CONTRACT_VERSION + }) + expect(db.getGate('gate_legacy')).toMatchObject({ run_id: adoptedRunId }) const run = db.createRun({ objective: 'verify repaired orchestration', @@ -164,8 +175,18 @@ describe('OrchestrationDb version-skew migration', () => { db.close() db = undefined db = new OrchestrationDb(dbPath) - expect(db.listTasks({ runId: LEGACY_RUN_ID }).map((row) => row.id)).toEqual(['task_legacy']) + expect(db.listTasks({ runId: adoptedRunId }).map((row) => row.id)).toEqual(['task_legacy']) expect(db.getRun(run.id)).toBeDefined() expect(db.getQuestion(question.message.id)).toMatchObject({ status: 'pending' }) }) + + it('does not repair an incomplete schema written by a future binary', () => { + const dbPath = createLegacySchemaClaimingVersion(20) + const raw = new Database(dbPath) + + expect(resolveOrchestrationMigrationStartVersion(raw, 20, 19)).toBe(20) + expect(raw.pragma('user_version', { simple: true })).toBe(20) + + raw.close() + }) }) diff --git a/src/main/runtime/orchestration/orchestration-worker-dispatch-db.test.ts b/src/main/runtime/orchestration/orchestration-worker-dispatch-db.test.ts index 34e9e456f6a..e9117b69846 100644 --- a/src/main/runtime/orchestration/orchestration-worker-dispatch-db.test.ts +++ b/src/main/runtime/orchestration/orchestration-worker-dispatch-db.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it } from 'vitest' -import { OrchestrationDb } from './db' +import { CURRENT_CONTRACT_VERSION, OrchestrationDb } from './db' describe('OrchestrationDb worker Dispatch state', () => { let db: OrchestrationDb | undefined @@ -44,6 +44,51 @@ describe('OrchestrationDb worker Dispatch state', () => { status: 'dispatched', assignee_handle: 'term_worker' }) + expect(d.listLegacyWorkerTerminalRecoveryRows()).toEqual([ + expect.objectContaining({ + dispatch_id: started.dispatch.id, + contract_version: CURRENT_CONTRACT_VERSION, + worker_state: 'ready', + agent_terminal_handle: 'term_worker' + }) + ]) + }) + + it('requeues an active Task before settling a worker whose terminal is missing', () => { + const d = createDb() + const task = d.createTask({ spec: 'recover missing worker' }) + const started = d.createStartingWorkerDispatch({ + taskId: task.id, + startOptions: { topology: 'current', agent: 'codex' } + }) + d.prepareStartingWorkerAuthority({ + dispatchId: started.dispatch.id, + handle: 'term_missing', + paneKey: 'tab_missing:11111111-1111-4111-8111-111111111111', + processIncarnation: 'pty-missing:22222222-2222-4222-8222-222222222222', + worktreeId: 'repo::worktree', + setupState: 'not_applicable', + effects: [] + }) + d.markWorkerDispatchReady(started.dispatch.id) + + expect( + d.reconcileMissingWorkerTerminal(started.dispatch.id, 'worker terminal is no longer live') + ).toMatchObject({ + state: 'abandoned', + stage: 'terminal_missing', + last_error: 'worker terminal is no longer live' + }) + expect(d.getDispatchContextById(started.dispatch.id)).toMatchObject({ + status: 'failed', + failure_count: 1, + last_failure: 'worker terminal is no longer live' + }) + expect(d.getTask(task.id)?.status).toBe('ready') + + d.reconcileMissingWorkerTerminal(started.dispatch.id, 'duplicate recovery') + expect(d.getDispatchContextById(started.dispatch.id)?.failure_count).toBe(1) + expect(d.getTask(task.id)?.status).toBe('ready') }) it('commits worker-start mutation acceptance with the starting Dispatch', () => { diff --git a/src/main/runtime/orchestration/types.ts b/src/main/runtime/orchestration/types.ts index eac7868da20..76b3d536fe8 100644 --- a/src/main/runtime/orchestration/types.ts +++ b/src/main/runtime/orchestration/types.ts @@ -14,6 +14,8 @@ export type MessageType = (typeof MESSAGE_TYPES)[number] export type MessagePriority = 'normal' | 'high' | 'urgent' +export type MessageDeliveryContract = 'legacy_direct' | 'current_delivery' | 'audit_only' + export type TaskStatus = 'pending' | 'ready' | 'dispatched' | 'completed' | 'failed' | 'blocked' export type DispatchStatus = 'pending' | 'dispatched' | 'completed' | 'failed' | 'circuit_broken' @@ -61,6 +63,46 @@ export type DeliveryRow = { acknowledged_at: string | null } +export type LegacyAdoptionRow = { + source_run_id: string + adopted_run_id: string + scheduler_state_lost: number + adopted_at: string +} + +export type LegacyPrincipalRole = 'worker' | 'coordinator' + +export type LegacyPrincipalStatus = 'committed' | 'settled' | 'revoked' + +export type LegacyCompatibilityPrincipalRow = { + id: string + run_id: string + dispatch_id: string | null + role: LegacyPrincipalRole + host_scope: string + terminal_handle: string + pane_key: string + launch_token_hash: string + process_incarnation: string | null + status: LegacyPrincipalStatus +} + +export type LegacyOperationReceiptRow = { + principal_id: string + operation_key: string + method: string + payload_hash: string + effect_id: string + response_json: string + completed_at: string +} + +export type LegacyMailReceiptRow = { + principal_id: string + message_id: string + acknowledged_at: string | null +} + export type QuestionStatus = 'pending' | 'answered' | 'closed' export type QuestionRow = { @@ -117,6 +159,19 @@ export type WorkerDispatchRow = { updated_at: string } +export type LegacyWorkerTerminalRecoveryRow = { + dispatch_id: string + task_id: string + dispatch_status: DispatchStatus + contract_version: number + assignee_handle: string | null + assignee_pane_key: string | null + process_incarnation: string | null + worker_state: WorkerDispatchState + worktree_id: string | null + agent_terminal_handle: string | null +} + export type FederatedDispatchRow = { dispatch_id: string environment_id: string @@ -170,6 +225,7 @@ export type FederationRelayItemRow = { export type MessageRow = { id: string run_id: string + delivery_contract?: MessageDeliveryContract from_handle: string to_handle: string subject: string @@ -204,6 +260,8 @@ export type DispatchContextRow = { id: string run_id: string task_id: string + contract_version: number + launch_token_hash: string | null assignee_handle: string | null assignee_pane_key: string | null capability_hash: string | null @@ -238,4 +296,5 @@ export type CoordinatorRun = { poll_interval_ms: number created_at: string completed_at: string | null + scheduler_lost_at: string | null } diff --git a/src/main/runtime/rpc/core.ts b/src/main/runtime/rpc/core.ts index db4c8ac08e1..93421c8b252 100644 --- a/src/main/runtime/rpc/core.ts +++ b/src/main/runtime/rpc/core.ts @@ -1,7 +1,7 @@ // Why: single boundary between raw RPC frames and OrcaRuntimeService; keeps schema, handler, and result type on one object. import { ZodError, type ZodType } from 'zod' import type { TerminalStreamFrame } from '../../../shared/terminal-stream-protocol' -import type { OrcaRuntimeService } from '../orca-runtime' +import type { OrcaRuntimeService, OrchestrationCompatibilityCallerAuthority } from '../orca-runtime' import type { DeviceCredentialInstalled, PairingGetEndpointsParams, @@ -9,6 +9,7 @@ import type { PairingProvisionRelayParams } from '../../../shared/mobile-relay-credential-contract' import type { RuntimeCapability } from '../../../shared/protocol-version' +import type { OrchestrationCompatibilityEvidence } from '../../../shared/orchestration-compatibility-evidence' export type PairingRpcContext = { getEndpoints(params: PairingGetEndpointsParams): Promise @@ -48,8 +49,18 @@ export type RpcRequest = { orchestrationCapability?: string orchestrationContractVersion?: number orchestrationRequestId?: string + compatibilityInvocationId?: string + orchestrationCompatibilityEvidence?: OrchestrationCompatibilityEvidence } +export type LegacyCoordinatorAuthorityProof = Readonly<{ + runId: string + principalId: string | null + terminalHandle: string + paneKey: string + consumerGeneration: number +}> + export type RpcContext = { runtime: OrcaRuntimeService // Why: lets long-poll handlers release immediately on client disconnect instead of running down timeoutMs. See design doc §3.1. @@ -77,6 +88,11 @@ export type RpcContext = { method: string payloadHash: string } + // Why: only the compatibility authority router can set this trusted scope; user params cannot bypass Run consumer binding. + legacyCoordinatorRunId?: string + legacyCoordinatorAuthority?: LegacyCoordinatorAuthorityProof + revalidateLegacyCoordinator?: () => string + orchestrationCompatibilityCallerAuthority?: OrchestrationCompatibilityCallerAuthority // Why: federation pins the authenticated saved-environment caller without exposing its token to handlers or storage. authenticatedCallerFingerprint?: string pairing?: PairingRpcContext diff --git a/src/main/runtime/rpc/dispatcher-error-response.ts b/src/main/runtime/rpc/dispatcher-error-response.ts new file mode 100644 index 00000000000..216c971b95c --- /dev/null +++ b/src/main/runtime/rpc/dispatcher-error-response.ts @@ -0,0 +1,43 @@ +import { InvalidArgumentError, ZodError, formatZodError } from './core' +import type { RpcEnvelopeMeta, RpcRequest, RpcResponse } from './core' +import { + computerErrorData, + errorResponse, + mapBrowserError, + mapEmulatorError, + mapRuntimeError +} from './errors' + +export function invalidArgumentResponse( + request: RpcRequest, + meta: RpcEnvelopeMeta, + message: string +): RpcResponse { + return errorResponse( + request.id, + meta, + 'invalid_argument', + message, + request.method.startsWith('computer.') ? computerErrorData('invalid_argument') : undefined + ) +} + +export function mapDispatcherError( + request: RpcRequest, + meta: RpcEnvelopeMeta, + error: unknown +): RpcResponse { + if (error instanceof ZodError) { + return invalidArgumentResponse(request, meta, formatZodError(error)) + } + if (error instanceof InvalidArgumentError) { + return invalidArgumentResponse(request, meta, error.message) + } + if (request.method.startsWith('browser.')) { + return mapBrowserError(request.id, meta, error) + } + if (request.method.startsWith('emulator.')) { + return mapEmulatorError(request.id, meta, error) + } + return mapRuntimeError(request.id, meta, error) +} diff --git a/src/main/runtime/rpc/dispatcher-feature-interactions.test.ts b/src/main/runtime/rpc/dispatcher-feature-interactions.test.ts index 64dc5ee4f7f..e33da2b2661 100644 --- a/src/main/runtime/rpc/dispatcher-feature-interactions.test.ts +++ b/src/main/runtime/rpc/dispatcher-feature-interactions.test.ts @@ -46,6 +46,10 @@ function makeRuntime(ui: PersistedUIState = getDefaultUIState()): OrcaRuntimeSer updateUIState: vi.fn((updates: Partial) => { currentUI = { ...currentUI, ...updates } return currentUI + }), + getOrchestrationDb: () => ({ + getLegacyAdoption: () => undefined, + resolveLegacyWorkerCandidate: () => undefined }) } as unknown as OrcaRuntimeService } diff --git a/src/main/runtime/rpc/dispatcher-stream-options.ts b/src/main/runtime/rpc/dispatcher-stream-options.ts new file mode 100644 index 00000000000..0e492c7ad9a --- /dev/null +++ b/src/main/runtime/rpc/dispatcher-stream-options.ts @@ -0,0 +1,18 @@ +import type { RuntimeCapability } from '../../../shared/protocol-version' +import type { TerminalStreamFrame } from '../../../shared/terminal-stream-protocol' +import type { PairingRpcContext } from './core' + +export type RpcDispatchStreamingOptions = { + connectionId?: string + signal?: AbortSignal + clientId?: string + pairedDeviceId?: string + clientKind?: 'mobile' | 'runtime' + clientCapabilities?: readonly RuntimeCapability[] + pairing?: PairingRpcContext + sendBinary?: (bytes: Uint8Array) => boolean | void + registerBinaryStreamHandler?: ( + streamId: number, + handler: (frame: TerminalStreamFrame) => void + ) => () => void +} diff --git a/src/main/runtime/rpc/dispatcher.ts b/src/main/runtime/rpc/dispatcher.ts index 71d327c97f6..ad2ba1a1eec 100644 --- a/src/main/runtime/rpc/dispatcher.ts +++ b/src/main/runtime/rpc/dispatcher.ts @@ -1,57 +1,44 @@ -// Why: the dispatcher is the one place that knows how to turn a validated -// RPC request into a response envelope. Splitting it from the transport -// makes it unit-testable without spinning up a socket, and keeps -// runtime-rpc.ts focused on framing/auth/connection bookkeeping. import { - ZodError, - InvalidArgumentError, buildRegistry, formatZodError, isStreamingMethod, type RpcAnyMethod, type RpcEnvelopeMeta, - type PairingRpcContext, type RpcRegistry, type RpcRequest, type RpcResponse } from './core' -import type { TerminalStreamFrame } from '../../../shared/terminal-stream-protocol' import type { FeatureInteractionId } from '../../../shared/feature-interactions' -import { - computerErrorData, - errorResponse, - mapBrowserError, - mapEmulatorError, - mapRuntimeError, - successResponse -} from './errors' +import { errorResponse, successResponse } from './errors' import { ALL_RPC_METHODS } from './methods' import { emulatorProbe, emulatorProbeError } from '../../emulator/emulator-probe' import type { OrcaRuntimeService } from '../orca-runtime' -import type { RuntimeCapability } from '../../../shared/protocol-version' import { - OrchestrationMutationExecutor, authenticatedCallerFingerprint, + getOrchestrationMutationExecutor, + type OrchestrationMutationExecutor, type DurableMutationInvocation } from './orchestration-mutation-executor' import { orchestrationMigrationFence } from './orchestration-contract-fence' -import { getRuntimeFeatureInteractionId } from './runtime-feature-interaction' +import { recordRuntimeFeatureInteraction } from './runtime-feature-interaction' +import { OrchestrationLegacyCompatibility } from './orchestration-legacy-compatibility' +import type { RpcDispatchStreamingOptions } from './dispatcher-stream-options' +import { invalidArgumentResponse, mapDispatcherError } from './dispatcher-error-response' -export type DispatcherOptions = { - runtime: OrcaRuntimeService - methods?: readonly RpcAnyMethod[] -} +export type DispatcherOptions = { runtime: OrcaRuntimeService; methods?: readonly RpcAnyMethod[] } export class RpcDispatcher { private readonly runtime: OrcaRuntimeService private readonly registry: RpcRegistry private readonly orchestrationMutations: OrchestrationMutationExecutor + private readonly legacyOrchestration: OrchestrationLegacyCompatibility constructor({ runtime, methods = ALL_RPC_METHODS }: DispatcherOptions) { this.runtime = runtime this.registry = buildRegistry(methods) - this.orchestrationMutations = new OrchestrationMutationExecutor(runtime) + this.orchestrationMutations = getOrchestrationMutationExecutor(runtime) + this.legacyOrchestration = new OrchestrationLegacyCompatibility(runtime) } async dispatch(request: RpcRequest, options?: { signal?: AbortSignal }): Promise { @@ -76,9 +63,6 @@ export class RpcDispatcher { return parsedParams.error } - // Why: streaming methods are not supported over one-shot transports like - // Unix sockets. They require a reply function that can be called multiple - // times, which is only available via dispatchStreaming. if (isStreamingMethod(method)) { return errorResponse( request.id, @@ -88,29 +72,60 @@ export class RpcDispatcher { ) } - const isEmulator = request.method.startsWith('emulator.') - if (isEmulator) { + if (request.method.startsWith('emulator.')) { emulatorProbe(`rpc ${request.method}`, request.params) } try { - const invoke = (mutation?: DurableMutationInvocation) => - method.handler(parsedParams.value, { + const compatibility = await this.legacyOrchestration.tryHandle( + request, + parsedParams.value, + options?.signal + ) + if (compatibility.handled) { + return successResponse(request.id, meta, compatibility.result) + } + const effectiveParams = compatibility.params ?? parsedParams.value + const legacyCoordinator = this.legacyOrchestration.createCoordinatorInvocation( + request, + compatibility.legacyCoordinatorAuthority + ) + const invoke = (mutation?: DurableMutationInvocation) => { + const legacyCoordinatorRunId = legacyCoordinator?.revalidate() + return method.handler(effectiveParams, { runtime: this.runtime, signal: options?.signal, requestId: request.id, orchestrationCapability: request.orchestrationCapability, - authenticatedCallerFingerprint: authenticatedCallerFingerprint(request), + authenticatedCallerFingerprint: + mutation?.identity.callerFingerprint ?? authenticatedCallerFingerprint(request), recordMutationReceipt: mutation?.recordReceipt, - orchestrationMutation: mutation?.identity + orchestrationMutation: mutation?.identity, + legacyCoordinatorRunId, + legacyCoordinatorAuthority: legacyCoordinator?.authority, + revalidateLegacyCoordinator: legacyCoordinator?.revalidate, + orchestrationCompatibilityCallerAuthority: + compatibility.orchestrationCompatibilityCallerAuthority }) - const result = await this.orchestrationMutations.run(request, parsedParams.value, invoke) - this.recordRuntimeFeatureInteraction(request.method, result, undefined, request.params) + } + const result = await this.orchestrationMutations.run( + request, + effectiveParams, + invoke, + legacyCoordinator?.mutationCallerFingerprint + ) + recordRuntimeFeatureInteraction( + this.runtime, + request.method, + result, + undefined, + request.params + ) return successResponse(request.id, meta, result) } catch (error) { - if (isEmulator) { + if (request.method.startsWith('emulator.')) { emulatorProbeError(`rpc ${request.method}`, error, { params: request.params }) } - return this.mapError(request, meta, error) + return mapDispatcherError(request, meta, error) } } @@ -120,20 +135,7 @@ export class RpcDispatcher { async dispatchStreaming( request: RpcRequest, reply: (response: string) => void, - options?: { - connectionId?: string - signal?: AbortSignal - clientId?: string - pairedDeviceId?: string - clientKind?: 'mobile' | 'runtime' - clientCapabilities?: readonly RuntimeCapability[] - pairing?: PairingRpcContext - sendBinary?: (bytes: Uint8Array) => boolean | void - registerBinaryStreamHandler?: ( - streamId: number, - handler: (frame: TerminalStreamFrame) => void - ) => () => void - } + options?: RpcDispatchStreamingOptions ): Promise { const meta = this.meta() const method = this.registry.get(request.method) @@ -160,8 +162,23 @@ export class RpcDispatcher { if (!isStreamingMethod(method)) { try { - const invoke = (mutation?: DurableMutationInvocation) => - method.handler(parsedParams.value, { + const compatibility = await this.legacyOrchestration.tryHandle( + request, + parsedParams.value, + options?.signal + ) + if (compatibility.handled) { + reply(JSON.stringify(successResponse(request.id, meta, compatibility.result))) + return + } + const effectiveParams = compatibility.params ?? parsedParams.value + const legacyCoordinator = this.legacyOrchestration.createCoordinatorInvocation( + request, + compatibility.legacyCoordinatorAuthority + ) + const invoke = (mutation?: DurableMutationInvocation) => { + const legacyCoordinatorRunId = legacyCoordinator?.revalidate() + return method.handler(effectiveParams, { runtime: this.runtime, signal: options?.signal, requestId: request.id, @@ -171,25 +188,44 @@ export class RpcDispatcher { clientKind: options?.clientKind, clientCapabilities: options?.clientCapabilities, orchestrationCapability: request.orchestrationCapability, - authenticatedCallerFingerprint: authenticatedCallerFingerprint(request), + authenticatedCallerFingerprint: + mutation?.identity.callerFingerprint ?? authenticatedCallerFingerprint(request), recordMutationReceipt: mutation?.recordReceipt, orchestrationMutation: mutation?.identity, pairing: options?.pairing, sendBinary: options?.sendBinary, - registerBinaryStreamHandler: options?.registerBinaryStreamHandler + registerBinaryStreamHandler: options?.registerBinaryStreamHandler, + legacyCoordinatorRunId, + legacyCoordinatorAuthority: legacyCoordinator?.authority, + revalidateLegacyCoordinator: legacyCoordinator?.revalidate, + orchestrationCompatibilityCallerAuthority: + compatibility.orchestrationCompatibilityCallerAuthority }) - const result = await this.orchestrationMutations.run(request, parsedParams.value, invoke) - this.recordRuntimeFeatureInteraction(request.method, result, undefined, request.params) + } + const result = await this.orchestrationMutations.run( + request, + effectiveParams, + invoke, + legacyCoordinator?.mutationCallerFingerprint + ) + recordRuntimeFeatureInteraction( + this.runtime, + request.method, + result, + undefined, + request.params + ) reply(JSON.stringify(successResponse(request.id, meta, result))) } catch (error) { - reply(JSON.stringify(this.mapError(request, meta, error))) + reply(JSON.stringify(mapDispatcherError(request, meta, error))) } return } const recordedStreamingFeatureInteractions = new Set() const emit = (result: unknown): void => { - this.recordRuntimeFeatureInteraction( + recordRuntimeFeatureInteraction( + this.runtime, request.method, result, recordedStreamingFeatureInteractions, @@ -212,20 +248,22 @@ export class RpcDispatcher { pairedDeviceId: options?.pairedDeviceId, clientKind: options?.clientKind, clientCapabilities: options?.clientCapabilities, + orchestrationCapability: request.orchestrationCapability, pairing: options?.pairing, sendBinary: options?.sendBinary, registerBinaryStreamHandler: options?.registerBinaryStreamHandler }, emit ) - this.recordRuntimeFeatureInteraction( + recordRuntimeFeatureInteraction( + this.runtime, request.method, result, recordedStreamingFeatureInteractions, request.params ) } catch (error) { - reply(JSON.stringify(this.mapError(request, meta, error))) + reply(JSON.stringify(mapDispatcherError(request, meta, error))) } } @@ -241,69 +279,13 @@ export class RpcDispatcher { const result = method.params.safeParse(rawParams) if (!result.success) { return { - error: this.invalidArgumentResponse(request, meta, formatZodError(result.error)) + error: invalidArgumentResponse(request, meta, formatZodError(result.error)) } } return { value: result.data } } - private mapError(request: RpcRequest, meta: RpcEnvelopeMeta, error: unknown): RpcResponse { - if (error instanceof ZodError) { - return this.invalidArgumentResponse(request, meta, formatZodError(error)) - } - if (error instanceof InvalidArgumentError) { - return this.invalidArgumentResponse(request, meta, error.message) - } - - // Why: browser methods throw BrowserError with a structured `code`; - // every other runtime error has a plain-message code. Routing by method - // prefix keeps the mapping a single decision rather than a per-method - // flag callers must remember to set. - if (request.method.startsWith('browser.')) { - return mapBrowserError(request.id, meta, error) - } - if (request.method.startsWith('emulator.')) { - return mapEmulatorError(request.id, meta, error) - } - return mapRuntimeError(request.id, meta, error) - } - - private invalidArgumentResponse( - request: RpcRequest, - meta: RpcEnvelopeMeta, - message: string - ): RpcResponse { - return errorResponse( - request.id, - meta, - 'invalid_argument', - message, - request.method.startsWith('computer.') ? computerErrorData('invalid_argument') : undefined - ) - } - private meta(): RpcEnvelopeMeta { return { runtimeId: this.runtime.getRuntimeId() } } - - private recordRuntimeFeatureInteraction( - method: string, - result: unknown, - alreadyRecorded?: Set, - rawParams?: unknown - ): void { - const id = getRuntimeFeatureInteractionId(method, result, rawParams) - if (!id) { - return - } - if (alreadyRecorded?.has(id)) { - return - } - try { - this.runtime.recordFeatureInteraction(id) - alreadyRecorded?.add(id) - } catch { - // Best-effort education state must not break runtime tools. - } - } } diff --git a/src/main/runtime/rpc/errors.ts b/src/main/runtime/rpc/errors.ts index d76b0e851c2..3cf66666ddf 100644 --- a/src/main/runtime/rpc/errors.ts +++ b/src/main/runtime/rpc/errors.ts @@ -88,6 +88,7 @@ const STRUCTURED_RUNTIME_PASSTHROUGH_CODES: ReadonlySet = new Set([ 'agent_unconfigured', 'terminal_worktree_mismatch', 'request_mismatch', + 'legacy_read_only', 'orchestration_migration_required', 'operation_unknown', 'question_not_found', diff --git a/src/main/runtime/rpc/methods/orchestration-federation-control-mail.test.ts b/src/main/runtime/rpc/methods/orchestration-federation-control-mail.test.ts index 8a5111047cb..b061b7dcb2d 100644 --- a/src/main/runtime/rpc/methods/orchestration-federation-control-mail.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-federation-control-mail.test.ts @@ -265,7 +265,7 @@ describe('orchestration federation control mail', () => { }) await expect(waiting).resolves.toMatchObject({ ok: true, - result: { count: 0, timedOut: true } + result: { count: 0 } }) }) diff --git a/src/main/runtime/rpc/methods/orchestration-federation.test.ts b/src/main/runtime/rpc/methods/orchestration-federation.test.ts index eb76d094368..302f4d58b1f 100644 --- a/src/main/runtime/rpc/methods/orchestration-federation.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-federation.test.ts @@ -224,6 +224,7 @@ describe('orchestration federation', () => { ]) ) expect(workerDb.listTasks()).toHaveLength(0) + expect(vi.mocked(workerRuntime.createManagedWorktree).mock.calls[0]?.[0].activate).toBe(false) expect(workerRuntime.sendTerminalAgentPrompt).toHaveBeenCalledWith( 'term_windows_worker', expect.stringContaining(`Your task ID is: ${task.id}`) diff --git a/src/main/runtime/rpc/methods/orchestration-gates.ts b/src/main/runtime/rpc/methods/orchestration-gates.ts index ca16b9719f0..3e85f08e267 100644 --- a/src/main/runtime/rpc/methods/orchestration-gates.ts +++ b/src/main/runtime/rpc/methods/orchestration-gates.ts @@ -103,8 +103,12 @@ export const ORCHESTRATION_GATE_METHODS: RpcMethod[] = [ defineMethod({ name: 'orchestration.gateCreate', params: GateCreateParams, - handler: (params, { runtime }) => { + handler: (params, { runtime, legacyCoordinatorRunId }) => { const db = runtime.getOrchestrationDb() + const task = db.getTask(params.task) + if (legacyCoordinatorRunId && task?.run_id !== legacyCoordinatorRunId) { + throw new Error(`Task not found: ${params.task}`) + } let options: string[] | undefined if (params.options) { try { @@ -129,8 +133,12 @@ export const ORCHESTRATION_GATE_METHODS: RpcMethod[] = [ defineMethod({ name: 'orchestration.gateResolve', params: GateResolveParams, - handler: (params, { runtime }) => { + handler: (params, { runtime, legacyCoordinatorRunId }) => { const db = runtime.getOrchestrationDb() + const existing = db.getGate(params.id) + if (legacyCoordinatorRunId && existing?.run_id !== legacyCoordinatorRunId) { + throw new Error(`Gate not found: ${params.id}`) + } const gate = db.resolveGate(params.id, params.resolution) if (!gate) { throw new Error(`Gate not found: ${params.id}`) diff --git a/src/main/runtime/rpc/methods/orchestration-runs.ts b/src/main/runtime/rpc/methods/orchestration-runs.ts index 498d6228365..020ec0a4f67 100644 --- a/src/main/runtime/rpc/methods/orchestration-runs.ts +++ b/src/main/runtime/rpc/methods/orchestration-runs.ts @@ -1,6 +1,6 @@ import { z } from 'zod' import { defineMethod, type RpcMethod } from '../core' -import { OptionalString, requiredString } from '../schemas' +import { OptionalBoolean, OptionalString, requiredString } from '../schemas' import type { OrcaRuntimeService } from '../../orca-runtime' import { OrchestrationError } from '../../orchestration/orchestration-error' @@ -11,7 +11,8 @@ const RunCreateParams = z.object({ const RunUseParams = z.object({ id: requiredString('Missing --id'), - from: requiredString('Missing coordinator terminal') + from: requiredString('Missing coordinator terminal'), + takeoverLegacy: OptionalBoolean }) const RunCurrentParams = z.object({ from: requiredString('Missing coordinator terminal') }) @@ -51,14 +52,33 @@ export const ORCHESTRATION_RUN_METHODS: RpcMethod[] = [ defineMethod({ name: 'orchestration.runUse', params: RunUseParams, - handler: (params, { runtime }) => { + handler: ( + params, + { + runtime, + legacyCoordinatorAuthority, + orchestrationCompatibilityCallerAuthority: callerAuthority + } + ) => { const paneKey = requireCallerPane(runtime, params.from) + if ( + params.takeoverLegacy && + (callerAuthority?.terminalHandle !== params.from || callerAuthority.paneKey !== paneKey) + ) { + throw new OrchestrationError( + 'legacy_read_only', + 'Legacy takeover must be invoked by the live coordinator agent terminal it will bind. No effects were applied.', + { effectsApplied: false } + ) + } const db = runtime.getOrchestrationDb() const priorRun = db.getCurrentRunForPane(paneKey) const run = db.bindRun({ runId: params.id, coordinatorHandle: params.from, - coordinatorPaneKey: paneKey + coordinatorPaneKey: paneKey, + takeoverLegacy: params.takeoverLegacy, + legacyCoordinatorAuthority }) if (!run) { throw new OrchestrationError( diff --git a/src/main/runtime/rpc/methods/orchestration-worker-topology.ts b/src/main/runtime/rpc/methods/orchestration-worker-topology.ts index 4605c9879a8..abeb652a19f 100644 --- a/src/main/runtime/rpc/methods/orchestration-worker-topology.ts +++ b/src/main/runtime/rpc/methods/orchestration-worker-topology.ts @@ -36,6 +36,21 @@ export type WorkerSetupReceipt = { | 'not_applicable' } +export function requireWorkerAuthority(runtime: OrcaRuntimeService, terminalHandle: string) { + const authority = runtime.getOrchestrationDispatchAuthority(terminalHandle) + const paneKey = authority?.paneKey ?? runtime.getTerminalPaneKey(terminalHandle) + const processIncarnation = + authority?.processIncarnation ?? runtime.getTerminalProcessIncarnation(terminalHandle) + if (!paneKey || !processIncarnation) { + throw new Error('stable_pane_required') + } + return { + paneKey, + processIncarnation, + ...(authority?.launchTokenHash ? { launchTokenHash: authority.launchTokenHash } : {}) + } +} + export async function createExistingWorktreeWorkerTerminal(args: { runtime: OrcaRuntimeService worktreeId: string diff --git a/src/main/runtime/rpc/methods/orchestration-workers.ts b/src/main/runtime/rpc/methods/orchestration-workers.ts index 9525de9d240..894be342f81 100644 --- a/src/main/runtime/rpc/methods/orchestration-workers.ts +++ b/src/main/runtime/rpc/methods/orchestration-workers.ts @@ -10,6 +10,7 @@ import { createExistingWorktreeWorkerTerminal, createWorkerWorktree, monitorWorkerSetup, + requireWorkerAuthority, type WorkerEffect, type WorkerSetupReceipt } from './orchestration-worker-topology' @@ -235,16 +236,11 @@ export const ORCHESTRATION_WORKER_START_METHODS: RpcMethod[] = [ : `Agent did not become ready (${wait.status}).` ) } - const paneKey = runtime.getTerminalPaneKey(terminalHandle) - const processIncarnation = runtime.getTerminalProcessIncarnation(terminalHandle) - if (!paneKey || !processIncarnation) { - throw new Error('stable_pane_required') - } + const terminalAuthority = requireWorkerAuthority(runtime, terminalHandle) const capability = db.prepareStartingWorkerAuthority({ dispatchId: started.dispatch.id, handle: terminalHandle, - paneKey, - processIncarnation, + ...terminalAuthority, worktreeId: resolvedWorktree.id, effects, setupState: setupReceipt.state diff --git a/src/main/runtime/rpc/methods/orchestration.test.ts b/src/main/runtime/rpc/methods/orchestration.test.ts index b0242738a7b..d58a8bd6b95 100644 --- a/src/main/runtime/rpc/methods/orchestration.test.ts +++ b/src/main/runtime/rpc/methods/orchestration.test.ts @@ -1918,6 +1918,30 @@ describe('orchestration RPC methods', () => { expect(db.getDispatchContextById(result.dispatch.id)?.assignee_pane_key).toBe('tab_w:leaf_w') }) + it('commits the target process launch token on a manual dispatch', async () => { + setup() + vi.spyOn(runtime, 'getOrchestrationDispatchAuthority').mockReturnValue({ + runtimeId: runtime.getRuntimeId(), + terminalHandle: 'term_a', + ptyId: 'pty_a', + worktreeId: 'repo::worktree', + paneKey: 'tab_w:leaf_w', + processIncarnation: 'runtime_test:term_a:1', + launchTokenHash: 'launch-token-hash', + hostScope: { kind: 'local', hostId: 'local' } + }) + const task = db.createTask({ spec: 'work' }) + + const result = (await call('orchestration.dispatch', { + task: task.id, + to: 'term_a' + })) as { dispatch: { id: string } } + + expect(db.getDispatchContextById(result.dispatch.id)?.launch_token_hash).toBe( + 'launch-token-hash' + ) + }) + it('rejects dispatch for a pending task', async () => { setup() const parent = db.createTask({ spec: 'parent' }) @@ -2162,6 +2186,32 @@ describe('orchestration RPC methods', () => { ) }) + it('commits the launched worker token with its durable authority', async () => { + setup() + mockCurrentWorkerStart() + vi.spyOn(runtime, 'getOrchestrationDispatchAuthority').mockReturnValue({ + runtimeId: runtime.getRuntimeId(), + terminalHandle: 'term_worker', + ptyId: 'pty_worker', + worktreeId: 'repo::worktree', + paneKey: 'tab_worker:leaf_worker', + processIncarnation: 'runtime_test:term_worker:1', + launchTokenHash: 'worker-launch-token-hash', + hostScope: { kind: 'local', hostId: 'local' } + }) + const task = db.createTask({ spec: 'persist worker identity' }) + + const result = (await call('orchestration.workerStart', { + task: task.id, + from: 'term_coord', + agent: 'codex' + })) as { dispatchId: string } + + expect(db.getDispatchContextById(result.dispatchId)?.launch_token_hash).toBe( + 'worker-launch-token-hash' + ) + }) + it('surfaces a worker terminal reveal failure without discarding the live worker', async () => { setup() mockCurrentWorkerStart() diff --git a/src/main/runtime/rpc/methods/orchestration.ts b/src/main/runtime/rpc/methods/orchestration.ts index bbb6067c47c..ee6e95e1cf1 100644 --- a/src/main/runtime/rpc/methods/orchestration.ts +++ b/src/main/runtime/rpc/methods/orchestration.ts @@ -2,7 +2,12 @@ import { z } from 'zod' import { defineMethod, type RpcMethod } from '../core' import { OptionalFiniteNumber, OptionalString, OptionalBoolean, requiredString } from '../schemas' -import type { MessageType, MessagePriority, TaskStatus } from '../../orchestration/db' +import { + LEGACY_CONTRACT_VERSION, + type MessageType, + type MessagePriority, + type TaskStatus +} from '../../orchestration/db' import { MESSAGE_TYPES } from '../../orchestration/types' import { buildDispatchPreamble } from '../../orchestration/preamble' import { formatMessageBanner } from '../../orchestration/formatter' @@ -51,6 +56,10 @@ function parseRemoteWorkerPayload(payload: string | undefined): Record { if ((params.question ? 1 : 0) + (params.resume ? 1 : 0) !== 1) { @@ -241,6 +255,7 @@ function resolveRunScope( callerTerminalHandle?: string callerPaneKey?: string requireCurrentConsumer: boolean + legacyCoordinatorRunId?: string } ): RunRow { const db = runtime.getOrchestrationDb() @@ -252,6 +267,9 @@ function resolveRunScope( if (!params.requireCurrentConsumer && explicit) { return explicit } + if (explicit && params.legacyCoordinatorRunId === explicit.id) { + return explicit + } if (!params.callerTerminalHandle) { throw new OrchestrationError( 'run_required', @@ -366,6 +384,49 @@ function resolveMessageRun( return { run, dispatchId: dispatch?.id ?? dispatchId } } +function legacyWorkerDeliveryContract( + runtime: OrcaRuntimeService, + runId: string | undefined, + recipient: string +): 'legacy_direct' | undefined { + if (!runId) { + return undefined + } + if (!recipient.startsWith('dispatch:')) { + return runtime + .getOrchestrationDb() + .resolveLegacyWorkerCandidate({ runId, terminalHandle: recipient }) + ? 'legacy_direct' + : undefined + } + const dispatch = runtime + .getOrchestrationDb() + .getDispatchContextById(recipient.slice('dispatch:'.length)) + return dispatch?.run_id === runId && + dispatch.contract_version === LEGACY_CONTRACT_VERSION && + (dispatch.status === 'pending' || dispatch.status === 'dispatched') + ? 'legacy_direct' + : undefined +} + +function interruptedAcknowledgedCheck( + runId: string, + acknowledged: string, + reason: 'consumer_fenced' | 'outcome_unknown' | 'waiter_exists' +): Record { + return { + runId, + deliveryId: null, + messages: [], + count: 0, + acknowledged, + timedOut: false, + cancelled: false, + connectionLost: false, + waitInterrupted: reason + } +} + function rejectFederatedExplicitTarget(params: { to?: string; run?: string }): void { if (params.to || params.run) { throw new OrchestrationError( @@ -382,7 +443,10 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ defineMethod({ name: 'orchestration.send', params: SendParams, - handler: async (params, { runtime, orchestrationCapability }) => { + handler: async ( + params, + { runtime, orchestrationCapability, legacyCoordinatorRunId, revalidateLegacyCoordinator } + ) => { const db = runtime.getOrchestrationDb() const from = params.from ?? 'unknown' // Why: caller-supplied pane fields are only compatibility metadata; lifecycle authority uses the runtime-observed pane plus capability. @@ -463,6 +527,15 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ runId: params.run, payload: params.payload }) + if ( + params.type === 'worker_done' && + !isWorkerReportOutcome(parseRemoteWorkerPayload(params.payload).outcome) + ) { + throw new OrchestrationError( + 'invalid_argument', + 'worker_done requires outcome=succeeded|failed for a current Dispatch.' + ) + } if (params.to?.startsWith('task:')) { throw new OrchestrationError( 'invalid_argument', @@ -514,6 +587,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ 'Coordinator-to-worker control mail cannot report worker lifecycle.' ) } + revalidateLegacyCoordinator?.() const relay = db.enqueueFederationRelay({ dispatchId, direction: 'to_worker', @@ -540,6 +614,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ } } // Point-to-point — existing single-recipient behavior + revalidateLegacyCoordinator?.() const msg = db.insertMessage({ from, to, @@ -550,7 +625,12 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ threadId: params.threadId, payload: params.payload, senderPaneKey, - runId: routing.run?.id + runId: routing.run?.id, + deliveryContract: legacyWorkerDeliveryContract( + runtime, + routing.run?.id ?? legacyCoordinatorRunId, + to + ) }) const dispatch = routing.dispatchId ? db.getDispatchContextById(routing.dispatchId) @@ -607,6 +687,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ throw new Error(`No recipients resolved for group address: ${to}`) } + revalidateLegacyCoordinator?.() const threadId = params.threadId ?? `thread_${Date.now()}` const messages = handles.map((handle) => db.insertMessage({ @@ -619,7 +700,12 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ threadId, payload: params.payload, senderPaneKey, - runId: routing.run?.id + runId: routing.run?.id, + deliveryContract: legacyWorkerDeliveryContract( + runtime, + routing.run?.id ?? legacyCoordinatorRunId, + handle + ) }) ) for (const message of messages) { @@ -633,7 +719,16 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ defineMethod({ name: 'orchestration.check', params: CheckParams, - handler: async (params, { runtime, signal }) => { + handler: async ( + params, + { + runtime, + signal, + legacyCoordinatorRunId, + revalidateLegacyCoordinator, + recordMutationReceipt + } + ) => { const db = runtime.getOrchestrationDb() const handle = params.terminal ?? 'unknown' const typeFilter = parseMessageTypes(params.types) @@ -646,7 +741,8 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ runId: params.run, callerTerminalHandle: handle, callerPaneKey: paneKey ?? undefined, - requireCurrentConsumer: true + requireCurrentConsumer: true, + legacyCoordinatorRunId }) const generation = run.consumer_generation const address = `run:${run.id}` @@ -659,6 +755,11 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ deliveryId: params.ack }) : undefined + if (acknowledged) { + recordMutationReceipt?.( + interruptedAcknowledgedCheck(run.id, acknowledged.delivery.id, 'outcome_unknown') + ) + } if (params.peek || params.all || params.unread === false) { const history = db.getRunMailboxHistory(run.id, 100, typeFilter) const messages = @@ -722,14 +823,28 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ signal, exclusive: true }) + try { + revalidateLegacyCoordinator?.() + } catch (error) { + if (!acknowledged) { + throw error + } + return interruptedAcknowledgedCheck(run.id, acknowledged.delivery.id, 'consumer_fenced') + } const latestRun = db.getRun(run.id) if (!latestRun || latestRun.consumer_generation !== generation) { + if (acknowledged) { + return interruptedAcknowledgedCheck(run.id, acknowledged.delivery.id, 'consumer_fenced') + } throw new OrchestrationError( 'consumer_fenced', 'This mailbox consumer was replaced while waiting.' ) } if (waitResult === 'waiter_exists') { + if (acknowledged) { + return interruptedAcknowledgedCheck(run.id, acknowledged.delivery.id, 'waiter_exists') + } throw new OrchestrationError( 'waiter_exists', `Run ${run.id} already has an active actionable waiter.` @@ -911,13 +1026,28 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ defineMethod({ name: 'orchestration.reply', params: ReplyParams, - handler: async (params, { runtime }) => { + handler: async (params, { runtime, legacyCoordinatorRunId }) => { const db = runtime.getOrchestrationDb() const original = db.getMessageById(params.id) if (!original) { throw new Error(`Message not found: ${params.id}`) } - if (original.run_id === ORCHESTRATION_LEGACY_RUN_ID) { + if ( + legacyCoordinatorRunId && + (original.run_id !== legacyCoordinatorRunId || + (params.run !== undefined && params.run !== legacyCoordinatorRunId)) + ) { + throw new OrchestrationError( + 'request_mismatch', + `Message ${params.id} does not belong to this adopted Run.`, + { effectsApplied: false } + ) + } + if ( + original.run_id === ORCHESTRATION_LEGACY_RUN_ID || + original.delivery_contract === 'legacy_direct' || + original.delivery_contract === 'audit_only' + ) { throw new OrchestrationError( 'legacy_read_only', 'Legacy orchestration messages are inspect-only; no reply was applied.', @@ -930,7 +1060,8 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ const run = resolveRunScope(runtime, { runId: params.run ?? question.run_id, callerTerminalHandle: params.from, - requireCurrentConsumer: true + requireCurrentConsumer: true, + legacyCoordinatorRunId }) const answered = db.answerQuestion({ messageId: question.message_id, @@ -993,7 +1124,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ defineMethod({ name: 'orchestration.taskCreate', params: TaskCreateParams, - handler: (params, { runtime }) => { + handler: (params, { runtime, legacyCoordinatorRunId }) => { const db = runtime.getOrchestrationDb() let deps: string[] | undefined if (params.deps) { @@ -1017,7 +1148,8 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ runId: resolveRunScope(runtime, { runId: params.run, callerTerminalHandle: params.callerTerminalHandle, - requireCurrentConsumer: true + requireCurrentConsumer: true, + legacyCoordinatorRunId }).id }) return { task } @@ -1027,7 +1159,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ defineMethod({ name: 'orchestration.taskList', params: TaskListParams, - handler: (params, { runtime }) => { + handler: (params, { runtime, legacyCoordinatorRunId }) => { const db = runtime.getOrchestrationDb() const explicitRun = params.run ? db.getRun(params.run) : undefined const run = @@ -1036,7 +1168,8 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ : resolveRunScope(runtime, { runId: params.run, callerTerminalHandle: params.callerTerminalHandle, - requireCurrentConsumer: params.run === undefined + requireCurrentConsumer: params.run === undefined, + legacyCoordinatorRunId }) // Why: listTasksWithDispatch adds assignee_handle + dispatch_id (NULL for non-dispatched), so legacy-shape consumers are unaffected. const joined = db.listTasksWithDispatch({ @@ -1063,12 +1196,13 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ defineMethod({ name: 'orchestration.taskUpdate', params: TaskUpdateParams, - handler: (params, { runtime }) => { + handler: (params, { runtime, legacyCoordinatorRunId }) => { const db = runtime.getOrchestrationDb() const run = resolveRunScope(runtime, { runId: params.run, callerTerminalHandle: params.callerTerminalHandle, - requireCurrentConsumer: true + requireCurrentConsumer: true, + legacyCoordinatorRunId }) const existing = db.getTask(params.id) if (!existing || existing.run_id !== run.id) { @@ -1088,7 +1222,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ defineMethod({ name: 'orchestration.dispatch', params: DispatchParams, - handler: async (params, { runtime }) => { + handler: async (params, { runtime, legacyCoordinatorRunId, revalidateLegacyCoordinator }) => { const db = runtime.getOrchestrationDb() const task = db.getTask(params.task) if (!task) { @@ -1097,7 +1231,8 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ const run = resolveRunScope(runtime, { runId: params.run, callerTerminalHandle: params.from, - requireCurrentConsumer: true + requireCurrentConsumer: true, + legacyCoordinatorRunId }) if (task.run_id !== run.id) { throw new OrchestrationError( @@ -1143,8 +1278,13 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ } } - const assigneePaneKey = runtime.getTerminalPaneKey(to) ?? undefined - const processIncarnation = runtime.getTerminalProcessIncarnation(to) ?? undefined + const dispatchAuthority = runtime.getOrchestrationDispatchAuthority(to) + const assigneePaneKey = + dispatchAuthority?.paneKey ?? runtime.getTerminalPaneKey(to) ?? undefined + const processIncarnation = + dispatchAuthority?.processIncarnation ?? + runtime.getTerminalProcessIncarnation(to) ?? + undefined if (params.inject && (!assigneePaneKey || !processIncarnation)) { throw new OrchestrationError( 'stable_pane_required', @@ -1152,7 +1292,13 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ ) } - const ctx = db.createDispatchContext(params.task, to, assigneePaneKey) + revalidateLegacyCoordinator?.() + const ctx = db.createDispatchContext( + params.task, + to, + assigneePaneKey, + dispatchAuthority?.launchTokenHash ?? undefined + ) const dispatchCapability = params.inject ? db.mintDispatchCapability({ dispatchId: ctx.id, diff --git a/src/main/runtime/rpc/orchestration-legacy-authority.ts b/src/main/runtime/rpc/orchestration-legacy-authority.ts new file mode 100644 index 00000000000..71e9f83acfa --- /dev/null +++ b/src/main/runtime/rpc/orchestration-legacy-authority.ts @@ -0,0 +1,257 @@ +import type { OrchestrationCompatibilityEvidence } from '../../../shared/orchestration-compatibility-evidence' +import type { + DispatchContextRow, + LegacyCompatibilityPrincipalRow, + LegacyPrincipalRole +} from '../orchestration/types' +import { OrchestrationError } from '../orchestration/orchestration-error' +import type { OrcaRuntimeService, OrchestrationCompatibilityCallerAuthority } from '../orca-runtime' +import { LEGACY_CONTRACT_VERSION } from '../orchestration/db' +import type { RpcRequest } from './core' +import { + equivalentLegacyPaneKey, + legacyCoordinatorReadOnly, + legacyReadOnlyError +} from './orchestration-legacy-process-identity' + +export type LegacyPrincipalCandidate = { + runId: string + role: LegacyPrincipalRole + dispatchId?: string + terminalHandle: string + paneKey: string +} + +export function resolveAttestedLegacyPrincipal(args: { + runtime: OrcaRuntimeService + evidence?: OrchestrationCompatibilityEvidence + candidate: LegacyPrincipalCandidate + authority?: OrchestrationCompatibilityCallerAuthority +}): LegacyCompatibilityPrincipalRow { + const authority = args.authority ?? verifyAttestedLegacyCandidate(args) + return args.runtime.getOrchestrationDb().commitLegacyCompatibilityPrincipal({ + runId: args.candidate.runId, + dispatchId: args.candidate.dispatchId, + role: args.candidate.role, + hostScope: JSON.stringify(authority.hostScope), + terminalHandle: args.candidate.terminalHandle, + paneKey: args.candidate.paneKey, + launchTokenHash: authority.launchTokenHash, + processIncarnation: authority.processIncarnation + }).principal +} + +export function verifyAttestedLegacyCandidate(args: { + runtime: OrcaRuntimeService + evidence?: OrchestrationCompatibilityEvidence + candidate: LegacyPrincipalCandidate +}) { + const authority = args.runtime.verifyOrchestrationCompatibilityCaller(args.evidence) + if ( + !authority || + !equivalentLegacyPaneKey(args.candidate.paneKey, authority.paneKey) || + args.candidate.terminalHandle !== authority.terminalHandle + ) { + throw legacyReadOnlyError() + } + if ( + args.candidate.role === 'worker' && + args.candidate.dispatchId && + !args.runtime.getOrchestrationDb().isDispatchProcessCurrent({ + dispatchId: args.candidate.dispatchId, + paneKey: authority.paneKey, + processIncarnation: authority.processIncarnation + }) + ) { + throw legacyReadOnlyError() + } + return authority +} + +export class LegacyCompatibilityAuthority { + constructor(private readonly runtime: OrcaRuntimeService) {} + + resolveWorkerDispatch( + request: RpcRequest, + target: { terminalHandle?: string; dispatchId?: string; taskId?: string } + ): DispatchContextRow | undefined { + const db = this.runtime.getOrchestrationDb() + const evidence = request.orchestrationCompatibilityEvidence + const adoptedRunId = db.getLegacyAdoption()?.adopted_run_id + const existing = adoptedRunId + ? db.resolveLegacyCompatibilityPrincipalByIdentity({ + runId: adoptedRunId, + role: 'worker', + terminalHandle: evidence?.terminalHandle, + paneKey: evidence?.paneKey + }) + : undefined + if (existing?.dispatch_id) { + verifyAttestedLegacyCandidate({ + runtime: this.runtime, + evidence, + candidate: candidateFromPrincipal(existing) + }) + const settledDispatch = db.getDispatchContextById(existing.dispatch_id) + if ( + settledDispatch && + (!target.dispatchId || target.dispatchId === settledDispatch.id) && + (!target.taskId || target.taskId === settledDispatch.task_id) + ) { + return settledDispatch + } + } + const dispatch = db.resolveLegacyWorkerCandidate({ + runId: adoptedRunId, + terminalHandle: evidence?.terminalHandle ?? target.terminalHandle, + paneKey: evidence?.paneKey, + dispatchId: target.dispatchId, + taskId: target.taskId + })?.dispatch + if (!dispatch && evidence) { + const retained = db.resolveLegacyWorkerCandidate({ + runId: adoptedRunId, + terminalHandle: target.terminalHandle, + dispatchId: target.dispatchId, + taskId: target.taskId + }) + if (retained) { + throw legacyReadOnlyError() + } + } + return dispatch + } + + resolveAskDispatch( + request: RpcRequest, + params: { from?: string; resume?: string } + ): DispatchContextRow | undefined { + const db = this.runtime.getOrchestrationDb() + if (params.resume) { + const question = db.getQuestion(params.resume) + return question ? db.getDispatchContextById(question.dispatch_id) : undefined + } + return this.resolveWorkerDispatch(request, { terminalHandle: params.from }) + } + + resolveCheckPrincipal( + request: RpcRequest, + terminalHandle?: string + ): LegacyCompatibilityPrincipalRow | undefined { + const db = this.runtime.getOrchestrationDb() + const adoption = db.getLegacyAdoption() + if (!adoption) { + return undefined + } + const evidence = request.orchestrationCompatibilityEvidence + const existingWorker = db.resolveLegacyCompatibilityPrincipalByIdentity({ + runId: adoption.adopted_run_id, + role: 'worker', + terminalHandle: evidence?.terminalHandle, + paneKey: evidence?.paneKey + }) + if (existingWorker) { + verifyAttestedLegacyCandidate({ + runtime: this.runtime, + evidence, + candidate: candidateFromPrincipal(existingWorker) + }) + return existingWorker + } + const worker = db.resolveLegacyWorkerCandidate({ + runId: adoption.adopted_run_id, + terminalHandle: evidence?.terminalHandle ?? terminalHandle, + paneKey: evidence?.paneKey + }) + if (worker) { + return this.attestWorker(request, worker.dispatch) + } + if (evidence) { + const retained = db.resolveLegacyWorkerCandidate({ + runId: adoption.adopted_run_id, + terminalHandle + }) + const settled = db.resolveLegacyCompatibilityPrincipalByIdentity({ + runId: adoption.adopted_run_id, + role: 'worker', + terminalHandle + }) + if (retained || settled) { + throw legacyReadOnlyError() + } + } + return this.attestCoordinator(request, adoption.adopted_run_id, false) + } + + attestWorker(request: RpcRequest, dispatch: DispatchContextRow): LegacyCompatibilityPrincipalRow { + if (dispatch.contract_version !== LEGACY_CONTRACT_VERSION) { + throw new OrchestrationError( + 'request_mismatch', + `Dispatch ${dispatch.id} does not use the legacy contract.` + ) + } + return resolveAttestedLegacyPrincipal({ + runtime: this.runtime, + evidence: request.orchestrationCompatibilityEvidence, + candidate: candidateFromDispatch(dispatch) + }) + } + + attestCoordinator( + request: RpcRequest, + runId: string, + required = true + ): LegacyCompatibilityPrincipalRow | undefined { + const db = this.runtime.getOrchestrationDb() + const candidate = db.resolveLegacyCoordinatorCandidate({ + runId, + terminalHandle: request.orchestrationCompatibilityEvidence?.terminalHandle, + paneKey: request.orchestrationCompatibilityEvidence?.paneKey + }) + if (!candidate) { + if (required) { + throw legacyCoordinatorReadOnly() + } + return undefined + } + const principal = resolveAttestedLegacyPrincipal({ + runtime: this.runtime, + evidence: request.orchestrationCompatibilityEvidence, + candidate: { + runId, + role: 'coordinator', + terminalHandle: candidate.terminalHandle, + paneKey: candidate.paneKey + } + }) + return principal + } +} + +function candidateFromDispatch(dispatch: DispatchContextRow): LegacyPrincipalCandidate { + if (!dispatch.assignee_handle || !dispatch.assignee_pane_key) { + throw new OrchestrationError( + 'legacy_read_only', + `Dispatch ${dispatch.id} lacks durable process identity. No effects were applied.` + ) + } + return { + runId: dispatch.run_id, + role: 'worker', + dispatchId: dispatch.id, + terminalHandle: dispatch.assignee_handle, + paneKey: dispatch.assignee_pane_key + } +} + +function candidateFromPrincipal( + principal: LegacyCompatibilityPrincipalRow +): LegacyPrincipalCandidate { + return { + runId: principal.run_id, + role: principal.role, + dispatchId: principal.dispatch_id ?? undefined, + terminalHandle: principal.terminal_handle, + paneKey: principal.pane_key + } +} diff --git a/src/main/runtime/rpc/orchestration-legacy-compatibility-dispatcher-test-fixture.ts b/src/main/runtime/rpc/orchestration-legacy-compatibility-dispatcher-test-fixture.ts new file mode 100644 index 00000000000..a7d98082a54 --- /dev/null +++ b/src/main/runtime/rpc/orchestration-legacy-compatibility-dispatcher-test-fixture.ts @@ -0,0 +1,176 @@ +import { createHash } from 'node:crypto' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { expect, vi } from 'vitest' +import type { OrchestrationCompatibilityEvidence } from '../../../shared/orchestration-compatibility-evidence' +import { ORCHESTRATION_CONTRACT_VERSION } from '../../../shared/protocol-version' +import Database from '../../sqlite/sync-database' +import { OrcaRuntimeService } from '../orca-runtime' +import { OrchestrationDb } from '../orchestration/db' +import type { RpcRequest, RpcResponse } from './core' +import { RpcDispatcher } from './dispatcher' +import { ORCHESTRATION_METHODS } from './methods/orchestration' + +export const WORKER_HANDLE = 'term_legacy_worker' +export const WORKER_PANE = 'tab_worker:33333333-3333-4333-8333-333333333333' +export const COORDINATOR_HANDLE = 'term_legacy_coord' +export const COORDINATOR_PANE = 'tab_coord:44444444-4444-4444-8444-444444444444' + +type Transport = 'dispatch' | 'websocket' + +export type LegacyCompatibilityDispatcherHarness = { + db: OrchestrationDb + dispatcher: RpcDispatcher + runtime: OrcaRuntimeService + adoptedRunId: string + taskId: string + dispatchId: string + notify: ReturnType + verify: ReturnType +} + +const tempDirs: string[] = [] +const databases: OrchestrationDb[] = [] + +export function cleanupLegacyCompatibilityDispatcherHarnesses(): void { + for (const database of databases.splice(0)) { + database.close() + } + for (const dir of tempDirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }) + } +} + +export function createHarness(): LegacyCompatibilityDispatcherHarness { + const dir = mkdtempSync(join(tmpdir(), 'orca-legacy-dispatcher-')) + tempDirs.push(dir) + const dbPath = join(dir, 'orchestration.db') + const before = new OrchestrationDb(dbPath) + const task = before.createTask({ + spec: 'legacy assignment', + createdByTerminalHandle: COORDINATOR_HANDLE + }) + const dispatch = before.createDispatchContext(task.id, WORKER_HANDLE, WORKER_PANE) + before.close() + + const raw = new Database(dbPath) + raw.exec(` + UPDATE dispatch_contexts SET process_incarnation = 'process-1'; + DROP INDEX IF EXISTS idx_messages_delivery_contract; + DROP TABLE legacy_mail_receipts; + DROP TABLE legacy_operation_receipts; + DROP TABLE legacy_compatibility_principals; + DROP TABLE legacy_adoptions; + `) + raw.pragma('user_version = 18') + raw.close() + + const db = new OrchestrationDb(dbPath) + databases.push(db) + const adoptedRunId = db.getLegacyAdoption()?.adopted_run_id as string + const runtime = new OrcaRuntimeService() + runtime.setOrchestrationDb(db) + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === COORDINATOR_HANDLE ? COORDINATOR_PANE : handle === WORKER_HANDLE ? WORKER_PANE : null + ) + const verify = vi + .spyOn(runtime, 'verifyOrchestrationCompatibilityCaller') + .mockImplementation((evidence) => { + const validWorker = + evidence?.terminalHandle === WORKER_HANDLE && evidence.paneKey === WORKER_PANE + const validCoordinator = + evidence?.terminalHandle === COORDINATOR_HANDLE && evidence.paneKey === COORDINATOR_PANE + if ((!validWorker && !validCoordinator) || !evidence?.launchToken) { + return null + } + return { + hostScope: { kind: 'local', hostId: 'local' }, + terminalHandle: evidence.terminalHandle as string, + paneKey: evidence.paneKey as string, + processIncarnation: 'process-1', + launchTokenHash: createHash('sha256').update(evidence.launchToken).digest('hex') + } + }) + const notify = vi.spyOn(runtime, 'notifyMessageArrived').mockImplementation(() => {}) + return { + db, + dispatcher: new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS }), + runtime, + adoptedRunId, + taskId: task.id, + dispatchId: dispatch.id, + notify, + verify + } +} + +export function evidence( + role: 'worker' | 'coordinator', + valid = true +): OrchestrationCompatibilityEvidence { + const worker = role === 'worker' + return { + terminalHandle: worker ? WORKER_HANDLE : COORDINATOR_HANDLE, + paneKey: valid ? (worker ? WORKER_PANE : COORDINATOR_PANE) : 'tab_wrong:wrong-leaf', + launchToken: `${role}-token` + } +} + +export function request( + method: string, + params: unknown, + proof: OrchestrationCompatibilityEvidence, + invocationId: string +): RpcRequest { + return { + id: `rpc_${invocationId}`, + authToken: 'caller-token', + method, + params, + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationRequestId: invocationId, + compatibilityInvocationId: invocationId, + orchestrationCompatibilityEvidence: proof + } +} + +export async function invoke( + dispatcher: RpcDispatcher, + rpcRequest: RpcRequest, + transport: Transport +): Promise { + if (transport === 'dispatch') { + return await dispatcher.dispatch(rpcRequest) + } + const replies: string[] = [] + await dispatcher.dispatchStreaming(rpcRequest, (reply) => replies.push(reply)) + expect(replies).toHaveLength(1) + return JSON.parse(replies[0]) as RpcResponse +} + +export function counts(db: OrchestrationDb): Record { + const sqlite = (db as unknown as { db: Database.Database }).db + return Object.fromEntries( + [ + 'messages', + 'legacy_compatibility_principals', + 'legacy_operation_receipts', + 'legacy_mail_receipts', + 'mutation_receipts' + ].map((table) => [ + table, + (sqlite.prepare(`SELECT COUNT(*) AS count FROM ${table}`).get() as { count: number }).count + ]) + ) +} + +export function escalationParams(harness: LegacyCompatibilityDispatcherHarness) { + return { + from: WORKER_HANDLE, + to: COORDINATOR_HANDLE, + subject: 'Blocked', + type: 'escalation', + payload: JSON.stringify({ taskId: harness.taskId, dispatchId: harness.dispatchId }) + } +} diff --git a/src/main/runtime/rpc/orchestration-legacy-compatibility-dispatcher.test.ts b/src/main/runtime/rpc/orchestration-legacy-compatibility-dispatcher.test.ts new file mode 100644 index 00000000000..3c087702627 --- /dev/null +++ b/src/main/runtime/rpc/orchestration-legacy-compatibility-dispatcher.test.ts @@ -0,0 +1,755 @@ +import { createHash } from 'node:crypto' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type Database from '../../sqlite/sync-database' +import { + cleanupLegacyCompatibilityDispatcherHarnesses, + COORDINATOR_HANDLE, + COORDINATOR_PANE, + counts, + createHarness, + escalationParams, + evidence, + invoke, + request, + WORKER_HANDLE, + WORKER_PANE +} from './orchestration-legacy-compatibility-dispatcher-test-fixture' + +afterEach(() => { + cleanupLegacyCompatibilityDispatcherHarnesses() +}) + +describe('legacy compatibility through RpcDispatcher', () => { + it('rejects malformed current-contract input before compatibility attestation', async () => { + const harness = createHarness() + const before = counts(harness.db) + const response = await harness.dispatcher.dispatch( + request( + 'orchestration.send', + { from: WORKER_HANDLE, type: 'escalation' }, + evidence('worker'), + 'malformed' + ) + ) + + expect(response).toMatchObject({ ok: false, error: { code: 'invalid_argument' } }) + expect(harness.verify).not.toHaveBeenCalled() + expect(counts(harness.db)).toEqual(before) + }) + + it.each([ + ['dispatch', true], + ['dispatch', false], + ['websocket', true], + ['websocket', false] + ] as const)( + '%s routes task-only escalation with valid proof=%s and zero partial effects', + async (transport, valid) => { + const harness = createHarness() + const before = counts(harness.db) + const response = await invoke( + harness.dispatcher, + request( + 'orchestration.send', + escalationParams(harness), + evidence('worker', valid), + `${transport}-${valid}` + ), + transport + ) + + if (!valid) { + expect(response).toMatchObject({ + ok: false, + error: { code: 'legacy_read_only' } + }) + expect(counts(harness.db)).toEqual(before) + expect(harness.notify).not.toHaveBeenCalled() + return + } + + expect(response).toMatchObject({ + ok: true, + result: { + message: { type: 'escalation', delivery_contract: 'legacy_direct' }, + legacyCompatibility: { replayed: false } + } + }) + expect(counts(harness.db)).toEqual({ + ...before, + messages: before.messages + 1, + legacy_compatibility_principals: before.legacy_compatibility_principals + 1, + legacy_operation_receipts: before.legacy_operation_receipts + 1 + }) + expect(harness.notify).toHaveBeenCalledOnce() + } + ) + + it('validates, infers, settles, and replays legacy worker completion exactly once', async () => { + const harness = createHarness() + const baseParams = { + from: WORKER_HANDLE, + to: COORDINATOR_HANDLE, + type: 'worker_done', + body: 'legacy result', + payload: JSON.stringify({ taskId: harness.taskId, dispatchId: harness.dispatchId }) + } + const before = counts(harness.db) + const invalid = await harness.dispatcher.dispatch( + request( + 'orchestration.send', + { ...baseParams, subject: 'Completed', payload: JSON.stringify({ outcome: 'maybe' }) }, + evidence('worker'), + 'invalid-outcome' + ) + ) + + expect(invalid).toMatchObject({ ok: false, error: { code: 'invalid_argument' } }) + expect(counts(harness.db)).toEqual(before) + + const firstRequest = request( + 'orchestration.send', + { ...baseParams, subject: 'Completed' }, + evidence('worker'), + 'completion' + ) + const first = await harness.dispatcher.dispatch(firstRequest) + const replay = await harness.dispatcher.dispatch({ ...firstRequest, id: 'rpc_replay' }) + const mismatch = await harness.dispatcher.dispatch({ + ...firstRequest, + id: 'rpc_mismatch', + params: { ...baseParams, subject: 'Changed completion' } + }) + + expect(first).toMatchObject({ + ok: true, + result: { + lifecycle: { action: 'settled', outcome: 'succeeded' }, + legacyCompatibility: { replayed: false } + } + }) + expect(replay).toMatchObject({ + ok: true, + result: { legacyCompatibility: { replayed: true } } + }) + expect(mismatch).toMatchObject({ ok: false, error: { code: 'request_mismatch' } }) + expect(harness.db.getTask(harness.taskId)?.status).toBe('completed') + expect(harness.db.getDispatchContextById(harness.dispatchId)?.status).toBe('completed') + expect(counts(harness.db)).toEqual({ + ...before, + messages: before.messages + 1, + legacy_compatibility_principals: before.legacy_compatibility_principals + 1, + legacy_operation_receipts: before.legacy_operation_receipts + 1 + }) + expect(harness.notify).toHaveBeenCalledOnce() + }) + + it('replays an A-era completion without touching a newer current attempt', async () => { + const harness = createHarness() + const payload = JSON.stringify({ + taskId: harness.taskId, + dispatchId: harness.dispatchId + }) + const completion = harness.db.insertMessage({ + runId: harness.adoptedRunId, + from: WORKER_HANDLE, + to: COORDINATOR_HANDLE, + subject: 'Failed: pinned A treated this as complete', + body: 'persisted result', + type: 'worker_done', + payload, + senderPaneKey: WORKER_PANE, + deliveryContract: 'legacy_direct' + }) + harness.db.settleWorkerReport({ + taskId: harness.taskId, + dispatchId: harness.dispatchId, + outcome: 'succeeded', + result: 'persisted result' + }) + harness.db.commitLegacyCompatibilityPrincipal({ + runId: harness.adoptedRunId, + dispatchId: harness.dispatchId, + role: 'worker', + hostScope: JSON.stringify({ kind: 'local', hostId: 'local' }), + terminalHandle: WORKER_HANDLE, + paneKey: WORKER_PANE, + launchTokenHash: createHash('sha256').update('worker-token').digest('hex'), + processIncarnation: 'process-1' + }) + harness.db.updateTaskStatus(harness.taskId, 'ready') + const currentDispatch = harness.db.createDispatchContext( + harness.taskId, + 'term_current_worker', + 'tab_current_worker:77777777-7777-4777-8777-777777777777', + 'current-launch-hash' + ) + const currentTaskBefore = harness.db.getTask(harness.taskId) + const currentDispatchBefore = harness.db.getDispatchContextById(currentDispatch.id) + const before = counts(harness.db) + + const response = await harness.dispatcher.dispatch( + request( + 'orchestration.send', + { + from: WORKER_HANDLE, + to: COORDINATOR_HANDLE, + subject: completion.subject, + body: completion.body, + type: 'worker_done', + payload + }, + evidence('worker'), + 'reconstruct-pinned-a-completion' + ) + ) + + expect(response).toMatchObject({ + ok: true, + result: { + message: { id: completion.id }, + lifecycle: { action: 'settled', outcome: 'succeeded', duplicate: true } + } + }) + expect(harness.db.getTask(harness.taskId)).toEqual(currentTaskBefore) + expect(harness.db.getDispatchContextById(harness.dispatchId)?.status).toBe('completed') + expect(harness.db.getDispatchContextById(currentDispatch.id)).toEqual(currentDispatchBefore) + expect(counts(harness.db)).toEqual({ + ...before, + legacy_operation_receipts: before.legacy_operation_receipts + 1 + }) + }) + + it('rejects a legacy lifecycle recipient outside the adopted Run with zero effects', async () => { + const harness = createHarness() + const before = counts(harness.db) + const response = await harness.dispatcher.dispatch( + request( + 'orchestration.send', + { + ...escalationParams(harness), + to: 'term_unrelated' + }, + evidence('worker'), + 'wrong-recipient' + ) + ) + + expect(response).toMatchObject({ + ok: false, + error: { code: 'request_mismatch' } + }) + expect(counts(harness.db)).toEqual(before) + expect(harness.verify).not.toHaveBeenCalled() + expect(harness.notify).not.toHaveBeenCalled() + }) + + it('rejects a reused pane whose live process incarnation is not the legacy worker', async () => { + const harness = createHarness() + const sqlite = (harness.db as unknown as { db: Database.Database }).db + sqlite + .prepare('UPDATE dispatch_contexts SET process_incarnation = ? WHERE id = ?') + .run('different-process', harness.dispatchId) + const before = counts(harness.db) + + const response = await harness.dispatcher.dispatch( + request('orchestration.send', escalationParams(harness), evidence('worker'), 'reused-pane') + ) + + expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } }) + expect(counts(harness.db)).toEqual(before) + expect(harness.notify).not.toHaveBeenCalled() + }) + + it('rejects a legacy question recipient outside the adopted Run with zero effects', async () => { + const harness = createHarness() + const before = counts(harness.db) + const response = await harness.dispatcher.dispatch( + request( + 'orchestration.ask', + { + from: WORKER_HANDLE, + to: 'term_unrelated', + question: 'Proceed?', + timeoutMs: 1 + }, + evidence('worker'), + 'wrong-question-recipient' + ) + ) + + expect(response).toMatchObject({ + ok: false, + error: { code: 'request_mismatch' } + }) + expect(counts(harness.db)).toEqual(before) + expect(harness.verify).not.toHaveBeenCalled() + expect(harness.notify).not.toHaveBeenCalled() + }) + + it('keeps distinct compatibility ask invocations on distinct questions', async () => { + const harness = createHarness() + const ask = { + from: WORKER_HANDLE, + to: COORDINATOR_HANDLE, + question: 'Same question?', + options: ['yes', 'no'], + timeoutMs: 1 + } + const first = await harness.dispatcher.dispatch( + request('orchestration.ask', ask, evidence('worker'), 'ask-one') + ) + const replay = await harness.dispatcher.dispatch( + request('orchestration.ask', ask, evidence('worker'), 'ask-one') + ) + const second = await harness.dispatcher.dispatch( + request('orchestration.ask', ask, evidence('worker'), 'ask-two') + ) + + expect(first).toMatchObject({ ok: true, result: { legacyCompatibility: { replayed: false } } }) + expect(replay).toMatchObject({ ok: true, result: { legacyCompatibility: { replayed: true } } }) + expect(second).toMatchObject({ ok: true, result: { legacyCompatibility: { replayed: false } } }) + const firstId = (first as { result: { messageId: string } }).result.messageId + const replayId = (replay as { result: { messageId: string } }).result.messageId + const secondId = (second as { result: { messageId: string } }).result.messageId + expect(replayId).toBe(firstId) + expect(secondId).not.toBe(firstId) + expect(harness.db.getMessageById(firstId)?.type).toBe('decision_gate') + + const coordinatorCheck = await harness.dispatcher.dispatch( + request( + 'orchestration.check', + { + terminal: COORDINATOR_HANDLE, + types: 'worker_done,escalation,decision_gate', + format: true + }, + evidence('coordinator'), + 'coordinator-check-decision-gates' + ) + ) + expect(coordinatorCheck).toMatchObject({ + ok: true, + result: { + messages: [ + { id: firstId, type: 'decision_gate' }, + { id: secondId, type: 'decision_gate' } + ], + count: 2 + } + }) + expect((coordinatorCheck as { result: { formatted: string } }).result.formatted).toContain( + `orca orchestration reply --id ${firstId}` + ) + }) + + it('keeps the normal wait budget when legacy check omits timeout', async () => { + const harness = createHarness() + const waitForMessage = vi + .spyOn(harness.runtime, 'waitForMessage') + .mockResolvedValue('timed_out') + let clockReads = 0 + const now = vi.spyOn(Date, 'now').mockImplementation(() => { + clockReads += 1 + return clockReads === 1 ? 0 : clockReads === 2 ? 1 : 120_001 + }) + + try { + const response = await harness.dispatcher.dispatch( + request( + 'orchestration.check', + { terminal: WORKER_HANDLE, wait: true }, + evidence('worker'), + 'legacy-check-default-timeout' + ) + ) + + expect(response).toMatchObject({ + ok: true, + result: { messages: [], count: 0, timedOut: true } + }) + expect(waitForMessage).toHaveBeenCalledOnce() + } finally { + now.mockRestore() + } + }) + + it('does not infer an outcome for a current Dispatch when legacy adoption exists', async () => { + const harness = createHarness() + const run = harness.db.createRun({ + objective: 'current work', + coordinatorHandle: 'term_current_coord', + coordinatorPaneKey: 'tab_current_coord:55555555-5555-4555-8555-555555555555' + }) + const task = harness.db.createTask({ spec: 'current assignment', runId: run.id }) + const dispatch = harness.db.createDispatchContext( + task.id, + 'term_current_worker', + 'tab_current_worker:66666666-6666-4666-8666-666666666666', + 'current-launch-hash' + ) + const before = counts(harness.db) + const response = await harness.dispatcher.dispatch( + request( + 'orchestration.send', + { + from: 'term_current_worker', + to: COORDINATOR_HANDLE, + subject: 'Completed', + type: 'worker_done', + payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id }) + }, + { + terminalHandle: 'term_current_worker', + paneKey: 'tab_current_worker:66666666-6666-4666-8666-666666666666', + launchToken: 'current-token' + }, + 'current-missing-outcome' + ) + ) + + expect(response).toMatchObject({ ok: false, error: { code: 'invalid_argument' } }) + expect(harness.db.getTask(task.id)?.status).toBe('dispatched') + expect(harness.db.getDispatchContextById(dispatch.id)?.status).toBe('dispatched') + expect(counts(harness.db)).toEqual(before) + }) + + it('rejects invalid typed ACKs and consumes only the filtered legacy page', async () => { + const harness = createHarness() + await harness.dispatcher.dispatch( + request( + 'orchestration.send', + { + ...escalationParams(harness), + type: 'worker_done', + subject: 'Completed', + payload: JSON.stringify({ taskId: harness.taskId, dispatchId: harness.dispatchId }) + }, + evidence('worker'), + 'settle-for-mail' + ) + ) + const status = harness.db.insertMessage({ + runId: harness.adoptedRunId, + from: COORDINATOR_HANDLE, + to: WORKER_HANDLE, + subject: 'status first', + type: 'status', + deliveryContract: 'legacy_direct' + }) + const question = harness.db.insertMessage({ + runId: harness.adoptedRunId, + from: COORDINATOR_HANDLE, + to: WORKER_HANDLE, + subject: 'question second', + type: 'question', + deliveryContract: 'legacy_direct' + }) + const check = await harness.dispatcher.dispatch( + request( + 'orchestration.check', + { terminal: WORKER_HANDLE, types: 'question' }, + evidence('worker'), + 'question-check' + ) + ) + expect(check).toMatchObject({ + ok: true, + result: { + messages: [{ id: question.id }], + legacyCompatibility: { ackMessageIds: [question.id] } + } + }) + + const invalid = await harness.dispatcher.dispatch( + request( + 'orchestration.check', + { + terminal: WORKER_HANDLE, + compatibilityAck: JSON.stringify({ + messageIds: [question.id], + types: ['not-a-message-type'] + }) + }, + evidence('worker'), + 'invalid-ack' + ) + ) + expect(invalid).toMatchObject({ ok: false, error: { code: 'invalid_argument' } }) + expect(harness.db.getMessageById(status.id)?.read).toBe(0) + expect(harness.db.getMessageById(question.id)?.read).toBe(0) + + const acknowledged = await harness.dispatcher.dispatch( + request( + 'orchestration.check', + { + terminal: WORKER_HANDLE, + compatibilityAck: JSON.stringify({ + messageIds: [question.id], + types: ['question'] + }) + }, + evidence('worker'), + 'valid-ack' + ) + ) + expect(acknowledged).toMatchObject({ + ok: true, + result: { acknowledged: [question.id], legacyCompatibility: { acknowledged: true } } + }) + expect(harness.db.getMessageById(status.id)?.read).toBe(0) + expect(harness.db.getMessageById(question.id)?.read).toBe(1) + }) + + it('rejects invalid legacy check types before attestation or mail consumption', async () => { + const harness = createHarness() + const message = harness.db.insertMessage({ + runId: harness.adoptedRunId, + from: COORDINATOR_HANDLE, + to: WORKER_HANDLE, + subject: 'retained status', + type: 'status', + deliveryContract: 'legacy_direct' + }) + const before = counts(harness.db) + const response = await harness.dispatcher.dispatch( + request( + 'orchestration.check', + { terminal: WORKER_HANDLE, types: 'status,not-a-message-type' }, + evidence('worker'), + 'invalid-types' + ) + ) + + expect(response).toMatchObject({ + ok: false, + error: { code: 'invalid_argument', message: 'Invalid --types: not-a-message-type' } + }) + expect(counts(harness.db)).toEqual(before) + expect(harness.db.getMessageById(message.id)?.read).toBe(0) + expect(harness.verify).not.toHaveBeenCalled() + expect(harness.notify).not.toHaveBeenCalled() + }) + + it('advertises only Run-addressed current delivery to a legacy coordinator', async () => { + const harness = createHarness() + harness.db.insertMessage({ + runId: harness.adoptedRunId, + from: 'term_current_worker', + to: WORKER_HANDLE, + subject: 'worker-only current mail', + deliveryContract: 'current_delivery' + }) + const workerOnly = await harness.dispatcher.dispatch( + request( + 'orchestration.check', + { terminal: COORDINATOR_HANDLE }, + evidence('coordinator'), + 'worker-current-mail' + ) + ) + expect(workerOnly).toMatchObject({ + ok: true, + result: { legacyCompatibility: { currentDelivery: undefined } } + }) + + harness.db.insertMessage({ + runId: harness.adoptedRunId, + from: 'term_current_worker', + to: `run:${harness.adoptedRunId}`, + subject: 'Run current mail', + deliveryContract: 'current_delivery' + }) + const runMail = await harness.dispatcher.dispatch( + request( + 'orchestration.check', + { terminal: COORDINATOR_HANDLE }, + evidence('coordinator'), + 'run-current-mail' + ) + ) + expect(runMail).toMatchObject({ + ok: true, + result: { + legacyCompatibility: { + currentDelivery: { + runId: harness.adoptedRunId, + checkCommand: expect.stringContaining(`--run ${harness.adoptedRunId}`) + } + } + } + }) + }) + + it('rejects a legacy coordinator reply outside the adopted Run with zero effects', async () => { + const harness = createHarness() + const unrelatedRun = harness.db.createRun({ + objective: 'unrelated current work', + coordinatorHandle: 'term_unrelated_coord', + coordinatorPaneKey: 'tab_unrelated_coord:55555555-5555-4555-8555-555555555555' + }) + const unrelated = harness.db.insertMessage({ + runId: unrelatedRun.id, + from: 'term_unrelated_worker', + to: `run:${unrelatedRun.id}`, + subject: 'Unrelated current status', + deliveryContract: 'current_delivery' + }) + const before = counts(harness.db) + + const response = await harness.dispatcher.dispatch( + request( + 'orchestration.reply', + { + id: unrelated.id, + body: 'Forged cross-Run reply', + from: COORDINATOR_HANDLE, + run: harness.adoptedRunId + }, + evidence('coordinator'), + 'cross-run-reply' + ) + ) + + expect(response).toMatchObject({ + ok: false, + error: { + code: 'request_mismatch', + data: { effectsApplied: false } + } + }) + expect(counts(harness.db)).toEqual(before) + expect(harness.db.getMessageById(unrelated.id)?.read).toBe(0) + expect(harness.notify).not.toHaveBeenCalled() + }) + + it('keeps explicit adopted Run task inspection available to unrelated callers', async () => { + const harness = createHarness() + const before = counts(harness.db) + const response = await harness.dispatcher.dispatch( + request( + 'orchestration.taskList', + { run: harness.adoptedRunId, callerTerminalHandle: 'term_unrelated' }, + { + terminalHandle: 'term_unrelated', + paneKey: 'tab_unrelated:55555555-5555-4555-8555-555555555555', + launchToken: 'unrelated-token' + }, + 'explicit-adopted-run-inspection' + ) + ) + + expect(response).toMatchObject({ + ok: true, + result: { + runId: harness.adoptedRunId, + tasks: [expect.objectContaining({ id: harness.taskId })] + } + }) + expect(counts(harness.db)).toEqual(before) + expect(harness.verify).not.toHaveBeenCalled() + }) + + it.each(['dispatch', 'websocket'] as const)( + '%s binds an attested coordinator once and rejects contradictory proof before binding', + async (transport) => { + const invalidHarness = createHarness() + const invalidBefore = counts(invalidHarness.db) + const rejected = await invoke( + invalidHarness.dispatcher, + request( + 'orchestration.runUse', + { id: invalidHarness.adoptedRunId, from: COORDINATOR_HANDLE }, + evidence('coordinator', false), + `run-use-invalid-${transport}` + ), + transport + ) + expect(rejected).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } }) + expect(invalidHarness.db.getRun(invalidHarness.adoptedRunId)?.consumer_generation).toBe(0) + expect(counts(invalidHarness.db)).toEqual(invalidBefore) + + const harness = createHarness() + const runUse = request( + 'orchestration.runUse', + { id: harness.adoptedRunId, from: COORDINATOR_HANDLE }, + evidence('coordinator'), + `run-use-${transport}` + ) + const first = await invoke(harness.dispatcher, runUse, transport) + const replay = await invoke( + harness.dispatcher, + { ...runUse, id: 'rpc_run-use-replay' }, + transport + ) + expect(first).toMatchObject({ + ok: true, + result: { binding: { consumerGeneration: 1 }, mutation: { replayed: false } } + }) + expect(replay).toMatchObject({ + ok: true, + result: { binding: { consumerGeneration: 1 }, mutation: { replayed: true } } + }) + expect(harness.db.getRun(harness.adoptedRunId)?.consumer_generation).toBe(1) + } + ) + + it('passes trusted coordinator scope to a failing handler without binding or receipts', async () => { + const harness = createHarness() + const before = counts(harness.db) + const response = await harness.dispatcher.dispatch( + request( + 'orchestration.taskUpdate', + { + id: 'task_missing', + status: 'completed', + callerTerminalHandle: COORDINATOR_HANDLE + }, + evidence('coordinator'), + 'missing-task' + ) + ) + + expect(response).toMatchObject({ ok: false, error: { code: 'task_not_found' } }) + expect(harness.db.getRun(harness.adoptedRunId)?.consumer_generation).toBe(0) + expect(counts(harness.db)).toEqual(before) + }) + + it('rejects stale legacy coordinator proof after current takeover', async () => { + const harness = createHarness() + const principal = harness.db.commitLegacyCompatibilityPrincipal({ + runId: harness.adoptedRunId, + role: 'coordinator', + hostScope: JSON.stringify({ kind: 'local', hostId: 'local' }), + terminalHandle: COORDINATOR_HANDLE, + paneKey: COORDINATOR_PANE, + launchTokenHash: createHash('sha256').update('coordinator-token').digest('hex'), + processIncarnation: 'process-1' + }).principal + harness.db.settleWorkerReport({ + taskId: harness.taskId, + dispatchId: harness.dispatchId, + outcome: 'succeeded', + result: 'done' + }) + harness.db.bindRun({ + runId: harness.adoptedRunId, + coordinatorHandle: 'term_current_coord', + coordinatorPaneKey: 'tab_current:55555555-5555-4555-8555-555555555555' + }) + expect(harness.db.getLegacyCompatibilityPrincipal(principal.id)?.status).toBe('revoked') + const before = counts(harness.db) + + const response = await harness.dispatcher.dispatch( + request( + 'orchestration.taskList', + {}, + evidence('coordinator'), + 'stale-coordinator-after-takeover' + ) + ) + + expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } }) + expect(counts(harness.db)).toEqual(before) + }) +}) diff --git a/src/main/runtime/rpc/orchestration-legacy-compatibility.ts b/src/main/runtime/rpc/orchestration-legacy-compatibility.ts new file mode 100644 index 00000000000..ff699d34636 --- /dev/null +++ b/src/main/runtime/rpc/orchestration-legacy-compatibility.ts @@ -0,0 +1,170 @@ +import { createHash } from 'node:crypto' +import type { LegacyCoordinatorAuthorityProof, RpcRequest } from './core' +import type { OrcaRuntimeService, OrchestrationCompatibilityCallerAuthority } from '../orca-runtime' +import { LegacyCompatibilityAuthority } from './orchestration-legacy-authority' +import { handleLegacyLifecycleSend } from './orchestration-legacy-lifecycle' +import { handleLegacyCheck, handleLegacyReply } from './orchestration-legacy-mail' +import { handleLegacyAsk } from './orchestration-legacy-question' +import type { + LegacyAskParams, + LegacyCheckParams, + LegacyReplyParams, + LegacySendParams +} from './orchestration-legacy-operation' +import { LegacyCoordinatorAuthority } from './orchestration-legacy-coordinator-authority' + +const COORDINATOR_PREFLIGHT_METHODS = new Set([ + 'orchestration.taskCreate', + 'orchestration.taskList', + 'orchestration.taskUpdate', + 'orchestration.dispatch', + 'orchestration.gateCreate', + 'orchestration.gateResolve', + 'orchestration.runUse', + 'orchestration.send', + 'orchestration.check', + 'orchestration.reply' +]) + +export type LegacyCompatibilityRoute = + | { handled: true; result: unknown } + | { + handled: false + params?: unknown + legacyCoordinatorAuthority?: LegacyCoordinatorAuthorityProof + orchestrationCompatibilityCallerAuthority?: OrchestrationCompatibilityCallerAuthority + } + +export type LegacyCoordinatorInvocation = Readonly<{ + authority: LegacyCoordinatorAuthorityProof + mutationCallerFingerprint: string + revalidate: () => string +}> + +export class OrchestrationLegacyCompatibility { + private readonly authority: LegacyCompatibilityAuthority + private readonly coordinatorAuthority: LegacyCoordinatorAuthority + + constructor(private readonly runtime: OrcaRuntimeService) { + this.authority = new LegacyCompatibilityAuthority(runtime) + this.coordinatorAuthority = new LegacyCoordinatorAuthority(runtime) + } + + async tryHandle( + request: RpcRequest, + params: unknown, + signal?: AbortSignal + ): Promise { + if (!request.method.startsWith('orchestration.')) { + return { handled: false } + } + const result = await this.route(request, params, signal) + if (result !== undefined) { + return { handled: true, result } + } + if (!COORDINATOR_PREFLIGHT_METHODS.has(request.method)) { + return { handled: false } + } + const values = params as Record + if (request.method === 'orchestration.runUse' && values.takeoverLegacy === true) { + const callerAuthority = this.runtime.verifyOrchestrationCompatibilityCaller( + request.orchestrationCompatibilityEvidence + ) + return { + handled: false, + ...(callerAuthority ? { orchestrationCompatibilityCallerAuthority: callerAuthority } : {}) + } + } + const requestedRunId = + request.method === 'orchestration.runUse' ? stringValue(values.id) : stringValue(values.run) + if (request.method === 'orchestration.taskList' && requestedRunId) { + return { handled: false } + } + const authority = this.coordinatorAuthority.resolve(request, requestedRunId) + return authority + ? { + handled: false, + params: { ...values, run: authority.runId }, + legacyCoordinatorAuthority: authority + } + : { handled: false } + } + + revalidateCoordinatorAuthority( + request: RpcRequest, + proof: LegacyCoordinatorAuthorityProof + ): string { + return this.coordinatorAuthority.revalidate(request, proof) + } + + createCoordinatorInvocation( + request: RpcRequest, + authority?: LegacyCoordinatorAuthorityProof + ): LegacyCoordinatorInvocation | undefined { + return authority + ? { + authority, + mutationCallerFingerprint: legacyCoordinatorMutationCallerFingerprint(authority), + revalidate: () => this.revalidateCoordinatorAuthority(request, authority) + } + : undefined + } + + private async route( + request: RpcRequest, + params: unknown, + signal?: AbortSignal + ): Promise { + if (request.method === 'orchestration.send') { + return await handleLegacyLifecycleSend({ + runtime: this.runtime, + authority: this.authority, + request, + params: params as LegacySendParams + }) + } + if (request.method === 'orchestration.check') { + return await handleLegacyCheck({ + runtime: this.runtime, + authority: this.authority, + request, + params: params as LegacyCheckParams, + signal + }) + } + if (request.method === 'orchestration.ask') { + return await handleLegacyAsk({ + runtime: this.runtime, + authority: this.authority, + request, + params: params as LegacyAskParams, + signal + }) + } + if (request.method === 'orchestration.reply') { + return await handleLegacyReply({ + runtime: this.runtime, + authority: this.authority, + request, + params: params as LegacyReplyParams + }) + } + return undefined + } +} + +function legacyCoordinatorMutationCallerFingerprint( + authority: LegacyCoordinatorAuthorityProof +): string { + return createHash('sha256') + .update( + ['legacy-coordinator-v1', authority.runId, authority.terminalHandle, authority.paneKey].join( + '\0' + ) + ) + .digest('hex') +} + +function stringValue(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined +} diff --git a/src/main/runtime/rpc/orchestration-legacy-coordinator-authority.ts b/src/main/runtime/rpc/orchestration-legacy-coordinator-authority.ts new file mode 100644 index 00000000000..a6223ba073a --- /dev/null +++ b/src/main/runtime/rpc/orchestration-legacy-coordinator-authority.ts @@ -0,0 +1,170 @@ +import type { OrcaRuntimeService, OrchestrationCompatibilityCallerAuthority } from '../orca-runtime' +import type { LegacyCompatibilityPrincipalRow } from '../orchestration/types' +import type { LegacyCoordinatorAuthorityProof, RpcRequest } from './core' +import { + resolveAttestedLegacyPrincipal, + verifyAttestedLegacyCandidate, + type LegacyPrincipalCandidate +} from './orchestration-legacy-authority' +import { + equivalentLegacyPaneKey, + legacyCoordinatorReadOnly +} from './orchestration-legacy-process-identity' + +export class LegacyCoordinatorAuthority { + constructor(private readonly runtime: OrcaRuntimeService) {} + + resolve( + request: RpcRequest, + requestedRunId?: string + ): LegacyCoordinatorAuthorityProof | undefined { + const db = this.runtime.getOrchestrationDb() + const adoption = db.getLegacyAdoption() + const requestedRun = requestedRunId ?? adoption?.adopted_run_id + if (!adoption || requestedRun !== adoption.adopted_run_id) { + return undefined + } + const candidate = db.resolveLegacyCoordinatorCandidate({ + runId: adoption.adopted_run_id, + terminalHandle: request.orchestrationCompatibilityEvidence?.terminalHandle, + paneKey: request.orchestrationCompatibilityEvidence?.paneKey + }) + if (!candidate) { + if (request.orchestrationCompatibilityEvidence) { + const caller = this.runtime.verifyOrchestrationCompatibilityCaller( + request.orchestrationCompatibilityEvidence + ) + const run = db.getRun(adoption.adopted_run_id) + if ( + caller && + run?.coordinator_handle === caller.terminalHandle && + run.coordinator_pane_key && + equivalentLegacyPaneKey(run.coordinator_pane_key, caller.paneKey) + ) { + return undefined + } + throw legacyCoordinatorReadOnly() + } + return undefined + } + const existing = db.getLegacyCoordinatorPrincipal(adoption.adopted_run_id) + const proofCandidate = existing + ? this.candidate(adoption.adopted_run_id, existing) + : this.candidate(adoption.adopted_run_id, candidate) + const attestation = verifyAttestedLegacyCandidate({ + runtime: this.runtime, + evidence: request.orchestrationCompatibilityEvidence, + candidate: proofCandidate + }) + const run = db.getRun(adoption.adopted_run_id) + if (!run || !this.bindingMatches(run, proofCandidate)) { + throw legacyCoordinatorReadOnly() + } + if (existing && !this.principalMatchesAttestation(existing, attestation)) { + throw legacyCoordinatorReadOnly() + } + const principal = existing + ? existing + : request.method === 'orchestration.runUse' + ? resolveAttestedLegacyPrincipal({ + runtime: this.runtime, + evidence: request.orchestrationCompatibilityEvidence, + candidate: proofCandidate, + authority: attestation + }) + : undefined + return { + runId: run.id, + principalId: principal?.id ?? null, + terminalHandle: proofCandidate.terminalHandle, + paneKey: proofCandidate.paneKey, + consumerGeneration: run.consumer_generation + } + } + + revalidate(request: RpcRequest, proof: LegacyCoordinatorAuthorityProof): string { + const db = this.runtime.getOrchestrationDb() + const adoption = db.getLegacyAdoption() + const run = db.getRun(proof.runId) + if ( + adoption?.adopted_run_id !== proof.runId || + !run || + run.consumer_generation !== proof.consumerGeneration + ) { + throw legacyCoordinatorReadOnly() + } + const candidate = this.candidate(proof.runId, proof) + const principal = db.getLegacyCoordinatorPrincipal(proof.runId) + if (proof.principalId) { + if ( + principal?.id !== proof.principalId || + principal.status !== 'committed' || + principal.terminal_handle !== proof.terminalHandle || + !equivalentLegacyPaneKey(principal.pane_key, proof.paneKey) + ) { + throw legacyCoordinatorReadOnly() + } + } else if ( + principal && + (principal.status !== 'committed' || + principal.terminal_handle !== proof.terminalHandle || + !equivalentLegacyPaneKey(principal.pane_key, proof.paneKey)) + ) { + throw legacyCoordinatorReadOnly() + } + const attestation = verifyAttestedLegacyCandidate({ + runtime: this.runtime, + evidence: request.orchestrationCompatibilityEvidence, + candidate + }) + if (principal && !this.principalMatchesAttestation(principal, attestation)) { + throw legacyCoordinatorReadOnly() + } + if (!this.bindingMatches(run, candidate)) { + throw legacyCoordinatorReadOnly() + } + return run.id + } + + private candidate( + runId: string, + identity: { + terminalHandle?: string + terminal_handle?: string + paneKey?: string + pane_key?: string + } + ): LegacyPrincipalCandidate { + return { + runId, + role: 'coordinator', + terminalHandle: identity.terminalHandle ?? (identity.terminal_handle as string), + paneKey: identity.paneKey ?? (identity.pane_key as string) + } + } + + private bindingMatches( + run: { coordinator_handle: string | null; coordinator_pane_key: string | null }, + candidate: LegacyPrincipalCandidate + ): boolean { + return ( + !run.coordinator_pane_key || + (run.coordinator_handle === candidate.terminalHandle && + equivalentLegacyPaneKey(run.coordinator_pane_key, candidate.paneKey)) + ) + } + + private principalMatchesAttestation( + principal: LegacyCompatibilityPrincipalRow, + attestation: OrchestrationCompatibilityCallerAuthority + ): boolean { + return ( + principal.status === 'committed' && + principal.host_scope === JSON.stringify(attestation.hostScope) && + principal.terminal_handle === attestation.terminalHandle && + equivalentLegacyPaneKey(principal.pane_key, attestation.paneKey) && + principal.launch_token_hash === attestation.launchTokenHash && + principal.process_incarnation === attestation.processIncarnation + ) + } +} diff --git a/src/main/runtime/rpc/orchestration-legacy-coordinator-race.test.ts b/src/main/runtime/rpc/orchestration-legacy-coordinator-race.test.ts new file mode 100644 index 00000000000..be5628df496 --- /dev/null +++ b/src/main/runtime/rpc/orchestration-legacy-coordinator-race.test.ts @@ -0,0 +1,664 @@ +import { createHash } from 'node:crypto' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { ORCHESTRATION_CONTRACT_VERSION } from '../../../shared/protocol-version' +import type { OrchestrationCompatibilityEvidence } from '../../../shared/orchestration-compatibility-evidence' +import Database from '../../sqlite/sync-database' +import { OrcaRuntimeService } from '../orca-runtime' +import { OrchestrationDb } from '../orchestration/db' +import type { RpcRequest, RpcResponse } from './core' +import { RpcDispatcher } from './dispatcher' +import { ORCHESTRATION_METHODS } from './methods/orchestration' + +const COORDINATOR_HANDLE = 'term_legacy_coord' +const COORDINATOR_PANE = 'tab_coord:44444444-4444-4444-8444-444444444444' +const WORKER_HANDLE = 'term_legacy_worker' +const WORKER_PANE = 'tab_worker:33333333-3333-4333-8333-333333333333' + +type Transport = 'dispatch' | 'websocket' + +type Harness = { + db: OrchestrationDb + dispatcher: RpcDispatcher + runtime: OrcaRuntimeService + adoptedRunId: string + taskId: string + dispatchId: string +} + +const tempDirs: string[] = [] +const databases: OrchestrationDb[] = [] + +afterEach(() => { + databases.splice(0).forEach((database) => database.close()) + tempDirs.splice(0).forEach((dir) => rmSync(dir, { recursive: true, force: true })) +}) + +function createHarness(): Harness { + const dir = mkdtempSync(join(tmpdir(), 'orca-legacy-coordinator-race-')) + tempDirs.push(dir) + const dbPath = join(dir, 'orchestration.db') + const before = new OrchestrationDb(dbPath) + const task = before.createTask({ + spec: 'legacy assignment', + createdByTerminalHandle: COORDINATOR_HANDLE + }) + const dispatch = before.createDispatchContext(task.id, WORKER_HANDLE, WORKER_PANE) + before.close() + + const raw = new Database(dbPath) + raw.exec(` + UPDATE dispatch_contexts SET process_incarnation = 'process-1'; + DROP INDEX IF EXISTS idx_messages_delivery_contract; + DROP TABLE legacy_mail_receipts; + DROP TABLE legacy_operation_receipts; + DROP TABLE legacy_compatibility_principals; + DROP TABLE legacy_adoptions; + `) + raw.pragma('user_version = 18') + raw.close() + + const db = new OrchestrationDb(dbPath) + databases.push(db) + const adoptedRunId = db.getLegacyAdoption()?.adopted_run_id as string + const runtime = new OrcaRuntimeService() + runtime.setOrchestrationDb(db) + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === COORDINATOR_HANDLE ? COORDINATOR_PANE : handle === WORKER_HANDLE ? WORKER_PANE : null + ) + vi.spyOn(runtime, 'verifyOrchestrationCompatibilityCaller').mockImplementation( + (compatibilityEvidence) => { + const coordinator = + compatibilityEvidence?.terminalHandle === COORDINATOR_HANDLE && + compatibilityEvidence.paneKey === COORDINATOR_PANE && + compatibilityEvidence.launchToken === 'coordinator-token' + const worker = + compatibilityEvidence?.terminalHandle === WORKER_HANDLE && + compatibilityEvidence.paneKey === WORKER_PANE && + compatibilityEvidence.launchToken === 'worker-token' + return coordinator || worker + ? { + hostScope: { kind: 'local', hostId: 'local' }, + terminalHandle: coordinator ? COORDINATOR_HANDLE : WORKER_HANDLE, + paneKey: coordinator ? COORDINATOR_PANE : WORKER_PANE, + processIncarnation: 'process-1', + launchTokenHash: createHash('sha256') + .update(coordinator ? 'coordinator-token' : 'worker-token') + .digest('hex') + } + : null + } + ) + return { + db, + dispatcher: new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS }), + runtime, + adoptedRunId, + taskId: task.id, + dispatchId: dispatch.id + } +} + +function evidence( + role: 'coordinator' | 'worker' = 'coordinator' +): OrchestrationCompatibilityEvidence { + return { + terminalHandle: role === 'coordinator' ? COORDINATOR_HANDLE : WORKER_HANDLE, + paneKey: role === 'coordinator' ? COORDINATOR_PANE : WORKER_PANE, + launchToken: role === 'coordinator' ? 'coordinator-token' : 'worker-token' + } +} + +function request( + method: string, + params: unknown, + invocationId: string, + callerEvidence = evidence() +): RpcRequest { + return { + id: `rpc_${invocationId}`, + authToken: 'caller-token', + method, + params, + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationRequestId: invocationId, + compatibilityInvocationId: invocationId, + orchestrationCompatibilityEvidence: callerEvidence + } +} + +async function invoke( + dispatcher: RpcDispatcher, + rpcRequest: RpcRequest, + transport: Transport +): Promise { + if (transport === 'dispatch') { + return await dispatcher.dispatch(rpcRequest) + } + const replies: string[] = [] + await dispatcher.dispatchStreaming(rpcRequest, (reply) => replies.push(reply)) + expect(replies).toHaveLength(1) + return JSON.parse(replies[0]) as RpcResponse +} + +function installTakeoverDuringMutationPreflight(harness: Harness): void { + const originalBegin = harness.db.beginMutationReceipt.bind(harness.db) + vi.spyOn(harness.db, 'beginMutationReceipt').mockImplementation((identity) => { + const begun = originalBegin(identity) + settleLegacyWorkerAndTakeOver(harness, 'settled during takeover') + return begun + }) +} + +function settleLegacyWorkerAndTakeOver(harness: Harness, result: string): void { + harness.db.settleWorkerReport({ + taskId: harness.taskId, + dispatchId: harness.dispatchId, + outcome: 'succeeded', + result + }) + harness.db.bindRun({ + runId: harness.adoptedRunId, + coordinatorHandle: 'term_current_coord', + coordinatorPaneKey: 'tab_current:55555555-5555-4555-8555-555555555555' + }) +} + +function mutationReceiptCount(db: OrchestrationDb): number { + const sqlite = (db as unknown as { db: Database.Database }).db + return ( + sqlite.prepare('SELECT COUNT(*) AS count FROM mutation_receipts').get() as { count: number } + ).count +} + +function messageCount(db: OrchestrationDb): number { + const sqlite = (db as unknown as { db: Database.Database }).db + return (sqlite.prepare('SELECT COUNT(*) AS count FROM messages').get() as { count: number }).count +} + +describe('legacy coordinator takeover races', () => { + it.each(['dispatch', 'websocket'] as const)( + '%s routes a pre-bind coordinator send to the retained worker mailbox', + async (transport) => { + const harness = createHarness() + + const response = await invoke( + harness.dispatcher, + request( + 'orchestration.send', + { from: COORDINATOR_HANDLE, to: WORKER_HANDLE, subject: 'continue safely' }, + `send-direct-${transport}` + ), + transport + ) + + expect(response).toMatchObject({ + ok: true, + result: { + message: { + run_id: harness.adoptedRunId, + to_handle: WORKER_HANDLE, + delivery_contract: 'legacy_direct' + } + } + }) + const workerCheck = await harness.dispatcher.dispatch( + request( + 'orchestration.check', + { terminal: WORKER_HANDLE }, + `worker-check-${transport}`, + evidence('worker') + ) + ) + expect(workerCheck).toMatchObject({ + ok: true, + result: { + messages: [ + { + subject: 'continue safely', + run_id: harness.adoptedRunId, + delivery_contract: 'legacy_direct' + } + ], + legacyCompatibility: { ackMessageIds: [expect.any(String)] } + } + }) + expect( + harness.db.getInbox(100).some((message) => message.run_id === 'run_legacy_local') + ).toBe(false) + } + ) + + it('replays coordinator mutations after transport authentication rotates', async () => { + const harness = createHarness() + const mutation = request( + 'orchestration.send', + { from: COORDINATOR_HANDLE, to: WORKER_HANDLE, subject: 'send once' }, + 'coordinator-restart-send' + ) + mutation.authToken = 'before-restart' + + const first = await harness.dispatcher.dispatch(mutation) + const restartedDispatcher = new RpcDispatcher({ + runtime: harness.runtime, + methods: ORCHESTRATION_METHODS + }) + const replay = await restartedDispatcher.dispatch({ + ...mutation, + id: 'rpc_coordinator-restart-send-retry', + authToken: 'after-restart' + }) + + expect(first).toMatchObject({ + ok: true, + result: { + message: { subject: 'send once' }, + mutation: { requestId: 'coordinator-restart-send', replayed: false } + } + }) + expect(replay).toMatchObject({ + ok: true, + result: { + message: { id: (first as { result: { message: { id: string } } }).result.message.id }, + mutation: { requestId: 'coordinator-restart-send', replayed: true } + } + }) + expect(messageCount(harness.db)).toBe(1) + expect(mutationReceiptCount(harness.db)).toBe(1) + }) + + it('routes an exact legacy Dispatch recipient to compatibility delivery', async () => { + const harness = createHarness() + + const response = await harness.dispatcher.dispatch( + request( + 'orchestration.send', + { + from: COORDINATOR_HANDLE, + to: `dispatch:${harness.dispatchId}`, + subject: 'dispatch guidance' + }, + 'send-dispatch' + ) + ) + + expect(response).toMatchObject({ + ok: true, + result: { + message: { + run_id: harness.adoptedRunId, + to_handle: `dispatch:${harness.dispatchId}`, + delivery_contract: 'legacy_direct' + } + } + }) + const workerCheck = await harness.dispatcher.dispatch( + request( + 'orchestration.check', + { terminal: WORKER_HANDLE }, + 'worker-check-dispatch', + evidence('worker') + ) + ) + expect(workerCheck).toMatchObject({ + ok: true, + result: { messages: [{ subject: 'dispatch guidance' }] } + }) + }) + + it('partitions a coordinator group send by legacy recipient contract', async () => { + const harness = createHarness() + vi.spyOn(harness.runtime, 'listTerminals').mockResolvedValue({ + terminals: [ + { handle: COORDINATOR_HANDLE }, + { handle: WORKER_HANDLE }, + { handle: 'term_current_worker' } + ], + totalCount: 3, + truncated: false + } as never) + + const response = await harness.dispatcher.dispatch( + request( + 'orchestration.send', + { from: COORDINATOR_HANDLE, to: '@all', subject: 'group guidance' }, + 'send-group' + ) + ) + + expect(response).toMatchObject({ ok: true, result: { recipients: 2 } }) + const messages = harness.db + .getInbox(100) + .filter((message) => message.subject === 'group guidance') + expect(messages).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + run_id: harness.adoptedRunId, + to_handle: WORKER_HANDLE, + delivery_contract: 'legacy_direct' + }), + expect.objectContaining({ + run_id: harness.adoptedRunId, + to_handle: 'term_current_worker', + delivery_contract: 'current_delivery' + }) + ]) + ) + }) + + it.each(['dispatch', 'websocket'] as const)( + '%s rejects a coordinator send after takeover during durable preflight', + async (transport) => { + const harness = createHarness() + installTakeoverDuringMutationPreflight(harness) + const before = messageCount(harness.db) + + const response = await invoke( + harness.dispatcher, + request( + 'orchestration.send', + { from: COORDINATOR_HANDLE, to: WORKER_HANDLE, subject: 'must not send' }, + `send-takeover-${transport}` + ), + transport + ) + + expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } }) + expect(messageCount(harness.db)).toBe(before) + expect(mutationReceiptCount(harness.db)).toBe(0) + } + ) + + it('revalidates a group send after asynchronous terminal discovery', async () => { + const harness = createHarness() + let resolveTerminals: + | ((value: { + terminals: { handle: string }[] + totalCount: number + truncated: boolean + }) => void) + | undefined + let signalListingStarted: (() => void) | undefined + const listingStarted = new Promise((resolve) => { + signalListingStarted = resolve + }) + vi.spyOn(harness.runtime, 'listTerminals').mockImplementation( + () => + new Promise((resolve) => { + resolveTerminals = resolve as typeof resolveTerminals + signalListingStarted?.() + }) as never + ) + const before = messageCount(harness.db) + const pending = harness.dispatcher.dispatch( + request( + 'orchestration.send', + { from: COORDINATOR_HANDLE, to: '@all', subject: 'must remain unsent' }, + 'send-group-takeover' + ) + ) + await listingStarted + settleLegacyWorkerAndTakeOver(harness, 'settled during terminal discovery') + resolveTerminals?.({ + terminals: [{ handle: COORDINATOR_HANDLE }, { handle: WORKER_HANDLE }], + totalCount: 2, + truncated: false + }) + + await expect(pending).resolves.toMatchObject({ + ok: false, + error: { code: 'legacy_read_only' } + }) + expect(messageCount(harness.db)).toBe(before) + expect(mutationReceiptCount(harness.db)).toBe(0) + }) + + it('reports a committed ACK when coordinator takeover interrupts its wait', async () => { + const harness = createHarness() + const incoming = harness.db.insertMessage({ + runId: harness.adoptedRunId, + from: WORKER_HANDLE, + to: `run:${harness.adoptedRunId}`, + subject: 'ready to acknowledge' + }) + const first = await harness.dispatcher.dispatch( + request( + 'orchestration.check', + { terminal: COORDINATOR_HANDLE, run: harness.adoptedRunId }, + 'check-delivery-before-takeover' + ) + ) + expect(first).toMatchObject({ + ok: true, + result: { deliveryId: expect.any(String), messages: [{ id: incoming.id }] } + }) + const deliveryId = (first as { result: { deliveryId: string } }).result.deliveryId + let resolveWait: (() => void) | undefined + let signalWaitStarted: (() => void) | undefined + const waitStarted = new Promise((resolve) => { + signalWaitStarted = resolve + }) + vi.spyOn(harness.runtime, 'waitForMessage').mockImplementation( + () => + new Promise((resolve) => { + resolveWait = () => resolve('notified') + signalWaitStarted?.() + }) + ) + const pending = harness.dispatcher.dispatch( + request( + 'orchestration.check', + { + terminal: COORDINATOR_HANDLE, + run: harness.adoptedRunId, + ack: deliveryId, + wait: true, + timeoutMs: 10_000 + }, + 'check-ack-takeover' + ) + ) + await waitStarted + expect(harness.db.getMessageById(incoming.id)?.read).toBe(1) + settleLegacyWorkerAndTakeOver(harness, 'settled during acknowledged wait') + resolveWait?.() + + await expect(pending).resolves.toMatchObject({ + ok: true, + result: { + acknowledged: deliveryId, + count: 0, + waitInterrupted: 'consumer_fenced', + mutation: { requestId: 'check-ack-takeover', replayed: false } + } + }) + const sqlite = (harness.db as unknown as { db: Database.Database }).db + expect(sqlite.prepare('SELECT state FROM mutation_receipts').get()).toEqual({ + state: 'completed' + }) + }) + + it('keeps concurrent duplicates on the live ACK-and-wait result', async () => { + const harness = createHarness() + harness.db.insertMessage({ + runId: harness.adoptedRunId, + from: WORKER_HANDLE, + to: `run:${harness.adoptedRunId}`, + subject: 'first delivery' + }) + const firstCheck = await harness.dispatcher.dispatch( + request( + 'orchestration.check', + { terminal: COORDINATOR_HANDLE, run: harness.adoptedRunId }, + 'check-first-delivery' + ) + ) + const deliveryId = (firstCheck as { result: { deliveryId: string } }).result.deliveryId + let resolveWait: (() => void) | undefined + let signalWaitStarted: (() => void) | undefined + const waitStarted = new Promise((resolve) => { + signalWaitStarted = resolve + }) + vi.spyOn(harness.runtime, 'waitForMessage').mockImplementation( + () => + new Promise((resolve) => { + resolveWait = () => resolve('notified') + signalWaitStarted?.() + }) + ) + const ackRequest = request( + 'orchestration.check', + { + terminal: COORDINATOR_HANDLE, + run: harness.adoptedRunId, + ack: deliveryId, + wait: true, + timeoutMs: 10_000 + }, + 'check-ack-concurrent' + ) + const original = harness.dispatcher.dispatch(ackRequest) + await waitStarted + const secondLiveDispatcher = new RpcDispatcher({ + runtime: harness.runtime, + methods: ORCHESTRATION_METHODS + }) + const duplicate = secondLiveDispatcher.dispatch({ + ...ackRequest, + id: 'rpc_check-ack-concurrent-duplicate' + }) + let duplicateSettled = false + void duplicate.then(() => { + duplicateSettled = true + }) + await Promise.resolve() + expect(duplicateSettled).toBe(false) + + const arrived = harness.db.insertMessage({ + runId: harness.adoptedRunId, + from: WORKER_HANDLE, + to: `run:${harness.adoptedRunId}`, + subject: 'arrived while waiting' + }) + resolveWait?.() + const [originalResult, duplicateResult] = await Promise.all([original, duplicate]) + expect(originalResult).toMatchObject({ + ok: true, + result: { + messages: [{ id: arrived.id }], + mutation: { requestId: 'check-ack-concurrent', replayed: false } + } + }) + expect(duplicateResult).toMatchObject({ + ok: true, + result: { + messages: [{ id: arrived.id }], + mutation: { requestId: 'check-ack-concurrent', replayed: true } + } + }) + }) + + it.each(['dispatch', 'websocket'] as const)( + '%s rejects run-use after takeover during durable preflight', + async (transport) => { + const harness = createHarness() + installTakeoverDuringMutationPreflight(harness) + + const response = await invoke( + harness.dispatcher, + request( + 'orchestration.runUse', + { id: harness.adoptedRunId, from: COORDINATOR_HANDLE }, + `run-use-takeover-${transport}` + ), + transport + ) + + expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } }) + expect(harness.db.getRun(harness.adoptedRunId)).toMatchObject({ + coordinator_handle: 'term_current_coord', + consumer_generation: 1 + }) + expect(mutationReceiptCount(harness.db)).toBe(0) + expect(harness.db.getLegacyCoordinatorPrincipal(harness.adoptedRunId)?.status).toBe('revoked') + } + ) + + it.each(['dispatch', 'websocket'] as const)( + '%s rejects a mutation after takeover during durable preflight', + async (transport) => { + const harness = createHarness() + const target = harness.db.createTask({ + spec: 'must remain ready', + runId: harness.adoptedRunId + }) + installTakeoverDuringMutationPreflight(harness) + + const response = await invoke( + harness.dispatcher, + request( + 'orchestration.taskUpdate', + { + id: target.id, + status: 'completed', + callerTerminalHandle: COORDINATOR_HANDLE + }, + `task-update-takeover-${transport}` + ), + transport + ) + + expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } }) + expect(harness.db.getTask(target.id)?.status).toBe('ready') + expect(mutationReceiptCount(harness.db)).toBe(0) + } + ) + + it('revalidates after asynchronous agent detection', async () => { + const harness = createHarness() + const target = harness.db.createTask({ + spec: 'must not dispatch after takeover', + runId: harness.adoptedRunId + }) + const targetHandle = 'term_current_worker' + const targetPane = 'tab_current_worker:77777777-7777-4777-8777-777777777777' + vi.spyOn(harness.runtime, 'getTerminalPaneKey').mockImplementation((handle: string) => + handle === COORDINATOR_HANDLE ? COORDINATOR_PANE : handle === targetHandle ? targetPane : null + ) + vi.spyOn(harness.runtime, 'getTerminalProcessIncarnation').mockReturnValue('current-process') + let resolveDetection: ((detected: boolean) => void) | undefined + let signalDetectionStarted: (() => void) | undefined + const detectionStarted = new Promise((resolve) => { + signalDetectionStarted = resolve + }) + vi.spyOn(harness.runtime, 'isTerminalRunningAgent').mockImplementation( + () => + new Promise((resolve) => { + resolveDetection = resolve + signalDetectionStarted?.() + }) + ) + const sendPrompt = vi + .spyOn(harness.runtime, 'sendTerminalAgentPrompt') + .mockResolvedValue({ handle: targetHandle, accepted: true, bytesWritten: 1 }) + + const pending = harness.dispatcher.dispatch( + request( + 'orchestration.dispatch', + { task: target.id, to: targetHandle, from: COORDINATOR_HANDLE, inject: true }, + 'dispatch-detection-takeover' + ) + ) + await detectionStarted + settleLegacyWorkerAndTakeOver(harness, 'settled during detection') + resolveDetection?.(true) + const response = await pending + + expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } }) + expect(harness.db.getTask(target.id)?.status).toBe('ready') + expect(harness.db.getDispatchContext(target.id)).toBeUndefined() + expect(sendPrompt).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/runtime/rpc/orchestration-legacy-lifecycle.ts b/src/main/runtime/rpc/orchestration-legacy-lifecycle.ts new file mode 100644 index 00000000000..9ffe17cca6e --- /dev/null +++ b/src/main/runtime/rpc/orchestration-legacy-lifecycle.ts @@ -0,0 +1,114 @@ +import type { RpcRequest } from './core' +import type { OrcaRuntimeService } from '../orca-runtime' +import type { MessageType } from '../orchestration/db' +import { OrchestrationError } from '../orchestration/orchestration-error' +import type { LegacyCompatibilityAuthority } from './orchestration-legacy-authority' +import { + inferLegacyWorkerOutcome, + isWorkerOutcome, + operationIdentity, + parseLegacyPayload, + stringValue, + type LegacySendParams +} from './orchestration-legacy-operation' + +export async function handleLegacyLifecycleSend(args: { + runtime: OrcaRuntimeService + authority: LegacyCompatibilityAuthority + request: RpcRequest + params: LegacySendParams +}): Promise { + const { runtime, authority, request, params } = args + if (!['heartbeat', 'worker_done', 'escalation'].includes(params.type ?? '')) { + return undefined + } + const payload = parseLegacyPayload(params.payload) + const dispatch = authority.resolveWorkerDispatch(request, { + terminalHandle: params.from, + dispatchId: stringValue(payload.dispatchId), + taskId: stringValue(payload.taskId) + }) + if (!dispatch) { + if (params.type === 'worker_done' && !isWorkerOutcome(payload.outcome)) { + throw new OrchestrationError( + 'invalid_argument', + 'worker_done requires outcome=succeeded|failed for a current Dispatch.' + ) + } + return undefined + } + if (!params.to) { + throw new OrchestrationError('invalid_argument', 'Legacy lifecycle mail requires --to.') + } + const db = runtime.getOrchestrationDb() + if (!db.isLegacyCoordinatorHandle(dispatch.run_id, params.to)) { + throw new OrchestrationError( + 'request_mismatch', + `Terminal ${params.to} is not a retained coordinator for this legacy Dispatch.` + ) + } + + if ( + params.type === 'worker_done' && + Object.hasOwn(payload, 'outcome') && + !isWorkerOutcome(payload.outcome) + ) { + throw new OrchestrationError( + 'invalid_argument', + 'Legacy worker_done outcome must be succeeded or failed when provided.' + ) + } + const principal = authority.attestWorker(request, dispatch) + const operation = operationIdentity(request, params.type as string, { ...params, payload }) + const inferredOutcome = + params.type === 'worker_done' + ? isWorkerOutcome(payload.outcome) + ? payload.outcome + : inferLegacyWorkerOutcome(params.subject) + : undefined + const existingId = + params.type === 'worker_done' + ? db.findLegacyWorkerCompletion({ + principalId: principal.id, + taskId: dispatch.task_id, + recipientHandle: params.to, + subject: params.subject, + body: params.body ?? '', + payload: params.payload ?? null + })?.id + : undefined + const committed = db.commitLegacyLifecycleOperation({ + principalId: principal.id, + operationKey: operation.key, + method: request.method, + payloadHash: operation.payloadHash, + message: { + existingId, + to: params.to, + subject: params.subject, + body: params.body, + type: params.type as MessageType, + priority: params.priority, + payload: params.payload + }, + lifecycle: + params.type === 'heartbeat' + ? { kind: 'heartbeat', at: new Date().toISOString() } + : params.type === 'worker_done' + ? { + kind: 'worker_report', + taskId: dispatch.task_id, + outcome: inferredOutcome as 'succeeded' | 'failed', + result: params.body || params.subject + } + : { kind: 'message_only' } + }) + if (!committed.duplicate) { + runtime.notifyMessageArrived(committed.message.to_handle, committed.message.type) + } + return { + message: committed.message, + ...(committed.settlement ? { lifecycle: committed.settlement } : {}), + legacyCompatibility: { replayed: committed.duplicate } + } +} diff --git a/src/main/runtime/rpc/orchestration-legacy-mail.ts b/src/main/runtime/rpc/orchestration-legacy-mail.ts new file mode 100644 index 00000000000..e41e0c11509 --- /dev/null +++ b/src/main/runtime/rpc/orchestration-legacy-mail.ts @@ -0,0 +1,181 @@ +import type { RpcRequest } from './core' +import type { OrcaRuntimeService } from '../orca-runtime' +import { OrchestrationError } from '../orchestration/orchestration-error' +import { formatMessageBanner } from '../orchestration/formatter' +import { ORCHESTRATION_MESSAGE_WAIT_DEFAULT_TIMEOUT_MS } from '../../../shared/orchestration-message-wait-timeout' +import type { LegacyCompatibilityAuthority } from './orchestration-legacy-authority' +import { + operationIdentity, + parseLegacyMailAck, + parseLegacyQuestionAck, + parseLegacyTypes, + supportedLegacyHints, + type LegacyCheckParams, + type LegacyReplyParams +} from './orchestration-legacy-operation' + +export async function handleLegacyCheck(args: { + runtime: OrcaRuntimeService + authority: LegacyCompatibilityAuthority + request: RpcRequest + params: LegacyCheckParams + signal?: AbortSignal +}): Promise { + const { runtime, authority, request, params, signal } = args + if (params.run) { + return undefined + } + const typeFilter = parseLegacyTypes(params.types) + const principal = authority.resolveCheckPrincipal(request, params.terminal) + if (!principal) { + return undefined + } + const db = runtime.getOrchestrationDb() + if (params.compatibilityQuestionAck) { + const answer = parseLegacyQuestionAck(params.compatibilityQuestionAck) + const acknowledged = db.acknowledgeLegacyQuestionAnswer({ + principalId: principal.id, + ...answer + }) + return { + messages: [], + count: 0, + acknowledged: [answer.answerMessageId], + duplicate: acknowledged.duplicate, + legacyCompatibility: { acknowledged: true } + } + } + if (params.compatibilityAck) { + const ack = parseLegacyMailAck(params.compatibilityAck) + const acknowledged = db.acknowledgeLegacyMail({ + principalId: principal.id, + messageIds: ack.messageIds, + types: ack.types + }) + return { + messages: [], + count: 0, + acknowledged: ack.messageIds, + duplicate: acknowledged.duplicate, + legacyCompatibility: { acknowledged: true } + } + } + + const history = params.all || (params.unread === false && !params.peek) + const readOnly = Boolean(params.peek || history) + const read = () => + history + ? db.getLegacyMailHistory({ principalId: principal.id, types: typeFilter }) + : db.getLegacyMailPage({ principalId: principal.id, types: typeFilter }) + let page = read() + const deadline = + Date.now() + Math.max(params.timeoutMs ?? ORCHESTRATION_MESSAGE_WAIT_DEFAULT_TIMEOUT_MS, 0) + while (page.messages.length === 0 && params.wait && !signal?.aborted && Date.now() < deadline) { + if (principal.role === 'coordinator' && db.hasPendingCurrentDelivery(principal.run_id)) { + break + } + await runtime.waitForMessage( + principal.role === 'worker' + ? `dispatch:${principal.dispatch_id as string}` + : principal.terminal_handle, + { + typeFilter, + timeoutMs: Math.min(1_000, Math.max(deadline - Date.now(), 1)), + signal + } + ) + page = read() + } + + const consuming = !readOnly + const messageIds = consuming ? page.messages.map((message) => message.id) : [] + const formattingAuthority = readOnly + ? 'legacy_read_only' + : page.recovery + ? 'legacy_recovery_replay' + : 'legacy_compatibility' + const formatted = + params.format || params.inject + ? page.messages + .map((message) => + formatMessageBanner(message, { + authority: formattingAuthority, + supportedActionHints: readOnly + ? [] + : supportedLegacyHints(message, principal, params.compatibilityCliCommand ?? 'orca') + }) + ) + .join('\n\n') + : undefined + const currentDelivery = + principal.role === 'coordinator' && db.hasPendingCurrentDelivery(principal.run_id) + ? { + runId: principal.run_id, + checkCommand: `${params.compatibilityCliCommand ?? 'orca'} orchestration check --run ${principal.run_id}`, + ackCommand: `${params.compatibilityCliCommand ?? 'orca'} orchestration check --run ${principal.run_id} --ack ` + } + : undefined + return { + runId: principal.run_id, + dispatchId: principal.dispatch_id, + messages: page.messages, + count: page.messages.length, + formatted, + timedOut: Boolean(params.wait && page.messages.length === 0 && !signal?.aborted), + cancelled: signal?.aborted === true, + connectionLost: signal?.aborted === true, + legacyCompatibility: { + recovery: page.recovery, + readOnly, + ackMessageIds: messageIds, + currentDelivery + } + } +} + +export async function handleLegacyReply(args: { + runtime: OrcaRuntimeService + authority: LegacyCompatibilityAuthority + request: RpcRequest + params: LegacyReplyParams +}): Promise { + const { runtime, authority, request, params } = args + const db = runtime.getOrchestrationDb() + const original = db.getMessageById(params.id) + if (!original || original.delivery_contract !== 'legacy_direct') { + return undefined + } + const principal = authority.attestCoordinator(request, original.run_id) + if (!principal) { + throw legacyCoordinatorReadOnly() + } + const operation = operationIdentity(request, 'reply', { + questionId: params.id, + body: params.body + }) + const committed = db.commitLegacyReplyOperation({ + principalId: principal.id, + operationKey: operation.key, + method: request.method, + payloadHash: operation.payloadHash, + questionId: params.id, + body: params.body + }) + if (!committed.duplicate) { + runtime.notifyMessageArrived(committed.question.asker_handle, 'status') + } + return { + message: committed.message, + question: committed.question, + duplicate: committed.duplicate, + legacyCompatibility: { replayed: committed.duplicate } + } +} + +function legacyCoordinatorReadOnly(): OrchestrationError { + return new OrchestrationError( + 'legacy_read_only', + 'This retained legacy coordinator could not prove its original process identity. No effects were applied.', + { effectsApplied: false } + ) +} diff --git a/src/main/runtime/rpc/orchestration-legacy-operation.ts b/src/main/runtime/rpc/orchestration-legacy-operation.ts new file mode 100644 index 00000000000..85efe72979a --- /dev/null +++ b/src/main/runtime/rpc/orchestration-legacy-operation.ts @@ -0,0 +1,209 @@ +import { createHash, randomUUID } from 'node:crypto' +import type { RpcRequest } from './core' +import type { MessageType } from '../orchestration/db' +import { MESSAGE_TYPES } from '../orchestration/types' +import type { LegacyCompatibilityPrincipalRow, MessageRow } from '../orchestration/types' +import { OrchestrationError } from '../orchestration/orchestration-error' + +const MESSAGE_TYPE_SET = new Set(MESSAGE_TYPES) + +export type LegacySendParams = { + from?: string + to?: string + subject: string + body?: string + type?: string + priority?: 'normal' | 'high' | 'urgent' + payload?: string +} + +export type LegacyCheckParams = { + terminal?: string + run?: string + unread?: boolean + peek?: boolean + all?: boolean + types?: string + format?: boolean + inject?: boolean + wait?: boolean + timeoutMs?: number + compatibilityAck?: string + compatibilityQuestionAck?: string + compatibilityCliCommand?: 'orca' | 'orca-ide' | 'orca-dev' +} + +export type LegacyAskParams = { + from?: string + to?: string + run?: string + question?: string + resume?: string + options?: string + timeoutMs?: number + compatibilityCliCommand?: 'orca' | 'orca-ide' | 'orca-dev' + compatibilityWindowsCommand?: 'orca' | 'orca-ide' +} + +export type LegacyReplyParams = { + id: string + body: string + from?: string + run?: string +} + +export function operationIdentity( + request: RpcRequest, + method: string, + payload: unknown +): { key: string; payloadHash: string } { + const canonical = JSON.stringify(canonicalize(payload)) + const payloadHash = createHash('sha256').update(canonical).digest('hex') + const semantic = + method === 'worker_done' || method === 'reply' || method === 'ask' + ? `${method}:${payloadHash}` + : `${method}:${randomUUID()}` + return { + key: request.compatibilityInvocationId + ? `invocation:${request.compatibilityInvocationId}` + : `semantic:${semantic}`, + payloadHash + } +} + +export function parseLegacyPayload(raw: string | undefined): Record { + if (!raw) { + return {} + } + try { + const parsed: unknown = JSON.parse(raw) + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { + return parsed as Record + } + } catch { + // Fall through to one stable validation error. + } + throw new OrchestrationError('invalid_argument', 'Message payload must be valid JSON.') +} + +export function parseLegacyMailAck(raw: string): { messageIds: string[]; types?: MessageType[] } { + try { + const parsed: unknown = JSON.parse(raw) + const messageIds = Array.isArray(parsed) + ? parsed + : parsed && typeof parsed === 'object' && !Array.isArray(parsed) + ? (parsed as { messageIds?: unknown }).messageIds + : undefined + const types = + parsed && typeof parsed === 'object' && !Array.isArray(parsed) + ? (parsed as { types?: unknown }).types + : undefined + if ( + Array.isArray(messageIds) && + messageIds.length > 0 && + messageIds.every((value) => typeof value === 'string' && value.length > 0) && + (types === undefined || + (Array.isArray(types) && + types.every((value) => typeof value === 'string' && MESSAGE_TYPE_SET.has(value)))) + ) { + return { + messageIds, + types: types && types.length > 0 ? (types as MessageType[]) : undefined + } + } + } catch { + // Fall through to one stable validation error. + } + throw new OrchestrationError('invalid_argument', 'Invalid compatibility acknowledgment.') +} + +export function parseLegacyQuestionAck(raw: string): { + questionId: string + answerMessageId: string +} { + try { + const parsed: unknown = JSON.parse(raw) + if ( + parsed && + typeof parsed === 'object' && + !Array.isArray(parsed) && + typeof (parsed as { questionId?: unknown }).questionId === 'string' && + typeof (parsed as { answerMessageId?: unknown }).answerMessageId === 'string' + ) { + return parsed as { questionId: string; answerMessageId: string } + } + } catch { + // Fall through to one stable validation error. + } + throw new OrchestrationError('invalid_argument', 'Invalid legacy question acknowledgment.') +} + +export function parseLegacyTypes(raw: string | undefined): MessageType[] | undefined { + const values = raw + ?.split(',') + .map((value) => value.trim()) + .filter(Boolean) + const invalidValues = values?.filter((value) => !MESSAGE_TYPE_SET.has(value)) + if (invalidValues && invalidValues.length > 0) { + throw new OrchestrationError('invalid_argument', `Invalid --types: ${invalidValues.join(',')}`) + } + return values && values.length > 0 ? (values as MessageType[]) : undefined +} + +export function parseLegacyOptions(raw: string | undefined): string[] { + return ( + raw + ?.split(',') + .map((value) => value.trim()) + .filter(Boolean) ?? [] + ) +} + +export function supportedLegacyHints( + message: MessageRow, + principal: LegacyCompatibilityPrincipalRow, + cliCommand: 'orca' | 'orca-ide' | 'orca-dev' +): string[] { + if ( + principal.role !== 'coordinator' || + (message.type !== 'question' && message.type !== 'decision_gate') + ) { + return [] + } + return [ + `${cliCommand} orchestration reply --id ${message.id} --from ${principal.terminal_handle} --body "..."` + ] +} + +export function inferLegacyWorkerOutcome(subject: string): 'succeeded' | 'failed' { + const normalized = subject.trim() + return normalized === 'Failed' || normalized.startsWith('Failed:') ? 'failed' : 'succeeded' +} + +export function isWorkerOutcome(value: unknown): value is 'succeeded' | 'failed' { + return value === 'succeeded' || value === 'failed' +} + +export function stringValue(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined +} + +export function normalizeLegacyText(value: string): string { + return value.replace(/\r\n/g, '\n').trim() +} + +function canonicalize(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(canonicalize) + } + if (!value || typeof value !== 'object') { + return value + } + const source = value as Record + return Object.fromEntries( + Object.keys(source) + .sort() + .filter((key) => source[key] !== undefined) + .map((key) => [key, canonicalize(source[key])]) + ) +} diff --git a/src/main/runtime/rpc/orchestration-legacy-process-identity.ts b/src/main/runtime/rpc/orchestration-legacy-process-identity.ts new file mode 100644 index 00000000000..35054c61279 --- /dev/null +++ b/src/main/runtime/rpc/orchestration-legacy-process-identity.ts @@ -0,0 +1,33 @@ +import { parsePaneKey } from '../../../shared/stable-pane-id' +import { OrchestrationError } from '../orchestration/orchestration-error' + +export function equivalentLegacyPaneKey( + a: string | null | undefined, + b: string | null | undefined +): boolean { + if (!a || !b) { + return false + } + if (a === b) { + return true + } + const aLeaf = parsePaneKey(a)?.leafId + const bLeaf = parsePaneKey(b)?.leafId + return Boolean(aLeaf && bLeaf && aLeaf === bLeaf) +} + +export function legacyReadOnlyError(): OrchestrationError { + return new OrchestrationError( + 'legacy_read_only', + 'This retained legacy assignment could not prove authority from its original live process. No effects were applied.', + { effectsApplied: false } + ) +} + +export function legacyCoordinatorReadOnly(): OrchestrationError { + return new OrchestrationError( + 'legacy_read_only', + 'This retained legacy coordinator could not prove its original process identity. No effects were applied.', + { effectsApplied: false } + ) +} diff --git a/src/main/runtime/rpc/orchestration-legacy-question-takeover.test.ts b/src/main/runtime/rpc/orchestration-legacy-question-takeover.test.ts new file mode 100644 index 00000000000..37fb70d116f --- /dev/null +++ b/src/main/runtime/rpc/orchestration-legacy-question-takeover.test.ts @@ -0,0 +1,364 @@ +import { createHash } from 'node:crypto' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { OrchestrationCompatibilityEvidence } from '../../../shared/orchestration-compatibility-evidence' +import { ORCHESTRATION_CONTRACT_VERSION } from '../../../shared/protocol-version' +import Database from '../../sqlite/sync-database' +import { OrchestrationDb } from '../orchestration/db' +import { OrcaRuntimeService } from '../orca-runtime' +import type { RpcRequest } from './core' +import { RpcDispatcher } from './dispatcher' +import { ORCHESTRATION_METHODS } from './methods/orchestration' + +const COORDINATOR_HANDLE = 'term_legacy_coord' +const CURRENT_COORDINATOR_HANDLE = 'term_current_coord' +const CURRENT_COORDINATOR_PANE = 'tab_current:55555555-5555-4555-8555-555555555555' +const WORKER_HANDLE = 'term_legacy_worker' +const WORKER_PANE = 'tab_worker:33333333-3333-4333-8333-333333333333' + +const tempDirs: string[] = [] +const databases: OrchestrationDb[] = [] + +afterEach(() => { + databases.splice(0).forEach((database) => database.close()) + tempDirs.splice(0).forEach((dir) => rmSync(dir, { recursive: true, force: true })) +}) + +function createHarness(options?: { seedCutoverQuestion?: boolean; seedCutoverAnswer?: boolean }): { + db: OrchestrationDb + dispatcher: RpcDispatcher + adoptedRunId: string + dispatchId: string + cutoverQuestionId?: string + cutoverAnswerId?: string +} { + const dir = mkdtempSync(join(tmpdir(), 'orca-legacy-question-takeover-')) + tempDirs.push(dir) + const dbPath = join(dir, 'orchestration.db') + const before = new OrchestrationDb(dbPath) + const task = before.createTask({ + spec: 'legacy assignment', + createdByTerminalHandle: COORDINATOR_HANDLE + }) + const dispatch = before.createDispatchContext(task.id, WORKER_HANDLE, WORKER_PANE) + const cutoverQuestion = options?.seedCutoverQuestion + ? before.insertMessage({ + from: WORKER_HANDLE, + to: COORDINATOR_HANDLE, + subject: 'Question', + body: 'Continue after update?', + type: 'decision_gate', + payload: JSON.stringify({ + question: 'Continue after update?', + options: ['yes', 'no'] + }) + }) + : undefined + const cutoverAnswer = + cutoverQuestion && options?.seedCutoverAnswer + ? before.insertMessage({ + from: COORDINATOR_HANDLE, + to: WORKER_HANDLE, + subject: 'Re: Question', + body: 'Continue.', + threadId: cutoverQuestion.id + }) + : undefined + before.close() + + const raw = new Database(dbPath) + raw.exec(` + UPDATE dispatch_contexts SET process_incarnation = 'process-1'; + DROP INDEX IF EXISTS idx_messages_delivery_contract; + DROP TABLE legacy_mail_receipts; + DROP TABLE legacy_operation_receipts; + DROP TABLE legacy_compatibility_principals; + DROP TABLE legacy_adoptions; + `) + raw.pragma('user_version = 18') + raw.close() + + const db = new OrchestrationDb(dbPath) + databases.push(db) + const runtime = new OrcaRuntimeService() + runtime.setOrchestrationDb(db) + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === WORKER_HANDLE + ? WORKER_PANE + : handle === CURRENT_COORDINATOR_HANDLE + ? CURRENT_COORDINATOR_PANE + : null + ) + vi.spyOn(runtime, 'verifyOrchestrationCompatibilityCaller').mockImplementation((evidence) => { + const worker = + evidence?.terminalHandle === WORKER_HANDLE && + evidence.paneKey === WORKER_PANE && + evidence.launchToken === 'worker-token' + const coordinator = + evidence?.terminalHandle === CURRENT_COORDINATOR_HANDLE && + evidence.paneKey === CURRENT_COORDINATOR_PANE && + evidence.launchToken === 'current-coordinator-token' + const launchToken = worker + ? 'worker-token' + : coordinator + ? 'current-coordinator-token' + : undefined + return launchToken + ? { + hostScope: { kind: 'local', hostId: 'local' }, + terminalHandle: evidence?.terminalHandle as string, + paneKey: evidence?.paneKey as string, + processIncarnation: 'process-1', + launchTokenHash: createHash('sha256').update(launchToken).digest('hex') + } + : null + }) + return { + db, + dispatcher: new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS }), + adoptedRunId: db.getLegacyAdoption()?.adopted_run_id as string, + dispatchId: dispatch.id, + ...(cutoverQuestion ? { cutoverQuestionId: cutoverQuestion.id } : {}), + ...(cutoverAnswer ? { cutoverAnswerId: cutoverAnswer.id } : {}) + } +} + +function request( + method: string, + params: unknown, + invocationId: string, + evidence?: OrchestrationCompatibilityEvidence +): RpcRequest { + return { + id: `rpc_${invocationId}`, + authToken: 'caller-token', + method, + params, + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationRequestId: invocationId, + compatibilityInvocationId: invocationId, + orchestrationCompatibilityEvidence: evidence + } +} + +const workerEvidence: OrchestrationCompatibilityEvidence = { + terminalHandle: WORKER_HANDLE, + paneKey: WORKER_PANE, + launchToken: 'worker-token' +} + +describe('legacy question takeover compatibility', () => { + it('reuses a pending ask that crossed the schema cutover', async () => { + const harness = createHarness({ seedCutoverQuestion: true }) + + const asked = await harness.dispatcher.dispatch( + request( + 'orchestration.ask', + { + from: WORKER_HANDLE, + to: COORDINATOR_HANDLE, + question: 'Continue after update?', + options: 'yes,no', + timeoutMs: 0 + }, + 'retry-cutover-ask', + workerEvidence + ) + ) + + expect(asked).toMatchObject({ + ok: true, + result: { + messageId: harness.cutoverQuestionId, + legacyCompatibility: { replayed: true } + } + }) + const sqlite = (harness.db as unknown as { db: Database.Database }).db + expect( + ( + sqlite + .prepare( + `SELECT COUNT(*) AS count + FROM messages + WHERE type IN ('decision_gate', 'question')` + ) + .get() as { count: number } + ).count + ).toBe(1) + expect( + (sqlite.prepare('SELECT COUNT(*) AS count FROM question_threads').get() as { count: number }) + .count + ).toBe(1) + }) + + it('resumes an answered pinned-A ask without duplicating it', async () => { + const harness = createHarness({ + seedCutoverQuestion: true, + seedCutoverAnswer: true + }) + + const resumed = await harness.dispatcher.dispatch( + request( + 'orchestration.ask', + { + from: WORKER_HANDLE, + resume: harness.cutoverQuestionId, + timeoutMs: 0 + }, + 'resume-answered-cutover-ask', + workerEvidence + ) + ) + + expect(resumed).toMatchObject({ + ok: true, + result: { + answer: 'Continue.', + answerMessageId: harness.cutoverAnswerId, + messageId: harness.cutoverQuestionId, + legacyCompatibility: { + answerAcknowledgement: { + questionId: harness.cutoverQuestionId, + answerMessageId: harness.cutoverAnswerId + } + } + } + }) + const sqlite = (harness.db as unknown as { db: Database.Database }).db + expect( + (sqlite.prepare('SELECT COUNT(*) AS count FROM question_threads').get() as { count: number }) + .count + ).toBe(1) + }) + + it('resumes and idempotently acknowledges the exact current Run answer', async () => { + const harness = createHarness() + const asked = await harness.dispatcher.dispatch( + request( + 'orchestration.ask', + { + from: WORKER_HANDLE, + to: COORDINATOR_HANDLE, + question: 'Continue after takeover?', + timeoutMs: 0 + }, + 'ask-before-takeover', + workerEvidence + ) + ) + const questionId = (asked as { result: { messageId: string } }).result.messageId + + await expect( + harness.dispatcher.dispatch( + request( + 'orchestration.runUse', + { + id: harness.adoptedRunId, + from: CURRENT_COORDINATOR_HANDLE, + takeoverLegacy: true + }, + 'take-over', + { + terminalHandle: CURRENT_COORDINATOR_HANDLE, + paneKey: CURRENT_COORDINATOR_PANE, + launchToken: 'current-coordinator-token' + } + ) + ) + ).resolves.toMatchObject({ ok: true }) + await expect( + harness.dispatcher.dispatch( + request( + 'orchestration.reply', + { + id: questionId, + body: 'Continue.', + from: CURRENT_COORDINATOR_HANDLE, + run: harness.adoptedRunId + }, + 'answer-after-takeover' + ) + ) + ).resolves.toMatchObject({ ok: true }) + + const resumed = await harness.dispatcher.dispatch( + request( + 'orchestration.ask', + { from: WORKER_HANDLE, resume: questionId, timeoutMs: 100 }, + 'resume-after-takeover', + workerEvidence + ) + ) + expect(resumed).toMatchObject({ + ok: true, + result: { + answer: 'Continue.', + legacyCompatibility: { + answerAcknowledgement: { + questionId, + answerMessageId: expect.any(String) + } + } + } + }) + const acknowledgement = ( + resumed as { + result: { + legacyCompatibility: { + answerAcknowledgement: { questionId: string; answerMessageId: string } + } + } + } + ).result.legacyCompatibility.answerAcknowledgement + expect(harness.db.getMessageById(acknowledgement.answerMessageId)).toMatchObject({ + run_id: harness.adoptedRunId, + from_handle: `run:${harness.adoptedRunId}`, + to_handle: `dispatch:${harness.dispatchId}`, + delivery_contract: 'current_delivery', + thread_id: questionId + }) + + const acknowledge = (invocationId: string) => + harness.dispatcher.dispatch( + request( + 'orchestration.check', + { + terminal: WORKER_HANDLE, + compatibilityQuestionAck: JSON.stringify(acknowledgement) + }, + invocationId, + workerEvidence + ) + ) + const sqlite = (harness.db as unknown as { db: Database.Database }).db + sqlite + .prepare('UPDATE messages SET to_handle = ? WHERE id = ?') + .run('dispatch:foreign', acknowledgement.answerMessageId) + await expect(acknowledge('ack-wrong-answer-route')).resolves.toMatchObject({ + ok: false, + error: { code: 'request_mismatch' } + }) + sqlite + .prepare('UPDATE messages SET to_handle = ? WHERE id = ?') + .run(`dispatch:${harness.dispatchId}`, acknowledgement.answerMessageId) + + await expect(acknowledge('ack-answer')).resolves.toMatchObject({ + ok: true, + result: { + acknowledged: [acknowledgement.answerMessageId], + duplicate: false, + legacyCompatibility: { acknowledged: true } + } + }) + await expect(acknowledge('ack-answer-repeat')).resolves.toMatchObject({ + ok: true, + result: { + acknowledged: [acknowledgement.answerMessageId], + duplicate: true, + legacyCompatibility: { acknowledged: true } + } + }) + }) +}) diff --git a/src/main/runtime/rpc/orchestration-legacy-question.ts b/src/main/runtime/rpc/orchestration-legacy-question.ts new file mode 100644 index 00000000000..69dec720a3a --- /dev/null +++ b/src/main/runtime/rpc/orchestration-legacy-question.ts @@ -0,0 +1,169 @@ +import { clampOrchestrationAskTimeoutMs } from '../../../shared/orchestration-ask-timeout' +import type { RpcRequest } from './core' +import type { OrcaRuntimeService } from '../orca-runtime' +import { OrchestrationError } from '../orchestration/orchestration-error' +import { LEGACY_CONTRACT_VERSION } from '../orchestration/db' +import type { LegacyCompatibilityAuthority } from './orchestration-legacy-authority' +import { + normalizeLegacyText, + operationIdentity, + parseLegacyOptions, + type LegacyAskParams +} from './orchestration-legacy-operation' + +export async function handleLegacyAsk(args: { + runtime: OrcaRuntimeService + authority: LegacyCompatibilityAuthority + request: RpcRequest + params: LegacyAskParams + signal?: AbortSignal +}): Promise { + const { runtime, authority, request, params, signal } = args + const dispatch = authority.resolveAskDispatch(request, params) + if (!dispatch || dispatch.contract_version !== LEGACY_CONTRACT_VERSION) { + return undefined + } + const db = runtime.getOrchestrationDb() + if (!params.resume) { + if (!params.to) { + throw new OrchestrationError('invalid_argument', 'Legacy ask requires --to.') + } + if (!db.isLegacyCoordinatorHandle(dispatch.run_id, params.to)) { + throw new OrchestrationError( + 'request_mismatch', + `Terminal ${params.to} is not a retained coordinator for this legacy Dispatch.` + ) + } + } + const principal = authority.attestWorker(request, dispatch) + const timeoutMs = clampOrchestrationAskTimeoutMs(params.timeoutMs) + let questionId = params.resume + let duplicate = true + if (questionId) { + const question = db.getQuestion(questionId) + if (!question || question.dispatch_id !== dispatch.id) { + throw new OrchestrationError( + 'question_not_found', + `Question ${questionId} does not belong to this legacy Dispatch.` + ) + } + } else { + const question = params.question as string + const options = parseLegacyOptions(params.options) + const recipient = params.to as string + const operation = operationIdentity(request, 'ask', { + question: normalizeLegacyText(question), + options: options.map(normalizeLegacyText), + recipient + }) + const priorReceipt = db.getLegacyOperationReceipt(principal.id, operation.key) + let existingQuestionId: string | undefined + if (!priorReceipt) { + const matches = db.findLegacyQuestionsBySemanticIdentity({ + principalId: principal.id, + question, + options, + recipientHandle: recipient + }) + const unclaimed = matches.filter((match) => !match.claimedByOperation) + const pending = unclaimed.filter((match) => match.question.status === 'pending') + const lostAnswer = unclaimed.find( + (match) => match.question.status === 'answered' && !match.answerAcknowledged + ) + if (lostAnswer) { + const cliCommand = + params.compatibilityCliCommand ?? params.compatibilityWindowsCommand ?? 'orca' + throw new OrchestrationError( + 'operation_unknown', + `A matching legacy answer may have been accepted before the update. Run ${cliCommand} orchestration check --terminal ${principal.terminal_handle} before asking again.` + ) + } + if (pending.length > 1) { + throw new OrchestrationError( + 'operation_unknown', + 'Multiple matching pending legacy questions exist; drain legacy check before retrying.' + ) + } + existingQuestionId = pending[0]?.question.message_id + } + const committed = db.commitLegacyAskOperation({ + question, + principalId: principal.id, + operationKey: operation.key, + method: request.method, + payloadHash: operation.payloadHash, + options, + recipientHandle: recipient, + existingQuestionId + }) + questionId = committed.question.message_id + duplicate = committed.duplicate || Boolean(existingQuestionId) + if (!duplicate) { + runtime.notifyMessageArrived(committed.message.to_handle, committed.message.type) + } + } + + if (params.compatibilityWindowsCommand && !params.resume) { + return { + answer: null, + messageId: questionId, + threadId: questionId, + timedOut: false, + cancelled: false, + connectionLost: false, + timeoutMs, + legacyCompatibility: { + resumeRequired: true, + resumeCommand: `${params.compatibilityWindowsCommand} orchestration ask --resume ${questionId}` + } + } + } + + const deadline = Date.now() + timeoutMs + while (true) { + const current = db.getQuestion(questionId as string) + if (!current || current.status === 'closed') { + throw new OrchestrationError( + 'dispatch_inactive', + `Question ${questionId as string} closed because its Dispatch is inactive.` + ) + } + if (current.status === 'answered') { + return { + answer: current.answer_body, + answerMessageId: current.answer_message_id, + messageId: current.message_id, + threadId: current.message_id, + timedOut: false, + cancelled: false, + connectionLost: false, + timeoutMs, + legacyCompatibility: { + replayed: duplicate, + answerAcknowledgement: current.answer_message_id + ? { + questionId: current.message_id, + answerMessageId: current.answer_message_id + } + : undefined + } + } + } + if (signal?.aborted || Date.now() >= deadline) { + return { + answer: null, + messageId: questionId, + threadId: questionId, + timedOut: !signal?.aborted, + cancelled: signal?.aborted === true, + connectionLost: signal?.aborted === true, + timeoutMs, + legacyCompatibility: { replayed: duplicate, ackMessageIds: [] } + } + } + await runtime.waitForMessage(principal.terminal_handle, { + timeoutMs: Math.min(1_000, Math.max(deadline - Date.now(), 1)), + signal + }) + } +} diff --git a/src/main/runtime/rpc/orchestration-legacy-takeover-dispatcher.test.ts b/src/main/runtime/rpc/orchestration-legacy-takeover-dispatcher.test.ts new file mode 100644 index 00000000000..aa4c7daef01 --- /dev/null +++ b/src/main/runtime/rpc/orchestration-legacy-takeover-dispatcher.test.ts @@ -0,0 +1,565 @@ +import { createHash } from 'node:crypto' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { OrchestrationCompatibilityEvidence } from '../../../shared/orchestration-compatibility-evidence' +import { ORCHESTRATION_CONTRACT_VERSION } from '../../../shared/protocol-version' +import Database from '../../sqlite/sync-database' +import { OrcaRuntimeService } from '../orca-runtime' +import { OrchestrationDb } from '../orchestration/db' +import type { RpcRequest } from './core' +import { RpcDispatcher } from './dispatcher' +import { ORCHESTRATION_METHODS } from './methods/orchestration' + +const WORKER_HANDLE = 'term_legacy_worker' +const WORKER_PANE = 'tab_worker:33333333-3333-4333-8333-333333333333' +const COORDINATOR_HANDLE = 'term_legacy_coord' +const COORDINATOR_PANE = 'tab_coord:44444444-4444-4444-8444-444444444444' +const CURRENT_COORDINATOR_HANDLE = 'term_current_coord' +const CURRENT_COORDINATOR_PANE = 'tab_current:55555555-5555-4555-8555-555555555555' + +type Harness = { + db: OrchestrationDb + dispatcher: RpcDispatcher + adoptedRunId: string + taskId: string + dispatchId: string +} + +const tempDirs: string[] = [] +const databases: OrchestrationDb[] = [] + +afterEach(() => { + for (const database of databases.splice(0)) { + database.close() + } + for (const dir of tempDirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }) + } +}) + +function createHarness(): Harness { + const dir = mkdtempSync(join(tmpdir(), 'orca-legacy-takeover-')) + tempDirs.push(dir) + const dbPath = join(dir, 'orchestration.db') + const before = new OrchestrationDb(dbPath) + const task = before.createTask({ + spec: 'legacy assignment', + createdByTerminalHandle: COORDINATOR_HANDLE + }) + const dispatch = before.createDispatchContext(task.id, WORKER_HANDLE, WORKER_PANE) + before.close() + + const raw = new Database(dbPath) + raw.exec(` + UPDATE dispatch_contexts SET process_incarnation = 'process-1'; + DROP INDEX IF EXISTS idx_messages_delivery_contract; + DROP TABLE legacy_mail_receipts; + DROP TABLE legacy_operation_receipts; + DROP TABLE legacy_compatibility_principals; + DROP TABLE legacy_adoptions; + `) + raw.pragma('user_version = 18') + raw.close() + + const db = new OrchestrationDb(dbPath) + databases.push(db) + const adoptedRunId = db.getLegacyAdoption()?.adopted_run_id as string + const runtime = new OrcaRuntimeService() + runtime.setOrchestrationDb(db) + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === COORDINATOR_HANDLE + ? COORDINATOR_PANE + : handle === WORKER_HANDLE + ? WORKER_PANE + : handle === CURRENT_COORDINATOR_HANDLE + ? CURRENT_COORDINATOR_PANE + : null + ) + vi.spyOn(runtime, 'verifyOrchestrationCompatibilityCaller').mockImplementation((proof) => { + const validWorker = proof?.terminalHandle === WORKER_HANDLE && proof.paneKey === WORKER_PANE + const validCoordinator = + proof?.terminalHandle === COORDINATOR_HANDLE && proof.paneKey === COORDINATOR_PANE + const validCurrentCoordinator = + proof?.terminalHandle === CURRENT_COORDINATOR_HANDLE && + proof.paneKey === CURRENT_COORDINATOR_PANE + if ((!validWorker && !validCoordinator && !validCurrentCoordinator) || !proof?.launchToken) { + return null + } + return { + hostScope: { kind: 'local', hostId: 'local' }, + terminalHandle: proof.terminalHandle as string, + paneKey: proof.paneKey as string, + processIncarnation: 'process-1', + launchTokenHash: createHash('sha256').update(proof.launchToken).digest('hex') + } + }) + vi.spyOn(runtime, 'notifyMessageArrived').mockImplementation(() => {}) + return { + db, + dispatcher: new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS }), + adoptedRunId, + taskId: task.id, + dispatchId: dispatch.id + } +} + +function evidence( + role: 'worker' | 'coordinator' | 'current-coordinator' +): OrchestrationCompatibilityEvidence { + const worker = role === 'worker' + const currentCoordinator = role === 'current-coordinator' + return { + terminalHandle: worker + ? WORKER_HANDLE + : currentCoordinator + ? CURRENT_COORDINATOR_HANDLE + : COORDINATOR_HANDLE, + paneKey: worker + ? WORKER_PANE + : currentCoordinator + ? CURRENT_COORDINATOR_PANE + : COORDINATOR_PANE, + launchToken: `${role}-token` + } +} + +function request( + method: string, + params: unknown, + proof: OrchestrationCompatibilityEvidence, + invocationId: string +): RpcRequest { + return { + id: `rpc_${invocationId}`, + authToken: 'caller-token', + method, + params, + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationRequestId: invocationId, + compatibilityInvocationId: invocationId, + orchestrationCompatibilityEvidence: proof + } +} + +function counts(db: OrchestrationDb): Record { + const sqlite = (db as unknown as { db: Database.Database }).db + return Object.fromEntries( + [ + 'messages', + 'legacy_compatibility_principals', + 'legacy_operation_receipts', + 'legacy_mail_receipts', + 'mutation_receipts' + ].map((table) => [ + table, + (sqlite.prepare(`SELECT COUNT(*) AS count FROM ${table}`).get() as { count: number }).count + ]) + ) +} + +function escalationParams(harness: Harness) { + return { + from: WORKER_HANDLE, + to: COORDINATOR_HANDLE, + subject: 'Blocked', + type: 'escalation', + payload: JSON.stringify({ taskId: harness.taskId, dispatchId: harness.dispatchId }) + } +} + +describe('legacy compatibility after explicit takeover', () => { + it('binds takeover to the authenticated invoking coordinator pane', async () => { + const harness = createHarness() + const before = harness.db.getRun(harness.adoptedRunId) + + const spoofed = await harness.dispatcher.dispatch( + request( + 'orchestration.runUse', + { + id: harness.adoptedRunId, + from: COORDINATOR_HANDLE, + takeoverLegacy: true + }, + evidence('current-coordinator'), + 'spoofed-takeover' + ) + ) + + expect(spoofed).toMatchObject({ + ok: false, + error: { code: 'legacy_read_only', data: { effectsApplied: false } } + }) + expect(harness.db.getRun(harness.adoptedRunId)).toEqual(before) + + const bound = await harness.dispatcher.dispatch( + request( + 'orchestration.runUse', + { + id: harness.adoptedRunId, + from: CURRENT_COORDINATOR_HANDLE, + takeoverLegacy: true + }, + evidence('current-coordinator'), + 'authenticated-takeover' + ) + ) + + expect(bound).toMatchObject({ + ok: true, + result: { + run: { + coordinator_handle: CURRENT_COORDINATOR_HANDLE, + coordinator_pane_key: CURRENT_COORDINATOR_PANE + } + } + }) + }) + + it('does not let an uncommitted legacy coordinator attest after explicit takeover', async () => { + const harness = createHarness() + harness.db.bindRun({ + runId: harness.adoptedRunId, + coordinatorHandle: 'term_current_coord', + coordinatorPaneKey: 'tab_current:55555555-5555-4555-8555-555555555555', + takeoverLegacy: true + }) + expect(harness.db.getLegacyCoordinatorPrincipal(harness.adoptedRunId)).toBeUndefined() + const before = counts(harness.db) + + const response = await harness.dispatcher.dispatch( + request( + 'orchestration.taskList', + {}, + evidence('coordinator'), + 'uncommitted-coordinator-after-takeover' + ) + ) + + expect(response).toMatchObject({ ok: false, error: { code: 'legacy_read_only' } }) + expect(harness.db.getLegacyCoordinatorPrincipal(harness.adoptedRunId)).toBeUndefined() + expect(counts(harness.db)).toEqual(before) + }) + + it.each(['escalation', 'worker_done'] as const)( + 'keeps the retained coordinator address routable for post-takeover %s', + async (type) => { + const harness = createHarness() + harness.db.bindRun({ + runId: harness.adoptedRunId, + coordinatorHandle: 'term_current_coord', + coordinatorPaneKey: 'tab_current:55555555-5555-4555-8555-555555555555', + takeoverLegacy: true + }) + + const response = await harness.dispatcher.dispatch( + request( + 'orchestration.send', + { + ...escalationParams(harness), + type, + subject: type === 'worker_done' ? 'Completed' : 'Blocked' + }, + evidence('worker'), + `post-takeover-${type}` + ) + ) + + expect(response).toMatchObject({ + ok: true, + result: { + message: { + to_handle: `run:${harness.adoptedRunId}`, + delivery_contract: 'current_delivery', + type + } + } + }) + } + ) + + it('routes future worker mail to Run Delivery after same-handle takeover', async () => { + const harness = createHarness() + harness.db.bindRun({ + runId: harness.adoptedRunId, + coordinatorHandle: COORDINATOR_HANDLE, + coordinatorPaneKey: COORDINATOR_PANE, + takeoverLegacy: true + }) + + const response = await harness.dispatcher.dispatch( + request( + 'orchestration.send', + escalationParams(harness), + evidence('worker'), + 'same-handle-takeover-mail' + ) + ) + + expect(response).toMatchObject({ + ok: true, + result: { + message: { + to_handle: `run:${harness.adoptedRunId}`, + delivery_contract: 'current_delivery', + type: 'escalation' + } + } + }) + }) + + it('keeps post-takeover legacy ask routed to the adopted Run', async () => { + const harness = createHarness() + harness.db.bindRun({ + runId: harness.adoptedRunId, + coordinatorHandle: 'term_current_coord', + coordinatorPaneKey: 'tab_current:55555555-5555-4555-8555-555555555555', + takeoverLegacy: true + }) + + const response = await harness.dispatcher.dispatch( + request( + 'orchestration.ask', + { + from: WORKER_HANDLE, + to: COORDINATOR_HANDLE, + question: 'Continue after takeover?', + timeoutMs: 1 + }, + evidence('worker'), + 'post-takeover-ask' + ) + ) + const messageId = (response as { result?: { messageId?: string } }).result?.messageId + + expect(response).toMatchObject({ + ok: true, + result: { timedOut: true, messageId: expect.any(String) } + }) + expect(harness.db.getMessageById(messageId as string)).toMatchObject({ + to_handle: `run:${harness.adoptedRunId}`, + delivery_contract: 'current_delivery', + type: 'question' + }) + }) + + it('keeps an active legacy worker usable across authenticated takeover', async () => { + const harness = createHarness() + await expect( + harness.dispatcher.dispatch( + request( + 'orchestration.runUse', + { id: harness.adoptedRunId, from: COORDINATOR_HANDLE }, + evidence('coordinator'), + 'bind-original-coordinator' + ) + ) + ).resolves.toMatchObject({ ok: true }) + const pendingAsk = await harness.dispatcher.dispatch( + request( + 'orchestration.ask', + { + from: WORKER_HANDLE, + to: COORDINATOR_HANDLE, + question: 'Continue with the migration?', + timeoutMs: 0 + }, + evidence('worker'), + 'pending-before-takeover' + ) + ) + const questionId = (pendingAsk as { result: { messageId: string } }).result.messageId + const unrelated = harness.db.insertMessage({ + runId: harness.adoptedRunId, + from: WORKER_HANDLE, + to: 'term_peer_worker', + subject: 'peer-only', + deliveryContract: 'legacy_direct' + }) + const currentRequest = (method: string, params: unknown, invocationId: string) => + request(method, params, evidence('current-coordinator'), invocationId) + const takeoverParams = { + id: harness.adoptedRunId, + from: CURRENT_COORDINATOR_HANDLE, + takeoverLegacy: true + } + + const takeover = await harness.dispatcher.dispatch( + currentRequest('orchestration.runUse', takeoverParams, 'explicit-takeover') + ) + const repeated = await harness.dispatcher.dispatch( + currentRequest('orchestration.runUse', takeoverParams, 'explicit-takeover-repeat') + ) + + expect(takeover).toMatchObject({ + ok: true, + result: { binding: { consumerGeneration: 2 } } + }) + expect(repeated).toMatchObject({ + ok: true, + result: { binding: { consumerGeneration: 2 } } + }) + expect(harness.db.getDispatchContextById(harness.dispatchId)?.status).toBe('dispatched') + expect(harness.db.getLegacyCoordinatorPrincipal(harness.adoptedRunId)?.status).toBe('revoked') + expect(harness.db.getMessageById(unrelated.id)).toMatchObject({ + to_handle: 'term_peer_worker', + delivery_contract: 'legacy_direct' + }) + + const promoted = await harness.dispatcher.dispatch( + currentRequest( + 'orchestration.check', + { + terminal: CURRENT_COORDINATOR_HANDLE, + run: harness.adoptedRunId, + format: true + }, + 'read-promoted-question' + ) + ) + expect(promoted).toMatchObject({ + ok: true, + result: { + messages: [ + { + id: questionId, + to_handle: `run:${harness.adoptedRunId}`, + delivery_contract: 'current_delivery' + } + ], + formatted: expect.not.stringContaining(`--from run:${harness.adoptedRunId}`) + } + }) + const deliveryId = (promoted as { result: { deliveryId: string } }).result.deliveryId + await expect( + harness.dispatcher.dispatch( + currentRequest( + 'orchestration.reply', + { + id: questionId, + body: 'Yes, continue.', + from: CURRENT_COORDINATOR_HANDLE, + run: harness.adoptedRunId + }, + 'reply-after-takeover' + ) + ) + ).resolves.toMatchObject({ ok: true, result: { question: { status: 'answered' } } }) + await expect( + harness.dispatcher.dispatch( + request( + 'orchestration.ask', + { from: WORKER_HANDLE, resume: questionId, timeoutMs: 100 }, + evidence('worker'), + 'resume-after-takeover' + ) + ) + ).resolves.toMatchObject({ ok: true, result: { answer: 'Yes, continue.' } }) + + const followUp = await harness.dispatcher.dispatch( + currentRequest( + 'orchestration.send', + { + from: CURRENT_COORDINATOR_HANDLE, + to: `dispatch:${harness.dispatchId}`, + subject: 'Continue from the new coordinator' + }, + 'current-coordinator-to-legacy-worker' + ) + ) + expect(followUp).toMatchObject({ + ok: true, + result: { + message: { + run_id: harness.adoptedRunId, + to_handle: `dispatch:${harness.dispatchId}`, + delivery_contract: 'legacy_direct' + } + } + }) + const workerFollowUp = await harness.dispatcher.dispatch( + request( + 'orchestration.check', + { terminal: WORKER_HANDLE }, + evidence('worker'), + 'legacy-worker-check-after-takeover' + ) + ) + expect(workerFollowUp).toMatchObject({ + ok: true, + result: { + messages: [{ subject: 'Continue from the new coordinator' }], + legacyCompatibility: { ackMessageIds: [expect.any(String)] } + } + }) + const [followUpMessageId] = ( + workerFollowUp as { result: { legacyCompatibility: { ackMessageIds: string[] } } } + ).result.legacyCompatibility.ackMessageIds + await expect( + harness.dispatcher.dispatch( + request( + 'orchestration.check', + { + terminal: WORKER_HANDLE, + compatibilityAck: JSON.stringify({ messageIds: [followUpMessageId] }) + }, + evidence('worker'), + 'legacy-worker-ack-after-takeover' + ) + ) + ).resolves.toMatchObject({ + ok: true, + result: { + acknowledged: [followUpMessageId], + legacyCompatibility: { acknowledged: true } + } + }) + + await harness.dispatcher.dispatch( + currentRequest( + 'orchestration.check', + { + terminal: CURRENT_COORDINATOR_HANDLE, + run: harness.adoptedRunId, + ack: deliveryId + }, + 'ack-promoted-question' + ) + ) + await expect( + harness.dispatcher.dispatch( + request( + 'orchestration.send', + { + from: WORKER_HANDLE, + to: COORDINATOR_HANDLE, + subject: 'Need current coordinator', + type: 'escalation', + payload: JSON.stringify({ taskId: harness.taskId }) + }, + evidence('worker'), + 'escalation-after-takeover' + ) + ) + ).resolves.toMatchObject({ + ok: true, + result: { + message: { + to_handle: `run:${harness.adoptedRunId}`, + delivery_contract: 'current_delivery' + } + } + }) + await expect( + harness.dispatcher.dispatch( + request( + 'orchestration.taskList', + { callerTerminalHandle: COORDINATOR_HANDLE }, + evidence('coordinator'), + 'old-coordinator-fenced' + ) + ) + ).resolves.toMatchObject({ ok: false, error: { code: 'legacy_read_only' } }) + }) +}) diff --git a/src/main/runtime/rpc/orchestration-mutation-executor.ts b/src/main/runtime/rpc/orchestration-mutation-executor.ts index 0ec659cc02d..66918510b56 100644 --- a/src/main/runtime/rpc/orchestration-mutation-executor.ts +++ b/src/main/runtime/rpc/orchestration-mutation-executor.ts @@ -22,13 +22,14 @@ export class OrchestrationMutationExecutor { async run( request: RpcRequest, params: unknown, - invoke: (mutation?: DurableMutationInvocation) => Promise | unknown + invoke: (mutation?: DurableMutationInvocation) => Promise | unknown, + callerFingerprintOverride?: string ): Promise { const requestId = request.orchestrationRequestId if (!requestId || !isOrchestrationMutation(request.method, params)) { return await invoke() } - const callerFingerprint = authenticatedCallerFingerprint(request) + const callerFingerprint = callerFingerprintOverride ?? authenticatedCallerFingerprint(request) const payloadHash = createHash('sha256') .update(JSON.stringify(canonicalize({ method: request.method, params }))) .digest('hex') @@ -55,6 +56,10 @@ export class OrchestrationMutationExecutor { : db.beginMutationReceipt(identity) if (begun.disposition === 'completed') { + const active = this.inFlight.get(key) + if (active) { + return attachMutationReceipt(await active, requestId, true) + } return attachMutationReceipt(JSON.parse(begun.row.receipt ?? 'null'), requestId, true) } if (begun.disposition === 'pending') { @@ -102,6 +107,20 @@ export class OrchestrationMutationExecutor { } } +const executorsByRuntime = new WeakMap() + +export function getOrchestrationMutationExecutor( + runtime: OrcaRuntimeService +): OrchestrationMutationExecutor { + const existing = executorsByRuntime.get(runtime) + if (existing) { + return existing + } + const executor = new OrchestrationMutationExecutor(runtime) + executorsByRuntime.set(runtime, executor) + return executor +} + export function authenticatedCallerFingerprint(request: RpcRequest): string { const callerToken = request.authToken || diff --git a/src/main/runtime/rpc/orchestration-mutation-ledger.test.ts b/src/main/runtime/rpc/orchestration-mutation-ledger.test.ts index 3407211ba6f..8e02d9c8e24 100644 --- a/src/main/runtime/rpc/orchestration-mutation-ledger.test.ts +++ b/src/main/runtime/rpc/orchestration-mutation-ledger.test.ts @@ -252,7 +252,10 @@ describe('durable orchestration mutation ledger', () => { }) it('recovers a lost ask acceptance without creating a second question', async () => { - const db = new OrchestrationDb(':memory:') + const dir = mkdtempSync(join(tmpdir(), 'orca-mutation-ask-recovery-')) + paths.push(dir) + const dbPath = join(dir, 'orchestration.db') + const db = new OrchestrationDb(dbPath) const runtime = new OrcaRuntimeService() runtime.setOrchestrationDb(db) vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue('tab_worker:leaf_worker') @@ -290,7 +293,16 @@ describe('durable orchestration mutation ledger', () => { const first = firstDispatcher.dispatch(askRequest, { signal: controller.signal }) await vi.waitFor(() => expect(db.getInbox(10)).toHaveLength(1)) - const restartedDispatcher = new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS }) + const restartedDb = new OrchestrationDb(dbPath) + const restartedRuntime = new OrcaRuntimeService() + restartedRuntime.setOrchestrationDb(restartedDb) + vi.spyOn(restartedRuntime, 'getTerminalPaneKey').mockReturnValue('tab_worker:leaf_worker') + vi.spyOn(restartedRuntime, 'getTerminalProcessIncarnation').mockReturnValue('runtime:pty:1') + vi.spyOn(restartedRuntime, 'notifyMessageArrived').mockImplementation(() => {}) + const restartedDispatcher = new RpcDispatcher({ + runtime: restartedRuntime, + methods: ORCHESTRATION_METHODS + }) const recovered = await restartedDispatcher.dispatch({ ...askRequest, id: 'rpc_ask_2' }) expect(recovered).toMatchObject({ ok: true, @@ -304,6 +316,7 @@ describe('durable orchestration mutation ledger', () => { controller.abort() await first + restartedDb.close() db.close() }) }) diff --git a/src/main/runtime/rpc/runtime-feature-interaction.ts b/src/main/runtime/rpc/runtime-feature-interaction.ts index 2a25bff1080..23f28001fcd 100644 --- a/src/main/runtime/rpc/runtime-feature-interaction.ts +++ b/src/main/runtime/rpc/runtime-feature-interaction.ts @@ -1,5 +1,6 @@ import type { FeatureInteractionId } from '../../../shared/feature-interactions' import { isBrowserPaneUiRuntimeRpcParams } from '../../../shared/runtime-rpc-feature-interaction-source' +import type { OrcaRuntimeService } from '../orca-runtime' export function getRuntimeFeatureInteractionId( method: string, @@ -37,6 +38,25 @@ export function getRuntimeFeatureInteractionId( return method.startsWith('orchestration.') ? 'agent-orchestration' : null } +export function recordRuntimeFeatureInteraction( + runtime: OrcaRuntimeService, + method: string, + result: unknown, + alreadyRecorded?: Set, + rawParams?: unknown +): void { + const id = getRuntimeFeatureInteractionId(method, result, rawParams) + if (!id || alreadyRecorded?.has(id)) { + return + } + try { + runtime.recordFeatureInteraction(id) + alreadyRecorded?.add(id) + } catch { + // Best-effort education state must not break runtime tools. + } +} + function hasBooleanResult(value: unknown, key: string): boolean { return ( value !== null && typeof value === 'object' && (value as Record)[key] === true diff --git a/src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts b/src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts index 1a3e8721064..cd3deff182b 100644 --- a/src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts +++ b/src/main/ssh/ssh-relay-session-reconnect-incarnation.test.ts @@ -1,20 +1,47 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' +import { randomUUID } from 'node:crypto' +import type * as NodeCrypto from 'node:crypto' import { SshRelaySession } from './ssh-relay-session' import { createMockDeps, mockDeploySuccess } from './ssh-relay-session-test-fixtures' -const { muxRequestMock } = vi.hoisted(() => ({ muxRequestMock: vi.fn() })) +type MockMuxInstance = { + requestHandlers: Map) => Promise> +} + +const { muxRequestMock, muxInstancesRaw } = vi.hoisted(() => ({ + muxRequestMock: vi.fn(), + muxInstancesRaw: [] as unknown[] +})) +const muxInstances = muxInstancesRaw as MockMuxInstance[] vi.mock('./ssh-relay-deploy', () => ({ deployAndLaunchRelay: vi.fn() })) vi.mock('./ssh-relay-deploy-helpers', () => ({ execCommand: vi.fn().mockResolvedValue('') })) +vi.mock('node:crypto', async (importOriginal) => { + const actual = await importOriginal() + return { ...actual, randomUUID: vi.fn() } +}) +vi.mock('./ssh-remote-orca-cli', () => ({ + runRemoteOrcaCli: vi.fn().mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' }) +})) vi.mock('./ssh-channel-multiplexer', () => ({ SshChannelMultiplexer: class MockSshChannelMultiplexer { + requestHandlers = new Map) => Promise>() notify = vi.fn() request = muxRequestMock onNotification = vi.fn().mockReturnValue(() => {}) - onRequest = vi.fn().mockReturnValue(() => {}) + onRequest = vi.fn( + (method: string, handler: (params: Record) => Promise) => { + this.requestHandlers.set(method, handler) + return () => this.requestHandlers.delete(method) + } + ) onDispose = vi.fn().mockReturnValue(() => {}) dispose = vi.fn() isDisposed = vi.fn().mockReturnValue(false) + + constructor() { + muxInstancesRaw.push(this) + } } })) vi.mock('../agent-hooks/remote-managed-hook-installers', () => ({ @@ -98,13 +125,74 @@ function emitExitDuringAttach(payload: { id: string; code: number; incarnationId describe('SshRelaySession reconnect incarnation ordering', () => { beforeEach(() => { vi.clearAllMocks() + muxInstances.splice(0) delete process.env.ORCA_FEATURE_REMOTE_AGENT_HOOKS muxRequestMock.mockReset() muxRequestMock.mockResolvedValue([]) + vi.mocked(randomUUID).mockReset() + vi.mocked(randomUUID).mockReturnValue('00000000-0000-4000-8000-000000000001') mockDeploySuccess() vi.mocked(getPtyIdsForConnection).mockReturnValue([]) }) + it('keeps the winning reconnect incarnation when a stale health check resolves last', async () => { + const initialIncarnation = '00000000-0000-4000-8000-000000000001' + const winningIncarnation = '00000000-0000-4000-8000-000000000002' + const staleIncarnation = '00000000-0000-4000-8000-000000000003' + let resolveStaleHealthCheck!: (value: unknown) => void + const staleHealthCheck = new Promise((resolve) => { + resolveStaleHealthCheck = resolve + }) + let resolveHomeCalls = 0 + muxRequestMock.mockImplementation((method: string) => { + if (method !== 'session.resolveHome') { + return Promise.resolve([]) + } + resolveHomeCalls += 1 + return resolveHomeCalls === 2 ? staleHealthCheck : Promise.resolve('/') + }) + vi.mocked(randomUUID) + .mockReturnValueOnce(initialIncarnation) + .mockReturnValueOnce(winningIncarnation) + .mockReturnValue(staleIncarnation) + const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() + const runtime = { + registerOrchestrationCompatibilitySshAttachment: vi.fn( + (_targetId: string, connectionIncarnation: string) => ({ + attachmentId: `attachment-${connectionIncarnation}`, + connectionIncarnation + }) + ), + releaseOrchestrationCompatibilitySshAttachment: vi.fn() + } + const session = new SshRelaySession( + 'target-1', + getMainWindow, + mockStore, + mockPortForward, + runtime as never + ) + await session.establish(mockConn) + + const staleReconnect = session.reconnect(mockConn) + await vi.waitFor(() => expect(resolveHomeCalls).toBe(2)) + await session.reconnect(mockConn) + expect(session.getState()).toBe('ready') + + resolveStaleHealthCheck('/') + await staleReconnect + + const winningCliHandler = muxInstances[2]?.requestHandlers.get('orca.cli') + expect(winningCliHandler).toBeDefined() + await winningCliHandler?.({ argv: ['status'], cwd: '/', env: {} }) + + expect(runtime.registerOrchestrationCompatibilitySshAttachment).toHaveBeenCalledWith( + 'target-1', + winningIncarnation + ) + expect(randomUUID).toHaveBeenCalledTimes(2) + }) + it('restores and persists exact incarnation proof from reconnect attach', async () => { const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() const incarnationId = 'incarnation-reconnect' diff --git a/src/main/ssh/ssh-relay-session.ts b/src/main/ssh/ssh-relay-session.ts index fdd0b115203..392a422e138 100644 --- a/src/main/ssh/ssh-relay-session.ts +++ b/src/main/ssh/ssh-relay-session.ts @@ -2,6 +2,7 @@ // Why: single authority for all relay lifecycle state per SSH target (previously scattered across module Maps/Sets with duplicated paths). import type { BrowserWindow } from 'electron' +import { randomUUID } from 'node:crypto' import { deployAndLaunchRelay } from './ssh-relay-deploy' import { execCommand } from './ssh-relay-deploy-helpers' import { isRelayVersionMismatchError } from './ssh-relay-version-mismatch-error' @@ -65,6 +66,10 @@ import { import type { Store } from '../persistence' import type { OrcaRuntimeService } from '../runtime/orca-runtime' import { runRemoteOrcaCli } from './ssh-remote-orca-cli' +import { + acknowledgeRemoteOrcaCliPostOutput, + parseRemoteOrcaCliPostOutput +} from './ssh-remote-orchestration-post-output' import { toSshExecutionHostId, type ExecutionHostId } from '../../shared/execution-host' import { isTerminalLeafId, makePaneKey } from '../../shared/stable-pane-id' import { isValidTerminalTabId } from '../../shared/terminal-tab-id' @@ -151,6 +156,7 @@ export class SshRelaySession { private remoteCliBridgeEnv: RemoteCliBridgeEnv | null = null private forwardedReattachReplayByPty = new Map() private pendingPtyReattaches = new Map() + private activeCompatibilityAttachmentIds = new Set() constructor( readonly targetId: string, @@ -278,8 +284,15 @@ export class SshRelaySession { // Why: round-trip the relay before registering providers so a closed --connect bridge fails fast instead of leaving a 'ready' session on a dead mux. await mux.request('session.resolveHome', { path: '~' }) + if (!ownsAttempt()) { + if (!mux.isDisposed()) { + mux.dispose() + } + throw new Error('Session disposed during establish') + } + const connectionIncarnation = randomUUID() - const registered = await this.registerProviders(mux, ownsAttempt) + const registered = await this.registerProviders(mux, ownsAttempt, connectionIncarnation) if (!registered) { if (!mux.isDisposed()) { mux.dispose() @@ -387,8 +400,9 @@ export class SshRelaySession { } return } + const connectionIncarnation = randomUUID() - const registered = await this.registerProviders(mux, ownsAttempt) + const registered = await this.registerProviders(mux, ownsAttempt, connectionIncarnation) if (!registered) { if (!mux.isDisposed()) { mux.dispose() @@ -499,7 +513,8 @@ export class SshRelaySession { // Why: shared by establish() and reconnect() so both use the exact same registration sequence. private async registerProviders( mux: SshChannelMultiplexer, - shouldContinue?: () => boolean + shouldContinue: (() => boolean) | undefined, + connectionIncarnation: string ): Promise { await this.registerRelayRoots(mux) if (shouldContinue && !shouldContinue()) { @@ -530,7 +545,7 @@ export class SshRelaySession { return false } - this.wireUpRemoteOrcaCli(mux) + this.wireUpRemoteOrcaCli(mux, connectionIncarnation) const ptyProvider = new SshPtyProvider(this.targetId, mux, this.remoteCliBridgeEnv ?? undefined) registerSshPtyProvider(this.targetId, ptyProvider) @@ -667,7 +682,7 @@ export class SshRelaySession { } } - private wireUpRemoteOrcaCli(mux: SshChannelMultiplexer): void { + private wireUpRemoteOrcaCli(mux: SshChannelMultiplexer, connectionIncarnation: string): void { mux.onRequest('orca.cli', async (params) => { if (!this.runtime) { throw new Error('Orca runtime is unavailable') @@ -687,12 +702,54 @@ export class SshRelaySession { ) : {} const stdin = typeof params.stdin === 'string' ? params.stdin : undefined - return await runRemoteOrcaCli(this.runtime, { - argv, - cwd, - env, - ...(stdin !== undefined ? { stdin } : {}) - }) + const runtimeAuthority = this.runtime.registerOrchestrationCompatibilitySshAttachment( + this.targetId, + connectionIncarnation + ) + this.activeCompatibilityAttachmentIds.add(runtimeAuthority.attachmentId) + try { + return await runRemoteOrcaCli(this.runtime, { + argv, + cwd, + env, + ...(stdin !== undefined ? { stdin } : {}), + runtimeAuthority + }) + } finally { + this.activeCompatibilityAttachmentIds.delete(runtimeAuthority.attachmentId) + this.runtime.releaseOrchestrationCompatibilitySshAttachment(runtimeAuthority.attachmentId) + } + }) + mux.onRequest('orca.cli.postOutput', async (params) => { + if (!this.runtime) { + throw new Error('Orca runtime is unavailable') + } + const rawEnv = params.env + const env = + rawEnv && typeof rawEnv === 'object' && !Array.isArray(rawEnv) + ? Object.fromEntries( + Object.entries(rawEnv).filter( + (entry): entry is [string, string] => + typeof entry[0] === 'string' && typeof entry[1] === 'string' + ) + ) + : {} + const runtimeAuthority = this.runtime.registerOrchestrationCompatibilitySshAttachment( + this.targetId, + connectionIncarnation + ) + this.activeCompatibilityAttachmentIds.add(runtimeAuthority.attachmentId) + try { + await acknowledgeRemoteOrcaCliPostOutput(this.runtime, { + postOutput: parseRemoteOrcaCliPostOutput(params.postOutput), + env, + runtimeAuthority + }) + return { acknowledged: true } + } finally { + this.activeCompatibilityAttachmentIds.delete(runtimeAuthority.attachmentId) + this.runtime.releaseOrchestrationCompatibilitySshAttachment(runtimeAuthority.attachmentId) + } }) } @@ -826,6 +883,10 @@ export class SshRelaySession { this.mux.dispose(reason) } this.mux = null + for (const attachmentId of this.activeCompatibilityAttachmentIds) { + this.runtime?.releaseOrchestrationCompatibilitySshAttachment(attachmentId) + } + this.activeCompatibilityAttachmentIds.clear() if (reason === 'shutdown') { clearPtyOwnershipForConnection(this.targetId) diff --git a/src/main/ssh/ssh-remote-cli-host-passthrough.test.ts b/src/main/ssh/ssh-remote-cli-host-passthrough.test.ts index a65ba6f91b5..1829becb6b2 100644 --- a/src/main/ssh/ssh-remote-cli-host-passthrough.test.ts +++ b/src/main/ssh/ssh-remote-cli-host-passthrough.test.ts @@ -22,6 +22,12 @@ import { import { resolveOrchestrationAskClientTimeoutMs } from '../../shared/orchestration-ask-timeout' import { remoteCliRequestTimeoutMs } from '../../relay/remote-cli-timeout' import { MAX_TIMER_DELAY_MS } from '../../shared/timer-delay' +import { + ORCHESTRATION_COMPATIBILITY_ATTACHMENT_ENV, + ORCHESTRATION_COMPATIBILITY_HOST_ID_ENV, + ORCHESTRATION_COMPATIBILITY_HOST_INCARNATION_ENV, + ORCHESTRATION_COMPATIBILITY_HOST_KIND_ENV +} from '../../shared/orchestration-compatibility-evidence' type FakeChild = EventEmitter & { stdout: EventEmitter @@ -65,7 +71,12 @@ describe('buildHostCliEnv', () => { ORCA_TERMINAL_HANDLE: 'term_remote', ORCA_WORKTREE_ID: 'repo::/home/alice/wt', ORCA_PANE_KEY: 'pane-9', + ORCA_AGENT_LAUNCH_TOKEN: 'launch-secret', ORCA_WORKSPACE_ID: 'ws-1', + [ORCHESTRATION_COMPATIBILITY_HOST_KIND_ENV]: 'wsl', + [ORCHESTRATION_COMPATIBILITY_HOST_ID_ENV]: 'caller-host', + [ORCHESTRATION_COMPATIBILITY_HOST_INCARNATION_ENV]: 'caller-incarnation', + [ORCHESTRATION_COMPATIBILITY_ATTACHMENT_ENV]: 'caller-attachment', // Why: these are remote-machine paths and must not leak into the host // subprocess (PATH would break host binary lookup; user-data would // retarget the CLI at a different local instance). @@ -73,13 +84,24 @@ describe('buildHostCliEnv', () => { ORCA_USER_DATA_PATH: '/remote/user-data' }, userDataPath: '/host/user-data', - remoteCwd: '/home/alice/wt/sub' + remoteCwd: '/home/alice/wt/sub', + runtimeAuthority: { + kind: 'ssh', + targetId: 'saved-target', + connectionIncarnation: 'connection-incarnation', + attachmentId: 'runtime-attachment' + } }) expect(env.ORCA_TERMINAL_HANDLE).toBe('term_remote') expect(env.ORCA_WORKTREE_ID).toBe('repo::/home/alice/wt') expect(env.ORCA_PANE_KEY).toBe('pane-9') + expect(env.ORCA_AGENT_LAUNCH_TOKEN).toBe('launch-secret') expect(env.ORCA_WORKSPACE_ID).toBe('ws-1') + expect(env[ORCHESTRATION_COMPATIBILITY_HOST_KIND_ENV]).toBe('ssh') + expect(env[ORCHESTRATION_COMPATIBILITY_HOST_ID_ENV]).toBe('saved-target') + expect(env[ORCHESTRATION_COMPATIBILITY_HOST_INCARNATION_ENV]).toBe('connection-incarnation') + expect(env[ORCHESTRATION_COMPATIBILITY_ATTACHMENT_ENV]).toBe('runtime-attachment') expect(env.PATH).toBe('/host/bin') expect(env.ORCA_USER_DATA_PATH).toBe('/host/user-data') expect(env.ORCA_CLI_CWD).toBe('/home/alice/wt/sub') diff --git a/src/main/ssh/ssh-remote-cli-host-passthrough.ts b/src/main/ssh/ssh-remote-cli-host-passthrough.ts index 09db31d4770..16510ddf479 100644 --- a/src/main/ssh/ssh-remote-cli-host-passthrough.ts +++ b/src/main/ssh/ssh-remote-cli-host-passthrough.ts @@ -17,20 +17,49 @@ import { parsePositiveSafeIntegerNumericText, parsePositiveSafeIntegerText } from '../../shared/timer-delay' +import { + ORCHESTRATION_COMPATIBILITY_ATTACHMENT_ENV, + ORCHESTRATION_COMPATIBILITY_HOST_ID_ENV, + ORCHESTRATION_COMPATIBILITY_HOST_INCARNATION_ENV, + ORCHESTRATION_COMPATIBILITY_HOST_KIND_ENV +} from '../../shared/orchestration-compatibility-evidence' + +export type SshCliRuntimeAuthority = { + kind: 'ssh' + targetId: string + connectionIncarnation: string + attachmentId: string +} export type RemoteOrcaCliRequest = { argv: string[] cwd: string env: Record stdin?: string + runtimeAuthority?: SshCliRuntimeAuthority } export type RemoteOrcaCliResult = { stdout: string stderr: string exitCode: number + postOutput?: RemoteOrcaCliPostOutput } +export type RemoteOrcaCliPostOutput = + | { + kind: 'legacy_check_ack' + terminal: string + messageIds: string[] + types?: string[] + } + | { + kind: 'legacy_question_ack' + terminal: string + questionId: string + answerMessageId: string + } + export type HostCliPassthroughOptions = { execPath?: string cliEntryPath?: string @@ -54,6 +83,7 @@ const REMOTE_CONTEXT_ENV_VARS = [ 'ORCA_TERMINAL_HANDLE', 'ORCA_WORKTREE_ID', 'ORCA_PANE_KEY', + 'ORCA_AGENT_LAUNCH_TOKEN', 'ORCA_WORKSPACE_ID' ] as const @@ -106,6 +136,7 @@ export function buildHostCliEnv(args: { remoteEnv: Record userDataPath: string remoteCwd: string + runtimeAuthority?: SshCliRuntimeAuthority }): NodeJS.ProcessEnv { const env: NodeJS.ProcessEnv = { ...args.hostEnv } for (const key of REMOTE_CONTEXT_ENV_VARS) { @@ -127,6 +158,17 @@ export function buildHostCliEnv(args: { env.ORCA_NODE_REPL_EXTERNAL_MODULE = args.hostEnv.NODE_REPL_EXTERNAL_MODULE ?? '' delete env.NODE_OPTIONS delete env.NODE_REPL_EXTERNAL_MODULE + delete env[ORCHESTRATION_COMPATIBILITY_HOST_KIND_ENV] + delete env[ORCHESTRATION_COMPATIBILITY_HOST_ID_ENV] + delete env[ORCHESTRATION_COMPATIBILITY_HOST_INCARNATION_ENV] + delete env[ORCHESTRATION_COMPATIBILITY_ATTACHMENT_ENV] + if (args.runtimeAuthority) { + env[ORCHESTRATION_COMPATIBILITY_HOST_KIND_ENV] = 'ssh' + env[ORCHESTRATION_COMPATIBILITY_HOST_ID_ENV] = args.runtimeAuthority.targetId + env[ORCHESTRATION_COMPATIBILITY_HOST_INCARNATION_ENV] = + args.runtimeAuthority.connectionIncarnation + env[ORCHESTRATION_COMPATIBILITY_ATTACHMENT_ENV] = args.runtimeAuthority.attachmentId + } env.ELECTRON_RUN_AS_NODE = '1' return env } @@ -177,7 +219,8 @@ export async function runHostOrcaCliPassthrough( hostEnv, remoteEnv: request.env, userDataPath, - remoteCwd: request.cwd + remoteCwd: request.cwd, + runtimeAuthority: request.runtimeAuthority }) return await new Promise((resolve, reject) => { diff --git a/src/main/ssh/ssh-remote-cli-in-process-result.ts b/src/main/ssh/ssh-remote-cli-in-process-result.ts new file mode 100644 index 00000000000..a35512e3451 --- /dev/null +++ b/src/main/ssh/ssh-remote-cli-in-process-result.ts @@ -0,0 +1,40 @@ +import type { RpcResponse } from '../runtime/rpc/core' +import type { ParsedRemoteCli } from './ssh-remote-cli-argument-error' +import type { RemoteOrcaCliResult } from './ssh-remote-cli-host-passthrough' +import { formatRemoteCli } from './ssh-remote-cli-format' +import { + formatRemoteOrchestrationAsk, + getRemoteCliExitCode +} from './ssh-remote-orchestration-ask-output' +import { formatRemoteOrchestrationCheck } from './ssh-remote-orchestration-check-output' +import { getRemoteCliPostOutput } from './ssh-remote-orchestration-post-output' +import { resolveRemoteCliHandle } from './ssh-remote-cli-args' + +export function formatInProcessRemoteCliResult( + parsed: ParsedRemoteCli, + env: Record, + response: RpcResponse, + json: boolean +): RemoteOrcaCliResult { + const command = parsed.commandPath.join(' ') + const formatted = + command === 'orchestration check' + ? formatRemoteOrchestrationCheck( + response, + json, + resolveRemoteCliHandle(parsed.flags, env, 'terminal'), + parsed.flags.has('format') + ) + : command === 'orchestration ask' + ? formatRemoteOrchestrationAsk(response, json) + : json + ? { stdout: `${JSON.stringify(response, null, 2)}\n`, stderr: '' } + : formatRemoteCli(response) + const postOutput = getRemoteCliPostOutput(parsed, env, response) + return { + stdout: formatted.stdout, + stderr: formatted.stderr, + exitCode: getRemoteCliExitCode(command, response), + ...(postOutput ? { postOutput } : {}) + } +} diff --git a/src/main/ssh/ssh-remote-orca-cli.test.ts b/src/main/ssh/ssh-remote-orca-cli.test.ts index 276eaac6500..f64a9d7bc26 100644 --- a/src/main/ssh/ssh-remote-orca-cli.test.ts +++ b/src/main/ssh/ssh-remote-orca-cli.test.ts @@ -24,7 +24,6 @@ const LEGACY_FALLBACK_OPTIONS: HostCliPassthroughOptions = { userDataPath: '/host/user-data', entryExists: () => false } - type FakeChild = EventEmitter & { stdout: EventEmitter stderr: EventEmitter @@ -80,6 +79,7 @@ describe('runRemoteOrcaCli', () => { } } }), + getLegacyAdoption: vi.fn(() => undefined), getActiveDispatchForIdentity: vi.fn(() => undefined), getCurrentRunForPane: vi.fn(() => undefined), findActiveRemoteAttachmentForPane: vi.fn(() => undefined) diff --git a/src/main/ssh/ssh-remote-orca-cli.ts b/src/main/ssh/ssh-remote-orca-cli.ts index caf1edb0e4a..acd0c94cfaf 100644 --- a/src/main/ssh/ssh-remote-orca-cli.ts +++ b/src/main/ssh/ssh-remote-orca-cli.ts @@ -1,10 +1,11 @@ import type { CliStatusResult, RuntimeStatus } from '../../shared/runtime-types' +import { randomUUID } from 'node:crypto' import type { RuntimeOrchestrationEnvelope } from '../../shared/runtime-rpc-envelope' +import { readOrchestrationCompatibilityEvidence } from '../../shared/orchestration-compatibility-evidence' import { ORCHESTRATION_CONTRACT_VERSION } from '../../shared/protocol-version' import { RpcDispatcher } from '../runtime/rpc/dispatcher' import type { RpcResponse } from '../runtime/rpc/core' import type { OrcaRuntimeService } from '../runtime/orca-runtime' -import { formatRemoteCli } from './ssh-remote-cli-format' import { HostCliUnavailableError, runHostOrcaCliPassthrough, @@ -24,9 +25,9 @@ import { buildRemoteCliError } from './ssh-remote-cli-error-response' import { getRemoteLinearHelp, tryDispatchRemoteLinearCli } from './ssh-remote-linear-cli' import { getRemoteOrchestrationPayload, - hasRemoteLifecycleRejection, resolveRemoteOrchestrationSender } from './ssh-remote-orchestration-send' +import { formatInProcessRemoteCliResult } from './ssh-remote-cli-in-process-result' export type { RemoteOrcaCliRequest, RemoteOrcaCliResult } from './ssh-remote-cli-host-passthrough' @@ -49,6 +50,7 @@ export async function runRemoteOrcaCli( ): Promise { const parsed = parseRemoteCliArgs(request.argv) const json = parsed.flags.has('json') + const command = parsed.commandPath.join(' ') const interactiveMessage = HOST_INTERACTIVE_COMMANDS[parsed.commandPath[0] ?? ''] if (interactiveMessage) { @@ -62,6 +64,17 @@ export async function runRemoteOrcaCli( return { stdout: '', stderr: `${interactiveMessage}\n`, exitCode: 1 } } + if (command === 'orchestration check' || command === 'orchestration ask') { + // Why: compatibility ACKs must wait until relay stdout is observable; a host CLI child can only flush into main's capture pipe. + return await runLegacyRemoteOrcaCli( + runtime, + request, + parsed, + json, + new HostCliUnavailableError('output-ordered orchestration bridge required') + ) + } + let passthroughFailure: HostCliUnavailableError | null = null try { return await runHostOrcaCliPassthrough(request, passthroughOptions) @@ -96,18 +109,10 @@ async function runLegacyRemoteOrcaCli( parsed, request.env, request.stdin, - passthroughFailure.message + passthroughFailure.message, + request.runtimeAuthority ) - const formatted = json - ? { stdout: `${JSON.stringify(response, null, 2)}\n`, stderr: '' } - : formatRemoteCli(response) - return { - stdout: formatted.stdout, - stderr: formatted.stderr, - // Why: the legacy SSH bridge bypasses the local CLI handler that turns - // a persisted lifecycle rejection into an unsuccessful command. - exitCode: response.ok && !hasRemoteLifecycleRejection(response.result) ? 0 : 1 - } + return formatInProcessRemoteCliResult(parsed, request.env, response, json) } catch (err) { const message = err instanceof Error ? err.message : String(err) const code = @@ -134,9 +139,23 @@ async function dispatchRemoteCli( parsed: ParsedRemoteCli, env: Record, stdin: string | undefined, - passthroughFailureReason: string + passthroughFailureReason: string, + runtimeAuthority: RemoteOrcaCliRequest['runtimeAuthority'] ): Promise { const command = parsed.commandPath.join(' ') + const inheritedEvidence = readOrchestrationCompatibilityEvidence(env) + const orchestrationCompatibilityEvidence = runtimeAuthority + ? { ...inheritedEvidence, host: runtimeAuthority } + : inheritedEvidence + const compatibilityEnvelope: RuntimeOrchestrationEnvelope = { + compatibilityInvocationId: randomUUID(), + orchestrationRequestId: + optionalRemoteCliString(parsed.flags, 'retry-request') ?? + (command === 'orchestration check' || command === 'orchestration ask' + ? randomUUID() + : undefined), + orchestrationCompatibilityEvidence + } const linearResponse = await tryDispatchRemoteLinearCli(dispatcher, parsed, env, stdin) if (linearResponse) { return linearResponse @@ -187,33 +206,74 @@ async function dispatchRemoteCli( senderPaneKey: env.ORCA_PANE_KEY || undefined }, { - orchestrationCapability: optionalRemoteCliString(parsed.flags, 'dispatch-capability'), - orchestrationRequestId: optionalRemoteCliString(parsed.flags, 'retry-request') + ...compatibilityEnvelope, + orchestrationCapability: optionalRemoteCliString(parsed.flags, 'dispatch-capability') } ) } case 'orchestration check': - return await call(dispatcher, 'orchestration.check', { - terminal: resolveRemoteCliHandle(parsed.flags, env, 'terminal'), - terminalPaneKey: parsed.flags.has('terminal') ? undefined : env.ORCA_PANE_KEY || undefined, - unread: parsed.flags.has('unread') ? true : undefined, - all: parsed.flags.has('all') ? true : undefined, - types: optionalRemoteCliString(parsed.flags, 'types'), - inject: parsed.flags.has('inject') ? true : undefined, - wait: parsed.flags.has('wait') ? true : undefined, - timeoutMs: optionalRemoteCliNumber(parsed.flags, 'timeout-ms') - }) + return await call( + dispatcher, + 'orchestration.check', + { + terminal: resolveRemoteCliHandle(parsed.flags, env, 'terminal'), + terminalPaneKey: parsed.flags.has('terminal') + ? undefined + : env.ORCA_PANE_KEY || undefined, + unread: parsed.flags.has('unread') ? true : parsed.flags.has('peek') ? false : undefined, + peek: parsed.flags.has('peek') ? true : undefined, + all: parsed.flags.has('all') ? true : undefined, + types: optionalRemoteCliString(parsed.flags, 'types'), + format: parsed.flags.has('format') ? true : undefined, + inject: parsed.flags.has('inject') ? true : undefined, + compatibilityCliCommand: 'orca', + run: optionalRemoteCliString(parsed.flags, 'run'), + ack: optionalRemoteCliString(parsed.flags, 'ack'), + wait: parsed.flags.has('wait') ? true : undefined, + timeoutMs: optionalRemoteCliNumber(parsed.flags, 'timeout-ms') + }, + compatibilityEnvelope + ) + case 'orchestration ask': + return await call( + dispatcher, + 'orchestration.ask', + { + to: optionalRemoteCliString(parsed.flags, 'to'), + question: optionalRemoteCliString(parsed.flags, 'question'), + resume: optionalRemoteCliString(parsed.flags, 'resume'), + options: optionalRemoteCliString(parsed.flags, 'options'), + timeoutMs: optionalRemoteCliNumber(parsed.flags, 'timeout-ms'), + from: resolveRemoteCliHandle(parsed.flags, env, 'from'), + run: optionalRemoteCliString(parsed.flags, 'run'), + compatibilityCliCommand: 'orca' + }, + { + ...compatibilityEnvelope, + orchestrationCapability: optionalRemoteCliString(parsed.flags, 'dispatch-capability') + } + ) case 'orchestration reply': - return await call(dispatcher, 'orchestration.reply', { - id: requiredRemoteCliString(parsed.flags, 'id'), - body: requiredRemoteCliString(parsed.flags, 'body'), - from: resolveRemoteCliHandle(parsed.flags, env, 'from') - }) + return await call( + dispatcher, + 'orchestration.reply', + { + id: requiredRemoteCliString(parsed.flags, 'id'), + body: requiredRemoteCliString(parsed.flags, 'body'), + from: resolveRemoteCliHandle(parsed.flags, env, 'from') + }, + compatibilityEnvelope + ) case 'orchestration inbox': - return await call(dispatcher, 'orchestration.inbox', { - limit: optionalRemoteCliNumber(parsed.flags, 'limit'), - terminal: optionalRemoteCliString(parsed.flags, 'terminal') - }) + return await call( + dispatcher, + 'orchestration.inbox', + { + limit: optionalRemoteCliNumber(parsed.flags, 'limit'), + terminal: optionalRemoteCliString(parsed.flags, 'terminal') + }, + compatibilityEnvelope + ) default: // Why: only reachable when the full host CLI could not be launched; // include that root cause so users can fix the install instead of @@ -239,6 +299,9 @@ async function call( orchestrationContractVersion: method.startsWith('orchestration.') ? ORCHESTRATION_CONTRACT_VERSION : undefined, - orchestrationRequestId: envelope?.orchestrationRequestId + orchestrationRequestId: envelope?.orchestrationRequestId, + compatibilityInvocationId: + envelope?.orchestrationRequestId ?? envelope?.compatibilityInvocationId, + orchestrationCompatibilityEvidence: envelope?.orchestrationCompatibilityEvidence }) } diff --git a/src/main/ssh/ssh-remote-orchestration-ask-output.ts b/src/main/ssh/ssh-remote-orchestration-ask-output.ts new file mode 100644 index 00000000000..d57493db5ce --- /dev/null +++ b/src/main/ssh/ssh-remote-orchestration-ask-output.ts @@ -0,0 +1,66 @@ +import type { RpcResponse } from '../runtime/rpc/core' +import { formatRemoteCli } from './ssh-remote-cli-format' +import { hasRemoteLifecycleRejection } from './ssh-remote-orchestration-send' + +export function formatRemoteOrchestrationAsk( + response: RpcResponse, + json: boolean +): { stdout: string; stderr: string } { + if (!response.ok || !isRecord(response.result)) { + return json + ? { stdout: `${JSON.stringify(response, null, 2)}\n`, stderr: '' } + : formatRemoteCli(response) + } + if (json) { + return { stdout: `${JSON.stringify(response.result)}\n`, stderr: '' } + } + if (isRecord(response.result.legacyCompatibility)) { + const compatibility = response.result.legacyCompatibility + if (compatibility.resumeRequired === true && typeof compatibility.resumeCommand === 'string') { + return { + stdout: + `Question ${String(response.result.messageId ?? 'unknown')} committed.\n` + + `Resume with: ${compatibility.resumeCommand}\n`, + stderr: '' + } + } + } + const answer = typeof response.result.answer === 'string' ? response.result.answer : '' + const thread = typeof response.result.threadId === 'string' ? response.result.threadId : 'unknown' + const timeoutMs = + typeof response.result.timeoutMs === 'number' ? response.result.timeoutMs : undefined + const stderr = response.result.timedOut + ? `ask timeout after ${timeoutMs ?? 0}ms (thread ${thread})\n` + : response.result.cancelled + ? response.result.connectionLost + ? `ask connection closed (question ${String(response.result.messageId ?? 'unknown')})\n` + : `ask cancelled (question ${String(response.result.messageId ?? 'unknown')})\n` + : '' + return { stdout: answer ? `${answer}\n` : '', stderr } +} + +export function getRemoteCliExitCode(command: string, response: RpcResponse): number { + if (!response.ok || hasRemoteLifecycleRejection(response.result)) { + return 1 + } + if ( + command === 'orchestration ask' && + isRecord(response.result) && + isRecord(response.result.legacyCompatibility) && + response.result.legacyCompatibility.resumeRequired === true + ) { + return 75 + } + if ( + command === 'orchestration ask' && + isRecord(response.result) && + (response.result.timedOut === true || response.result.cancelled === true) + ) { + return 1 + } + return 0 +} + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) +} diff --git a/src/main/ssh/ssh-remote-orchestration-check-output.ts b/src/main/ssh/ssh-remote-orchestration-check-output.ts new file mode 100644 index 00000000000..78ec3f4c544 --- /dev/null +++ b/src/main/ssh/ssh-remote-orchestration-check-output.ts @@ -0,0 +1,48 @@ +import { + formatOrchestrationCheckText, + prepareOrchestrationCheckOutput, + type OrchestrationCheckOutput, + type OrchestrationMessageSummary +} from '../../shared/orchestration-check-output' +import type { RpcResponse } from '../runtime/rpc/core' +import { formatRemoteCli } from './ssh-remote-cli-format' + +export function formatRemoteOrchestrationCheck( + response: RpcResponse, + json: boolean, + terminal: string, + formattedRequested: boolean +): { stdout: string; stderr: string } { + if (!response.ok || !isRecord(response.result)) { + return json + ? { stdout: `${JSON.stringify(response, null, 2)}\n`, stderr: '' } + : formatRemoteCli(response) + } + const messages = Array.isArray(response.result.messages) + ? response.result.messages.filter(isMessageSummary) + : [] + const count = typeof response.result.count === 'number' ? response.result.count : messages.length + const result = prepareOrchestrationCheckOutput( + { ...(response.result as OrchestrationCheckOutput), messages, count }, + terminal, + formattedRequested + ) + if (json) { + return { + stdout: `${JSON.stringify({ ...response, result }, null, 2)}\n`, + stderr: '' + } + } + return { + stdout: `${formatOrchestrationCheckText(result, terminal)}\n`, + stderr: '' + } +} + +function isMessageSummary(value: unknown): value is OrchestrationMessageSummary { + return isRecord(value) && typeof value.id === 'string' && typeof value.from_handle === 'string' +} + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) +} diff --git a/src/main/ssh/ssh-remote-orchestration-compatibility.test.ts b/src/main/ssh/ssh-remote-orchestration-compatibility.test.ts new file mode 100644 index 00000000000..45e6eb57f41 --- /dev/null +++ b/src/main/ssh/ssh-remote-orchestration-compatibility.test.ts @@ -0,0 +1,401 @@ +import { createHash } from 'node:crypto' +import { describe, expect, it, vi } from 'vitest' + +vi.mock('electron', () => ({ + app: { + isPackaged: false, + getAppPath: () => '/host/app' + } +})) +vi.mock('../persistence', () => ({ + getCanonicalUserDataPath: () => '/host/user-data' +})) + +import { OrchestrationDb } from '../runtime/orchestration/db' +import { OrcaRuntimeService } from '../runtime/orca-runtime' +import type Database from '../sqlite/sync-database' +import type { HostCliPassthroughOptions } from './ssh-remote-cli-host-passthrough' +import { runRemoteOrcaCli } from './ssh-remote-orca-cli' +import { acknowledgeRemoteOrcaCliPostOutput } from './ssh-remote-orchestration-post-output' + +const LEGACY_FALLBACK_OPTIONS: HostCliPassthroughOptions = { + execPath: '/host/electron', + cliEntryPath: '/host/app/out/cli/index.js', + userDataPath: '/host/user-data', + entryExists: () => false +} +const WORKER_HANDLE = 'term_legacy_ssh_worker' +const WORKER_PANE = 'tab_legacy_ssh:33333333-3333-4333-8333-333333333333' +const COORDINATOR_HANDLE = 'term_legacy_ssh_coord' +const COORDINATOR_PANE = 'tab_legacy_coord:44444444-4444-4444-8444-444444444444' +const WORKER_ENV = { + ORCA_TERMINAL_HANDLE: WORKER_HANDLE, + ORCA_PANE_KEY: WORKER_PANE, + ORCA_AGENT_LAUNCH_TOKEN: 'legacy-ssh-token' +} +const COORDINATOR_ENV = { + ORCA_TERMINAL_HANDLE: COORDINATOR_HANDLE, + ORCA_PANE_KEY: COORDINATOR_PANE, + ORCA_AGENT_LAUNCH_TOKEN: 'legacy-ssh-coordinator-token' +} +const RUNTIME_AUTHORITY = { + kind: 'ssh' as const, + targetId: 'saved-target', + connectionIncarnation: 'connection-1', + attachmentId: 'attachment-1' +} + +function createLegacyRuntime() { + const db = new OrchestrationDb(':memory:') + const run = db.createRun({ + objective: 'Adopted legacy SSH work', + coordinatorHandle: COORDINATOR_HANDLE, + coordinatorPaneKey: COORDINATOR_PANE + }) + const task = db.createTask({ + spec: 'legacy SSH assignment', + runId: run.id, + createdByTerminalHandle: COORDINATOR_HANDLE + }) + const dispatch = db.createDispatchContext(task.id, WORKER_HANDLE, WORKER_PANE) + const sqlite = (db as unknown as { db: Database.Database }).db + sqlite + .prepare( + `UPDATE dispatch_contexts + SET contract_version = 0, process_incarnation = 'process-1' + WHERE id = ?` + ) + .run(dispatch.id) + sqlite + .prepare( + `INSERT INTO legacy_adoptions (source_run_id, adopted_run_id, scheduler_state_lost) + VALUES ('run_legacy_local', ?, 1)` + ) + .run(run.id) + sqlite + .prepare( + `UPDATE runs + SET coordinator_handle = NULL, coordinator_pane_key = NULL, consumer_generation = 0 + WHERE id = ?` + ) + .run(run.id) + + const runtime = new OrcaRuntimeService() + runtime.setOrchestrationDb(db) + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === WORKER_HANDLE ? WORKER_PANE : handle === COORDINATOR_HANDLE ? COORDINATOR_PANE : null + ) + vi.spyOn(runtime, 'verifyOrchestrationCompatibilityCaller').mockImplementation((evidence) => { + const worker = + evidence?.terminalHandle === WORKER_HANDLE && + evidence.paneKey === WORKER_PANE && + evidence.launchToken === WORKER_ENV.ORCA_AGENT_LAUNCH_TOKEN + const coordinator = + evidence?.terminalHandle === COORDINATOR_HANDLE && + evidence.paneKey === COORDINATOR_PANE && + evidence.launchToken === COORDINATOR_ENV.ORCA_AGENT_LAUNCH_TOKEN + if ( + (!worker && !coordinator) || + evidence.host?.kind !== 'ssh' || + evidence.host.targetId !== RUNTIME_AUTHORITY.targetId || + evidence.host.connectionIncarnation !== RUNTIME_AUTHORITY.connectionIncarnation + ) { + return null + } + const identity = worker + ? { + terminalHandle: WORKER_HANDLE, + paneKey: WORKER_PANE, + launchToken: WORKER_ENV.ORCA_AGENT_LAUNCH_TOKEN + } + : { + terminalHandle: COORDINATOR_HANDLE, + paneKey: COORDINATOR_PANE, + launchToken: COORDINATOR_ENV.ORCA_AGENT_LAUNCH_TOKEN + } + return { + hostScope: { kind: 'ssh', targetId: RUNTIME_AUTHORITY.targetId }, + terminalHandle: identity.terminalHandle, + paneKey: identity.paneKey, + processIncarnation: 'process-1', + launchTokenHash: createHash('sha256').update(identity.launchToken).digest('hex') + } + }) + vi.spyOn(runtime, 'notifyMessageArrived').mockImplementation(() => {}) + return { db, runtime, run: db.getRun(run.id)!, dispatch } +} + +describe('legacy SSH orchestration fallback', () => { + it('acknowledges a consuming check only after remote output', async () => { + const { db, runtime, run } = createLegacyRuntime() + const message = db.insertMessage({ + runId: run.id, + deliveryContract: 'legacy_direct', + from: COORDINATOR_HANDLE, + to: WORKER_HANDLE, + subject: 'retained SSH mail' + }) + const request = { + argv: ['orchestration', 'check', '--unread', '--inject', '--json'], + cwd: '/home/alice/repo', + env: WORKER_ENV, + runtimeAuthority: RUNTIME_AUTHORITY + } + + try { + const first = await runRemoteOrcaCli(runtime, request, LEGACY_FALLBACK_OPTIONS) + const replay = await runRemoteOrcaCli(runtime, request, LEGACY_FALLBACK_OPTIONS) + + expect(JSON.parse(first.stdout)).toMatchObject({ + result: { + messages: [{ id: message.id }], + formatted: expect.stringContaining('retained SSH mail') + } + }) + expect(first.postOutput).toEqual({ + kind: 'legacy_check_ack', + terminal: WORKER_HANDLE, + messageIds: [message.id] + }) + expect(JSON.parse(replay.stdout)).toMatchObject({ + result: { messages: [{ id: message.id }] } + }) + expect(db.getMessageById(message.id)?.read).toBe(0) + + await acknowledgeRemoteOrcaCliPostOutput(runtime, { + postOutput: first.postOutput!, + env: WORKER_ENV, + runtimeAuthority: RUNTIME_AUTHORITY + }) + const afterAck = await runRemoteOrcaCli(runtime, request, LEGACY_FALLBACK_OPTIONS) + + expect(JSON.parse(afterAck.stdout)).toMatchObject({ result: { messages: [], count: 0 } }) + expect(afterAck.postOutput).toBeUndefined() + expect(db.getMessageById(message.id)?.read).toBe(1) + } finally { + db.close() + } + }) + + it('keeps peek formatted and non-consuming', async () => { + const { db, runtime, run } = createLegacyRuntime() + const message = db.insertMessage({ + runId: run.id, + deliveryContract: 'legacy_direct', + from: COORDINATOR_HANDLE, + to: WORKER_HANDLE, + subject: 'inspect retained SSH mail' + }) + + try { + const peek = await runRemoteOrcaCli( + runtime, + { + argv: ['orchestration', 'check', '--peek', '--format', '--json'], + cwd: '/home/alice/repo', + env: WORKER_ENV, + runtimeAuthority: RUNTIME_AUTHORITY + }, + LEGACY_FALLBACK_OPTIONS + ) + + const output = JSON.parse(peek.stdout) as { + result: { formatted: string } + } + expect(output).toMatchObject({ + result: { + messages: [{ id: message.id }], + formatted: expect.stringContaining('inspect retained SSH mail'), + legacyCompatibility: { readOnly: true } + } + }) + expect(output.result.formatted).toContain(`${message.id} [legacy, read-only]`) + expect(output.result.formatted).toContain( + 'Inspection only: reply and acknowledgment are unavailable.' + ) + expect(output.result.formatted).not.toContain('orchestration reply') + expect(peek.postOutput).toBeUndefined() + expect(db.getMessageById(message.id)?.read).toBe(0) + } finally { + db.close() + } + }) + + it('reads and acknowledges current Run delivery', async () => { + const { db, runtime, run } = createLegacyRuntime() + const message = db.insertMessage({ + runId: run.id, + from: WORKER_HANDLE, + to: `run:${run.id}`, + subject: 'current Run mail' + }) + const baseRequest = { + cwd: '/home/alice/repo', + env: COORDINATOR_ENV, + runtimeAuthority: RUNTIME_AUTHORITY + } + const spawn = vi.fn() + const hostCliAvailable = { + ...LEGACY_FALLBACK_OPTIONS, + entryExists: () => true, + spawn: spawn as HostCliPassthroughOptions['spawn'] + } + + try { + const checked = await runRemoteOrcaCli( + runtime, + { + ...baseRequest, + argv: ['orchestration', 'check', '--run', run.id] + }, + hostCliAvailable + ) + expect(checked.stdout).toContain('Delivery ') + expect(checked.stdout).toContain(`${message.id} [status] from=${WORKER_HANDLE}`) + expect(spawn).not.toHaveBeenCalled() + + const checkedJson = await runRemoteOrcaCli( + runtime, + { + ...baseRequest, + argv: ['orchestration', 'check', '--run', run.id, '--json'] + }, + hostCliAvailable + ) + const result = JSON.parse(checkedJson.stdout) as { + result: { deliveryId: string; messages: { id: string }[] } + } + expect(result.result.messages).toEqual([expect.objectContaining({ id: message.id })]) + + const acknowledged = await runRemoteOrcaCli( + runtime, + { + ...baseRequest, + argv: [ + 'orchestration', + 'check', + '--run', + run.id, + '--ack', + result.result.deliveryId, + '--json' + ] + }, + hostCliAvailable + ) + + expect(JSON.parse(acknowledged.stdout)).toMatchObject({ + result: { acknowledged: result.result.deliveryId } + }) + expect(db.getMessageById(message.id)?.read).toBe(1) + } finally { + db.close() + } + }) + + it('replays an SSH legacy ask by retry request without creating another question', async () => { + const { db, runtime } = createLegacyRuntime() + const argv = [ + 'orchestration', + 'ask', + '--to', + COORDINATOR_HANDLE, + '--question', + 'Continue?', + '--timeout-ms', + '1', + '--retry-request', + 'ssh-question-1', + '--json' + ] + const request = { + argv, + cwd: '/home/alice/repo', + env: WORKER_ENV, + runtimeAuthority: RUNTIME_AUTHORITY + } + + try { + const first = await runRemoteOrcaCli(runtime, request, LEGACY_FALLBACK_OPTIONS) + const replay = await runRemoteOrcaCli(runtime, request, LEGACY_FALLBACK_OPTIONS) + const sqlite = (db as unknown as { db: Database.Database }).db + + const firstResult = JSON.parse(first.stdout) as { messageId: string; timedOut: boolean } + const replayResult = JSON.parse(replay.stdout) as { messageId: string; timedOut: boolean } + expect(first).toMatchObject({ exitCode: 1 }) + expect(firstResult).toMatchObject({ messageId: expect.any(String), timedOut: true }) + expect(replay).toMatchObject({ exitCode: 1 }) + expect(replayResult).toMatchObject({ messageId: firstResult.messageId, timedOut: true }) + expect( + ( + sqlite.prepare('SELECT COUNT(*) AS count FROM question_threads').get() as { + count: number + } + ).count + ).toBe(1) + } finally { + db.close() + } + }) + + it('resumes and acknowledges a question answer', async () => { + const { db, runtime, run, dispatch } = createLegacyRuntime() + const pending = db.createQuestion({ + runId: run.id, + dispatchId: dispatch.id, + askerHandle: WORKER_HANDLE, + question: 'Proceed?' + }) + const answer = db.answerQuestion({ + messageId: pending.question.message_id, + runId: run.id, + consumerGeneration: run.consumer_generation, + body: 'yes' + }) + const sqlite = (db as unknown as { db: Database.Database }).db + sqlite + .prepare( + `UPDATE messages + SET delivery_contract = 'legacy_direct', read = 0 + WHERE id = ?` + ) + .run(answer.message.id) + + try { + const result = await runRemoteOrcaCli( + runtime, + { + argv: ['orchestration', 'ask', '--resume', pending.question.message_id, '--json'], + cwd: '/home/alice/repo', + env: WORKER_ENV, + runtimeAuthority: RUNTIME_AUTHORITY + }, + LEGACY_FALLBACK_OPTIONS + ) + + expect(result.exitCode).toBe(0) + expect(JSON.parse(result.stdout)).toMatchObject({ + answer: 'yes', + messageId: pending.question.message_id, + timedOut: false + }) + expect(result.postOutput).toEqual({ + kind: 'legacy_question_ack', + terminal: WORKER_HANDLE, + questionId: pending.question.message_id, + answerMessageId: answer.message.id + }) + expect(db.getMessageById(answer.message.id)?.read).toBe(0) + + await acknowledgeRemoteOrcaCliPostOutput(runtime, { + postOutput: result.postOutput!, + env: WORKER_ENV, + runtimeAuthority: RUNTIME_AUTHORITY + }) + + expect(db.getMessageById(answer.message.id)?.read).toBe(1) + } finally { + db.close() + } + }) +}) diff --git a/src/main/ssh/ssh-remote-orchestration-post-output.ts b/src/main/ssh/ssh-remote-orchestration-post-output.ts new file mode 100644 index 00000000000..5d833651350 --- /dev/null +++ b/src/main/ssh/ssh-remote-orchestration-post-output.ts @@ -0,0 +1,143 @@ +import { randomUUID } from 'node:crypto' +import { readOrchestrationCompatibilityEvidence } from '../../shared/orchestration-compatibility-evidence' +import { ORCHESTRATION_CONTRACT_VERSION } from '../../shared/protocol-version' +import type { OrcaRuntimeService } from '../runtime/orca-runtime' +import type { RpcResponse } from '../runtime/rpc/core' +import { RpcDispatcher } from '../runtime/rpc/dispatcher' +import type { + RemoteOrcaCliPostOutput, + RemoteOrcaCliRequest +} from './ssh-remote-cli-host-passthrough' +import { RemoteCliArgumentError, type ParsedRemoteCli } from './ssh-remote-cli-argument-error' +import { optionalRemoteCliString, resolveRemoteCliHandle } from './ssh-remote-cli-args' + +export async function acknowledgeRemoteOrcaCliPostOutput( + runtime: OrcaRuntimeService, + args: { + postOutput: RemoteOrcaCliPostOutput + env: Record + runtimeAuthority?: RemoteOrcaCliRequest['runtimeAuthority'] + } +): Promise { + const inheritedEvidence = readOrchestrationCompatibilityEvidence(args.env) + const orchestrationCompatibilityEvidence = args.runtimeAuthority + ? { ...inheritedEvidence, host: args.runtimeAuthority } + : inheritedEvidence + const params = + args.postOutput.kind === 'legacy_check_ack' + ? { + terminal: args.postOutput.terminal, + compatibilityAck: JSON.stringify({ + messageIds: args.postOutput.messageIds, + types: args.postOutput.types + }) + } + : { + terminal: args.postOutput.terminal, + compatibilityQuestionAck: JSON.stringify({ + questionId: args.postOutput.questionId, + answerMessageId: args.postOutput.answerMessageId + }) + } + const response = await new RpcDispatcher({ runtime }).dispatch({ + id: `remote-cli-post-output-${randomUUID()}`, + authToken: 'remote-cli', + method: 'orchestration.check', + params, + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + compatibilityInvocationId: randomUUID(), + orchestrationCompatibilityEvidence + }) + if (!response.ok) { + throw new Error(response.error.message) + } +} + +export function parseRemoteOrcaCliPostOutput(value: unknown): RemoteOrcaCliPostOutput { + if (!isRecord(value) || typeof value.kind !== 'string' || typeof value.terminal !== 'string') { + throw invalidPostOutput() + } + if ( + value.kind === 'legacy_check_ack' && + Array.isArray(value.messageIds) && + value.messageIds.every((id) => typeof id === 'string') && + (value.types === undefined || + (Array.isArray(value.types) && value.types.every((type) => typeof type === 'string'))) + ) { + return { + kind: value.kind, + terminal: value.terminal, + messageIds: value.messageIds, + ...(value.types === undefined ? {} : { types: value.types }) + } + } + if ( + value.kind === 'legacy_question_ack' && + typeof value.questionId === 'string' && + typeof value.answerMessageId === 'string' + ) { + return { + kind: value.kind, + terminal: value.terminal, + questionId: value.questionId, + answerMessageId: value.answerMessageId + } + } + throw invalidPostOutput() +} + +export function getRemoteCliPostOutput( + parsed: ParsedRemoteCli, + env: Record, + response: RpcResponse +): RemoteOrcaCliPostOutput | undefined { + if (!response.ok || !isRecord(response.result)) { + return undefined + } + const compatibility = response.result.legacyCompatibility + if (!isRecord(compatibility)) { + return undefined + } + const command = parsed.commandPath.join(' ') + if ( + command === 'orchestration check' && + Array.isArray(compatibility.ackMessageIds) && + compatibility.ackMessageIds.length > 0 && + compatibility.ackMessageIds.every((id) => typeof id === 'string') + ) { + const types = optionalRemoteCliString(parsed.flags, 'types') + ?.split(',') + .map((type) => type.trim()) + .filter(Boolean) + return { + kind: 'legacy_check_ack', + terminal: resolveRemoteCliHandle(parsed.flags, env, 'terminal'), + messageIds: compatibility.ackMessageIds, + ...(types ? { types } : {}) + } + } + const acknowledgement = compatibility.answerAcknowledgement + if ( + command === 'orchestration ask' && + response.result.answer !== null && + isRecord(acknowledgement) && + typeof acknowledgement.questionId === 'string' && + typeof acknowledgement.answerMessageId === 'string' + ) { + return { + kind: 'legacy_question_ack', + terminal: resolveRemoteCliHandle(parsed.flags, env, 'from'), + questionId: acknowledgement.questionId, + answerMessageId: acknowledgement.answerMessageId + } + } + return undefined +} + +function invalidPostOutput(): RemoteCliArgumentError { + return new RemoteCliArgumentError('invalid_argument', 'Invalid SSH CLI post-output action.') +} + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) +} diff --git a/src/main/startup/desktop-startup-ordering.test.ts b/src/main/startup/desktop-startup-ordering.test.ts index 6c5564e3e19..07f75b5dd31 100644 --- a/src/main/startup/desktop-startup-ordering.test.ts +++ b/src/main/startup/desktop-startup-ordering.test.ts @@ -73,6 +73,13 @@ describe('startup ordering', () => { expect(desktopStartup).not.toContain('await managedWslCliReconciliationReady') expect(barrier).toContain('managedWslCliStartupBarrierReady') expect(barrier).not.toContain('managedWslCliReconciliationReady') + expect(barrier).toContain("ipcMain.handle('app:recoverLegacyWorkerTerminalsForRendererStartup'") + expect(barrier).toContain('recoverLegacyWorkerTerminalsForRendererStartup({') + expect(barrier).toContain('localPtyProviderStartupReady,') + expect(barrier).toContain('await runtime?.refreshRestoredOrchestrationAuthority()') + expect(barrier).toContain( + 'return runtime?.reconcileLegacyWorkerTerminals({ materializeRenderer: true })' + ) }) it('exposes managed WSL reconciliation status to headless serve clients and diagnostics', () => { diff --git a/src/main/startup/legacy-worker-renderer-recovery.test.ts b/src/main/startup/legacy-worker-renderer-recovery.test.ts new file mode 100644 index 00000000000..9373a59b487 --- /dev/null +++ b/src/main/startup/legacy-worker-renderer-recovery.test.ts @@ -0,0 +1,117 @@ +import { describe, expect, it, vi } from 'vitest' +import { recoverLegacyWorkerTerminalsForRendererStartup } from './legacy-worker-renderer-recovery' + +describe('legacy worker renderer recovery', () => { + it('hydrates after bounded barriers while provider startup remains pending', async () => { + let resolveFirstWindow!: () => void + let resolveWslBarrier!: () => void + let resolveProvider!: () => void + const firstWindowReady = new Promise((resolve) => { + resolveFirstWindow = resolve + }) + const wslBarrierReady = new Promise((resolve) => { + resolveWslBarrier = resolve + }) + const providerReady = new Promise((resolve) => { + resolveProvider = resolve + }) + const reconcile = vi.fn().mockResolvedValue(undefined) + const startup = recoverLegacyWorkerTerminalsForRendererStartup({ + firstWindowStartupServicesReady: firstWindowReady, + managedWslCliStartupBarrierReady: wslBarrierReady, + localPtyProviderStartupReady: providerReady, + reconcile, + onDeferredRecoveryError: vi.fn() + }) + + resolveFirstWindow() + await Promise.resolve() + expect(reconcile).not.toHaveBeenCalled() + + resolveWslBarrier() + await startup + expect(reconcile).toHaveBeenCalledTimes(1) + + resolveProvider() + await vi.waitFor(() => expect(reconcile).toHaveBeenCalledTimes(2)) + }) + + it('retries after initial recovery when the provider is already ready', async () => { + const reconcile = vi.fn().mockResolvedValue(undefined) + + await recoverLegacyWorkerTerminalsForRendererStartup({ + firstWindowStartupServicesReady: Promise.resolve(), + managedWslCliStartupBarrierReady: Promise.resolve(), + localPtyProviderStartupReady: Promise.resolve(), + reconcile, + onDeferredRecoveryError: vi.fn() + }) + + await vi.waitFor(() => expect(reconcile).toHaveBeenCalledTimes(2)) + }) + + it('contains provider startup rejection after initial recovery', async () => { + const providerError = new Error('provider failed') + const reconcile = vi.fn().mockResolvedValue(undefined) + let reportError!: (error: unknown) => void + const reportedError = new Promise((resolve) => { + reportError = resolve + }) + + await recoverLegacyWorkerTerminalsForRendererStartup({ + firstWindowStartupServicesReady: Promise.resolve(), + managedWslCliStartupBarrierReady: Promise.resolve(), + localPtyProviderStartupReady: Promise.reject(providerError), + reconcile, + onDeferredRecoveryError: reportError + }) + + await expect(reportedError).resolves.toBe(providerError) + expect(reconcile).toHaveBeenCalledTimes(1) + }) + + it('contains deferred recovery rejection', async () => { + const recoveryError = new Error('recovery failed') + const reconcile = vi.fn().mockResolvedValueOnce(undefined).mockRejectedValueOnce(recoveryError) + let reportError!: (error: unknown) => void + const reportedError = new Promise((resolve) => { + reportError = resolve + }) + + await recoverLegacyWorkerTerminalsForRendererStartup({ + firstWindowStartupServicesReady: Promise.resolve(), + managedWslCliStartupBarrierReady: Promise.resolve(), + localPtyProviderStartupReady: Promise.resolve(), + reconcile, + onDeferredRecoveryError: reportError + }) + + await expect(reportedError).resolves.toBe(recoveryError) + expect(reconcile).toHaveBeenCalledTimes(2) + }) + + it('contains initial recovery rejection and still retries when the provider becomes ready', async () => { + const initialError = new Error('initial recovery failed') + let resolveProvider!: () => void + const providerReady = new Promise((resolve) => { + resolveProvider = resolve + }) + const reconcile = vi.fn().mockRejectedValueOnce(initialError).mockResolvedValueOnce(undefined) + const onDeferredRecoveryError = vi.fn() + + await expect( + recoverLegacyWorkerTerminalsForRendererStartup({ + firstWindowStartupServicesReady: Promise.resolve(), + managedWslCliStartupBarrierReady: Promise.resolve(), + localPtyProviderStartupReady: providerReady, + reconcile, + onDeferredRecoveryError + }) + ).resolves.toBeUndefined() + + expect(onDeferredRecoveryError).toHaveBeenCalledWith(initialError) + expect(reconcile).toHaveBeenCalledTimes(1) + resolveProvider() + await vi.waitFor(() => expect(reconcile).toHaveBeenCalledTimes(2)) + }) +}) diff --git a/src/main/startup/legacy-worker-renderer-recovery.ts b/src/main/startup/legacy-worker-renderer-recovery.ts new file mode 100644 index 00000000000..fa149c189bd --- /dev/null +++ b/src/main/startup/legacy-worker-renderer-recovery.ts @@ -0,0 +1,33 @@ +type LegacyWorkerRendererRecoveryOptions = { + firstWindowStartupServicesReady: Promise + managedWslCliStartupBarrierReady: Promise + localPtyProviderStartupReady: Promise + reconcile: () => Promise | undefined + onDeferredRecoveryError: (error: unknown) => void +} + +export async function recoverLegacyWorkerTerminalsForRendererStartup( + options: LegacyWorkerRendererRecoveryOptions +): Promise { + const providerStartupResult = options.localPtyProviderStartupReady.then( + () => ({ ok: true as const }), + (error: unknown) => ({ ok: false as const, error }) + ) + await Promise.all([ + options.firstWindowStartupServicesReady, + options.managedWslCliStartupBarrierReady + ]) + void providerStartupResult + .then(async (result) => { + if (!result.ok) { + throw result.error + } + await options.reconcile() + }) + .catch(options.onDeferredRecoveryError) + try { + await options.reconcile() + } catch (error) { + options.onDeferredRecoveryError(error) + } +} diff --git a/src/main/window/attach-main-window-services.test.ts b/src/main/window/attach-main-window-services.test.ts index 833f158fc56..bc1e7ae258d 100644 --- a/src/main/window/attach-main-window-services.test.ts +++ b/src/main/window/attach-main-window-services.test.ts @@ -829,4 +829,73 @@ describe('attachMainWindowServices', () => { await expect(revealPromise).resolves.toEqual({ tabId: 'tab-1', title: 'SSH tmux' }) expect(removeListenerMock).toHaveBeenCalledWith('terminal:tabCreateReply', handler) }) + + it('requires an exact renderer identity receipt for recovered worker reveals', async () => { + const sendMock = vi.fn() + const mainWindow = createMainWindow({ send: sendMock }) + const runtime = createRuntime() + + attachMainWindowServices(mainWindow as never, createStore(), runtime as never) + + const notifier = runtime.setNotifier.mock.calls[0][0] as { + revealTerminalSession: ( + worktreeId: string, + opts: { + ptyId: string + tabId: string + leafId: string + expectedProcessIdentity: { terminalHandle: string; incarnationId: string } + } + ) => Promise + } + const opts = { + ptyId: 'pty-worker', + tabId: 'tab-worker', + leafId: 'leaf-worker', + expectedProcessIdentity: { + terminalHandle: 'term_worker', + incarnationId: 'inc-worker' + } + } + const mismatch = notifier.revealTerminalSession('worktree-1', opts) + const mismatchPayload = sendMock.mock.calls.at(-1)?.[1] + const mismatchHandler = onMock.mock.calls.findLast( + ([channel]) => channel === 'terminal:tabCreateReply' + )?.[1] + mismatchHandler?.( + { sender: mainWindow.webContents }, + { + requestId: mismatchPayload.requestId, + tabId: 'tab-worker', + identity: { + worktreeId: 'worktree-1', + tabId: 'tab-worker', + leafId: 'leaf-worker', + ptyId: 'pty-replacement' + } + } + ) + await expect(mismatch).rejects.toThrow('terminal_reveal_identity_mismatch') + + const exact = notifier.revealTerminalSession('worktree-1', opts) + const exactPayload = sendMock.mock.calls.at(-1)?.[1] + const exactHandler = onMock.mock.calls.findLast( + ([channel]) => channel === 'terminal:tabCreateReply' + )?.[1] + const identity = { + worktreeId: 'worktree-1', + tabId: 'tab-worker', + leafId: 'leaf-worker', + ptyId: 'pty-worker' + } + exactHandler?.( + { sender: mainWindow.webContents }, + { requestId: exactPayload.requestId, tabId: 'tab-worker', identity } + ) + await expect(exact).resolves.toEqual({ + tabId: 'tab-worker', + title: undefined, + identity + }) + }) }) diff --git a/src/main/window/attach-main-window-services.ts b/src/main/window/attach-main-window-services.ts index 21a5654e587..4d4448c04bf 100644 --- a/src/main/window/attach-main-window-services.ts +++ b/src/main/window/attach-main-window-services.ts @@ -49,6 +49,7 @@ import type { RuntimeMarkdownSaveTabResult } from '../../shared/mobile-markdown-document' import type { RuntimeMobileSessionTabMove } from '../../shared/runtime-types' +import type { TerminalTabCreateReply } from '../../shared/terminal-reveal-identity' import { isNativeFileDropPayload, type NativeFileDropPayload } from '../../shared/native-file-drop' import { requestMobileMarkdownFromRenderer } from './mobile-markdown-request-relay' import { requestTerminalTabCloseFromRenderer } from './terminal-tab-close-request-relay' @@ -347,14 +348,20 @@ function registerRuntimeWindowLifecycle( revealTerminalSession: (worktreeId, opts) => new Promise((resolve, reject) => { const requestId = randomUUID() + const expectedIdentity = opts.expectedProcessIdentity + ? opts.tabId && opts.leafId + ? { worktreeId, tabId: opts.tabId, leafId: opts.leafId, ptyId: opts.ptyId } + : null + : undefined + if (expectedIdentity === null) { + reject(new Error('terminal_reveal_identity_required')) + return + } const timer = setTimeout(() => { ipcMain.removeListener('terminal:tabCreateReply', handler) reject(new Error('Terminal reveal timed out')) }, 10_000) - const handler = ( - event: Electron.IpcMainEvent, - reply: { requestId: string; tabId?: string; title?: string; error?: string } - ): void => { + const handler = (event: Electron.IpcMainEvent, reply: TerminalTabCreateReply): void => { // Why: requestId is renderer-supplied, so only the targeted main window may satisfy the reveal. if (event.sender !== mainWindow.webContents || reply.requestId !== requestId) { return @@ -365,7 +372,22 @@ function registerRuntimeWindowLifecycle( reject(new Error(reply.error)) return } - resolve({ tabId: reply.tabId!, title: reply.title }) + if ( + expectedIdentity && + (!reply.identity || + reply.identity.worktreeId !== expectedIdentity.worktreeId || + reply.identity.tabId !== expectedIdentity.tabId || + reply.identity.leafId !== expectedIdentity.leafId || + reply.identity.ptyId !== expectedIdentity.ptyId) + ) { + reject(new Error('terminal_reveal_identity_mismatch')) + return + } + resolve({ + tabId: reply.tabId!, + title: reply.title, + ...(reply.identity ? { identity: reply.identity } : {}) + }) } ipcMain.on('terminal:tabCreateReply', handler) send('ui:createTerminal', { @@ -387,9 +409,16 @@ function registerRuntimeWindowLifecycle( ...(opts.splitDirection !== undefined ? { splitDirection: opts.splitDirection } : {}), ...(opts.splitTelemetrySource !== undefined ? { splitTelemetrySource: opts.splitTelemetrySource } - : {}) + : {}), + ...(opts.focus !== undefined ? { focus: opts.focus } : {}) }) }), + resolveLegacyWorkerTerminalRecovery: (paneKey, resolution, ptyId) => + send('agentStatus:legacyWorkerTerminalRecovery', { + paneKey, + resolution, + ...(ptyId ? { ptyId } : {}) + }), splitTerminal: (tabId, paneRuntimeId, opts) => { send('ui:splitTerminal', { tabId, diff --git a/src/preload/api-types.ts b/src/preload/api-types.ts index 9431db3396a..b601ce39fac 100644 --- a/src/preload/api-types.ts +++ b/src/preload/api-types.ts @@ -18,6 +18,7 @@ import type { TerminalTabCloseRequest, TerminalTabCloseResponse } from '../shared/terminal-tab-close' +import type { TerminalTabCreateReply } from '../shared/terminal-reveal-identity' import type { LocalLogTailChangedPayload, LocalLogTailReadArgs, @@ -971,6 +972,8 @@ export type AppApi = { /** Resolves when the daemon PTY provider and hook receiver have either * started or failed open for the first BrowserWindow. */ awaitFirstWindowStartupServices: () => Promise + /** Reconciles legacy worker authority around persisted terminal reconnect. */ + recoverLegacyWorkerTerminalsForRendererStartup: () => Promise /** Emits a startup benchmark marker when ORCA_STARTUP_DIAGNOSTICS is enabled. */ startupDiagnostic: (event: string, details?: Record) => Promise /** macOS active input mode, or layout ID when no IME is selected (e.g. `com.apple.keylayout.PolishPro`). @@ -3135,6 +3138,7 @@ export type PreloadApi = { title?: string ptyId?: string activate?: boolean + focus?: boolean presentation?: RuntimeTerminalPresentation tabId?: string leafId?: string @@ -3149,12 +3153,7 @@ export type PreloadApi = { onRequestTerminalTabMount: ( callback: (data: { worktreeId: string; tabId?: string; ptyId?: string }) => void ) => () => void - replyTerminalCreate: (reply: { - requestId: string - tabId?: string - title?: string - error?: string - }) => void + replyTerminalCreate: (reply: TerminalTabCreateReply) => void onSplitTerminal: ( callback: (data: { tabId: string @@ -3517,6 +3516,13 @@ export type PreloadApi = { /** Listen for PTYs on a legacy numeric pane key that have registry-backed UUID pane proof. */ onMigrationUnsupported: (callback: (entry: MigrationUnsupportedPtyEntry) => void) => () => void onMigrationUnsupportedClear: (callback: (data: { ptyId: string }) => void) => () => void + onLegacyWorkerTerminalRecovery: ( + callback: (data: { + paneKey: string + resolution: 'adopted' | 'exited' | 'rolled_back' + ptyId?: string + }) => void + ) => () => void getMigrationUnsupportedSnapshot: () => Promise /** Drop a paneKey from the main-process hook cache and on-disk last-status file. Fire-and-forget. */ drop: (paneKey: string) => void diff --git a/src/preload/index.ts b/src/preload/index.ts index 8df862d0c4c..3388c8af33e 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -13,6 +13,7 @@ import type { CliInstallStatus } from '../shared/cli-install-types' import type { AgentHookInstallStatus } from '../shared/agent-hook-types' import type { CodexConfigSyncStatus } from '../shared/codex-config-sync-types' import type { TerminalPaneSplitSource } from '../shared/feature-education-telemetry' +import type { TerminalTabCreateReply } from '../shared/terminal-reveal-identity' import type { ProjectExecutionRuntimeResolution } from '../shared/project-execution-runtime' import type { StartupCommandDelivery } from '../shared/codex-startup-delivery' import type { @@ -487,6 +488,8 @@ const api = { }, awaitFirstWindowStartupServices: (): Promise => ipcRenderer.invoke('app:awaitFirstWindowStartupServices'), + recoverLegacyWorkerTerminalsForRendererStartup: (): Promise => + ipcRenderer.invoke('app:recoverLegacyWorkerTerminalsForRendererStartup'), startupDiagnostic: (event: string, details?: Record): Promise => startupDiagnosticsEnabled ? ipcRenderer.invoke('app:startupDiagnostic', event, details) @@ -3696,6 +3699,7 @@ const api = { title?: string ptyId?: string activate?: boolean + focus?: boolean presentation?: RuntimeTerminalPresentation tabId?: string leafId?: string @@ -3720,6 +3724,7 @@ const api = { title?: string ptyId?: string activate?: boolean + focus?: boolean presentation?: RuntimeTerminalPresentation tabId?: string leafId?: string @@ -3751,12 +3756,7 @@ const api = { ipcRenderer.on('terminal:requestTabMount', listener) return () => ipcRenderer.removeListener('terminal:requestTabMount', listener) }, - replyTerminalCreate: (reply: { - requestId: string - tabId?: string - title?: string - error?: string - }): void => { + replyTerminalCreate: (reply: TerminalTabCreateReply): void => { ipcRenderer.send('terminal:tabCreateReply', reply) }, onSplitTerminal: ( @@ -4646,6 +4646,24 @@ const api = { ipcRenderer.on('agentStatus:migrationUnsupportedClear', listener) return () => ipcRenderer.removeListener('agentStatus:migrationUnsupportedClear', listener) }, + onLegacyWorkerTerminalRecovery: ( + callback: (data: { + paneKey: string + resolution: 'adopted' | 'exited' | 'rolled_back' + ptyId?: string + }) => void + ): (() => void) => { + const listener = ( + _event: Electron.IpcRendererEvent, + data: { + paneKey: string + resolution: 'adopted' | 'exited' | 'rolled_back' + ptyId?: string + } + ) => callback(data) + ipcRenderer.on('agentStatus:legacyWorkerTerminalRecovery', listener) + return () => ipcRenderer.removeListener('agentStatus:legacyWorkerTerminalRecovery', listener) + }, getMigrationUnsupportedSnapshot: (): Promise => ipcRenderer.invoke('agentStatus:getMigrationUnsupportedSnapshot'), /** Drop the cached hook status for a paneKey on both sides (memory + on-disk) so a relaunch can't resurrect a dismissed row. */ diff --git a/src/relay/relay.ts b/src/relay/relay.ts index 82110b58db2..6daf87b309c 100644 --- a/src/relay/relay.ts +++ b/src/relay/relay.ts @@ -198,6 +198,8 @@ async function runOrcaCliMode(sockPath: string, argv: string[]): Promise { let nextSeq = 1 let highestReceivedSeq = 0 const requestId = 1 + const postOutputRequestId = 2 + let initialExitCode = 0 const sendRequest = (): void => { const env = pickRemoteCliEnv(process.env) @@ -219,6 +221,59 @@ async function runOrcaCliMode(sockPath: string, argv: string[]): Promise { sock.write(frame) } + const finish = (exitCode: number): void => { + sock.destroy() + process.exit(exitCode) + } + + const sendPostOutput = (postOutput: unknown): void => { + sock.write( + encodeJsonRpcFrame( + { + jsonrpc: '2.0', + id: postOutputRequestId, + method: 'orca.cli.postOutput', + params: { postOutput, env: pickRemoteCliEnv(process.env) } + }, + nextSeq++, + highestReceivedSeq + ) + ) + } + + const writeOutput = ( + result: { stdout?: unknown; stderr?: unknown }, + onFlushed: (error?: Error) => void + ): void => { + const writes = [ + [process.stdout, result.stdout], + [process.stderr, result.stderr] + ].filter((entry): entry is [NodeJS.WriteStream, string] => typeof entry[1] === 'string') + if (writes.length === 0) { + onFlushed() + return + } + let pending = writes.length + let completed = false + for (const [stream, output] of writes) { + stream.write(output, (error) => { + if (completed) { + return + } + if (error) { + completed = true + onFlushed(error) + return + } + pending -= 1 + if (pending === 0) { + completed = true + onFlushed() + } + }) + } + } + const decoder = new FrameDecoder((frame: DecodedFrame) => { if (frame.id > highestReceivedSeq) { highestReceivedSeq = frame.id @@ -227,28 +282,41 @@ async function runOrcaCliMode(sockPath: string, argv: string[]): Promise { return } const msg = parseJsonRpcMessage(frame.payload) - if (!('id' in msg) || msg.id !== requestId || !('result' in msg || 'error' in msg)) { + if ( + !('id' in msg) || + (msg.id !== requestId && msg.id !== postOutputRequestId) || + !('result' in msg || 'error' in msg) + ) { return } const response = msg as JsonRpcResponse if (response.error) { process.stderr.write(`${response.error.message}\n`) - sock.destroy() - process.exit(1) + finish(1) + return + } + if (response.id === postOutputRequestId) { + finish(initialExitCode) + return } const result = (response.result ?? {}) as { stdout?: unknown stderr?: unknown exitCode?: unknown + postOutput?: unknown } - if (typeof result.stdout === 'string' && result.stdout.length > 0) { - process.stdout.write(result.stdout) - } - if (typeof result.stderr === 'string' && result.stderr.length > 0) { - process.stderr.write(result.stderr) - } - sock.destroy() - process.exit(typeof result.exitCode === 'number' ? result.exitCode : 0) + initialExitCode = typeof result.exitCode === 'number' ? result.exitCode : 0 + writeOutput(result, (error) => { + if (error) { + finish(1) + return + } + if (result.postOutput === undefined) { + finish(initialExitCode) + return + } + sendPostOutput(result.postOutput) + }) }) const connectTimeout = setTimeout(() => { @@ -443,6 +511,12 @@ async function main(): Promise { timeoutMs: remoteCliRequestTimeoutMs(params) }) }) + dispatcher.onRequest('orca.cli.postOutput', async (params, context) => { + return await dispatcher.requestAnyClient('orca.cli.postOutput', params, { + excludeClientId: context.clientId, + timeoutMs: remoteCliRequestTimeoutMs(params) + }) + }) function configureRelayGraceTime(params: Record): { graceTimeMs: number } { const seconds = Number(params.graceTimeSeconds) diff --git a/src/relay/remote-cli-env.test.ts b/src/relay/remote-cli-env.test.ts index 1546230a5dd..68941e09768 100644 --- a/src/relay/remote-cli-env.test.ts +++ b/src/relay/remote-cli-env.test.ts @@ -8,6 +8,7 @@ describe('pickRemoteCliEnv', () => { ORCA_TERMINAL_HANDLE: 'term_ssh', ORCA_WORKTREE_ID: 'repo::remote', ORCA_PANE_KEY: 'pane-1', + ORCA_AGENT_LAUNCH_TOKEN: 'launch-secret', ORCA_WORKSPACE_ID: 'workspace-1', ORCA_USER_DATA_PATH: '/tmp/orca', PATH: '/usr/bin', @@ -17,6 +18,7 @@ describe('pickRemoteCliEnv', () => { ORCA_TERMINAL_HANDLE: 'term_ssh', ORCA_WORKTREE_ID: 'repo::remote', ORCA_PANE_KEY: 'pane-1', + ORCA_AGENT_LAUNCH_TOKEN: 'launch-secret', ORCA_WORKSPACE_ID: 'workspace-1', ORCA_USER_DATA_PATH: '/tmp/orca', PATH: '/usr/bin' diff --git a/src/relay/remote-cli-env.ts b/src/relay/remote-cli-env.ts index d8bc800c660..6161862c54c 100644 --- a/src/relay/remote-cli-env.ts +++ b/src/relay/remote-cli-env.ts @@ -4,6 +4,7 @@ export function pickRemoteCliEnv(env: NodeJS.ProcessEnv): Record 'ORCA_TERMINAL_HANDLE', 'ORCA_WORKTREE_ID', 'ORCA_PANE_KEY', + 'ORCA_AGENT_LAUNCH_TOKEN', 'ORCA_WORKSPACE_ID', 'ORCA_USER_DATA_PATH', 'PATH', diff --git a/src/renderer/src/App.tsx b/src/renderer/src/App.tsx index 26a434525a4..1d90f1cf5e0 100644 --- a/src/renderer/src/App.tsx +++ b/src/renderer/src/App.tsx @@ -1063,10 +1063,16 @@ function App(): React.JSX.Element { await timeRendererStartupStep('first-window-services-await', () => window.api.app.awaitFirstWindowStartupServices() ) + await timeRendererStartupStep('recover-legacy-worker-terminals-pre-reconnect', () => + window.api.app.recoverLegacyWorkerTerminalsForRendererStartup() + ) reconnectStarted = true await timeRendererStartupStep('reconnect-terminals', () => actions.reconnectPersistedTerminals(abortController.signal) ) + await timeRendererStartupStep('recover-legacy-worker-terminals-post-reconnect', () => + window.api.app.recoverLegacyWorkerTerminalsForRendererStartup() + ) syncZoomCSSVar() // Why (issue #1158): unlock the session writer only after hydration and all dependent steps succeeded, so a mid-startup throw can't serialize partially-mutated state to disk. actions.setHydrationSucceeded(true) @@ -1141,7 +1147,9 @@ function App(): React.JSX.Element { if (!reconnectStarted) { try { await window.api.app.awaitFirstWindowStartupServices() + await window.api.app.recoverLegacyWorkerTerminalsForRendererStartup() await actions.reconnectPersistedTerminals(abortController.signal) + await window.api.app.recoverLegacyWorkerTerminalsForRendererStartup() } catch (reconnectErr) { console.error( '[startup] reconnectPersistedTerminals failed in error path:', diff --git a/src/renderer/src/app-startup-routing.test.ts b/src/renderer/src/app-startup-routing.test.ts index 31015a530a9..85dd64ba574 100644 --- a/src/renderer/src/app-startup-routing.test.ts +++ b/src/renderer/src/app-startup-routing.test.ts @@ -123,11 +123,24 @@ describe('renderer startup runtime routing', () => { it('waits for first-window startup services before terminal reconnect', () => { const source = readFileSync(join(process.cwd(), 'src/renderer/src/App.tsx'), 'utf8') - const reconnectIndex = source.indexOf('await actions.reconnectPersistedTerminals') const servicesIndex = source.indexOf('await window.api.app.awaitFirstWindowStartupServices()') + const preReconnectRecoveryIndex = source.indexOf( + 'window.api.app.recoverLegacyWorkerTerminalsForRendererStartup()', + servicesIndex + ) + const reconnectIndex = source.indexOf( + 'actions.reconnectPersistedTerminals(abortController.signal)', + preReconnectRecoveryIndex + ) + const postReconnectRecoveryIndex = source.indexOf( + 'window.api.app.recoverLegacyWorkerTerminalsForRendererStartup()', + reconnectIndex + ) expect(servicesIndex).toBeGreaterThanOrEqual(0) - expect(servicesIndex).toBeLessThan(reconnectIndex) + expect(preReconnectRecoveryIndex).toBeGreaterThan(servicesIndex) + expect(reconnectIndex).toBeGreaterThan(preReconnectRecoveryIndex) + expect(postReconnectRecoveryIndex).toBeGreaterThan(reconnectIndex) }) it('keeps the persisted Automations view from starting its own bootstrap worktree scan', () => { diff --git a/src/renderer/src/components/terminal-pane/pty-connection.test.ts b/src/renderer/src/components/terminal-pane/pty-connection.test.ts index bdeca30e7f2..c5a413be990 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection.test.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection.test.ts @@ -10109,6 +10109,178 @@ describe('connectPanePty', () => { expect(mockStoreState.clearSleepingAgentSession).not.toHaveBeenCalled() }) + it('does not resume a live provider session while legacy worker recovery owns the pane', async () => { + const { connectPanePty } = await import('./pty-connection') + const retainedPtyId = 'wt-1@@lost-pty' + const transport = createMockTransport() + transport.connect.mockImplementation(async ({ sessionId }: { sessionId?: string }) => + sessionId + ? { + id: 'fresh-pty', + coldRestore: { scrollback: 'cold-payload', cwd: '/tmp/wt-1' } + } + : 'fresh-pty' + ) + transportFactoryQueue.push(transport) + const paneKey = makePaneKey('tab-1', LEAF_1) + mockStoreState = { + ...mockStoreState, + tabsByWorktree: { + 'wt-1': [{ id: 'tab-1', ptyId: retainedPtyId }] + }, + settings: { + ...mockStoreState.settings, + agentCmdOverrides: {} + }, + agentStatusByPaneKey: { + [paneKey]: { + paneKey, + state: 'working', + prompt: 'finish the task', + agentType: 'codex', + providerSession: { key: 'session_id', id: 'codex-session-1' } + } + }, + sleepingAgentSessionsByPaneKey: { + [paneKey]: { + paneKey, + tabId: 'tab-1', + worktreeId: 'wt-1', + agent: 'codex', + providerSession: { key: 'session_id', id: 'codex-session-1' }, + prompt: 'finish the task', + state: 'working', + capturedAt: 1, + updatedAt: 1, + automaticResumeBlockedBy: 'legacy-orchestration-worker' + } + } + } as StoreState + + connectPanePty( + createPane(1) as never, + createManager(1) as never, + createDeps({ + restoredLeafId: LEAF_1, + restoredPtyIdByLeafId: { [LEAF_1]: retainedPtyId } + }) as never + ) + await flushAsyncTicks(20) + await new Promise((resolve) => setTimeout(resolve, 70)) + + expect(transport.connect).not.toHaveBeenCalled() + expect(transport.attach).toHaveBeenCalledWith( + expect.objectContaining({ existingPtyId: retainedPtyId }) + ) + const attachOptions = transport.attach.mock.calls[0]?.[0] as Record + expect(attachOptions).not.toHaveProperty('cols') + expect(attachOptions).not.toHaveProperty('rows') + expect(mockStoreState.registerAgentLaunchConfig).not.toHaveBeenCalled() + expect(mockStoreState.clearSleepingAgentSession).not.toHaveBeenCalled() + }) + + it('does not replace a missing retained legacy worker over direct SSH', async () => { + const { connectPanePty } = await import('./pty-connection') + const retainedPtyId = toAppSshPtyId('ssh-a', 'missing-legacy-worker') + const transport = createMockTransport() + transport.getConnectionId.mockReturnValue('ssh-a') + transport.attach.mockImplementation(() => { + throw new Error('remote PTY missing') + }) + transportFactoryQueue.push(transport) + const paneKey = makePaneKey('tab-1', LEAF_1) + mockStoreState = { + ...mockStoreState, + tabsByWorktree: { + 'wt-1': [{ id: 'tab-1', ptyId: retainedPtyId }] + }, + repos: [{ id: 'repo1', connectionId: 'ssh-a' }], + sshConnectionStates: new Map([['ssh-a', { status: 'connected' }]]), + sleepingAgentSessionsByPaneKey: { + [paneKey]: { + paneKey, + tabId: 'tab-1', + worktreeId: 'wt-1', + agent: 'codex', + providerSession: { key: 'session_id', id: 'codex-session-1' }, + prompt: 'finish the task', + state: 'working', + capturedAt: 1, + updatedAt: 1, + automaticResumeBlockedBy: 'legacy-orchestration-worker' + } + } + } as StoreState + const deps = createDeps({ + restoredLeafId: LEAF_1, + restoredPtyIdByLeafId: { [LEAF_1]: retainedPtyId } + }) + + connectPanePty(createPane(1) as never, createManager(1) as never, deps as never) + await flushAsyncTicks(20) + await new Promise((resolve) => setTimeout(resolve, 70)) + + expect(transport.attach).toHaveBeenCalledWith( + expect.objectContaining({ existingPtyId: retainedPtyId }) + ) + expect(transport.connect).not.toHaveBeenCalled() + expect(deps.clearTabPtyId).not.toHaveBeenCalled() + expect(mockStoreState.registerAgentLaunchConfig).not.toHaveBeenCalled() + }) + + it('preserves a missing retained legacy worker through direct SSH reconnect', async () => { + const { connectPanePty } = await import('./pty-connection') + const retainedPtyId = toAppSshPtyId('ssh-a', 'missing-legacy-worker') + const transport = createMockTransport() + transport.getConnectionId.mockReturnValue('ssh-a') + transport.attach.mockImplementation(() => { + throw new Error('remote PTY missing') + }) + transportFactoryQueue.push(transport) + const paneKey = makePaneKey('tab-1', LEAF_1) + mockStoreState = { + ...mockStoreState, + tabsByWorktree: { + 'wt-1': [{ id: 'tab-1', ptyId: retainedPtyId }] + }, + repos: [{ id: 'repo1', connectionId: 'ssh-a' }], + sshConnectionStates: new Map([['ssh-a', { status: 'disconnected' }]]), + deferredSshReconnectTargets: ['ssh-a'], + deferredSshSessionIdsByTabId: { 'tab-1': retainedPtyId }, + sleepingAgentSessionsByPaneKey: { + [paneKey]: { + paneKey, + tabId: 'tab-1', + worktreeId: 'wt-1', + agent: 'codex', + providerSession: { key: 'session_id', id: 'codex-session-1' }, + prompt: 'finish the task', + state: 'working', + capturedAt: 1, + updatedAt: 1, + automaticResumeBlockedBy: 'legacy-orchestration-worker' + } + } + } as StoreState + const deps = createDeps({ + restoredLeafId: LEAF_1, + restoredPtyIdByLeafId: { [LEAF_1]: retainedPtyId } + }) + + connectPanePty(createPane(1) as never, createManager(1) as never, deps as never) + await flushAsyncTicks(20) + await new Promise((resolve) => setTimeout(resolve, 70)) + + expect(window.api.ssh.connect).toHaveBeenCalledWith({ targetId: 'ssh-a' }) + expect(transport.attach).toHaveBeenCalledWith( + expect.objectContaining({ existingPtyId: retainedPtyId }) + ) + expect(transport.connect).not.toHaveBeenCalled() + expect(mockStoreState.removeDeferredSshSessionId).not.toHaveBeenCalled() + expect(deps.clearTabPtyId).not.toHaveBeenCalled() + expect(mockStoreState.registerAgentLaunchConfig).not.toHaveBeenCalled() + }) + it('ignores stale live launch config when cold restore identity lookup rejects it', async () => { const { connectPanePty } = await import('./pty-connection') const transport = createMockTransport('fresh-pty') diff --git a/src/renderer/src/components/terminal-pane/pty-connection.ts b/src/renderer/src/components/terminal-pane/pty-connection.ts index 27c52889e0d..a6b571c951f 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection.ts @@ -1176,6 +1176,9 @@ export function connectPanePty( const [paneKey, record] = selectedLegacyMatch return { paneKey, record } } + const isLegacyWorkerAutomaticResumeBlocked = (): boolean => + getSleepingRecordForPane(useAppStore.getState())?.record.automaticResumeBlockedBy === + 'legacy-orchestration-worker' const clearSleepingRecordProviderDuplicates = ( state: ReturnType, consumed: { paneKey: string; record: SleepingAgentSessionRecord } @@ -4776,6 +4779,9 @@ export function connectPanePty( const entry = state.agentStatusByPaneKey[cacheKey] const sleepingRecordEntry = getSleepingRecordForPane(state) const sleepingRecord = sleepingRecordEntry?.record + if (isLegacyWorkerAutomaticResumeBlocked()) { + return null + } const useLiveEntry = entry && entry.state !== 'done' const agent = useLiveEntry ? entry.agentType : sleepingRecord?.agent if (!agent || !isResumableTuiAgent(agent)) { @@ -5002,6 +5008,9 @@ export function connectPanePty( startupOverride?: PendingStartupCommand | null, options: FreshSpawnOptions = {} ): Promise => { + if (isLegacyWorkerAutomaticResumeBlocked()) { + return Promise.resolve(null) + } if (useAppStore.getState().deleteStateByWorktreeId?.[deps.worktreeId]?.isDeleting) { // Why: the worktree is being deleted; its PTYs were just killed for the // filesystem teardown. A fresh shell must not spawn into a directory the @@ -7919,6 +7928,30 @@ export function connectPanePty( return true } + const attachRetainedLegacyPty = (ptyId: string): boolean => { + try { + clearPaneMode2031State() + clearHiddenOutputRestoreState() + const outputCallbacks = captureTransportOutputCallbacks(reportError) + transport.attach({ + existingPtyId: ptyId, + callbacks: outputCallbacks.callbacks + }) + const attachedPtyId = transport.getPtyId() ?? ptyId + bindActivePanePty(attachedPtyId, { + updateTabPtyId: 'if-missing', + sampleVisibleForegroundAgent: true + }) + if (isRemoteRuntimePtyId(attachedPtyId)) { + registerPaneSerializerFor(attachedPtyId) + } + return true + } catch (err) { + reportError(err instanceof Error ? err.message : String(err)) + return false + } + } + // Why: trigger the deferred SSH connect per-tab (not per-target) so multiple tabs for one target reattach independently. // Must run before session-id resolution: the SSH provider isn't registered until connect succeeds. if (connectionId) { @@ -7952,7 +7985,8 @@ export function connectPanePty( console.warn( `[pty-connection] SSH tab=${deps.tabId} connectionId=${connectionId} pendingSessionId=${pendingSessionId} sshConnected=${gate.sshConnected}` ) - if (gate.enterDeferredFlow) { + const legacyWorkerOwnsPane = isLegacyWorkerAutomaticResumeBlocked() + if (gate.enterDeferredFlow && (!legacyWorkerOwnsPane || !gate.sshConnected)) { void (async () => { // Why: for a passphrase target with no cached credential, don't auto-fire ssh.connect — a prompt popping just from focusing a tab / Cmd+J would surprise the user. // Wait for a user-initiated connect first; no-passphrase targets return false here and auto-connect as before. @@ -8057,6 +8091,13 @@ export function connectPanePty( return } if (pendingSessionId) { + if (isLegacyWorkerAutomaticResumeBlocked()) { + if (attachRetainedLegacyPty(pendingSessionId)) { + useAppStore.getState().removeDeferredSshSessionId(deps.tabId) + scheduleRuntimeGraphSync() + } + return + } console.warn( `[pty-connection] Attempting reattach for tab=${deps.tabId} sessionId=${pendingSessionId}` ) @@ -8281,14 +8322,18 @@ export function connectPanePty( : null // Why: after a daemon crash + cold restore, a stale session-to-tab mapping can make a tab hold a ptyId from another worktree. // Restoring it would paint the wrong terminal content, so drop the reattach and spawn fresh. - const deferredReattachSessionId = - runtimeHostPtyWakeHint ?? - (candidateReattachSessionId && - !isRemoteRuntimePtyId(candidateReattachSessionId) && - !candidateHasEagerBuffer && - isSessionOwnedByWorktree(candidateReattachSessionId, deps.worktreeId) - ? candidateReattachSessionId - : null) + const legacyAttachOnlyPtyId = isLegacyWorkerAutomaticResumeBlocked() + ? candidateReattachSessionId + : null + const deferredReattachSessionId = legacyAttachOnlyPtyId + ? null + : (runtimeHostPtyWakeHint ?? + (candidateReattachSessionId && + !isRemoteRuntimePtyId(candidateReattachSessionId) && + !candidateHasEagerBuffer && + isSessionOwnedByWorktree(candidateReattachSessionId, deps.worktreeId) + ? candidateReattachSessionId + : null)) recordPtyConnectDiagnostic( `pane=${pane.id} tab=${deps.tabId} restored=${restoredPtyId} existing=${existingPtyId} detached=${detachedRemoteLeafPtyId ?? detachedLivePtyId} reattach=${deferredReattachSessionId} hasTransport=${hadExistingPaneTransportAtConnect} pendingKey=${pendingSpawnKey}` ) @@ -8435,36 +8480,48 @@ export function connectPanePty( }) }) armDirectSshPaneRetryTimeout(trackedReattachPromise, directSshRetryAttempt) - } else if (detachedRemoteLeafPtyId || detachedLivePtyId || eagerLivePtyId) { + } else if ( + legacyAttachOnlyPtyId || + detachedRemoteLeafPtyId || + detachedLivePtyId || + eagerLivePtyId + ) { // Why: mirrored web-leaf panes must attach to their exact remote PTY, not spawn a replacement host tab. // eagerLivePtyId covers a still-live background PTY (e.g. an automation agent) with a live eager buffer to adopt. - const attachPtyId = detachedRemoteLeafPtyId ?? detachedLivePtyId ?? eagerLivePtyId! + const attachPtyId = + legacyAttachOnlyPtyId ?? detachedRemoteLeafPtyId ?? detachedLivePtyId ?? eagerLivePtyId! recordPtyConnectDiagnostic(`pane=${pane.id} -> ATTACH detached=${attachPtyId}`) allowInitialIdleCacheSeed = false - // Why: surface synchronous attach failures via reportError so the pane shows a diagnostic instead of a blank surface. - // On throw, clear the stale ptyId from the tab and fresh-spawn — else the next remount reads the same dead id and loops here. - try { - clearPaneMode2031State() - clearHiddenOutputRestoreState() - const outputCallbacks = captureTransportOutputCallbacks(reportError) - transport.attach({ - existingPtyId: attachPtyId, - cols, - rows, - callbacks: outputCallbacks.callbacks - }) - const attachedPtyId = transport.getPtyId() ?? attachPtyId - bindActivePanePty(attachedPtyId, { - updateTabPtyId: 'if-missing', - sampleVisibleForegroundAgent: true - }) - if (attachPtyId === eagerLivePtyId || isRemoteRuntimePtyId(attachedPtyId)) { - registerPaneSerializerFor(attachedPtyId) + if (legacyAttachOnlyPtyId) { + if (attachRetainedLegacyPty(legacyAttachOnlyPtyId) && connectionId) { + useAppStore.getState().removeDeferredSshSessionId(deps.tabId) + } + } else { + // Why: surface synchronous attach failures via reportError so the pane shows a diagnostic instead of a blank surface. + // On throw, clear the stale ptyId from the tab and fresh-spawn — else the next remount reads the same dead id and loops here. + try { + clearPaneMode2031State() + clearHiddenOutputRestoreState() + const outputCallbacks = captureTransportOutputCallbacks(reportError) + transport.attach({ + existingPtyId: attachPtyId, + cols, + rows, + callbacks: outputCallbacks.callbacks + }) + const attachedPtyId = transport.getPtyId() ?? attachPtyId + bindActivePanePty(attachedPtyId, { + updateTabPtyId: 'if-missing', + sampleVisibleForegroundAgent: true + }) + if (attachPtyId === eagerLivePtyId || isRemoteRuntimePtyId(attachedPtyId)) { + registerPaneSerializerFor(attachedPtyId) + } + } catch (err) { + reportError(err instanceof Error ? err.message : String(err)) + deps.clearTabPtyId(deps.tabId, attachPtyId) + startFreshSpawn() } - } catch (err) { - reportError(err instanceof Error ? err.message : String(err)) - deps.clearTabPtyId(deps.tabId, attachPtyId) - startFreshSpawn() } } else { allowInitialIdleCacheSeed = false diff --git a/src/renderer/src/components/terminal-pane/use-terminal-pane-lifecycle.test.ts b/src/renderer/src/components/terminal-pane/use-terminal-pane-lifecycle.test.ts index a051000b1f9..74e53a0b6be 100644 --- a/src/renderer/src/components/terminal-pane/use-terminal-pane-lifecycle.test.ts +++ b/src/renderer/src/components/terminal-pane/use-terminal-pane-lifecycle.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, vi } from 'vitest' import { + applyTerminalPaneCloseRequest, applyTerminalScrollbackRowsToMountedPanes, clearQueuedInitialCwdAfterFirstPane, getPreviousVisibleForTerminalPane, @@ -9,11 +10,154 @@ import { resolvePaneSeedCwd, resolveQueuedInitialCwd, resetTerminalKeyboardProtocolAfterInterrupt, + retireMountedTerminalPaneSurface, shouldDetachPaneTransportOnUnmount, splitPaneWithOneShotStartup, suppressIntentionalPaneCloseExit } from './use-terminal-pane-lifecycle' +describe('applyTerminalPaneCloseRequest', () => { + it('detaches a rolled-back split surface without closing its PTY', () => { + const manager = { + getPanes: vi.fn(() => [{ id: 1 }, { id: 2 }]), + getNumericIdForLeaf: vi.fn(() => 2), + closePane: vi.fn(), + detachPaneForExternalMove: vi.fn(() => true), + retirePanePreservingPty: vi.fn(() => true) + } + const closeTab = vi.fn() + const closeTabPreservingPty = vi.fn() + + expect( + applyTerminalPaneCloseRequest({ + detail: { + tabId: 'legacy-worker', + leafId: '11111111-1111-4111-8111-111111111111', + preservePty: true + }, + manager, + closeTab, + closeTabPreservingPty + }) + ).toBe('pane') + expect(manager.detachPaneForExternalMove).toHaveBeenCalledWith(2) + expect(manager.closePane).not.toHaveBeenCalled() + expect(closeTab).not.toHaveBeenCalled() + expect(closeTabPreservingPty).not.toHaveBeenCalled() + }) + + it('uses non-destructive tab close semantics for the last rolled-back pane', () => { + const closeTab = vi.fn() + const closeTabPreservingPty = vi.fn() + + expect( + applyTerminalPaneCloseRequest({ + detail: { + tabId: 'legacy-worker', + paneRuntimeId: 1, + preservePty: true + }, + manager: { + getPanes: vi.fn(() => [{ id: 1 }]), + getNumericIdForLeaf: vi.fn(() => 1), + closePane: vi.fn(), + detachPaneForExternalMove: vi.fn(() => true), + retirePanePreservingPty: vi.fn(() => true) + }, + closeTab, + closeTabPreservingPty + }) + ).toBe('tab') + expect(closeTabPreservingPty).toHaveBeenCalledOnce() + expect(closeTab).not.toHaveBeenCalled() + }) + + it('ignores a delayed rollback after the pane PTY identity changed', () => { + const manager = { + getPanes: vi.fn(() => [{ id: 1 }]), + getNumericIdForLeaf: vi.fn(() => 1), + closePane: vi.fn(), + detachPaneForExternalMove: vi.fn(() => true), + retirePanePreservingPty: vi.fn(() => true) + } + const closeTab = vi.fn() + const closeTabPreservingPty = vi.fn() + + expect( + applyTerminalPaneCloseRequest({ + detail: { + tabId: 'legacy-worker', + leafId: '11111111-1111-4111-8111-111111111111', + preservePty: true, + expectedPtyId: 'pty-legacy' + }, + manager, + closeTab, + closeTabPreservingPty, + getPtyIdForLeaf: () => 'pty-replacement' + }) + ).toBe('ignored') + expect(manager.detachPaneForExternalMove).not.toHaveBeenCalled() + expect(closeTabPreservingPty).not.toHaveBeenCalled() + }) + + it('retires a mounted rollback pane without detaching it as a movable surface', () => { + const manager = { + getPanes: vi.fn(() => [{ id: 1 }, { id: 2 }]), + getNumericIdForLeaf: vi.fn(() => 2), + closePane: vi.fn(), + detachPaneForExternalMove: vi.fn(() => true), + retirePanePreservingPty: vi.fn(() => true) + } + + expect( + applyTerminalPaneCloseRequest({ + detail: { + tabId: 'legacy-worker', + leafId: '11111111-1111-4111-8111-111111111111', + preservePty: true, + retireSurface: true, + expectedPtyId: 'pty-legacy' + }, + manager, + closeTab: vi.fn(), + closeTabPreservingPty: vi.fn(), + getPtyIdForLeaf: () => 'pty-legacy' + }) + ).toBe('pane') + expect(manager.retirePanePreservingPty).toHaveBeenCalledWith(2) + expect(manager.detachPaneForExternalMove).not.toHaveBeenCalled() + expect(manager.closePane).not.toHaveBeenCalled() + }) + + it('retires mounted authority and binding while preserving the process and sleeping fence', () => { + const retireAgentPaneAuthority = vi.fn() + const syncPanePtyLayoutBinding = vi.fn() + const clearTabPtyId = vi.fn() + const transport = { detach: vi.fn(), destroy: vi.fn() } + + retireMountedTerminalPaneSurface({ + paneKey: 'legacy-worker:11111111-1111-4111-8111-111111111111', + paneId: 2, + tabId: 'legacy-worker', + ptyId: 'pty-legacy', + retireAgentPaneAuthority, + syncPanePtyLayoutBinding, + clearTabPtyId, + transport + }) + + expect(retireAgentPaneAuthority).toHaveBeenCalledWith( + 'legacy-worker:11111111-1111-4111-8111-111111111111', + { preserveSleepingAgentSession: true } + ) + expect(syncPanePtyLayoutBinding).toHaveBeenCalledWith(2, null) + expect(clearTabPtyId).toHaveBeenCalledWith('legacy-worker', 'pty-legacy') + expect(transport.detach).toHaveBeenCalledOnce() + expect(transport.destroy).not.toHaveBeenCalled() + }) +}) + describe('resetTerminalKeyboardProtocolAfterInterrupt', () => { it('does not write to an xterm whose pipeline is certified dead', async () => { const { _resetWritePipelineHealthForTests, notifyUndeliverableWrite } = diff --git a/src/renderer/src/components/terminal-pane/use-terminal-pane-lifecycle.ts b/src/renderer/src/components/terminal-pane/use-terminal-pane-lifecycle.ts index 4ad241c9fa3..0f7199c48ac 100644 --- a/src/renderer/src/components/terminal-pane/use-terminal-pane-lifecycle.ts +++ b/src/renderer/src/components/terminal-pane/use-terminal-pane-lifecycle.ts @@ -503,6 +503,77 @@ export function getPreviousVisibleForTerminalPane(args: { return args.previous.isVisible } +type TerminalPaneCloseManager = { + closePane: (paneId: number) => void + detachPaneForExternalMove: (paneId: number) => boolean + retirePanePreservingPty: (paneId: number) => boolean + getNumericIdForLeaf: (leafId: string) => number | null + getPanes: () => unknown[] +} + +export function applyTerminalPaneCloseRequest(args: { + detail: CloseTerminalPaneDetail + manager: TerminalPaneCloseManager + closeTab: () => void + closeTabPreservingPty: () => void + getPtyIdForLeaf?: (leafId: string) => string | undefined +}): 'ignored' | 'pane' | 'tab' { + if ( + args.detail.expectedPtyId && + (!args.detail.leafId || + args.getPtyIdForLeaf?.(args.detail.leafId) !== args.detail.expectedPtyId) + ) { + return 'ignored' + } + const paneRuntimeId = + args.detail.paneRuntimeId ?? + (args.detail.leafId ? args.manager.getNumericIdForLeaf(args.detail.leafId) : null) + if (paneRuntimeId === null || paneRuntimeId === undefined) { + return 'ignored' + } + if (args.manager.getPanes().length <= 1) { + if (args.detail.preservePty) { + args.closeTabPreservingPty() + } else { + args.closeTab() + } + return 'tab' + } + if (args.detail.preservePty) { + if (args.detail.retireSurface) { + args.manager.retirePanePreservingPty(paneRuntimeId) + } else { + args.manager.detachPaneForExternalMove(paneRuntimeId) + } + } else { + args.manager.closePane(paneRuntimeId) + } + return 'pane' +} + +export function retireMountedTerminalPaneSurface(args: { + paneKey: string + paneId: number + tabId: string + ptyId: string | null + retireAgentPaneAuthority: ( + paneKey: string, + options?: { preserveSleepingAgentSession?: boolean } + ) => void + syncPanePtyLayoutBinding: (paneId: number, ptyId: string | null) => void + clearTabPtyId: (tabId: string, ptyId: string) => void + transport?: { detach?: () => void; destroy?: () => void } +}): void { + args.retireAgentPaneAuthority(args.paneKey, { + preserveSleepingAgentSession: true + }) + if (args.ptyId) { + args.syncPanePtyLayoutBinding(args.paneId, null) + args.clearTabPtyId(args.tabId, args.ptyId) + } + args.transport?.detach?.() +} + /** Wires mounted terminal panes to renderer state and terminal event handling. */ export function useTerminalPaneLifecycle({ tabId, @@ -1105,6 +1176,7 @@ export function useTerminalPaneLifecycle({ onPaneClosed: (paneId, closedPane) => { onPtyRecoveryStateRef?.current?.(paneId, null) const isDetachedToTab = closedPane?.reason === 'detach' + const isRetiredSurface = closedPane?.reason === 'retire' const linkProviderDisposable = linkProviderDisposablesRef.current.get(paneId) if (linkProviderDisposable) { linkProviderDisposable.dispose() @@ -1181,14 +1253,25 @@ export function useTerminalPaneLifecycle({ panePtyBindings.delete(paneId) } const leafId = closedPane?.leafId - if (leafId && !isDetachedToTab) { + if (leafId && isRetiredSurface) { + retireMountedTerminalPaneSurface({ + paneKey: makePaneKey(tabId, leafId), + paneId, + tabId, + ptyId: closedPtyId, + retireAgentPaneAuthority: useAppStore.getState().retireAgentPaneAuthority, + syncPanePtyLayoutBinding, + clearTabPtyId, + ...(transport ? { transport } : {}) + }) + } else if (leafId && !isDetachedToTab) { // Why: revoke only this pane's authority; an exact tombstone blocks queued hooks without suppressing siblings. const paneKey = makePaneKey(tabId, leafId) useAppStore.getState().retireAgentPaneAuthority(paneKey) } - if (transport) { + if (transport && !isRetiredSurface) { if (isDetachedToTab) { - // Why: detach hands the PTY to a new tab, so drop renderer listeners without a process teardown. + // Why: detach hands the PTY to a new tab, so drop renderer listeners without process teardown. transport.detach?.() } else { const ptyId = suppressIntentionalPaneCloseExit( @@ -1542,16 +1625,32 @@ export function useTerminalPaneLifecycle({ if (!mgr) { return } - if (mgr.getPanes().length <= 1) { - // Why: route through closeTerminalTab (not raw closeTab) so a pinned tab hits the confirmation guard — this was the one path that silently dropped pinned tabs. - closeTerminalTab(tabId) - } else { - mgr.closePane(detail.paneRuntimeId) - scheduleRuntimeGraphSync() - syncCanExpandState() - queueResizeAll(isActive) - persistLayoutSnapshot() + const result = applyTerminalPaneCloseRequest({ + detail, + manager: mgr, + getPtyIdForLeaf: (leafId) => + useAppStore.getState().terminalLayoutsByTabId[tabId]?.ptyIdsByLeafId?.[leafId], + closeTab: () => closeTerminalTab(tabId), + closeTabPreservingPty: () => { + const store = useAppStore.getState() + if (detail.retireSurface && detail.leafId) { + store.retireAgentPaneAuthority(makePaneKey(tabId, detail.leafId), { + preserveSleepingAgentSession: true + }) + } + store.closeTab(tabId, { + reason: 'pty-exit', + captureRecentlyClosed: false + }) + } + }) + if (result !== 'pane') { + return } + scheduleRuntimeGraphSync() + syncCanExpandState() + queueResizeAll(isActive) + persistLayoutSnapshot() } window.addEventListener(CLOSE_TERMINAL_PANE_EVENT, onCliClosePane) diff --git a/src/renderer/src/constants/terminal.ts b/src/renderer/src/constants/terminal.ts index 1c0b5a5b713..a5af687c466 100644 --- a/src/renderer/src/constants/terminal.ts +++ b/src/renderer/src/constants/terminal.ts @@ -69,7 +69,11 @@ export type RequestActiveTerminalPaneSplitDetail = { export type CloseTerminalPaneDetail = { tabId: string - paneRuntimeId: number + paneRuntimeId?: number + leafId?: string + preservePty?: boolean + retireSurface?: boolean + expectedPtyId?: string } export type BackgroundMountTerminalWorktreeDetail = { diff --git a/src/renderer/src/hooks/legacy-worker-terminal-recovery-event.test.ts b/src/renderer/src/hooks/legacy-worker-terminal-recovery-event.test.ts new file mode 100644 index 00000000000..6b6b3aa8862 --- /dev/null +++ b/src/renderer/src/hooks/legacy-worker-terminal-recovery-event.test.ts @@ -0,0 +1,129 @@ +import { describe, expect, it, vi } from 'vitest' +import { + resolveLegacyWorkerTerminalRecoveryAction, + rollbackLegacyWorkerTerminalSurfaceInStore +} from './legacy-worker-terminal-recovery-event' + +const LEAF_ID = '11111111-1111-4111-8111-111111111111' + +describe('legacy worker terminal recovery events', () => { + it('removes a rolled-back surface without clearing its sleeping recovery fence', () => { + expect( + resolveLegacyWorkerTerminalRecoveryAction({ + paneKey: `legacy-worker:${LEAF_ID}`, + resolution: 'rolled_back', + ptyId: 'pty-legacy' + }) + ).toEqual({ + kind: 'rollback-surface', + detail: { + tabId: 'legacy-worker', + leafId: LEAF_ID, + preservePty: true, + retireSurface: true, + expectedPtyId: 'pty-legacy' + } + }) + }) + + it('clears sleeping recovery only after an adopted or exited resolution', () => { + expect( + resolveLegacyWorkerTerminalRecoveryAction({ + paneKey: `legacy-worker:${LEAF_ID}`, + resolution: 'adopted' + }) + ).toEqual({ + kind: 'clear-sleeping', + paneKey: `legacy-worker:${LEAF_ID}` + }) + }) + + it('removes an unmounted split surface only when its PTY identity still matches', () => { + const setTabLayout = vi.fn() + const clearTabPtyId = vi.fn() + const closeTab = vi.fn() + const retireAgentPaneAuthority = vi.fn() + const siblingLeafId = '22222222-2222-4222-8222-222222222222' + const store = { + tabsByWorktree: { + worktree: [{ id: 'legacy-worker' }] + }, + terminalLayoutsByTabId: { + 'legacy-worker': { + root: { + type: 'split' as const, + direction: 'horizontal' as const, + ratio: 0.5, + first: { type: 'leaf' as const, leafId: LEAF_ID }, + second: { type: 'leaf' as const, leafId: siblingLeafId } + }, + activeLeafId: LEAF_ID, + expandedLeafId: null, + ptyIdsByLeafId: { + [LEAF_ID]: 'pty-legacy', + [siblingLeafId]: 'pty-sibling' + } + } + }, + setTabLayout, + clearTabPtyId, + closeTab, + retireAgentPaneAuthority + } + + expect( + rollbackLegacyWorkerTerminalSurfaceInStore(store as never, { + tabId: 'legacy-worker', + leafId: LEAF_ID, + preservePty: true, + expectedPtyId: 'pty-legacy' + }) + ).toBe('removed') + expect(setTabLayout).toHaveBeenCalledWith( + 'legacy-worker', + expect.objectContaining({ + root: { type: 'leaf', leafId: siblingLeafId }, + ptyIdsByLeafId: { [siblingLeafId]: 'pty-sibling' } + }) + ) + expect(clearTabPtyId).toHaveBeenCalledWith('legacy-worker', 'pty-legacy') + expect(retireAgentPaneAuthority).toHaveBeenCalledWith(`legacy-worker:${LEAF_ID}`, { + preserveSleepingAgentSession: true + }) + expect(closeTab).not.toHaveBeenCalled() + }) + + it('does not remove a replacement surface that reused the same pane', () => { + const setTabLayout = vi.fn() + const clearTabPtyId = vi.fn() + const closeTab = vi.fn() + const store = { + tabsByWorktree: { + worktree: [{ id: 'legacy-worker' }] + }, + terminalLayoutsByTabId: { + 'legacy-worker': { + root: { type: 'leaf' as const, leafId: LEAF_ID }, + activeLeafId: LEAF_ID, + expandedLeafId: null, + ptyIdsByLeafId: { [LEAF_ID]: 'pty-replacement' } + } + }, + setTabLayout, + clearTabPtyId, + closeTab + } + + expect( + rollbackLegacyWorkerTerminalSurfaceInStore(store as never, { + tabId: 'legacy-worker', + leafId: LEAF_ID, + preservePty: true, + expectedPtyId: 'pty-legacy' + }) + ).toBe('identity-mismatch') + expect(setTabLayout).not.toHaveBeenCalled() + expect(clearTabPtyId).not.toHaveBeenCalled() + expect(closeTab).not.toHaveBeenCalled() + }) +}) diff --git a/src/renderer/src/hooks/legacy-worker-terminal-recovery-event.ts b/src/renderer/src/hooks/legacy-worker-terminal-recovery-event.ts new file mode 100644 index 00000000000..61940acedd7 --- /dev/null +++ b/src/renderer/src/hooks/legacy-worker-terminal-recovery-event.ts @@ -0,0 +1,83 @@ +import type { CloseTerminalPaneDetail } from '@/constants/terminal' +import { detachTerminalLayoutLeaf } from '@/components/terminal-pane/terminal-layout-leaf-detach' +import type { AppState } from '@/store' +import { makePaneKey, parsePaneKey } from '../../../shared/stable-pane-id' + +type LegacyWorkerTerminalRecoveryEvent = { + paneKey: string + resolution: 'adopted' | 'exited' | 'rolled_back' + ptyId?: string +} + +export type LegacyWorkerTerminalRecoveryAction = + | { kind: 'clear-sleeping'; paneKey: string } + | { kind: 'rollback-surface'; detail: CloseTerminalPaneDetail } + | { kind: 'ignore' } + +export function resolveLegacyWorkerTerminalRecoveryAction( + event: LegacyWorkerTerminalRecoveryEvent +): LegacyWorkerTerminalRecoveryAction { + if (event.resolution !== 'rolled_back') { + return { kind: 'clear-sleeping', paneKey: event.paneKey } + } + const pane = parsePaneKey(event.paneKey) + return pane && event.ptyId + ? { + kind: 'rollback-surface', + detail: { + tabId: pane.tabId, + leafId: pane.leafId, + preservePty: true, + retireSurface: true, + expectedPtyId: event.ptyId + } + } + : { kind: 'ignore' } +} + +type LegacyWorkerTerminalRecoveryStore = Pick< + AppState, + | 'tabsByWorktree' + | 'terminalLayoutsByTabId' + | 'setTabLayout' + | 'clearTabPtyId' + | 'closeTab' + | 'retireAgentPaneAuthority' +> + +export function rollbackLegacyWorkerTerminalSurfaceInStore( + store: LegacyWorkerTerminalRecoveryStore, + detail: CloseTerminalPaneDetail +): 'removed' | 'already-removed' | 'identity-mismatch' { + const tabExists = Object.values(store.tabsByWorktree).some((tabs) => + tabs.some((tab) => tab.id === detail.tabId) + ) + if (!tabExists) { + return 'already-removed' + } + if (!detail.leafId || !detail.expectedPtyId) { + return 'identity-mismatch' + } + const layout = store.terminalLayoutsByTabId[detail.tabId] + const boundPtyId = layout?.ptyIdsByLeafId?.[detail.leafId] + if (!boundPtyId) { + return 'already-removed' + } + if (boundPtyId !== detail.expectedPtyId) { + return 'identity-mismatch' + } + const detached = detachTerminalLayoutLeaf(layout, detail.leafId) + if (detached) { + store.retireAgentPaneAuthority(makePaneKey(detail.tabId, detail.leafId), { + preserveSleepingAgentSession: true + }) + store.setTabLayout(detail.tabId, detached.sourceLayout) + store.clearTabPtyId(detail.tabId, detail.expectedPtyId) + } else { + store.closeTab(detail.tabId, { + reason: 'pty-exit', + captureRecentlyClosed: false + }) + } + return 'removed' +} diff --git a/src/renderer/src/hooks/useIpcEvents.test.ts b/src/renderer/src/hooks/useIpcEvents.test.ts index 39ef0eaea7c..0765019ca83 100644 --- a/src/renderer/src/hooks/useIpcEvents.test.ts +++ b/src/renderer/src/hooks/useIpcEvents.test.ts @@ -1504,7 +1504,6 @@ describe('useIpcEvents updater integration', () => { openAgentTabsInChatByDefault: false } } - vi.doMock('react', async () => { const actual = await vi.importActual('react') return { @@ -1789,7 +1788,11 @@ describe('useIpcEvents updater integration', () => { }) it('surfaces terminal creates without stealing focus unless requested', async () => { - const createTab = vi.fn(() => ({ id: 'tab-new' })) + const createTab = vi.fn( + (_worktreeId: string, _groupId?: string, _tabType?: string, options?: { id?: string }) => ({ + id: options?.id ?? 'tab-new' + }) + ) const setActiveView = vi.fn() const setActiveWorktree = vi.fn() const markWorktreeVisited = vi.fn() @@ -1876,6 +1879,20 @@ describe('useIpcEvents updater integration', () => { activeRuntimeEnvironmentId: undefined as string | undefined } } + updateTabPtyId.mockImplementation((tabId: string, ptyId: string) => { + storeState.ptyIdsByTabId[tabId] = [ + ...new Set([...(storeState.ptyIdsByTabId[tabId] ?? []), ptyId]) + ] + for (const tabs of Object.values(storeState.tabsByWorktree)) { + const tab = tabs.find((candidate) => candidate.id === tabId) + if (tab) { + tab.ptyId = ptyId + } + } + }) + setTabLayout.mockImplementation((tabId: string, layout: unknown) => { + storeState.terminalLayoutsByTabId[tabId] = layout + }) const createTerminalListenerRef: { current: | ((data: { @@ -1888,6 +1905,7 @@ describe('useIpcEvents updater integration', () => { title?: string ptyId?: string activate?: boolean + focus?: boolean presentation?: 'background' | 'focused' tabId?: string leafId?: string @@ -2036,6 +2054,7 @@ describe('useIpcEvents updater integration', () => { title?: string ptyId?: string activate?: boolean + focus?: boolean presentation?: 'background' | 'focused' tabId?: string leafId?: string @@ -2278,6 +2297,7 @@ describe('useIpcEvents updater integration', () => { setActiveTabType.mockClear() setActiveTab.mockClear() revealWorktreeInSidebar.mockClear() + dispatchEvent.mockClear() createTerminalListenerRef.current({ worktreeId: 'wt-2', title: 'Runner', @@ -2586,6 +2606,7 @@ describe('useIpcEvents updater integration', () => { setActiveTab.mockClear() revealWorktreeInSidebar.mockClear() replyTerminalCreate.mockClear() + dispatchEvent.mockClear() requestTerminalCreateListenerRef.current({ requestId: 'req-renderer-backed-background', worktreeId: 'wt-2', @@ -2603,6 +2624,12 @@ describe('useIpcEvents updater integration', () => { expect(setActiveTabType).not.toHaveBeenCalled() expect(setActiveTab).not.toHaveBeenCalled() expect(revealWorktreeInSidebar).not.toHaveBeenCalled() + expect(dispatchEvent).toHaveBeenCalledWith( + expect.objectContaining({ + type: 'orca-background-mount-terminal-worktree', + detail: { worktreeId: 'wt-2', tabIds: ['tab-new'] } + }) + ) expect(replyTerminalCreate).toHaveBeenCalledWith({ requestId: 'req-renderer-backed-background', tabId: 'tab-new', @@ -2706,6 +2733,7 @@ describe('useIpcEvents updater integration', () => { setActiveTabType.mockClear() setActiveTab.mockClear() revealWorktreeInSidebar.mockClear() + dispatchEvent.mockClear() createTerminalListenerRef.current({ worktreeId: 'wt-2', ptyId: 'pty-bg-3', @@ -2723,6 +2751,50 @@ describe('useIpcEvents updater integration', () => { expect(setActiveTabType).not.toHaveBeenCalled() expect(setActiveTab).not.toHaveBeenCalled() expect(revealWorktreeInSidebar).not.toHaveBeenCalled() + expect(dispatchEvent).toHaveBeenCalledOnce() + expect(dispatchEvent).toHaveBeenCalledWith( + expect.objectContaining({ + type: 'orca-background-mount-terminal-worktree', + detail: { worktreeId: 'wt-2', tabIds: ['tab-cli-bg-reveal'] } + }) + ) + + createTab.mockClear() + setActiveView.mockClear() + setActiveWorktree.mockClear() + setActiveTabType.mockClear() + setActiveTab.mockClear() + revealWorktreeInSidebar.mockClear() + focusRuntimeTerminalSurface.mockClear() + focusTerminalTabSurface.mockClear() + dispatchEvent.mockClear() + createTerminalListenerRef.current({ + worktreeId: 'wt-2', + ptyId: 'pty-recovery-bg', + activate: true, + focus: false, + tabId: 'tab-recovery-bg' + }) + + expect(createTab).toHaveBeenCalledWith('wt-2', undefined, undefined, { + initialPtyId: 'pty-recovery-bg', + activate: false, + id: 'tab-recovery-bg' + }) + expect(setActiveView).not.toHaveBeenCalled() + expect(setActiveWorktree).not.toHaveBeenCalled() + expect(setActiveTabType).not.toHaveBeenCalled() + expect(setActiveTab).not.toHaveBeenCalled() + expect(revealWorktreeInSidebar).not.toHaveBeenCalled() + expect(focusRuntimeTerminalSurface).not.toHaveBeenCalled() + expect(focusTerminalTabSurface).not.toHaveBeenCalled() + expect(dispatchEvent).toHaveBeenCalledOnce() + expect(dispatchEvent).toHaveBeenCalledWith( + expect.objectContaining({ + type: 'orca-background-mount-terminal-worktree', + detail: { worktreeId: 'wt-2', tabIds: ['tab-recovery-bg'] } + }) + ) storeState.tabsByWorktree = { 'wt-2': [{ id: 'tab-existing', ptyId: 'pty-bg', title: 'Terminal 1' }] @@ -2805,7 +2877,13 @@ describe('useIpcEvents updater integration', () => { expect(replyTerminalCreate).toHaveBeenCalledWith({ requestId: 'req-adopt-pending', tabId: pendingTabId, - title: 'Terminal 3' + title: 'Terminal 3', + identity: { + worktreeId: 'wt-2', + tabId: pendingTabId, + leafId: pendingLeafId, + ptyId: 'serve-cf39bedb-a33a-417c-9ab6-f304dc27a6c0' + } }) storeState.tabsByWorktree = { @@ -2871,7 +2949,13 @@ describe('useIpcEvents updater integration', () => { expect(replyTerminalCreate).toHaveBeenCalledWith({ requestId: 'req-split', tabId: 'tab-existing', - title: 'Terminal 1' + title: 'Terminal 1', + identity: { + worktreeId: 'wt-2', + tabId: 'tab-existing', + leafId: 'leaf-split', + ptyId: 'pty-split' + } }) storeState.terminalLayoutsByTabId = { @@ -4933,6 +5017,9 @@ describe('useIpcEvents agent status snapshot integration', () => { function buildWindowApi(args: { onSet: (cb: (data: AgentStatusSetData) => void) => () => void onClear?: (cb: (data: AgentStatusClearIpcPayload) => void) => () => void + onLegacyWorkerTerminalRecovery?: ( + cb: (data: { paneKey: string; resolution: 'adopted' | 'exited' }) => void + ) => () => void getSnapshot?: () => Promise drop?: (paneKey: string) => void remoteWorkspace?: Record @@ -5044,6 +5131,8 @@ describe('useIpcEvents agent status snapshot integration', () => { agentStatus: { onSet: args.onSet, onClear: args.onClear ?? vi.fn(() => () => {}), + onLegacyWorkerTerminalRecovery: + args.onLegacyWorkerTerminalRecovery ?? vi.fn(() => () => {}), getSnapshot: args.getSnapshot ?? vi.fn(() => Promise.resolve([])), drop: args.drop ?? vi.fn() }, @@ -5198,6 +5287,51 @@ describe('useIpcEvents agent status snapshot integration', () => { vi.unstubAllGlobals() }) + it('retires the exact sleeping record after adopted or exited legacy worker recovery', async () => { + const clearSleepingAgentSession = vi.fn() + const setSleepingAgentAutomaticResumeBlocked = vi.fn() + let listener: + | ((data: { paneKey: string; resolution: 'adopted' | 'exited' }) => void) + | undefined + const storeState = buildStoreState({ + clearSleepingAgentSession, + setSleepingAgentAutomaticResumeBlocked + }) + + stubReactSyncEffect() + vi.doMock('../store', () => ({ + useAppStore: { + subscribe: vi.fn(() => () => {}), + getState: () => storeState + } + })) + stubAuxiliaryModules() + vi.stubGlobal( + 'window', + buildWindowApi({ + onSet: () => () => {}, + onLegacyWorkerTerminalRecovery: (callback) => { + listener = callback + return () => {} + } + }) + ) + + const { useIpcEvents } = await import('./useIpcEvents') + useIpcEvents() + await Promise.resolve() + expect(listener).toBeTypeOf('function') + + listener?.({ paneKey: 'tab-adopted:leaf-adopted', resolution: 'adopted' }) + expect(clearSleepingAgentSession).toHaveBeenCalledWith('tab-adopted:leaf-adopted') + expect(setSleepingAgentAutomaticResumeBlocked).not.toHaveBeenCalled() + + clearSleepingAgentSession.mockClear() + listener?.({ paneKey: 'tab-exited:leaf-exited', resolution: 'exited' }) + expect(clearSleepingAgentSession).toHaveBeenCalledWith('tab-exited:leaf-exited') + expect(setSleepingAgentAutomaticResumeBlocked).not.toHaveBeenCalled() + }) + it.each([ { partialAuthority: { providerEpoch: 'epoch-current' } }, { partialAuthority: { connectionGeneration: 7 } } diff --git a/src/renderer/src/hooks/useIpcEvents.ts b/src/renderer/src/hooks/useIpcEvents.ts index 68a94915300..37e9303cf66 100644 --- a/src/renderer/src/hooks/useIpcEvents.ts +++ b/src/renderer/src/hooks/useIpcEvents.ts @@ -101,9 +101,14 @@ import { track } from '@/lib/telemetry' import { singlePaneLayoutSnapshot } from '@/store/slices/terminal-helpers' import { buildWorkspaceSessionPayload } from '@/lib/workspace-session' import { persistWorkspaceSessionByHost } from '@/lib/workspace-session-host-persistence' +import { verifyTerminalRevealIdentity } from '@/lib/terminal-reveal-identity' import { getLinearIssueWorkspaceName } from '../../../shared/workspace-name' import type { RuntimeClientEvent } from '../../../shared/runtime-client-events' import { applyHostWorktreeTerminalSleepState } from '@/components/terminal-pane/pty-shutdown-exit-deferral' +import { + resolveLegacyWorkerTerminalRecoveryAction, + rollbackLegacyWorkerTerminalSurfaceInStore +} from './legacy-worker-terminal-recovery-event' import type { AppState } from '../store/types' import { guardPinnedTabClose, resolvePinnedTabLabel } from '../store/pinned-tab-close-guard' import { @@ -181,10 +186,14 @@ const browserAutomationBootstrapLeaseByPageId = new Map { try { const store = useAppStore.getState() - const terminalPresentation = resolveTerminalPresentation({ presentation, activate }) + const terminalPresentation = resolveTerminalPresentation({ + presentation, + activate, + focus + }) const shouldActivate = terminalPresentation === 'focused' const shouldSurfaceOwner = terminalPresentation !== 'background' if (shouldActivate) { @@ -1567,11 +1581,24 @@ export function useIpcEvents(): void { ...(launchAgent ? { launchAgent } : {}) }) } + if (ptyId && terminalPresentation === 'background') { + requestBackgroundTerminalWorktreeMount({ worktreeId, tabIds: [tab.id] }) + } if (requestId) { + const identity = + ptyId && tabId && leafId + ? verifyTerminalRevealIdentity(useAppStore.getState(), { + worktreeId, + tabId, + leafId, + ptyId + }) + : undefined window.api.ui.replyTerminalCreate({ requestId, tabId: tab.id, - title: title ?? tab.title + title: title ?? tab.title, + ...(identity ? { identity } : {}) }) } } catch (err) { @@ -3338,6 +3365,21 @@ export function useIpcEvents(): void { if (unsubscribeMigrationUnsupportedClear) { unsubs.push(unsubscribeMigrationUnsupportedClear) } + const unsubscribeLegacyWorkerTerminalRecovery = + window.api.agentStatus.onLegacyWorkerTerminalRecovery?.((event) => { + const action = resolveLegacyWorkerTerminalRecoveryAction(event) + if (action.kind === 'rollback-surface') { + window.dispatchEvent( + new CustomEvent(CLOSE_TERMINAL_PANE_EVENT, { detail: action.detail }) + ) + rollbackLegacyWorkerTerminalSurfaceInStore(useAppStore.getState(), action.detail) + } else if (action.kind === 'clear-sleeping') { + useAppStore.getState().clearSleepingAgentSession(action.paneKey) + } + }) + if (unsubscribeLegacyWorkerTerminalRecovery) { + unsubs.push(unsubscribeLegacyWorkerTerminalRecovery) + } // Why: main hook server is the durable source of truth; pull the snapshot only after tabs are ready so early startup pushes can be ignored, not buffered. requestAgentStatusSnapshotIfReady() diff --git a/src/renderer/src/lib/pane-manager/pane-manager-types.ts b/src/renderer/src/lib/pane-manager/pane-manager-types.ts index 4f409f16628..4629cf54e3c 100644 --- a/src/renderer/src/lib/pane-manager/pane-manager-types.ts +++ b/src/renderer/src/lib/pane-manager/pane-manager-types.ts @@ -26,7 +26,7 @@ export type PaneSpawnHints = { export type ClosedPaneInfo = { paneId: number leafId: TerminalLeafId - reason?: 'close' | 'detach' + reason?: 'close' | 'detach' | 'retire' } export type PaneExternalDropTarget = { diff --git a/src/renderer/src/lib/pane-manager/pane-manager.ts b/src/renderer/src/lib/pane-manager/pane-manager.ts index ff567643367..f4ab728ad1c 100644 --- a/src/renderer/src/lib/pane-manager/pane-manager.ts +++ b/src/renderer/src/lib/pane-manager/pane-manager.ts @@ -47,6 +47,7 @@ import { PaneIdentityRegistry } from './pane-identity-registry' import { closeManagedPane, detachManagedPaneForExternalMove, + retireManagedPanePreservingPty, splitManagedPane } from './pane-split-close' import { FIRST_PANE_ID } from '../../../../shared/pane-key' @@ -190,6 +191,22 @@ export class PaneManager { }) } + retirePanePreservingPty(paneId: number): boolean { + return retireManagedPanePreservingPty({ + paneId, + activePaneId: this.activePaneId, + panes: this.panes, + root: this.root, + styleOptions: this.styleOptions, + managerOptions: this.options, + getDragCallbacks: () => this.getDragCallbacks(), + releasePaneIdentity: (numericPaneId) => this.identities.release(numericPaneId), + setActivePaneId: (id) => { + this.activePaneId = id + } + }) + } + getPanes(): ManagedPane[] { return Array.from(this.panes.values()).map(toPublicPane) } diff --git a/src/renderer/src/lib/pane-manager/pane-split-close.ts b/src/renderer/src/lib/pane-manager/pane-split-close.ts index 4bc6018372e..e27ffad1a9a 100644 --- a/src/renderer/src/lib/pane-manager/pane-split-close.ts +++ b/src/renderer/src/lib/pane-manager/pane-split-close.ts @@ -166,7 +166,10 @@ type CloseManagedPaneArgs = { setActivePaneId: (paneId: number | null) => void } -function teardownManagedPane(args: CloseManagedPaneArgs, reason: 'close' | 'detach'): void { +function teardownManagedPane( + args: CloseManagedPaneArgs, + reason: 'close' | 'detach' | 'retire' +): void { const pane = args.panes.get(args.paneId) if (!pane) { return @@ -204,6 +207,14 @@ export function detachManagedPaneForExternalMove(args: CloseManagedPaneArgs): bo return true } +export function retireManagedPanePreservingPty(args: CloseManagedPaneArgs): boolean { + if (!args.panes.has(args.paneId) || args.panes.size <= 1) { + return false + } + teardownManagedPane(args, 'retire') + return true +} + function removePaneContainer(args: CloseManagedPaneArgs, pane: ManagedPaneInternal): void { const paneContainer = pane.container const parent = paneContainer.parentElement diff --git a/src/renderer/src/lib/resume-sleeping-agent-session-legacy-worker.test.ts b/src/renderer/src/lib/resume-sleeping-agent-session-legacy-worker.test.ts new file mode 100644 index 00000000000..1ec95258f05 --- /dev/null +++ b/src/renderer/src/lib/resume-sleeping-agent-session-legacy-worker.test.ts @@ -0,0 +1,53 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { SleepingAgentSessionRecord } from '../../../shared/agent-session-resume' +import { useAppStore } from '@/store' +import { resumeSleepingAgentSessionsForWorktree } from './resume-sleeping-agent-session' + +const initialAppStoreState = useAppStore.getState() + +afterEach(() => { + vi.unstubAllGlobals() + useAppStore.setState(initialAppStoreState, true) +}) + +describe('legacy worker sleeping-session recovery', () => { + it('never resumes a proven-exited legacy worker on workspace activation', () => { + const record: SleepingAgentSessionRecord = { + paneKey: 'tab-legacy:leaf-legacy', + tabId: 'tab-legacy', + worktreeId: 'wt-legacy', + agent: 'claude', + providerSession: { key: 'session_id', id: 'session-legacy' }, + prompt: 'continue legacy work', + state: 'working', + capturedAt: 1, + updatedAt: 1, + origin: 'live', + automaticResumeBlockedBy: 'legacy-orchestration-worker' + } + useAppStore.setState({ + tabsByWorktree: { + 'wt-legacy': [ + { + id: 'tab-legacy', + ptyId: null, + worktreeId: 'wt-legacy', + title: 'Legacy worker', + customTitle: null, + color: null, + sortOrder: 0, + createdAt: 1 + } + ] + }, + sleepingAgentSessionsByPaneKey: { [record.paneKey]: record } + } as never) + + expect(resumeSleepingAgentSessionsForWorktree('wt-legacy')).toBe(0) + expect(useAppStore.getState().sleepingAgentSessionsByPaneKey[record.paneKey]).toBe(record) + + useAppStore.getState().clearSleepingAgentSession(record.paneKey) + expect(resumeSleepingAgentSessionsForWorktree('wt-legacy')).toBe(0) + expect(useAppStore.getState().sleepingAgentSessionsByPaneKey[record.paneKey]).toBeUndefined() + }) +}) diff --git a/src/renderer/src/lib/resume-sleeping-agent-session.ts b/src/renderer/src/lib/resume-sleeping-agent-session.ts index 5097d74cea9..198c5d71925 100644 --- a/src/renderer/src/lib/resume-sleeping-agent-session.ts +++ b/src/renderer/src/lib/resume-sleeping-agent-session.ts @@ -173,6 +173,9 @@ export function resumeSleepingAgentSessionsForWorktree( if (options?.skipClaimKeys?.has(claimKey)) { continue } + if (record.automaticResumeBlockedBy === 'legacy-orchestration-worker') { + continue + } if (isInvalidWorktreeActivationRecord(record)) { state.clearSleepingAgentSession(record.paneKey) continue diff --git a/src/renderer/src/lib/terminal-reveal-identity.test.ts b/src/renderer/src/lib/terminal-reveal-identity.test.ts new file mode 100644 index 00000000000..48c0585d1a2 --- /dev/null +++ b/src/renderer/src/lib/terminal-reveal-identity.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from 'vitest' + +import { verifyTerminalRevealIdentity } from './terminal-reveal-identity' + +const expected = { + worktreeId: 'worktree-1', + tabId: 'tab-1', + leafId: 'leaf-1', + ptyId: 'pty-1' +} + +const mismatchCases: { + name: string + state: Parameters[0] +}[] = [ + { + name: 'worktree', + state: { + tabsByWorktree: { 'worktree-2': [{ id: 'tab-1' }] }, + terminalLayoutsByTabId: { + 'tab-1': { ptyIdsByLeafId: { 'leaf-1': 'pty-1' } } + } + } + }, + { + name: 'leaf', + state: { + tabsByWorktree: { 'worktree-1': [{ id: 'tab-1' }] }, + terminalLayoutsByTabId: { + 'tab-1': { ptyIdsByLeafId: { 'leaf-2': 'pty-1' } } + } + } + }, + { + name: 'pty', + state: { + tabsByWorktree: { 'worktree-1': [{ id: 'tab-1' }] }, + terminalLayoutsByTabId: { + 'tab-1': { ptyIdsByLeafId: { 'leaf-1': 'pty-2' } } + } + } + } +] + +describe('verifyTerminalRevealIdentity', () => { + it('returns the exact fresh renderer binding', () => { + expect( + verifyTerminalRevealIdentity( + { + tabsByWorktree: { 'worktree-1': [{ id: 'tab-1' }] }, + terminalLayoutsByTabId: { + 'tab-1': { ptyIdsByLeafId: { 'leaf-1': 'pty-1' } } + } + }, + expected + ) + ).toEqual(expected) + }) + + it.each(mismatchCases)('rejects a mismatched $name owner', ({ state }) => { + expect(() => verifyTerminalRevealIdentity(state, expected)).toThrow( + 'terminal_reveal_identity_mismatch' + ) + }) +}) diff --git a/src/renderer/src/lib/terminal-reveal-identity.ts b/src/renderer/src/lib/terminal-reveal-identity.ts new file mode 100644 index 00000000000..d352e96aedb --- /dev/null +++ b/src/renderer/src/lib/terminal-reveal-identity.ts @@ -0,0 +1,23 @@ +import type { TerminalRevealIdentity } from '../../../shared/terminal-reveal-identity' + +type TerminalRevealState = { + tabsByWorktree: Record + terminalLayoutsByTabId: Record< + string, + { ptyIdsByLeafId?: Record } | undefined + > +} + +export function verifyTerminalRevealIdentity( + state: TerminalRevealState, + expected: TerminalRevealIdentity +): TerminalRevealIdentity { + const ownsTab = state.tabsByWorktree[expected.worktreeId]?.some( + (tab) => tab.id === expected.tabId + ) + const boundPtyId = state.terminalLayoutsByTabId[expected.tabId]?.ptyIdsByLeafId?.[expected.leafId] + if (!ownsTab || boundPtyId !== expected.ptyId) { + throw new Error('terminal_reveal_identity_mismatch') + } + return expected +} diff --git a/src/renderer/src/store/slices/agent-pane-authority.test.ts b/src/renderer/src/store/slices/agent-pane-authority.test.ts index f456688be94..31e5e242c28 100644 --- a/src/renderer/src/store/slices/agent-pane-authority.test.ts +++ b/src/renderer/src/store/slices/agent-pane-authority.test.ts @@ -72,6 +72,39 @@ describe('agent pane authority', () => { expect(retirePaneAuthority).toHaveBeenCalledWith(TARGET) }) + it('can retire live pane authority while retaining a migration recovery fence', () => { + const store = createTestStore() + store.getState().setAgentStatus(TARGET, { state: 'working', prompt: 'target' }) + store.getState().registerAgentLaunchConfig(TARGET, { agentArgs: '', agentEnv: {} }) + store.setState({ + sleepingAgentSessionsByPaneKey: { + [TARGET]: { + paneKey: TARGET, + tabId: 'tab-target', + worktreeId: 'wt-1', + agent: 'codex', + providerSession: { key: 'session_id', id: 'session-1' }, + prompt: 'continue', + state: 'working', + capturedAt: 1, + updatedAt: 1, + automaticResumeBlockedBy: 'legacy-orchestration-worker' + } + } + }) + + store.getState().retireAgentPaneAuthority(TARGET, { preserveSleepingAgentSession: true }) + + const state = store.getState() + expect(state.agentStatusByPaneKey[TARGET]).toBeUndefined() + expect(state.agentLaunchConfigByPaneKey[TARGET]).toBeUndefined() + expect(state.sleepingAgentSessionsByPaneKey[TARGET]).toMatchObject({ + automaticResumeBlockedBy: 'legacy-orchestration-worker' + }) + expect(state.recentlyRetiredAgentStatusPaneKeys[TARGET]).toBe(true) + expect(retirePaneAuthority).toHaveBeenCalledWith(TARGET) + }) + it('keeps a physical pane routed through chained detaches until its current owner closes', () => { const store = createTestStore() store.getState().setAgentStatus(SOURCE, { state: 'working', prompt: 'source' }) diff --git a/src/renderer/src/store/slices/agent-status-provider-session.test.ts b/src/renderer/src/store/slices/agent-status-provider-session.test.ts index 30262d079f0..b450d42ec96 100644 --- a/src/renderer/src/store/slices/agent-status-provider-session.test.ts +++ b/src/renderer/src/store/slices/agent-status-provider-session.test.ts @@ -182,6 +182,64 @@ describe('recordAgentProviderSession', () => { ).toBeUndefined() }) + it('preserves the legacy resume fence only for the same Pi session identity', () => { + const store = createTestStore() + const makeRecord = (transcriptPath: string): SleepingAgentSessionRecord => ({ + paneKey: 'tab-1:leaf-1', + tabId: 'tab-1', + worktreeId: 'wt-1', + agent: 'pi', + providerSession: { + key: 'session_id', + id: 'pi-session-1', + transcriptPath + }, + prompt: '', + state: 'working', + capturedAt: 10, + updatedAt: 10, + automaticResumeBlockedBy: 'legacy-orchestration-worker', + origin: 'live' + }) + store.setState({ + sleepingAgentSessionsByPaneKey: { + 'tab-1:leaf-1': makeRecord('/tmp/pi-session-1.jsonl') + } + } as Partial) + + store.getState().recordAgentProviderSession( + 'tab-1:leaf-1', + 'pi', + { + key: 'session_id', + id: 'pi-session-1', + transcriptPath: '/tmp/pi-session-1.jsonl' + }, + { updatedAt: 20 }, + { tabId: 'tab-1', worktreeId: 'wt-1' } + ) + + expect( + store.getState().sleepingAgentSessionsByPaneKey['tab-1:leaf-1']?.automaticResumeBlockedBy + ).toBe('legacy-orchestration-worker') + + store.getState().recordAgentProviderSession( + 'tab-1:leaf-1', + 'pi', + { + key: 'session_id', + id: 'pi-session-1', + transcriptPath: '/tmp/pi-session-2.jsonl' + }, + { updatedAt: 30 }, + { tabId: 'tab-1', worktreeId: 'wt-1' } + ) + + expect( + store.getState().sleepingAgentSessionsByPaneKey['tab-1:leaf-1']?.automaticResumeBlockedBy + ).toBeUndefined() + }) + it.each(PI_COMPATIBLE_CASES)( 'keeps a completed $label session resumable through manual worktree sleep', async ({ agent, label }) => { diff --git a/src/renderer/src/store/slices/agent-status.ts b/src/renderer/src/store/slices/agent-status.ts index 563a75c617b..337bbdd2e23 100644 --- a/src/renderer/src/store/slices/agent-status.ts +++ b/src/renderer/src/store/slices/agent-status.ts @@ -140,7 +140,10 @@ export type AgentStatusSlice = { /** Exact pane authorities retired while sibling panes in the tab stay live. */ recentlyRetiredAgentStatusPaneKeys: Record - retireAgentPaneAuthority: (paneKey: string) => void + retireAgentPaneAuthority: ( + paneKey: string, + options?: { preserveSleepingAgentSession?: boolean } + ) => void transferAgentPaneAuthority: (args: { fromPaneKey: string toPaneKey: string @@ -233,6 +236,7 @@ export type AgentStatusSlice = { captureAllSleepingAgentSessions: (mode: AllAgentSessionCaptureMode) => void clearSleepingAgentSession: (paneKey: string) => void clearSleepingAgentSessionsByPaneKey: (paneKeys: readonly string[]) => void + setSleepingAgentAutomaticResumeBlocked: (paneKey: string, blocked: boolean) => void clearSleepingAgentSessionsByWorktree: (worktreeId: string) => void pruneSleepingAgentSessions: (validWorktreeIds: Set) => void @@ -1223,7 +1227,7 @@ export const createAgentStatusSlice: StateCreator freshness.schedule(), - retireAgentPaneAuthority: (paneKey) => { + retireAgentPaneAuthority: (paneKey, options) => { const ownerPaneKey = resolveAgentPaneAuthorityKey(paneKey) const retiredPaneKeys = retireAgentPaneAuthorityAliases(paneKey) const retiredPaneKeySet = new Set(retiredPaneKeys) @@ -1251,10 +1255,9 @@ export const createAgentStatusSlice: StateCreator clearSleepingAgentSessionsByPaneKey([paneKey]), clearSleepingAgentSessionsByPaneKey, + setSleepingAgentAutomaticResumeBlocked: (paneKey, blocked) => { + set((s) => { + const current = s.sleepingAgentSessionsByPaneKey[paneKey] + if ( + !current || + (blocked + ? current.automaticResumeBlockedBy === 'legacy-orchestration-worker' + : current.automaticResumeBlockedBy === undefined) + ) { + return s + } + const next = { ...current } + if (blocked) { + next.automaticResumeBlockedBy = 'legacy-orchestration-worker' + } else { + delete next.automaticResumeBlockedBy + } + return { + sleepingAgentSessionsByPaneKey: { + ...s.sleepingAgentSessionsByPaneKey, + [paneKey]: next + } + } + }) + }, clearSleepingAgentSessionsByWorktree: (worktreeId) => { set((s) => { diff --git a/src/renderer/src/web/web-preload-api.ts b/src/renderer/src/web/web-preload-api.ts index f193f035cfd..8ce018f4fbd 100644 --- a/src/renderer/src/web/web-preload-api.ts +++ b/src/renderer/src/web/web-preload-api.ts @@ -544,6 +544,7 @@ function createWebPreloadApi(): Partial { writeJson(UI_STORAGE_KEY, mergeWebUIState(readLocalWebUIState(), ui)) }, awaitFirstWindowStartupServices: () => Promise.resolve(), + recoverLegacyWorkerTerminalsForRendererStartup: () => Promise.resolve(), startupDiagnostic: () => Promise.resolve(), getKeyboardInputSourceId: () => Promise.resolve(null), setUnreadDockBadgeCount: () => Promise.resolve(), @@ -835,6 +836,7 @@ function createWebPreloadApi(): Partial { inferQuestionAnswered: () => Promise.resolve(false), onMigrationUnsupported: () => noopUnsubscribe, onMigrationUnsupportedClear: () => noopUnsubscribe, + onLegacyWorkerTerminalRecovery: () => noopUnsubscribe, getMigrationUnsupportedSnapshot: () => Promise.resolve([]), drop: () => {}, dropByTabPrefix: () => {}, diff --git a/src/shared/agent-session-resume.ts b/src/shared/agent-session-resume.ts index 50581c5c919..19ccde5e783 100644 --- a/src/shared/agent-session-resume.ts +++ b/src/shared/agent-session-resume.ts @@ -60,6 +60,9 @@ export type SleepingAgentSessionRecord = { * so only the pane's own cold-restore path may consume them — activation * launching a tab too would duplicate a warm-reattached session (#5232). */ origin?: 'worktree-sleep' | 'quit' | 'live' + /** Prevents provider-session relaunch while main reconciles a durable + * orchestration assignment against authoritative PTY inventory. */ + automaticResumeBlockedBy?: 'legacy-orchestration-worker' } const RESUMABLE_TUI_AGENT_SET: ReadonlySet = new Set(RESUMABLE_TUI_AGENTS) diff --git a/src/shared/orchestration-check-output.test.ts b/src/shared/orchestration-check-output.test.ts new file mode 100644 index 00000000000..00ffbb79da3 --- /dev/null +++ b/src/shared/orchestration-check-output.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from 'vitest' +import { prepareOrchestrationCheckOutput } from './orchestration-check-output' + +describe('prepareOrchestrationCheckOutput', () => { + it('keeps mixed read-only mail safe and current Run replies executable', () => { + const prepared = prepareOrchestrationCheckOutput( + { + count: 2, + messages: [ + { + id: 'msg_current', + run_id: 'run_adopted', + delivery_contract: 'current_delivery', + from_handle: 'term_worker', + to_handle: 'run:run_adopted', + subject: 'Question' + }, + { + id: 'msg_legacy', + run_id: 'run_legacy_local', + delivery_contract: 'audit_only', + from_handle: 'term_legacy', + to_handle: 'term_coord', + subject: 'Old reply' + } + ], + formatted: '[Reply: unsafe stale formatter output]' + }, + 'term_current_coord', + true + ) + + expect(prepared.formatted).toContain( + '[Reply: orca orchestration reply --id msg_current --body "..."]' + ) + expect(prepared.formatted).not.toContain('--from run:run_adopted') + expect(prepared.formatted).toContain( + '[Inspection only: reply and acknowledgment are unavailable.]' + ) + expect(prepared.formatted).not.toContain('unsafe stale formatter output') + }) +}) diff --git a/src/shared/orchestration-check-output.ts b/src/shared/orchestration-check-output.ts new file mode 100644 index 00000000000..e736b92f4dd --- /dev/null +++ b/src/shared/orchestration-check-output.ts @@ -0,0 +1,199 @@ +import { ORCHESTRATION_LEGACY_RUN_ID } from './orchestration-rpc-contract' + +export type OrchestrationMessageSummary = { + id: string + run_id?: string + delivery_contract?: 'legacy_direct' | 'current_delivery' | 'audit_only' + from_handle: string + to_handle?: string + subject?: string + type?: string + body?: string + payload?: string | null + priority?: string + read?: number +} + +export type LegacyCompatibilityResult = { + recovery?: boolean + readOnly?: boolean + ackMessageIds?: string[] + answerAcknowledgement?: { + questionId: string + answerMessageId: string + } + currentDelivery?: { + runId: string + checkCommand: string + ackCommand: string + } + resumeRequired?: boolean + resumeCommand?: string +} + +export type OrchestrationCheckOutput = { + messages: OrchestrationMessageSummary[] + count: number + formatted?: string + deliveryId?: string | null + timedOut?: boolean + cancelled?: boolean + connectionLost?: boolean + legacyCompatibility?: LegacyCompatibilityResult +} + +export function formatMessageReadOnlyTag( + message: OrchestrationMessageSummary, + legacyCompatibilityActive = false +): string { + return isLegacyReadOnlyMessage(message, legacyCompatibilityActive) ? ' [legacy, read-only]' : '' +} + +export function isLegacyReadOnlyMessage( + message: OrchestrationMessageSummary, + legacyCompatibilityActive = false +): boolean { + return ( + message.run_id === ORCHESTRATION_LEGACY_RUN_ID || + (message.delivery_contract === 'legacy_direct' && !legacyCompatibilityActive) || + message.delivery_contract === 'audit_only' + ) +} + +export function formatOrchestrationCheckText( + result: OrchestrationCheckOutput, + checkedTerminal: string +): string { + const prepared = prepareOrchestrationCheckOutput( + result, + checkedTerminal, + result.formatted !== undefined + ) + const compatibilityActive = Boolean( + prepared.legacyCompatibility && !prepared.legacyCompatibility.readOnly + ) + const legacyHeader = prepared.legacyCompatibility + ? prepared.legacyCompatibility.readOnly + ? '[LEGACY READ-ONLY]\n' + : prepared.legacyCompatibility.recovery + ? '[LEGACY RECOVERY REPLAY — MAY HAVE BEEN SEEN]\n' + : '[LEGACY COMPATIBILITY]\n' + : '' + const deliveryNotice = formatCurrentDeliveryNotice(prepared.legacyCompatibility?.currentDelivery) + if (prepared.formatted) { + return `${legacyHeader}${prepared.formatted}${deliveryNotice}` + } + if (prepared.count === 0) { + if (prepared.timedOut) { + return `${legacyHeader}Wait timed out; no messages were consumed.${deliveryNotice}` + } + if (prepared.cancelled) { + const cancelled = prepared.connectionLost + ? 'Wait cancelled because the connection closed; no messages were consumed.' + : 'Wait cancelled; no messages were consumed.' + return `${legacyHeader}${cancelled}${deliveryNotice}` + } + return `${legacyHeader}No messages.${deliveryNotice}` + } + const rendered = prepared.messages + .map( + (message) => + `${message.id}${formatMessageReadOnlyTag( + message, + compatibilityActive + )} [${message.type ?? 'status'}] from=${message.from_handle} "${message.subject}"` + ) + .join('\n') + const output = prepared.deliveryId ? `Delivery ${prepared.deliveryId}\n${rendered}` : rendered + return `${legacyHeader}${output}${deliveryNotice}` +} + +export function prepareOrchestrationCheckOutput( + result: T, + checkedTerminal: string, + formattedRequested: boolean +): T { + const compatibilityActive = Boolean( + result.legacyCompatibility && !result.legacyCompatibility.readOnly + ) + if ( + !formattedRequested || + !result.messages.some((message) => isLegacyReadOnlyMessage(message, compatibilityActive)) + ) { + return result + } + return { + ...result, + formatted: formatLegacyAwareCheckMessages(result.messages, checkedTerminal, compatibilityActive) + } +} + +function formatMessagePriorityTag(message: OrchestrationMessageSummary): string { + return message.priority === 'urgent' ? ' [URGENT]' : message.priority === 'high' ? ' [HIGH]' : '' +} + +function escapeTerminalControlCharacters(value: string): string { + return [...value] + .map((character) => { + const code = character.charCodeAt(0) + if (character === '\n' || (code >= 0x20 && code < 0x7f) || code > 0x9f) { + return character + } + return `\\x${code.toString(16).padStart(2, '0')}` + }) + .join('') +} + +function formatQuotedMessageField(label: string, value?: string): string { + return `[${label}]\n${escapeTerminalControlCharacters(value ?? '') + .split('\n') + .map((line) => ` ${line}`) + .join('\n')}` +} + +function formatLegacyAwareCheckMessages( + messages: OrchestrationMessageSummary[], + checkedTerminal: string, + legacyCompatibilityActive = false +): string { + return messages + .map((message) => { + const legacyReadOnly = isLegacyReadOnlyMessage(message, legacyCompatibilityActive) + const lines = [ + `${message.id}${formatMessageReadOnlyTag(message, legacyCompatibilityActive)}${formatMessagePriorityTag(message)} [${message.type ?? 'status'}] from=${message.from_handle}`, + formatQuotedMessageField('subject', message.subject) + ] + if (legacyReadOnly) { + lines.push('[Inspection only: reply and acknowledgment are unavailable.]') + } + if (message.body) { + lines.push(formatQuotedMessageField('body', message.body)) + } + if (message.payload) { + lines.push(formatQuotedMessageField('payload', message.payload)) + } + if (!legacyReadOnly) { + const replyTarget = message.to_handle ?? checkedTerminal + const replyFrom = + replyTarget.startsWith('run:') || replyTarget.startsWith('dispatch:') + ? '' + : ` --from ${replyTarget}` + lines.push(`[Reply: orca orchestration reply --id ${message.id}${replyFrom} --body "..."]`) + } + return lines.join('\n') + }) + .join('\n\n') +} + +function formatCurrentDeliveryNotice( + delivery: LegacyCompatibilityResult['currentDelivery'] +): string { + if (!delivery) { + return '' + } + return ( + `\n[CURRENT RUN MAIL WAITING]\n` + + `Read: ${delivery.checkCommand}\n` + + `Then acknowledge the delivery ID it prints: ${delivery.ackCommand}` + ) +} diff --git a/src/shared/orchestration-compatibility-evidence.test.ts b/src/shared/orchestration-compatibility-evidence.test.ts new file mode 100644 index 00000000000..28da8335f7a --- /dev/null +++ b/src/shared/orchestration-compatibility-evidence.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from 'vitest' +import { + ORCHESTRATION_COMPATIBILITY_ATTACHMENT_ENV, + ORCHESTRATION_COMPATIBILITY_HOST_ID_ENV, + ORCHESTRATION_COMPATIBILITY_HOST_INCARNATION_ENV, + ORCHESTRATION_COMPATIBILITY_HOST_KIND_ENV, + readOrchestrationCompatibilityEvidence, + redactOrchestrationCompatibilitySecrets +} from './orchestration-compatibility-evidence' + +describe('orchestration compatibility evidence', () => { + it('reads inherited pane evidence and a complete runtime-stamped WSL scope', () => { + expect( + readOrchestrationCompatibilityEvidence({ + ORCA_TERMINAL_HANDLE: 'term_wsl', + ORCA_PANE_KEY: 'tab:leaf', + ORCA_AGENT_LAUNCH_TOKEN: 'launch-secret', + [ORCHESTRATION_COMPATIBILITY_HOST_KIND_ENV]: 'wsl', + [ORCHESTRATION_COMPATIBILITY_HOST_ID_ENV]: 'local', + [ORCHESTRATION_COMPATIBILITY_HOST_INCARNATION_ENV]: 'Ubuntu' + }) + ).toEqual({ + terminalHandle: 'term_wsl', + paneKey: 'tab:leaf', + launchToken: 'launch-secret', + host: { kind: 'wsl', hostId: 'local', distro: 'Ubuntu' } + }) + }) + + it('rejects partial host stamps instead of accepting caller-chosen scope', () => { + expect( + readOrchestrationCompatibilityEvidence({ + ORCA_PANE_KEY: 'tab:leaf', + [ORCHESTRATION_COMPATIBILITY_HOST_KIND_ENV]: 'ssh', + [ORCHESTRATION_COMPATIBILITY_HOST_ID_ENV]: 'saved-target', + [ORCHESTRATION_COMPATIBILITY_HOST_INCARNATION_ENV]: 'connection-only' + }) + ).toEqual({ paneKey: 'tab:leaf' }) + }) + + it('redacts evidence from nested CLI error and structured-log values', () => { + const redacted = redactOrchestrationCompatibilitySecrets({ + compatibilityInvocationId: 'safe-operation-id', + orchestrationCompatibilityEvidence: { + launchToken: 'secret', + host: { + kind: 'ssh', + [ORCHESTRATION_COMPATIBILITY_ATTACHMENT_ENV]: 'attachment-secret' + } + } + }) + + expect(redacted).toEqual({ + compatibilityInvocationId: 'safe-operation-id', + orchestrationCompatibilityEvidence: '[redacted]' + }) + expect(JSON.stringify(redacted)).not.toContain('secret') + }) + + it('redacts detached SSH attachment proof fields', () => { + expect( + redactOrchestrationCompatibilitySecrets({ + host: { + targetId: 'saved-target', + connectionIncarnation: 'connection-secret', + attachmentId: 'attachment-secret' + } + }) + ).toEqual({ + host: { + targetId: 'saved-target', + connectionIncarnation: '[redacted]', + attachmentId: '[redacted]' + } + }) + }) + + it('redacts cyclic arrays without recursing forever', () => { + const cyclic: unknown[] = [] + cyclic.push(cyclic) + + expect(redactOrchestrationCompatibilitySecrets(cyclic)).toEqual(['[circular]']) + }) +}) diff --git a/src/shared/orchestration-compatibility-evidence.ts b/src/shared/orchestration-compatibility-evidence.ts new file mode 100644 index 00000000000..72c7a03c20c --- /dev/null +++ b/src/shared/orchestration-compatibility-evidence.ts @@ -0,0 +1,107 @@ +const MAX_EVIDENCE_FIELD_LENGTH = 4_096 + +export const ORCHESTRATION_COMPATIBILITY_HOST_KIND_ENV = + 'ORCA_ORCHESTRATION_COMPATIBILITY_HOST_KIND' +export const ORCHESTRATION_COMPATIBILITY_HOST_ID_ENV = 'ORCA_ORCHESTRATION_COMPATIBILITY_HOST_ID' +export const ORCHESTRATION_COMPATIBILITY_HOST_INCARNATION_ENV = + 'ORCA_ORCHESTRATION_COMPATIBILITY_HOST_INCARNATION' +export const ORCHESTRATION_COMPATIBILITY_ATTACHMENT_ENV = + 'ORCA_ORCHESTRATION_COMPATIBILITY_ATTACHMENT' + +export type OrchestrationCompatibilityHostStamp = + | { + kind: 'wsl' + hostId: string + distro: string + } + | { + kind: 'ssh' + targetId: string + connectionIncarnation: string + attachmentId: string + } + +export type OrchestrationCompatibilityEvidence = { + terminalHandle?: string + paneKey?: string + launchToken?: string + host?: OrchestrationCompatibilityHostStamp +} + +const SECRET_KEYS = new Set([ + 'launchToken', + 'connectionIncarnation', + 'attachmentId', + 'compatibilityEvidence', + 'orchestrationCompatibilityEvidence', + 'ORCA_AGENT_LAUNCH_TOKEN', + ORCHESTRATION_COMPATIBILITY_HOST_INCARNATION_ENV, + ORCHESTRATION_COMPATIBILITY_ATTACHMENT_ENV +]) + +function boundedValue(value: string | undefined): string | undefined { + const trimmed = value?.trim() + return trimmed && trimmed.length <= MAX_EVIDENCE_FIELD_LENGTH ? trimmed : undefined +} + +export function readOrchestrationCompatibilityEvidence( + env: Readonly> +): OrchestrationCompatibilityEvidence | undefined { + const terminalHandle = boundedValue(env.ORCA_TERMINAL_HANDLE) + const paneKey = boundedValue(env.ORCA_PANE_KEY) + const launchToken = boundedValue(env.ORCA_AGENT_LAUNCH_TOKEN) + const host = readHostStamp(env) + if (!terminalHandle && !paneKey && !launchToken && !host) { + return undefined + } + return { + ...(terminalHandle ? { terminalHandle } : {}), + ...(paneKey ? { paneKey } : {}), + ...(launchToken ? { launchToken } : {}), + ...(host ? { host } : {}) + } +} + +function readHostStamp( + env: Readonly> +): OrchestrationCompatibilityHostStamp | undefined { + const kind = boundedValue(env[ORCHESTRATION_COMPATIBILITY_HOST_KIND_ENV]) + const hostId = boundedValue(env[ORCHESTRATION_COMPATIBILITY_HOST_ID_ENV]) + const incarnation = boundedValue(env[ORCHESTRATION_COMPATIBILITY_HOST_INCARNATION_ENV]) + const attachment = boundedValue(env[ORCHESTRATION_COMPATIBILITY_ATTACHMENT_ENV]) + if (kind === 'wsl' && hostId && incarnation) { + return { kind, hostId, distro: incarnation } + } + if (kind === 'ssh' && hostId && incarnation && attachment) { + return { + kind, + targetId: hostId, + connectionIncarnation: incarnation, + attachmentId: attachment + } + } + return undefined +} + +export function redactOrchestrationCompatibilitySecrets(value: unknown): unknown { + return redactValue(value, new WeakSet()) +} + +function redactValue(value: unknown, seen: WeakSet): unknown { + if (typeof value !== 'object' || value === null) { + return value + } + if (seen.has(value)) { + return '[circular]' + } + seen.add(value) + if (Array.isArray(value)) { + return value.map((entry) => redactValue(entry, seen)) + } + return Object.fromEntries( + Object.entries(value).map(([key, entry]) => [ + key, + SECRET_KEYS.has(key) ? '[redacted]' : redactValue(entry, seen) + ]) + ) +} diff --git a/src/shared/orchestration-message-wait-timeout.ts b/src/shared/orchestration-message-wait-timeout.ts new file mode 100644 index 00000000000..608b807bc99 --- /dev/null +++ b/src/shared/orchestration-message-wait-timeout.ts @@ -0,0 +1 @@ +export const ORCHESTRATION_MESSAGE_WAIT_DEFAULT_TIMEOUT_MS = 2 * 60 * 1000 diff --git a/src/shared/remote-runtime-client.ts b/src/shared/remote-runtime-client.ts index a522e709e71..dafa2261feb 100644 --- a/src/shared/remote-runtime-client.ts +++ b/src/shared/remote-runtime-client.ts @@ -107,7 +107,9 @@ export async function sendRemoteRuntimeRequest( params, orchestrationCapability: envelope?.orchestrationCapability, orchestrationContractVersion: envelope?.orchestrationContractVersion, - orchestrationRequestId: envelope?.orchestrationRequestId + orchestrationRequestId: envelope?.orchestrationRequestId, + compatibilityInvocationId: envelope?.compatibilityInvocationId, + orchestrationCompatibilityEvidence: envelope?.orchestrationCompatibilityEvidence }) ) } diff --git a/src/shared/runtime-rpc-envelope.ts b/src/shared/runtime-rpc-envelope.ts index d12c762dd9d..d407de72cb9 100644 --- a/src/shared/runtime-rpc-envelope.ts +++ b/src/shared/runtime-rpc-envelope.ts @@ -2,6 +2,7 @@ // Keeping the envelope contract here avoids making those clients import each // other just to validate the shared RPC frame shape. import { z } from 'zod' +import type { OrchestrationCompatibilityEvidence } from './orchestration-compatibility-evidence' // Why: clients and runtimes update independently; strip additive envelope // fields while continuing to validate every known discriminator and field. @@ -78,6 +79,8 @@ export type RuntimeOrchestrationEnvelope = { orchestrationCapability?: string orchestrationContractVersion?: number orchestrationRequestId?: string + compatibilityInvocationId?: string + orchestrationCompatibilityEvidence?: OrchestrationCompatibilityEvidence } export type RuntimeRpcKeepaliveFrame = z.infer diff --git a/src/shared/terminal-reveal-identity.ts b/src/shared/terminal-reveal-identity.ts new file mode 100644 index 00000000000..514e45e3b6d --- /dev/null +++ b/src/shared/terminal-reveal-identity.ts @@ -0,0 +1,14 @@ +export type TerminalRevealIdentity = { + worktreeId: string + tabId: string + leafId: string + ptyId: string +} + +export type TerminalTabCreateReply = { + requestId: string + tabId?: string + title?: string + identity?: TerminalRevealIdentity + error?: string +} diff --git a/src/shared/workspace-session-sleeping-agents.ts b/src/shared/workspace-session-sleeping-agents.ts index 5b8cc8be708..58f19ce089e 100644 --- a/src/shared/workspace-session-sleeping-agents.ts +++ b/src/shared/workspace-session-sleeping-agents.ts @@ -97,7 +97,8 @@ const sleepingAgentSessionRecordSchema = z interrupted: z.boolean().optional(), connectionId: z.string().nullable().optional(), launchConfig: sleepingAgentLaunchConfigSchema.optional(), - origin: z.enum(['worktree-sleep', 'quit', 'live']).optional() + origin: z.enum(['worktree-sleep', 'quit', 'live']).optional(), + automaticResumeBlockedBy: z.enum(['legacy-orchestration-worker']).optional() }) .refine( (record) => getAgentResumeArgv(record.agent, record.providerSession) !== null, diff --git a/tests/e2e/global-setup.ts b/tests/e2e/global-setup.ts index b67eb68d7c6..cea4e7717a7 100644 --- a/tests/e2e/global-setup.ts +++ b/tests/e2e/global-setup.ts @@ -20,11 +20,13 @@ import { prepareDockerSshRelayImage } from './helpers/docker-ssh-relay-image' /** Temp file where the test repo path is stored for the fixture to read. */ export const TEST_REPO_PATH_FILE = path.join(os.tmpdir(), 'orca-e2e-test-repo-path.txt') const ELECTRON_E2E_BUILD_TIMEOUT_MS = 300_000 +const CLI_E2E_BUILD_TIMEOUT_MS = 120_000 const WEB_E2E_BUILD_TIMEOUT_MS = 300_000 export default function globalSetup(): void { const root = process.cwd() const outMain = path.join(root, 'out', 'main', 'index.js') + const outCli = path.join(root, 'out', 'cli', 'index.js') const outWeb = path.join(root, 'out', 'web', 'web-index.html') // ── 1. Build the Electron app ────────────────────────────────────── @@ -44,6 +46,17 @@ export default function globalSetup(): void { }) console.error('[e2e] Build complete.') } + if (process.env.SKIP_BUILD && existsSync(outCli)) { + console.error('[e2e] SKIP_BUILD set and out/cli/index.js exists — skipping CLI build') + } else { + console.error('[e2e] Building bundled CLI...') + execSync('pnpm run build:cli', { + cwd: root, + stdio: 'inherit', + timeout: CLI_E2E_BUILD_TIMEOUT_MS + }) + console.error('[e2e] CLI build complete.') + } if (process.env.ORCA_E2E_WEB_CLIENT === '1') { if (process.env.SKIP_BUILD && existsSync(outWeb)) { console.error('[e2e] SKIP_BUILD set and web client exists — skipping web build') diff --git a/tests/e2e/orchestration-legacy-worker-missing-terminal-recovery.spec.ts b/tests/e2e/orchestration-legacy-worker-missing-terminal-recovery.spec.ts new file mode 100644 index 00000000000..f554fd323ac --- /dev/null +++ b/tests/e2e/orchestration-legacy-worker-missing-terminal-recovery.spec.ts @@ -0,0 +1,347 @@ +import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import type { ElectronApplication } from '@stablyai/playwright-test' +import { test, expect } from './helpers/orca-app' +import { TEST_REPO_PATH_FILE } from './global-setup' +import { attachRepoAndOpenTerminal, createRestartSession } from './helpers/orca-restart' +import { + ensureTerminalVisible, + getActiveTabId, + waitForActiveWorktree, + waitForSessionReady +} from './helpers/store' +import { waitForActivePaneHookDescriptor, waitForActivePanePtyId } from './helpers/terminal' +import { RuntimeClient } from '../../src/cli/runtime-client' +import { DaemonClient } from '../../src/main/daemon/client' +import { getDaemonSocketPath, getDaemonTokenPath } from '../../src/main/daemon/daemon-spawner' +import Database from '../../src/main/sqlite/sync-database' +import { LEGACY_CONTRACT_VERSION } from '../../src/main/runtime/orchestration/db' +import { DEFAULT_LOCAL_ORCA_PROFILE_ID } from '../../src/shared/orca-profiles' +import type { RuntimeTerminalListResult, RuntimeTerminalRead } from '../../src/shared/runtime-types' + +const PROVIDER_SESSION_ID = 'e2e-missing-legacy-worker' +const fakeCliDir = mkdtempSync(path.join(os.tmpdir(), 'orca-e2e-missing-legacy-worker-')) +const spawnLedgerPath = path.join(fakeCliDir, 'spawn.jsonl') +const interruptionLedgerPath = path.join(fakeCliDir, 'interruption.jsonl') +const fakeCodexSource = ` +const { appendFileSync } = require('node:fs') +function appendLedger(envName, event) { + const ledgerPath = process.env[envName] + if (!ledgerPath) return + try { + appendFileSync(ledgerPath, JSON.stringify({ pid: process.pid, ...event }) + '\\n') + } catch {} +} +if (process.argv.slice(2).includes('app-server')) { + process.stderr.write("error: unrecognized subcommand 'app-server'\\n") + process.exit(2) +} +appendLedger('ORCA_E2E_SPAWN_LEDGER', { event: 'spawn' }) +process.stdout.write('\\u001b]0;Codex Ready\\u0007OpenAI Codex\\nmodel: e2e\\ndirectory: e2e\\n') +let acknowledged = false +process.stdin.on('data', (chunk) => { + const input = chunk.toString() + if (input.includes('\\x03')) { + appendLedger('ORCA_E2E_INTERRUPTION_LEDGER', { event: 'stdin-ctrl-c' }) + } + if (!acknowledged && input.includes('\\r')) { + acknowledged = true + process.stdout.write('ACK\\n') + } +}) +for (const signal of ['SIGINT', 'SIGHUP', 'SIGTERM']) { + process.on(signal, () => { + appendLedger('ORCA_E2E_INTERRUPTION_LEDGER', { event: 'signal', signal }) + process.exit(0) + }) +} +process.stdin.resume() +setInterval(() => {}, 60_000) +` + +if (process.platform === 'win32') { + writeFileSync(path.join(fakeCliDir, 'fake-codex.js'), fakeCodexSource) + writeFileSync( + path.join(fakeCliDir, 'codex.cmd'), + '@echo off\r\nnode "%~dp0\\fake-codex.js" %*\r\n' + ) +} else { + const executable = path.join(fakeCliDir, 'codex') + writeFileSync(executable, `#!/usr/bin/env node\n${fakeCodexSource}`) + chmodSync(executable, 0o755) +} + +type LedgerEvent = { pid: number; event: string; signal?: string } + +function readLedger(ledgerPath: string): LedgerEvent[] { + if (!existsSync(ledgerPath)) { + return [] + } + return readFileSync(ledgerPath, 'utf8') + .split(/\r?\n/) + .filter(Boolean) + .map((line) => JSON.parse(line) as LedgerEvent) +} + +function isProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0) + return true + } catch { + return false + } +} + +async function removeDetachedDaemonSession(userDataDir: string, ptyId: string): Promise { + const daemonDir = path.join(userDataDir, 'daemon') + const client = new DaemonClient({ + socketPath: getDaemonSocketPath(daemonDir), + tokenPath: getDaemonTokenPath(daemonDir) + }) + try { + await client.ensureConnected() + await client.request('kill', { sessionId: ptyId, immediate: true }) + } finally { + client.disconnect() + } +} + +async function detachedDaemonSessionExists(userDataDir: string, ptyId: string): Promise { + const daemonDir = path.join(userDataDir, 'daemon') + const client = new DaemonClient({ + socketPath: getDaemonSocketPath(daemonDir), + tokenPath: getDaemonTokenPath(daemonDir) + }) + try { + await client.ensureConnected() + const result = await client.request<{ sessions: { sessionId: string }[] }>( + 'listSessions', + undefined + ) + return result.sessions.some((session) => session.sessionId === ptyId) + } finally { + client.disconnect() + } +} + +function persistedDataPath(userDataDir: string): string { + return path.join(userDataDir, 'profiles', DEFAULT_LOCAL_ORCA_PROFILE_ID, 'orca-data.json') +} + +function hasPersistedResumeRecord(userDataDir: string, paneKey: string): boolean { + const data = JSON.parse(readFileSync(persistedDataPath(userDataDir), 'utf8')) as { + workspaceSession?: { + sleepingAgentSessionsByPaneKey?: Record + } + } + return ( + data.workspaceSession?.sleepingAgentSessionsByPaneKey?.[paneKey]?.providerSession?.id === + PROVIDER_SESSION_ID + ) +} + +function markDispatchLegacy(userDataDir: string, dispatchId: string): void { + const db = new Database(path.join(userDataDir, 'orchestration.db')) + try { + db.prepare( + `UPDATE dispatch_contexts + SET contract_version = ?, capability_hash = NULL, capability_revoked_at = NULL, + launch_token_hash = NULL + WHERE id = ?` + ).run(LEGACY_CONTRACT_VERSION, dispatchId) + } finally { + db.close() + } +} + +function readSettledDispatch(userDataDir: string, dispatchId: string): unknown { + const db = new Database(path.join(userDataDir, 'orchestration.db')) + try { + return db + .prepare( + `SELECT dc.status AS dispatch_status, wd.state AS worker_state, wd.stage + FROM dispatch_contexts dc + INNER JOIN worker_dispatches wd ON wd.dispatch_id = dc.id + WHERE dc.id = ?` + ) + .get(dispatchId) + } finally { + db.close() + } +} + +test.describe.configure({ mode: 'serial' }) + +test.afterAll(() => { + rmSync(fakeCliDir, { recursive: true, force: true }) +}) + +test('a missing legacy worker cannot spawn a replacement during restart recovery', async (// oxlint-disable-next-line no-empty-pattern -- This restart test owns both Electron launches. +{}, testInfo) => { + test.setTimeout(300_000) + rmSync(spawnLedgerPath, { force: true }) + rmSync(interruptionLedgerPath, { force: true }) + const repoPath = existsSync(TEST_REPO_PATH_FILE) + ? readFileSync(TEST_REPO_PATH_FILE, 'utf8').trim() + : '' + test.skip(!repoPath || !existsSync(repoPath), 'Global setup did not produce a seeded test repo') + + const session = createRestartSession(testInfo, { + PATH: `${fakeCliDir}${path.delimiter}${process.env.PATH ?? ''}`, + ORCA_E2E_SPAWN_LEDGER: spawnLedgerPath, + ORCA_E2E_INTERRUPTION_LEDGER: interruptionLedgerPath + }) + let firstApp: ElectronApplication | null = null + let secondApp: ElectronApplication | null = null + + try { + const first = await session.launch() + firstApp = first.app + const worktreeId = await attachRepoAndOpenTerminal(first.page, repoPath) + await waitForSessionReady(first.page) + await ensureTerminalVisible(first.page) + await getActiveTabId(first.page) + await waitForActivePanePtyId(first.page) + const coordinatorPane = await waitForActivePaneHookDescriptor(first.page) + const firstClient = new RuntimeClient(session.userDataDir, 30_000, null, null) + const coordinator = await firstClient.call<{ terminal: { handle: string } }>( + 'terminal.resolvePane', + { paneKey: coordinatorPane.paneKey } + ) + const coordinatorTerminal = await firstClient.call<{ + terminal: { worktreeId: string } + }>('terminal.show', { terminal: coordinator.result.terminal.handle }) + await expect + .poll(async () => { + const listed = await firstClient.call<{ worktrees: { id: string }[] }>('worktree.list', {}) + return listed.result.worktrees.some( + (candidate) => candidate.id === coordinatorTerminal.result.terminal.worktreeId + ) + }) + .toBe(true) + const run = await firstClient.call<{ run: { id: string } }>('orchestration.runCreate', { + objective: 'Missing legacy worker recovery', + from: coordinator.result.terminal.handle + }) + const task = await firstClient.call<{ task: { id: string } }>('orchestration.taskCreate', { + spec: 'Respond ACK and remain idle', + run: run.result.run.id, + callerTerminalHandle: coordinator.result.terminal.handle + }) + await firstClient.call('orchestration.workerStart', { + task: task.result.task.id, + from: coordinator.result.terminal.handle, + agent: 'codex', + timeoutMs: 15_000 + }) + + let worker = ( + await firstClient.call('terminal.list') + ).result.terminals.find((terminal) => terminal.title === 'Codex Ready') + await expect + .poll(async () => { + const listed = await firstClient.call('terminal.list') + worker = listed.result.terminals.find((terminal) => terminal.title === 'Codex Ready') + return worker?.ptyId ?? null + }) + .toBeTruthy() + const workerPaneKey = `${worker!.tabId}:${worker!.leafId}` + await expect + .poll(async () => { + const read = await firstClient.call<{ terminal: RuntimeTerminalRead }>('terminal.read', { + terminal: worker!.handle, + limit: 100 + }) + return read.result.terminal.tail.join('\n') + }) + .toContain('ACK') + const dispatch = await firstClient.call<{ + dispatch: { id: string } | null + }>('orchestration.dispatchShow', { task: task.result.task.id }) + expect(dispatch.result.dispatch?.id).toBeTruthy() + await expect.poll(() => readLedger(spawnLedgerPath)).toHaveLength(1) + const [initialSpawn] = readLedger(spawnLedgerPath) + + const transcriptPath = session.seedCodexResumeRollout(PROVIDER_SESSION_ID, repoPath) + await first.page.evaluate( + ({ paneKey, tabId, workerWorktreeId, terminalHandle, transcript }) => { + window.__store?.getState().setAgentStatus( + paneKey, + { state: 'working', prompt: 'Respond ACK and remain idle', agentType: 'codex' }, + 'Codex Ready', + undefined, + { tabId, worktreeId: workerWorktreeId, terminalHandle }, + { + providerSession: { + key: 'session_id', + id: 'e2e-missing-legacy-worker', + transcriptPath: transcript + }, + launchConfig: { + agentCommand: 'codex', + agentArgs: '--dangerously-bypass-approvals-and-sandbox', + agentEnv: {} + } + } + ) + window.__store?.getState().captureAllSleepingAgentSessions('quit') + }, + { + paneKey: workerPaneKey, + tabId: worker!.tabId, + workerWorktreeId: worker!.worktreeId, + terminalHandle: worker!.handle, + transcript: transcriptPath + } + ) + await expect.poll(() => hasPersistedResumeRecord(session.userDataDir, workerPaneKey)).toBe(true) + markDispatchLegacy(session.userDataDir, dispatch.result.dispatch!.id) + + await session.close(firstApp) + firstApp = null + await removeDetachedDaemonSession(session.userDataDir, worker!.ptyId) + await expect + .poll(() => detachedDaemonSessionExists(session.userDataDir, worker!.ptyId)) + .toBe(false) + await expect.poll(() => isProcessAlive(initialSpawn.pid)).toBe(false) + rmSync(interruptionLedgerPath, { force: true }) + + const second = await session.launch() + secondApp = second.app + await waitForSessionReady(second.page) + expect(await waitForActiveWorktree(second.page)).toBe(worktreeId) + const secondClient = new RuntimeClient(session.userDataDir, 30_000, null, null) + await expect + .poll(async () => { + const listed = await secondClient.call('terminal.list') + return listed.result.terminals.filter( + (terminal) => terminal.ptyId === worker!.ptyId || terminal.title === 'Codex Ready' + ) + }) + .toEqual([]) + await expect( + second.page.locator(`[data-testid="sortable-tab"][data-tab-id="${worker!.tabId}"]`) + ).toHaveCount(0) + await expect + .poll(() => readSettledDispatch(session.userDataDir, dispatch.result.dispatch!.id)) + .toEqual({ + dispatch_status: 'failed', + worker_state: 'abandoned', + stage: 'terminal_missing' + }) + await expect + .poll(() => hasPersistedResumeRecord(session.userDataDir, workerPaneKey)) + .toBe(false) + expect(readLedger(spawnLedgerPath)).toEqual([initialSpawn]) + expect(readLedger(interruptionLedgerPath)).toEqual([]) + } finally { + if (secondApp) { + await session.close(secondApp).catch(() => undefined) + } + if (firstApp) { + await session.close(firstApp).catch(() => undefined) + } + await session.dispose() + } +}) diff --git a/tests/e2e/orchestration-legacy-worker-restart-recovery.spec.ts b/tests/e2e/orchestration-legacy-worker-restart-recovery.spec.ts new file mode 100644 index 00000000000..5d17f1c0712 --- /dev/null +++ b/tests/e2e/orchestration-legacy-worker-restart-recovery.spec.ts @@ -0,0 +1,770 @@ +import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import type { ElectronApplication, Page } from '@stablyai/playwright-test' +import { test, expect } from './helpers/orca-app' +import { TEST_REPO_PATH_FILE } from './global-setup' +import { attachRepoAndOpenTerminal, createRestartSession } from './helpers/orca-restart' +import { + ensureTerminalVisible, + getActiveTabId, + switchToOtherWorktree, + switchToWorktree, + waitForActiveWorktree, + waitForSessionReady +} from './helpers/store' +import { waitForActivePaneHookDescriptor, waitForActivePanePtyId } from './helpers/terminal' +import { RuntimeClient } from '../../src/cli/runtime-client' +import Database from '../../src/main/sqlite/sync-database' +import { + CURRENT_CONTRACT_VERSION, + LEGACY_CONTRACT_VERSION, + LEGACY_RUN_ID +} from '../../src/main/runtime/orchestration/db' +import { DEFAULT_LOCAL_ORCA_PROFILE_ID } from '../../src/shared/orca-profiles' +import type { RuntimeTerminalListResult, RuntimeTerminalRead } from '../../src/shared/runtime-types' + +const PROVIDER_SESSION_ID = 'e2e-legacy-orchestration-worker' +const fakeCliDir = mkdtempSync(path.join(os.tmpdir(), 'orca-e2e-legacy-worker-')) +const spawnLedgerPath = path.join(fakeCliDir, 'spawn.jsonl') +const interruptionLedgerPath = path.join(fakeCliDir, 'interruption.jsonl') +const authorityLedgerPath = path.join(fakeCliDir, 'authority.jsonl') +const lifecycleLedgerPath = path.join(fakeCliDir, 'lifecycle.jsonl') +const fakeCodexSource = ` +const { appendFileSync } = require('node:fs') +const { spawnSync } = require('node:child_process') +function appendLedger(envName, event) { + const ledgerPath = process.env[envName] + if (!ledgerPath) return + try { + appendFileSync(ledgerPath, JSON.stringify({ pid: process.pid, at: Date.now(), ...event }) + '\\n') + } catch {} +} +async function emitAuthorityHook() { + const port = process.env.ORCA_AGENT_HOOK_PORT + const token = process.env.ORCA_AGENT_HOOK_TOKEN + const launchToken = process.env.ORCA_AGENT_LAUNCH_TOKEN + if (!port || !token || !launchToken || !process.env.ORCA_PANE_KEY) return + try { + const response = await fetch('http://127.0.0.1:' + port + '/hook/codex', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Orca-Agent-Hook-Token': token + }, + body: JSON.stringify({ + paneKey: process.env.ORCA_PANE_KEY, + tabId: process.env.ORCA_TAB_ID, + worktreeId: process.env.ORCA_WORKTREE_ID, + env: process.env.ORCA_AGENT_HOOK_ENV, + version: process.env.ORCA_AGENT_HOOK_VERSION, + launchToken, + payload: { + hook_event_name: 'UserPromptSubmit', + prompt: 'Respond ACK and remain idle' + } + }) + }) + appendLedger('ORCA_E2E_AUTHORITY_LEDGER', { event: 'authority-hook', status: response.status }) + } catch (error) { + appendLedger('ORCA_E2E_AUTHORITY_LEDGER', { + event: 'authority-hook-error', + error: error instanceof Error ? error.message : String(error) + }) + } +} +if (process.argv.slice(2).includes('app-server')) { + process.stderr.write("error: unrecognized subcommand 'app-server'\\n") + process.exit(2) +} +appendLedger('ORCA_E2E_SPAWN_LEDGER', { event: 'spawn', argv: process.argv.slice(2) }) +process.stdout.write('\\u001b]0;Codex Ready\\u0007OpenAI Codex\\nmodel: e2e\\ndirectory: e2e\\n') +void emitAuthorityHook() +let acknowledged = false +let lifecycleSent = false +process.stdin.on('data', (chunk) => { + const input = chunk.toString() + if (input.includes('\\x03')) { + appendLedger('ORCA_E2E_INTERRUPTION_LEDGER', { event: 'stdin-ctrl-c' }) + } + if (!acknowledged && input.includes('\\r')) { + acknowledged = true + process.stdout.write('ACK\\n') + } + const legacyCompletion = input.match(/ORCA_E2E_RUN_LEGACY_DONE:([A-Za-z0-9+/=]+)/) + if (!lifecycleSent && legacyCompletion) { + lifecycleSent = true + const identity = JSON.parse(Buffer.from(legacyCompletion[1], 'base64').toString('utf8')) + const cliEntry = process.env.ORCA_E2E_CLI_ENTRY + const args = [ + 'orchestration', + 'send', + '--to', + identity.coordinatorHandle, + '--type', + 'worker_done', + '--subject', + 'Completed', + '--body', + 'E2E retained legacy completion', + '--payload', + JSON.stringify({ + taskId: identity.taskId, + dispatchId: identity.dispatchId, + filesModified: [] + }), + '--json' + ] + const result = cliEntry + ? spawnSync(process.execPath, [cliEntry, ...args], { + env: process.env, + encoding: 'utf8' + }) + : { status: 127, stdout: '', stderr: 'ORCA_E2E_CLI_ENTRY missing' } + appendLedger('ORCA_E2E_LIFECYCLE_LEDGER', { + event: 'legacy-command', + argv: args, + status: result.status, + stdout: result.stdout, + stderr: result.stderr + }) + process.stdout.write(String(result.stdout || '') + String(result.stderr || '')) + } +}) +for (const signal of ['SIGINT', 'SIGHUP', 'SIGTERM']) { + process.on(signal, () => { + appendLedger('ORCA_E2E_INTERRUPTION_LEDGER', { event: 'signal', signal }) + process.exit(0) + }) +} +process.stdin.resume() +setInterval(() => {}, 60_000) +` + +if (process.platform === 'win32') { + writeFileSync(path.join(fakeCliDir, 'fake-codex.js'), fakeCodexSource) + writeFileSync( + path.join(fakeCliDir, 'codex.cmd'), + '@echo off\r\nnode "%~dp0\\fake-codex.js" %*\r\n' + ) +} else { + const executable = path.join(fakeCliDir, 'codex') + writeFileSync(executable, `#!/usr/bin/env node\n${fakeCodexSource}`) + chmodSync(executable, 0o755) +} + +type LedgerEvent = { + pid: number + event: string + argv?: string[] + signal?: string + status?: number + stdout?: string + stderr?: string + error?: string +} + +type PersistedWorkspaceSession = { + activeTabId?: string | null + activeTabIdByWorktree?: Record + tabsByWorktree?: Record + terminalLayoutsByTabId?: Record + unifiedTabs?: Record + tabGroups?: Record< + string, + { activeTabId: string | null; tabOrder: string[]; recentTabIds?: string[] }[] + > + sleepingAgentSessionsByPaneKey?: Record< + string, + { providerSession?: { id?: unknown }; automaticResumeBlockedBy?: string } + > + terminalPtyIncarnationsByPaneKey?: Record + terminalSurfaceTombstonesByPaneKey?: Record +} + +type PersistedData = { + workspaceSession?: PersistedWorkspaceSession +} + +function readLedger(ledgerPath: string): LedgerEvent[] { + if (!existsSync(ledgerPath)) { + return [] + } + return readFileSync(ledgerPath, 'utf8') + .split(/\r?\n/) + .filter(Boolean) + .map((line) => JSON.parse(line) as LedgerEvent) +} + +function isProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0) + return true + } catch { + return false + } +} + +function persistedDataPath(userDataDir: string): string { + return path.join(userDataDir, 'profiles', DEFAULT_LOCAL_ORCA_PROFILE_ID, 'orca-data.json') +} + +function readPersistedData(userDataDir: string): PersistedData { + return JSON.parse(readFileSync(persistedDataPath(userDataDir), 'utf8')) as PersistedData +} + +function hasPersistedResumeRecord(userDataDir: string, paneKey: string): boolean { + return ( + readPersistedData(userDataDir).workspaceSession?.sleepingAgentSessionsByPaneKey?.[paneKey] + ?.providerSession?.id === PROVIDER_SESSION_ID + ) +} + +async function readRendererRecoveryState( + page: Page, + paneKey: string, + tabId: string +): Promise<{ sleeping: boolean; resumeClaim: boolean; pendingStartup: boolean }> { + return page.evaluate( + ({ workerPaneKey, workerTabId }) => { + const state = window.__store?.getState() + return { + sleeping: Boolean(state?.sleepingAgentSessionsByPaneKey[workerPaneKey]), + resumeClaim: Boolean(state?.automaticAgentResumeClaimsByTabId[workerTabId]), + pendingStartup: Boolean(state?.pendingStartupByTabId[workerTabId]) + } + }, + { workerPaneKey: paneKey, workerTabId: tabId } + ) +} + +function stripLegacyWorkerRendererBinding( + userDataDir: string, + input: { + worktreeId: string + coordinatorTabId: string + workerTabId: string + workerPaneKey: string + } +): void { + const data = readPersistedData(userDataDir) + const session = data.workspaceSession + if (!session) { + throw new Error('Expected a persisted workspace session') + } + const sleeping = session.sleepingAgentSessionsByPaneKey?.[input.workerPaneKey] + if (sleeping?.providerSession?.id !== PROVIDER_SESSION_ID) { + throw new Error('Expected the legacy worker resume record before removing its tab binding') + } + session.tabsByWorktree = { + ...session.tabsByWorktree, + [input.worktreeId]: (session.tabsByWorktree?.[input.worktreeId] ?? []).filter( + (tab) => tab.id !== input.workerTabId + ) + } + delete session.terminalLayoutsByTabId?.[input.workerTabId] + if (session.unifiedTabs?.[input.worktreeId]) { + session.unifiedTabs[input.worktreeId] = session.unifiedTabs[input.worktreeId].filter( + (tab) => tab.id !== input.workerTabId && tab.entityId !== input.workerTabId + ) + } + for (const group of session.tabGroups?.[input.worktreeId] ?? []) { + group.tabOrder = group.tabOrder.filter((tabId) => tabId !== input.workerTabId) + group.recentTabIds = group.recentTabIds?.filter((tabId) => tabId !== input.workerTabId) + if (group.activeTabId === input.workerTabId) { + group.activeTabId = input.coordinatorTabId + } + } + session.activeTabId = input.coordinatorTabId + session.activeTabIdByWorktree = { + ...session.activeTabIdByWorktree, + [input.worktreeId]: input.coordinatorTabId + } + delete session.terminalPtyIncarnationsByPaneKey?.[input.workerPaneKey] + delete session.terminalSurfaceTombstonesByPaneKey?.[input.workerPaneKey] + writeFileSync(persistedDataPath(userDataDir), `${JSON.stringify(data, null, 2)}\n`, 'utf8') +} + +function assertDispatchRemainsCurrent( + userDataDir: string, + input: { + dispatchId: string + terminalHandle: string + paneKey: string + processIncarnation: string + worktreeId: string + } +): void { + const db = new Database(path.join(userDataDir, 'orchestration.db')) + try { + const authority = db + .prepare( + `SELECT dc.status AS dispatch_status, dc.assignee_handle, dc.assignee_pane_key, + dc.process_incarnation, dc.contract_version, dc.capability_hash, + wd.state AS worker_state, wd.worktree_id, wd.agent_terminal_handle + FROM dispatch_contexts dc + INNER JOIN worker_dispatches wd ON wd.dispatch_id = dc.id + WHERE dc.id = ?` + ) + .get(input.dispatchId) + expect(authority).toEqual({ + dispatch_status: 'dispatched', + assignee_handle: input.terminalHandle, + assignee_pane_key: input.paneKey, + process_incarnation: input.processIncarnation, + contract_version: CURRENT_CONTRACT_VERSION, + capability_hash: expect.any(String), + worker_state: 'ready', + worktree_id: input.worktreeId, + agent_terminal_handle: input.terminalHandle + }) + } finally { + db.close() + } +} + +function markAssignmentAsPreUpdateLegacy( + userDataDir: string, + input: { + taskId: string + dispatchId: string + terminalHandle: string + paneKey: string + processIncarnation: string + worktreeId: string + } +): void { + const db = new Database(path.join(userDataDir, 'orchestration.db')) + try { + const authority = db + .prepare( + `SELECT dc.status AS dispatch_status, dc.assignee_handle, dc.assignee_pane_key, + dc.process_incarnation, wd.state AS worker_state, wd.worktree_id, + wd.agent_terminal_handle + FROM dispatch_contexts dc + INNER JOIN worker_dispatches wd ON wd.dispatch_id = dc.id + WHERE dc.id = ?` + ) + .get(input.dispatchId) + expect(authority).toEqual({ + dispatch_status: 'dispatched', + assignee_handle: input.terminalHandle, + assignee_pane_key: input.paneKey, + process_incarnation: input.processIncarnation, + worker_state: 'ready', + worktree_id: input.worktreeId, + agent_terminal_handle: input.terminalHandle + }) + db.exec('BEGIN IMMEDIATE') + db.prepare('UPDATE tasks SET run_id = ? WHERE id = ?').run(LEGACY_RUN_ID, input.taskId) + db.prepare( + `UPDATE dispatch_contexts + SET run_id = ?, contract_version = ?, capability_hash = NULL, + capability_revoked_at = NULL, launch_token_hash = NULL + WHERE id = ?` + ).run(LEGACY_RUN_ID, LEGACY_CONTRACT_VERSION, input.dispatchId) + db.exec(` + DROP INDEX IF EXISTS idx_messages_delivery_contract; + DROP TABLE legacy_mail_receipts; + DROP TABLE legacy_operation_receipts; + DROP TABLE legacy_compatibility_principals; + DROP TABLE legacy_adoptions; + `) + db.pragma('user_version = 18') + db.exec('COMMIT') + } finally { + db.close() + } +} + +test.describe.configure({ mode: 'serial' }) + +test.afterAll(() => { + rmSync(fakeCliDir, { recursive: true, force: true }) +}) + +for (const contractVersion of [LEGACY_CONTRACT_VERSION, CURRENT_CONTRACT_VERSION]) { + const contractLabel = contractVersion === LEGACY_CONTRACT_VERSION ? 'legacy' : 'current' + test(`adopts one live ${contractLabel} worker after restart without replaying resume`, async (// oxlint-disable-next-line no-empty-pattern -- This lifecycle test owns both Electron launches and intentionally opts out of the default app fixture. + {}, testInfo) => { + test.setTimeout(300_000) + rmSync(spawnLedgerPath, { force: true }) + rmSync(interruptionLedgerPath, { force: true }) + rmSync(authorityLedgerPath, { force: true }) + rmSync(lifecycleLedgerPath, { force: true }) + const repoPath = existsSync(TEST_REPO_PATH_FILE) + ? readFileSync(TEST_REPO_PATH_FILE, 'utf8').trim() + : '' + test.skip(!repoPath || !existsSync(repoPath), 'Global setup did not produce a seeded test repo') + + const session = createRestartSession(testInfo, { + PATH: `${fakeCliDir}${path.delimiter}${process.env.PATH ?? ''}`, + ORCA_E2E_SPAWN_LEDGER: spawnLedgerPath, + ORCA_E2E_INTERRUPTION_LEDGER: interruptionLedgerPath, + ORCA_E2E_AUTHORITY_LEDGER: authorityLedgerPath, + ORCA_E2E_LIFECYCLE_LEDGER: lifecycleLedgerPath, + ORCA_E2E_CLI_ENTRY: path.join(process.cwd(), 'out', 'cli', 'index.js') + }) + let firstApp: ElectronApplication | null = null + let secondApp: ElectronApplication | null = null + + try { + const first = await session.launch() + firstApp = first.app + const worktreeId = await attachRepoAndOpenTerminal(first.page, repoPath) + await waitForSessionReady(first.page) + await ensureTerminalVisible(first.page) + const coordinatorTabId = await getActiveTabId(first.page) + expect(coordinatorTabId).toBeTruthy() + await waitForActivePanePtyId(first.page) + const coordinatorPane = await waitForActivePaneHookDescriptor(first.page) + const firstClient = new RuntimeClient(session.userDataDir, 30_000, null, null) + const coordinator = await firstClient.call<{ terminal: { handle: string } }>( + 'terminal.resolvePane', + { paneKey: coordinatorPane.paneKey } + ) + const coordinatorTerminal = await firstClient.call<{ + terminal: { worktreeId: string } + }>('terminal.show', { terminal: coordinator.result.terminal.handle }) + await expect + .poll(async () => { + const listed = await firstClient.call<{ worktrees: { id: string }[] }>( + 'worktree.list', + {} + ) + return listed.result.worktrees.some( + (candidate) => candidate.id === coordinatorTerminal.result.terminal.worktreeId + ) + }) + .toBe(true) + const run = await firstClient.call<{ run: { id: string } }>('orchestration.runCreate', { + objective: 'Legacy worker restart recovery', + from: coordinator.result.terminal.handle + }) + const task = await firstClient.call<{ task: { id: string } }>('orchestration.taskCreate', { + spec: 'Respond ACK and remain idle', + run: run.result.run.id, + callerTerminalHandle: coordinator.result.terminal.handle + }) + const started = await firstClient.call<{ + effects: { kind: string; role?: string; id?: string }[] + }>('orchestration.workerStart', { + task: task.result.task.id, + from: coordinator.result.terminal.handle, + agent: 'codex', + timeoutMs: 15_000 + }) + const workerHandle = started.result.effects.find( + (effect) => effect.kind === 'terminal' && effect.role === 'agent' + )?.id + expect(workerHandle).toBeTruthy() + + let worker = ( + await firstClient.call('terminal.list') + ).result.terminals.find((terminal) => terminal.title === 'Codex Ready') + await expect + .poll(async () => { + const listed = await firstClient.call('terminal.list') + worker = listed.result.terminals.find((terminal) => terminal.title === 'Codex Ready') + return worker?.ptyId ?? null + }) + .toBeTruthy() + expect(worker?.incarnationId).toBeTruthy() + const workerPaneKey = `${worker!.tabId}:${worker!.leafId}` + await expect + .poll(async () => { + const read = await firstClient.call<{ terminal: RuntimeTerminalRead }>('terminal.read', { + terminal: worker!.handle, + limit: 200 + }) + return read.result.terminal.tail.join('\n') + }) + .toContain('ACK') + const initialWorker = { + ptyId: worker!.ptyId, + incarnationId: worker!.incarnationId, + worktreeId: worker!.worktreeId, + tabId: worker!.tabId, + leafId: worker!.leafId + } + const initialDispatch = await firstClient.call<{ + dispatch: { + id: string + task_id: string + assignee_handle: string + assignee_pane_key: string + process_incarnation: string + } | null + }>('orchestration.dispatchShow', { task: task.result.task.id }) + expect(initialDispatch.result.dispatch).toEqual( + expect.objectContaining({ + task_id: task.result.task.id, + assignee_pane_key: workerPaneKey, + process_incarnation: `${initialWorker.ptyId}:${initialWorker.incarnationId}` + }) + ) + const dispatchHandle = initialDispatch.result.dispatch!.assignee_handle + await expect.poll(() => readLedger(spawnLedgerPath)).toHaveLength(1) + const [initialSpawn] = readLedger(spawnLedgerPath) + expect(isProcessAlive(initialSpawn.pid)).toBe(true) + expect(readLedger(interruptionLedgerPath)).toEqual([]) + await expect + .poll(() => readLedger(authorityLedgerPath)) + .toEqual([expect.objectContaining({ event: 'authority-hook', status: 204 })]) + + const transcriptPath = session.seedCodexResumeRollout(PROVIDER_SESSION_ID, repoPath) + await first.page.evaluate( + ({ paneKey, tabId, worktreeId: workerWorktreeId, terminalHandle, transcript }) => { + window.__store?.getState().setAgentStatus( + paneKey, + { state: 'working', prompt: 'Respond ACK and remain idle', agentType: 'codex' }, + 'Codex Ready', + undefined, + { tabId, worktreeId: workerWorktreeId, terminalHandle }, + { + providerSession: { + key: 'session_id', + id: 'e2e-legacy-orchestration-worker', + transcriptPath: transcript + }, + launchConfig: { + agentCommand: 'codex', + agentArgs: '--dangerously-bypass-approvals-and-sandbox', + agentEnv: {} + } + } + ) + window.__store?.getState().captureAllSleepingAgentSessions('quit') + }, + { + paneKey: workerPaneKey, + tabId: worker!.tabId, + worktreeId: worker!.worktreeId, + terminalHandle: worker!.handle, + transcript: transcriptPath + } + ) + await expect + .poll(() => hasPersistedResumeRecord(session.userDataDir, workerPaneKey), { + timeout: 30_000 + }) + .toBe(true) + + await session.close(firstApp) + firstApp = null + expect(readLedger(spawnLedgerPath)).toEqual([initialSpawn]) + expect(readLedger(interruptionLedgerPath)).toEqual([]) + expect(isProcessAlive(initialSpawn.pid)).toBe(true) + + stripLegacyWorkerRendererBinding(session.userDataDir, { + worktreeId, + coordinatorTabId: coordinatorTabId!, + workerTabId: worker!.tabId, + workerPaneKey + }) + const dispatchIdentity = { + taskId: task.result.task.id, + dispatchId: initialDispatch.result.dispatch!.id, + terminalHandle: dispatchHandle, + paneKey: workerPaneKey, + processIncarnation: `${initialWorker.ptyId}:${initialWorker.incarnationId}`, + worktreeId: initialWorker.worktreeId + } + if (contractVersion === LEGACY_CONTRACT_VERSION) { + markAssignmentAsPreUpdateLegacy(session.userDataDir, dispatchIdentity) + } else { + assertDispatchRemainsCurrent(session.userDataDir, dispatchIdentity) + } + + const second = await session.launch() + secondApp = second.app + await waitForSessionReady(second.page) + expect(await waitForActiveWorktree(second.page)).toBe(worktreeId) + const secondClient = new RuntimeClient(session.userDataDir, 30_000, null, null) + let recovered = ( + await secondClient.call('terminal.list') + ).result.terminals.find((terminal) => terminal.ptyId === initialWorker.ptyId) + await expect + .poll(async () => { + const listed = await secondClient.call('terminal.list') + const matches = listed.result.terminals.filter( + (terminal) => terminal.ptyId === initialWorker.ptyId + ) + recovered = matches[0] + return matches + }) + .toEqual([ + expect.objectContaining({ + ...initialWorker, + connected: true, + writable: true + }) + ]) + + const recoveredTab = second.page.locator( + `[data-testid="sortable-tab"][data-tab-id="${initialWorker.tabId}"]` + ) + await expect(recoveredTab).toBeVisible() + await expect(recoveredTab).toHaveCount(1) + await expect(recoveredTab).toHaveAttribute('data-active', 'false') + await expect( + second.page.locator(`[data-testid="sortable-tab"][data-tab-id="${coordinatorTabId!}"]`) + ).toHaveAttribute('data-active', 'true') + await expect + .poll(async () => { + const read = await secondClient.call<{ terminal: RuntimeTerminalRead }>('terminal.read', { + terminal: recovered!.handle, + limit: 200 + }) + return read.result.terminal.tail.join('\n') + }) + .toContain('ACK') + + let assignmentRunId = run.result.run.id + if (contractVersion === LEGACY_CONTRACT_VERSION) { + const runs = await secondClient.call<{ + runs: { id: string; objective: string }[] + }>('orchestration.runList') + assignmentRunId = runs.result.runs.find( + (candidate) => + candidate.objective === 'Recovered orchestration work from a contract update' + )!.id + } + const restoredRun = await secondClient.call<{ run: { id: string } }>( + 'orchestration.runShow', + { id: assignmentRunId } + ) + expect(restoredRun.result.run.id).toBe(assignmentRunId) + const tasks = await secondClient.call<{ tasks: { id: string }[] }>('orchestration.taskList', { + run: assignmentRunId + }) + expect(tasks.result.tasks).toEqual( + expect.arrayContaining([expect.objectContaining({ id: task.result.task.id })]) + ) + const recoveredDispatch = await secondClient.call<{ + dispatch: { + id: string + task_id: string + assignee_handle: string + assignee_pane_key: string + process_incarnation: string + contract_version: number + } | null + }>('orchestration.dispatchShow', { task: task.result.task.id }) + expect(recoveredDispatch.result.dispatch).toEqual( + expect.objectContaining({ + id: initialDispatch.result.dispatch!.id, + task_id: task.result.task.id, + assignee_handle: dispatchHandle, + assignee_pane_key: workerPaneKey, + process_incarnation: `${initialWorker.ptyId}:${initialWorker.incarnationId}`, + contract_version: contractVersion + }) + ) + await expect + .poll(async () => ({ + renderer: await readRendererRecoveryState( + second.page, + workerPaneKey, + initialWorker.tabId + ), + persisted: hasPersistedResumeRecord(session.userDataDir, workerPaneKey) + })) + .toEqual({ + renderer: { sleeping: false, resumeClaim: false, pendingStartup: false }, + persisted: false + }) + expect(readLedger(spawnLedgerPath)).toEqual([initialSpawn]) + expect(readLedger(interruptionLedgerPath)).toEqual([]) + expect(isProcessAlive(initialSpawn.pid)).toBe(true) + + if (contractVersion === LEGACY_CONTRACT_VERSION) { + const legacyCompletion = Buffer.from( + JSON.stringify({ + coordinatorHandle: coordinator.result.terminal.handle, + taskId: task.result.task.id, + dispatchId: initialDispatch.result.dispatch!.id + }) + ).toString('base64') + await secondClient.call('terminal.send', { + terminal: recovered!.handle, + text: `ORCA_E2E_RUN_LEGACY_DONE:${legacyCompletion}`, + enter: true + }) + await expect + .poll(() => readLedger(lifecycleLedgerPath), { timeout: 30_000 }) + .toEqual([ + expect.objectContaining({ + event: 'legacy-command', + pid: initialSpawn.pid, + argv: [ + 'orchestration', + 'send', + '--to', + coordinator.result.terminal.handle, + '--type', + 'worker_done', + '--subject', + 'Completed', + '--body', + 'E2E retained legacy completion', + '--payload', + JSON.stringify({ + taskId: task.result.task.id, + dispatchId: initialDispatch.result.dispatch!.id, + filesModified: [] + }), + '--json' + ], + status: 0, + stderr: '' + }) + ]) + await expect + .poll(async () => { + const dispatch = await secondClient.call<{ + dispatch: { id: string; status: string } | null + }>('orchestration.dispatchShow', { task: task.result.task.id }) + const listedTasks = await secondClient.call<{ + tasks: { id: string; status: string }[] + }>('orchestration.taskList', { run: assignmentRunId }) + return { + dispatch: dispatch.result.dispatch?.status, + task: listedTasks.result.tasks.find( + (candidate) => candidate.id === task.result.task.id + )?.status + } + }) + .toEqual({ dispatch: 'completed', task: 'completed' }) + expect(readLedger(spawnLedgerPath)).toEqual([initialSpawn]) + expect(isProcessAlive(initialSpawn.pid)).toBe(true) + } else { + expect(readLedger(lifecycleLedgerPath)).toEqual([]) + } + + const otherWorktreeId = await switchToOtherWorktree(second.page, worktreeId) + expect(otherWorktreeId).toBeTruthy() + await switchToWorktree(second.page, worktreeId) + await expect(recoveredTab).toBeVisible() + await expect(recoveredTab).toHaveCount(1) + await expect(recoveredTab).toHaveAttribute('data-active', 'false') + await expect + .poll(async () => + readRendererRecoveryState(second.page, workerPaneKey, initialWorker.tabId) + ) + .toEqual({ sleeping: false, resumeClaim: false, pendingStartup: false }) + expect(readLedger(spawnLedgerPath)).toEqual([initialSpawn]) + expect(readLedger(interruptionLedgerPath)).toEqual([]) + expect(isProcessAlive(initialSpawn.pid)).toBe(true) + await expect(second.page.locator('body')).not.toContainText('Conversation interrupted') + } finally { + if (secondApp) { + await session.close(secondApp).catch(() => undefined) + } + if (firstApp) { + await session.close(firstApp).catch(() => undefined) + } + await session.dispose() + } + }) +} diff --git a/tests/e2e/orchestration-worker-terminal-visibility.spec.ts b/tests/e2e/orchestration-worker-terminal-visibility.spec.ts index 23d1adf68d8..af603c8ca1f 100644 --- a/tests/e2e/orchestration-worker-terminal-visibility.spec.ts +++ b/tests/e2e/orchestration-worker-terminal-visibility.spec.ts @@ -1,4 +1,4 @@ -import { chmodSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import os from 'node:os' import path from 'node:path' import { test as base, expect } from './helpers/orca-app' @@ -15,19 +15,40 @@ import { RuntimeClient } from '../../src/cli/runtime-client' import type { RuntimeTerminalListResult, RuntimeTerminalRead } from '../../src/shared/runtime-types' const fakeCliDir = mkdtempSync(path.join(os.tmpdir(), 'orca-e2e-orchestration-worker-')) +const spawnLedgerPath = path.join(fakeCliDir, 'spawn.jsonl') +const interruptionLedgerPath = path.join(fakeCliDir, 'interruption.jsonl') const fakeCodexSource = ` +const { appendFileSync } = require('node:fs') +function appendLedger(envName, event) { + const ledgerPath = process.env[envName] + if (!ledgerPath) return + try { + appendFileSync(ledgerPath, JSON.stringify({ pid: process.pid, at: Date.now(), ...event }) + '\\n') + } catch {} +} if (process.argv.slice(2).includes('app-server')) { process.stderr.write("error: unrecognized subcommand 'app-server'\\n") process.exit(2) } +appendLedger('ORCA_E2E_SPAWN_LEDGER', { event: 'spawn', startedAt: Date.now() }) process.stdout.write('\\u001b]0;Codex Ready\\u0007OpenAI Codex\\nmodel: e2e\\ndirectory: e2e\\n') let acknowledged = false process.stdin.on('data', (chunk) => { - if (!acknowledged && chunk.toString().includes('\\r')) { + const input = chunk.toString() + if (input.includes('\\x03')) { + appendLedger('ORCA_E2E_INTERRUPTION_LEDGER', { event: 'stdin-ctrl-c' }) + } + if (!acknowledged && input.includes('\\r')) { acknowledged = true process.stdout.write('ACK\\n') } }) +for (const signal of ['SIGINT', 'SIGHUP', 'SIGTERM']) { + process.on(signal, () => { + appendLedger('ORCA_E2E_INTERRUPTION_LEDGER', { event: 'signal', signal }) + process.exit(0) + }) +} process.stdin.resume() setInterval(() => {}, 60_000) ` @@ -47,7 +68,9 @@ if (process.platform === 'win32') { const test = base.extend({ launchEnv: [ { - PATH: `${fakeCliDir}${path.delimiter}${process.env.PATH ?? ''}` + PATH: `${fakeCliDir}${path.delimiter}${process.env.PATH ?? ''}`, + ORCA_E2E_SPAWN_LEDGER: spawnLedgerPath, + ORCA_E2E_INTERRUPTION_LEDGER: interruptionLedgerPath }, { option: true } ] @@ -57,7 +80,33 @@ test.afterAll(() => { rmSync(fakeCliDir, { recursive: true, force: true }) }) -test('worker-start materializes one inactive terminal tab before workspace re-entry', async ({ +type LedgerEvent = { + pid: number + event: string + startedAt?: number + signal?: string +} + +function readLedger(ledgerPath: string): LedgerEvent[] { + if (!existsSync(ledgerPath)) { + return [] + } + return readFileSync(ledgerPath, 'utf8') + .split(/\r?\n/) + .filter(Boolean) + .map((line) => JSON.parse(line) as LedgerEvent) +} + +function isProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0) + return true + } catch { + return false + } +} + +test('worker-start preserves one live inactive worker across workspace re-entry', async ({ orcaPage, electronApp }) => { @@ -124,6 +173,33 @@ test('worker-start materializes one inactive terminal tab before workspace re-en return read.result.terminal.tail.join('\n') }) .toContain('ACK') + const initialWorkerIdentity = { + ptyId: workerTerminal!.ptyId, + incarnationId: workerTerminal!.incarnationId, + worktreeId: workerTerminal!.worktreeId, + tabId: workerTerminal!.tabId, + leafId: workerTerminal!.leafId + } + const initialDispatch = await client.call<{ + dispatch: { id: string; task_id: string; assignee_handle: string } | null + }>('orchestration.dispatchShow', { task: task.result.task.id }) + expect(initialDispatch.result.dispatch).toEqual( + expect.objectContaining({ + task_id: task.result.task.id, + assignee_handle: workerHandle + }) + ) + await expect.poll(() => readLedger(spawnLedgerPath)).toHaveLength(1) + const [spawn] = readLedger(spawnLedgerPath) + expect(spawn).toEqual( + expect.objectContaining({ + event: 'spawn', + pid: expect.any(Number), + startedAt: expect.any(Number) + }) + ) + expect(isProcessAlive(spawn.pid)).toBe(true) + expect(readLedger(interruptionLedgerPath)).toEqual([]) const workerTab = orcaPage.locator( `[data-testid="sortable-tab"][data-tab-id="${workerTerminal!.tabId}"]` ) @@ -156,4 +232,27 @@ test('worker-start materializes one inactive terminal tab before workspace re-en await expect( orcaPage.locator(`[data-testid="sortable-tab"][data-tab-title="${workerTabTitle}"]`) ).toHaveCount(1) + const terminalsAfterReturn = await client.call('terminal.list') + const workerAfterReturn = terminalsAfterReturn.result.terminals.find( + (terminal) => terminal.ptyId === initialWorkerIdentity.ptyId + ) + expect(workerAfterReturn).toEqual(expect.objectContaining(initialWorkerIdentity)) + const dispatchAfterReturn = await client.call<{ + dispatch: { id: string; task_id: string; assignee_handle: string } | null + }>('orchestration.dispatchShow', { task: task.result.task.id }) + expect(dispatchAfterReturn.result.dispatch).toEqual(initialDispatch.result.dispatch) + expect(readLedger(spawnLedgerPath)).toEqual([spawn]) + expect(readLedger(interruptionLedgerPath)).toEqual([]) + expect(isProcessAlive(spawn.pid)).toBe(true) + const workerOutputAfterReturn = await client.call<{ terminal: RuntimeTerminalRead }>( + 'terminal.read', + { + terminal: workerAfterReturn!.handle, + limit: 200 + } + ) + expect(workerOutputAfterReturn.result.terminal.tail.join('\n')).not.toContain( + 'Conversation interrupted' + ) + await expect(orcaPage.locator('body')).not.toContainText('Conversation interrupted') }) diff --git a/tools/win-update-e2e/README.md b/tools/win-update-e2e/README.md index 6c771bb1197..cc94aefc552 100644 --- a/tools/win-update-e2e/README.md +++ b/tools/win-update-e2e/README.md @@ -54,6 +54,7 @@ Or directly: `node tools/win-update-e2e/run.mjs --from ... --to ... --expect ... | `--install-dir ` | Isolated-install mode (see below) — install into `` | | `--asset-pattern ` | gh asset glob (default `*windows-setup.exe`) | | `--soak-seconds ` | Post-relaunch window-watch soak (default `180`) | +| `--require-distinct-artifacts` | Require distinct N/B paths, versions, and SHA-256 hashes | | `--keep-install` | Skip teardown/uninstall for debugging (ignored in isolated) | ### Profiles @@ -210,6 +211,14 @@ powershell -File tools/win-update-e2e/window-enum.ps1 ## Known limitations +- **This is installer/session scaffolding, not the orchestration cutover gate.** + `--require-distinct-artifacts` makes the existing Windows update journey fail + closed unless it received two provably different packages. A CI follow-up + must still drive the packaged updater with a blocked packaged `orca.exe` ask, + prove launcher status `75` and the exact native/WSL resume command, then + record one question and reply on B. That proof requires distinct installed + A/B artifacts and a physical Windows host with WSL; this harness does not + simulate either contract. - **Scrollback fidelity is best-effort.** A production build renders the terminal with WebGL, so xterm text is not reliably in the DOM and the e2e `SerializeAddon` is not exposed. When text cannot be read the check reports diff --git a/tools/win-update-e2e/cli-args.mjs b/tools/win-update-e2e/cli-args.mjs index ed50412314e..e98ffda89eb 100644 --- a/tools/win-update-e2e/cli-args.mjs +++ b/tools/win-update-e2e/cli-args.mjs @@ -52,6 +52,9 @@ Options: --keep-install Skip teardown/uninstall (leaves the app installed) --asset-pattern gh release asset glob (default: *windows-setup.exe) --soak-seconds Post-relaunch window watch duration (default: 180) + --require-distinct-artifacts + Fail unless N and N+1 have distinct paths, versions, + and SHA-256 hashes. Intended for compatibility gates. -h, --help Show this help ` @@ -71,6 +74,7 @@ export function parseArgs(argv) { installDir: takeValue(argv, '--install-dir'), keepInstall: argv.includes('--keep-install'), allowExistingInstall: argv.includes('--allow-existing-install'), + requireDistinctArtifacts: argv.includes('--require-distinct-artifacts'), usage: USAGE } diff --git a/tools/win-update-e2e/installer-pair-proof.mjs b/tools/win-update-e2e/installer-pair-proof.mjs new file mode 100644 index 00000000000..f97101c0a8b --- /dev/null +++ b/tools/win-update-e2e/installer-pair-proof.mjs @@ -0,0 +1,35 @@ +import { createHash } from 'node:crypto' +import { createReadStream } from 'node:fs' +import path from 'node:path' + +async function sha256(filePath) { + const hash = createHash('sha256') + for await (const chunk of createReadStream(filePath)) { + hash.update(chunk) + } + return hash.digest('hex') +} + +export async function proveDistinctInstallerPair({ fromPath, toPath, fromVersion, toVersion }) { + const [fromHash, toHash] = await Promise.all([sha256(fromPath), sha256(toPath)]) + const proof = { + from: { path: path.resolve(fromPath), version: fromVersion, sha256: fromHash }, + to: { path: path.resolve(toPath), version: toVersion, sha256: toHash } + } + const errors = [] + if (proof.from.path.toLowerCase() === proof.to.path.toLowerCase()) { + errors.push('base and candidate installer paths are identical') + } + if (fromHash === toHash) { + errors.push('base and candidate installer SHA-256 hashes are identical') + } + if (!fromVersion || !toVersion) { + errors.push('both installer versions must be readable') + } else if (fromVersion === toVersion) { + errors.push(`base and candidate installer versions are identical (${fromVersion})`) + } + if (errors.length > 0) { + throw new Error(`Distinct installer proof failed: ${errors.join('; ')}`) + } + return proof +} diff --git a/tools/win-update-e2e/installer-pair-proof.test.mjs b/tools/win-update-e2e/installer-pair-proof.test.mjs new file mode 100644 index 00000000000..203170f5eef --- /dev/null +++ b/tools/win-update-e2e/installer-pair-proof.test.mjs @@ -0,0 +1,88 @@ +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { parseArgs } from './cli-args.mjs' +import { proveDistinctInstallerPair } from './installer-pair-proof.mjs' + +const scratchDirs = [] + +async function artifactPair(fromBody = 'artifact-a', toBody = 'artifact-b') { + const directory = await mkdtemp(path.join(os.tmpdir(), 'orca-installer-pair-proof-')) + scratchDirs.push(directory) + const fromPath = path.join(directory, 'orca-a.exe') + const toPath = path.join(directory, 'orca-b.exe') + await Promise.all([writeFile(fromPath, fromBody), writeFile(toPath, toBody)]) + return { fromPath, toPath } +} + +afterEach(async () => { + await Promise.all(scratchDirs.splice(0).map((directory) => rm(directory, { recursive: true }))) +}) + +describe('packaged installer pair proof', () => { + it('requires an explicit compatibility-gate flag', () => { + const opts = parseArgs([ + '--from', + 'a.exe', + '--to', + 'b.exe', + '--expect', + 'cold-restore', + '--require-distinct-artifacts' + ]) + + expect(opts.requireDistinctArtifacts).toBe(true) + }) + + it('records distinct paths, versions, and hashes', async () => { + const pair = await artifactPair() + + const proof = await proveDistinctInstallerPair({ + ...pair, + fromVersion: '1.4.130', + toVersion: '1.4.131' + }) + + expect(proof.from.path).not.toBe(proof.to.path) + expect(proof.from.version).not.toBe(proof.to.version) + expect(proof.from.sha256).not.toBe(proof.to.sha256) + }) + + it('fails closed for byte-identical artifacts at different paths', async () => { + const pair = await artifactPair('same-artifact', 'same-artifact') + + await expect( + proveDistinctInstallerPair({ + ...pair, + fromVersion: '1.4.130', + toVersion: '1.4.131' + }) + ).rejects.toThrow('SHA-256 hashes are identical') + }) + + it('fails closed for an identical path or version', async () => { + const pair = await artifactPair() + + await expect( + proveDistinctInstallerPair({ + fromPath: pair.fromPath, + toPath: pair.fromPath, + fromVersion: '1.4.130', + toVersion: '1.4.130' + }) + ).rejects.toThrow(/paths are identical.*hashes are identical.*versions are identical/) + }) + + it('fails closed when either package version is unavailable', async () => { + const pair = await artifactPair() + + await expect( + proveDistinctInstallerPair({ + ...pair, + fromVersion: null, + toVersion: '1.4.131' + }) + ).rejects.toThrow('both installer versions must be readable') + }) +}) diff --git a/tools/win-update-e2e/run.mjs b/tools/win-update-e2e/run.mjs index 7c8d8cde6c5..d9f8d8b5b10 100644 --- a/tools/win-update-e2e/run.mjs +++ b/tools/win-update-e2e/run.mjs @@ -12,7 +12,13 @@ import { execFileSync } from 'node:child_process' import { assertWin32 } from './platform-guard.mjs' import { parseArgs } from './cli-args.mjs' import { preflight } from './preflight.mjs' -import { resolveInstaller, silentInstall, silentUninstall } from './installer-steps.mjs' +import { + getExeVersion, + resolveInstaller, + silentInstall, + silentUninstall +} from './installer-steps.mjs' +import { proveDistinctInstallerPair } from './installer-pair-proof.mjs' import { backupInstallState, restoreInstallState } from './registry-shortcut-backup.mjs' import { launchInstalledApp, @@ -177,6 +183,24 @@ async function runProof(ctx, args) { releaseTag: opts.fromRelease, assetPattern: opts.assetPattern }) + const toInstaller = resolveInstaller({ + localPath: opts.to, + releaseTag: opts.toRelease, + assetPattern: opts.assetPattern + }) + if (opts.requireDistinctArtifacts) { + const artifactProof = await proveDistinctInstallerPair({ + fromPath: fromInstaller, + toPath: toInstaller, + fromVersion: getExeVersion(fromInstaller), + toVersion: getExeVersion(toInstaller) + }) + log( + 'artifact-proof', + `A ${artifactProof.from.version} ${artifactProof.from.sha256}; ` + + `B ${artifactProof.to.version} ${artifactProof.to.sha256}` + ) + } log('install-base', `installing ${fromInstaller}`) const base = silentInstall(fromInstaller, { installDir }) // Track the install now (not only after the update at L238) so a failure @@ -230,11 +254,6 @@ async function runProof(ctx, args) { log('watch', `started (duration ${watchDuration}s) -> ${watchOut}`) // --- Update: install N+1 --- - const toInstaller = resolveInstaller({ - localPath: opts.to, - releaseTag: opts.toRelease, - assetPattern: opts.assetPattern - }) log('update', `installing ${toInstaller}`) const updated = silentInstall(toInstaller, { installDir }) // Record the exact dir the harness installed into so non-isolated teardown