diff --git a/.github/workflows/mobile.yml b/.github/workflows/mobile.yml index 59f6cf20bf4..6dbfc02aa3c 100644 --- a/.github/workflows/mobile.yml +++ b/.github/workflows/mobile.yml @@ -15,8 +15,13 @@ on: # Why: this job holds the only checks that load the Fastfile, so edits to # it or to the release workflow it guards must re-run them. - '.github/workflows/mobile.yml' + - '.github/actions/install-node-dependencies/**' - '.github/workflows/mobile-ios-release.yml' +concurrency: + group: mobile-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: verify: runs-on: ubuntu-latest @@ -35,10 +40,7 @@ jobs: - name: Checkout uses: actions/checkout@v6 - - name: Setup Node.js - uses: actions/setup-node@v6 - with: - node-version-file: package.json + - uses: ./.github/actions/install-node-dependencies # bundler-cache installs mobile/Gemfile.lock, so this job is also what # proves the pinned fastlane the release workflow depends on still @@ -50,23 +52,6 @@ jobs: bundler-cache: true working-directory: mobile - - name: Setup pnpm - uses: pnpm/setup@v2 - with: - install: false - - # Why: the mobile typecheck imports shared types from ../src/shared, and - # some of those files import runtime deps (tweetnacl, ws) resolved from - # the repo-root node_modules. Without a root install, tsc fails with - # "Cannot find module 'tweetnacl'/'ws'". Mobile is a separate pnpm project - # (not in the root workspace), so this is a distinct install. - # --ignore-scripts skips the root postinstall (Electron native-module - # rebuild) which is irrelevant to a type-only check and would only add - # time and failure surface on this ubuntu mobile runner. - - name: Install root dependencies - working-directory: . - run: pnpm install --frozen-lockfile --ignore-scripts - - name: Install dependencies run: pnpm install --frozen-lockfile diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index a749214e232..ba2eaf83192 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -41,6 +41,10 @@ jobs: managed_hook_node18: ${{ steps.filter.outputs.managed_hook_node18 }} package: ${{ steps.filter.outputs.package }} package_windows: ${{ steps.filter.outputs.package_windows }} + e2e_should_run: ${{ steps.e2e_filter.outputs.should_run }} + test_files: ${{ steps.e2e_filter.outputs.test_files }} + ssh_source_changed: ${{ steps.e2e_filter.outputs.ssh_source_changed }} + native_ime_source_changed: ${{ steps.e2e_filter.outputs.native_ime_source_changed }} steps: - name: Checkout uses: actions/checkout@v6 @@ -66,6 +70,37 @@ jobs: printf '%s\n' "$CHANGED" printf '%s\n' "$CHANGED" | node config/scripts/pr-code-change-scope.mjs | tee -a "$GITHUB_OUTPUT" + # Reuse the path-detector checkout instead of queuing another runner. + - name: Filter changed E2E specs + id: e2e_filter + if: github.event.pull_request.draft != true && steps.filter.outputs.should_run == 'true' + run: | + set -euo pipefail + BASE="${{ github.event.pull_request.base.sha }}" + HEAD="${{ github.event.pull_request.head.sha }}" + CHANGED="$(git diff --name-only --diff-filter=AMCR --merge-base "$BASE" "$HEAD")" + # Source routes are executable contracts so a test can prove exact + # authorities, exclusions, and sentinels without evaluating workflow shell. + TEST_FILES_JSON="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs)" + echo "test_files=$TEST_FILES_JSON" >> "$GITHUB_OUTPUT" + # Why a separate signal: the Docker-SSH lane must trigger on SSH source, not on a + # spec name surviving in a route's list. Same routes, so the two cannot drift. + SSH_SOURCE_CHANGED="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --ssh-source)" + echo "ssh_source_changed=$SSH_SOURCE_CHANGED" >> "$GITHUB_OUTPUT" + echo "SSH source changed: $SSH_SOURCE_CHANGED" + # Why its own signal: the real-IME lane is a whole ibus session, not a spec, so it must + # trigger on IME source rather than on a spec name in some route's list. + NATIVE_IME_SOURCE_CHANGED="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --native-ime-source)" + echo "native_ime_source_changed=$NATIVE_IME_SOURCE_CHANGED" >> "$GITHUB_OUTPUT" + echo "Native IME source changed: $NATIVE_IME_SOURCE_CHANGED" + if [ "$TEST_FILES_JSON" != '[]' ]; then + echo "should_run=true" >> "$GITHUB_OUTPUT" + echo "Changed E2E specs: $TEST_FILES_JSON" + else + echo "should_run=false" >> "$GITHUB_OUTPUT" + echo "No changed E2E specs" + fi + static_analysis: name: static analysis needs: [code_paths] @@ -712,7 +747,11 @@ jobs: - name: Package unpacked app env: ORCA_REUSE_PREPARED_NATIVE_RUNTIME: '1' - run: pnpm exec electron-builder --config config/electron-builder.config.cjs --linux AppImage deb rpm --x64 --publish never + # PR artifacts are only inspected locally; gzip avoids release-size xz compression. + run: >- + pnpm exec electron-builder --config config/electron-builder.config.cjs + --linux AppImage deb rpm --x64 --publish never + --config.deb.compression=gz --config.rpm.compression=gzip - name: Verify root-package marker payloads run: | @@ -861,65 +900,10 @@ jobs: - name: Smoke packaged CLI run: node config/scripts/smoke-packaged-cli.mjs --app-dir=dist/win-unpacked - # Why: PR E2E is advisory and only validates changed specs; scheduled and - # release runs retain full-suite coverage. - e2e-paths: - name: detect changed e2e specs - needs: [code_paths] - runs-on: ubuntu-latest - if: github.event.pull_request.draft != true && needs.code_paths.outputs.should_run == 'true' - # Why: detector only needs to read the checkout; do not inherit repo defaults. - permissions: - contents: read - outputs: - should_run: ${{ steps.filter.outputs.should_run }} - test_files: ${{ steps.filter.outputs.test_files }} - ssh_source_changed: ${{ steps.filter.outputs.ssh_source_changed }} - native_ime_source_changed: ${{ steps.filter.outputs.native_ime_source_changed }} - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - # Why blob:none: full history is needed for the merge-base diff, but historical - # file contents are not. Blobs are ~89% of this repo's pack, and Git fetches the - # few this job actually reads on demand. - fetch-depth: 0 - filter: blob:none - persist-credentials: false - - - name: Filter changed E2E specs - id: filter - run: | - set -euo pipefail - BASE="${{ github.event.pull_request.base.sha }}" - HEAD="${{ github.event.pull_request.head.sha }}" - CHANGED="$(git diff --name-only --diff-filter=AMCR --merge-base "$BASE" "$HEAD")" - # Source routes are executable contracts so a test can prove exact - # authorities, exclusions, and sentinels without evaluating workflow shell. - TEST_FILES_JSON="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs)" - echo "test_files=$TEST_FILES_JSON" >> "$GITHUB_OUTPUT" - # Why a separate signal: the Docker-SSH lane must trigger on SSH source, not on a - # spec name surviving in a route's list. Same routes, so the two cannot drift. - SSH_SOURCE_CHANGED="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --ssh-source)" - echo "ssh_source_changed=$SSH_SOURCE_CHANGED" >> "$GITHUB_OUTPUT" - echo "SSH source changed: $SSH_SOURCE_CHANGED" - # Why its own signal: the real-IME lane is a whole ibus session, not a spec, so it must - # trigger on IME source rather than on a spec name in some route's list. - NATIVE_IME_SOURCE_CHANGED="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --native-ime-source)" - echo "native_ime_source_changed=$NATIVE_IME_SOURCE_CHANGED" >> "$GITHUB_OUTPUT" - echo "Native IME source changed: $NATIVE_IME_SOURCE_CHANGED" - if [ "$TEST_FILES_JSON" != '[]' ]; then - echo "should_run=true" >> "$GITHUB_OUTPUT" - echo "Changed E2E specs: $TEST_FILES_JSON" - else - echo "should_run=false" >> "$GITHUB_OUTPUT" - echo "No changed E2E specs" - fi - e2e: name: e2e - needs: e2e-paths - if: needs.e2e-paths.outputs.should_run == 'true' + needs: code_paths + if: needs.code_paths.outputs.e2e_should_run == 'true' # Why: reusable e2e.yml only checkouts, builds, and uploads artifacts. permissions: contents: read @@ -928,8 +912,8 @@ jobs: # The synthetic pull-request merge ref can disappear while this reusable # workflow is queued. The head SHA is immutable and works for every PR. ref: ${{ github.event.pull_request.head.sha }} - test_files: ${{ needs.e2e-paths.outputs.test_files }} - ssh_source_changed: ${{ needs.e2e-paths.outputs.ssh_source_changed }} + test_files: ${{ needs.code_paths.outputs.test_files }} + ssh_source_changed: ${{ needs.code_paths.outputs.ssh_source_changed }} # Why this is not in verify's needs: it is the first PR-gate run of a harness whose reliability # is only known from nightly main runs (20/20 green, 2026-08-09..2026-08-29, p50 3m25s). It @@ -939,8 +923,8 @@ jobs: # require `success || skipped` outside the strict loop — see the note on `e2e`. terminal_ime_native: name: real IME - needs: e2e-paths - if: needs.e2e-paths.outputs.native_ime_source_changed == 'true' + needs: code_paths + if: needs.code_paths.outputs.native_ime_source_changed == 'true' # Why: the reusable workflow only checks out, builds, and uploads artifacts. permissions: contents: read diff --git a/.github/workflows/skill-update-roundtrip.yml b/.github/workflows/skill-update-roundtrip.yml index 71fcf264f69..239f1b2f27c 100644 --- a/.github/workflows/skill-update-roundtrip.yml +++ b/.github/workflows/skill-update-roundtrip.yml @@ -22,6 +22,10 @@ on: - main paths: *skill-roundtrip-paths +concurrency: + group: skill-roundtrip-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: roundtrip: strategy: diff --git a/cloud/docs/relay-improvement-checklist-2026-09.md b/cloud/docs/relay-improvement-checklist-2026-09.md new file mode 100644 index 00000000000..91f1cc742ef --- /dev/null +++ b/cloud/docs/relay-improvement-checklist-2026-09.md @@ -0,0 +1,189 @@ +# Relay improvement: implementation checklist, lanes, and disruption + +Companion to [`relay-improvement-roadmap-2026-09.md`](./relay-improvement-roadmap-2026-09.md) (item numbers +match). This file answers three questions per item: what are the concrete steps, what can run in parallel, +and will a user notice. + +## Status as of 2026-09-04 22:30Z + +Three buckets. "Merged" means the code is on `main` and nothing in production has changed yet. "Deployed" means users are already getting it. "Awaiting owner" means I will not touch production without a go. + +**Deployed to production** +- Auth instance cap 20 + dead-family audit fix (orca-cloud #474) as revision `orca-cloud-auth-00031-tox`. +- Dynamic NAT ports in both regions (stablyai/orca #18693). Zero drops and zero proxy dial errors since. +- Nine alert policies with log metrics: 4 auth (#475), 3 relay Cloud SQL/NAT (#18693), 1 cell process-exit (#18717), all on the relay Slack channel. + +**Merged, ships with the next relay cell image roll (Roll 1 carries `519f4914`; Roll 2 needs a fresh image build)** +- Per-cell inventory locks, delta counters, pool `statement_timeout` (#18722). Roll 2. +- Cells dial Cloud SQL with `--private-ip` when configured (#18720). Inert until 2.1 applies. +- Phone shows a clear "sign in on the desktop again" state when the desktop is signed out (#18698). + +**Merged, ships with the next auth deploy** +- Refresh rotation grace window (orca-cloud #478). Startup adds one nullable column (brief exclusive lock on `refresh_tokens`). +- Pruning job code (orca-cloud #476) is in the image; the job itself is Terraform-disabled until 1.2. + +**Merged, ships with the next desktop release** +- Never replay a refresh token after a timeout; ±10 % jitter on relay lease renewal (#18719). +- Renderer learns when a cloud session is revoked (#18694). + +**Merged, not applied** +- Incident dashboard (#18717) blocked behind the runtime-metric label drift (5.x first item). +- Monitor probe fix (#18723) is live in the workflow; the same-cap roll gate has not yet produced a green dry-run since. + +**Awaiting owner go (production mutations)** +1. Roll 1 cell image roll (1.1): dry-run gate, then c8 canary, then batches. +2. Auth deploy carrying #478 (3.1): quiet minute for the column add. +3. orca-cloud #477 private IP (2.1): merge arms an instance restart and a one-way door. Recommendation: hold. +4. Runtime-metric `region` label drift (5.x): intentional replacement of 21 metrics, or drop the label. +5. Enable pruning (1.2): first budget 20k rows; needs a Terraform apply. +6. Paging channel for auth alerts (5.2): needs the destination from you. + +**Open code follow-ups (no gate, nobody assigned)** +- Monitor summary Markdown does not render `tolerated: true` continuity events (added by #18798); the state artifact has them, the checkpoint table does not. +- Relay container boot races the `cloud-sql-proxy` sidecar: c13's fresh container exited twice (`applyPostgresSchema` connection timeout, 2 s each) before the proxy was listening. Make schema apply wait for the proxy or order the containers. +- `cloud-deploy-relay-production-capacity-job.yml` (~line 416) has the same wave-0 single-shot preflight carve-out that #18778 removes from the same-cap job; its single-evidence path never retries freshness-only failures. +- `cloud/package.json` `test` names every dev-script test file explicitly; an unregistered `*.test.mjs` is silently never run in CI (found by #18769). Needs a glob or a ratchet that fails on an unlisted test file. +- Same-cap job's verify step uses bare `curl --fail-with-body` against the just-rolled cell; one 503 at the LB warm-up edge failed c8 canary #2 (run 33935407461) after the transition verifier had already passed. Needs a bounded retry, same rule as #18723/#18740. +- `verify-mutation` in `cloud-deploy-relay-production.yml`, the multi-target workflow, and the capacity workflow still binds to an exact commit; same exposure #18754 fixed for the same-cap and rehome paths. +- `incident-live-preflight-cli.ts` reports only `source/code` (`active-probe/threshold_max`) with no signal name or observed value, so a failed mutation preflight (c27 recovery #3, run 33986948522) cannot be attributed to an endpoint without an out-of-band probe. Print the signal and observed/threshold pair. Related: the 2 000 ms `endpointLatencyMs` bar is shared by US and Asia cells while Asia /health round trips from a US runner sit at 0.7–1.3 s idle; consider a per-region bar or the p50 of the gate window instead of one shot. Gates #44 and #45 (2026-09-05) both froze on `cell.production-gce-c27.latency_ms` at 2.6–2.7 s with c28 showing the identical tail under operator probes; the bar is now blocking Asia rolls. **Fix: stablyai/orca #18877** (per-region `cellEndpointLatencyMs`, us-central1 2 000 / asia-east2 4 000, plus signal/observed/threshold in preflight messages). Residual: `probeEndpointHealth` in `resource-inventory.ts` still uses the flat 2 000 bar to decide whether to retry after the 10 s readiness-cache wait, so a healthy Asia cell over 2 s costs one extra probe per sample (latency, not verdict); thread the region bar into the retry decision. +- The root oxlint config ignores `cloud/**`, so `check:code-quality:changed` never inspects relay-ops or the cloud dev scripts; typecheck + vitest is the only gate there. +- Monitor bars that froze on non-health today: `directorInstancesMin: 5` with `latest-sum` (one-minute instance recycle), `endpointLatencyMs: 2000` on a US-runner probe to asia-east2, `cloudDataMaxAgeMs: 180000` vs Cloud Monitoring publish lag up to 255 s. Recalibrate with a week of data. +- `parsed()` in `resource-inventory.ts` still returns null on a 200 with a malformed MIG body; a second path to `runtime_power_unknown`. +- Deploy script strips `ORCA_CLOUD_REFRESH_TOKEN_TTL_DAYS` on every release (3.1 first item). +- `assignOnce` placement lock still global (4.1 remainder). +- Region preference (4.2), retries-bar recalibration after a week of Roll 2 data (4.4), pruner `stopReason` alert (1.5). +- Full apps-root apply for 4 unrelated drifts (1.4), from a host with the 1Password account. + +## Uplift ranking (reliability gained per unit of effort) + +| Rank | Item | Why it ranks here | +|---|---|---| +| 1 | 1.1 cell image roll | Removes the only crash mode we have seen in production. 22 of 23 cells still have it. One afternoon. | +| 2 | 3.1 refresh rotation grace window | Turns the entire "slow auth → mass sign-out" class into a slowdown. One day. | +| 3 | 4.1 inventory lock contention | The floor under every 503 and slow phone accept, every day, not just incidents. One week. | +| — | 2.2 relay/auth database split | **Deferred 2026-09-04** to ~2026-11-01. Biggest structural fix, but the concrete cause is fixed and alerts now page; see roadmap 2.2 for re-open triggers. | +| 4 | 1.2 + 1.3 pruning and reclaim | Defuses the 63 M-row time bomb. Low effort, mostly waiting. | +| 5 | 5.1 + 5.2 crash alert, page a human | Cheapest detection uplift; today's incident ran 4 h unpaged. | +| 6 | 2.1 private IP | Durable version of a fix that already landed (dynamic NAT ports). Do it on the existing instance. | +| 7 | 4.3 + 3.2 desktop hardening | Small, ride the normal desktop release. | +| 8 | 4.2, 4.4, 5.4, 1.4, 1.5 | Housekeeping and quality-of-life. | + +## The shared bottleneck: cell rolls + +Every change to what runs on a cell (image, proxy flag, env, relay code) needs a same-cap roll: drain → +recreate → verify, one wave at a time, gated by the 15-minute monitor, about an afternoon. Each wave forces +the desktops on that cell to re-dial (c7 canary: 807 controls re-dialed in ~10 s) and phones on those +desktops reconnect on their normal retry. Users see a few seconds of "reconnecting" per wave. + +So batch. Two rolls, not five: + +- **Roll 1 (now):** current image only (1.1). Do not wait for anything else. +- **Roll 2 (week 2–3):** proxy `--private-ip` (2.1) + relay pool `statement_timeout` (2.3) + lock-contention + fix (4.1), all in one image/template. Prerequisite: 2.1's peering and private IP exist first. + +## Lanes (independent; different people can own them) + +``` +Lane A data plane 1.1 roll ──────────────────► Roll 2 (2.1 flag + 2.3 + 4.1) ──► 4.4 recalibrate +Lane B auth/DB 1.2 enable pruning ──(10 d)──► 1.3 reclaim 3.1 grace window (any time) +Lane C network 2.1 peering + private IP ─────┐ (feeds Roll 2) (2.2 DB split deferred) +Lane D desktop 3.2 no same-token retry, 4.3 lease jitter (any release; wire-compatible) +Lane E observability 1.5, 5.1, 5.2, 5.4 (Terraform only, any time) +Lane F director 4.2 region preference (Cloud Run deploy, any time) +Misc 1.4 full apps-root apply (any time; see its check) +``` + +Hard dependencies: Roll 2 waits on 2.1's network work; 1.3 waits on 1.2 finishing. Everything else is +independent. (2.2 deferred; if revived, do it after 2.1 so the new instance is private from day one.) + +## Disruption summary + +| Item | User-visible? | What they see | Mitigation | +|---|---|---|---| +| 1.1 / Roll 2 | **Yes, transient** | Per wave, desktops on that cell reconnect within seconds; phones follow on retry. | Waves gated by the monitor; run in the US night. Already rehearsed on c7. | +| 1.2 pruning | No | Background deletes, 5k rows per batch. | Small first budget; watch `stopReason` and Cloud SQL write throughput. Stop the scheduler if checkpoint alerts fire. | +| 1.3 reclaim | **Depends on tool** | `VACUUM FULL` takes an exclusive lock on `refresh_tokens`: sign-in and refresh block for its duration (minutes to tens of minutes on 16 GB). `pg_repack` holds only brief locks. | Use `pg_repack`. If VACUUM FULL, announce a maintenance window. | +| 1.4 full apps apply | Should be none, **verify** | Terraform will create a new auth revision (env added). Traffic is pinned to `00031-tox` by name, so the new revision should receive 0 %. | Confirm in the plan that no `traffic` change appears. If it does, stop: the Terraform image variable is not the serving image. | +| 1.5, 5.x alerts | No | | | +| 2.1 private IP | **Yes, certain** | Google: "Configuring an existing Cloud SQL instance to use private IP causes the instance to restart, resulting in downtime." No in-place path, HA does not avoid it. Expect 1–2 min DB unavailability: sign-in fails, relay renewals retry. **One-way door**: private IP cannot be disabled and the VPC link cannot be removed once set. The proxy flag change rides Roll 2. | Off-peak; only after Roll 1 (old image dies on a 2 min DB blip). Owner decision required before the foundation apply. | +| 2.2 DB split (deferred) | **Yes, scheduled** | Relay unavailable for the cutover (drain all cells → copy relay tables → flip `DATABASE_URL` → restart). Minutes if rehearsed. Desktops and phones reconnect automatically after. | Rehearse on staging; do it in the US night; announce. | +| 2.3 statement timeout | No beyond Roll 2 | | | +| 3.1 grace window | No | Auth deploys are no-traffic candidate → smoke → promote. | Security trade-off: a stolen token replayed inside the window is served once instead of revoking. 60 s is the usual choice. | +| 3.2, 4.3 desktop | No | Normal app update. | | +| 4.1 lock fix | No beyond Roll 2 | | Verify against real Postgres on 55440 with concurrent probes before shipping. | +| 4.2 region preference | **Minor, Asia users** | Phones that start being placed in Asia reconnect once to a nearer cell. | Roll out behind the existing region-preference flag. | +| 4.4 | No | | | + +## Checklists + +### 1.1 Cell image roll (Roll 1) +- [x] Confirm fleet is quiet: 15-min monitor dry-run passes. #19 green 23:07:53Z (run 33927238469). Canary then failed the evidence provenance check because main moved during the gate; re-gating with a same-commit chain. +- [x] Confirm director is on 519f4914 and c7 on 85bf6799 (confirmed 2026-09-04 via instance-template census; 20 serving cells still on `5aedbca5`) (`verify` mode of the same-cap workflow). +- [x] Dispatch `cloud-deploy-relay-production-same-cap` waves per the plan in the findings doc; one wave, verify, next. Done 2026-09-05 01:14Z–22:27Z: c8 canary, US batches c9–c10, c13–c16, c19–c26 at protocol 1, then Asia c27 (recovered via `mode=rollback` re-entry after gate freezes on the flat latency bar, fixed by #18877), c28, c29 as single-cell canaries at protocol 0. +- [x] After each wave: the transition verifier passed at migration-only and again at general on every cell (assignments carried, heartbeat fresh, hard cap 3 000); no `container die` fleet-wide across the whole roll. The 4408/1006 burst per wave was not measured separately; the verifier's assignment count before and after each restart is the recovery evidence recorded. +- [x] Record image census in the findings doc. 2026-09-05 22:27Z: all 19 general cells on `519f4914` except c7 on `85bf6799`; existing-only c1–c6, c11, c12 and migration-only c17, c18 untouched on their older images by design. Selector at gen 148. + +### 1.2 Enable pruning +- [x] `auth_token_pruner_image` = digest of `orca-cloud-auth-00031-tox` (`343a0915…`; it contains the entrypoint). orca-cloud #479 merged. +- [x] `auth_token_pruner_enabled = true`, `auth_token_pruner_max_rows_per_run = 20000` for the first day (orca-cloud #479). +- [x] Targeted plan asserted 9 create / 0 change / 0 destroy. Applied 2026-09-05 02:06Z. +- [x] Trigger one run by hand; read the summary event. 02:18Z: `time-budget`, 73 batches, 365k scanned, 1 040 deleted (1 021 revoked, 19 expired), no errors. Scan-bound. +- [ ] Raise the budget to the default 200k after a clean day; watch Cloud SQL write MB/s and the checkpoint alert. +- [ ] 1.5: log metric + policy on `stopReason != complete`. + +### 1.3 Reclaim +- [ ] Wait for steady-state runs deleting ~0 rows. +- [ ] `pg_repack -t refresh_tokens` off-peak (needs the extension; check `pg_available_extensions`). Not `VACUUM FULL` without a window. +- [ ] Confirm table + index size and `disk/utilization` dropped. + +### 1.4 Full apps-root apply +- [ ] Run from CI or a host with the 1Password account (local plan fails on the Cloudflare data source). +- [ ] Plan shows exactly the four known drifts and **no traffic change** on `google_cloud_run_v2_service.auth`. +- [ ] Apply; confirm `status.traffic` still pins `00031-tox` at 100 %. + +### 2.1 Private IP (PRs open: orca-cloud #477 foundation, stablyai/orca #18720 relay flag) +- [ ] **Owner decision**: the foundation apply restarts the instance and is irreversible on Google's side. Merging #477 arms the next foundation apply; hold the merge until the window is chosen. +- [ ] Director is out of scope: it uses the Cloud Run built-in connector (managed Google path, not the relay VPC NAT), so it consumed none of the exhausted ports; moving it needs Direct VPC egress + a separate DSN secret. Own PR if ever wanted. +- [ ] Step 7 (`ipv4_enabled=false`) is blocked until humans have IAP/bastion access and the director is moved; it breaks both today. +- [ ] Allocate a `/24` private services range on the relay VPC; `google_service_networking_connection`. +- [ ] Add `ip_configuration.private_network` to `google_sql_database_instance.auth` (foundation root). Plan must show update, not replace. +- [ ] Apply off-peak; expect a possible restart. Watch auth 5xx alert and relay `sqlFailures`. +- [ ] Cell template: proxy args add `--private-ip` (code merged #18720; flag not set). Director: Direct VPC egress or connector, then the same flag. Both ride Roll 2. +- [ ] After Roll 2: NAT `port_usage` for relay gateways drops to ~0; then consider `ipv4_enabled = false` (removes the public IP; breaks the local `cloud-sql-proxy --token` workflow unless it also goes private). + +### 2.2 Database split (deferred to ~2026-11-01; checklist kept for when it is revived) +- [ ] New `google_sql_database_instance.relay` (private IP from day one, its own size and flags). Staging first. +- [ ] Relay schema applies cleanly to an empty instance (it does at startup). +- [ ] Rehearsal on staging: drain → `pg_dump` relay tables → restore → flip `relay_database_url` secret → restart director + cells → phones/desktops reconnect. Time it. +- [ ] Production: announce a window; same steps; verify `orca_relay_runtime_metrics` controls recover to pre-cutover count. +- [ ] Update `production-cloud-sql-app-consumers` budget test and both alert policies' `database_id`. + +### 2.3 Relay pool statement timeout (merged stablyai/orca #18722; ships Roll 2) +- [x] `statement_timeout` on the relay `pg.Pool` (5 s, env-configurable; schema pool untimed; `57014` retryable), below the control-renewal deadline; DDL on an untimed connection (same pattern as auth #476). +- [x] Postgres test on 55440: a held lock fails the query fast and the bounded retry takes over. + +### 3.1 Refresh rotation grace window (orca-cloud #478 merged 2026-09-04; deploy pending owner go) +- [ ] Fix the deploy-script env strip for `ORCA_CLOUD_REFRESH_TOKEN_TTL_DAYS` (pre-existing; found by #478). +- [x] `rotateRefreshToken`: if `rotated_at` within 60 s and not revoked, return the existing successor (idempotent), no revoke, no audit. +- [x] Outside the window or a third presentation: unchanged (revoke + audit). +- [x] Tests: replay inside window returns same successor; outside revokes; concurrent double-present yields one successor. +- [x] Deploy via `deploy-auth-production` (candidate → smoke → promote). Deployed 2026-09-04 23:15Z as `orca-cloud-auth-00035-gos`, cap 20 kept, 0 5xx; `successor_material` column present; sealed successors being written. (candidate → smoke → promote). + +### 3.2 / 4.3 Desktop (merged stablyai/orca #18719; ships next desktop release) +- [x] 3.2: on refresh timeout, re-read stored session before retrying; do not re-send a token already rotated locally. +- [x] 4.3: ±10 % jitter on control lease renewal; unit test on the distribution; wire-compatible (server accepts early renewals already). + +### 4.1 Lock contention (partial: stablyai/orca #18722 merged; ships Roll 2) +- [x] Replace the global `FOR UPDATE` over `relay_cells` with per-cell row locks; counters delta-only. Remaining: `assignOnce` placement lock is still global (optimistic snapshot follow-up). with per-cell row locks or `pg_advisory_xact_lock(cell)`; counters delta-only. +- [x] Postgres tests on 55440 with concurrent probes (in #18722). Staging load run still owed; `postgres_retries` per hour drops in staging load run. +- [ ] Ships in Roll 2; then 4.4 recalibrates the retries bar from a week of data. + +### 4.2 Region preference +- [ ] Director: honor requested region when the preferred region has headroom, else sticky. Behind the existing flag. +- [ ] Measure with `orca_relay_runtime_metrics` region counters before/after. + +### 5.x Observability +- [x] **Relay-root runtime-metric drift**: resolved by dropping the `region` label to match live state (stablyai/orca #18734). Applied 2026-09-04 23:11Z: 8 never-applied `control_*` renewal metrics + the incident dashboard created, 0 destroyed, 21 live metrics untouched. +- [x] 5.1 `container die` log metric per cell (`relay_cell_process_exit`, applied 2026-09-04 via #18717), > 3 / 15 min, relay channel. +- [ ] 5.2 Add a paging channel (**needs owner input**: destination) to `auth_alert_notification_channels` for refresh rejections + latency. +- [x] 5.4 One dashboard (applied 2026-09-04 23:11Z): `orca_relay_cloud_sql_wal_checkpoint`, NAT drops, `orca_auth_refresh_401`, summed `controls`. diff --git a/cloud/docs/relay-improvement-roadmap-2026-09.md b/cloud/docs/relay-improvement-roadmap-2026-09.md new file mode 100644 index 00000000000..64f33c69a70 --- /dev/null +++ b/cloud/docs/relay-improvement-roadmap-2026-09.md @@ -0,0 +1,67 @@ +# Relay improvement roadmap (written 2026-09-04, after the auth/relay outage) + +Owner-facing list of what is left to make the relay more robust, in priority order. Evidence and history +for every item is in [`relay-reconnect-2026-09-findings.md`](./relay-reconnect-2026-09-findings.md) +(Findings 1–13). Everything already landed on 2026-09-04 is listed at the end so this file is complete on +its own. + +## 1. Finish what 2026-09-04 started (this week) + +| # | Item | Why | How | Size | +|---|---|---|---|---| +| 1.1 | **Roll all 23 cells onto the current relay image** | Every cell still runs the image that exits the whole process on a Postgres connect timeout (Finding 6). The fixed image runs only on the director and c7. Any future DB stall repeats the 200-crashes-in-48h pattern. | `cloud-deploy-relay-production-same-cap` waves, gated by the 15-min monitor. Roll inputs and canary results are in the findings doc ("Roll inputs", "Canary blast radius"). | one afternoon | +| 1.2 | **Enable the refresh_tokens pruning job** (orca-cloud #476, merged, off) | `refresh_tokens` is 63 M rows / 26 GB and grows forever; its size is what turned a slow disk into a sign-out storm (Finding 13). | Build an auth image from main (the 21:04Z deploy already contains the entrypoint: `orca-cloud-auth-00031-tox`, digest `343a0915…`), set `auth_token_pruner_enabled = true` and the image digest in `infra/terraform-apps/environments/production.tfvars`, apply targeted. First run with a small `auth_token_pruner_max_deleted_rows`. Watch the run summary's `stopReason`, not the exit code. ~48 M rows drain in ~10 days at 200k/hour. | 1 hour + 10 days of watching | +| 1.3 | **Reclaim the disk after pruning** | Deletes leave dead tuples; the 16 GB table does not shrink on its own. | `pg_repack` (or `VACUUM FULL` in a maintenance window; it takes an exclusive lock) on `refresh_tokens` off-peak, after 1.2 finishes. | 1 evening | +| 1.4 | **Full Terraform apply of the orca-cloud apps root** | The production plan carries four drifts from other merged work: `ORCA_CLOUD_REFRESH_TOKEN_TTL_DAYS` env on the auth service (#476), a skill-share log exclusion filter change, skill pressure threshold 16→8, an artifacts bucket lifecycle rule. Locally it also fails on the 1Password Cloudflare data source. | Run from CI or a machine with the 1Password account; review the four drifts as ordinary changes. | 30 min | +| 1.5 | **Alert on the pruning job** | A run that only ever times out exits 0 and reads as green. | Log metric on the job's summary event where `stopReason != "complete"`, policy on the relay channel. | 1 hour | + +## 2. Remove the shared fate between auth and relay (2.1 and 2.3 this quarter; 2.2 deferred) + +| # | Item | Why | How | Size | +|---|---|---|---|---| +| 2.1 | **Private IP for Cloud SQL, `--private-ip` on the cell proxies** (do this on the existing shared instance; do not wait for 2.2) | Cells reach the database's public IP through Cloud NAT. Dynamic port allocation (landed) raised the ceiling from 64 to 4096 ports per VM, but the NAT is still in the path and its logs are still the only place port exhaustion shows up (Finding 11). | Add a private IP to `orca-cloud-auth-db` (foundation root, orca-cloud), peer the relay VPC, switch the proxy flag in the cell template, roll. | 1–2 days | +| 2.2 | **Split the relay database from the auth database** — *DEFERRED 2026-09-04 (owner decision): revisit ~2026-11-01 once pruning is done and there is a month of alert history* | One Cloud SQL instance serves `orca_auth`, `orca_relay`, `orca_push`, `orca_skills`. The auth table's growth stalled the relay for a day (Findings 10, 13). Deferral rationale: the concrete cause is fixed (disk 250 GB, WAL 16 GB, index, pruning), 2.3 + 1.1 turn a future stall into retries, and the checkpoint/disk/headroom alerts now page. Re-open if the checkpoint-loop or connection-headroom alert fires, or a large new auth-side table is planned. | New instance for `orca_relay`; migrate with a short relay drain. Relay state is small so the cutover is minutes. | 1–2 weeks incl. rehearsal on staging | +| 2.3 | **Statement timeouts on the relay pool** (the auth pool got one in #476) | A relay query stuck behind a checkpoint fsync should fail fast and let the bounded retry take over rather than hold a pool slot for seconds. | `statement_timeout` on the relay `pg.Pool` in `cloud/apps/relay`, tuned under the lease renewal deadline. | half a day | + +## 3. Make the desktop refresh path forgiving (next 2 weeks) + +| # | Item | Why | How | Size | +|---|---|---|---|---| +| 3.1 | **Refresh-token rotation grace window** | The server revokes the whole family the first time a just-rotated token is presented again. On 2026-09-04 that turned a 30 s server slowdown into 21,605 sign-outs. A short window (e.g. 60 s) where the immediately-previous token is still accepted, returning the same new token, is standard practice. | In `apps/auth/src/tokens/refresh-tokens.ts`: accept `rotated_at` within the window, return the successor instead of revoking. Keep true reuse (outside the window, or a third presentation) as revocation. | 1 day incl. tests | +| 3.2 | **Do not retry `/refresh` with the same token on timeout** | Desktop's 30 s `CLOUD_REQUEST_TIMEOUT_MS` expiring is treated like a network error and retried with a token the server may already have rotated. | In `src/main/orca-profiles/profile-cloud-session-refresh.ts`: on timeout, re-read the stored session first, and prefer a longer single attempt for the refresh call specifically. | half a day | +| 3.3 | **Un-revoke is impossible; make sign-out recovery obvious instead** | Server-side un-revoke does not help because the desktop deletes its local token on the 401. Landed: desktop notices immediately (#18694) and the phone says "desktop signed out" (#18698). | Nothing more unless we want a re-auth deep link from the phone to the desktop. | — | + +## 4. Chronic relay issues already characterised + +| # | Item | Why | How | Size | +|---|---|---|---|---| +| 4.1 | **Cell-inventory lock contention** (partial: PR #18722 narrowed the remaining non-placement sites; `assignOnce` placement lock is the follow-up) | `postgres_retries` is a global `FOR UPDATE` over the 23-row `relay_cells` table with a 1 s `lock_timeout`; it is the floor under every 503 and every slow phone accept (Findings 2, 5; memory `relay-cell-inventory-lock-contention`). | Per-cell row locks or an advisory lock keyed by cell; move capacity counters to delta writes. Verify against real Postgres on 55440. | 1 week | +| 4.2 | **Region preference is mostly inert** | Phones request an Asia cell on ~19 % of attempts and get one ~6 % of the time; the sticky lane wins silently, so Asia users ride the US path more than intended (memory `relay-region-preference-mostly-inert`). | Let a region preference override stickiness when the preferred region has headroom; measure with `orca_relay_runtime_metrics` region counters. | 2–3 days | +| 4.3 | **Desktop lease-rotation waves** | A cell recreate seeds a fleet-wide 1006/4408 reconnect burst ~54 min later, every ~54 min (Finding 3). | Jitter the desktop control lease renewal by ±10 % so the cohort spreads out. | half a day, desktop + wire-compatible | +| 4.4 | **Raise `postgres_retries` gate calibration** | The 300 bar was recalibrated (PR #18580) but should track the post-lock-fix baseline once 4.1 lands. | Re-derive from a week of `orca_relay_postgres_transaction_retry` counts. | 1 hour | + +## 5. Observability still missing + +| # | Item | Why | How | +|---|---|---|---| +| 5.1 | **Cell crash-rate alert** | 201 process exits in 48 h with no page (Finding 6). | Log metric on `container die` for `resource.type="gce_instance"` relay cells, > 3 per 15 min per cell. In `cloud/infra/terraform/relay-observability.tf`. | +| 5.2 | **Page a person for auth alerts** | Today's four auth policies (orca-cloud #475) route to the relay Slack channel only. A repeat of 2026-09-04 deserves a page. | Add a PagerDuty/phone notification channel to `auth_alert_notification_channels` for refresh rejections and latency. | +| 5.3 | **Pruning job alert** | See 1.5. | | +| 5.4 | **Dashboard that puts the four signals side by side** | Diagnosis took hours because checkpoint state, NAT drops, auth 401 rate, and fleet controls live in four consoles. | One Cloud Monitoring dashboard: `orca_relay_cloud_sql_wal_checkpoint`, NAT `dropped_sent_packets_count`, `orca_auth_refresh_401`, summed `controls`. | + +## Landed on 2026-09-04 (for completeness) + +- Auth service cap 2 → 20 (service-level manual scaling removed); Cloud SQL disk 49 → 250 GB PD-SSD; + `max_wal_size` 16384; partial index `refresh_tokens_family_unrevoked` built concurrently by hand. +- orca-cloud #474: the above in Terraform + deploy workflow; replayed dead token answers 401 without + re-revoking or re-auditing. Deployed as `orca-cloud-auth-00031-tox` 21:04Z. +- orca-cloud #475: auth alerts (refresh 401 > 100/5 min, 429 > 20/5 min, 5xx > 10/5 min, p99 > 10 s). Applied. +- orca-cloud #476: batched `refresh_tokens` pruner (disabled), auth pool `statement_timeout` 10 s, schema + DDL on an untimed connection. +- stablyai/orca #18693: both relay NATs on dynamic port allocation 64..4096 (applied US 21:01Z, Asia 21:05Z); + alerts for Cloud SQL WAL-checkpoint loop, disk > 70 %, NAT `OUT_OF_RESOURCES` drops. Applied. +- stablyai/orca #18694: desktop learns of a revoked session immediately, panes re-fetch on mount, pairing + notice says "Sign in again to use Orca Relay". +- stablyai/orca #18698: phone shows "Desktop signed out — sign in to Orca on your desktop to reconnect" via + the WebSocket close reason (only additive slot old phones tolerate). +- Director on image 519f4914; c7 on 85bf6799; other 22 cells still on the old image (see 1.1). diff --git a/cloud/docs/relay-reconnect-2026-09-findings.md b/cloud/docs/relay-reconnect-2026-09-findings.md new file mode 100644 index 00000000000..426a120c251 --- /dev/null +++ b/cloud/docs/relay-reconnect-2026-09-findings.md @@ -0,0 +1,991 @@ +# Relay reconnect investigation: findings and evidence + +Working notes for the 2026-09-04 mobile relay reconnect incident and the cell roll that follows. +Kept current across context compactions. Newest section first. All times UTC. Host ids are log digests, +never raw ids. Nothing here is a production mutation record unless the "Mutations" section says so. + +## Status board + +| Item | State | Where | +|---|---|---| +| PR #18565 relay accept abandonment + lease jitter + desktop rotation spread + phone probe fail-fast | Open, CI fully green again after the doc move (05:45Z), CodeRabbit + Pullfrog cleared, 3 review rounds; not merged (owner has not asked) | https://github.com/stablyai/orca/pull/18565 | +| PR #18569 monitor `relayPostgresRetryExhausted` 0 -> 300 | **Merged** 2026-09-04 ~04:20Z as 4101505b6b | https://github.com/stablyai/orca/pull/18569 | +| Same-cap `verify` of c7 (read-only) | **Passed** run 33836527159 | confirms identities, selector gen 110, rehome gen 12, protocol 1, digests | +| Monitor dry-run #1 | Froze min 5: `relay.postgres_retries` 380 > 300 | run 33836470590 | +| Monitor dry-run #2 | Green to min 13, froze 04:49Z: `director.concurrency` 76.7 > 64 (six-cell crash storm, Finding 6) | run 33837160275 | +| Monitor dry-run #3 | Froze min 3 at 05:01Z: `relay.postgres_retries` 339 > 300; no crash, concurrency 5–8 | run 33838698725 | +| Owner decision 2026-09-04 ~05:10Z | **Option B approved**: "you can raise the bar. or remove it altogether ... whats the most logical move". Kept the bar (removal would leave contention unwatched during the roll) and recalibrated from measured data. | this thread | +| PR #18580 monitor `relayPostgresRetries` 300 -> 2000 | Open, awaiting CI; mutation-checked (300 fails the new test) | https://github.com/stablyai/orca/pull/18580 | +| PR #18565 CI | Was red on `root directory guard` because this findings file sat at repo root; moved to `cloud/docs/` in 8ebff89106 | | +| PR #18580 | **Merged** 2026-09-04 05:23Z as 79d5fb469a (Pullfrog cancelled by the merge; independent Opus review requested instead, per owner) | | +| Monitor dry-run #4 | Froze min 12 at 05:37:35Z: `cell.production-gce-c27.health`/`.ready` = 0. Retries green all 12 samples under the new 2000 bar. Cause: c27 (asia-east2) container died 3x 05:37:00–05:38:01Z, Finding 6 crash class. | run 33840364323 | +| Monitor dry-run #5 | Froze at sample 1 (05:41Z): c27 health/ready still 0. MIG autoheal `recreateInstance` on c27 fired 05:38:12Z after the 3 crashes; instance RECREATING, process up with 0 controls (was ~395). Second c27 recreate in 7 h (Finding 3 seed pattern). Waiting for c27 to settle before dry-run #6. | run 33841327879 | +| Monitor dry-run #6 | **Passed** 06:06:31Z: 16 samples, no freeze (started 05:47:42Z) | run 33841783747 attempt 1 | +| c7 `canary-apply` | **Succeeded.** Dispatched 06:07:15Z; drain 06:10Z; MIG recreate 06:16–06:23Z; new image listening 06:23:42Z; verify + trust proof passed; restored to `admission=general` 06:25:21Z; canary authority sealed. c7 is on `85bf6799…`. | run 33843071283 | +| PR #18581 doc reconcile (Aug 23 figure: 2,200–3,000 raw log lines vs 1,510 on the gate metric) | **Merged** | https://github.com/stablyai/orca/pull/18581 | +| Same-cap `verify` c7 target=519f4914 rollback=85bf6799, gen 112 | **Passed** (read-only) | run 33856355648 | +| Monitor dry-run #7 (gen 112) | Froze at sample 1 (09:05:31Z): `director.errors` 4 > 0, the four 2.0 s pg-connect 500s from the 09:00 cascade still inside the 5-min delta window. Dispatched 4 min too early. | run 33856521278 | +| Monitor dry-run #8 (gen 112) | Green for 15 of 16 samples (09:09:38–09:24), froze on the final sample 09:25:22Z: `director.errors` 1 > 0. The one 500 was `/v1/admin/evacuation-status` at 09:23:50Z, 2.01 s latency = director pg-connect timeout, called by **the monitor's own collector** (`incident-monitor-sources.ts:492`). First evacuation-status 500 since Sep 1. The gate froze on a request it made itself. | run 33856905229 | +| Monitor dry-run #9 (gen 112) | Froze: c13/c23 crashed 50 s after dispatch, then c14/c20/c9 at 09:34. | run 33858650691 | +| Monitor dry-run #10 | Dispatched 09:46:13Z; froze at sample 5 (09:56:59Z): `director.errors` 12. All twelve at 09:55:17–21Z, 0.8–2.1 s latency, 10 on `/v1/regions` + 2 on `/v1/assign`; c16 and c8 crashed at 09:55:19 in the same second. A single 4-second Postgres connect stall hit director and cells together. | run 33859947207 | +| Monitor dry-run #11 | Froze at sample 2 (10:08:07Z): `director.concurrency` 79.8 > 64, the c8/c20 re-dial. They crashed 10:05:54, 3 s before the waiter's quiet check passed (log ingestion lag). | run 33861578009 | +| Monitor dry-run #12 | Dispatched 10:17:38Z after 10 quiet min; froze at sample 2 (10:19:24Z): `cell.production-gce-c16.health` 0. c16 did **not** crash (no container die, MIG NONE/HEALTHY, readiness=true throughout, `/health` 200 in 230 ms at 10:21). At 10:19:07–16 it logged "control activity renewal failed" x4 and a burst of 1006 closes, sqlFailures 1 -> 14, sqlLatencyMsMax 2588: a pg stall on the old image that did not reach the unhandled path. The probe's single fetch (30 s timeout) came back unavailable during that stall and `unavailableIsZero` turned it into health=0. | run 33862504601 | +| Monitor dry-run #13 | Green 14 of 16 samples (10:48:38–11:03), froze 11:04:43Z: c9 crashed 11:04:23, c28 11:04:25 (then looped 11:05:04, 11:05:41); c15 probe also read 0 (stall, no crash). Missed by ~90 s. **Dispatched by hand 10:48:15Z** into a 43-min crash lull (last die 10:05:54; last director 500 10:31:49). The re-armed waiter never fired: its MIG-stable check used `grep -vc True`, which exits 1 when nothing matches, so `&&` short-circuited on the *healthy* case. Waiter armed 10:20Z: 10-min quiet + every MIG stable + 60 s recheck, then dispatch, then canary c7 on green. Held at 10:24 and 10:31 by lone director `/v1/assign` 500s (2 s pg-connect stalls, no cell crash). Director 500 events since 08:46: 6 (gaps 2.7/21/31/29/7.6 min). At 10:39 the waiter was re-armed with a 6-min director-500 window (the monitor's own delta is 5 min) instead of 10, since the gate only needs the 15 min *after* dispatch to be clean. Cell crashes have stopped since 10:05 (33+ min, longest gap since 08:40). 12 dry-runs: 1 pass (#6), 11 freezes, none on a real fleet-health regression. | Cascade gaps since 09:00: 31, 2.9, 5.1, 16.1, 4.0 min (median 5); a 15-min clean window is ~28% per attempt at this rate. | | +| Monitor dry-run #14 | Dispatched 11:26:53Z by the fixed waiter (first autonomous dispatch); c14, c23, c25, c15, c24, c19 died 11:30:59–11:31:08 (six cells, 13 min after the last cascade). Froze on c8 (and others) health/ready probes. Waiter re-armed 11:06Z (grep bug fixed: `grep -c` under `|| true`), same chain; held through the 11:17 cascade and c14/c28 recreates. 13 dry-runs: 1 pass, 12 freezes. Since 08:40: 10 cascades, 75 container dies, gaps 20/31/3/5/16/4/6.5/58/13 min; only 3 windows of >=17 clean minutes existed in 2.6 h, and dry-runs hit two of them (#6 passed, #13 lost the third by 90 s). | +| Monitor dry-run #15 | Waiter armed 11:33Z (6-min director-500 window, 8-min crash window, all MIGs stable), chained canary; still holding at 12:04Z. Since 11:00: 8 cascades, 98 dies, gaps 13/13.6/3.6/14.5/4.4/6.1/3.0 min, **max gap 14.5 min**, so no 15-min clean window has existed in the last hour. 14 dry-runs: 1 pass, 13 freezes. | +| Monitor dry-run #15 verdict | Dispatched 12:28:49Z; froze at sample 2 (12:30:41Z): **12 cells** health/ready = 0 at once (c4, c5, c7, c10, c15, c16, c18, c20, c22, c25, c27, c28), including c4/c5 (0 controls all day, `/health` 200 in 190 ms a minute later) and c7 (new image). Six old-image cells also crashed 12:30:02–21. This was a fleet-wide SQL stall, not a cascade: every cell's `sqlLatencyMsMax` hit 4–6 s (c7 4865, director 5140), director pool waiting 1258, 15 cell pg-connect timeouts, director sqlFailures 92. Cloud SQL CPU 0.73, backends 160, new connections normal, memory 0.46, so the *instance* was not saturated; something held the database for ~5 s. Postgres log 12:31:23–28 shows a burst of `could not obtain lock on row in relation "relay_cells"` from NOWAIT (single-row and full-inventory) sweeps, i.e. the row locks were held during recovery. Cloud SQL transactions/min flat (~30k), reads flat, +network flat: the database was neither busy nor saturated, it was *waiting*. The stall bracket +(12:30:02–12:30:41) is where every cell's SQL max hit 4–6 s at once. Lock retries in that window were +ordinary (49/29/13 per min). Best reading: a ~5 s Postgres-side wait event shared by every session +(lock on a hot row held across a long transaction, or an instance-level pause), not CPU/IO. Cell +`sqlLatencyMsMax` was already 1.5–2.2 s fleet-wide in the four minutes before, i.e. the old cells' 1 s +`lock_timeout` plus queueing. | run 33872946111 | +| Monitor dry-run #16 | Dispatched 12:38:57Z; froze at sample 1 (12:40:11Z): `cell.production-gce-c27.latency_ms` 2071 > 2000, a fifth distinct freeze signal, the probe's own round-trip absorbing a checkpoint sync. **Loop stopped by me at 12:41Z**: with the disk in the checkpoint loop (Finding 10) no bar can hold for 15 min, so further dry-runs only burn the shared rollout lease. 16 dry-runs: 1 pass, 15 freezes. Re-arm after the disk change lands. | +| Cloud SQL checkpoint loop | **Broke on its own 12:39–12:45Z**: disk writes 48 -> 4 MB/s at 12:39 with transactions and network flat and no Cloud SQL operation; 12:40:17 checkpoint synced 0.047 s; 12:45:53 checkpoint was `time`-triggered again (first since 11:55) with sync 0.096 s and write spread over 269 s. Cause of the break unknown (most likely WAL fell back under `max_wal_size` once a burst of full-page writes aged out). It can re-enter the loop on the next large checkpoint; the disk-size fix remains the durable one. | +| Monitor dry-run #17 | Dispatched ~12:49Z (all guards clean); froze at sample 1 (12:52:05Z): `director.errors` 4, from the c9/c22 crash loop that began 12:50:34, ~90 s after dispatch. Checkpoints stayed healthy (85 ms), so this is the old image's baseline crash rate, not the disk. 17 dry-runs: 1 pass, 16 freezes. | +| Monitor dry-run #18 | **Dispatched by mistake 13:48:56Z into the outage**: my gcloud credentials expired ~13:45Z, every guard query returned empty, and the waiter's `grep -c . || true` read empty as "quiet". Froze at sample 1 (13:49:43Z) on `director.ready=0`, `auth.health=0`, and cell probes; no canary dispatched, no production mutation. All waiter loops killed at 13:51Z. Lesson: a quiet-window check must fail closed when its data source errors. Waiter had been re-armed 12:53Z. | +| Gate decision | Owner asked at 09:36Z to choose: A keep looping / B recalibrate `directorErrors` 0 -> small n / C human bypass. Ten dry-runs, four froze on this bar. Recommendation B+A. Note: B alone would not have passed #9 or #10 (cell health probes and a 12-error burst); it fixes the single-500 false freezes (#7, #8) only. | | +| Batch roll | **Deferred by plan**: roll once with the lock-fix image instead of twice. | | +| PR #18606 lock removal (root cause) | **Merged** 09:2xZ as 7b108abf71 after review, fix, re-verify; CI green | https://github.com/stablyai/orca/pull/18606 | +| Image publish for 7b108abf71 | **Done** 08:36:49Z run 33854111305: `sha256:519f4914217f08cabcdcd34825965db8473ec37c6591553a3af0d65dcdeeb183` | | +| Director deploy on 519f4914 | **Succeeded** 08:45Z run 33854355791; serving `orca-cloud-relay-00570-siv`, rollback tag on 00569-ret (also 519f4914), 00565-fes (85bf6799) still deployable. Dispatched 08:37:45Z (blue/green; prior revision 00565-fes on 85bf6799 kept as rollback). Note: `predecessor-image-digest` is a required input even with bootstrap=false; pass the serving digest. | `cloud-deploy-relay-production-director.yml` | +| c7 on new image, 2 h in | 817 controls, **0 container die** since restore (was ~1 per 15 min on old image); `sqlLatencyMsMax` still 1.0 s = lock wait unchanged, which #18606 targets | | +| Terraform alert `relay_postgres_retry_exhausted` at `> 0` | Firing continuously since #18521; recalibration not done (own change) | `cloud/infra/terraform/relay-observability.tf:447,469` | + +## Mutations performed (complete list) + +1. Merged PR #18569 to main (code/docs only). +2. Merged PR #18580 and #18581 to main (monitor bar + docs). +2b. Merged PR #18606 to main (relay lock change; no serving effect until the image is deployed). +2c. Dispatched `cloud-publish-relay-production` for 7b108abf71 (builds and pushes an image; changes nothing serving). Done: 519f4914. +2d. Dispatched `cloud-deploy-relay-production-director` on 519f4914 (preserve placement, no prune, rehome gen 12). Succeeded 08:45Z; serving revision 00570-siv. Rollback: `gcloud run services update-traffic orca-cloud-relay --region us-central1 --to-revisions orca-cloud-relay-00565-fes=100` (85bf6799, still Ready). Not needed so far. +3. 2026-09-04 06:07:15Z: dispatched `cloud-deploy-relay-production-same-cap` `canary-apply` for production-gce-c7 only (run 33843071283). Completed successfully 06:26Z: c7 isolated, drained (807 controls re-dialed), template + MIG rolled to 85bf6799, verified, restored to general admission. Selector generation advanced 110 -> 112 (isolate + restore). +4. Nothing else. Both monitor dispatches were `mode=dry-run` (read-only). The same-cap dispatch was `mode=verify` (read-only, confirmed by step gates `if: inputs.mode != 'verify'` on every mutating step). + +## Finding 6 (2026-09-04 ~05:00Z): the old cell image crashes the whole process on a Postgres connect timeout + +**This is the most important open finding.** The 23 GCE cells run image `sha256:5aedbca5…` = orca-cloud +commit e3e92d95d3 (2026-08-14). In that build `beginProof` is called as `void this.beginProof(...)`. +When `verifyCellAssignment` inside it throws (pg-pool `timeout exceeded when trying to connect`, 2 s +`connectionTimeoutMillis`), the rejection is unhandled and Node exits 1. Docker restarts the container +in ~1 s, but every control on that cell (~800 hosts) drops and re-dials `/v1/assign` at once. + +Evidence, cell c7 instance 4545742188814054238, 2026-09-04: + +``` +04:46:47.951 stderr [orca-relay] control activity renewal failed (x5) +04:46:49.527 stderr Error: timeout exceeded when trying to connect + at pg-pool/index.js:45:11 + at async PostgresPoolPressure.connect (postgres-pool-pressure.js:30:20) + at async PostgresDatabase.query (database.js:645:24) + at async RelayAssignmentStore.verifyCellAssignment (assignment-store.js:2024:22) + at async HostSessionRegistry.beginProof (host-session-registry.js:376:15) +04:46:49.527 stderr Node.js v24.19.0 +04:46:49.835 dockerd: container die … exitCode=1 image=…relay@sha256:5aed… +04:46:50.258 dockerd: container start +04:46:52.761 stdout [orca-relay] listening on https://c7.relay.onorca.dev +``` + +2026-09-04 05:36:59–05:38:01Z: c27 died 3x in 62 s plus one other instance (5464389947731541178); this froze dry-run #4 on c27's health probe. + +Fleet-wide `container die … exitCode=1` on the relay image, last 48 h: **201 events on 19 instances** +(c28 x38, c29 x37, c27 x19). Hourly counts track the lock-contention curve (peak 23/h at 21Z Sep 3). +Every one has the same `Node.js v24…` crash banner. On 2026-09-04 04:46:35–04:47:41Z six cells +(c7, c8, c19, c21, c22, c25) died within 66 s: ~4,800 hosts re-dialed, `/v1/assign` returned 16,321 +503s in one minute (baseline ~20), director concurrency hit 85 (Cloud Run cap 80), Cloud SQL +`new_connection_count` 119 -> 287/min. Fleet recovered by 04:51Z. That is what froze dry-run #2. + +Fix status: `guardSessionTask` wrapping `beginProof` landed in orca-cloud #436 (2026-08-27) and is in +the target image `sha256:85bf6799…` (main 11aace8dec). The roll is the fix. Not caused by anything in +this session: the same-cap verify finished ~04:25Z and never reached a mutating step; no compute +operations exist for those instances; heap/event-loop were flat before the crash. + +Autoheal amplifier: MIG health check is `/health` every 10 s, timeout 5 s, unhealthy after 3, so a +crash loop of ~30 s+ triggers `compute.instances.repair.recreateInstance`. All ~20 recreates in the +48 h to 2026-09-04 05:40Z were the three Asia cells (c27 x6, c28 x7, c29 x8; gcloud prints local +-07:00 times). c27 recreated 05:38:12Z after 3 crashes in 62 s; its ~395 controls went to 0 and the +monitor's `cell.production-gce-c27.health/ready` probe read 0 for the whole recreate (~several min), +freezing dry-runs #4 and #5. Each recreate also seeds a Finding 3 rotation cohort. Rolling the Asia +cells early in the batch phase should be weighed against the canary-first rule; c7 stays the canary. + +Implication for the gate: the monitor's `director.concurrency` freeze is *correctly* detecting these +crash storms. A dry-run only passes in a 15-minute window with no cell crash, roughly 1 in 3 windows +at current rates. Retrying in quiet hours is legitimate; the bar is not wrong. + +## Finding 5: `relay.postgres_retries` at 300 is 3x under today's baseline + +Retries per 5 min, cells + director, last 24 h: p50 579, p90 1039, p99 1398, max 1505; **65% of +windows over 300**. Quiet hours (03–08Z) p50 235, max 512. When the 300 bar was set (2026-08-26) +healthy bursts reached 234. Baseline has roughly tripled in 10 days. Skill notes say do not raise this +bar; I have not. Best odds for a clean 15 min are 02–04Z and 17–18Z (9/12 five-minute windows under +300 in each). + +## Finding 4: exhausted-retry bar was the wrong single blocker (fixed) + +`relayPostgresRetryExhausted: 0` never cleared after #18521 reached the director (22:12Z Sep 3): 236/236 +five-minute windows non-zero; post-#18521 p50 42 / p90 147 / max 220; Aug 23 incident peak 467. +Recalibrated to 300 in #18569 (merged). Dry-run #1 immediately revealed Finding 5 behind it. + +## Finding 3: the 00:50Z control-close wave was desktop lease rotation, not a rollout + +2026-09-04 00:49–00:51Z: 2,745 control closes on 19 instances; 1157/1632 code 1006 and 973/1030 code +4408 `control rebound` had ageMs in the 53-minute bin. Relay grants a flat 55 min lease; desktops +rebind 60–120 s early; so every host that (re)connected in the same minute rebinds as one cohort +forever. Seed: c27 MIG autoheal recreate 23:23Z (`compute.instances.repair.recreateInstance`) dumped +~420 controls. Harmonics at 23:55, 00:04, 00:25, 00:49Z. Each rebind is an `activateControl` +transaction that can take the inventory lock. Fix in #18565: relay lease 55 min ± 5 min (symmetric, +so mean rebind rate unchanged), desktop early window 1–6 min. + +## Finding 2: fleet-wide lock contention, worse on Sep 3 + +| window | 55P03 retries/h (cells) | cell sqlFailures/h | +|---|---|---| +| Sep 2 18Z – Sep 3 07Z | 660–1470 | 680–1620 | +| Sep 3 08Z–16Z | 3600–7100 | 3700–7700 | +| Sep 3 23Z | 7468 | 7585 | + +100% of sampled retries are 55P03; director phase is `cell-inventory`. Every cell pins +`sqlLatencyMsMax` at 1.0–1.2 s = the pre-#18521 1 s pool `lock_timeout`. Not load (controls flat +~26k, Cloud SQL CPU 46–53%). No `cloud-*` workflow explains the 08Z step. The lock is a global +`SELECT * FROM relay_cells FOR UPDATE` (23 rows) taken by assignment, control activation, activity +acquire, and sweeps, held to COMMIT. + +## Finding 1: root cause of the phone's 24 s hang (the original symptom) + +`acceptClient` runs four serialized Postgres calls; the fourth (`acquireActivity`) contends for the +global lock. Under contention the cell finishes after the phone's 12 s bound, then +`PendingHostDataReservation.bind` throws `host_data_reservation_already_bound` because the phone's +close already released the reservation. Every "first frame handler failed already_bound" line is that +post-mortem (31 events 23:06–01:01Z across 12 instances). Fix in #18565: abandon the accept after each +DB step once the socket is closed; new event `orca_relay_client_accept_abandoned {stage, elapsedMs}` +and metric fields `clientAcceptsAbandonedByStageDelta` / `clientAcceptAbandonedMsMax`. Phone side: +direct probe now fails fast on `reconnecting` so relay recovery is not queued behind three doomed +LAN redials (~3.5 s saved per foreground). #18518 (merged, not yet on the phone) covers the +stage-aware dial bound. + +Host 666077865f2e: stable throughout. 4408 rotation 00:27:45Z; 1006 quit 00:52:24Z on old adhoc; +sticky reassignment to c27 00:52:35Z on new build; rotation closes 01:44:55Z and 02:23:15Z with +splices intact. No drain/4404/wrong-cell. + +## Finding 7 (2026-09-04 ~05:10Z): retries bar recalibration basis (PR #18580) + +Chose 2000 over removal. The metric is the gate's own source (`orca_relay_postgres_retries` +log metric, director + cells summed per five minutes, ALIGN_DELTA 300 s): + +| window | p50 | p90 | p99 | max | > 300 | +|---|---|---|---|---|---| +| 2026-09-01 | 56 | 105 | 206 | 456 | 0% | +| 2026-09-02 | 109 | 186 | 294 | 377 | 1% | +| 2026-09-03 | 430 | 924 | 1320 | 1504 | 55% | +| 2026-09-04 to 05Z | 285 | 1012 | 1211 | 1211 | 44% | + +15-minute pass rate, last 24 h: bar 300 -> 22%, 800 -> 66%, 1000 -> 86%, 1500 -> 99%, 2000 -> 100%. +Aug 23 incident on this metric: 1510 then 646 (single windows), so retries no longer separate an +incident from baseline; exhausted (467 vs bar 300; healthy 72 h max 184), director concurrency, +and pool bars carry that role. Note: my earlier "p99 1398 / 65% over 300" in Finding 5 came from +raw log line counts; the metric-based numbers above are what the gate actually evaluates. +Baseline tripled between Sep 2 and Sep 3 with no deploy; still unexplained (Finding 2). + +## Decision needed from the owner (resolved: B) + +The same-cap roll is blocked only by the monitor gate, and the gate is blocked by `relayPostgresRetries: 300` +(Finding 5: 65% of windows breach it; even the 04:55Z quiet window hit 339). Three options: + +- A. Keep waiting for a naturally quiet 15 min. Odds per attempt ~1 in 3 in quiet hours, lower by day. + Each attempt is free and read-only. Could take hours. +- B. Recalibrate `relayPostgresRetries` from measured data, same method as #18569: 24 h p99 is 1398, the + Aug 23 incident ran 2200–3000, so ~1500 clears healthy windows with ~1.5–2x incident separation + (less margin than the exhausted bar had). Overrides the "do not raise" note in the skill facts. + Argument for: the roll being gated is the thing that reduces retries. Argument against: the bar is + doing its job of saying contention is high. +- C. A human dispatches the roll with a different gate policy. Not something I can or should do. + +My recommendation: B, with the number chosen from the table in Finding 5 and the roll following +immediately so the bar can be re-tightened after the fleet is on the 500 ms lock wait. + +## Finding 12 (2026-09-04 13:12Z): **INCIDENT IN PROGRESS. The auth service is at its 2-instance cap and rejecting 90% of desktop token calls with 429; the relay fleet has emptied.** + +Timeline: 13:04–13:06 the old-image cascades and NAT stalls drove ~1,400 desktops to re-dial. Their relay +JWTs (5-min TTL) expired mid-storm, so they hit `orca-cloud-auth` `/v1/desktop/auth/refresh` and +`/v1/desktop/auth/relay-token` together. The auth service is Cloud Run `maxScale=2`, `concurrency=80`, +1 vCPU throttled (`auth_max_instances = 2` in orca-cloud `infra/terraform-apps/environments/production.tfvars`, +applied by `deploy-auth-production.yml`). Both instances pinned at concurrency 85 from 13:02; from 13:07 +Cloud Run's front door returns **429 "no available instance"** (0 s latency, never reaches the container): +12,045 at 13:07, 54,292 at 13:08, 46,025 at 13:08, 42,529 at 13:09. Sep 3 total auth 429s: **0**. +Without a fresh relay token every desktop's `/v1/assign` gets 401 (1,433 distinct hosts 401'd, 0 got 200 +since 13:07) and every cell closes its control with `4401 relay authorization expired`. Fleet controls: +13,375 (12:55) -> 7,633 (13:08) -> **249 (13:12)**, splices 1. Auth container CPU 0.15–0.5, so the cap is +the limit, not the code. Every desktop is now in its refresh-retry loop hammering the same 2 instances: +this is a self-sustaining thundering herd and will not clear on its own. At 13:14Z: fleet **30 controls** +across 23 cells; successful relay-token issuance 5,000–6,500/min until 13:05, then 1,059 / 734 / 733 / +443 / 220 / 214 / 148 / **4** per minute through 13:13; auth 429s 54k -> 25k/min only because desktops +are backing off, not because the service recovered. Note `AUTH_MAX_INSTANCES: 2` is also hardcoded in +orca-cloud `.github/workflows/deploy-auth-production.yml` (lines 33–34), so a redeploy would re-pin it; +change both the workflow env and the tfvars. + +**Immediate mitigation (owner action, not applied):** raise the auth service's max instances. Fastest: +`gcloud run services update orca-cloud-auth --region us-central1 --max-instances 20` (or `10`, matching +the other apps' `max_instances = 10`), then land the same in `auth_max_instances` so Terraform does not +revert it. Auth is stateless behind Cloud SQL (`refresh_tokens` table); backends 210 of 400, so 20 +instances x a small pool is within budget. Also consider the desktop's refresh backoff: it re-dials on +401 immediately with no jitter, so a 429 storm sustains itself. + +**13:51Z status: my gcloud session lost auth at ~13:45Z; all production monitoring from this session is +blind until re-authenticated (`gcloud auth login`, interactive). Last confirmed state 13:40Z: fleet 0 +controls, auth maxScale 2, 7,600 auth 429/min. All autonomous dispatch loops are stopped.** + +**17:19Z–17:21Z MITIGATION APPLIED (owner said "fix it NOW").** State at 17:19Z, four hours in: all 23 +cells at 0 controls, auth 429 ~2,000/min, auth 2xx ~40/min, and the 2xx that got through took 13–28 s +(both instances saturated). Mutation 1: `gcloud run services update orca-cloud-auth --max-instances 20` +created revision `orca-cloud-auth-00018-4jc` (same image `auth@sha256:1710ff6c`, same env/concurrency, +only maxScale 2 -> 20) but the service pins traffic to `00023-qud` **by revision name**, so the new revision +was immediately `Retired` and nothing changed. Mutation 2 (17:21:30Z): `gcloud run services update-traffic +--to-revisions orca-cloud-auth-00018-4jc=100`. Lesson: the auth service's traffic block is name-pinned +(the deploy workflow does an explicit traffic switch), so a bare `services update` never reaches users. +Terraform still says `auth_max_instances = 2`; the next `deploy-auth-production.yml` run will revert this +unless the tfvars and the workflow's `AUTH_MAX_INSTANCES` are changed first. + +## Finding 13 (2026-09-04 17:19Z–18:10Z): **the auth outage is a database problem, not (only) a Cloud Run cap; `refresh_tokens` has 63 M rows and reuse-revokes scan whole families** + +Mutations this window (all online, no restarts, all by hand in project onorca-cloud): +1. 17:19Z `gcloud run services update orca-cloud-auth --max-instances 20` → new revision `00018-4jc`, but traffic is + pinned by revision name so it was `Retired`; 17:21:30Z `update-traffic --to-revisions 00018-4jc=100`. +2. Still 2 instances at 17:31Z: the SERVICE has its own `scaling.maxInstanceCount=2` in **manual scaling mode** + (`run.googleapis.com/maxScale: '2'` on service metadata, set by Terraform `infra/terraform-apps/auth.tf`), which + overrides the revision cap. `--scaling=auto` then `--max 20` at 17:31:45Z. Instances 2→20 by 17:38Z; 429s fell + 6,000/2 min → 60/2 min at 17:36Z and controls briefly reached 11. +3. Then latency, not capacity, became the wall: every refresh took 100+ s inside Postgres (desktop client timeout + is 30 s, `CLOUD_REQUEST_TIMEOUT_MS`), so 20 instances × 80 concurrency filled again with requests nobody was + waiting for, and 429s returned (~1,500/2 min from 17:40Z). +4. 17:27Z Cloud SQL disk 62 GB → 250 GB (IOPS ceiling 1,470 → ~7,500). 18:00Z `max_wal_size` 1.5 GB → 16 GB + (the checkpoint loop: `checkpoint starting: wal` every 45–60 s since 13:06Z). +5. 18:07Z `CREATE INDEX CONCURRENTLY refresh_tokens_family_unrevoked ON refresh_tokens(family_id) WHERE + revoked_at IS NULL` (an earlier attempt with `AND rotated_at IS NULL` was wrong for the revoke predicate; its + invalid remnant `refresh_tokens_family_live` was dropped). + +Evidence: `refresh_tokens` = 63.3 M live tuples, 16 GB table + 10 GB indexes; every refresh inserts a row and +nothing ever deletes (30-day TTL rows are never pruned). Query Insights 17:33–17:39Z: `UPDATE refresh_tokens SET +revoked_at = $1 WHERE family_id = $2 AND revoked_at IS NULL` = 21,000 s of execution per 6 min, ~90–120 k rows +updated per minute; io_time 15,000 s read; pg_stat_activity 180+ backends in `IO/DataFileRead` on that statement, +200 backends total for orca_auth (20 instances × pool max 10). `session-refresh-reuse-detected` audit events per +hour: ~100 all day → 8,805 (13Z), 15,511, 19,486, 24,897, 26,935 (17Z). Mechanism: a desktop's refresh times out +client-side at 30 s, the server had already rotated the token, the desktop retries with the same token, the +server calls that reuse and revokes the family (Bitmap scan on `refresh_tokens_family` + heap filter over every +row the family ever had), then the desktop retries the dead token again, and each retry re-runs the same +full-family scan (already-revoked families short-circuit nowhere). Reuse-detected 401 also **signs the user out** +on the desktop (`isOrcaCloudAuthFailure` → `clearCloudSessionIfUnchanged`), so every user who hit this during the +outage must sign in again. + +Durable fixes (orca-cloud PR in preparation on branch `auth-revoke-only-live-tokens`): `AUTH_MAX_INSTANCES` and +`auth_max_instances` → 20; Terraform disk 250 + `max_wal_size=16384`; the partial index in the schema; an +`already-revoked` short-circuit in `rotateRefreshToken` that skips the family UPDATE and the audit insert. Still +open after that: prune `refresh_tokens` (expired or revoked rows older than N days), a server-side statement +timeout shorter than the desktop's 30 s so the client and server agree on failure, and an alert on auth 429s. + +**19:11Z RESOLVED at the database layer.** `refresh_tokens_family_unrevoked` went valid at 19:11:17Z (build +18:07–19:11, two full table scans of 2.1 M blocks under load). Within 60 s: refresh latency 100 s → 0.1 s, auth 429 +→ 0, active orca_auth backends 200 → 2, checkpoints back on the 5-min timer (`checkpoint starting: time` at 18:35, +18:41, 19:00, 19:11). Director `/v1/assign` returning 200. Fleet controls 0 → 17 by 19:14Z. + +**Residual: mass sign-out.** 19:11–19:14Z: 3,857 refresh 401s from 3,829 distinct IPs, then near zero. Every one is +a desktop whose family was revoked by reuse-detection during the outage; the desktop clears its cloud session on +401 (`clearCloudSessionIfUnchanged`) and stops retrying. Those users must sign in again before the relay sees +them. Fresh `/session` sign-ins: 1, 5, 3 per minute at 19:10–19:12. Recovery of controls is now paced by users +signing in, not by infrastructure. Total `session-refresh-reuse-detected` events 13:00–19:00Z ≈ 100k, against a +~100/hour baseline. +**Affected-user count (19:22Z, from `refresh_tokens`):** 23,318 live token families revoked in the window, +**21,605 distinct users**. Only ~3,800 desktops had seen their 401 by 19:15Z; the rest were closed or asleep +and will find themselves signed out on next launch, so sign-ins will trickle for days. + +**Desktop UX finding (owner's own Mac, 19:22Z):** a revoked desktop keeps showing the account card as +"Connected" and the pairing pane as "Orca Relay: Unavailable" / `relay_control_not_active` indefinitely; the +local trace writes no relay events. Only quit + relaunch surfaced the sign-out prompt, after which sign-in → +relay-token → `/v1/assign` 200 (0.15 s) → working pairing, all within 10 s. Follow-ups: the relay coordinator's +401 path should flip the account card to reconnect-required immediately, and the pairing error should say "Sign +in again to use Relay" when the cause is an auth failure. Announcement wording: "If Relay shows Unavailable, quit +and reopen Orca, then sign in when prompted." + +orca-cloud PR #474 (branch `auth-revoke-only-live-tokens`): caps → 20, disk 250 / max_wal_size 16384 in +Terraform, partial index in the schema, `already-revoked` short-circuit. Do not deploy auth to any environment +with a large `refresh_tokens` before building the index concurrently there. + +**Wave 1 of the roadmap (2026-09-04 21:35Z onward):** five Opus agents in isolated worktrees: 3.1 grace window +(orca-cloud), 4.1+2.3 relay locks + pool timeout, 3.2+4.3 desktop refresh/jitter, 5.1+5.4 observability, +2.1 private IP (plan only, both repos). First back: stablyai/orca PR #18717 (crash alert + dashboard). Its key +finding: cell exits log to `cos_system` with uppercase `jsonPayload.MESSAGE` and `SYSLOG_IDENTIFIER=docker`, +so every earlier `jsonPayload.message:"container die"` count in this doc that read 0 was querying the wrong +field. Verified: 87 exits 12–13Z on the agent's filter, 0 in the last 6 h. Monitor dry-run 33922255205 +dispatched 21:41Z as the Roll 1 gate. +Dry-run 33922255205 froze at 21:46Z on `signal_missing cloud_sql.backends`. Cause: Cloud Monitoring published +no `num_backends` point for the auth instance between 21:40 and 21:46 (every other minute of the last 100 has +one; measured directly via the timeSeries API). A Google-side publish gap, not a database or monitor defect; +the monitor's freeze-on-missing rule is correct. The 12–13Z monitor failures were a different cause (active +probes reading 0 during the crash cascade). Re-dispatched at 21:50Z. +Dry-run #2 (33922844671) froze at 21:52:21Z on `auth.health observed 0` — verdict read from the state.json +artifact, not the log (the log only prints checkpoints). Auth served `/health` 200 continuously, including the +21:52:05 probe. Cause: the probe requires `/health` AND `/ready` on the first attempt; auth has no `/ready` +(404 by design), so every auth sample takes the forced 11 s retry, and on the third sample the retry fetch threw +at the network layer on the runner (no request reached Cloud Run) and `check()` recorded the exception as +health=false. Neither freeze was fleet health. Fix delegated (relay-ops: a thrown fetch is not a reading; auth +does not require `/ready`). **Sequencing constraint for Roll 1:** monitor evidence must be < 5 min old at +canary dispatch, so the owner's go must precede the dry-run, and a green dry-run must be followed by the +canary dispatch immediately. + +stablyai/orca PR #18719 (3.2 + 4.3, desktop): the replay engine was not the refresh function but +`RelayAuthCoordinator.scheduleRetry`, since `shouldRetryRelayConnectionError` treats any non-HTTP error +(including a refresh `TimeoutError`) as retryable and re-reads the same stored token on backoff. Fix: refresh +gets one 60 s attempt; an ambiguous failure (no status line) records the token and blocks re-sending it for +30 s (bounded, not permanent); definitive 5xx gets exactly one retry after re-reading the store; a 401 on an +ambiguously-attempted token logs `orca_cloud_refresh_possible_replay`. Lease renewal gets ±10 % full jitter +(base shrunk so the latest sample stays ≥ 90 s before expiry); server resets the full 55-min TTL on any rebind +(`host-session-registry.ts:736-743`) so early renewal is free. Verified the retry-path claim and both server +cites against main. + +2.1 private IP: orca-cloud PR #477 (foundation: servicenetworking API, /24 peering range 10.42.128.0, private +network on the instance, `prevent_destroy`; real production plan 3 add / 1 in-place change, staging unchanged) +and stablyai/orca PR #18720 (relay: `relay_cloud_sql_private_ip` variable, conditional `--private-ip` in the +cell startup template; default false renders byte-identical to main). Findings that change the plan: Google +states the private-IP change **restarts the instance** with no in-place path, and it is a one-way door (cannot +disable private IP or remove the network link). The director uses the Cloud Run built-in connector, not the +relay VPC NAT, so it never consumed the exhausted ports and is out of scope. Disabling public IP later breaks +the local proxy workflow and the director. #18720 merges (inert); #477 held for owner decision. + +4.1 + 2.3 relay: stablyai/orca PR #18722. Premise correction: #18521 and #18606 had already bounded and +narrowed most of the fleet-wide lock before today; what remained were the sticky-refresh retry (all 23 rows → +the one pinned row), reservation reconciliation (23 → the 2 involved rows), a dead pool-default fallback, and +an absolute counter write (→ delta with capacity guard). Placement (`assignOnce`) deliberately keeps the +ordered inventory lock: least-loaded selection is fleet-wide and dynamic target-only locking previously caused +cross-cell cycles; converting it to optimistic snapshot + conditional delta is the remaining 55P03 floor and a +follow-up. Pool `statement_timeout` was already 5 s but hardcoded; now env-configurable, `57014` added to the +retryable set (it was terminal before), schema DDL on an untimed max:1 pool. Independently re-ran the new and +adjacent suites here against 55440: 66/66. Harness note: 55440 is not idempotent across full runs (2 +pre-existing failures on a second run); reset the schema between runs. Rollout: director first, watch +`orca_relay_postgres_transaction_exhausted` and `cellInventoryHoldMsP95` before cells. + +#18719 first CI run failed only on `windows-host-job.win32.test.ts` (EPERM on temp-dir cleanup), a Windows +PTY test the PR does not touch and which no other recent run failed on; rerun dispatched rather than waved. + +3.1 grace window: orca-cloud PR #478 merged (not yet deployed; deploy is an owner gate because the startup +schema apply adds a nullable column to `refresh_tokens` with a brief ACCESS EXCLUSIVE). Semantics: within +`ORCA_CLOUD_REFRESH_ROTATION_GRACE_MS` (60 s default, 300 s cap, 0 = off) a re-presented rotated token gets the +SAME successor refresh token + a fresh access token, no revoke, no audit, provided the successor is still the +live head. Third presentation / outside window / revoked family: unchanged (revoke + audit). Successor plaintext +is stored sealed (AES-256-GCM, key = HKDF of the predecessor token; the DB never holds the key). Cost stated +plainly: a stolen token replayed inside 60 s is served once instead of tripping detection; DB-read + stolen +predecessor recovers the successor offline until pruned. Rotation now runs in one transaction (proved by a +forced-INSERT-failure rollback test; the 8-way race alone did not kill the non-transactional mutant). Verified +locally 27/27 incl. the Postgres suite against 55440, and CI ran it on PG 16 and 17 (4/4 each, not skipped). +Deploy wiring: env is set by BOTH Terraform and the deploy workflow, with a test pinning all three sources to +one value. **Pre-existing bug surfaced:** the deploy script strips every env var it does not own, so the +Terraform-set `ORCA_CLOUD_REFRESH_TOKEN_TTL_DAYS` (from #476) silently reverts to the compiled default on each +release. Latent only because both defaults are 30. Follow-up: add it to `authEnvironment` + the workflow env. + +Monitor probe fix: stablyai/orca PR #18723. A thrown fetch (DNS/TCP/TLS/8 s abort) is now "no reading" and is +re-asked once after 1 s; only a second throw is `false`. A non-ok HTTP answer is still `false` with no extra +retry. `latencyMs` is the slowest answering round trip, never a sleep. `requiresReady` is per endpoint: auth +(no `/ready` by design) is judged on `/health` + latency; director and cells unchanged. No threshold or rule +touched; `auth.ready` had no consumer. 81/81 relay-ops tests and 9/9 evidence-script tests locally. The monitor +runs at `main` head, so once merged the next dry-run uses it. + +Applying #18717 (22:10Z): the cell-exit log metric `orca_relay_cell_process_exit` is created; the alert policy +raced descriptor propagation (404) and is being retried. **Not applied, deliberately:** the dashboard. Its +targeted plan drags in `google_logging_metric.relay_snapshot[*]`, and that plan is `32 to add, 21 to destroy`: +the Terraform source adds a `region` label to every runtime metric (`EXTRACT(jsonPayload.region)`) which the +live metrics do not have, and a label change on a log metric is a delete+create. Replacing 21 live metrics +resets their history and would blank the 14 existing relay alert policies during the swap. That is +pre-existing drift in the relay root (unapplied since the region work), not something #18717 introduced. It +needs its own reviewed apply in a quiet window, ideally with the runtime-metric replacement acknowledged as +intentional. Dashboard apply waits on that. + +**Wave 1 closed 22:20Z.** Merged: orca-cloud #478 (grace window); stablyai/orca #18717 (crash alert + +dashboard TF), #18719 (desktop no-replay + jitter), #18720 (private-IP flag, off), #18722 (relay per-cell +locks + pool timeout), #18723 (monitor probe fix). Applied to production: cell-exit log metric + alert policy. +Held for owner: orca-cloud #477 private IP (restart, one-way); the dashboard apply (behind the runtime-metric +label drift); the auth deploy carrying #478; Roll 1. Every wave-1 code change now sits on main un-deployed: +the next relay image build carries #18722 + #18723's monitor runs at main head already; the next auth deploy +carries #478. + +**Landing (2026-09-04 20:50Z–21:02Z, owner: "if you are confident the cloud changes are valid, you can land them"):** + +- Merged: orca-cloud #474, #475, #476; stablyai/orca #18693, #18694, #18698. Neither repo has branch + protection or environment reviewers; `verify` / `cloud-verify` green on main after each. +- Applied to production by targeted saved plans (each plan asserted create-only / exact-attribute before + apply, via `terraform show -json`): 4 relay resources (WAL-checkpoint log metric + 3 alert policies), 8 auth + resources (3 log metrics, propagation sleep, 4 alert policies), and the us-central1 NAT + (`enable_dynamic_port_allocation` false→true, ports 64..4096). Google's docs: switching to dynamic does not + break existing connections when max ≥ 1024 and max ≥ old min; only lowering max or reverting to static is + disruptive. asia-east2 NAT deliberately left for after a US soak. +- Not applied: the untargeted apps-root plan also carries 4 unrelated drifts (`ORCA_CLOUD_REFRESH_TOKEN_TTL_DAYS` + env on the auth service from #476, a skill log exclusion filter change, skill pressure threshold 16→8, an + artifacts bucket lifecycle rule) and fails on the 1Password Cloudflare data source locally. The foundation + root plans clean (disk 250 / max_wal_size already match). Those drifts belong to whoever runs the next full + apps apply in CI. +- `deploy-auth-production` on main 8034955 (run 33919143723) **succeeded 21:04Z**: serving revision + `orca-cloud-auth-00031-tox` at 100%, previous `00018-4jc`, cap 20, smoke passed on both URLs. First 15 min on + the new revision: 31×200 / 1×401 on `/refresh`, max latency 56 ms, no 5xx. The new + `refresh_token_prune_cursor` table exists, so the new schema applied. +- US NAT soak (21:01–21:06Z): 0 drops, 0 proxy dial errors, 0 cell exits, port_usage 11, sqlMax ~1.07 s. + Asia NAT then applied 21:05:28Z from the pre-verified saved plan (same three attributes). The deploy script strips env vars it does not own, so the Terraform + TTL var will not be on the new revision until the full apps apply lands; the auth code defaults to 30 d. +- Terraform locally needs `GOOGLE_OAUTH_ACCESS_TOKEN="$(gcloud auth print-access-token)"`; ADC is stale. + +**Alerting + NAT follow-ups (19:58Z, superseded by the landing block above):** + +- stablyai/orca PR #18693 (`relay-nat-ports-and-sql-alerts`): both relay NATs switch to dynamic port + allocation (64–4096 per VM); new relay-channel alerts for the Cloud SQL WAL checkpoint loop (log metric on + `checkpoint starting: wal`, > 3 per 5 min), Cloud SQL disk > 70%, and NAT `OUT_OF_RESOURCES` drops. No + existing workflow applies these resources; the PR body carries the targeted plan. +- orca-cloud PR #475 (`auth-observability-alerts`): log metrics + policies for auth refresh 401 (> 100 per 5 + min; Sep 3 baseline 20–80 per hour), 429 (> 20 per 5 min; baseline 0), 5xx (> 10 per 5 min), and Cloud Run + p99 latency > 10 s. Production routes to the relay Slack channel. +- Desktop stale auth-status fix: stablyai/orca PR #18694 (`desktop-cloud-session-revoked-status`). Main pushes + an auth-status-changed IPC when a 401 clears the session; panes re-fetch on mount; the pairing notice says + "Your Orca account session expired. Sign in again to use Orca Relay" and hides Retry. StrictMode regression + test verified red on the old guard. Does not help desktops already revoked today (session cleared before + this code); it fixes every future revocation. +- orca-cloud PR #476 (`auth-refresh-token-pruning`): batched `refresh_tokens` pruner as a scheduled Cloud Run + job (revoked rows kept 30 d, rotated rows 60 d against a 30 d TTL, 5k-row batches, 200 ms pauses, persisted + cursor, per-run budget) plus a 10 s `statement_timeout` on the auth request pool with schema DDL on an + untimed connection. Merges cleanly onto #474 and does not need its index (walks the primary key; + EXPLAIN-asserted no seq scan). CI ran the Postgres integration tests for real on PG 16 and 17. Ships + `auth_token_pruner_enabled = false` in both environments: enabling needs an image digest from a build that + contains the new entrypoint. Operating rules once enabled: monitor the run summary's `stopReason` and + `deletedRows`, not the exit code (a run that only ever times out exits 0); ~48 M rows drain in ~10 days at + 200k/hour; deleting them leaves dead tuples, so the 16 GB is not reclaimed without a separate VACUUM FULL or + pg_repack pass, which is its own change. +- Phone-side copy when the desktop is signed out: stablyai/orca PR #18698 (`phone-desktop-signed-out-reason`). + Real path traced: the director resolves the phone to the host's last cell (durable assignment row), and the + cell's `acceptClient` rejects with 4404. The only additive slot every shipped peer tolerates is the WebSocket + close *reason* (relay-hello and resolve schemas are zod strict; a new close code drops old phones off the + host-offline cadence). Desktop closes its control with reason `signed-out` only when the cloud session is gone + (null context after a 401, or explicit sign-out); quit and relaunch stay reasonless. Cell remembers it per + host for the dormant-assignment TTL, forgets on re-auth, and echoes it as the 4404 close reason; phone + renders "Desktop signed out — sign in to Orca on your desktop to reconnect" with the same retry cadence. + Old×new matrix in the PR body; nothing changes for any old peer. Merges cleanly with #18694. + +## What actually blocks the roll now (12:58Z summary for the owner) + +0. **Cloud NAT ports** (Finding 11, found 12:55Z): every us-central1 cell reaches Cloud SQL's public IP + through a NAT with the default 64 ports/VM; port_usage pinned at 64 and 1,514 dropped SYNs to + Cloud SQL:3307 in one 4-min window. This is the 2 s connect stall that kills old-image cells and is + still active after the disk loop broke. Fix: `min_ports_per_vm = 1024` (or dynamic allocation) on + `google_compute_router_nat.relay_gce` in `cloud/infra/terraform/relay-gce-foundation.tf`, targeted + apply; durable fix is a private IP on the Cloud SQL instance. Online, no VM restart. +1. **Cloud SQL disk** (Finding 10): 49 GB PD-SSD saturated since 11:58Z, checkpoint loop, fleet-wide + 4–6 s stalls every ~45 s. Fix: bigger disk and/or `max_wal_size`. Owner: `stablyai/orca-cloud` + `infra/terraform-foundation/database.tf` `google_sql_database_instance.auth` (no `disk_size`, + `disk_autoresize`, or `database_flags` set today, so Terraform is at defaults: 10 GB initial, autoresize + grew it to 49 GB). Add `disk_size = 200` (+ `disk_autoresize = true`) and optionally + `database_flags { name = "max_wal_size" value = "4096" }`; production tfvars are + `infra/terraform-foundation/environments/production.tfvars`; applied by `deploy-production.yml` in + that repo. Online, no restart for disk; `max_wal_size` is also a non-restart flag. Note Terraform + `disk_size` below the live 49 GB would be a destructive shrink, so 200 is safe and 49 is the floor. **This is now the first thing to do**; nothing else can pass a + 15-min gate while it persists, and it is also what is killing the old-image cells several times an hour. +2. **Old cell image** (Finding 6): dies on every stall. Fixed by rolling 519f4914 (canary inputs ready). +3. **Gate policy**: `directorErrors: 0` and per-cell health probes freeze on any single stall. Recalibrate + after 1 and 2, or bypass by hand for the canary. + +## Plan agreed with the owner (2026-09-04 ~06:45Z), in execution order + +Owner: "feel free to improve operations to make things more effective ... continue driving everything e2e +until this process is complete." Owner has had multi-day experiences with cell rolls and does not want a +9-hour sequential roll. + +1. **Lock-removal PR** (root cause). *Status 08:55Z: pushed as branch `relay-single-row-reservation` + (2 commits). Opus adversarial review found one real defect: `acquireActivity` moving a client-chosen + activity id across cells locked the old cell's row before the new one, cycling with placement's + ascending inventory lock (reviewer reproduced it as paired 55P03s on real Postgres; no 40P01 because + lock_timeout == deadlock_timeout == 1 s). Fixed with `lockCellRows` (ordered, 500 ms bound); census now + fails on any inline `relay_cells FOR UPDATE` outside the named helpers. Three-cell Postgres test moves + an activity high->low while the target row is held; 5/5 revert-mutants fail it. 480 SQLite tests + + tsc green. Also fixed a pre-existing test leak (`relay_cell_connection_snapshots`) that made + `assignment-control-supersession-postgres` fail on reruns. Reviewer re-verified 65569be3de: cycle + repro completes in 7 ms (was 1022 ms + paired 55P03); no remaining out-of-order pair in the store; + flagged two evasions in the new census guard, closed in the third commit (whole-statement scan, + covers query() too, mutation-checked with both evasions). Headroom Postgres test's one failure is + pre-existing on main (verified by swapping in main's store).* Make `activateControl` superseded-control cleanup, `acquireActivity` + existing-lease branch, and `changeActivity` use the existing single-row + `adjustCellReservationAtomically` instead of the 23-row `lockCellInventory`. Keep the global lock only + for placement (`resolve`/assignment) and sweeps. Real-Postgres contention test on port 55440. +2. **Faster same-cap rollout workflow.** (a) paced drain instead of `graceMs: 0` so a cell's ~800 hosts + re-dial over minutes, not one second (director cap is 5 x 80 = 400 in-flight); (b) cells in a batch run + in parallel once drains are paced; (c) post-canary batches use a short freshness check instead of a new + 15-min dry-run, since the in-job safety recheck already runs before each drain; (d) job timeout > 75 min. + Target: 22 cells in ~6 batches x ~25 min. +3. **Build image** with (1) merged, then one roll of the fleet with (2). Asia cells c27/c28/c29 first. +4. Re-tighten the monitor retries bar; recalibrate the Terraform exhausted alert. +5. Consider deleting the 55-min control lease rebind entirely (no recorded reason; liveness is the 75 s + watchdog + 90 s activity lease). Separate PR after (1) so its effect is measurable. + +## Faster same-cap rollout: design (step 2 of the plan), from reading the real limits + +What actually bounds parallelism today (measured on the c7 canary, run 33843071283): + +| step | c7 duration | bound by | +|---|---|---| +| prechecks (recheck, backend init, resolve, verify) | 43 s | none | +| isolate + drain + transition wait | 7 min | drain is `graceMs: 0`; `verify-relay-capacity-transition --activity restart-safe` polls until leases drain | +| Terraform template + MIG recreate + wait-until stable | 8 min | GCE recreate; per cell, independent | +| verify new incarnation + trust proof + restore | 1.5 min | none | + +Real constraints: (1) the director is 5 x 80 = 400 in-flight `/v1/assign`; a `graceMs: 0` drain of ~800 +hosts pins it at cap for ~2 min (observed 79.75/84.75 p99). (2) `production-cloud-sql-rollout` lease and +workflow concurrency group serialise the whole run, by design, and the per-cell job shares it via +`holder-key`. Nothing else forbids parallel cells. + +Changes, smallest first: +1. **Paced drain.** `HostSessionRegistry.drain(graceMs)` already sends `drain {graceMs}` and closes each + session after `graceMs`, but the desktop's `handleDrain` re-dials immediately regardless of graceMs + (`relay-origin-pool.ts:150-162`), so graceMs only delays the *close*, not the stampede. Fix on the + cell: stagger the drain *send* across sessions over a window (e.g. 800 sessions over 120 s = ~7/s), + which needs no desktop change and works for every desktop version in the field. New admin body field + `spreadMs` (optional, default 0 keeps today's behaviour); canary script passes `spreadMs: 120000`. + Requires the cell to be on an image with the change, so it applies to batches after the first + post-lock-fix roll, not to this one. +2. **Parallel cells in a batch.** In `cloud-deploy-relay-production-same-cap.yml` make `cell_2..cell_4` + `needs: [gate]` instead of chaining, gated on the same evidence (drop the `+75 min x wave-index` + allowance, it exists only because of chaining). Each job already takes the rollout lease with the + run's `holder-key`, so they re-enter it rather than fail. With paced drains, 4 cells x ~800 hosts + over 120 s is ~27 dials/s, well under the director cap. Raise `timeout-minutes` to 90. +3. **Post-canary batches skip the 15-min dry-run.** The in-job "Recheck aggregate SQL, pool, + reconnect, migration, and selector safety" step (`pnpm incident:relay-preflight`) already runs a + live one-shot check before each drain. For `batch-apply` with a sealed `canary-run-id` from the + same commit, accept a dry-run of any age (the canary's) plus that live recheck; keep the 15-min + requirement for `canary-apply`. Change lands in `relay-monitor-evidence.mjs verify-authority` + + `relay-production-same-cap-wave.mjs` + their node:test suites. + +**Correction after reading the cell job (07:35Z):** (2) parallel cells is not a flag flip. Each cell job +asserts the exact selector generation `expected + 2 x wave-index` and exact memberships derived from +predecessors having completed (`ISOLATED_*`/`RESTORED_*` in the job, `applyExactAdmissionSelector` +compare-and-swap), and all cells share one Terraform state lock. Making that concurrent means a batch-level +isolate/restore in the gate and a rewrite of the 650-line job's expectations. That is the multi-day trap +the owner described. Deferred. + +What is cheap and removes most of the wall-clock: (3). The per-batch 15-min dry-run costs 15 min each +*and* fails ~50% of the time on old-image crashes, which is where hours go. Implement: `batch-apply` with a +verified canary authority accepts a passed dry-run up to 6 h old and may re-use one already consumed +(the consumed-marker check exists to stop replaying stale evidence; the canary binding plus the in-job +live preflight at drain time replace it). Files: `relay-monitor-evidence.mjs` (`--after-canary`), +`incident-live-preflight-cli.ts` (same flag), the same-cap workflow + job, and both test suites. +Revised expectation: 22 cells = 6 sequential batches x ~70 min = ~7 h wall-clock but *unattended-safe* +and with one dry-run total, versus today's 6 dry-runs at ~50% each. (1) paced drain rides the lock-fix +image. + +## Recommended next steps (superseded by the plan above; kept for history) + +1. Resolve the gate decision above, then: monitor dry-run -> c7 `canary-apply` only -> verify -> stop. + Each rolled cell leaves the Finding 6 crash class. +2. Merge #18565; publish; a later same-cap roll carries it to cells. +3. Remove the global inventory lock from per-connection paths (`acquireActivity` existing-lease + branch, `activateControl` superseded-control cleanup, `changeActivity`) by using the existing + `adjustCellReservationAtomically` single-row update. Own PR, after the roll. +4. Recalibrate the Terraform alert `relay_postgres_retry_exhausted` to 300/300 s (observability root). +5. Whether to raise `relayPostgresRetries` is a human call; the data is in Finding 5. + +## Canary blast radius (read before dispatching c7) + +- What `canary-apply` does to c7, in order: isolate (selector -> migration-only, no new + assignments), `/v1/admin/drain graceMs:0` (every control on c7 re-dials the director and is + reassigned), Terraform template + MIG update to the target image, wait stable, verify new + incarnation + exact digest + protocol, prove per-host trust, restore c7 to general admission. + On any failure c7 is left isolated (migration-only) with rehome disabled; nothing else is touched. +- c7 at 05:20Z: 788 controls, 5 splices, 800 connections. So ~790 desktops re-dial once. The fleet + already absorbs this exact event 201 times / 48 h uncontrolled (Finding 6); the controlled version + isolates first, so no new assignment lands on c7 mid-roll. Expect a director concurrency blip, not + a freeze-class one (six cells at once gave 85; one cell should stay well under 64). +- Precedent: the identical workflow (pre-move, in orca-cloud) ran 9 successful `apply` canaries and + batches on 2026-08-27 (last: c20 -> 5aedbca5). Its failures that day all stopped at the read-only + "Recheck aggregate SQL..." or "Require durable rehome disabled" step, before `MUTATION_STARTED`. + The moved copy in this repo has one run: the read-only `verify` of c7 (passed, including WIF auth). +- c7 side note: MIG autoheal recreated the c7 instance four times on 2026-09-01 08:02-08:42 PDT + at ~13 min spacing. Same crash class as Finding 6 (health check failing during restart loops). + +### Canary observed effect (c7 drain, 2026-09-04 06:10Z) + +- c7 807 controls -> 0 between 06:08:52Z and 06:10:52Z. Director `/v1/assign`: 200s 32 (06:09) -> 2628 (06:10) + -> 340 (06:11); 5xx 1969 (06:10) -> 31 (06:11). Director max-concurrency p99 7.9 -> 79.75 (06:10) -> 84.75 + (06:11), i.e. at the Cloud Run cap of 80 for ~2 min. My pre-dispatch estimate ("well under 64") was wrong. +- Confounder: c10 (us-central1, instance 2803000337345335589) crashed 06:09:56Z on the old-image class + (Node.js banner + container die), so ~1,600 hosts re-dialed in the same minute, not ~800. Coincidental; + the fleet has one of these every ~15 min. +- Recovery: 06:13 903 / 06:14 1471 assign 200s from 640 distinct desktop IPs; 503s 78 -> 183 -> 29/min. + No cell crash 06:12–06:16Z. Drain step passed ~06:16Z; template/MIG apply started. +- 06:16:03–06:17:08Z, during c7's template apply (not its drain): c27 (x4) and c29 (x3) crash-looped on the + old-image pg-pool connect timeout in `beginProof`, both MIGs autoheal-recreated (c27's second recreate in + 40 min). Fleet 23 -> 21 reporting cells, controls 13286 -> 12462, assign 503s 1000/min at 06:17, director + concurrency p99 74.8. Cloud SQL CPU 0.70 max, backends 174 max (bar 250). Same multi-cell pattern occurred + at 01:31Z (4 cells) and 04:47Z (5 cells) with nothing rolling; the c7 drain's SQL load 6 min earlier may + have nudged the pool timeouts but the class is pre-existing. c7 MIG RECREATING onto new template + `…20260904061618…` = the expected image swap. +- 06:20Z: 849 assign 503s. Closes 06:19:30–06:21: 162x1006 age<5min (hosts bouncing off the recreating + c27/c29), 73x4408 + 53x1006 in the 50-min age bin (Finding 3 rotation cohort). Not roll-caused. + c7 MIG `recreating=1` on the new template since 06:16:18Z; c27 and c29 MIGs also RECREATING (autoheal). +- 06:23:16Z c7 instance restarted in place (MIG RECREATE keeps name/id relay-c7-bwjc / 4545742188814054238), + pulled `relay@sha256:85bf6799…` 06:23:37Z, listening + readiness true 06:23:42Z. Apply step passed 06:24Z; + verify step running. Isolate -> ready on new image took ~14 min end to end. +- Post-restore c7 on new image (06:25:42–06:26:42Z): controls 143 -> 273 -> 377 refilling, sqlQueries + ~1,500/30 s, `sqlLatencyMsMax` 518 -> 1003 -> 1155 ms, still 55P03 `cell-inventory` retries. So the new + image alone does not remove lock waits; the request-path 500 ms cap from #18521 applies to the director's + paths, and cell-side `acquireActivity`/`activateControl` still ride the global lock (step 3 in next steps). + Watch: does c7's sqlLatencyMsMax settle below the old 1.0–1.2 s pin once refill finishes, and does c7 stop + appearing in `container die` (the real win: guardSessionTask). +- 08:25Z (2 h after restore): c7 817 controls, 0 crashes since 06:25Z. Fleet crashes last 2 h: c27 x6, + c28 x5, all old-image Asia cells. The new image stops the crash class as predicted; it does not move + lock latency (c7 sqlLatencyMsMax 1005 ms), which is #18606's job. +- Implication for the batch phase: every drain will push director concurrency past the monitor's 64 bar + for ~1-2 min. The batch job rechecks safety *before* it drains (read-only step), so that is fine per wave, + but never run a monitor dry-run concurrently with a wave, and prefer batches of 2 over 4 until the fleet + is on the new image and the crash class is gone. + +## Post-merge dispatch plan for #18606 (image -> director -> cells) + +1. `gh workflow run cloud-publish-relay-production.yml --ref main -f mode=publish` (after the squash lands + on main). Resolve the digest by tag, never by parsing the log (it mixes relay and fence-broker digests): + `gcloud artifacts docker images describe us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay:sha- --format='value(image_summary.digest)'`. +2. Director: `gh workflow run cloud-deploy-relay-production-director.yml --ref main -f image-digest= + -f regional-placement-mode=preserve -f prune-incompatible-revisions=false -f expected-rehome-generation=12 + -f bootstrap-runtime-identity=false -f predecessor-image-digest=` + (no monitor evidence needed; requires rehome disabled at gen 12, which it is). Last run 33826514754 used + the same shape. Watch director `orca_relay_postgres_transaction_retry` per minute before/after. +3. Cells: same-cap `verify` c7 with target=, rollback=85bf6799; fresh dry-run; `canary-apply` c7; + then batches (3 per batch, Asia c27/c29/c28 first). Each batch: new dry-run unless the batch-reuse + change (design section above) has shipped. + +## Finding 8 (2026-09-04 08:40Z): ten-cell crash cascade during the director deploy, not caused by it + +Timeline: candidate revision 00570-siv created 08:38:39Z, first log 08:39:20Z; traffic still 100% on +00565-fes through 08:43 (assign logs by revision). Cell crashes: c28 (5031087219978409220) looped 08:37:55– +08:40:07 (9x), then at 08:40:20–08:40:45Z **ten** instances died within 25 s (c10 2803…, 5110…, 532…, 5464…, +7536…, 7726…, 8671…, 8928…, 8966…). All old-image `beginProof` pg-pool timeouts. Fleet controls 13,423 -> +6,157 by 08:43; assign 503s 3,912 (08:42) and 4,624 (08:43) per minute, director concurrency 85 (cap 80), +Cloud Run autoscaled 5 -> 10 instances, Cloud SQL CPU 0.55 -> 0.99. Deploy finished cleanly at 08:45Z with +the new director taking the tail of the storm; by 08:46 503s were ~30/15 s, controls 7,913 and rising, +director lock retries 29/min (vs 105–157/min pre-deploy) and exhausted 2/min (vs 65/min at 08:36). +Same class as 01:31Z (4 cells) and 04:47Z (5 cells) today; this was the biggest. c7, on the new image +since 06:25Z, did not crash. What triggered the pool timeouts fleet-wide at 08:40 is not established; Cloud +SQL CPU was 0.78–0.88 in the minutes before, the highest of the day, so the cells' 2 s connect timeout is +the plausible tipping point under a busy database. Every cell still on 5aedbca5 remains exposed to this. + +## Finding 9 (2026-09-04 08:56Z): #18606 on the director cut lock retries ~10x + +`orca_relay_postgres_retries` per 5 min, director only: 08:21–08:41 windows 419–689 (old image, incl. the +crash storm); 08:46/08:51/08:56 (new image 519f4914, refilling ~7k hosts): **61 / 69 / 54**. Exhausted: +104–178 -> **11 / 14 / 12**. Inventory hold p95 ~200 ms, max 255 ms, ~366 holds/min. Cells (still old +image) 17–44 -> 0–3, because the director no longer holds the 23-row lock on their behalf. This is the +first direct measurement of the root-cause fix under real load. Cloud SQL CPU peaked 0.99 during the +cascade and is decaying (0.86 at 08:55); the monitor freezes above 0.80, so no dry-run until it clears. + +Fourth cascade 09:00:12–09:00:18Z: c23, c8, c16, c26, c22 (five cells, 11 container-die events in 6 s, +all `5aedbca5`, exitCode 1, Node banner, pg-pool `client closed the connection` burst right before). Cloud +SQL CPU 0.84 -> 0.78 in the preceding minutes, director concurrency 18–22 (idle), so this one fired +*without* a database or director spike. Fleet had just recovered to 13,015. Cadence today: 01:31 (4), +04:47 (5), 08:40 (10), 09:00 (5), 09:31 (c13, c23), 09:34 (c23 again, c14, c20, c9; c14/c20 crash-looping), +09:39 (c21, c24), 09:55 (c16, c8), 09:59 (c20), 10:05 (c8, c20), 10:19 (c16 stalled, no crash), then a 58-min +lull, 11:04 (c9; c28 died 13x in 4 min, autoheal recreate 11:09Z, its 3rd recreate today), 11:17 (c10, c28 +again, c22, c23, c14 x9 looping; 23 dies in ~90 s; fleet 13.3k -> 10.8k), 11:31 (c14, c23, c25, c15, c24, c19), 11:34 (c20, c26, c29 x4, c14, c27 x3, c25; fleet 13.1k -> 10.3k). +Three cascades in 17 min. 11:38–11:45 c27 crash-looped 17x and c28 4x (Asia cells), c29 recreating. +11:59 (c21, c9, c10, c23), 12:02 (c19; 4,109 assign 503s that minute, mostly hosts bouncing off the +recreating cells, code 1006 age<5min x217), 12:09–12:12 (c19, c27 x6, c28 x5, c13, c22, c15, c26, c14; +8 cells, c27/c28 recreating again). Cloud SQL CPU 0.62–0.85 through it. 12:20 (six more cells). Cascade +cadence since 11:00 is now ~every 8 min; the waiter has held correctly the whole time and there has been +no dispatchable window. Loop continues unattended; findings stop logging each cascade from here unless the +class changes. Every cell that has died today is on 5aedbca5; c7 (85bf6799, +5.5 h) has not. Cell dies per hour today: +01Z 5, 02Z 7, 03Z 4, 04Z 7, 05Z 4, 06Z 9, 07Z 11, 08Z 30, 09Z 26, 10Z 2, 11Z 68+ (to 11:42). +Director concurrency pinned at 85 for 09:32–09:33; 503s 4,141 and 4,396 per minute. 09:39: c21, c24 +(2,870 503s). Crashes per instance 08:10–09:40Z: c28 x14, c27 x5, c23 x5, c22 x4, c14 x4, c13/c20 x3, +then c26/c9/c24/c16/c8 x2. Mean gap between cascades since 08:40: ~12 min. Every 15-min gate attempt +now has well under even odds; the c7-style canary that ends this needs a gate it can pass. The old image is now cascading roughly hourly regardless of load; the +only cell on a fixed image (c7) has 0 crashes in 2.5 h across all four. + +Director 500s: 4 in the 09:00 window, all 2.0 s latency on `/v1/assign` or `/v1/resolve` = pg-pool connect +timeout surfacing as a 500. Pre-existing (Sep 3: 03h/08h/16h one each, same 2.0 s shape; 06:09Z today on +the old image during the c7 drain). The monitor's `directorErrors: 0` bar freezes on any of these, so a +dry-run needs a 15-min window with none; at ~1 per cascade that is a real but modest constraint. + +**Gate observation (09:26Z):** `directorErrors: 0` counts every non-503 5xx on the director, including +the monitor's own admin calls. The director on 519f4914 still sees an occasional 2.0 s pg-pool connect +timeout (~1 per 20 min under today's Cloud SQL load), which surfaces as a 500 on whichever request drew +it. Two consecutive dry-runs (#7, #8) froze on exactly this: one, isolated, 2 s 500. That bar was set for +"unexpected director 5xx"; a single connect timeout that the client retries is not an incident. Candidate +recalibration (own PR, not done): `directorErrors` 0 -> 2 per 5 min, or exclude the monitor's own +user-agent. Not changing it unasked; noting that at ~3 per hour the 15-min gate passes ~1 in 2 attempts. + +**Did the director deploy make cells crash more? (checked 09:45Z)** Cell `container die` per 30 min: +06:00 9, 07:00 2, 07:30 9, **08:30 30** (director candidate 08:38, traffic 08:43–08:45; the 10-cell burst +was 08:40:20, before the move), 09:00 11, 09:30 12. Per hour today 05:4 06:9 07:11 08:30 09:23 vs Sep 3 +same hours 2/7/8. So today is 2–3x worse than yesterday and was rising before the deploy; after the deploy +it is ~11–12 per 30 min, in line with 06:00–07:30. Cloud SQL backends (~230 max) and new connections +(~5k/30 min) are flat across the deploy. Latest crash (c21 09:39:11) is `Connection terminated due to +connection timeout` with cause `Connection terminated unexpectedly` in `verifyCellAssignment` <- +`beginProof`, the same unhandled path. Conclusion: no evidence the deploy worsened it; the old image's +crash rate simply climbed all day. Director lock retries stayed ~10x lower after the deploy. + +**Checkpoint-phase check (10:00Z, negative result):** Postgres checkpoints complete every 5 min at ~:07. +Cell crashes bucketed by phase within that 5-min cycle show a mild :00–:29 s cluster today (22 of 103) +that is absent on Sep 3 (7 of 114), so checkpoints are not the trigger. Disk write bytes in cascade +minutes are at or below the median except 09:00. Cloud SQL memory 0.47, transaction rate flat. The +09:55 stall (11 director + 4 cell pg-connect timeouts in the same 4 s) came with `could not obtain lock +on row in relation "relay_cells"` from a NOWAIT sweep at 09:55:36, i.e. someone was holding the full +inventory at that moment. On the new director that can only be placement or a sweep; on the old cells it +is still every rebind. What stalls *connections* (not locks) for 2 s fleet-wide remains unexplained; +Cloud SQL is `db-custom-4-15360` REGIONAL PD_SSD 49 GB at 0.5–0.75 CPU when it happens. + +**Stall census (10:01Z):** 33 pg-connect-timeout stall events today (clusters of timeouts < 20 s apart). +Before 08:35 they were 1–9 timeouts each and 10–60 min apart; from 08:35 the big ones are 16, 22, 21, +17 timeouts and 5–30 min apart. No second-of-minute phase (start seconds spread across all buckets), so +not a fixed timer. Cloud SQL backends by state at 09:55: active peaked 42 at 09:52, idle-in-transaction +≤ 10, nothing near the 400 ceiling; memory 0.47; disk normal. Each stall is a few seconds where *new* +connections to Cloud SQL (via the auth proxy socket) time out at the 2 s `connectionTimeoutMillis`, +hitting every process that happens to need a fresh pool connection in that window. Old-image cells die +on it (unhandled), new-image director logs a 2 s 500 and continues. Root cause of the stall itself is +outside the relay code (Cloud SQL proxy or instance); not chased further here. + +## Finding 10 (2026-09-04 12:40Z): Cloud SQL disk write saturation since 11:58Z is driving the stalls + +`orca-cloud-auth-db` is `db-custom-4-15360` on a **49 GB PD-SSD** (81% used). PD-SSD performance scales +with size: 49 GB gives roughly 1,470 write IOPS and ~23 MB/s write throughput. Measured: + +| | before 11:58Z | 11:59Z onward | +|---|---|---| +| disk write MB/s | 4–6 | **30–50** (over the ~23 MB/s cap) | +| disk write IOPS | 500–800 | 800–1,475 (at the ~1,470 cap in 11:59, 12:15, 12:24, 12:34) | +| checkpoint `sync=` | 0.07–0.2 s (Sep 3 max 0.65 s, 290 checkpoints) | 2–20 s; 27 of 39 checkpoints in 12Z were >= 2 s | +| checkpoints per hour | 12 (timed, every 5 min) | 39 (WAL-triggered, every ~45 s; `write=` fell from 270 s to 30 s) | +| Cloud SQL CPU / memory | 0.5–0.8 / 0.47 | same (not the bottleneck) | + +Every 4 s+ fleet-wide SQL stall since 11:04 (11:04, 11:17, 11:31, 11:34, 12:09, 12:10, 12:18, 12:20, +12:30) sits inside a slow checkpoint `sync` window; the 12:30:49 checkpoint synced 5.88 s (longest file +5.47 s), matching the 12:30:02–41 stall. During fsync the WAL writer stalls and every session waits, which +is why the stall hit all 23 cells and the director at once regardless of the relay lock changes. The +old-image cells then die on the pool timeout; the new image survives. What raised write volume ~8x at +11:58Z is not established (autovacuum ran on every relay table 11:55–11:57 and checkpoints are being +forced by WAL volume, so a write amplifier inside Postgres is the leading candidate; relay transaction +rate and Cloud SQL network bytes were flat). This is the first cause found today that is *upstream* of +the relay code and it explains the afternoon acceleration (11Z 68 dies, 12Z 47 by 12:34). + +Corrections after digging (12:45Z): relay query volume, renewals, reconnects, and assignments per 5 min +were **flat** across 11:58 (sqlQ ~330k, renewals ~115k), so the relay did not start writing more. WAL +recycling per checkpoint went 7 -> 10–11 files (16 MB each) at 45 s intervals, i.e. WAL output rose from +~0.4 MB/s to ~4 MB/s while data-file writes rose to 30–50 MB/s; checkpoints switched from `time` to `wal` +triggered at 11:58:24. No Postgres slow-statement or "checkpoints too frequently" lines. This is +write amplification inside Postgres (full-page writes after each of the now-frequent checkpoints on +hot pages, plus autovacuum on every relay table each minute) on a disk too small for its IOPS ceiling, +not new relay load. Instance label `managed_by=terraform`, created 2026-07-09; the instance resource is +**not** in `cloud/infra/terraform` (only the database, user, and secret are, via +`local.relay_database_instance_name`), so it lives in the other Terraform root (orca-cloud, per +[[orca-cloud-terraform-split-findings]]). `storageAutoResize=true` with limit 0, so Cloud SQL will grow +the disk only when it fills, not when IOPS saturate; disk is 81% full. + +Onset precisely: the 11:55:37 `time` checkpoint wrote 67,258 buffers (10.5% of shared_buffers, the +day's largest) over 163 s and completed 11:58:24. Every checkpoint since has been `wal`-triggered at +~45 s spacing (`max_wal_size` reached), each writing 13–20k buffers with 9–11 WAL files recycled. This is a +self-sustaining loop: a checkpoint completes -> every subsequent write to a hot page emits a full-page +image into WAL -> WAL fills `max_wal_size` in ~45 s -> next checkpoint -> repeat. The relay's hot rows +(`relay_cells`, `relay_assignments`, activity leases, cell runtime) are updated tens of thousands of +times a minute, so full-page-write amplification is large. Before 11:58 the 5-min timed checkpoints kept +WAL well under the limit; a one-off larger checkpoint tipped it over and the disk's write ceiling keeps +it there. Query Insights: io_time +30% in the 12:00 bucket, lock_time flat. + +**Owning workflow / mitigation (not applied):** raise the Cloud SQL data disk (PD-SSD IOPS and MB/s scale +linearly with GB; 49 -> 200 GB roughly quadruples the ceiling, online, no restart) in the Terraform root +that owns `google_sql_database_instance` for `orca-cloud-auth-db`, applied through that root's workflow. +A second, flag-level lever is raising `max_wal_size` (default 1 GB) so timed checkpoints resume; that is +also a Cloud SQL instance setting in the owning Terraform root. Per the standing rule, not applied from +this session. Until then the fleet-wide 4–6 s stalls recur on +every slow checkpoint sync, the old-image cells die on each one, and no 15-min gate window will exist. + +## Finding 11 (2026-09-04 12:55Z): **Cloud NAT port exhaustion** on the us-central1 cells is the second stall class + +`google_compute_router_nat.relay_gce` (us-central1, `AUTO_ONLY` IPs, no `min_ports_per_vm`, no dynamic +port allocation, i.e. the default **64 ports per VM**). `router.googleapis.com/nat/port_usage` per VM +hit **64 = the cap** in exactly the minutes the cells' Cloud SQL proxies logged `dial tcp +35.188.82.89:3307: i/o timeout` (12:20–12:22, 12:41–12:43, 12:51–12:53), and +`nat/dropped_sent_packets_count` went 0 -> 56/552/590, 82/272/133, 395/1565/1842 in those same minutes. +Hourly: port_usage max was 25–50 all of Sep 3 and until 10Z today, 64 in 11Z and 12Z; dropped packets 0 +until 11Z (219), then 5,491 in 12Z. Open NAT connections rose 400–600 -> 815–874. Every cell's Cloud SQL +traffic egresses through this NAT to the instance's public IP (the instance has no private IP: +`ipv4Enabled=true`, `privateNetwork` unset). When a VM's 64 ports fill, new TCP SYNs to 3307 are dropped, +the proxy's dial times out, and the relay pool's 2 s `connectionTimeoutMillis` fires: that is the exact +2 s stall the old image dies on and the new director surfaces as a 500. The dial timeouts hit c7 and c8 +hardest because they carry the most controls and open the most DB connections. + +What raised port demand today: each old-image crash re-opens a full pool through fresh NAT ports, the +autoheal recreates do the same, and the 55P03 retry storms keep more connections mid-transaction, so +crashes and NAT exhaustion feed each other. This is why the afternoon accelerated even after the disk +loop broke at 12:39. + +**Owning change (not applied):** `cloud/infra/terraform/relay-gce-foundation.tf` +`google_compute_router_nat.relay_gce` (this repo): set `min_ports_per_vm = 1024` (or enable +`enable_dynamic_port_allocation = true` with `max_ports_per_vm = 4096`) and, if needed, add manual NAT IPs +(each IP supplies 64,512 ports across VMs). Online change, no VM restart. The durable fix is giving the +Cloud SQL instance a **private IP** and pointing the proxy at `--private-ip`, which takes DB traffic off +NAT entirely; that is a Cloud SQL instance change in the orca-cloud foundation root plus a startup-script +flag here. Per the standing rule, not applied from this session. + +Direct proof: `resource.type="nat_gateway" AND jsonPayload.allocation_status="DROPPED"` shows **1,514 +dropped allocations to 35.188.82.89:3307** in 12:50–12:54 alone, every one of them the Cloud SQL public +IP. The NAT has zero manual IPs (AUTO_ONLY) and no port settings in Terraform, so it is at Google's +default 64 ports/VM. No workflow in this repo applies `relay-gce-foundation.tf` broadly (the roll +workflows apply cell templates with `-target`), so the NAT change needs a targeted apply of +`google_compute_router_nat.relay_gce`, which is an owner-run Terraform step. + +Original write-up of the symptom before the NAT correlation follows. + +The 12:50:30–12:50:50 stall (every cell 3.7–3.9 s SQL max, six old-image cells died) happened with +checkpoints healthy (85 ms) and disk at 6 MB/s, so it is not Finding 10. The cells' Cloud SQL Auth Proxy +logged `failed to connect to instance: dial error: dial tcp 35.188.82.89:3307: i/o timeout`. Count of +those per hour today: 08Z 1, 11Z 15, **12Z 416**; all of Sep 3: 4. Cloud SQL `up`/backends/connections +did not blip. So new TCP connections to the instance's public IP on 3307 are timing out from the cells' +proxies in bursts, which is exactly the "2 s connect timeout" the old image dies on. Query Insights for +12:49–12:54 attributes 1,380 s of lock wait to the placement CTE (`WITH assignment_state AS +MATERIALIZED …`) and 469 s to the single-row reservation UPDATE: the lock queue is the *consequence* of +connections stalling mid-transaction, not the cause. Not chased further; candidates are the proxy's +connection churn under the crash loops (each recreated cell opens a fresh pool) and the instance's +public-IP path. Relay code cannot fix this; it is Cloud SQL / network. Dial timeouts by minute today: 12:20 24, 12:21 +66, 12:41 22, 12:42 6, 12:51 160, 12:52 137, i.e. bursts of 20–160 s each, and they hit c7 (new image, +89 today) and c8 (93) hardest, so it is not the old image's connection churn either. Cloud SQL `up`=1 +throughout. The proxy dials the instance's public IP `35.188.82.89:3307`; a burst of i/o timeouts to a +healthy instance points at the path (public-IP egress / NAT / proxy connection limits), not at Postgres. +That is the same 2 s that the old image dies on and that the new director surfaces as a 500. + +## Roll inputs (verified by the read-only `verify` run) + +**Image census from instance templates, 2026-09-04 21:45Z (authoritative, read from `gcloud compute +instance-templates`):** 20 serving cells on `5aedbca5` (c8, c9, c10, c13–c16, c19–c29) — the image that exits +the process on a Postgres connect timeout (Finding 6); c7 on `85bf6799`; c4, c5, c17, c18 (draining / +migration-only) on `0e83408b` / `36a56b10`; c1, c2, c3, c6, c11, c12 (existing-only) on Jul/Aug images. Target +for Roll 1 is `519f4914` (director already on it). Monitor dry-run dispatched 21:45Z as the roll gate; waves +require owner go. + + +- target-image-digest `sha256:519f4914217f08cabcdcd34825965db8473ec37c6591553a3af0d65dcdeeb183` (lock fix; supersedes 85bf6799 as target) +- previous target `sha256:85bf67993869a769642995d0863f4c2b6b569c3850c2d8390ec2ca5f2b179e28` (c7 is on this; use as c7's rollback) +- rollback-image-digest `sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563` +- target/rollback rehome protocol 1 / 1; expected-rehome-generation 12; selector generation **112** (110 before the c7 canary) +- existing-only c1,c11,c12,c2,c3,c4,c5,c6; migration-only c17,c18; general c10,c13–c16,c19–c29,c7,c8,c9 +- confirmation for canary: `ROLL_RELAY_SAME_CAP production-gce-c7` +- monitor evidence is single-use and must be < 5 min old at dispatch (plus 75 min per predecessor wave) +- monitor dry-run dispatch (read-only, runs at `main` head so a merged bar change applies immediately): + `gh workflow run cloud-monitor-relay-production.yml --ref main -f mode=dry-run -f expected-selector-generation=110 + -f expected-existing-only-cells= -f expected-migration-only-cells=production-gce-c17,production-gce-c18 + -f expected-general-cells= -f migration-policy=strict -f recovery-source-cell-id=none -f capacity-cell-id=none` + +## Queries that worked (copy-paste) + +- Cell metrics: `resource.type="gce_instance" AND jsonPayload.event="orca_relay_runtime_metrics"` +- Container crashes: `resource.type="gce_instance" AND jsonPayload.MESSAGE:"container die" AND jsonPayload.MESSAGE:"relay@sha256"` +- Crash banner: `resource.type="gce_instance" AND jsonPayload.message:"Node.js v24"` +- Retries: `jsonPayload.event="orca_relay_postgres_transaction_retry"` (no resource filter to get both) +- Director lines are `textPayload`; cell lines are `jsonPayload.message` +- Cloud Run concurrency: Monitoring API `run.googleapis.com/container/max_request_concurrencies` +- Dry-run final state: download artifact `relay-monitor-dry-run--`, read `*.state.json` (the log's `schemaVersion` lines are only checkpoints, not the final verdict) + +## 2026-09-04 22:50Z onward: owner go received; driving the gates + +Owner: "sure, feel free to drive these." Sequence chosen: Roll 1 first (highest uplift), auth deploy with +#478 second, pruner enable third, label drift resolved by matching Terraform to live state, #477 still held. + +| Step | Result | +| --- | --- | +| Monitor dry-run #19 (gen 112, strict) | **Passed** 23:07:53Z, run 33927238469 attempt 1. First green since the probe fix (#18723). 16 samples, no freeze. Dispatched 22:51:33Z after confirming: 0 `container die` in 3 h, director 5xx in the last 4 h were all 503s (excluded by the `director.errors` filter). | +| c8 `canary-apply` onto 519f4914 (rollback 5aedbca5) | **Failed at 23:09:07Z before any mutation**: `relay monitor evidence provenance does not match` in `verify-authority`. Run 33928330631. Gate job passed, `cell_1 / rollout` failed on the manifest check, `seal_canary` skipped, lease released. Cause: the manifest binds `commitSha`; the dry-run ran at main `264c9ed8d2`, the canary dispatched at `--ref main` resolved to `4fab8e2f15` because unrelated PRs merged to main during the 15-minute gate. Verified no side effects: c8 MIG still on template `…c8-20260827…` (5aedbca5), stable, 25 controls; no `/v1/admin/drain` or isolate calls in the director log. | +| Constraint learned | Both workflows must run at the **same main commit**. The production environment's deployment branch policy allows only `main`, and the job gates on `github.ref == 'refs/heads/main'`, so a pinned tag/branch is not an option. Any merge to stablyai/orca main during the 15-minute dry-run invalidates the evidence. Mitigation for the retry: dispatch the canary within seconds of the green, and do not merge anything to stablyai/orca main myself during the window. A durable fix (accept evidence whose commit is an ancestor with identical workflow/script content) is a follow-up, not a same-day change to a safety check. | +| Label drift (5.x) | Resolved by dropping the `region` label from Terraform to match the 21 live metrics (stablyai/orca #18734, merged). Targeted plan asserted `27 no-op, 9 create, 0 destroy`; applied 23:11Z: 8 `orca_relay_control_*` renewal metrics that had never been applied, plus `google_monitoring_dashboard.relay_incident`. `orca_relay_controls` createTime unchanged (2026-07-13), label extractors unchanged. | +| Pruner enable (1.2) | orca-cloud #479 merged: `auth_token_pruner_enabled = true`, image digest of `00031-tox`, `max_rows_per_run = 20000`. Targeted plan asserted 9 create / 0 change / 0 destroy (job, scheduler at `41 * * * *` UTC, two service accounts, five IAM grants). **Not yet applied**: waiting until the roll canary has landed so the first hourly run does not overlap a drain. | +| Auth deploy with #478 (3.1) | Dispatched 23:13Z from orca-cloud main `f0fa4b5` (run 33928663526). Candidate startup adds nullable `successor_material` under a brief ACCESS EXCLUSIVE lock. | +| Auth deploy result | **Succeeded** 23:15:37Z: `orca-cloud-auth-00035-gos` serving 100 %, cap 20 preserved, 0 5xx. `refresh_tokens.successor_material` present (nullable text); 298 sealed successors written in the first 15 min against 924 rotations; `session-refresh-reuse-detected` at baseline (5 / 15 min). Grace window is live. | +| Monitor dry-run #20 | Froze 23:35:38Z on `runtime_power_unknown cell.production-gce-c11.powered`. Two window restarts earlier (23:24, 23:25) on `signal_stale auth.errors` (Cloud Monitoring publish lag 181–255 s vs 180 s bar). Cause: one transient rejection of the per-cell MIG GET in `readResourceInventory` yields `targetSize: null` → `runtimeKnown=false` → hard freeze. c11 is a parked existing-only cell (MIG size 0, stable) and was fine. Not fleet health. Fix delegated: stablyai/orca #18740 (retry the MIG read once, mirroring #18723). Run 33928912676. | +| Monitor dry-run #21 | **Green** 23:54Z at main `8064d1f991`, but main had moved to `0a821e5bc8` during the window; the chain re-gated instead of dispatching (the canary would have failed provenance again). Run 33930229711. | +| Monitor dry-run #22 | **Green** 00:10Z at `0a821e5bc8`; main moved to `2e80972450`. Re-gated. Run 33931177390. | +| Monitor dry-run #23 | Froze 00:18:31Z on `cell.production-gce-c29.latency_ms` 2635 > 2000, the probe's own round-trip from a US runner to asia-east2; c29 controls 17→19 and `sqlLatencyMsMax` flat ~1050 through the minute, no crash, no checkpoint stall. c29 probe max was 0 in the three previous gates, so a one-off. Run 33932092775. | +| Blocking constraint | Main receives unrelated merges every 5–10 min (23:08, 23:15, 23:17, 23:40, 23:42, …). A 15-min gate bound to an exact commit cannot be consumed under that traffic. Delegated a durable fix: `verify-authority` accepts evidence whose commit is an ancestor of the canary commit **and** has no diff on the monitor/deployer trusted paths; fails closed on shallow clones or unknown commits. Chain re-armed on dry-run #24 (run 33932679796) meanwhile. | +| Monitor dry-run #24 | Froze 00:28:00Z on `director.instances` 4 < 5. Cloud Run active-instance count read 4 for exactly one minute (00:27), 5 in every other minute for 3 h; min/max scale is pinned at 5; no new revision. A routine single-instance recycle. Not fleet health. Bar `directorInstancesMin: 5` with `latest-sum` cannot tolerate that; recalibrate to 4 or use a 3-min window minimum (follow-up, not same-day). Run 33932679796. Chain dispatched #25 (run 33933193511) at `86cd327749`. | +| Monitor dry-run #25 | **Green** 00:46Z at `86cd327749`; main moved to `8096cb2803`. Fourth green gate lost to unrelated main traffic (#19, #21, #22, #25). Run 33933193511. Chain's re-gate #26 (run 33934079533) cancelled by me. | +| Fixes merged 00:55Z | stablyai/orca #18740 (MIG inventory read retried once before `runtime_power_unknown`; 2 tests) and #18754 (`verify-authority` and the batch canary authority accept evidence sealed at an **ancestor** commit when every trusted monitor/deployer path is byte-identical; fails closed on shallow clones and unknown commits; deploy/rehome jobs now check out with `fetch-depth: 0`; 5 new tests, 18/18 pass). Reviewed both diffs; trusted-path set verified to exist on main. | +| Monitor dry-run #27 | Dispatched 00:56Z at `74ad08ec66` (first gate whose evidence the new rule can consume). Run 33934541092. Chain re-armed with the same ancestor + identical-trusted-code rule so an unrelated merge no longer forces a re-gate. | +| Monitor dry-run #27 | **Green** 01:11:35Z at `74ad08ec66`; main had moved to `38bde20121` with identical trusted code, so the new rule (#18754) let the chain dispatch. Run 33934541092. | +| c8 `canary-apply` #2 (run 33935407461) | Provenance check **passed** (first consumption of ancestor evidence). Isolate → gen 113, drain, template+MIG applied 01:14–01:22, new c8 came up on `519f4914` and `relay_capacity_transition_verified` (migration-only, image exact, heartbeat fresh) at 01:23:50. Then the step's next call, `curl --fail-with-body` to c8 `/v1/admin/runtime-status`, got a **503 with a 27-byte body** at 01:23:51 and the step exited 22. Director `cell-status` at 01:23:50.8 returned 200; c8's own logs show nothing at that second; c8 health/ready both 200 seconds later; backend HEALTHY (the health check had just flipped TIMEOUT→HEALTHY at 01:22:16 and UNKNOWN→HEALTHY at 01:23:47 as the new instance warmed). Read: a single 503 at the load-balancer/warm-up edge on a curl with no retry, on a cell that was already verified healthy one line earlier. Failsafe ran: c8 kept **migration-only**, rehome control disabled, selector gen 113. c8 is serving (40 controls at 01:39, sqlLatencyMsMax ~30 ms) on the target image, just not admitted for general traffic. Nothing to roll back. | +| Recovery | The job has an explicit resume path: `mode=rollback` with `rollback-image-digest` = the image the cell already runs skips isolate/apply, verifies, and restores general admission (`ROLLBACK_RESUME=true`). Dispatched gate #28 (run 33936966508) at gen 113 with c8 in migration-only; on green the chain dispatches that resume for c8 with rollback digest `519f4914` and target `5aedbca5` (the validator only requires them to differ). | +| Follow-up | The verify step's bare `curl --fail-with-body` needs the same "no reading is not a verdict" retry the monitor got (#18723/#18740); a 503 immediately after `verify-relay-capacity-transition` passed is not evidence of a bad cell. | +| Monitor dry-run #28 | **Green** 01:58:59Z at gen 113 with c8 in migration-only. Run 33936966508. | +| c8 recovery (run 33937756402, `mode=rollback`, rollback digest = 519f4914) | **Succeeded** 02:02Z. `ROLLBACK_RESUME=true` path: isolate/apply skipped, converged-Terraform check passed, verify passed (`relay_capacity_transition_verified` general, image `519f4914`, heartbeat fresh), activate → **gen 114**, c8 general. No restart, no drain. c8 at 43 controls, sqlLatencyMsMax 36 ms. **c8 is the second cell on 519f4914** (with c7 on 85bf6799). Because the recovery ran as `rollback`, `seal_canary` was skipped, so no canary authority exists for a `batch-apply`; the next cell runs as another `canary-apply`. | +| Merged 02:05Z | stablyai/orca #18769: bounded retries on every admin-endpoint curl/fetch in the same-cap job and the rehome/canary/verify scripts (`--retry 3 --retry-delay 2 --retry-connrefused`, per-attempt bodies to a file; script helper 2 attempts on network error or 500/502/503/504 only; 4xx never retried; 650/650 tests). Trusted-path change, so the next gate runs at a commit containing it. | +| Pruner enabled (1.2) | Terraform applied 02:06Z (8 creates, then the deploy-identity job IAM grant after a propagation 404, 9/9). Job `orca-cloud-auth-token-pruner`, image `343a0915…`, scheduler `41 * * * *` UTC, budget 20 000 rows/run. First run by hand (exec `sf5ct`): cold start 3m20s, then `stopReason: time-budget` at 480 s: 73 batches, 365 000 scanned, **1 040 deleted** (1 021 revoked, 19 expired, 0 rotated), ~6.4 s/batch of 5 000, `completedFullPass: false`. No errors, no lock-wait or checkpoint alert. Scan-bound, not budget-bound: at this pace a full pass over the table takes many hourly runs, and the row budget is never the limiter. Leave the budget alone; watch hourly runs for `stopReason` and a rising `deletedRows` as the cursor reaches the rotated backlog. | +| Monitor dry-run #29 | **Green** 02:20:58Z at gen 114, main `e2b70a5eba` (contains #18740, #18754, #18769). Run 33938052374. | +| c9 `canary-apply` (run 33938818286) | **Succeeded end to end** 02:21–02:34Z: isolate → gen 115, drain, template+MIG to `519f4914`, verify passed on the first try (retry-hardened step), trust proof, activate → **gen 116**, general. `seal_canary` **succeeded**: batch authority now exists. c9 at 38 controls, sqlLatencyMsMax 33 ms. No `container die` in 30 min. Three cells on new images (c7 `85bf6799`, c8 and c9 `519f4914`); 17 serving cells still on `5aedbca5`. | +| Monitor dry-run #30 | Dispatched 02:36Z at gen 116 (run 33939533990). On green the chain dispatches **batch 1**: `batch-apply` c10,c13,c14,c15 bound to canary run 33938818286 (sealed at gen 116, same commit `e2b70a5eba`). Preflight: all four on `5aedbca5`, MIGs stable, no crash in 20 min. Sequential cells inside the job (wave-index 0..3), each with its own isolate/drain/apply/verify/restore, so ~12 min per cell, ~50 min total. | +| Monitor dry-run #30 verdict | **Green** 02:52:15Z at gen 116, `e2b70a5eba`. | +| Batch 1 (run 33940290163) | Dispatched 02:52:27Z: `batch-apply` c10,c13,c14,c15, canary authority run 33938818286, same commit. | +| Batch 1 attempt 1 (run 33940290163) | **Failed at 02:54:39Z in the live preflight, before any mutation**: `relay live preflight failed: cloud-monitoring/signal_stale`. The step's `--retry-freshness` (5 attempts, 15 s apart, freshness-only codes) is passed only for `WAVE_INDEX != 0`; the first cell takes a single sample, so one Cloud Monitoring publish lag > 180 s at that instant fails the batch. Every candidate series was current again by the time I checked. c10 untouched (template `…c10-20260827…`, 47 controls), no selector write, gen still 116, failsafe no-op. Gate #31 dispatched 02:57Z (run 33940508865); chain re-dispatches the same batch (canary authority 33938818286 still valid: same gen 116, same commit). Fix delegated: wave 0 gets the same freshness retry. | +| Monitor dry-run #31 | **Green** 03:13:26Z at gen 116; main at `cb7f7dd11a` with identical trusted code. Run 33940508865. | +| Batch 1 attempt 2 (run 33941253533) | Dispatched 03:13:38Z: c10,c13,c14,c15, canary authority 33938818286. Runs at `cb7f7dd11a` (batch authority is accepted across the ancestor since trusted paths are unchanged). | +| Merged 03:14Z | stablyai/orca #18778: `--retry-freshness` on every same-cap wave including the first, and the retry loop now stops before the next wait would push evidence past the wave's age bound (it was checked only at entry before). Twin carve-out in the capacity job filed as a follow-up. | +| Batch 1 cell 1 (c10) | **Succeeded** 03:14–03:27Z (preflight, drain, apply, verify, restore). c13 started 03:27Z. | +| Batch 1 cell 2 (c13) | **Succeeded** 03:27–03:38Z. c14 started 03:38Z. | +| Batch 1 cell 3 (c14) | **Succeeded** 03:38–03:50Z. c15 started 03:50Z. | +| Batch 1 complete (run 33941253533) | **All four succeeded** 03:13–04:00Z: c10, c13, c14, c15 on `519f4914`, selector **gen 124**. Fleet at 936 controls, 23 cells. Two `container die` at 03:35:41/44 were **c13's new container** exiting during boot (`applyPostgresSchema` → `Connection terminated due to connection timeout`, exit 1, 2 s runtime each) because the `cloud-sql-proxy` sidecar had not finished starting; the third start at 03:35:45 succeeded and c13 has been serving since (57 controls). A boot-order race in the container spec, not a serving-cell crash. Follow-up: schema pool should wait for the proxy socket, or the container should depend on the proxy's readiness. **8 cells on new images** (c7 85bf6799; c8, c9, c10, c13, c14, c15 519f4914), 12 on `5aedbca5`: c16, c19–c26 (US), c27–c29 (Asia). | +| Monitor dry-run #32 | **Green** 04:19:50Z at gen 124, main `436ef827dd` (contains #18778). Run 33943539025. | +| c16 `canary-apply` (run 33944255902) | Dispatched 04:20:02Z. On success it seals the authority for batch 2 (c19,c20,c21,c22). | +| c16 canary (run 33944255902) | **Succeeded** 04:20–04:32Z, activate → gen 126, batch authority sealed. 9 cells on new images. | +| Monitor dry-run #33 | Failed 04:58:56Z on `continuity_deadline_exceeded` (1 500 004 ms > 1 500 000 ms). One `signal_stale cloud_sql.lock_waits` at 04:46 (189 s vs 180 s bar, Cloud Monitoring publish lag) restarted the 15-min window at sample 12; the restart could not complete inside the 25-min continuity cap. No health failure at any sample; no `container die` since c16's own boot race at 04:30. Run 33944873727. Chain re-gates. Note for recalibration: `cloudDataMaxAgeMs: 180000` vs observed Cloud Monitoring publish lag of 181–255 s has now cost three gates (#20 twice, #33). | +| Freshness recalibration | stablyai/orca #18798 (open, merge after batch 2 dispatch): `cloudDataMaxAgeMs` 180 s → 330 s, derived from Google's documented visibility delays (Cloud Run 60+120 s, Cloud SQL 60+165 s) and the 5-min window-sum query (a label series that stops emitting reads as up to 300 s old while its sum is complete, which is the 255 s `auth.errors` case) plus ~30 s collect latency. Director-admin and the lock-wait carry keep their own 180 s pins. A freshness-only failure may miss 2 consecutive samples without restarting the window; the sample still counts and is still threshold-checked; a 3rd miss, collector failure, runner gap, or any breach restarts/freezes as before. 92/92 tests. | +| Monitor dry-run #34 | **Green** 05:17:31Z at gen 126, `436ef827dd`. Run 33946093029. | +| Batch 2 (run 33946819345) | Dispatched 05:17:43Z: c19,c20,c21,c22, canary authority 33944255902 (c16). | +| Merged 05:19Z | stablyai/orca #18798 (freshness bar 330 s + two-sample tolerance). Next gate runs at a commit containing it. | +| Batch 2 cell 1 (c19) | **Succeeded** 05:19–05:32Z. c20 started. | +| Batch 2 cell 2 (c20) | **Succeeded** 05:32–05:43Z. c21 started. | +| Batch 2 cell 3 (c21) | **Succeeded** 05:43–05:59Z. c22 started. | +| Batch 2 complete (run 33946819345) | **All four succeeded** 05:17–06:12Z: c19, c20, c21, c22 on `519f4914`, selector **gen 134**. Fleet at 1 090 controls, 23 cells, refresh 401s at baseline (1–4 per 3 min). One `container die` at 06:08:45 was **c22's new container** exiting during boot (exit 1, 2 s runtime; started 06:08:43, restarted 06:08:46 and serving since), the same proxy-sidecar boot race seen on c13 and c16. No serving-cell crash. **Census: 15 of 23 serving cells on new images** (c7 `85bf6799`; c8–c10, c13–c16, c19–c22 `519f4914`), 7 on `5aedbca5`: c23–c26 (US), c27–c29 (Asia). Next: gate at gen 134 → canary c23 → batch c24,c25,c26; then canary c27 → batch c28,c29. | +| Monitor dry-run #35 | **Green** 06:32:01Z at gen 134, `b33d1972bc` (contains #18798, first gate at the 330 s freshness bar). Run 33949334606. | +| c23 `canary-apply` (run 33950075843) | Dispatched 06:32:13Z at main `b0c67eaf88` (ancestor gate SHA, identical trusted code). On success it seals the authority for batch 3 (c24,c25,c26). | +| c23 canary (run 33950075843) | **Succeeded** 06:32–06:46Z, activate → gen 136, batch authority sealed. No `container die` during boot. 16 of 23 serving cells on new images; 6 on `5aedbca5` (c24–c26 US, c27–c29 Asia). | +| Monitor dry-run #36 | Dispatched 06:46Z at gen 136, run 33950746574 (`58553bfe1c`). On green the chain dispatches batch 3 (c24,c25,c26) under canary authority 33950075843. | +| Monitor dry-run #36 result | **Green** 07:02:49Z at gen 136, `58553bfe1c`. | +| Batch 3 (run 33951468008) | Dispatched 07:03Z: c24,c25,c26, canary authority 33950075843 (c23). | +| Batch 3 cell 1 (c24) | **Succeeded** 07:04–07:18Z. c25 started. | +| Batch 3 cell 2 (c25) | **Succeeded** 07:18–07:31Z. c26 started. | +| Batch 3 complete (run 33951468008) | **All three succeeded** 07:03–07:44Z: c24, c25, c26 on `519f4914`, selector **gen 142**. Fleet at ~1 230 controls, 23 cells, refresh 401s at baseline. **Zero `container die`** during the batch (no boot race on c24–c26). **All 20 US serving cells now on new images** (c7 `85bf6799`; c8–c10, c13–c16, c19–c26 `519f4914`). Remaining on `5aedbca5`: c27, c28, c29 (asia-east2, probe hard cap 3000 ms). | +| Monitor dry-run #37 | Dispatched 07:48Z at gen 142, run 33953555224 (`4c5077d57a`). On green the chain dispatches the c27 canary (first Asia cell). | +| Monitor dry-run #37 result | **Green** 08:04:24Z at gen 142, `4c5077d57a`. | +| c27 `canary-apply` (run 33954264945) | Dispatched 08:04Z, first Asia cell (asia-east2-a). On success it seals the authority for batch 4 (c28,c29). | +| c27 canary (run 33954264945) | **Failed closed before any mutation** 08:07:25Z at "Verify exact current generation, digest, cap, and rollback point": `runtime predecessor mismatch fields=regionalRehomeProtocol`. **Operator input error, not a cell fault**: the chain script hardcoded `target-rehome-protocol=1 / rollback-rehome-protocol=1` for every cell, but `relay_region_rehome_source_cell_ids` lists only the 16 US cells (c7–c10, c13–c16, c19–c26), so the Asia startup template omits `ORCA_RELAY_REHOME_*` and c27–c29 report protocol 0 by design. `MUTATION_STARTED` never set, failsafe no-op, selector stays gen 142, c27 still serving on `5aedbca5`, no `container die`. Gate #37 evidence consumed. Fix: chain script now takes `PROTO`; Asia round dispatches with protocol 0 (the per-host trust proof step is protocol-gated and skips, as designed for non-source cells). Follow-up: the job already reads `relay_region_rehome_source_cell_ids`; it could derive the expected protocol from membership instead of trusting the operator input. | +| Monitor dry-run #38 | Dispatched 08:12Z at gen 142, run 33954621425 (`e95d247be1`). On green the chain dispatches the c27 canary with protocol 0. | +| Monitor dry-run #38 result | **Green** 08:28:36Z at gen 142, `e95d247be1`. | +| c27 `canary-apply` #2 (run 33955359385) | Dispatched 08:28Z with `target/rollback-rehome-protocol=0`. | +| c27 canary #2 (run 33955359385) | **Failed closed, no mutation** 08:31:19Z. Predecessor check passed with protocol 0; the isolate step then died at argument parsing: `production capacity target is not approved`. The same-cap job shells out to `prepare-relay-production-capacity-canary.mjs` for isolate/drain/activate, whose `PRODUCTION_CAPACITY_CELL_IDS` allowlist is the 16 US capacity cells (c7–c26), while the same-cap wave validator (`SAME_CAP_CELLS`) approves all 19 serving cells including c27–c29. The Asia cells have never been through this job (their Aug 14 rollout used the asia-topology workflow). Both the isolate step and the failsafe threw before any HTTP call, so `MUTATION_STARTED=true` was written but nothing was isolated: selector stays gen 142, c27 general and serving on `5aedbca5`, no `container die`. Gate #38 evidence consumed. Fix: stablyai/orca #18811 (`--approved-cells same-cap` on all four invocations, default unchanged for the US capacity job, census test over every `SAME_CAP_CELLS` member × isolate/drain/activate + the job's cell-shape bash block; 525/525 script tests). Sweep of the other job scripts found no further Asia blocker; gate #39 (run 33955668701) dispatched at gen 142 to prove the selector is unchanged before the next attempt. | +| Monitor dry-run #39 | **Green** 08:51:28Z at gen 142: independent proof the selector was untouched by both failed c27 attempts. Not used for dispatch (its commit predates #18811). | +| Merged 08:51Z | stablyai/orca #18811 → main `12e05203a4`. | +| Monitor dry-run #40 | Dispatched 08:51Z at gen 142 on main `12e05203a4` (contains #18811), run 33956408337. On green the chain dispatches the c27 canary, protocol 0, third attempt. | +| Monitor dry-run #40 result | **Green** 09:08:03Z at gen 142, `12e05203a4`. | +| c27 `canary-apply` #3 (run 33957151726) | Dispatched 09:08Z, protocol 0, on main containing #18811. | +| c27 canary #3 (run 33957151726) | **Failed after isolate; failsafe held** 09:17:21Z. Live check 09:26Z: c27 at 0 controls (drained), template still `…20260814235757`, c28/c29 absorbed the hosts (37 each), fleet 1 404 controls / 23 cells, refresh 401s baseline, no `container die` in 60 m. Predecessor check and allowlist passed; isolate → **gen 143** (c27 migration-only), drain sent (graceMs 0, hosts reconnected via director to c28/c29/US). Terraform plan built correctly (template replace + MIG update to `519f4914`), then `validate-relay-capacity-plan.mjs --mode same-cap-cell` rejected it: `cell plan does not contain the reviewed image and capacity`. Its same-cap rule demands exactly one `ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT` and one `ORCA_RELAY_REHOME_AUDIENCE` printf in the startup script; Asia templates omit both because c27–c29 are not rehome sources (same root as attempt 1, third US-only assumption in the job). **No apply ran**: c27 template unchanged, still `5aedbca5`, isolated and draining (drain is one-way in-process; only a restart clears it). Failsafe re-asserted migration-only at gen 143 and rehome disabled. Recovery plan: fix validator (protocol-0 path: require the rehome lines *absent*), merge, gate at gen 143, then `mode=rollback` with rollback-image=`519f4914` (the failed-canary re-entry path; accepts draining + migration-only) to restart c27 onto the target image and restore it; then single-cell canaries for c28 and c29 (batch needs ≥2 cells). | +| Plan-validator fix | stablyai/orca #18818 (merged 09:41Z → main `9f2a9a248e`): `validate-relay-capacity-plan.mjs --regional-rehome-protocol 0|1` in same-cap-cell mode; protocol 0 requires the rehome lines *absent*, protocol 1 unchanged; both plan-validation calls in the job pass `DESIRED_REHOME_PROTOCOL`; census test now validates a correct plan for every `SAME_CAP_CELLS` member at its tfvars-derived protocol. 529/529. Residual: the operator-supplied protocol is still unbound for Asia cells (no `SOURCE_CELLS` cross-check outside us-central1), so a wrong value fails late at plan validation rather than early; deriving it from membership is the checklist follow-up. | +| Monitor dry-run #41 | Dispatched 09:42Z at gen 143 (c27 expected migration-only) on main `9f2a9a248e` (contains #18811 + #18818), run 33958728141. On green: c27 recovery via `mode=rollback`, rollback-image `519f4914`, protocol 0, confirmation `ROLL_BACK_RELAY_SAME_CAP`. | +| Monitor dry-run #41 result | **Green** 09:58:51Z at gen 143, `9f2a9a248e`. | +| c27 recovery #1 (run 33959789773, `mode=rollback`) | **Failed closed, no mutation** 10:09:21Z at `Verify monitor evidence provenance`: `relay monitor dry-run authority is incomplete or stale`. The dry-run authority is valid for 5 min after `completedAt` at wave 0 (`EVIDENCE_MAX_AGE_MS`); the gate completed 09:58:51Z but the operator poller (20 s `gh run view` loop) only observed completion at 10:07:09Z during a local network outage, so the dispatch landed at 10:07:11Z, 8 m 20 s after completion. Failed before the rollout lease, isolate, or any Terraform step; c27 unchanged (migration-only, drained, `5aedbca5`, gen 143). Every prior canary dispatched ≤15 s after gate green, so this is a dispatch-latency miss, not a job defect; the freshness bound behaved as designed. | +| Monitor dry-run #42 | Dispatched 18:39Z at gen 143 on main `af82126058` (trusted paths byte-identical to `9f2a9a248e`), run 33984753269. Recovery script re-armed behind it (same `mode=rollback` onto `519f4914`, protocol 0). | +| Monitor dry-run #42 result | **Green** 18:55:47Z at gen 143, `af82126058`. | +| c27 recovery #2 (run 33985902062, `mode=rollback`) | **Failed closed, no mutation** 19:05:02Z, same `authority is incomplete or stale`. Dispatch landed 19:02:39Z, 6 m 52 s after the gate completed. Root cause of both misses is the operator laptop sleeping during the 15 min gate wait (`pmset -g log`: asleep 18:52:28Z → 19:02:17Z; the morning miss coincided with a sleep/dark-wake cycle too), so the 20 s poller never ran inside the 5 min window. Not a job or evidence defect: the freshness bound did its job. Operator fix: poller now runs under `caffeinate -i`. | +| Monitor dry-run #43 | Dispatched 19:06Z at gen 143 on main `af82126058`, run 33986121849. Recovery armed behind it under `caffeinate`. | +| Monitor dry-run #43 result | **Green** 19:22:53Z at gen 143, `af82126058`. | +| c27 recovery #3 (run 33986948522, `mode=rollback`) | **Failed closed, no mutation** 19:25:48Z. Dispatched 13 s after gate green (authority accepted this time), then the live preflight recheck failed: `relay live preflight failed: active-probe/threshold_max`. That is the 2 000 ms `endpointLatencyMs` bar on one endpoint's slowest /health or /ready round trip from the runner (8 s fetch timeout, one retry). The error names no endpoint and the job log prints none; gate #43 had zero failures across 16 samples, so this was a transient probe slow-down in the ~3 min between gate and preflight. Live probe 19:32Z from the operator: director and auth ~130–190 ms, US cells ≤540 ms, Asia cells 690–1 315 ms (c28/c29 /health ~1.3 s, the closest to the bar; c27 ~0.9 s). Existing-only cells c1–c3, c6, c11, c12 return 503 on both paths as expected (unpowered). Failed before the rollout lease, isolate, or any Terraform step; c27 unchanged. Follow-up (checklist): preflight should print the failing signal and observed value. | +| Monitor dry-run #44 | Dispatched 19:33Z at gen 143 on main `062db77118`, run 33987646501. Recovery re-armed behind it. | +| Monitor dry-run #44 result | **Frozen red** 19:50:01Z after 13 samples: `active-probe/threshold_max cell.production-gce-c27.latency_ms observed=2568 threshold=2000`. No other failure, no continuity event, no `container die` fleet-wide in 60 m. `/health` is a static JSON reply (`app.ts`), so the slow round trip was `/ready` (the probe reports the max of the two) or the path to the cell. Cloud SQL logs for 19:49:38Z–19:51:58Z show six `could not obtain lock on row in relation "relay_cells"` errors and a time-triggered checkpoint completing at 19:50:36Z (write phase 270 s, the spread target, not a stall). c27 is drained with 0 controls, so its `/ready` dependency check was the only thing it was doing. Recovery script stopped as designed (no auto re-gate). Operator probe 19:53Z: c27 and c28 both bimodal, ~0.27 s or ~0.89 s per `/health` from the US, identical shape, nothing c27-specific. Attributing the one 2.6 s sample to the same shared-DB contention that produced the lock errors is the best available reading; the retry at gate #45 tests whether it recurs. | +| Monitor dry-run #45 | Dispatched 19:53Z at gen 143 on main `062db77118`, run 33988383401. Recovery re-armed behind it. | +| Monitor dry-run #45 result | **Frozen red** 19:54:47Z after 3 samples, same signal: `cell.production-gce-c27.latency_ms observed=2668 threshold=2000`. Two gates in a row now attribute a >2 s round trip to c27 while every other cell passes. | +| c27 `/ready` tail analysis | `/health` is static; `/ready` (`relay-readiness.ts`) fetches the auth JWKS (2 s timeout) then runs `SELECT 1`, cached 10 s. Operator probes 19:57Z–20:00Z, 15 each from the US: c27 and c28 have the **same** tail (0.27 s / 0.88 s modes, then 1.3 s, then 2.17–2.27 s at the top); US cells c8/c20 sit at 0.08–0.18 s. Auth JWKS latency over the last hour: 400 requests, max 20 ms, none over 1 s. So the tail is cell→Cloud SQL (US) round trips plus the runner→Asia hop, not auth and not c27-specific; c27 is drained (0 controls) so nothing local competes. Cloud SQL `could not obtain lock on row in relation "relay_cells"` runs at 17–78 per 10 min all day (NOWAIT inventory locks, expected under placement bursts) with no spike in the failing minutes. The bar (`endpointLatencyMs` 2 000 ms, one shot per minute, max of two paths) leaves Asia cells ~10% of samples from tripping; the gate got unlucky twice on c27 and lucky on c28/c29. Not a health finding. | +| Monitor dry-run #46 | Dispatched 20:01Z at gen 143 on main `062db77118`, run 33988810139. Recovery re-armed behind it. If this also freezes on an Asia probe, the next move is a per-region latency bar (or p50 over the window) in `incident-monitor.ts`, reviewed and merged before further Asia gates rather than retrying blindly. | +| Monitor dry-run #46 result | **Frozen red** 20:07:44Z after 7 samples, third time on `cell.production-gce-c27.latency_ms` (observed 2 685). Operator 40-sample `/ready` probe per Asia cell at 20:10Z: c27 p50 0.88 s / p90 2.15 s / max 2.26 s / 6 over 2 s; c28 p50 0.88 / p90 1.25 / max 2.25 / 1 over; c29 p50 0.88 / p90 0.89 / max 1.27 / 0 over. All 200. `/ready` (`relay-readiness.ts`) fetches the auth JWKS in us-central1 then `SELECT 1` on Cloud SQL in us-central1, so an Asia cell's readiness is two trans-Pacific hops plus the runner→Asia hop; the fleet-wide 2 000 ms bar was calibrated on US cells (0.08–0.5 s). c27 being drained and idle has no local load, so this is path latency, not health. **Stopped retrying gates.** Fix in flight: per-region `cell..latency_ms` bar (us-central1 stays 2 000, asia-east2 4 000; hard faults still caught by the health/ready equal-1 checks and the 8 s probe timeout) plus attributable preflight failure messages, via review + CI before the next Asia gate. | +| Merged 20:33Z | stablyai/orca #18877 → main `a3c1d32995`: per-region `cellEndpointLatencyMs` (us-central1 2 000, asia-east2 4 000; director/auth rules and the `endpointLatencyMs` key unchanged), region carried from tfvars onto every cell expectation, preflight failures now print `source/code signal observed= threshold=`. relay-ops 95/95, cloud suite 633 + 529 + 148 green. | +| Monitor dry-run #47 | Dispatched 20:34Z at gen 143 on main `a3c1d32995` (first gate with the per-region bar), run 33989896150. Recovery re-armed behind it. | +| Monitor dry-run #47 result | **Green** 20:38:09Z at gen 143 on `a3c1d32995`: first gate under the per-region bar, 16/16 samples, no Asia latency failure. | +| c27 recovery #4 (run 33990715317, `mode=rollback`) | **Success** 20:51Z. Dispatched 13 s after gate green. Isolate re-asserted migration-only at gen 143 (already isolated, no change), Terraform applied the same-cap template `…20260905204141` and the MIG replaced the instance, new incarnation on `519f4914`, protocol 0, transition verifier passed at migration-only (1 180 assignments carried, hard cap 3 000, heartbeat fresh), then activate → **gen 144**, c27 general, verifier passed again. No `container die` fleet-wide 19:55Z–20:52Z. c27 now runs the target image; c28/c29 remain on `5aedbca5` (template `…20260814235757`). | +| Monitor dry-run #48 | Dispatched 20:53Z at gen 144 (c27 back in general, MIG = c17,c18) on main `61ebffa86e` (trusted paths identical to `a3c1d32995`), run 33991385880. On green the chain dispatches the c28 `canary-apply`, protocol 0. | +| Monitor dry-run #48 result | **Green** 21:08Z at gen 144, 16/16 samples, no Asia latency failure. Main had moved to `5cec2c2dfc`; the chain verified the trusted paths were identical to the gate commit and dispatched 12 s after green. | +| c28 canary (run 33992169289, `canary-apply`) | **Success** 21:27Z. Isolate → migration-only at **gen 145**, drain already clear, verifier passed on the old image (1 220 assignments carried, hard cap 3 000, heartbeat fresh), Terraform applied same-cap template `…20260905211352`, new incarnation on `519f4914` at protocol 0, verifier passed again at migration-only, activate → **gen 146**, c28 general, verifier passed (1 219 assignments). Seal step recorded the canary. No `container die` fleet-wide 21:08Z–21:30Z. Only c29 remains on `5aedbca5`. | +| Monitor dry-run #49 | Dispatched 21:33Z at gen 146 (c28 back in general, MIG = c17,c18) on main `dce5ebd83d` (trusted paths identical to `a3c1d32995`), run 33993075948. On green the chain dispatches the c29 `canary-apply`, protocol 0, the last Roll 1 cell. | +| Monitor dry-run #49 result | **Frozen red** 21:52:24Z, `active-probe/continuity_deadline_exceeded observed=1500005 threshold=1500000`. One continuity event at 21:41:27Z, `cloud-monitoring/collector_failed` (a Cloud Monitoring read failed, not tolerated), which reset the continuous window at sample 14; the restarted window reached 10 samples before the 25-minute lineage cap (`INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS`) expired. No health failure in any of the 25 samples, no Asia latency failure, no `container die`. Monitor-side transient, not a fleet finding. The chain re-gated automatically after its 2-minute back-off. | +| Monitor dry-run #50 | Dispatched 21:54Z at gen 146 on main `51eed5a1bc`, run 33994385666. **Green** 22:10Z, 16/16 samples. Main had moved to `d7767fb196`; trusted paths identical to `a3c1d32995`. Chain dispatched the c29 `canary-apply` (run 33995164002, protocol 0) 12 s after green. | +| c29 canary (run 33995164002, `canary-apply`) | **Success** 22:27Z. Isolate → migration-only at **gen 147**, verifier passed on the old image (1 199 assignments), Terraform applied same-cap template `…20260905221622`, new incarnation on `519f4914` at protocol 0, verifier passed at migration-only, activate → **gen 148**, c29 general, verifier passed (1 199 assignments carried). No `container die` fleet-wide 22:11Z–22:30Z. | +| **Roll 1 complete** | Image census 22:30Z from MIG templates: c8–c10, c13–c16, c19–c29 on `519f4914` (18 cells); c7 on `85bf6799` (the earlier rehearsal image, carries the same fix); existing-only c1–c6, c11, c12 and migration-only c17, c18 untouched by design. No serving cell remains on `5aedbca5`. Selector gen 148, membership unchanged from the start of the roll. Zero relay container exits fleet-wide across the roll (01:14Z–22:30Z). Gates used: #19–#50; freezes were all monitor-side (provenance, freshness, flat Asia latency bar, one Cloud Monitoring collector failure), none a fleet health finding. Roll 2 (fresh image with #18722 + #18720) is the next data-plane step and waits on the owner's private-IP window decision. | diff --git a/cloud/docs/relay-roll2-plan-2026-09.md b/cloud/docs/relay-roll2-plan-2026-09.md new file mode 100644 index 00000000000..f84de39163f --- /dev/null +++ b/cloud/docs/relay-roll2-plan-2026-09.md @@ -0,0 +1,154 @@ +# Relay Roll 2 and close-out plan (2026-09-05) + +Owner-approved scope 2026-09-05: finish the relay reliability work with one more cell image roll, +deferring the Cloud SQL private-IP move (2.1, orca-cloud #477) to a separate owner decision. Roll 1 +is complete (see `relay-reconnect-2026-09-findings.md`, "Roll 1 complete"); every serving cell runs +`519f4914` except c7 on `85bf6799`. + +Estimate: about two working days of effort over one week of calendar time. The cell roll itself is +6 to 7 hours of mostly unattended wall clock, run in the US night. + +## Phase 0. Land the code (half a day, no production change) + +### 0a. Split PR #18565 + +The branch mixes three relay/mobile/desktop fixes with the operator record. Split so the record +lands regardless of how the code review goes. + +- **Docs PR** (new branch off main): `relay-reconnect-2026-09-findings.md`, + `relay-improvement-checklist-2026-09.md`, `relay-improvement-roadmap-2026-09.md`, this file. + Docs only, merge on CI green. +- **Code PR** (rebase #18565 onto main, resolve two conflicts): + - `cloud/apps/relay/src/host-session-registry.ts`: conflict with #18698 (signed-out signal). + Keep both; the accept-abandonment and lease changes are orthogonal to the signed-out path. + - `src/main/runtime/relay/relay-origin-pool.ts`: **drop this branch's version**. #18719 already + merged the desktop early-window jitter (1 to 6 min). Also drop + `relay-session-broker.test.ts` additions that only exercise the dropped change. + - Keep: relay accept abandonment (`orca_relay_client_accept_abandoned` event), relay-side lease + jitter, mobile direct-probe fail-fast, and their tests. + +### 0b. Lengthen the control lease (same code PR) + +In `cloud/apps/relay/src/host-session-registry.ts`: + +``` +CONTROL_LEASE_MS = 6 * 60 * 60 * 1000 // was 55 min +CONTROL_LEASE_JITTER_MS = 30 * 60 * 1000 // was 5 min +``` + +Why 6 h: the lease bounds how long a host stays on a cell after a missed drain and is the only +passive rebalancing; 6 h keeps both and cuts control-activation traffic on the inventory lock by +about 6x. Nothing else depends on it: the relay JWT (5 min) is refreshed by the desktop on its own +schedule and liveness is the 75 s silence watchdog. Wire-safe: the relay sends `leaseExpiresAt` in +the hello ack and old desktops schedule from that value. + +Update the comment above the constants and the three assertions in +`host-session-client-accept.test.ts` that pin the lease arithmetic. Check that nothing in +`cloud/apps/relay-ops` or the monitor thresholds assumes a 55 min rotation period (grep +`55`, `CONTROL_LEASE`, `rotation`). + +### 0c. Review and merge + +Review rounds per the standing process (Opus review, then Codex pass). Merge order: docs PR first +(no dependency), then the code PR. Record the merge SHA of the code PR; that is the Roll 2 image +source. + +## Phase 1. Build and stage the image (half a day) + +Roll 2 image = code PR merge SHA. It carries, relative to `519f4914`: + +| Change | PR | Effect | +|---|---|---| +| Per-cell inventory locks, delta counters | #18722 | Removes the global `relay_cells FOR UPDATE` behind the phone accept hang | +| Relay pool `statement_timeout` 5 s | #18722 | A relay query can no longer hang a cell | +| Accept abandonment | #18565 | Cell stops finishing accepts for phones that already closed | +| Control lease 6 h ± 30 min | #18565 | Fewer, spread-out rebinds | +| `--private-ip` proxy flag support | #18720 | Code only; flag stays unset until 2.1 | + +Steps, in order (from the findings doc's post-merge dispatch plan): + +1. `gh workflow run cloud-publish-relay-production.yml --ref main -f mode=publish`. Resolve the + digest by tag, not from the log: + `gcloud artifacts docker images describe us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay:sha- --format='value(image_summary.digest)'`. +2. Staging: `cloud-deploy-relay-staging.yml` with the new digest; paired phone plus desktop smoke + (connect, background, reconnect). Confirm `orca_relay_client_accept_abandoned` appears only when + a client closes early, and that `sqlLatencyMsMax` no longer pins at the lock timeout. +3. Director: `cloud-deploy-relay-production-director.yml -f image-digest= + -f regional-placement-mode=preserve -f prune-incompatible-revisions=false + -f expected-rehome-generation=12 -f bootstrap-runtime-identity=false + -f predecessor-image-digest=`. Blue/green; prior revision stays as rollback. + Watch director `orca_relay_postgres_transaction_retry` per minute before and after. The director + goes first so the per-cell locks are live before any cell restart burst. +4. Same-cap `verify` mode against c7 with target=, rollback=`519f4914`. Read-only. + +Go/no-go for Phase 2: director serving the new image for at least 30 min, retries per minute at or +below the pre-deploy baseline, no `container die`, no auth 5xx. + +## Phase 2. Roll the cells (one US night, mostly unattended) + +Same machinery as Roll 1: `cloud-monitor-relay-production.yml` dry-run gate, then +`cloud-deploy-relay-production-same-cap.yml`. Cells roll one at a time by design (exact selector +assertions, single Terraform state, and one cell's ~1.2k-host reconnect burst per restart). Do not +add parallelism for this roll. + +Inputs: target=, rollback=`519f4914` (c7: rollback=`85bf6799`). Selector membership is +unchanged from the end of Roll 1 (gen 148; existing-only c1–c6, c11, c12; migration-only c17, c18). + +Order: + +1. **c7 canary** (`canary-apply`, protocol 1). c7 is the rehearsal cell and the only one not on + `519f4914`. +2. **c8 canary**, then **batch c9, c10, c13, c14**. +3. **c15 canary**, then **batch c16, c19, c20, c21**. +4. **c22 canary**, then **batch c23, c24, c25, c26**. +5. **Asia c27, c28, c29** as three single canaries at protocol 0 (`PROTO=0`). Batch mode cannot + take Asia cells yet and needs at least two cells. + +Each batch needs a same-commit canary authority; each wave needs a fresh 15 min gate. Use the +chain script pattern from Roll 1 (wait gate green, check trusted-path ancestry, dispatch within 5 min, +log `CANARY `) under `caffeinate -i`. Budget: 11 to 13 min per cell plus 15 min per gate, +about 6 to 7 h total. + +Per wave checks (same as Roll 1): transition verifier passes at migration-only and again at general +with assignments carried; no `container die` fleet-wide; selector generation advances by exactly 2 +per cell. After the Asia cells: image census from MIG templates; every general cell on the new digest. + +Failure handling: a failed canary re-enters through `mode=rollback` with rollback-digest = desired +image (Roll 1 c27 pattern). A gate freeze on an Asia latency probe despite the 4 000 ms bar is a +stop-and-investigate, not a retry. Monitor-side freezes (freshness, continuity deadline) re-gate +after a 2 min back-off; the chain does this on its own. + +Record every gate and wave in the findings doc as in Roll 1. + +## Phase 3. After the roll (spread over the following week) + +- **4.4 Recalibrate the retries bar.** After one week of `orca_relay_postgres_transaction_retry` + on the new image, re-derive the `postgres_retries` monitor threshold from the new baseline + (PR against `cloud/apps/relay-ops/src/incident-monitor.ts` thresholds). About 2 h. +- **1.2 Pruner budget.** Raise `auth_token_pruner_max_rows_per_run` to the default 200k after a + clean day; watch Cloud SQL write MB/s and the checkpoint alert. Then **1.5** log metric plus + policy on `stopReason != complete`. +- **1.3 Reclaim.** Once pruner runs delete ~0 rows: `pg_repack -t refresh_tokens` off-peak (check + `pg_available_extensions` first; not `VACUUM FULL`). Confirm table, index, and `disk/utilization` + dropped. +- **Monitor residuals** already in the checklist: `probeEndpointHealth` retry decision still uses the + flat 2 000 ms bar; operator protocol unbound for Asia; `probe-relay-rehome-trust` regex. +- Update the checklist status header; tick 2.3, 4.1, 4.3 relay-side as deployed. + +## Deferred, owner decision required + +- **2.1 Private IP** (orca-cloud #477). One-way door with a Cloud SQL restart. When chosen: apply the + foundation off-peak, then a template-only change that sets the `--private-ip` proxy flag. That is + another cell roll unless bundled with a future image. +- **5.2 Paging channel** for auth alerts: needs a destination. +- **Parallel cell rolls** (2 or 3 at a time): about 1.5 days (relax exact-selector assertions to + "exact except in-flight", single coordinator Terraform apply, parallel job shape, tests). Only + worth building if more image rolls are planned after Roll 2, and only once the per-cell locks are + live so a multi-cell reconnect burst is safe. +- **2.2 Database split**: deferred to ~2026-11-01. + +## Not in this plan + +Desktop and mobile changes already merged (#18719 desktop early-window jitter and no same-token +refresh retry; #18565 mobile fail-fast once merged) ship with the next desktop and mobile releases +on their own schedules. No relay action needed. diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 4a6b0299685..6fc938fe986 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -10,6 +10,102 @@ } }, "gates": [ + { + "id": "terminal-output.prestarted-shell-snapshot-adoption", + "title": "Prestarted shell adoption paints covered output once", + "maturity": "experimental", + "protection": "partial", + "owner": "terminal-runtime", + "layer": "renderer-transport-and-live-electron", + "surfaces": [ + "backend-created first terminal", + "daemon snapshot adoption", + "deferred live output" + ], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "wsl", "ssh", "remote-runtime"], + "coveredPlatforms": ["macos", "linux", "windows"], + "coveredProviders": ["local", "daemon", "wsl"], + "coverageNotes": "macOS daemon-backed Electron journey verifies same PID and terminal identity plus rendered output. Focused renderer contracts pass on Linux, Windows and WSL. Neighboring SSH model and replay contracts pass locally; no new live SSH or paired-runtime journey.", + "motivatingLinks": [ + "https://github.com/user-attachments/assets/e8c6d1dc-6150-4c3d-b55a-3d12efefdd04", + "https://github.com/user-attachments/assets/b0328f88-34ac-4d51-8119-9efe17072435" + ], + "invariant": "Adopting a prestarted terminal preserves its existing process and paints snapshot-covered startup output once while retaining subsequent live output. Missing sequence proof or blank snapshots must not authorize dropping output.", + "oracle": "Pass snapshot sequence and proven zero keyboard flags through real IPC transport projection. Deliver snapshot-covered and newer output before reattach resolves; drain replay parse callbacks and require one startup marker and the newer output. Repeat with no sequence and blank snapshot to retain unproven bytes. In Electron select a prestarted workspace, type a generated marker and compare PID and stable terminal identities before and after.", + "commands": [ + "pnpm test src/renderer/src/components/terminal-pane/pty-transport-connect-spawn.test.ts src/renderer/src/components/terminal-pane/pty-connection-deferred-reattach-live-output.test.ts", + "pnpm test src/renderer/src/components/terminal-pane/pty-transport-connect-spawn.test.ts src/renderer/src/components/terminal-pane/pty-connection-deferred-reattach-live-output.test.ts src/renderer/src/components/terminal-pane/pty-connection-hidden-snapshot-live-overlap.test.ts src/renderer/src/components/terminal-pane/pty-connection-replay-payload-handling.test.ts src/renderer/src/components/terminal-pane/pty-connection/reattach-payload-ssh-reconnect-model-paint.test.ts", + "pnpm test src/renderer/src/components/terminal-pane/pty-connection src/renderer/src/components/terminal-pane/pty-transport" + ], + "testFiles": [ + "src/renderer/src/components/terminal-pane/pty-transport-connect-spawn.test.ts", + "src/renderer/src/components/terminal-pane/pty-connection-deferred-reattach-live-output.test.ts" + ], + "assertionRefs": [ + { + "file": "src/renderer/src/components/terminal-pane/pty-transport-connect-spawn.test.ts", + "assertions": [ + "zero and nonzero snapshot sequence and proven zero keyboard flags survive IPC projection" + ] + }, + { + "file": "src/renderer/src/components/terminal-pane/pty-connection-deferred-reattach-live-output.test.ts", + "assertions": [ + "startup output covered by the snapshot is painted once", + "new output remains visible", + "legacy unsequenced and blank snapshots retain bytes" + ] + } + ], + "evidenceRuns": [ + { + "date": "2026-09-04", + "runner": "local", + "platform": "macos", + "command": "pnpm test src/renderer/src/components/terminal-pane/pty-transport-connect-spawn.test.ts src/renderer/src/components/terminal-pane/pty-connection-deferred-reattach-live-output.test.ts src/renderer/src/components/terminal-pane/pty-connection-hidden-snapshot-live-overlap.test.ts src/renderer/src/components/terminal-pane/pty-connection-replay-payload-handling.test.ts src/renderer/src/components/terminal-pane/pty-connection/reattach-payload-ssh-reconnect-model-paint.test.ts", + "result": "passed", + "durationSeconds": 2.34, + "summary": "5 suites / 48 tests pass. Focused 2-suite runs independently pass 27 tests on Linux, Windows and WSL." + }, + { + "date": "2026-09-04", + "runner": "local", + "platform": "macos", + "command": "pnpm test src/renderer/src/components/terminal-pane/pty-connection src/renderer/src/components/terminal-pane/pty-transport", + "result": "passed", + "durationSeconds": 5.67, + "summary": "Broader connection/transport gate: 77 files and 813 tests passed, including neighboring restore, reconnect, input and replay behavior. Log: artifacts/worktree-create/orca-draft-replay-broader-gate.log." + } + ], + "runtimeBudget": { + "p95Seconds": 15, + "scope": "focused renderer transport and deferred-adoption contracts" + }, + "flakeHistory": { + "status": "unknown", + "evidence": "Focused local and remote runs pass; no CI soak history." + }, + "redGreenEvidence": { + "status": "partial", + "evidence": "Metadata tests fail before forwarding. Corrected parse-draining regression observes two startup markers when the baseline installation is removed, and one after restoration. Initial missing-live-output failure was a harness parse-drain omission and is not red proof. Before/fixed Electron screenshots show duplicate/single startup output." + }, + "performanceBudget": { + "required": true, + "evidence": "Reuses existing snapshot baseline reconciliation with no new scan, timer or subprocess. Corrected daemon-backed rendered trial reaches replay at 116.6 ms and generated keyboard output at 177 ms after selecting the prestarted workspace. This measures selection/adoption, not ordinary composer creation." + }, + "promotionCriteria": [ + "Meet manifest CI and soak policy.", + "Retain intentional-break and rendered identity/output proof.", + "Exercise live SSH and paired-runtime snapshot adoption before claiming full provider coverage." + ], + "knownGaps": [ + "Composer draft creation and cancellation are not implemented by this gate.", + "No new live SSH, Windows or WSL UI run; remote evidence is focused contract tests.", + "Mixed-version snapshots without sequence proof intentionally retain legacy behavior." + ], + "demotionRule": "Keep experimental or demote if adoption duplicates covered output, drops newer or unproven output, changes terminal ownership, or flakes without explanation." + }, { "id": "cmd-j-tabs.host-qualified-candidate-ownership", "title": "Cmd-J tab candidates retain execution-host ownership", diff --git a/config/scripts/pr-code-change-scope.test.mjs b/config/scripts/pr-code-change-scope.test.mjs index 4642372135c..f31822e5b93 100644 --- a/config/scripts/pr-code-change-scope.test.mjs +++ b/config/scripts/pr-code-change-scope.test.mjs @@ -414,10 +414,11 @@ describe('PR Checks skip wiring', () => { }) it('skips e2e detection on docs-only PRs without dropping the draft gate', () => { - expect(prWorkflow.jobs['e2e-paths'].needs).toEqual(['code_paths']) - expect(prWorkflow.jobs['e2e-paths'].if).toBe( - "github.event.pull_request.draft != true && needs.code_paths.outputs.should_run == 'true'" + const filter = prWorkflow.jobs.code_paths.steps.find((step) => step.id === 'e2e_filter') + expect(filter.if).toBe( + "github.event.pull_request.draft != true && steps.filter.outputs.should_run == 'true'" ) + expect(prWorkflow.jobs['e2e-paths']).toBeUndefined() }) it('lets verify pass skipped jobs the classifier turned off', () => { diff --git a/config/scripts/pr-e2e-gate-contract.test.mjs b/config/scripts/pr-e2e-gate-contract.test.mjs index ceac6b8cc6e..67e271868df 100644 --- a/config/scripts/pr-e2e-gate-contract.test.mjs +++ b/config/scripts/pr-e2e-gate-contract.test.mjs @@ -39,7 +39,7 @@ const nativeImeSpec = readFileSync( 'utf8' ) -const filterStep = prWorkflow.jobs['e2e-paths'].steps.find( +const filterStep = prWorkflow.jobs.code_paths.steps.find( (step) => step.name === 'Filter changed E2E specs' ) const rollbackStep = prWorkflow.jobs.static_analysis.steps.find( @@ -106,16 +106,16 @@ describe('PR E2E gate contract', () => { // Why: without this the job could lose its filter and run on every PR — the // cost the path filter exists to avoid — while the gate assertions above // stay green. - expect(prWorkflow.jobs.e2e.needs).toBe('e2e-paths') - expect(prWorkflow.jobs.e2e.if).toBe("needs.e2e-paths.outputs.should_run == 'true'") - expect(prWorkflow.jobs['e2e-paths'].outputs.should_run).toBe( - '${{ steps.filter.outputs.should_run }}' + expect(prWorkflow.jobs.e2e.needs).toBe('code_paths') + expect(prWorkflow.jobs.e2e.if).toBe("needs.code_paths.outputs.e2e_should_run == 'true'") + expect(prWorkflow.jobs.code_paths.outputs.e2e_should_run).toBe( + '${{ steps.e2e_filter.outputs.should_run }}' ) - expect(prWorkflow.jobs['e2e-paths'].outputs.test_files).toBe( - '${{ steps.filter.outputs.test_files }}' + expect(prWorkflow.jobs.code_paths.outputs.test_files).toBe( + '${{ steps.e2e_filter.outputs.test_files }}' ) expect(prWorkflow.jobs.e2e.with.ref).toBe('${{ github.event.pull_request.head.sha }}') - expect(prWorkflow.jobs.e2e.with.test_files).toBe('${{ needs.e2e-paths.outputs.test_files }}') + expect(prWorkflow.jobs.e2e.with.test_files).toBe('${{ needs.code_paths.outputs.test_files }}') }) it('enforces every job verify depends on', () => { @@ -360,11 +360,11 @@ describe('PR E2E gate contract', () => { expect(sshLaneCondition).toContain("inputs.ssh_source_changed == 'true' ||") expect(e2eWorkflow.on.workflow_call.inputs.ssh_source_changed.type).toBe('string') - expect(prWorkflow.jobs['e2e-paths'].outputs.ssh_source_changed).toBe( - '${{ steps.filter.outputs.ssh_source_changed }}' + expect(prWorkflow.jobs.code_paths.outputs.ssh_source_changed).toBe( + '${{ steps.e2e_filter.outputs.ssh_source_changed }}' ) expect(prWorkflow.jobs.e2e.with.ssh_source_changed).toBe( - '${{ needs.e2e-paths.outputs.ssh_source_changed }}' + '${{ needs.code_paths.outputs.ssh_source_changed }}' ) expect(filterStep.run).toContain('pr-e2e-source-routing.mjs --ssh-source') expect(filterStep.run).toContain('ssh_source_changed=$SSH_SOURCE_CHANGED') @@ -565,12 +565,12 @@ describe('PR E2E gate contract', () => { expect(prWorkflow.jobs.terminal_ime_native.uses).toBe( './.github/workflows/terminal-ime-e2e.yml' ) - expect(prWorkflow.jobs.terminal_ime_native.needs).toBe('e2e-paths') + expect(prWorkflow.jobs.terminal_ime_native.needs).toBe('code_paths') expect(prWorkflow.jobs.terminal_ime_native.if).toBe( - "needs.e2e-paths.outputs.native_ime_source_changed == 'true'" + "needs.code_paths.outputs.native_ime_source_changed == 'true'" ) - expect(prWorkflow.jobs['e2e-paths'].outputs.native_ime_source_changed).toBe( - '${{ steps.filter.outputs.native_ime_source_changed }}' + expect(prWorkflow.jobs.code_paths.outputs.native_ime_source_changed).toBe( + '${{ steps.e2e_filter.outputs.native_ime_source_changed }}' ) expect(filterStep.run).toContain('pr-e2e-source-routing.mjs --native-ime-source') expect(filterStep.run).toContain('native_ime_source_changed=$NATIVE_IME_SOURCE_CHANGED') diff --git a/config/scripts/run-electron-vite-dev.mjs b/config/scripts/run-electron-vite-dev.mjs index dfb0a0aceb7..c520083cb6a 100644 --- a/config/scripts/run-electron-vite-dev.mjs +++ b/config/scripts/run-electron-vite-dev.mjs @@ -616,7 +616,7 @@ if (!isHelpOrVersion && process.env.ORCA_DEV_INSTANCE_LABEL) { // Why: automation launches this app while someone is working; announce that the // window will come up without taking the foreground so the mode is visible in logs. if (!isHelpOrVersion && process.env.ORCA_BACKGROUND_LAUNCH === '1') { - console.error('[orca-dev] Background launch: window shows without stealing focus') + console.error('[orca-dev] Background launch: window stays off screen; automate through CDP') } let forwardedExtras = [] if (!userPassedPort && !isHelpOrVersion) { diff --git a/docs/reference/ci-runner-efficiency.md b/docs/reference/ci-runner-efficiency.md new file mode 100644 index 00000000000..e569f749102 --- /dev/null +++ b/docs/reference/ci-runner-efficiency.md @@ -0,0 +1,99 @@ +# CI efficiency and runner capacity + +Audit date: September 5, 2026. No paid capacity or provider configuration changed. + +## Measurements and changes + +Three recent successful PR runs used 54.6–64.9 aggregate runner minutes: +[33998366568](https://github.com/stablyai/orca/actions/runs/33998366568), +[33998220287](https://github.com/stablyai/orca/actions/runs/33998220287), and +[33998181502](https://github.com/stablyai/orca/actions/runs/33998181502). +These are sums of active job durations, excluding skipped jobs; they are not +billing minutes or queue time. This small sample is not a historical average. + +- Consolidate E2E routing into the existing code-path detector. The removed + detector occupied 20–22 seconds and required another runner allocation and + full-history checkout per nondraft code PR. The same routing commands remain, + including SSH and native IME selection; actual E2E results remain advisory. + A routing-script error now fails the required code-path detector. +- Use gzip for PR-only Debian/RPM artifacts. The two sampled Linux packaging + jobs took 8m10s and 8m19s overall; one spent 3m47s in electron-builder. Its + default Debian/RPM compression is xz. PR artifacts are inspected on the same + runner, so their download size offers no benefit. Keep all AppImage, Debian, + RPM, payload, launcher, and shutdown checks. Release compression is unchanged. + Compression savings need a hosted run; do not equate the full packaging step + with removable compression time. +- Cancel superseded Mobile Checks and Skill update round-trip PR runs. The + skill matrix has 13 jobs. Preserve non-cancelling main/merge-group skill runs, + with separate concurrency groups per event. +- Reuse the existing script-free root dependency action in Mobile Checks, + including the pnpm cache keyed by both root and mobile lockfiles. The root + install remains necessary because mobile types import root dependencies. + +The repository already has eight unit shards, path-scoped platform checks, +native caches, one shared E2E build, PR cancellation, incremental TypeScript +caching, and changed-spec E2E routing. Increasing shards would increase setup +work and simultaneous runner demand. Do not adjust the count without comparing +critical-path time and aggregate job time on the same commit. + +## Runner recommendations + +The repository is **public**, verified using the GitHub API. Standard +GitHub-hosted Linux, Windows, and macOS runners have free compute minutes for +public repositories. Queue pressure and third-party provider allowances still +matter; artifact storage and larger runners have separate billing rules. +See [GitHub Actions billing](https://docs.github.com/en/billing/concepts/product-billing/github-actions). + +1. Keep standard GitHub-hosted runners as the default. Ask GitHub Support for a + higher concurrent-job limit before paying for more capacity. The documented + standard limits depend on the account plan (Free: 20 total/5 macOS; Team: + 60/5; Enterprise: 500/50), and increases are subject to approval. The actual + account entitlement was not verified. See [limits](https://docs.github.com/en/actions/reference/limits). +2. Reserve existing Blacksmith allowance for macOS if that is the priority. + Blacksmith documents 3,000 free x64 2-vCPU-equivalent minutes per organization; + a 6-vCPU Mac minute consumes 20 equivalents, or 150 actual Mac minutes if + it uses the entire free pool. Cloud workflows also use Blacksmith Linux. + Moving Linux to hosted GitHub saves shared allowance, but does not necessarily + free Mac hardware capacity. Account-specific contracts and usage were not + inspected. See [Blacksmith runners](https://docs.blacksmith.sh/blacksmith-runners/overview). +3. Treat Ubicloud as an optional small Linux overflow trial. Its documented + $2.50 monthly credit buys 1,250 premium 2-vCPU minutes at $0.002/minute, or + 2,000 standard 2-vCPU minutes at $0.00125/minute. New accounts default to + premium and require a credit card. No enforceable hard spending cap was + verified, so changing runner labels cannot guarantee the no-spend constraint. + One PR's roughly 55–65 runner minutes also makes clear how small this pool + is relative to repository activity (hardware speeds differ). + See [pricing](https://ubicloud.com/docs/about/pricing) and + [setup](https://ubicloud.com/docs/github-actions-integration/quickstart). + +## Machines that also run coding agents + +Do not register the credentialed host directly as a public-PR runner. A PR can +execute arbitrary build/test code, and a persistent host lets it access local +credentials or affect subsequent jobs. Docker alone is not adequate isolation +when it exposes the host home, Docker socket, SSH agent, or office network. + +A possible no-new-hardware experiment is a disposable VM per job, preferably on +a dedicated spare machine, with a just-in-time single-job runner, no shared +home/keychain/SSH agent or host mounts, restricted network access, and CPU/RAM +limits that leave room for coding agents. Destroy the VM after every job; +ephemeral runner registration by itself does not clean the machine. Start with +trusted branch/manual workloads and keep public fork PRs on hosted runners. +Provisioning and ongoing patching are real operational costs even when the +machine is already owned. See GitHub's +[self-hosted runner security guidance](https://docs.github.com/en/actions/security-for-github-actions/security-guides/security-hardening-for-github-actions). + +## Release waits + +The latest successful sampled Windows release used 13m59s of a 21m56s job in +signing wait/download steps. The same release held an Ubuntu job for 11m38s +polling the isolated Mac build. These are stronger occupancy opportunities than +small checkout savings, especially when approval takes hours. + +[Windows signing without occupying a runner](windows-signing-runner-time.md) +describes a staged, same-run design, required protected environments, and +rehearsal criteria. No callback integration or protected Windows signing +environments currently exist. An environment-gated design adds a GitHub +approval after each SignPath approval and changes the current automatic inner +signing timeout fallback; those are explicit release-policy decisions, so this +PR leaves production signing behavior unchanged. diff --git a/docs/reference/windows-signing-runner-time.md b/docs/reference/windows-signing-runner-time.md new file mode 100644 index 00000000000..fb02ba6bdca --- /dev/null +++ b/docs/reference/windows-signing-runner-time.md @@ -0,0 +1,137 @@ +# Windows signing without occupying a runner during approval + +Status: implementation proposal; production signing behavior is unchanged. + +## Measured cost + +In [release run 33821033674](https://github.com/stablyai/orca/actions/runs/33821033674) +(September 4, 2026), the Windows job took 21m56s. The inner-binary download step +took 13m19s and the installer download step took 40s: 13m59s, or 64% of the job, +was spent in the signing download/wait steps. These durations include the +download itself, so they are an upper bound on removable idle time, not a +prediction of net savings after transferring state between jobs. + +`release-cut.yml` submits both requests with `wait-for-completion: false`, but +then invokes `Get-SignedArtifact` on the same Windows runner with one-hour and +four-hour completion timeouts. The six-hour job timeout accommodates both +waits. Changing the submission flag again, polling less often, or running the +wait inside a container does not release the runner slot. + +This is runner occupancy, not a billing estimate. Standard GitHub-hosted +runners in a public repository may be free; removing the waits still releases +concurrency for other work. Check actual billing before assigning dollar savings. + +The same release also occupied an Ubuntu runner for 11m38s while +`run-release-mac-build-workflow.mjs` waited on the isolated macOS workflow. +That is a separate orchestration optimization. Windows development-channel +builds deliberately ship unsigned and have no SignPath wait to remove. + +## Proposed execution graph + +Keep all Windows stages in the original `release-cut.yml` run to preserve the +current SignPath GitHub artifact provenance boundary: + +1. `build-windows` builds and uploads the unpacked app, original installer, + updater metadata, and inner-signing manifest. It submits the inner request, + sends the existing notification, exposes the request ID, and finishes. +2. `package-windows` depends on that job and uses a protected environment named + `windows-inner-signing`. Its runner is allocated only after GitHub approval. + It restores the exact build, downloads the signed binaries with a short, + bounded completion wait, applies the existing signature restoration and + signed `elevate.exe` cache replacement, builds the NSIS installer, uploads it, + submits the second signing request, notifies approvers, and finishes. +3. `finalize-windows` depends on packaging and uses a second protected environment + named `windows-installer-signing`. After approval it downloads the signed + installer, regenerates its blockmap and `latest.yml`, runs existing outer and + inner signature checks, uploads evidence, and uploads the assets to the draft. +4. `publish-release` depends on finalization as well as the existing Linux, macOS, + and blocking release gates. It remains the only job that publishes the draft. + +The approver signs in SignPath, waits for that request to finish, and then +approves the corresponding pending GitHub job. Each notification should link +to both places and explain the order. GitHub approval is an extra action; +approving in SignPath alone does not release an environment gate. + +## Required configuration + +The repository environments were inspected through the GitHub API on +September 5, 2026. Neither Windows environment exists. `adhoc-mac-build` has no +protection rules; it cannot be reused as an approval gate. No SignPath callback +handler was found in the repository's workflows, scripts, application, or cloud +code. + +Before enabling the graph: + +1. Create both environments in repository Settings → Environments. +2. Add the release approvers as required reviewers for each environment. Decide + whether a release initiator may approve their own job, and configure that + consistently with the existing SignPath policy. +3. Restrict deployment branches to the trusted refs used to dispatch release + workflows, and check that the release workflow's ref passes the restriction. + The workflow ref and the checked-out release tag are different concepts. +4. Read back both environments through the API and verify that + `required_reviewers` rules exist before changing the release graph. Merely + referring to a new environment name in YAML can create an unprotected + environment and silently leave the wait on the runner. +5. Add a preflight assertion for those rules so accidental removal fails before + any signing request is submitted. Verify the API access required for this + assertion using the release workflow's token; do not assume an administrator's + local `gh` access proves workflow-token access. + +An automatic alternative requires a SignPath completion callback and an +authenticated integration that releases the corresponding deployment gate. +Confirm the Foundation plan supports the necessary callback before choosing +that architecture. Do not introduce a long-running GitHub polling job as the +callback substitute: it would continue occupying a slot. + +## State and failure contracts + +- Use artifacts from this exact run and attempt, with a manifest containing the + tag, tag commit SHA, workflow SHA, request IDs, artifact IDs, and SHA-256 hashes. + Artifact names alone are insufficient. Preserve the original unsigned + installer for the existing inner-signing fallback. +- Restore `dist/win-unpacked`, the staging list, the installer, and updater + metadata as one checkpoint. Use an archive to preserve the tree. Do not ship + a fresh rebuild of the app after approving a different binary tree. +- Each new Windows runner needs the pinned Node/pnpm toolchain, build + dependencies, SignPath module, and electron-builder tool cache. The second + runner must populate the NSIS cache before replacing `elevate.exe`; the old + code assumes the first installer build already populated that cache. +- Retain checkout-from-tag behavior and the existing support for release tags + that predate the composite action. Explicitly restore new orchestration code + from the workflow SHA when necessary. +- Preserve the rule that rerunning a workflow never submits a new signing + request. A resume must consume the recorded request and artifacts. Test failed + stage reruns, whole-workflow reruns, and missing/expired checkpoints separately. +- Keep installer signature checks blocking. Keep inner verification evidence + and its current warning-only policy unless changed in a separate decision. +- Resolve the current one-hour inner-signing fallback deliberately: an + environment approval can remain pending longer than one hour and rejection + skips dependent jobs. It cannot reproduce the existing automatic timeout + fallback by itself. A first migration should explicitly document the new + manual release/cancellation behavior; silently treating rejected approval as + permission to ship is not acceptable. +- Keep the release-wide concurrency lock while the graph waits, preventing + another release from overtaking this draft. This saves worker occupancy, but + does not shorten the serialized release queue's human approval time. + +## Validation before production + +First adapt `windows-signing-rehearsal.yml` to exercise the same staged code +using the auto-approved test-signing policy. Then run a manual rehearsal with +the protected environments and confirm that pending approval has no allocated +Windows runner. Verify signed bytes through the existing extraction-based +installer checks, not only the outer installer signature. + +Cover approval before SignPath completion, rejected approval, missing signed +files, changed checkpoint hashes, lost checkpoints, expired artifacts, failed +packaging, and stage reruns without duplicate submissions. Confirm no release +becomes public until all platform and signature gates pass. Compare transferred +artifact/setup time with the original 13m59s wait sample to measure net savings. + +A separate `workflow_dispatch` continuation can avoid environment provisioning, +but changes this design substantially: the original release run finishes, +workflow-level concurrency no longer protects the pending draft, and SignPath +must accept artifacts assembled from a prior run. That option needs a durable +release state machine and provenance validation before production use; it is +not a drop-in replacement for the two download steps. diff --git a/electron.vite.config.ts b/electron.vite.config.ts index 4ed4641cde1..90dc637c204 100644 --- a/electron.vite.config.ts +++ b/electron.vite.config.ts @@ -253,6 +253,9 @@ export const electronViteConfig: UserConfig = { 'agent-hooks/managed-agent-hook-controls': resolve( 'src/main/agent-hooks/managed-agent-hook-controls.ts' ), + 'codex/managed-home-shell-preflight': resolve( + 'src/main/codex/managed-home-shell-preflight.ts' + ), // Why: account import mutates the user's macOS Keychain from the CLI. 'claude-accounts/keychain': resolve('src/main/claude-accounts/keychain.ts') }, diff --git a/skill-guides/orchestration/references/recovery-and-cleanup.md b/skill-guides/orchestration/references/recovery-and-cleanup.md index 41a62620ac3..a07c40e66bf 100644 --- a/skill-guides/orchestration/references/recovery-and-cleanup.md +++ b/skill-guides/orchestration/references/recovery-and-cleanup.md @@ -93,6 +93,20 @@ When a worker's terminal accepted input but the submit is unconfirmed, use `terminal send --wait-submit `: it observes the accepted prompt for that long and, on timeout, returns the input-accepted receipt without resending. +## Refused starts + +`dispatch` and `worker-start` refuse the following preflight cases with a stable +`error.code`; read it before choosing a recovery, and treat `error.data.nextSteps` +as the exact recovery text. Older hosts may omit `data`, so treat every field as +optional. + +| Code | Meaning | Recovery | +| -------------------- | --------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | +| `task_not_found` | No Task with that id, or not in the bound Run (`data.taskId`, `data.runId`) | Check `task-list --json`; create the Task with `task-create` if it does not exist | +| `task_not_startable` | Task cannot start now: not `ready`, or invalid `--retry-of` (`data.status`, `data.unmetDependencies`, `data.retryOf`) | Wait for running dependencies with `check --wait`; retry or unblock failed ones; inspect `dispatch-show` if already dispatched | +| `inject_rejected` | `--inject` refused because no recognized agent runs in the target (`data.terminal`, `data.reason`) | Start a recognized agent there or pick another terminal; or dispatch without `--inject` and use `terminal send` | +| `runtime_error` | Any other failure, including a target terminal that already owns an active Dispatch | Read the message, inspect state, and do not retry unchanged | + ## Retry, stop, and abandon Retry only a positively proven failed or stopped attempt. Placement is never diff --git a/src/cli/bundled-skill-guides.ts b/src/cli/bundled-skill-guides.ts index 4ebfffe644c..1fb195bdf36 100644 --- a/src/cli/bundled-skill-guides.ts +++ b/src/cli/bundled-skill-guides.ts @@ -39,7 +39,7 @@ const ORCA_PER_WORKSPACE_ENV_MARKDOWN = "---\nname: orca-per-workspace-env\ndesc const ORCHESTRATION_MARKDOWN = "---\nname: orchestration\ndescription: >-\n Coordinate supervised Orca workers: threaded messages, blocking ask/reply,\n task dispatch, worker_done/escalation waits, task DAGs, decision gates,\n coordinator loops, and decomposing work across agents. Use `orca-cli` for full\n ownership handoffs — \"hand off\", \"handoff\", \"handover\", \"give this to another\n agent\", \"another worktree\" — unless asked to supervise, monitor, or coordinate\n a DAG, and for terminal control, lightweight terminal prompts, shell commands,\n Orca worktree management, and reading or waiting on terminals. Use Computer\n Use for external browser windows, webviews, Orca app UI, or desktop UI outside\n Orca's embedded browser only when the task requires OS/window-level control\n such as focus, menus, dialogs, coordinates, or screenshots. Use `orca-cli` for\n Orca's embedded pages and a page-automation tool such as Playwright or CDP for\n external pages.\n---\n\n# Orca orchestration\n\nOrchestration is Orca's structured coordination layer. It records who owns work,\nwhich attempt is authoritative, and when supervised work has settled.\n\n## Outcome\n\n**Result:** every in-scope Task has one explicit outcome and every settled worker\nterminal has a next owner or cleanup decision. **Next consumer:** the user who\nrequested supervision. **Done:** all expected Dispatches have settled, every\ndelivered message was processed before acknowledgment, each settled worker was\nreused, explicitly retained, or released, and the turn ends only when the report\nto that user names, per Task, its outcome, the evidence behind it, and any\nunresolved blocker.\n\n**Safe failure:** preserve work and authority and report the state as unknown or\n`unverifiable`. Only positive proof of exit authorizes stop, abandon, or retry,\nand only an accepted settlement authorizes release. Every other observation,\nabsence included, is a checkpoint.\n\n## Classify the role\n\n| Current context | Role | Route |\n| ---------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------- | ------------------------------------------------------------------------------ |\n| The user explicitly asks to supervise, monitor, wait for results, track completion, coordinate a DAG, use a decision gate, or manage ask/reply | Coordinator | Use the supervised loop below |\n| The current prompt contains a live injected preamble with Task and Dispatch IDs | Dispatched worker | Follow the preamble and the worker obligations below |\n| The user asks to hand off ownership or start another agent/worktree without supervision | Handoff owner | Use `orca-cli`; create no Run, Task, or Dispatch and do not monitor completion |\n| A message carries a legacy authority label | Compatibility operator | Load the legacy contract reference before any lifecycle mutation |\n| No live preamble and no explicit supervision | Ordinary terminal agent | Do not emit lifecycle messages; use `orca-cli` for terminal/worktree work |\n\nModel or effort selection does not make a handoff supervised. Never substitute a\nnon-Orca subagent tool when Orca orchestration provenance was requested.\n\n## Authority and safety floor\n\n- A Run is a durable namespace and coordinator inbox; it does not schedule or\n place workers. A Task is work. A Dispatch is one authoritative Task attempt.\n- Lifecycle authority comes from the active Dispatch, not a terminal title,\n copied ID, old database row, provider transcript, or visible pane.\n- Workers use the exact executable, handle, capability, Task ID, and Dispatch ID\n in the live preamble. Never reconstruct, translate, or broaden those arguments.\n- After remote start, address the worker by Dispatch ID. The execution host owns\n process, filesystem, transcript, stop, and cleanup facts. Preserve the verdicts\n `live` / `unverifiable` / `exited`; contact loss is not process death.\n- Liveness is layered: `worker-list`'s `projection.liveness` is the fleet verdict\n for the agent; `worker-show`'s `observation.status` is PTY liveness only. A live\n terminal can still hold a dead or stuck agent.\n- Folder workspaces are valid; never require Git or assume a worktree.\n- Clients and remote servers update independently. Treat unknown optional fields\n as absent. A new stream operation requires advertised capability because old\n decoders may silently drop unknown opcodes. Never fall back to local execution\n when remote authority or capability is unproven.\n- Use the executable you used to run `skills get` for the entire run. In the\n examples below, replace `ORCA` with it; do not create a shell variable or run\n `ORCA` literally. If it fails, report that exact error instead of switching.\n- A successful `orchestration send` proves durable enqueue; its wake or nudge is\n best-effort attention only and does not prove the recipient read or accepted it.\n\n## Worker obligations\n\nThe injected preamble is authoritative. A dispatched worker must:\n\n1. Do only the current Task and use the preamble's `ask` command for a blocking\n coordinator question. Never open a local question TUI the coordinator cannot\n answer. Resume the same message ID after an ask timeout.\n2. Send heartbeats only at the cadence in the preamble. A heartbeat proves\n liveness, not completion.\n3. Read coordinator follow-ups at each natural checkpoint — before starting a\n new file, after a test run — and once more immediately before `worker_done`:\n `ORCA orchestration check --terminal --json`.\n4. Send `worker_done` exactly once, from the dispatched terminal, with a\n three-sentence executive summary, both lifecycle IDs, and explicit\n `--outcome succeeded` or `--outcome failed`. Never encode failure only in prose.\n5. Append `--files-modified` and `--report-path` only with real values when\n applicable. After `worker_done`, end the dispatched turn and idle; do not poll\n or start new work.\n\nA direct user instruction after completion starts new user-owned work and takes\nprecedence over the idle rule. Do not reuse the settled lifecycle IDs.\n\n## Canonical supervised loop\n\nConfirm the runtime, bind one Run, and start the full independent wave before\nwaiting. `worker-start --spec` creates the Task and its attempt in one call:\n\n```text\nORCA status --json\nORCA orchestration run-create --objective \"\" --json\nORCA orchestration worker-start --spec \"\" --worktree current --agent codex --json\nORCA orchestration worker-start --spec \"\" --worktree current --agent claude --json\nORCA orchestration check --wait --types \"worker_done,escalation,question\" --timeout-ms 900000 --json\n```\n\nIf `worker-start` exits non-zero, do not relaunch. Read the receipt's\n`failedStage` and `residualResources`, then load\n`references/recovery-and-cleanup.md`.\n\nUse `task-create` plus `worker-start --task ` for planned fan-out with\ndependencies or a retry of a known Task. Use dependencies only for real ordering\nand prefer parallel waves over chains deeper than three or four steps; nested\nworkers obey the depth limit, and a new Run does not reset the caller's depth.\n\nA consuming `check` names its caller with `--terminal `, never `--from`;\nomit it inside the coordinator's own Orca terminal. It returns the bound Run's\noldest FIFO Delivery and replays that batch until acknowledged. Process every\nmessage: reply to questions, validate each `worker_done` against the expected\nactive Dispatch, and decide each settled terminal's next owner before the ack:\n\n```text\nORCA orchestration reply --id --body \"\" --json\nORCA orchestration worker-release --dispatch --json\nORCA orchestration check --ack --wait --types \"worker_done,escalation,question\" --timeout-ms 900000 --json\n```\n\nKeep waiting until every expected Dispatch settles. A timeout or empty result is\na checkpoint, not a failure. Do not stop, retry, release, or launch a duplicate\neditor without the positive proof `## Outcome` requires.\n\nAfter three consecutive empty waits, stop waiting blindly and enumerate with\n`ORCA orchestration worker-list --include-remote --json` (defaults to the bound\nRun; `--run ` overrides; the receipt's `scope` names which), acting on\neach row's `projection.attention` categories, `projection.attention.requiresAction`, and literal `projection.nextAction` argv.\nAn `inspect` `nextAction` on a `live` row with `attention.requiresAction` false\nis informational, not a command to re-run: keep waiting with `check --wait`.\nLeave the wait only on positive proof the agent stopped: `exited` liveness, the\nworker's own observation of process exit, or a transcript whose final agent turn\nsent no `worker_done`. Then load `references/recovery-and-cleanup.md` and choose\n`worker-stop` or `worker-abandon` explicitly. `unverifiable` is absence,\nincluding when `worker-show` reports `agentWait` null. Absence never authorizes\nstop, abandon, retry, or release; keep waiting or inspect.\n\n`worker-start` is the normal path, composing placement, terminal readiness,\nprompt injection, and supervised resource ownership. `dispatch --inject` leaves\nan operator-created process unsupervised and is only for an expressiveness gap.\n\n## Task-spec contract\n\nEvery Task spec must be self-contained and name:\n\n- **Target:** the files, component, or environment in scope.\n- **Change:** the concrete result to produce.\n- **Constraints:** invariants, compatibility rules, and do-not-touch boundaries.\n- **Ownership:** what this worker may edit and any coordination boundary.\n- **Observable acceptance:** the test, output, or evidence that proves completion.\n\n## Completion accounting\n\nAfter an accepted success or failure report, immediately do exactly one:\n\n1. Reuse the same proven agent terminal for an immediate follow-up Dispatch.\n2. Record user-requested retention with `worker-retain`.\n3. Run `worker-release`.\n\nRelease is post-settlement cleanup, not cancellation. Only an accepted\nsettlement authorizes it; no other observation does. If release is uncertain,\nfollow its exact recovery receipt and never substitute `terminal close`.\n\nA valid `worker_done` settles the Task and Dispatch automatically; do not follow\nit with `task-update --status completed`. Enumerate the terminals still owing a\ndecision with `worker-list --run --terminal-state reclaimable --json`,\nand do not end the coordinator turn until it returns none.\n\n## Conditional references\n\nThis compact guide is sufficient for the normal local loop. At an action gate\nbelow, run `ORCA skills get orchestration --reference references/.md` and\nread only that document; `--references` lists the names. If the CLI rejects\n`--reference`, run `ORCA skills get orchestration --full` once instead: it\nreturns this exact kernel and every reference, so read only the named one. If an\nolder CLI rejects `--full`, keep this kernel's safety floor, use that command's\n`--help`, and never guess newer flags.\n\n| Action gate | Bundled reference |\n| ------------------------------------------------------------------------------------------------------------- | ----------------------------------------- |\n| Expanded DAG waves, launch model/effort, same-terminal reuse, or review ownership | `references/coordinator-loop.md` |\n| You are a dispatched worker and the live preamble does not answer your question, or `check` returned an error | `references/worker-contract.md` |\n| New worktree, exact workspace, SSH, WSL, or connected-server placement | `references/placement-and-remote.md` |\n| Inbox replay, follow-up messages, group addresses, or decision gates | `references/messaging-and-gates.md` |\n| Failed/stopped/unknown attempts, retry, stop, abandon, retain, or uncertain release | `references/recovery-and-cleanup.md` |\n| Custom argv or terminal topology that `worker-start` cannot express | `references/low-level-topology.md` |\n| Any legacy label, adopted Run, compatibility receipt, or takeover | `references/legacy-contract-migration.md` |\n\nRetired scheduler commands are not aliases for Run creation. Recovery commands\nmust provide their exact next action; follow it with the same selected executable.\n" // oxfmt-ignore -const ORCHESTRATION_FULL_MARKDOWN = "---\nname: orchestration\ndescription: >-\n Coordinate supervised Orca workers: threaded messages, blocking ask/reply,\n task dispatch, worker_done/escalation waits, task DAGs, decision gates,\n coordinator loops, and decomposing work across agents. Use `orca-cli` for full\n ownership handoffs — \"hand off\", \"handoff\", \"handover\", \"give this to another\n agent\", \"another worktree\" — unless asked to supervise, monitor, or coordinate\n a DAG, and for terminal control, lightweight terminal prompts, shell commands,\n Orca worktree management, and reading or waiting on terminals. Use Computer\n Use for external browser windows, webviews, Orca app UI, or desktop UI outside\n Orca's embedded browser only when the task requires OS/window-level control\n such as focus, menus, dialogs, coordinates, or screenshots. Use `orca-cli` for\n Orca's embedded pages and a page-automation tool such as Playwright or CDP for\n external pages.\n---\n\n# Orca orchestration\n\nOrchestration is Orca's structured coordination layer. It records who owns work,\nwhich attempt is authoritative, and when supervised work has settled.\n\n## Outcome\n\n**Result:** every in-scope Task has one explicit outcome and every settled worker\nterminal has a next owner or cleanup decision. **Next consumer:** the user who\nrequested supervision. **Done:** all expected Dispatches have settled, every\ndelivered message was processed before acknowledgment, each settled worker was\nreused, explicitly retained, or released, and the turn ends only when the report\nto that user names, per Task, its outcome, the evidence behind it, and any\nunresolved blocker.\n\n**Safe failure:** preserve work and authority and report the state as unknown or\n`unverifiable`. Only positive proof of exit authorizes stop, abandon, or retry,\nand only an accepted settlement authorizes release. Every other observation,\nabsence included, is a checkpoint.\n\n## Classify the role\n\n| Current context | Role | Route |\n| ---------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------- | ------------------------------------------------------------------------------ |\n| The user explicitly asks to supervise, monitor, wait for results, track completion, coordinate a DAG, use a decision gate, or manage ask/reply | Coordinator | Use the supervised loop below |\n| The current prompt contains a live injected preamble with Task and Dispatch IDs | Dispatched worker | Follow the preamble and the worker obligations below |\n| The user asks to hand off ownership or start another agent/worktree without supervision | Handoff owner | Use `orca-cli`; create no Run, Task, or Dispatch and do not monitor completion |\n| A message carries a legacy authority label | Compatibility operator | Load the legacy contract reference before any lifecycle mutation |\n| No live preamble and no explicit supervision | Ordinary terminal agent | Do not emit lifecycle messages; use `orca-cli` for terminal/worktree work |\n\nModel or effort selection does not make a handoff supervised. Never substitute a\nnon-Orca subagent tool when Orca orchestration provenance was requested.\n\n## Authority and safety floor\n\n- A Run is a durable namespace and coordinator inbox; it does not schedule or\n place workers. A Task is work. A Dispatch is one authoritative Task attempt.\n- Lifecycle authority comes from the active Dispatch, not a terminal title,\n copied ID, old database row, provider transcript, or visible pane.\n- Workers use the exact executable, handle, capability, Task ID, and Dispatch ID\n in the live preamble. Never reconstruct, translate, or broaden those arguments.\n- After remote start, address the worker by Dispatch ID. The execution host owns\n process, filesystem, transcript, stop, and cleanup facts. Preserve the verdicts\n `live` / `unverifiable` / `exited`; contact loss is not process death.\n- Liveness is layered: `worker-list`'s `projection.liveness` is the fleet verdict\n for the agent; `worker-show`'s `observation.status` is PTY liveness only. A live\n terminal can still hold a dead or stuck agent.\n- Folder workspaces are valid; never require Git or assume a worktree.\n- Clients and remote servers update independently. Treat unknown optional fields\n as absent. A new stream operation requires advertised capability because old\n decoders may silently drop unknown opcodes. Never fall back to local execution\n when remote authority or capability is unproven.\n- Use the executable you used to run `skills get` for the entire run. In the\n examples below, replace `ORCA` with it; do not create a shell variable or run\n `ORCA` literally. If it fails, report that exact error instead of switching.\n- A successful `orchestration send` proves durable enqueue; its wake or nudge is\n best-effort attention only and does not prove the recipient read or accepted it.\n\n## Worker obligations\n\nThe injected preamble is authoritative. A dispatched worker must:\n\n1. Do only the current Task and use the preamble's `ask` command for a blocking\n coordinator question. Never open a local question TUI the coordinator cannot\n answer. Resume the same message ID after an ask timeout.\n2. Send heartbeats only at the cadence in the preamble. A heartbeat proves\n liveness, not completion.\n3. Read coordinator follow-ups at each natural checkpoint — before starting a\n new file, after a test run — and once more immediately before `worker_done`:\n `ORCA orchestration check --terminal --json`.\n4. Send `worker_done` exactly once, from the dispatched terminal, with a\n three-sentence executive summary, both lifecycle IDs, and explicit\n `--outcome succeeded` or `--outcome failed`. Never encode failure only in prose.\n5. Append `--files-modified` and `--report-path` only with real values when\n applicable. After `worker_done`, end the dispatched turn and idle; do not poll\n or start new work.\n\nA direct user instruction after completion starts new user-owned work and takes\nprecedence over the idle rule. Do not reuse the settled lifecycle IDs.\n\n## Canonical supervised loop\n\nConfirm the runtime, bind one Run, and start the full independent wave before\nwaiting. `worker-start --spec` creates the Task and its attempt in one call:\n\n```text\nORCA status --json\nORCA orchestration run-create --objective \"\" --json\nORCA orchestration worker-start --spec \"\" --worktree current --agent codex --json\nORCA orchestration worker-start --spec \"\" --worktree current --agent claude --json\nORCA orchestration check --wait --types \"worker_done,escalation,question\" --timeout-ms 900000 --json\n```\n\nIf `worker-start` exits non-zero, do not relaunch. Read the receipt's\n`failedStage` and `residualResources`, then load\n`references/recovery-and-cleanup.md`.\n\nUse `task-create` plus `worker-start --task ` for planned fan-out with\ndependencies or a retry of a known Task. Use dependencies only for real ordering\nand prefer parallel waves over chains deeper than three or four steps; nested\nworkers obey the depth limit, and a new Run does not reset the caller's depth.\n\nA consuming `check` names its caller with `--terminal `, never `--from`;\nomit it inside the coordinator's own Orca terminal. It returns the bound Run's\noldest FIFO Delivery and replays that batch until acknowledged. Process every\nmessage: reply to questions, validate each `worker_done` against the expected\nactive Dispatch, and decide each settled terminal's next owner before the ack:\n\n```text\nORCA orchestration reply --id --body \"\" --json\nORCA orchestration worker-release --dispatch --json\nORCA orchestration check --ack --wait --types \"worker_done,escalation,question\" --timeout-ms 900000 --json\n```\n\nKeep waiting until every expected Dispatch settles. A timeout or empty result is\na checkpoint, not a failure. Do not stop, retry, release, or launch a duplicate\neditor without the positive proof `## Outcome` requires.\n\nAfter three consecutive empty waits, stop waiting blindly and enumerate with\n`ORCA orchestration worker-list --include-remote --json` (defaults to the bound\nRun; `--run ` overrides; the receipt's `scope` names which), acting on\neach row's `projection.attention` categories, `projection.attention.requiresAction`, and literal `projection.nextAction` argv.\nAn `inspect` `nextAction` on a `live` row with `attention.requiresAction` false\nis informational, not a command to re-run: keep waiting with `check --wait`.\nLeave the wait only on positive proof the agent stopped: `exited` liveness, the\nworker's own observation of process exit, or a transcript whose final agent turn\nsent no `worker_done`. Then load `references/recovery-and-cleanup.md` and choose\n`worker-stop` or `worker-abandon` explicitly. `unverifiable` is absence,\nincluding when `worker-show` reports `agentWait` null. Absence never authorizes\nstop, abandon, retry, or release; keep waiting or inspect.\n\n`worker-start` is the normal path, composing placement, terminal readiness,\nprompt injection, and supervised resource ownership. `dispatch --inject` leaves\nan operator-created process unsupervised and is only for an expressiveness gap.\n\n## Task-spec contract\n\nEvery Task spec must be self-contained and name:\n\n- **Target:** the files, component, or environment in scope.\n- **Change:** the concrete result to produce.\n- **Constraints:** invariants, compatibility rules, and do-not-touch boundaries.\n- **Ownership:** what this worker may edit and any coordination boundary.\n- **Observable acceptance:** the test, output, or evidence that proves completion.\n\n## Completion accounting\n\nAfter an accepted success or failure report, immediately do exactly one:\n\n1. Reuse the same proven agent terminal for an immediate follow-up Dispatch.\n2. Record user-requested retention with `worker-retain`.\n3. Run `worker-release`.\n\nRelease is post-settlement cleanup, not cancellation. Only an accepted\nsettlement authorizes it; no other observation does. If release is uncertain,\nfollow its exact recovery receipt and never substitute `terminal close`.\n\nA valid `worker_done` settles the Task and Dispatch automatically; do not follow\nit with `task-update --status completed`. Enumerate the terminals still owing a\ndecision with `worker-list --run --terminal-state reclaimable --json`,\nand do not end the coordinator turn until it returns none.\n\n## Conditional references\n\nThis compact guide is sufficient for the normal local loop. At an action gate\nbelow, run `ORCA skills get orchestration --reference references/.md` and\nread only that document; `--references` lists the names. If the CLI rejects\n`--reference`, run `ORCA skills get orchestration --full` once instead: it\nreturns this exact kernel and every reference, so read only the named one. If an\nolder CLI rejects `--full`, keep this kernel's safety floor, use that command's\n`--help`, and never guess newer flags.\n\n| Action gate | Bundled reference |\n| ------------------------------------------------------------------------------------------------------------- | ----------------------------------------- |\n| Expanded DAG waves, launch model/effort, same-terminal reuse, or review ownership | `references/coordinator-loop.md` |\n| You are a dispatched worker and the live preamble does not answer your question, or `check` returned an error | `references/worker-contract.md` |\n| New worktree, exact workspace, SSH, WSL, or connected-server placement | `references/placement-and-remote.md` |\n| Inbox replay, follow-up messages, group addresses, or decision gates | `references/messaging-and-gates.md` |\n| Failed/stopped/unknown attempts, retry, stop, abandon, retain, or uncertain release | `references/recovery-and-cleanup.md` |\n| Custom argv or terminal topology that `worker-start` cannot express | `references/low-level-topology.md` |\n| Any legacy label, adopted Run, compatibility receipt, or takeover | `references/legacy-contract-migration.md` |\n\nRetired scheduler commands are not aliases for Run creation. Recovery commands\nmust provide their exact next action; follow it with the same selected executable.\n\n---\n\n# Bundled references\n\nThese references belong to the version-matched guide above. Read only the documents named by its action gates.\n\n\n\n# Coordinator loop\n\nLoad this reference for expanded DAG waves, per-invocation launch preferences,\nsame-terminal reuse, or review ownership. The compact guide remains the source\nof truth for the loop order and completion boundary.\n\n## Ready waves\n\nCreate independent Tasks before the first wait. Encode only real dependencies,\nthen use the ready view as external memory:\n\n```text\nORCA orchestration task-create --spec \"\" --deps --json\nORCA orchestration task-list --ready --brief --json\n```\n\n`--brief` collapses whitespace and caps echoed specs at 160 characters;\n`spec_truncated` identifies shortened rows. Omit it when full specs are needed or\nwhen an older CLI rejects the flag. A nested worker must respect\n`nested_worker_depth_exceeded`; creating another Run does not reset depth.\n\n## Launch preferences\n\nFor a fresh Claude, Codex, or Cursor terminal, `--model` accepts an opaque\nprovider model ID. Add `--effort` only when that model supports it:\n\n```text\nORCA orchestration worker-start --task --worktree current --agent claude --model opus --effort high --json\n```\n\n`--effort` requires `--model`; neither option combines with `--terminal`. A\nconnected worker server must advertise launch-preference support before Orca\nforwards either field. Compare `launch.requested` with `launch.effective`; never\nclaim a model or effort from requested arguments alone.\n\n## Reuse after settlement\n\nChoose the terminal's next owner before acknowledging the Delivery. When the\nsame exact agent has immediate follow-up work, recover the proven handle and\ntransfer cleanup ownership to the new Dispatch:\n\n```text\nORCA orchestration worker-show --dispatch --json\nORCA orchestration worker-start --task --terminal --json\n```\n\nOtherwise explicitly retain or release the settled worker. Do not leave it live\nonly to inspect output; archived output remains available through `worker-read`.\n\n## Review ownership\n\nA review-only `worker_done` authorizes synthesis of findings, not coordinator\nfile edits. Dispatch or hand off fixes unless the user explicitly assigned them\nto the coordinator. If the user's plan names a next owner, post-review fixes and\nPR preparation remain with that owner; the coordinator routes and synthesizes.\n\n\n\n# Legacy contract migration\n\nLoad this reference only for an authority label, adopted Run, compatibility or\nrecovery receipt, or explicit legacy takeover. A newly created attempt always\nuses the current grammar.\n\n## Authority labels\n\n- `[LEGACY COMPATIBILITY]` is live and attested. Run only the exact supported\n command printed with the message, using the same selected executable and\n arguments supplied by the original prompt.\n- `[LEGACY RECOVERY REPLAY — MAY HAVE BEEN SEEN]` is one bounded,\n at-least-once cutover replay. Process it idempotently and acknowledge only\n through the exact displayed guidance.\n- `[LEGACY READ-ONLY]` is inspection-only. It has no reply, acknowledgment, or\n lifecycle mutation.\n- An unlabeled current message uses the current guide and grammar.\n\nAn explicitly selected current Run, attested current binding, current Dispatch,\nor federated attachment takes precedence over legacy fallback. A retained\nadoption record alone does not grant mutation authority. If liveness, principal\nownership, capability, or the exact legacy contract is unproven, degrade to\nread-only inspection and never fall back to local execution.\n\nAdoption preserves the live agent process, PTY/session, terminal handle,\ntab/pane, worktree or folder workspace, Task, and Dispatch. It never restarts or\nreplaces the worker and never revives the retired scheduler. Loss of lifecycle\nauthority does not invalidate the existing process, assignment, or filesystem\nwork. Exact recovery may restore the same PTY once in its original inactive\nbackground tab; it must not spawn, write, signal, stop, switch, focus, split, or\ninject a terminal.\n\n## Compatibility recovery\n\nWhen a compatibility response returns structured next-step arguments, execute\nthose exact arguments with the same selected CLI executable. Do not translate\nfrom memory, broaden the recipient, or retry as a current mutation unless the\nreceipt explicitly authorizes it.\n\nA pending ask, reply, final Dispatch settlement, and consuming check have\ndurable recovery identities. Heartbeat and escalation remain at-least-once\nacross a manual contract-boundary retry. If an ask may already have been\nanswered, run the exact non-consuming recovery check printed by Orca before\ncreating any new question. Never guess among identical question threads.\n\nOn packaged Windows, a legacy ask uses a two-step commit/resume protocol. The\ninitial command commits the question, prints its exact\n`ask --resume ` command, and exits with launcher status `75`. Run\nthat exact resume after the launcher or update boundary. For an attested WSL\nlaunch, preserve the printed `orca-ide` executable and distro route. Older WSL\nworkers without launch proof remain lifecycle read-only even while their\nterminal and filesystem work continue.\n\n## Read-only inspection and takeover\n\nRead-only inspection does not consume mail:\n\n```text\nORCA orchestration run-list --json\nORCA orchestration run-show --id run_legacy_local --json\nORCA orchestration run-show --id --json\nORCA orchestration task-list --run --json\nORCA orchestration inbox --full --json\nORCA orchestration check --terminal --peek --format --json\nORCA terminal read --terminal --json\nORCA terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\n```\n\n`run_legacy_local` is an empty audit tombstone after adoption. Find the ordinary\nRun whose objective is `Recovered orchestration work from a contract update`.\n\nOnly when the original coordinator is unavailable or cannot prove retained\nauthority may a new live coordinator take over from its own terminal:\n\n```text\nORCA orchestration run-use --id --takeover-legacy --json\nORCA orchestration check --run --json\n```\n\nTakeover binds the authenticated invoking terminal; `--from` cannot nominate\nanother coordinator. It fences only the old coordinator and moves pending mail\ninto current Run delivery. It preserves live workers, Tasks, Dispatches, processes, and files.\nNever take over while the original coordinator is actively coordinating.\n\nDo not launch a replacement editor merely because Orca updated or authority is\nunclear. Keep the original worker as the only editor until a stable handoff\npoint, then use a fresh current Dispatch in a conflict-free placement.\n\n\n\n# Low-level topology\n\nLoad this reference only when `worker-start` cannot express required custom argv\nor terminal topology. It is not the normal supervised loop and is never a full\nhandoff recipe.\n\n```text\nORCA terminal create --worktree active --title --command \"\" --json\nORCA terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\nORCA orchestration dispatch --task --to --inject --json\n```\n\nWait for readiness only when startup could lose injected input. Prefer\nagent-first `worker-start` whenever its argv and topology are sufficient.\n\n`dispatch --inject` creates authoritative Task/Dispatch context but deliberately\nkeeps an operator-created process unsupervised: it creates no supervised worker\nresource row. `worker-show`, `worker-read`, and `worker-list` report the lane as\n`unsupervised`; `worker-stop` and `worker-abandon` do not close that process, and\nsettled retain/release take no process action.\n\nUse `worker-start --terminal ` when lifecycle ownership of an existing\nagent terminal is required. Never imply that low-level dispatch retroactively\nowns a process, never use it to route around the nested-depth limit, and never\nuse it for an ownership handoff.\n\n\n\n# Messaging and gates\n\nLoad this reference for inbox replay, attempt-specific guidance, group\naddresses, blocking questions, or coordinator-managed DAG decisions.\n\nA successful `send` proves durable enqueue. Wake and nudge are best-effort\nattention only: neither proves the recipient read the message, began a turn, or\naccepted steering.\n\n## Coordinator delivery loop\n\n`check` names its caller with `--terminal ` and is the only verb that\nrejects `--from`. Omit `--terminal` inside an Orca terminal, where Orca resolves\nthe caller; pass it explicitly from anywhere else, including a dispatched\nworker reading coordinator follow-ups.\n\nA consuming coordinator `check` returns the bound Run's oldest FIFO Delivery,\nup to 50 messages, and replays that exact batch until acknowledged. Process\nevery row and required terminal ownership decision before `--ack`. Type filters\ndecide when a waiter wakes; they do not authorize skipping older actionable\nmail. A Delivery therefore always carries the whole FIFO batch whatever its\ntypes, and a `check` without `--wait` hands that batch over unfiltered.\n`--peek` and `--all` are read-only inspection, not progress through the\ncoordinator inbox.\n\nAn empty wait or timeout is a checkpoint. Continue rolling waits until every\nexpected Dispatch settles. Heartbeat or visible activity means alive, not done.\n\n## Addresses\n\nUse a stable Dispatch address for attempt-specific coordinator guidance:\n\n```text\nORCA orchestration send --to dispatch: --subject \"Follow-up\" --body \"\" --json\n```\n\nDo not substitute a remote terminal handle. Omit `--from` for ordinary\ncoordinator calls; a dispatched worker instead copies the exact `--from` and\ncapability arguments in its preamble. `check` is the exception: it identifies\nits caller with `--terminal`, never `--from`.\n\nGroup addresses include `@all`, `@idle`, `@claude`, `@codex`, `@opencode`,\n`@gemini`, `@droid`, `@grok`, `@cursor`, and `@worktree:`. Use them only for\nintentional fan-out status or questions. `worker_done`, heartbeat, and other\nDispatch lifecycle messages never target groups.\n\n## Questions and gates\n\nA worker uses `ask`; its timeout leaves one durable question pending, which the\nworker resumes by message ID. The coordinator answers that message with `reply`.\n\nUse a gate only for a coordinator-owned Task-DAG decision:\n\n```text\nORCA orchestration gate-create --task --question \"\" --options --json\nORCA orchestration gate-resolve --id --resolution \"\" --json\nORCA orchestration gate-list --task --json\n```\n\nPass `json_array` using the quoting rules of the active shell; do not copy POSIX\nsingle-quote syntax into PowerShell or `cmd.exe`.\n\nDo not create a gate merely to answer a worker's `ask`.\n\n\n\n# Placement and remote execution\n\nLoad this reference before creating a new worktree or placing work through SSH,\nWSL, or another connected Orca server.\n\n## Placement choices\n\nA fresh worker means a fresh agent terminal, not a new Git worktree. Use the\ncurrent or an exact existing workspace by default. Create a worktree only when\nthe user requested one or a concrete checkout or filesystem conflict makes\nsharing unsafe.\n\n```text\n# Current workspace; setup is not rerun.\nORCA orchestration worker-start --task --worktree current --agent codex --json\n\n# Stacked child worktree.\nORCA orchestration worker-start --task --worktree new-child --name --agent codex --setup run --json\n\n# Independent top-level worktree.\nORCA orchestration worker-start --task --worktree new-top-level --name --agent codex --setup run --json\n```\n\nCurrent and exact existing workspaces create a fresh terminal unless\n`--terminal` is explicit. Folder workspaces are first-class; do not invoke Git\nor require worktree lineage when the selected workspace is a folder.\n\nRegister a folder workspace through project setup. `repo add --path `\nrequires a valid Git repository and rejects a plain directory:\n\n```text\nORCA project setup-existing-folder --project --host --path --kind folder --json\n```\n\nThen place work on the returned workspace with an exact selector. A worktree\nselector needs the full `::` value Orca returned, passed as\n`id:`; a bare repo id is not a worktree id. `new-child` and\n`new-top-level` are worktree creation and do not apply to a folder.\n\nNew worktrees use agent-first creation and run setup by default. Preserve the\nrepository's startup policy: `start-immediately` can report setup as `running`,\nwhile `wait-for-setup` gates prompt delivery on success. Orca lineage, Git base,\nfilesystem isolation, coordination parentage, UI grouping, and execution host\nare separate decisions.\n\n## Connected servers\n\nThe Run and Tasks remain authoritative on the current server. `--on` selects\nonly the worker's execution server and appears only on `worker-start`:\n\n```text\nORCA orchestration worker-start --task --on --worktree new-top-level --repo --name --agent codex --setup run --json\n```\n\nRemote `current` and `new-child` are invalid because they are ambiguous across\nservers. Use an exact discovered remote workspace, or `new-top-level` with an\nexact remote repository selector. After start, route every follow-up, read,\nstop, and cleanup by Dispatch ID; never repeat `--on` or substitute a remote\nterminal handle.\n\n```text\nORCA orchestration worker-show --dispatch --json\nORCA orchestration worker-read --dispatch --limit 50 --json\nORCA orchestration send --to dispatch: --subject \"Follow-up\" --body \"\" --json\nORCA orchestration worker-list --run --include-remote --json\n```\n\n`worker-list` reads local fleet state only; enumerate remote workers with\n`--include-remote` or every one of them reads `unverifiable`. Scope every list\nwith `--run `: unscoped, it reports every Dispatch this runtime has\nrecorded, and the workers you are waiting on are lost in that history.\n\n## Execution-host and mixed-version floor\n\nThe execution host owns process, filesystem, transcript, stop, and cleanup\nfacts. Render only `live`, `unverifiable`, or `exited`. Connection loss, relay\nabsence, missing client inventory, or timeout yields `unverifiable`, never\nsynthetic exit and never a client-local substitute action.\n\nClients and servers update independently. Optional response fields may be\nabsent. Forward model/effort, transcript reads, cleanup, or another new remote\noperation only when the peer advertises the relevant capability; unknown stream\nopcodes can be silently dropped. A narrow unsupported response may degrade to a\ndocumented older path, but must not broaden the target or cross the execution\nboundary. Changing host-published content reaches old clients even without a\nwire-shape change, so preserve established semantics or negotiate the behavior.\n\nFor WSL, use the exact executable and arguments returned by Orca so the distro\nand packaged launcher remain bound. Do not translate a printed `orca-ide`\nrecovery command into a PATH-resolved local command.\n\n\n\n# Recovery and cleanup\n\nLoad this reference only after a failed/stopped/unknown attempt, explicit retry\ndecision, stop/abandon request, retention request, or uncertain release.\n\n| Proven state | Safe action |\n| ----------------------- | ------------------------------------------------------------------ |\n| `ready` or active | Keep waiting; optionally read bounded output |\n| `failed` or `stopped` | Start a replacement with `--retry-of`; repeat placement explicitly |\n| `outcome_unknown` | Inspect, then choose `worker-stop` or explicit `worker-abandon` |\n| Accepted `worker_done` | Reuse, retain, or release |\n| Remote contact lost | Preserve `unverifiable`; do not stop or retry from absence alone |\n| `unverifiable` liveness | Keep waiting or inspect; never stop, abandon, retry, or release |\n| Proven `exited` agent | Enumerate with `worker-list`; follow its `nextAction` |\n\n## Inspect before acting\n\n```text\nORCA orchestration worker-list --run --json\nORCA orchestration worker-list --run --include-remote --json\nORCA orchestration worker-show --dispatch --json\nORCA orchestration worker-read --dispatch --limit 50 --json\n```\n\n`worker-list` is the enumerating command and the authority on agent liveness:\neach row carries `projection.liveness`, `projection.attention.categories`,\n`projection.attention.requiresAction`, and a literal `projection.nextAction`\nargv to run. Always scope it with `--run `; an unscoped list reports\nevery Dispatch this runtime has ever recorded and buries the live ones.\n`worker-show`'s `observation.status` is PTY liveness only, so a `live` terminal\nwhose agent died at a trust prompt still reads `live` there.\n\nWhen the two disagree, the fleet verdict decides — unless the fleet row is\n`unverifiable` for a reason that names a gap on this client rather than a fact\nabout the worker. `missing_status`, `host_unavailable`, and\n`capability_unsupported` are such gaps: the first means this runtime holds no\nstatus row, the second that it could not ask the execution host at all, and the\nthird that a stale peer answered but lacks the fleet-snapshot capability.\nAgainst any of them, a `worker-show` verdict sourced from the execution host is\nthe better evidence and outranks the row. Only `host_unavailable` is contact\nloss; the other two mean the host was never asked or answered without the\ncapability.\n\nThis never promotes absence. `unverifiable` from either command still authorizes\nnothing — only a positive `live` or `exited` verdict does.\n\nA worker started with `--on ` reads `unverifiable` until you\nenumerate with `--include-remote`, which asks its execution host for the\nverdict. Past 100 rows the response pages, so follow `page.nextCursor` with\n`--cursor ` until `page.hasMore` is false.\n\n## Stall needs positive evidence\n\nLeave the wait only on positive proof the agent stopped: `exited` liveness, the\nworker's own observation of process exit, or a transcript whose final agent turn\nsent no `worker_done`. Only then choose `worker-stop` or `worker-abandon`.\n\n`unverifiable` is always absence — `missing_status`, `stale_status`,\n`restored_unconfirmed`, or a remote worker with no connection — and a null\n`agentWait` or an unchanged `worker-read` tail is that same absence seen again.\nAbsence never authorizes stop, abandon, retry, or release: keep waiting, or\ninspect until you hold one of the positive signals above. A `nextAction` that\nnames an inspecting command is asking for evidence, not for cleanup.\n\n`worker-read --source auto` uses a proven provider transcript when available and\notherwise returns bounded terminal output with a typed `fallbackReason`.\nContinue with its top-level cursor, which is pinned to that source. If Orca\nreports `source_changed`, restart without the old cursor. A bounded initial\ntranscript tail can return an EOF cursor that follows only newly appended records;\nread `contentComplete`, `clipping`, and `warnings` before assuming omitted older\nrecords are pageable. Never guess a provider session ID, transcript path, or\nremote terminal handle.\n\n## Was the mutation applied?\n\nWhen a mutation's response was lost and named no Dispatch, do not replay blind.\nEvery orchestration mutation accepts `--retry-request `, which reuses one\noperation identity so Orca can replay, join, or recover it instead of starting a\nduplicate. Ask what happened first:\n\n```text\nORCA orchestration request-show --request --json\n```\n\n`completed` means the mutation already took effect; read its recorded receipt\ninstead of rerunning. `pending` means the original mutation is still running or\nOrca restarted before recording its outcome; replay the original command with\n`--retry-request `. `absent` means this runtime holds no receipt\nunder your caller identity — that is not proof nothing happened, so inspect the\naffected Task, Dispatch, and terminal before deciding whether to retry.\n\nWhen a worker's terminal accepted input but the submit is unconfirmed, use\n`terminal send --wait-submit `: it observes the accepted prompt for that\nlong and, on timeout, returns the input-accepted receipt without resending.\n\n## Retry, stop, and abandon\n\nRetry only a positively proven failed or stopped attempt. Placement is never\nsilently inherited:\n\n```text\nORCA orchestration worker-start --task --retry-of --worktree --agent --json\n```\n\nAfter three consecutive failures for one Task, its dispatch context\ncircuit-breaks and the Task is failed. Do not route around that boundary with a\nnew Run or an unrelated Dispatch.\n\nFor `outcome_unknown`, inspect first, then make an explicit choice:\n\n```text\nORCA orchestration worker-stop --dispatch --json\nORCA orchestration worker-abandon --dispatch --json\n```\n\n`worker-stop` closes only the exact proven supervised agent terminal. It never\ndeletes the worktree, setup terminal, configured tabs, or unrelated processes.\n`worker-abandon` fences orchestration while accepting that resources may remain\nlive; it performs no remote, process, or filesystem action.\n\n## Retain and release\n\n```text\nORCA orchestration worker-retain --dispatch --json\nORCA orchestration worker-release --dispatch --json\n```\n\nRetain only when the user explicitly wants the settled terminal kept live.\nRelease works after succeeded and failed reports, archives readable output, and\ncloses only the exact terminal owned by that settled Dispatch. Replays may call\nrelease again safely. Reused, pre-existing, setup, coordinator, active,\nuser-taken-over, and unproven terminals are retained.\n\nA `worker-start` that failed before its agent was ready still owns the terminal\nit created. Its receipt names `worker-release`, and `worker-list` reports that\nrow as `reclaimable`; release it there rather than closing the terminal by hand.\n\nNever release because of timeout, TUI idle, heartbeat, status, question,\nescalation, or stale/rejected completion. If the receipt says `release_pending`\nor `release_unknown`, follow its exact recovery action. Never substitute\n`terminal close`.\n\n`orchestration reset` is destructive recovery. Do not run it during active\ncoordination unless the user explicitly abandons that state.\n\n\n\n# Worker contract\n\nThe injected preamble is authoritative. Copy its command rather than\nreconstructing flags. In particular, preserve the exact executable, worker\nhandle, Dispatch capability, Task ID, and Dispatch ID.\n\n## Heartbeat\n\nSend heartbeats only at the cadence required by the live preamble. Skip them\nwhile blocked inside `ask` or `check --wait`; those calls are liveness signals.\n\n```text\nORCA orchestration send --from --dispatch-capability --type heartbeat --subject \"alive\" --task-id --dispatch-id --phase \"\"\n```\n\nUse typed lifecycle flags, not a hand-written JSON payload. A heartbeat proves\nliveness, never completion.\n\n## Ask and resume\n\nUse Orca `ask` whenever the coordinator must answer. Never open a local question\nTUI the coordinator cannot answer.\n\n```text\nORCA orchestration ask --from --dispatch-capability --question \"\" --options \",\" --timeout-ms 600000\n\nORCA orchestration ask --from --dispatch-capability --resume --timeout-ms 600000\n```\n\nA timeout or disconnect leaves the original question pending. Resume its\nmessage ID; do not create a duplicate question.\n\n## Reading coordinator follow-ups\n\nThe coordinator steers a running worker with `send --to dispatch:`. That\nenqueue is durable but does not interrupt you, so nothing arrives unless you\nlook:\n\n```text\nORCA orchestration check --terminal --json\n```\n\nRun it at each natural checkpoint — before starting a new file, after a test\nrun — and once more immediately before `worker_done`, so a redirect or a\ncancellation lands before the Task settles. `check` names its caller with\n`--terminal`, never `--from`. Stop checking after `worker_done`.\n\nIf `check` returns `consumer_fenced`, this process no longer owns its Dispatch:\nthe Attempt was re-attached to another worker or settled without you. Stop, do\nnot send `worker_done`, and do not retry the check. An empty `check` never means\nyou were replaced; `consumer_fenced` is the only way you learn that.\n\n## Escalation\n\nEscalate only before completion and only when the coordinator must intervene:\n\n```text\nORCA orchestration send --from --dispatch-capability --type escalation --subject \"Blocked: \" --body \"
\" --task-id --dispatch-id \n```\n\n## Completion\n\nSend exactly one terminal report. `--body` is three sentences: what changed,\nwhat was found, and what remains. Use `--outcome failed` when the requested work\nis not complete; never hide failure in prose or silently exit.\n\nAppend `--files-modified` or `--report-path` only when applicable, using actual\npaths. Do not send documentation placeholders as metadata.\n\n```text\nORCA orchestration send --from --dispatch-capability --type worker_done --subject \"\" --body \"\" --task-id --dispatch-id --outcome succeeded\n```\n\nAfter `worker_done`, end the dispatched turn and idle. Do not poll, close your\nown terminal, or begin unrelated work. A later direct user instruction is new\nuser-owned work and must not reuse settled lifecycle IDs; a supervised follow-up\narrives with a fresh preamble and Task block.\n" +const ORCHESTRATION_FULL_MARKDOWN = "---\nname: orchestration\ndescription: >-\n Coordinate supervised Orca workers: threaded messages, blocking ask/reply,\n task dispatch, worker_done/escalation waits, task DAGs, decision gates,\n coordinator loops, and decomposing work across agents. Use `orca-cli` for full\n ownership handoffs — \"hand off\", \"handoff\", \"handover\", \"give this to another\n agent\", \"another worktree\" — unless asked to supervise, monitor, or coordinate\n a DAG, and for terminal control, lightweight terminal prompts, shell commands,\n Orca worktree management, and reading or waiting on terminals. Use Computer\n Use for external browser windows, webviews, Orca app UI, or desktop UI outside\n Orca's embedded browser only when the task requires OS/window-level control\n such as focus, menus, dialogs, coordinates, or screenshots. Use `orca-cli` for\n Orca's embedded pages and a page-automation tool such as Playwright or CDP for\n external pages.\n---\n\n# Orca orchestration\n\nOrchestration is Orca's structured coordination layer. It records who owns work,\nwhich attempt is authoritative, and when supervised work has settled.\n\n## Outcome\n\n**Result:** every in-scope Task has one explicit outcome and every settled worker\nterminal has a next owner or cleanup decision. **Next consumer:** the user who\nrequested supervision. **Done:** all expected Dispatches have settled, every\ndelivered message was processed before acknowledgment, each settled worker was\nreused, explicitly retained, or released, and the turn ends only when the report\nto that user names, per Task, its outcome, the evidence behind it, and any\nunresolved blocker.\n\n**Safe failure:** preserve work and authority and report the state as unknown or\n`unverifiable`. Only positive proof of exit authorizes stop, abandon, or retry,\nand only an accepted settlement authorizes release. Every other observation,\nabsence included, is a checkpoint.\n\n## Classify the role\n\n| Current context | Role | Route |\n| ---------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------- | ------------------------------------------------------------------------------ |\n| The user explicitly asks to supervise, monitor, wait for results, track completion, coordinate a DAG, use a decision gate, or manage ask/reply | Coordinator | Use the supervised loop below |\n| The current prompt contains a live injected preamble with Task and Dispatch IDs | Dispatched worker | Follow the preamble and the worker obligations below |\n| The user asks to hand off ownership or start another agent/worktree without supervision | Handoff owner | Use `orca-cli`; create no Run, Task, or Dispatch and do not monitor completion |\n| A message carries a legacy authority label | Compatibility operator | Load the legacy contract reference before any lifecycle mutation |\n| No live preamble and no explicit supervision | Ordinary terminal agent | Do not emit lifecycle messages; use `orca-cli` for terminal/worktree work |\n\nModel or effort selection does not make a handoff supervised. Never substitute a\nnon-Orca subagent tool when Orca orchestration provenance was requested.\n\n## Authority and safety floor\n\n- A Run is a durable namespace and coordinator inbox; it does not schedule or\n place workers. A Task is work. A Dispatch is one authoritative Task attempt.\n- Lifecycle authority comes from the active Dispatch, not a terminal title,\n copied ID, old database row, provider transcript, or visible pane.\n- Workers use the exact executable, handle, capability, Task ID, and Dispatch ID\n in the live preamble. Never reconstruct, translate, or broaden those arguments.\n- After remote start, address the worker by Dispatch ID. The execution host owns\n process, filesystem, transcript, stop, and cleanup facts. Preserve the verdicts\n `live` / `unverifiable` / `exited`; contact loss is not process death.\n- Liveness is layered: `worker-list`'s `projection.liveness` is the fleet verdict\n for the agent; `worker-show`'s `observation.status` is PTY liveness only. A live\n terminal can still hold a dead or stuck agent.\n- Folder workspaces are valid; never require Git or assume a worktree.\n- Clients and remote servers update independently. Treat unknown optional fields\n as absent. A new stream operation requires advertised capability because old\n decoders may silently drop unknown opcodes. Never fall back to local execution\n when remote authority or capability is unproven.\n- Use the executable you used to run `skills get` for the entire run. In the\n examples below, replace `ORCA` with it; do not create a shell variable or run\n `ORCA` literally. If it fails, report that exact error instead of switching.\n- A successful `orchestration send` proves durable enqueue; its wake or nudge is\n best-effort attention only and does not prove the recipient read or accepted it.\n\n## Worker obligations\n\nThe injected preamble is authoritative. A dispatched worker must:\n\n1. Do only the current Task and use the preamble's `ask` command for a blocking\n coordinator question. Never open a local question TUI the coordinator cannot\n answer. Resume the same message ID after an ask timeout.\n2. Send heartbeats only at the cadence in the preamble. A heartbeat proves\n liveness, not completion.\n3. Read coordinator follow-ups at each natural checkpoint — before starting a\n new file, after a test run — and once more immediately before `worker_done`:\n `ORCA orchestration check --terminal --json`.\n4. Send `worker_done` exactly once, from the dispatched terminal, with a\n three-sentence executive summary, both lifecycle IDs, and explicit\n `--outcome succeeded` or `--outcome failed`. Never encode failure only in prose.\n5. Append `--files-modified` and `--report-path` only with real values when\n applicable. After `worker_done`, end the dispatched turn and idle; do not poll\n or start new work.\n\nA direct user instruction after completion starts new user-owned work and takes\nprecedence over the idle rule. Do not reuse the settled lifecycle IDs.\n\n## Canonical supervised loop\n\nConfirm the runtime, bind one Run, and start the full independent wave before\nwaiting. `worker-start --spec` creates the Task and its attempt in one call:\n\n```text\nORCA status --json\nORCA orchestration run-create --objective \"\" --json\nORCA orchestration worker-start --spec \"\" --worktree current --agent codex --json\nORCA orchestration worker-start --spec \"\" --worktree current --agent claude --json\nORCA orchestration check --wait --types \"worker_done,escalation,question\" --timeout-ms 900000 --json\n```\n\nIf `worker-start` exits non-zero, do not relaunch. Read the receipt's\n`failedStage` and `residualResources`, then load\n`references/recovery-and-cleanup.md`.\n\nUse `task-create` plus `worker-start --task ` for planned fan-out with\ndependencies or a retry of a known Task. Use dependencies only for real ordering\nand prefer parallel waves over chains deeper than three or four steps; nested\nworkers obey the depth limit, and a new Run does not reset the caller's depth.\n\nA consuming `check` names its caller with `--terminal `, never `--from`;\nomit it inside the coordinator's own Orca terminal. It returns the bound Run's\noldest FIFO Delivery and replays that batch until acknowledged. Process every\nmessage: reply to questions, validate each `worker_done` against the expected\nactive Dispatch, and decide each settled terminal's next owner before the ack:\n\n```text\nORCA orchestration reply --id --body \"\" --json\nORCA orchestration worker-release --dispatch --json\nORCA orchestration check --ack --wait --types \"worker_done,escalation,question\" --timeout-ms 900000 --json\n```\n\nKeep waiting until every expected Dispatch settles. A timeout or empty result is\na checkpoint, not a failure. Do not stop, retry, release, or launch a duplicate\neditor without the positive proof `## Outcome` requires.\n\nAfter three consecutive empty waits, stop waiting blindly and enumerate with\n`ORCA orchestration worker-list --include-remote --json` (defaults to the bound\nRun; `--run ` overrides; the receipt's `scope` names which), acting on\neach row's `projection.attention` categories, `projection.attention.requiresAction`, and literal `projection.nextAction` argv.\nAn `inspect` `nextAction` on a `live` row with `attention.requiresAction` false\nis informational, not a command to re-run: keep waiting with `check --wait`.\nLeave the wait only on positive proof the agent stopped: `exited` liveness, the\nworker's own observation of process exit, or a transcript whose final agent turn\nsent no `worker_done`. Then load `references/recovery-and-cleanup.md` and choose\n`worker-stop` or `worker-abandon` explicitly. `unverifiable` is absence,\nincluding when `worker-show` reports `agentWait` null. Absence never authorizes\nstop, abandon, retry, or release; keep waiting or inspect.\n\n`worker-start` is the normal path, composing placement, terminal readiness,\nprompt injection, and supervised resource ownership. `dispatch --inject` leaves\nan operator-created process unsupervised and is only for an expressiveness gap.\n\n## Task-spec contract\n\nEvery Task spec must be self-contained and name:\n\n- **Target:** the files, component, or environment in scope.\n- **Change:** the concrete result to produce.\n- **Constraints:** invariants, compatibility rules, and do-not-touch boundaries.\n- **Ownership:** what this worker may edit and any coordination boundary.\n- **Observable acceptance:** the test, output, or evidence that proves completion.\n\n## Completion accounting\n\nAfter an accepted success or failure report, immediately do exactly one:\n\n1. Reuse the same proven agent terminal for an immediate follow-up Dispatch.\n2. Record user-requested retention with `worker-retain`.\n3. Run `worker-release`.\n\nRelease is post-settlement cleanup, not cancellation. Only an accepted\nsettlement authorizes it; no other observation does. If release is uncertain,\nfollow its exact recovery receipt and never substitute `terminal close`.\n\nA valid `worker_done` settles the Task and Dispatch automatically; do not follow\nit with `task-update --status completed`. Enumerate the terminals still owing a\ndecision with `worker-list --run --terminal-state reclaimable --json`,\nand do not end the coordinator turn until it returns none.\n\n## Conditional references\n\nThis compact guide is sufficient for the normal local loop. At an action gate\nbelow, run `ORCA skills get orchestration --reference references/.md` and\nread only that document; `--references` lists the names. If the CLI rejects\n`--reference`, run `ORCA skills get orchestration --full` once instead: it\nreturns this exact kernel and every reference, so read only the named one. If an\nolder CLI rejects `--full`, keep this kernel's safety floor, use that command's\n`--help`, and never guess newer flags.\n\n| Action gate | Bundled reference |\n| ------------------------------------------------------------------------------------------------------------- | ----------------------------------------- |\n| Expanded DAG waves, launch model/effort, same-terminal reuse, or review ownership | `references/coordinator-loop.md` |\n| You are a dispatched worker and the live preamble does not answer your question, or `check` returned an error | `references/worker-contract.md` |\n| New worktree, exact workspace, SSH, WSL, or connected-server placement | `references/placement-and-remote.md` |\n| Inbox replay, follow-up messages, group addresses, or decision gates | `references/messaging-and-gates.md` |\n| Failed/stopped/unknown attempts, retry, stop, abandon, retain, or uncertain release | `references/recovery-and-cleanup.md` |\n| Custom argv or terminal topology that `worker-start` cannot express | `references/low-level-topology.md` |\n| Any legacy label, adopted Run, compatibility receipt, or takeover | `references/legacy-contract-migration.md` |\n\nRetired scheduler commands are not aliases for Run creation. Recovery commands\nmust provide their exact next action; follow it with the same selected executable.\n\n---\n\n# Bundled references\n\nThese references belong to the version-matched guide above. Read only the documents named by its action gates.\n\n\n\n# Coordinator loop\n\nLoad this reference for expanded DAG waves, per-invocation launch preferences,\nsame-terminal reuse, or review ownership. The compact guide remains the source\nof truth for the loop order and completion boundary.\n\n## Ready waves\n\nCreate independent Tasks before the first wait. Encode only real dependencies,\nthen use the ready view as external memory:\n\n```text\nORCA orchestration task-create --spec \"\" --deps --json\nORCA orchestration task-list --ready --brief --json\n```\n\n`--brief` collapses whitespace and caps echoed specs at 160 characters;\n`spec_truncated` identifies shortened rows. Omit it when full specs are needed or\nwhen an older CLI rejects the flag. A nested worker must respect\n`nested_worker_depth_exceeded`; creating another Run does not reset depth.\n\n## Launch preferences\n\nFor a fresh Claude, Codex, or Cursor terminal, `--model` accepts an opaque\nprovider model ID. Add `--effort` only when that model supports it:\n\n```text\nORCA orchestration worker-start --task --worktree current --agent claude --model opus --effort high --json\n```\n\n`--effort` requires `--model`; neither option combines with `--terminal`. A\nconnected worker server must advertise launch-preference support before Orca\nforwards either field. Compare `launch.requested` with `launch.effective`; never\nclaim a model or effort from requested arguments alone.\n\n## Reuse after settlement\n\nChoose the terminal's next owner before acknowledging the Delivery. When the\nsame exact agent has immediate follow-up work, recover the proven handle and\ntransfer cleanup ownership to the new Dispatch:\n\n```text\nORCA orchestration worker-show --dispatch --json\nORCA orchestration worker-start --task --terminal --json\n```\n\nOtherwise explicitly retain or release the settled worker. Do not leave it live\nonly to inspect output; archived output remains available through `worker-read`.\n\n## Review ownership\n\nA review-only `worker_done` authorizes synthesis of findings, not coordinator\nfile edits. Dispatch or hand off fixes unless the user explicitly assigned them\nto the coordinator. If the user's plan names a next owner, post-review fixes and\nPR preparation remain with that owner; the coordinator routes and synthesizes.\n\n\n\n# Legacy contract migration\n\nLoad this reference only for an authority label, adopted Run, compatibility or\nrecovery receipt, or explicit legacy takeover. A newly created attempt always\nuses the current grammar.\n\n## Authority labels\n\n- `[LEGACY COMPATIBILITY]` is live and attested. Run only the exact supported\n command printed with the message, using the same selected executable and\n arguments supplied by the original prompt.\n- `[LEGACY RECOVERY REPLAY — MAY HAVE BEEN SEEN]` is one bounded,\n at-least-once cutover replay. Process it idempotently and acknowledge only\n through the exact displayed guidance.\n- `[LEGACY READ-ONLY]` is inspection-only. It has no reply, acknowledgment, or\n lifecycle mutation.\n- An unlabeled current message uses the current guide and grammar.\n\nAn explicitly selected current Run, attested current binding, current Dispatch,\nor federated attachment takes precedence over legacy fallback. A retained\nadoption record alone does not grant mutation authority. If liveness, principal\nownership, capability, or the exact legacy contract is unproven, degrade to\nread-only inspection and never fall back to local execution.\n\nAdoption preserves the live agent process, PTY/session, terminal handle,\ntab/pane, worktree or folder workspace, Task, and Dispatch. It never restarts or\nreplaces the worker and never revives the retired scheduler. Loss of lifecycle\nauthority does not invalidate the existing process, assignment, or filesystem\nwork. Exact recovery may restore the same PTY once in its original inactive\nbackground tab; it must not spawn, write, signal, stop, switch, focus, split, or\ninject a terminal.\n\n## Compatibility recovery\n\nWhen a compatibility response returns structured next-step arguments, execute\nthose exact arguments with the same selected CLI executable. Do not translate\nfrom memory, broaden the recipient, or retry as a current mutation unless the\nreceipt explicitly authorizes it.\n\nA pending ask, reply, final Dispatch settlement, and consuming check have\ndurable recovery identities. Heartbeat and escalation remain at-least-once\nacross a manual contract-boundary retry. If an ask may already have been\nanswered, run the exact non-consuming recovery check printed by Orca before\ncreating any new question. Never guess among identical question threads.\n\nOn packaged Windows, a legacy ask uses a two-step commit/resume protocol. The\ninitial command commits the question, prints its exact\n`ask --resume ` command, and exits with launcher status `75`. Run\nthat exact resume after the launcher or update boundary. For an attested WSL\nlaunch, preserve the printed `orca-ide` executable and distro route. Older WSL\nworkers without launch proof remain lifecycle read-only even while their\nterminal and filesystem work continue.\n\n## Read-only inspection and takeover\n\nRead-only inspection does not consume mail:\n\n```text\nORCA orchestration run-list --json\nORCA orchestration run-show --id run_legacy_local --json\nORCA orchestration run-show --id --json\nORCA orchestration task-list --run --json\nORCA orchestration inbox --full --json\nORCA orchestration check --terminal --peek --format --json\nORCA terminal read --terminal --json\nORCA terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\n```\n\n`run_legacy_local` is an empty audit tombstone after adoption. Find the ordinary\nRun whose objective is `Recovered orchestration work from a contract update`.\n\nOnly when the original coordinator is unavailable or cannot prove retained\nauthority may a new live coordinator take over from its own terminal:\n\n```text\nORCA orchestration run-use --id --takeover-legacy --json\nORCA orchestration check --run --json\n```\n\nTakeover binds the authenticated invoking terminal; `--from` cannot nominate\nanother coordinator. It fences only the old coordinator and moves pending mail\ninto current Run delivery. It preserves live workers, Tasks, Dispatches, processes, and files.\nNever take over while the original coordinator is actively coordinating.\n\nDo not launch a replacement editor merely because Orca updated or authority is\nunclear. Keep the original worker as the only editor until a stable handoff\npoint, then use a fresh current Dispatch in a conflict-free placement.\n\n\n\n# Low-level topology\n\nLoad this reference only when `worker-start` cannot express required custom argv\nor terminal topology. It is not the normal supervised loop and is never a full\nhandoff recipe.\n\n```text\nORCA terminal create --worktree active --title --command \"\" --json\nORCA terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\nORCA orchestration dispatch --task --to --inject --json\n```\n\nWait for readiness only when startup could lose injected input. Prefer\nagent-first `worker-start` whenever its argv and topology are sufficient.\n\n`dispatch --inject` creates authoritative Task/Dispatch context but deliberately\nkeeps an operator-created process unsupervised: it creates no supervised worker\nresource row. `worker-show`, `worker-read`, and `worker-list` report the lane as\n`unsupervised`; `worker-stop` and `worker-abandon` do not close that process, and\nsettled retain/release take no process action.\n\nUse `worker-start --terminal ` when lifecycle ownership of an existing\nagent terminal is required. Never imply that low-level dispatch retroactively\nowns a process, never use it to route around the nested-depth limit, and never\nuse it for an ownership handoff.\n\n\n\n# Messaging and gates\n\nLoad this reference for inbox replay, attempt-specific guidance, group\naddresses, blocking questions, or coordinator-managed DAG decisions.\n\nA successful `send` proves durable enqueue. Wake and nudge are best-effort\nattention only: neither proves the recipient read the message, began a turn, or\naccepted steering.\n\n## Coordinator delivery loop\n\n`check` names its caller with `--terminal ` and is the only verb that\nrejects `--from`. Omit `--terminal` inside an Orca terminal, where Orca resolves\nthe caller; pass it explicitly from anywhere else, including a dispatched\nworker reading coordinator follow-ups.\n\nA consuming coordinator `check` returns the bound Run's oldest FIFO Delivery,\nup to 50 messages, and replays that exact batch until acknowledged. Process\nevery row and required terminal ownership decision before `--ack`. Type filters\ndecide when a waiter wakes; they do not authorize skipping older actionable\nmail. A Delivery therefore always carries the whole FIFO batch whatever its\ntypes, and a `check` without `--wait` hands that batch over unfiltered.\n`--peek` and `--all` are read-only inspection, not progress through the\ncoordinator inbox.\n\nAn empty wait or timeout is a checkpoint. Continue rolling waits until every\nexpected Dispatch settles. Heartbeat or visible activity means alive, not done.\n\n## Addresses\n\nUse a stable Dispatch address for attempt-specific coordinator guidance:\n\n```text\nORCA orchestration send --to dispatch: --subject \"Follow-up\" --body \"\" --json\n```\n\nDo not substitute a remote terminal handle. Omit `--from` for ordinary\ncoordinator calls; a dispatched worker instead copies the exact `--from` and\ncapability arguments in its preamble. `check` is the exception: it identifies\nits caller with `--terminal`, never `--from`.\n\nGroup addresses include `@all`, `@idle`, `@claude`, `@codex`, `@opencode`,\n`@gemini`, `@droid`, `@grok`, `@cursor`, and `@worktree:`. Use them only for\nintentional fan-out status or questions. `worker_done`, heartbeat, and other\nDispatch lifecycle messages never target groups.\n\n## Questions and gates\n\nA worker uses `ask`; its timeout leaves one durable question pending, which the\nworker resumes by message ID. The coordinator answers that message with `reply`.\n\nUse a gate only for a coordinator-owned Task-DAG decision:\n\n```text\nORCA orchestration gate-create --task --question \"\" --options --json\nORCA orchestration gate-resolve --id --resolution \"\" --json\nORCA orchestration gate-list --task --json\n```\n\nPass `json_array` using the quoting rules of the active shell; do not copy POSIX\nsingle-quote syntax into PowerShell or `cmd.exe`.\n\nDo not create a gate merely to answer a worker's `ask`.\n\n\n\n# Placement and remote execution\n\nLoad this reference before creating a new worktree or placing work through SSH,\nWSL, or another connected Orca server.\n\n## Placement choices\n\nA fresh worker means a fresh agent terminal, not a new Git worktree. Use the\ncurrent or an exact existing workspace by default. Create a worktree only when\nthe user requested one or a concrete checkout or filesystem conflict makes\nsharing unsafe.\n\n```text\n# Current workspace; setup is not rerun.\nORCA orchestration worker-start --task --worktree current --agent codex --json\n\n# Stacked child worktree.\nORCA orchestration worker-start --task --worktree new-child --name --agent codex --setup run --json\n\n# Independent top-level worktree.\nORCA orchestration worker-start --task --worktree new-top-level --name --agent codex --setup run --json\n```\n\nCurrent and exact existing workspaces create a fresh terminal unless\n`--terminal` is explicit. Folder workspaces are first-class; do not invoke Git\nor require worktree lineage when the selected workspace is a folder.\n\nRegister a folder workspace through project setup. `repo add --path `\nrequires a valid Git repository and rejects a plain directory:\n\n```text\nORCA project setup-existing-folder --project --host --path --kind folder --json\n```\n\nThen place work on the returned workspace with an exact selector. A worktree\nselector needs the full `::` value Orca returned, passed as\n`id:`; a bare repo id is not a worktree id. `new-child` and\n`new-top-level` are worktree creation and do not apply to a folder.\n\nNew worktrees use agent-first creation and run setup by default. Preserve the\nrepository's startup policy: `start-immediately` can report setup as `running`,\nwhile `wait-for-setup` gates prompt delivery on success. Orca lineage, Git base,\nfilesystem isolation, coordination parentage, UI grouping, and execution host\nare separate decisions.\n\n## Connected servers\n\nThe Run and Tasks remain authoritative on the current server. `--on` selects\nonly the worker's execution server and appears only on `worker-start`:\n\n```text\nORCA orchestration worker-start --task --on --worktree new-top-level --repo --name --agent codex --setup run --json\n```\n\nRemote `current` and `new-child` are invalid because they are ambiguous across\nservers. Use an exact discovered remote workspace, or `new-top-level` with an\nexact remote repository selector. After start, route every follow-up, read,\nstop, and cleanup by Dispatch ID; never repeat `--on` or substitute a remote\nterminal handle.\n\n```text\nORCA orchestration worker-show --dispatch --json\nORCA orchestration worker-read --dispatch --limit 50 --json\nORCA orchestration send --to dispatch: --subject \"Follow-up\" --body \"\" --json\nORCA orchestration worker-list --run --include-remote --json\n```\n\n`worker-list` reads local fleet state only; enumerate remote workers with\n`--include-remote` or every one of them reads `unverifiable`. Scope every list\nwith `--run `: unscoped, it reports every Dispatch this runtime has\nrecorded, and the workers you are waiting on are lost in that history.\n\n## Execution-host and mixed-version floor\n\nThe execution host owns process, filesystem, transcript, stop, and cleanup\nfacts. Render only `live`, `unverifiable`, or `exited`. Connection loss, relay\nabsence, missing client inventory, or timeout yields `unverifiable`, never\nsynthetic exit and never a client-local substitute action.\n\nClients and servers update independently. Optional response fields may be\nabsent. Forward model/effort, transcript reads, cleanup, or another new remote\noperation only when the peer advertises the relevant capability; unknown stream\nopcodes can be silently dropped. A narrow unsupported response may degrade to a\ndocumented older path, but must not broaden the target or cross the execution\nboundary. Changing host-published content reaches old clients even without a\nwire-shape change, so preserve established semantics or negotiate the behavior.\n\nFor WSL, use the exact executable and arguments returned by Orca so the distro\nand packaged launcher remain bound. Do not translate a printed `orca-ide`\nrecovery command into a PATH-resolved local command.\n\n\n\n# Recovery and cleanup\n\nLoad this reference only after a failed/stopped/unknown attempt, explicit retry\ndecision, stop/abandon request, retention request, or uncertain release.\n\n| Proven state | Safe action |\n| ----------------------- | ------------------------------------------------------------------ |\n| `ready` or active | Keep waiting; optionally read bounded output |\n| `failed` or `stopped` | Start a replacement with `--retry-of`; repeat placement explicitly |\n| `outcome_unknown` | Inspect, then choose `worker-stop` or explicit `worker-abandon` |\n| Accepted `worker_done` | Reuse, retain, or release |\n| Remote contact lost | Preserve `unverifiable`; do not stop or retry from absence alone |\n| `unverifiable` liveness | Keep waiting or inspect; never stop, abandon, retry, or release |\n| Proven `exited` agent | Enumerate with `worker-list`; follow its `nextAction` |\n\n## Inspect before acting\n\n```text\nORCA orchestration worker-list --run --json\nORCA orchestration worker-list --run --include-remote --json\nORCA orchestration worker-show --dispatch --json\nORCA orchestration worker-read --dispatch --limit 50 --json\n```\n\n`worker-list` is the enumerating command and the authority on agent liveness:\neach row carries `projection.liveness`, `projection.attention.categories`,\n`projection.attention.requiresAction`, and a literal `projection.nextAction`\nargv to run. Always scope it with `--run `; an unscoped list reports\nevery Dispatch this runtime has ever recorded and buries the live ones.\n`worker-show`'s `observation.status` is PTY liveness only, so a `live` terminal\nwhose agent died at a trust prompt still reads `live` there.\n\nWhen the two disagree, the fleet verdict decides — unless the fleet row is\n`unverifiable` for a reason that names a gap on this client rather than a fact\nabout the worker. `missing_status`, `host_unavailable`, and\n`capability_unsupported` are such gaps: the first means this runtime holds no\nstatus row, the second that it could not ask the execution host at all, and the\nthird that a stale peer answered but lacks the fleet-snapshot capability.\nAgainst any of them, a `worker-show` verdict sourced from the execution host is\nthe better evidence and outranks the row. Only `host_unavailable` is contact\nloss; the other two mean the host was never asked or answered without the\ncapability.\n\nThis never promotes absence. `unverifiable` from either command still authorizes\nnothing — only a positive `live` or `exited` verdict does.\n\nA worker started with `--on ` reads `unverifiable` until you\nenumerate with `--include-remote`, which asks its execution host for the\nverdict. Past 100 rows the response pages, so follow `page.nextCursor` with\n`--cursor ` until `page.hasMore` is false.\n\n## Stall needs positive evidence\n\nLeave the wait only on positive proof the agent stopped: `exited` liveness, the\nworker's own observation of process exit, or a transcript whose final agent turn\nsent no `worker_done`. Only then choose `worker-stop` or `worker-abandon`.\n\n`unverifiable` is always absence — `missing_status`, `stale_status`,\n`restored_unconfirmed`, or a remote worker with no connection — and a null\n`agentWait` or an unchanged `worker-read` tail is that same absence seen again.\nAbsence never authorizes stop, abandon, retry, or release: keep waiting, or\ninspect until you hold one of the positive signals above. A `nextAction` that\nnames an inspecting command is asking for evidence, not for cleanup.\n\n`worker-read --source auto` uses a proven provider transcript when available and\notherwise returns bounded terminal output with a typed `fallbackReason`.\nContinue with its top-level cursor, which is pinned to that source. If Orca\nreports `source_changed`, restart without the old cursor. A bounded initial\ntranscript tail can return an EOF cursor that follows only newly appended records;\nread `contentComplete`, `clipping`, and `warnings` before assuming omitted older\nrecords are pageable. Never guess a provider session ID, transcript path, or\nremote terminal handle.\n\n## Was the mutation applied?\n\nWhen a mutation's response was lost and named no Dispatch, do not replay blind.\nEvery orchestration mutation accepts `--retry-request `, which reuses one\noperation identity so Orca can replay, join, or recover it instead of starting a\nduplicate. Ask what happened first:\n\n```text\nORCA orchestration request-show --request --json\n```\n\n`completed` means the mutation already took effect; read its recorded receipt\ninstead of rerunning. `pending` means the original mutation is still running or\nOrca restarted before recording its outcome; replay the original command with\n`--retry-request `. `absent` means this runtime holds no receipt\nunder your caller identity — that is not proof nothing happened, so inspect the\naffected Task, Dispatch, and terminal before deciding whether to retry.\n\nWhen a worker's terminal accepted input but the submit is unconfirmed, use\n`terminal send --wait-submit `: it observes the accepted prompt for that\nlong and, on timeout, returns the input-accepted receipt without resending.\n\n## Refused starts\n\n`dispatch` and `worker-start` refuse the following preflight cases with a stable\n`error.code`; read it before choosing a recovery, and treat `error.data.nextSteps`\nas the exact recovery text. Older hosts may omit `data`, so treat every field as\noptional.\n\n| Code | Meaning | Recovery |\n| -------------------- | --------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ |\n| `task_not_found` | No Task with that id, or not in the bound Run (`data.taskId`, `data.runId`) | Check `task-list --json`; create the Task with `task-create` if it does not exist |\n| `task_not_startable` | Task cannot start now: not `ready`, or invalid `--retry-of` (`data.status`, `data.unmetDependencies`, `data.retryOf`) | Wait for running dependencies with `check --wait`; retry or unblock failed ones; inspect `dispatch-show` if already dispatched |\n| `inject_rejected` | `--inject` refused because no recognized agent runs in the target (`data.terminal`, `data.reason`) | Start a recognized agent there or pick another terminal; or dispatch without `--inject` and use `terminal send` |\n| `runtime_error` | Any other failure, including a target terminal that already owns an active Dispatch | Read the message, inspect state, and do not retry unchanged |\n\n## Retry, stop, and abandon\n\nRetry only a positively proven failed or stopped attempt. Placement is never\nsilently inherited:\n\n```text\nORCA orchestration worker-start --task --retry-of --worktree --agent --json\n```\n\nAfter three consecutive failures for one Task, its dispatch context\ncircuit-breaks and the Task is failed. Do not route around that boundary with a\nnew Run or an unrelated Dispatch.\n\nFor `outcome_unknown`, inspect first, then make an explicit choice:\n\n```text\nORCA orchestration worker-stop --dispatch --json\nORCA orchestration worker-abandon --dispatch --json\n```\n\n`worker-stop` closes only the exact proven supervised agent terminal. It never\ndeletes the worktree, setup terminal, configured tabs, or unrelated processes.\n`worker-abandon` fences orchestration while accepting that resources may remain\nlive; it performs no remote, process, or filesystem action.\n\n## Retain and release\n\n```text\nORCA orchestration worker-retain --dispatch --json\nORCA orchestration worker-release --dispatch --json\n```\n\nRetain only when the user explicitly wants the settled terminal kept live.\nRelease works after succeeded and failed reports, archives readable output, and\ncloses only the exact terminal owned by that settled Dispatch. Replays may call\nrelease again safely. Reused, pre-existing, setup, coordinator, active,\nuser-taken-over, and unproven terminals are retained.\n\nA `worker-start` that failed before its agent was ready still owns the terminal\nit created. Its receipt names `worker-release`, and `worker-list` reports that\nrow as `reclaimable`; release it there rather than closing the terminal by hand.\n\nNever release because of timeout, TUI idle, heartbeat, status, question,\nescalation, or stale/rejected completion. If the receipt says `release_pending`\nor `release_unknown`, follow its exact recovery action. Never substitute\n`terminal close`.\n\n`orchestration reset` is destructive recovery. Do not run it during active\ncoordination unless the user explicitly abandons that state.\n\n\n\n# Worker contract\n\nThe injected preamble is authoritative. Copy its command rather than\nreconstructing flags. In particular, preserve the exact executable, worker\nhandle, Dispatch capability, Task ID, and Dispatch ID.\n\n## Heartbeat\n\nSend heartbeats only at the cadence required by the live preamble. Skip them\nwhile blocked inside `ask` or `check --wait`; those calls are liveness signals.\n\n```text\nORCA orchestration send --from --dispatch-capability --type heartbeat --subject \"alive\" --task-id --dispatch-id --phase \"\"\n```\n\nUse typed lifecycle flags, not a hand-written JSON payload. A heartbeat proves\nliveness, never completion.\n\n## Ask and resume\n\nUse Orca `ask` whenever the coordinator must answer. Never open a local question\nTUI the coordinator cannot answer.\n\n```text\nORCA orchestration ask --from --dispatch-capability --question \"\" --options \",\" --timeout-ms 600000\n\nORCA orchestration ask --from --dispatch-capability --resume --timeout-ms 600000\n```\n\nA timeout or disconnect leaves the original question pending. Resume its\nmessage ID; do not create a duplicate question.\n\n## Reading coordinator follow-ups\n\nThe coordinator steers a running worker with `send --to dispatch:`. That\nenqueue is durable but does not interrupt you, so nothing arrives unless you\nlook:\n\n```text\nORCA orchestration check --terminal --json\n```\n\nRun it at each natural checkpoint — before starting a new file, after a test\nrun — and once more immediately before `worker_done`, so a redirect or a\ncancellation lands before the Task settles. `check` names its caller with\n`--terminal`, never `--from`. Stop checking after `worker_done`.\n\nIf `check` returns `consumer_fenced`, this process no longer owns its Dispatch:\nthe Attempt was re-attached to another worker or settled without you. Stop, do\nnot send `worker_done`, and do not retry the check. An empty `check` never means\nyou were replaced; `consumer_fenced` is the only way you learn that.\n\n## Escalation\n\nEscalate only before completion and only when the coordinator must intervene:\n\n```text\nORCA orchestration send --from --dispatch-capability --type escalation --subject \"Blocked: \" --body \"
\" --task-id --dispatch-id \n```\n\n## Completion\n\nSend exactly one terminal report. `--body` is three sentences: what changed,\nwhat was found, and what remains. Use `--outcome failed` when the requested work\nis not complete; never hide failure in prose or silently exit.\n\nAppend `--files-modified` or `--report-path` only when applicable, using actual\npaths. Do not send documentation placeholders as metadata.\n\n```text\nORCA orchestration send --from --dispatch-capability --type worker_done --subject \"\" --body \"\" --task-id --dispatch-id --outcome succeeded\n```\n\nAfter `worker_done`, end the dispatched turn and idle. Do not poll, close your\nown terminal, or begin unrelated work. A later direct user instruction is new\nuser-owned work and must not reuse settled lifecycle IDs; a supervised follow-up\narrives with a fresh preamble and Task block.\n" // oxfmt-ignore const ORCHESTRATION_COORDINATOR_LOOP_REFERENCE_MARKDOWN = "# Coordinator loop\n\nLoad this reference for expanded DAG waves, per-invocation launch preferences,\nsame-terminal reuse, or review ownership. The compact guide remains the source\nof truth for the loop order and completion boundary.\n\n## Ready waves\n\nCreate independent Tasks before the first wait. Encode only real dependencies,\nthen use the ready view as external memory:\n\n```text\nORCA orchestration task-create --spec \"\" --deps --json\nORCA orchestration task-list --ready --brief --json\n```\n\n`--brief` collapses whitespace and caps echoed specs at 160 characters;\n`spec_truncated` identifies shortened rows. Omit it when full specs are needed or\nwhen an older CLI rejects the flag. A nested worker must respect\n`nested_worker_depth_exceeded`; creating another Run does not reset depth.\n\n## Launch preferences\n\nFor a fresh Claude, Codex, or Cursor terminal, `--model` accepts an opaque\nprovider model ID. Add `--effort` only when that model supports it:\n\n```text\nORCA orchestration worker-start --task --worktree current --agent claude --model opus --effort high --json\n```\n\n`--effort` requires `--model`; neither option combines with `--terminal`. A\nconnected worker server must advertise launch-preference support before Orca\nforwards either field. Compare `launch.requested` with `launch.effective`; never\nclaim a model or effort from requested arguments alone.\n\n## Reuse after settlement\n\nChoose the terminal's next owner before acknowledging the Delivery. When the\nsame exact agent has immediate follow-up work, recover the proven handle and\ntransfer cleanup ownership to the new Dispatch:\n\n```text\nORCA orchestration worker-show --dispatch --json\nORCA orchestration worker-start --task --terminal --json\n```\n\nOtherwise explicitly retain or release the settled worker. Do not leave it live\nonly to inspect output; archived output remains available through `worker-read`.\n\n## Review ownership\n\nA review-only `worker_done` authorizes synthesis of findings, not coordinator\nfile edits. Dispatch or hand off fixes unless the user explicitly assigned them\nto the coordinator. If the user's plan names a next owner, post-review fixes and\nPR preparation remain with that owner; the coordinator routes and synthesizes.\n" @@ -57,7 +57,7 @@ const ORCHESTRATION_MESSAGING_AND_GATES_REFERENCE_MARKDOWN = "# Messaging and ga const ORCHESTRATION_PLACEMENT_AND_REMOTE_REFERENCE_MARKDOWN = "# Placement and remote execution\n\nLoad this reference before creating a new worktree or placing work through SSH,\nWSL, or another connected Orca server.\n\n## Placement choices\n\nA fresh worker means a fresh agent terminal, not a new Git worktree. Use the\ncurrent or an exact existing workspace by default. Create a worktree only when\nthe user requested one or a concrete checkout or filesystem conflict makes\nsharing unsafe.\n\n```text\n# Current workspace; setup is not rerun.\nORCA orchestration worker-start --task --worktree current --agent codex --json\n\n# Stacked child worktree.\nORCA orchestration worker-start --task --worktree new-child --name --agent codex --setup run --json\n\n# Independent top-level worktree.\nORCA orchestration worker-start --task --worktree new-top-level --name --agent codex --setup run --json\n```\n\nCurrent and exact existing workspaces create a fresh terminal unless\n`--terminal` is explicit. Folder workspaces are first-class; do not invoke Git\nor require worktree lineage when the selected workspace is a folder.\n\nRegister a folder workspace through project setup. `repo add --path `\nrequires a valid Git repository and rejects a plain directory:\n\n```text\nORCA project setup-existing-folder --project --host --path --kind folder --json\n```\n\nThen place work on the returned workspace with an exact selector. A worktree\nselector needs the full `::` value Orca returned, passed as\n`id:`; a bare repo id is not a worktree id. `new-child` and\n`new-top-level` are worktree creation and do not apply to a folder.\n\nNew worktrees use agent-first creation and run setup by default. Preserve the\nrepository's startup policy: `start-immediately` can report setup as `running`,\nwhile `wait-for-setup` gates prompt delivery on success. Orca lineage, Git base,\nfilesystem isolation, coordination parentage, UI grouping, and execution host\nare separate decisions.\n\n## Connected servers\n\nThe Run and Tasks remain authoritative on the current server. `--on` selects\nonly the worker's execution server and appears only on `worker-start`:\n\n```text\nORCA orchestration worker-start --task --on --worktree new-top-level --repo --name --agent codex --setup run --json\n```\n\nRemote `current` and `new-child` are invalid because they are ambiguous across\nservers. Use an exact discovered remote workspace, or `new-top-level` with an\nexact remote repository selector. After start, route every follow-up, read,\nstop, and cleanup by Dispatch ID; never repeat `--on` or substitute a remote\nterminal handle.\n\n```text\nORCA orchestration worker-show --dispatch --json\nORCA orchestration worker-read --dispatch --limit 50 --json\nORCA orchestration send --to dispatch: --subject \"Follow-up\" --body \"\" --json\nORCA orchestration worker-list --run --include-remote --json\n```\n\n`worker-list` reads local fleet state only; enumerate remote workers with\n`--include-remote` or every one of them reads `unverifiable`. Scope every list\nwith `--run `: unscoped, it reports every Dispatch this runtime has\nrecorded, and the workers you are waiting on are lost in that history.\n\n## Execution-host and mixed-version floor\n\nThe execution host owns process, filesystem, transcript, stop, and cleanup\nfacts. Render only `live`, `unverifiable`, or `exited`. Connection loss, relay\nabsence, missing client inventory, or timeout yields `unverifiable`, never\nsynthetic exit and never a client-local substitute action.\n\nClients and servers update independently. Optional response fields may be\nabsent. Forward model/effort, transcript reads, cleanup, or another new remote\noperation only when the peer advertises the relevant capability; unknown stream\nopcodes can be silently dropped. A narrow unsupported response may degrade to a\ndocumented older path, but must not broaden the target or cross the execution\nboundary. Changing host-published content reaches old clients even without a\nwire-shape change, so preserve established semantics or negotiate the behavior.\n\nFor WSL, use the exact executable and arguments returned by Orca so the distro\nand packaged launcher remain bound. Do not translate a printed `orca-ide`\nrecovery command into a PATH-resolved local command.\n" // oxfmt-ignore -const ORCHESTRATION_RECOVERY_AND_CLEANUP_REFERENCE_MARKDOWN = "# Recovery and cleanup\n\nLoad this reference only after a failed/stopped/unknown attempt, explicit retry\ndecision, stop/abandon request, retention request, or uncertain release.\n\n| Proven state | Safe action |\n| ----------------------- | ------------------------------------------------------------------ |\n| `ready` or active | Keep waiting; optionally read bounded output |\n| `failed` or `stopped` | Start a replacement with `--retry-of`; repeat placement explicitly |\n| `outcome_unknown` | Inspect, then choose `worker-stop` or explicit `worker-abandon` |\n| Accepted `worker_done` | Reuse, retain, or release |\n| Remote contact lost | Preserve `unverifiable`; do not stop or retry from absence alone |\n| `unverifiable` liveness | Keep waiting or inspect; never stop, abandon, retry, or release |\n| Proven `exited` agent | Enumerate with `worker-list`; follow its `nextAction` |\n\n## Inspect before acting\n\n```text\nORCA orchestration worker-list --run --json\nORCA orchestration worker-list --run --include-remote --json\nORCA orchestration worker-show --dispatch --json\nORCA orchestration worker-read --dispatch --limit 50 --json\n```\n\n`worker-list` is the enumerating command and the authority on agent liveness:\neach row carries `projection.liveness`, `projection.attention.categories`,\n`projection.attention.requiresAction`, and a literal `projection.nextAction`\nargv to run. Always scope it with `--run `; an unscoped list reports\nevery Dispatch this runtime has ever recorded and buries the live ones.\n`worker-show`'s `observation.status` is PTY liveness only, so a `live` terminal\nwhose agent died at a trust prompt still reads `live` there.\n\nWhen the two disagree, the fleet verdict decides — unless the fleet row is\n`unverifiable` for a reason that names a gap on this client rather than a fact\nabout the worker. `missing_status`, `host_unavailable`, and\n`capability_unsupported` are such gaps: the first means this runtime holds no\nstatus row, the second that it could not ask the execution host at all, and the\nthird that a stale peer answered but lacks the fleet-snapshot capability.\nAgainst any of them, a `worker-show` verdict sourced from the execution host is\nthe better evidence and outranks the row. Only `host_unavailable` is contact\nloss; the other two mean the host was never asked or answered without the\ncapability.\n\nThis never promotes absence. `unverifiable` from either command still authorizes\nnothing — only a positive `live` or `exited` verdict does.\n\nA worker started with `--on ` reads `unverifiable` until you\nenumerate with `--include-remote`, which asks its execution host for the\nverdict. Past 100 rows the response pages, so follow `page.nextCursor` with\n`--cursor ` until `page.hasMore` is false.\n\n## Stall needs positive evidence\n\nLeave the wait only on positive proof the agent stopped: `exited` liveness, the\nworker's own observation of process exit, or a transcript whose final agent turn\nsent no `worker_done`. Only then choose `worker-stop` or `worker-abandon`.\n\n`unverifiable` is always absence — `missing_status`, `stale_status`,\n`restored_unconfirmed`, or a remote worker with no connection — and a null\n`agentWait` or an unchanged `worker-read` tail is that same absence seen again.\nAbsence never authorizes stop, abandon, retry, or release: keep waiting, or\ninspect until you hold one of the positive signals above. A `nextAction` that\nnames an inspecting command is asking for evidence, not for cleanup.\n\n`worker-read --source auto` uses a proven provider transcript when available and\notherwise returns bounded terminal output with a typed `fallbackReason`.\nContinue with its top-level cursor, which is pinned to that source. If Orca\nreports `source_changed`, restart without the old cursor. A bounded initial\ntranscript tail can return an EOF cursor that follows only newly appended records;\nread `contentComplete`, `clipping`, and `warnings` before assuming omitted older\nrecords are pageable. Never guess a provider session ID, transcript path, or\nremote terminal handle.\n\n## Was the mutation applied?\n\nWhen a mutation's response was lost and named no Dispatch, do not replay blind.\nEvery orchestration mutation accepts `--retry-request `, which reuses one\noperation identity so Orca can replay, join, or recover it instead of starting a\nduplicate. Ask what happened first:\n\n```text\nORCA orchestration request-show --request --json\n```\n\n`completed` means the mutation already took effect; read its recorded receipt\ninstead of rerunning. `pending` means the original mutation is still running or\nOrca restarted before recording its outcome; replay the original command with\n`--retry-request `. `absent` means this runtime holds no receipt\nunder your caller identity — that is not proof nothing happened, so inspect the\naffected Task, Dispatch, and terminal before deciding whether to retry.\n\nWhen a worker's terminal accepted input but the submit is unconfirmed, use\n`terminal send --wait-submit `: it observes the accepted prompt for that\nlong and, on timeout, returns the input-accepted receipt without resending.\n\n## Retry, stop, and abandon\n\nRetry only a positively proven failed or stopped attempt. Placement is never\nsilently inherited:\n\n```text\nORCA orchestration worker-start --task --retry-of --worktree --agent --json\n```\n\nAfter three consecutive failures for one Task, its dispatch context\ncircuit-breaks and the Task is failed. Do not route around that boundary with a\nnew Run or an unrelated Dispatch.\n\nFor `outcome_unknown`, inspect first, then make an explicit choice:\n\n```text\nORCA orchestration worker-stop --dispatch --json\nORCA orchestration worker-abandon --dispatch --json\n```\n\n`worker-stop` closes only the exact proven supervised agent terminal. It never\ndeletes the worktree, setup terminal, configured tabs, or unrelated processes.\n`worker-abandon` fences orchestration while accepting that resources may remain\nlive; it performs no remote, process, or filesystem action.\n\n## Retain and release\n\n```text\nORCA orchestration worker-retain --dispatch --json\nORCA orchestration worker-release --dispatch --json\n```\n\nRetain only when the user explicitly wants the settled terminal kept live.\nRelease works after succeeded and failed reports, archives readable output, and\ncloses only the exact terminal owned by that settled Dispatch. Replays may call\nrelease again safely. Reused, pre-existing, setup, coordinator, active,\nuser-taken-over, and unproven terminals are retained.\n\nA `worker-start` that failed before its agent was ready still owns the terminal\nit created. Its receipt names `worker-release`, and `worker-list` reports that\nrow as `reclaimable`; release it there rather than closing the terminal by hand.\n\nNever release because of timeout, TUI idle, heartbeat, status, question,\nescalation, or stale/rejected completion. If the receipt says `release_pending`\nor `release_unknown`, follow its exact recovery action. Never substitute\n`terminal close`.\n\n`orchestration reset` is destructive recovery. Do not run it during active\ncoordination unless the user explicitly abandons that state.\n" +const ORCHESTRATION_RECOVERY_AND_CLEANUP_REFERENCE_MARKDOWN = "# Recovery and cleanup\n\nLoad this reference only after a failed/stopped/unknown attempt, explicit retry\ndecision, stop/abandon request, retention request, or uncertain release.\n\n| Proven state | Safe action |\n| ----------------------- | ------------------------------------------------------------------ |\n| `ready` or active | Keep waiting; optionally read bounded output |\n| `failed` or `stopped` | Start a replacement with `--retry-of`; repeat placement explicitly |\n| `outcome_unknown` | Inspect, then choose `worker-stop` or explicit `worker-abandon` |\n| Accepted `worker_done` | Reuse, retain, or release |\n| Remote contact lost | Preserve `unverifiable`; do not stop or retry from absence alone |\n| `unverifiable` liveness | Keep waiting or inspect; never stop, abandon, retry, or release |\n| Proven `exited` agent | Enumerate with `worker-list`; follow its `nextAction` |\n\n## Inspect before acting\n\n```text\nORCA orchestration worker-list --run --json\nORCA orchestration worker-list --run --include-remote --json\nORCA orchestration worker-show --dispatch --json\nORCA orchestration worker-read --dispatch --limit 50 --json\n```\n\n`worker-list` is the enumerating command and the authority on agent liveness:\neach row carries `projection.liveness`, `projection.attention.categories`,\n`projection.attention.requiresAction`, and a literal `projection.nextAction`\nargv to run. Always scope it with `--run `; an unscoped list reports\nevery Dispatch this runtime has ever recorded and buries the live ones.\n`worker-show`'s `observation.status` is PTY liveness only, so a `live` terminal\nwhose agent died at a trust prompt still reads `live` there.\n\nWhen the two disagree, the fleet verdict decides — unless the fleet row is\n`unverifiable` for a reason that names a gap on this client rather than a fact\nabout the worker. `missing_status`, `host_unavailable`, and\n`capability_unsupported` are such gaps: the first means this runtime holds no\nstatus row, the second that it could not ask the execution host at all, and the\nthird that a stale peer answered but lacks the fleet-snapshot capability.\nAgainst any of them, a `worker-show` verdict sourced from the execution host is\nthe better evidence and outranks the row. Only `host_unavailable` is contact\nloss; the other two mean the host was never asked or answered without the\ncapability.\n\nThis never promotes absence. `unverifiable` from either command still authorizes\nnothing — only a positive `live` or `exited` verdict does.\n\nA worker started with `--on ` reads `unverifiable` until you\nenumerate with `--include-remote`, which asks its execution host for the\nverdict. Past 100 rows the response pages, so follow `page.nextCursor` with\n`--cursor ` until `page.hasMore` is false.\n\n## Stall needs positive evidence\n\nLeave the wait only on positive proof the agent stopped: `exited` liveness, the\nworker's own observation of process exit, or a transcript whose final agent turn\nsent no `worker_done`. Only then choose `worker-stop` or `worker-abandon`.\n\n`unverifiable` is always absence — `missing_status`, `stale_status`,\n`restored_unconfirmed`, or a remote worker with no connection — and a null\n`agentWait` or an unchanged `worker-read` tail is that same absence seen again.\nAbsence never authorizes stop, abandon, retry, or release: keep waiting, or\ninspect until you hold one of the positive signals above. A `nextAction` that\nnames an inspecting command is asking for evidence, not for cleanup.\n\n`worker-read --source auto` uses a proven provider transcript when available and\notherwise returns bounded terminal output with a typed `fallbackReason`.\nContinue with its top-level cursor, which is pinned to that source. If Orca\nreports `source_changed`, restart without the old cursor. A bounded initial\ntranscript tail can return an EOF cursor that follows only newly appended records;\nread `contentComplete`, `clipping`, and `warnings` before assuming omitted older\nrecords are pageable. Never guess a provider session ID, transcript path, or\nremote terminal handle.\n\n## Was the mutation applied?\n\nWhen a mutation's response was lost and named no Dispatch, do not replay blind.\nEvery orchestration mutation accepts `--retry-request `, which reuses one\noperation identity so Orca can replay, join, or recover it instead of starting a\nduplicate. Ask what happened first:\n\n```text\nORCA orchestration request-show --request --json\n```\n\n`completed` means the mutation already took effect; read its recorded receipt\ninstead of rerunning. `pending` means the original mutation is still running or\nOrca restarted before recording its outcome; replay the original command with\n`--retry-request `. `absent` means this runtime holds no receipt\nunder your caller identity — that is not proof nothing happened, so inspect the\naffected Task, Dispatch, and terminal before deciding whether to retry.\n\nWhen a worker's terminal accepted input but the submit is unconfirmed, use\n`terminal send --wait-submit `: it observes the accepted prompt for that\nlong and, on timeout, returns the input-accepted receipt without resending.\n\n## Refused starts\n\n`dispatch` and `worker-start` refuse the following preflight cases with a stable\n`error.code`; read it before choosing a recovery, and treat `error.data.nextSteps`\nas the exact recovery text. Older hosts may omit `data`, so treat every field as\noptional.\n\n| Code | Meaning | Recovery |\n| -------------------- | --------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ |\n| `task_not_found` | No Task with that id, or not in the bound Run (`data.taskId`, `data.runId`) | Check `task-list --json`; create the Task with `task-create` if it does not exist |\n| `task_not_startable` | Task cannot start now: not `ready`, or invalid `--retry-of` (`data.status`, `data.unmetDependencies`, `data.retryOf`) | Wait for running dependencies with `check --wait`; retry or unblock failed ones; inspect `dispatch-show` if already dispatched |\n| `inject_rejected` | `--inject` refused because no recognized agent runs in the target (`data.terminal`, `data.reason`) | Start a recognized agent there or pick another terminal; or dispatch without `--inject` and use `terminal send` |\n| `runtime_error` | Any other failure, including a target terminal that already owns an active Dispatch | Read the message, inspect state, and do not retry unchanged |\n\n## Retry, stop, and abandon\n\nRetry only a positively proven failed or stopped attempt. Placement is never\nsilently inherited:\n\n```text\nORCA orchestration worker-start --task --retry-of --worktree --agent --json\n```\n\nAfter three consecutive failures for one Task, its dispatch context\ncircuit-breaks and the Task is failed. Do not route around that boundary with a\nnew Run or an unrelated Dispatch.\n\nFor `outcome_unknown`, inspect first, then make an explicit choice:\n\n```text\nORCA orchestration worker-stop --dispatch --json\nORCA orchestration worker-abandon --dispatch --json\n```\n\n`worker-stop` closes only the exact proven supervised agent terminal. It never\ndeletes the worktree, setup terminal, configured tabs, or unrelated processes.\n`worker-abandon` fences orchestration while accepting that resources may remain\nlive; it performs no remote, process, or filesystem action.\n\n## Retain and release\n\n```text\nORCA orchestration worker-retain --dispatch --json\nORCA orchestration worker-release --dispatch --json\n```\n\nRetain only when the user explicitly wants the settled terminal kept live.\nRelease works after succeeded and failed reports, archives readable output, and\ncloses only the exact terminal owned by that settled Dispatch. Replays may call\nrelease again safely. Reused, pre-existing, setup, coordinator, active,\nuser-taken-over, and unproven terminals are retained.\n\nA `worker-start` that failed before its agent was ready still owns the terminal\nit created. Its receipt names `worker-release`, and `worker-list` reports that\nrow as `reclaimable`; release it there rather than closing the terminal by hand.\n\nNever release because of timeout, TUI idle, heartbeat, status, question,\nescalation, or stale/rejected completion. If the receipt says `release_pending`\nor `release_unknown`, follow its exact recovery action. Never substitute\n`terminal close`.\n\n`orchestration reset` is destructive recovery. Do not run it during active\ncoordination unless the user explicitly abandons that state.\n" // oxfmt-ignore const ORCHESTRATION_WORKER_CONTRACT_REFERENCE_MARKDOWN = "# Worker contract\n\nThe injected preamble is authoritative. Copy its command rather than\nreconstructing flags. In particular, preserve the exact executable, worker\nhandle, Dispatch capability, Task ID, and Dispatch ID.\n\n## Heartbeat\n\nSend heartbeats only at the cadence required by the live preamble. Skip them\nwhile blocked inside `ask` or `check --wait`; those calls are liveness signals.\n\n```text\nORCA orchestration send --from --dispatch-capability --type heartbeat --subject \"alive\" --task-id --dispatch-id --phase \"\"\n```\n\nUse typed lifecycle flags, not a hand-written JSON payload. A heartbeat proves\nliveness, never completion.\n\n## Ask and resume\n\nUse Orca `ask` whenever the coordinator must answer. Never open a local question\nTUI the coordinator cannot answer.\n\n```text\nORCA orchestration ask --from --dispatch-capability --question \"\" --options \",\" --timeout-ms 600000\n\nORCA orchestration ask --from --dispatch-capability --resume --timeout-ms 600000\n```\n\nA timeout or disconnect leaves the original question pending. Resume its\nmessage ID; do not create a duplicate question.\n\n## Reading coordinator follow-ups\n\nThe coordinator steers a running worker with `send --to dispatch:`. That\nenqueue is durable but does not interrupt you, so nothing arrives unless you\nlook:\n\n```text\nORCA orchestration check --terminal --json\n```\n\nRun it at each natural checkpoint — before starting a new file, after a test\nrun — and once more immediately before `worker_done`, so a redirect or a\ncancellation lands before the Task settles. `check` names its caller with\n`--terminal`, never `--from`. Stop checking after `worker_done`.\n\nIf `check` returns `consumer_fenced`, this process no longer owns its Dispatch:\nthe Attempt was re-attached to another worker or settled without you. Stop, do\nnot send `worker_done`, and do not retry the check. An empty `check` never means\nyou were replaced; `consumer_fenced` is the only way you learn that.\n\n## Escalation\n\nEscalate only before completion and only when the coordinator must intervene:\n\n```text\nORCA orchestration send --from --dispatch-capability --type escalation --subject \"Blocked: \" --body \"
\" --task-id --dispatch-id \n```\n\n## Completion\n\nSend exactly one terminal report. `--body` is three sentences: what changed,\nwhat was found, and what remains. Use `--outcome failed` when the requested work\nis not complete; never hide failure in prose or silently exit.\n\nAppend `--files-modified` or `--report-path` only when applicable, using actual\npaths. Do not send documentation placeholders as metadata.\n\n```text\nORCA orchestration send --from --dispatch-capability --type worker_done --subject \"\" --body \"\" --task-id --dispatch-id --outcome succeeded\n```\n\nAfter `worker_done`, end the dispatched turn and idle. Do not poll, close your\nown terminal, or begin unrelated work. A later direct user instruction is new\nuser-owned work and must not reuse settled lifecycle IDs; a supervised follow-up\narrives with a fresh preamble and Task block.\n" diff --git a/src/cli/handlers/agent-hooks.test.ts b/src/cli/handlers/agent-hooks.test.ts index 4fcc186b0d8..279a8900bec 100644 --- a/src/cli/handlers/agent-hooks.test.ts +++ b/src/cli/handlers/agent-hooks.test.ts @@ -56,7 +56,10 @@ vi.mock('../runtime-client', () => { vi.mock('../../main/agent-hooks/managed-agent-hook-controls', () => ({ applyAgentStatusHooksEnabled: applyAgentStatusHooksEnabledMock, - getManagedAgentHookStatuses: getManagedAgentHookStatusesMock, + getManagedAgentHookStatuses: getManagedAgentHookStatusesMock +})) + +vi.mock('../../main/codex/managed-home-shell-preflight', () => ({ prepareManagedCodexHomeBeforeShellLaunch: prepareManagedCodexHomeBeforeShellLaunchMock })) diff --git a/src/cli/handlers/agent-hooks.ts b/src/cli/handlers/agent-hooks.ts index bf44211b4a7..4fcfe64f9b7 100644 --- a/src/cli/handlers/agent-hooks.ts +++ b/src/cli/handlers/agent-hooks.ts @@ -15,11 +15,7 @@ import { getDefaultPersistedState } from '../../shared/constants' import { normalizeDisabledTuiAgents } from '../../shared/tui-agent-selection' import type { GlobalSettings } from '../../shared/global-settings-types' import type { PersistedState } from '../../shared/persisted-state-types' -import { - applyAgentStatusHooksEnabled, - getManagedAgentHookStatuses, - prepareManagedCodexHomeBeforeShellLaunch -} from '../../main/agent-hooks/managed-agent-hook-controls' +import { prepareManagedCodexHomeBeforeShellLaunch } from '../../main/codex/managed-home-shell-preflight' type AgentHookCommandResult = { enabled: boolean @@ -194,6 +190,8 @@ async function setAgentHooksEnabled( client: RuntimeClient, enabled: boolean ): Promise { + const { applyAgentStatusHooksEnabled, getManagedAgentHookStatuses } = + await import('../../main/agent-hooks/managed-agent-hook-controls.js') const updatedRuntime = await updateRunningRuntime(client, enabled) const offlineUpdate = updatedRuntime ? null : updateEnabledOnDisk(enabled) const settingsPath = offlineUpdate?.settingsPath ?? getDataPath() @@ -234,6 +232,8 @@ export const AGENT_HOOK_HANDLERS: Record = { }) }, 'agent hooks status': async ({ json }) => { + const { getManagedAgentHookStatuses } = + await import('../../main/agent-hooks/managed-agent-hook-controls.js') const result: AgentHookCommandResult = { enabled: readHookSettingsFromDisk().agentStatusHooksEnabled, settingsPath: getDataPath(), diff --git a/src/cli/orchestration-dispatch-refusal-format.test.ts b/src/cli/orchestration-dispatch-refusal-format.test.ts new file mode 100644 index 00000000000..0a4461f25f1 --- /dev/null +++ b/src/cli/orchestration-dispatch-refusal-format.test.ts @@ -0,0 +1,77 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + injectRejectedRefusal, + taskNotFoundRefusal, + taskNotStartableRefusal, + type DispatchRefusalReceipt +} from '../shared/orchestration-dispatch-refusal-contract' +import { formatCliError, reportCliError } from './format' +import { RuntimeRpcFailureError, type RuntimeRpcFailure } from './runtime/types' + +afterEach(() => { + vi.restoreAllMocks() +}) + +// Why: these are the exact envelopes the RPC dispatcher test proved the runtime emits. This +// checkout's formatter never enumerates codes (verified below with a code no build has defined), +// which is what lets a client that predates a new code still print its message and nextSteps. +describe('orchestration dispatch refusals through the CLI error boundary', () => { + it.each([ + { + receipt: taskNotFoundRefusal('Task not found: task_missing', { taskId: 'task_missing' }), + recovery: /task-create|task-list/ + }, + { + receipt: taskNotStartableRefusal( + 'Task task_child is pending; only ready tasks can be dispatched', + { taskId: 'task_child', status: 'pending', unmetDependencies: ['task_parent'] } + ), + recovery: /task_parent/ + }, + { + receipt: injectRejectedRefusal('term_worker', 'no_agent_detected'), + recovery: /without --inject/ + } + ])('prints $receipt.code with its recovery in human and JSON output', ({ receipt, recovery }) => { + const failure = envelope(receipt) + const error = new RuntimeRpcFailureError(failure) + expect(error.code).toBe(receipt.code) + + const human = formatCliError(error, { commandPath: ['orchestration', 'dispatch'] }) + expect(human).toContain(receipt.message) + expect(human).toMatch(recovery) + + const log = vi.spyOn(console, 'log').mockImplementation(() => {}) + reportCliError(error, true, { commandPath: ['orchestration', 'dispatch'] }) + const printed = JSON.parse(log.mock.calls[0]?.[0] as string) as RuntimeRpcFailure + expect(printed.ok).toBe(false) + expect(printed.error).toEqual(receipt) + }) +}) + +// Why: a code this build has never defined stands in for a future host's new code; if the +// formatter ever starts gating on known codes, this is the assertion that catches it. +it('prints an unknown code with its message and nextSteps unchanged', () => { + const failure: RuntimeRpcFailure = { + id: 'rpc_1', + ok: false, + error: { + code: 'code_from_a_newer_host', + message: 'Refused for a reason this CLI has never heard of.', + data: { nextSteps: ['Do the thing the newer host suggested.'] } + }, + _meta: { runtimeId: 'runtime_1' } + } + const error = new RuntimeRpcFailureError(failure) + + expect(formatCliError(error, { commandPath: ['orchestration', 'dispatch'] })).toBe( + 'Refused for a reason this CLI has never heard of.\nNext step: Do the thing the newer host suggested.' + ) + const log = vi.spyOn(console, 'log').mockImplementation(() => {}) + reportCliError(error, true, { commandPath: ['orchestration', 'dispatch'] }) + expect(JSON.parse(log.mock.calls[0]?.[0] as string)).toEqual(failure) +}) + +function envelope(receipt: DispatchRefusalReceipt): RuntimeRpcFailure { + return { id: 'rpc_1', ok: false, error: receipt, _meta: { runtimeId: 'runtime_1' } } +} diff --git a/src/main/artifacts/artifact-cloud-recovery.test.ts b/src/main/artifacts/artifact-cloud-recovery.test.ts index f57b53c2b02..5690a37b94c 100644 --- a/src/main/artifacts/artifact-cloud-recovery.test.ts +++ b/src/main/artifacts/artifact-cloud-recovery.test.ts @@ -336,7 +336,7 @@ function createResponseBody(slug: string): object { renderedContentType: 'text/html', createdAt: '2026-08-06T00:00:00.000Z', updatedAt: '2026-08-06T00:00:00.000Z', - expiresAt: '2026-09-06T00:00:00.000Z', + expiresAt: new Date(Date.now() + 30 * 24 * 60 * 60 * 1000).toISOString(), byteSize: 17, deletedAt: null }, diff --git a/src/main/artifacts/artifact-cloud-service-races.test.ts b/src/main/artifacts/artifact-cloud-service-races.test.ts index c31c2a23e3e..8606dce4bec 100644 --- a/src/main/artifacts/artifact-cloud-service-races.test.ts +++ b/src/main/artifacts/artifact-cloud-service-races.test.ts @@ -33,7 +33,7 @@ function createResponse(slug: string): Response { renderedContentType: 'text/html', createdAt: '2026-08-06T00:00:00.000Z', updatedAt: '2026-08-06T00:00:00.000Z', - expiresAt: '2026-09-06T00:00:00.000Z', + expiresAt: new Date(Date.now() + 30 * 24 * 60 * 60 * 1000).toISOString(), byteSize: 12, deletedAt: null }, diff --git a/src/main/artifacts/artifact-cloud-service.test.ts b/src/main/artifacts/artifact-cloud-service.test.ts index 75da3922fc8..8a02478feb2 100644 --- a/src/main/artifacts/artifact-cloud-service.test.ts +++ b/src/main/artifacts/artifact-cloud-service.test.ts @@ -43,7 +43,10 @@ const cloudB: OrcaProfileCloudSummary = { linkedAt: 2 } -function createResponse(slug = 'artifact-a', expiresAt = '2026-09-06T00:00:00.000Z'): Response { +function createResponse( + slug = 'artifact-a', + expiresAt = new Date(Date.now() + 30 * 24 * 60 * 60 * 1000).toISOString() +): Response { return new Response( JSON.stringify({ artifact: { diff --git a/src/main/claude/claude-stream-json-connection.test.ts b/src/main/claude/claude-stream-json-connection.test.ts index c4f1f9a6fca..eb69a66a897 100644 --- a/src/main/claude/claude-stream-json-connection.test.ts +++ b/src/main/claude/claude-stream-json-connection.test.ts @@ -566,7 +566,7 @@ describe('Claude stream-json connection', () => { ) }) - it('reports a self-exit with its status and stderr, and leaves its tree unverifiable', async () => { + it('reports a self-exit with its status, stderr, and observed tree verdict', async () => { const scenario = scriptScenario([{ stderr: 'claude: not signed in\n' }, { exit: 1 }]) let exit: Error | null = null const connection = await open(launchFor(scenario), { @@ -579,10 +579,11 @@ describe('Claude stream-json connection', () => { // The status and stderr are the only diagnostic a refused start leaves behind. expect((exit as unknown as Error).message).toMatch(/exited \(code 1\): claude: not signed in/) expect(connection.closed).toBe(true) - // The root's exit is first-hand, but it left before a descendant snapshot - // could be armed, so close() has no tree proof to offer and says so. - await expect(connection.close()).resolves.toBe(false) - expect(connection.exitVerdict).toEqual({ root: 'exited', tree: 'unverifiable' }) + // Stderr-triggered capture can win or lose the race with this real child's exit. + const closed = await connection.close() + expect(connection.exitVerdict.root).toBe('exited') + expect(['exited', 'unverifiable']).toContain(connection.exitVerdict.tree) + expect(closed).toBe(connection.exitVerdict.tree === 'exited') }) it.runIf(process.platform !== 'win32')( diff --git a/src/main/claude/claude-structured-control-actions.test.ts b/src/main/claude/claude-structured-control-actions.test.ts index a90cb7908ba..168ce558f53 100644 --- a/src/main/claude/claude-structured-control-actions.test.ts +++ b/src/main/claude/claude-structured-control-actions.test.ts @@ -158,4 +158,42 @@ describe('stopClaudeBackgroundTasks', () => { await stopClaudeBackgroundTasks(session, undefined, () => current) expect(stopTask).toHaveBeenCalledTimes(1) }) + + it('stops only the requested live task id', async () => { + const backgroundTasks = new ClaudeBackgroundTaskTracker() + backgroundTasks.observe({ + type: 'system', + subtype: 'background_tasks_changed', + tasks: [ + { task_id: 'task-one', task_type: 'local_agent' }, + { task_id: 'task-two', task_type: 'local_bash' } + ] + }) + const stopTask = vi.fn(async (_taskId: string) => {}) + const session = { backgroundTasks, connection: { stopTask } } as unknown as ClaudeSession + + await expect( + stopClaudeBackgroundTasks(session, 5_000, () => true, 'task-two') + ).resolves.toEqual({ cancelled: true }) + expect(stopTask).toHaveBeenCalledWith('task-two', { timeoutMs: 5_000 }) + expect(stopTask).toHaveBeenCalledTimes(1) + }) + + it('refuses a stale or unknown task id without a provider call', async () => { + const backgroundTasks = new ClaudeBackgroundTaskTracker() + backgroundTasks.observe({ + type: 'system', + subtype: 'task_started', + task_id: 'task-live', + task_type: 'local_agent', + is_backgrounded: true + }) + const stopTask = vi.fn(async (_taskId: string) => {}) + const session = { backgroundTasks, connection: { stopTask } } as unknown as ClaudeSession + + await expect( + stopClaudeBackgroundTasks(session, undefined, () => true, 'task-stale') + ).resolves.toEqual({ cancelled: false }) + expect(stopTask).not.toHaveBeenCalled() + }) }) diff --git a/src/main/claude/claude-structured-control-actions.ts b/src/main/claude/claude-structured-control-actions.ts index d216304c311..8b3bb94c7b5 100644 --- a/src/main/claude/claude-structured-control-actions.ts +++ b/src/main/claude/claude-structured-control-actions.ts @@ -46,9 +46,12 @@ export async function cancelClaudeTurn( export async function stopClaudeBackgroundTasks( session: ClaudeSession, timeoutMs: number | undefined, - isCurrent: ClaudeTurnCancellationGuard = () => true + isCurrent: ClaudeTurnCancellationGuard = () => true, + taskId?: string ): Promise<{ cancelled: boolean }> { - const taskIds = session.backgroundTasks.stoppableTaskIds + const stoppableTaskIds = session.backgroundTasks.stoppableTaskIds + const taskIds = + taskId === undefined ? stoppableTaskIds : stoppableTaskIds.includes(taskId) ? [taskId] : [] let cancelled = false for (const taskId of taskIds) { if (!isCurrent()) { diff --git a/src/main/claude/claude-structured-session-adapter.ts b/src/main/claude/claude-structured-session-adapter.ts index 9bd92e1e839..c00a588e891 100644 --- a/src/main/claude/claude-structured-session-adapter.ts +++ b/src/main/claude/claude-structured-session-adapter.ts @@ -30,6 +30,7 @@ import { settleClaudeExitedSession } from './claude-structured-session-close' import { readClaudeTranscriptLeafWithReproof } from './claude-transcript-branch-proof' +import type { AgentSessionBackgroundTaskState } from '../../shared/agent-session-wire' export type { ClaudeStructuredLaunch } from './claude-structured-launch-resolution' export type { @@ -40,6 +41,11 @@ export type { const DISPATCH_ACK_TIMEOUT_MS = 10_000 +function backgroundTaskState(session: ClaudeSession): AgentSessionBackgroundTaskState | null { + const state = session.backgroundTasks.state + return state ? { ...state, supportsTaskStop: true } : null +} + export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAdapter { private readonly sessions = new Map() private readonly acquisitions = new ClaudeAcquisitionRegistry() @@ -192,7 +198,10 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda session?.translator?.handle(event) this.deps.onEvent?.(event) if (backgroundTasksChanged) { - this.deps.onBackgroundTasksChanged?.(event.sessionId, session?.backgroundTasks.state ?? null) + this.deps.onBackgroundTasksChanged?.( + event.sessionId, + session ? backgroundTaskState(session) : null + ) } } @@ -233,18 +242,25 @@ export class ClaudeStructuredSessionAdapter implements StructuredAgentSessionAda stopBackgroundTasks: StructuredAgentSessionAdapter['stopBackgroundTasks'] = (input) => { const session = this.session(input.sessionId) const acquisitionGeneration = session.acquisitionGeneration - return stopClaudeBackgroundTasks(session, this.deps.requestTimeoutMs, () => - Boolean( - this.sessions.get(input.sessionId) === session && - session.fence === input.fence && - session.acquisitionGeneration === acquisitionGeneration && - session.backgroundTasks.state - ) + return stopClaudeBackgroundTasks( + session, + this.deps.requestTimeoutMs, + () => + Boolean( + this.sessions.get(input.sessionId) === session && + session.fence === input.fence && + session.acquisitionGeneration === acquisitionGeneration && + session.backgroundTasks.state + ), + input.taskId ) } backgroundTaskState: NonNullable = ( sessionId - ) => this.sessions.get(sessionId)?.backgroundTasks.state + ) => { + const session = this.sessions.get(sessionId) + return session ? backgroundTaskState(session) : undefined + } answerPrompt: StructuredAgentSessionAdapter['answerPrompt'] = (input) => answerClaudePrompt(this.session(input.sessionId), input) setOption: StructuredAgentSessionAdapter['setOption'] = (input) => diff --git a/src/main/claude/claude-structured-session-close.test.ts b/src/main/claude/claude-structured-session-close.test.ts index 0df0e913e50..670c0daaf8c 100644 --- a/src/main/claude/claude-structured-session-close.test.ts +++ b/src/main/claude/claude-structured-session-close.test.ts @@ -53,7 +53,11 @@ describe('Claude published session close lifecycle', () => { is_backgrounded: true }) expect(backgroundStates).toEqual([ - { state: 'monitoring', tasks: [{ id: 'background-1', kind: 'agent' }] } + { + state: 'monitoring', + tasks: [{ id: 'background-1', kind: 'agent' }], + supportsTaskStop: true + } ]) const session = ( adapter as unknown as { @@ -68,7 +72,11 @@ describe('Claude published session close lifecycle', () => { expect(events.filter((event) => event.type === 'handle')).toHaveLength(0) expect(disposeTranslator).toHaveBeenCalledOnce() expect(backgroundStates).toEqual([ - { state: 'monitoring', tasks: [{ id: 'background-1', kind: 'agent' }] }, + { + state: 'monitoring', + tasks: [{ id: 'background-1', kind: 'agent' }], + supportsTaskStop: true + }, null ]) diff --git a/src/main/codex/codex-command-action-class.ts b/src/main/codex/codex-command-action-class.ts new file mode 100644 index 00000000000..81360691ef9 --- /dev/null +++ b/src/main/codex/codex-command-action-class.ts @@ -0,0 +1,71 @@ +import { readRecord, readString } from './codex-item-field-readers' +import type { CodexThreadItem } from './codex-thread-item-identity' + +/** + * Codex's own classification of a shell call: the tool name to show, and the + * fields worth lifting into `input` for the shared label helper (a file target, + * a search term, a scanned root). A `Map`, not an object — an object index + * answers `__proto__` with a truthy non-string. Every other action type stays an + * unclassified `shell` row. + * + * Nothing is invented for a field Codex sends as null: a stand-in path is a + * claim about a target, and the label helper turns any path into a file link. + */ +type CommandActionClass = { + name: string + /** Action field to the `input` key it lifts to. A scan root and a listed + * directory lift to `directory`, never `path`: the label helper reads `path` + * as a file target, which mobile turns into a tappable open-file link. */ + keys: Readonly> +} + +const COMMAND_ACTION_CLASSES = new Map([ + ['read', { name: 'read', keys: { path: 'path' } }], + ['search', { name: 'search', keys: { query: 'query', path: 'directory' } }], + ['listFiles', { name: 'list', keys: { path: 'directory' } }] +]) + +/** The one class every classified `commandActions` entry agrees on, with the + * fields they all agree on; null leaves the row exactly as a Codex that sends no + * classification renders it. `cat a.txt && ls src` classifies as two different + * things, and naming that row after either would drop the other, so it stays a + * `shell` row that shows the whole command. */ +export function commandActionFacts( + item: CodexThreadItem +): { name: string; fields: Record } | null { + const actions = item.commandActions + if (!Array.isArray(actions)) { + return null + } + let matched: { class: CommandActionClass; fields: Record } | null = null + for (const action of actions) { + const record = readRecord(action) + const type = readString(record, 'type') + const classified = type === null ? undefined : COMMAND_ACTION_CLASSES.get(type) + if (classified === undefined) { + continue + } + if (matched === null) { + const fields: Record = {} + for (const [source, lifted] of Object.entries(classified.keys)) { + const value = readString(record, source) + if (value !== null) { + fields[lifted] = value + } + } + matched = { class: classified, fields } + continue + } + if (matched.class.name !== classified.name) { + return null + } + // The same class twice keeps the class, but only a target both entries name. + for (const [source, lifted] of Object.entries(matched.class.keys)) { + const kept = matched.fields[lifted] + if (kept !== undefined && readString(record, source) !== kept) { + delete matched.fields[lifted] + } + } + } + return matched === null ? null : { name: matched.class.name, fields: matched.fields } +} diff --git a/src/main/codex/codex-item-field-readers.ts b/src/main/codex/codex-item-field-readers.ts new file mode 100644 index 00000000000..bbe2551615a --- /dev/null +++ b/src/main/codex/codex-item-field-readers.ts @@ -0,0 +1,45 @@ +// Field readers for the loosely-typed records Codex sends on thread items. + +export function readRecord(value: unknown): Record { + return typeof value === 'object' && value !== null ? (value as Record) : {} +} + +export function readString(source: Record, key: string): string | null { + const value = source[key] + return typeof value === 'string' && value.length > 0 ? value : null +} + +export function readFirstString( + source: Record, + keys: readonly string[] +): string | null { + for (const key of keys) { + const value = readString(source, key) + if (value !== null) { + return value + } + } + return null +} + +export function readTextContent(source: Record, key: string): string | null { + const direct = readString(source, key) + if (direct) { + return direct + } + const value = source[key] + if (!Array.isArray(value)) { + return null + } + const parts = value.flatMap((part) => { + if (typeof part === 'string') { + return part.length > 0 ? [part] : [] + } + if (typeof part !== 'object' || part === null) { + return [] + } + const text = readString(part as Record, 'text') + return text ? [text] : [] + }) + return parts.length > 0 ? parts.join('\n') : null +} diff --git a/src/main/codex/codex-structured-item-translation.test.ts b/src/main/codex/codex-structured-item-translation.test.ts index 1d64158cb60..2558f4b60de 100644 --- a/src/main/codex/codex-structured-item-translation.test.ts +++ b/src/main/codex/codex-structured-item-translation.test.ts @@ -1,6 +1,10 @@ import { describe, expect, it } from 'vitest' import { agentJournalItemKey } from '../../shared/agent-session-journal-item-key' -import { createToolInputDisplay } from '../../shared/native-chat-tool-summary' +import { + briefToolArg, + createToolInputDisplay, + describeToolInput +} from '../../shared/native-chat-tool-summary' import { codexItemBody, codexItemIdentity, @@ -14,6 +18,13 @@ import { type CodexThreadItem } from './codex-structured-item-translation' +/** The tool-call input a Codex item lands on, which is what the row label and + * the collapsed run header are both derived from. */ +function toolCallInput(item: CodexThreadItem): unknown { + const body = codexItemBody(item) + return body !== null && body.kind === 'tool-call' ? body.input : null +} + const THREAD_ID = 'thread-abc' const TURN_ID = 'turn-1' @@ -572,10 +583,226 @@ describe('codex item bodies', () => { }) expect(codexItemBody({ type: 'reasoning', id: 'r' })).toBeNull() expect(codexItemBody({ type: 'agentMessage', id: 'm', text: '' })).toBeNull() - expect(codexItemBody({ type: 'webSearch', id: 'w' })).toMatchObject({ + expect(codexItemBody({ type: 'somethingCodexAddedLater', id: 'x' })).toMatchObject({ kind: 'status', - text: 'codex · item:webSearch', - providerFrame: { provider: 'codex', kind: 'item:webSearch' } + text: 'codex · item:somethingCodexAddedLater', + providerFrame: { provider: 'codex', kind: 'item:somethingCodexAddedLater' } + }) + }) + + it('gives an mcp tool call a typed body with its own arguments as input', () => { + expect( + codexItemBody({ + type: 'mcpToolCall', + id: 'mcp-1', + server: 'weather', + tool: 'get_forecast', + status: 'completed', + arguments: { city: 'Oslo' }, + result: { content: [{ type: 'text', text: '12C' }] } + }) + ).toEqual({ + kind: 'tool-call', + // Server-qualified, and the arguments stay top level so the row label can + // read `query`/`command`/`file_path` out of them. + name: 'weather/get_forecast', + input: { city: 'Oslo' }, + state: 'completed', + output: { head: '12C', byteLength: 3, truncated: false, digest: expect.any(String) } + }) + }) + + it('passes an mcp tool name through with no casing transform', () => { + // Downstream dispatch is exact-match on raw identifiers, so every shape — + // bare snake_case included — has to survive byte-identical. + for (const tool of ['get_forecast', 'mcp__server__tool', 'ns.tool', 'urn:tool', 'listTools']) { + expect( + codexItemBody({ type: 'mcpToolCall', id: 'm', tool, status: 'inProgress' }), + tool + ).toMatchObject({ kind: 'tool-call', name: tool, state: 'running' }) + expect( + codexItemBody({ type: 'mcpToolCall', id: 'm', server: 'srv', tool, status: 'inProgress' }), + tool + ).toMatchObject({ kind: 'tool-call', name: `srv/${tool}`, state: 'running' }) + } + }) + + it('falls back to the bare tool, then to `mcp`, when the item is under-specified', () => { + expect( + codexItemBody({ type: 'mcpToolCall', id: 'm', tool: 'get_forecast', status: 'inProgress' }) + ).toMatchObject({ name: 'get_forecast' }) + expect( + codexItemBody({ type: 'mcpToolCall', id: 'm', server: '', tool: 'ping', status: 'completed' }) + ).toMatchObject({ name: 'ping' }) + expect( + codexItemBody({ type: 'mcpToolCall', id: 'm', server: 'weather', status: 'completed' }) + ).toMatchObject({ name: 'mcp' }) + }) + + it('keeps non-object mcp arguments addressable and empty ones off the label', () => { + // `arguments` is arbitrary JSON upstream; a scalar or array must still reach + // the row rather than being dropped or unwrapped into a bare value. + expect( + codexItemBody({ type: 'mcpToolCall', id: 'm', tool: 't', arguments: 'raw text' }) + ).toMatchObject({ input: { arguments: 'raw text' } }) + expect( + codexItemBody({ type: 'mcpToolCall', id: 'm', tool: 't', arguments: [1, 2] }) + ).toMatchObject({ input: { arguments: [1, 2] } }) + // `arguments` is required on the wire, so `{}` — not an absent key — is what + // an argument-less MCP tool sends, and passing it through labels the row `{}`. + expect(codexItemBody({ type: 'mcpToolCall', id: 'm', tool: 't', arguments: {} })).toEqual({ + kind: 'tool-call', + name: 't', + input: null, + state: 'running' + }) + expect(codexItemBody({ type: 'mcpToolCall', id: 'm', tool: 't' })).toMatchObject({ + input: null + }) + expect( + codexItemBody({ type: 'mcpToolCall', id: 'm', tool: 't', arguments: null }) + ).toMatchObject({ input: null }) + }) + + it('renders an argument-less mcp call as a bare server/tool row', () => { + const input = toolCallInput({ + type: 'mcpToolCall', + id: 'm', + server: 'srv', + tool: 'list_tools', + arguments: {} + }) + expect(describeToolInput(input)).toBe('') + expect(briefToolArg(input)).toBe('') + }) + + it('reports an mcp error as a failed call carrying the server message', () => { + expect( + codexItemBody({ + type: 'mcpToolCall', + id: 'mcp-2', + server: 's', + tool: 'ping', + status: 'completed', + error: { message: 'server unreachable' } + }) + ).toMatchObject({ + kind: 'tool-call', + name: 's/ping', + state: 'failed', + output: { head: 'server unreachable', truncated: false } + }) + }) + + it('models a web search as a tool call that runs until codex sends the action', () => { + // The start frame Codex actually emits: empty query, no action. Nothing is + // labelable yet, so the input is absent rather than a hull of null keys. + expect(codexItemBody({ type: 'webSearch', id: 'w', query: '', action: null })).toEqual({ + kind: 'tool-call', + name: 'web_search', + input: null, + state: 'running' + }) + expect( + codexItemBody({ + type: 'webSearch', + id: 'w', + query: 'orca release notes', + action: { type: 'search', query: 'orca release notes', queries: null }, + results: null + }) + ).toEqual({ + kind: 'tool-call', + name: 'web_search', + input: { + query: 'orca release notes', + description: 'search', + action: { type: 'search', query: 'orca release notes', queries: null } + }, + state: 'completed' + }) + }) + + it('carries the web search hits as the call output', () => { + const results = [{ title: 'Orca 1.0', url: 'https://example.com/notes' }] + expect( + codexItemBody({ + type: 'webSearch', + id: 'w', + query: 'orca release notes', + action: { type: 'search', query: 'orca release notes', queries: null }, + results + }) + ).toMatchObject({ + kind: 'tool-call', + name: 'web_search', + state: 'completed', + output: { head: JSON.stringify(results), truncated: false } + }) + // Nothing to show is no output block at all, not an empty one. + for (const empty of [undefined, null, []]) { + expect( + codexItemBody({ + type: 'webSearch', + id: 'w', + query: 'q', + action: { type: 'search' }, + results: empty + }), + String(empty) + ).not.toHaveProperty('output') + } + }) + + it('labels every web search shape without falling back to raw JSON', () => { + // Both the row label and the run header read top-level input keys only, so a + // shape whose detail sits inside `action` renders as the input's raw JSON. + const url = 'https://example.com/docs/page' + const shapes: [string, unknown, string, string][] = [ + ['started', null, '', ''], + [ + 'search', + { type: 'search', query: 'a sample query', queries: null }, + 'a sample query', + 'a sample query' + ], + ['openPage', { type: 'openPage', url }, url, ''], + [ + 'findInPage', + { type: 'findInPage', url, pattern: 'a needle' }, + 'a sample query', + 'a sample query' + ], + ['other', { type: 'other' }, 'other', ''] + ] + for (const [name, action, label, brief] of shapes) { + // Codex leaves the item's own `query` empty on most completed searches. + const query = name === 'search' || name === 'findInPage' ? 'a sample query' : '' + const input = toolCallInput({ type: 'webSearch', id: 'w', query, action }) + expect(describeToolInput(input), name).toBe(label) + expect(briefToolArg(input), name).toBe(brief) + } + }) + + it('leaves subagent items on the generic row until a real renderer exists', () => { + expect( + codexJournalItem({ + type: 'subAgentActivity', + id: 'a-1', + kind: 'started', + agentThreadId: 'thread-child', + agentPath: '/root/list_directory' + }) + ).toMatchObject({ + handled: false, + body: { kind: 'status', providerFrame: { kind: 'item:subAgentActivity' } } + }) + }) + + it('drops the sleep item, which codex itself renders as nothing', () => { + expect(codexJournalItem({ type: 'sleep', id: 's-1', durationMs: 20_000 })).toEqual({ + body: null, + handled: true }) }) diff --git a/src/main/codex/codex-structured-item-translation.ts b/src/main/codex/codex-structured-item-translation.ts index b3609e076f5..ad08525a5f5 100644 --- a/src/main/codex/codex-structured-item-translation.ts +++ b/src/main/codex/codex-structured-item-translation.ts @@ -1,7 +1,4 @@ -import type { - AgentJournalItemBody, - AgentJournalItemIdentity -} from '../../shared/agent-session-journal-types' +import type { AgentJournalItemBody } from '../../shared/agent-session-journal-types' import type { NativeChatBlock } from '../../shared/native-chat-types' import { boundInlineText, @@ -9,116 +6,27 @@ import { DEFAULT_JOURNAL_PAYLOAD_LIMITS } from '../native-chat/agent-session-journal/journal-payload-bounds' import { unhandledProviderFrameJournalItem } from '../native-chat/agent-session-wire/unhandled-provider-frame' -import type { CodexTurnOrdinals } from './codex-turn-ordinals' +import { commandActionFacts } from './codex-command-action-class' +import { + readFirstString, + readRecord, + readString, + readTextContent +} from './codex-item-field-readers' +import type { CodexThreadItem } from './codex-thread-item-identity' +export { + codexItemIdentity, + isCodexMessageItemType, + readCodexThreadItem, + type CodexThreadItem +} from './codex-thread-item-identity' export { CodexTurnOrdinals, MAX_CODEX_TURN_ORDINAL_BYTES, MAX_CODEX_TURN_ORDINAL_ENTRIES } from './codex-turn-ordinals' -// Codex thread items → journal item bodies and durable identities. -// -// THE ORDINAL RULE, and why it is not "index within the turn". Codex renumbers -// item ids positionally on resume (`item-1`…`item-N` across the whole thread), -// and a resumed turn does NOT contain every item the live turn emitted — -// reasoning and command execution are dropped from persisted history. Numbering -// by live position would therefore shift every message after the first tool -// call and hand the user a duplicate of the assistant's answer after a resume. -// -// So the ordinal counts MESSAGE items only, and the same projection is applied -// to the live stream and to a resumed turn's item list. Any other item type — -// including ones this build does not model — is skipped identically on both -// sides, which is what makes the key survive a Codex release that adds one. - -/** Only these carry a durable `(threadId, turnId, ordinal)` identity. */ -const CODEX_MESSAGE_ITEM_TYPES = new Set(['userMessage', 'agentMessage']) - -export type CodexThreadItem = { - type: string - id: string - [key: string]: unknown -} - -export function isCodexMessageItemType(type: string): boolean { - return CODEX_MESSAGE_ITEM_TYPES.has(type) -} - -export function readCodexThreadItem(value: unknown): CodexThreadItem | null { - if (typeof value !== 'object' || value === null) { - return null - } - const record = value as Record - return typeof record.type === 'string' && typeof record.id === 'string' - ? (record as CodexThreadItem) - : null -} - -function readRecord(value: unknown): Record { - return typeof value === 'object' && value !== null ? (value as Record) : {} -} - -/** - * Durable identity for a Codex item, or null for one that has none. - * - * Non-message items fall back to the `orca` namespace keyed by the Codex item - * id. That id is unstable across resume, so those rows are live-session detail - * that a recovered journal simply will not contain — which is correct: Codex - * itself does not persist them either. - */ -export function codexItemIdentity(input: { - threadId: string - turnId: string | null - item: CodexThreadItem - ordinals: CodexTurnOrdinals -}): AgentJournalItemIdentity { - const { item, turnId } = input - if (turnId && isCodexMessageItemType(item.type)) { - return { - provider: 'codex', - threadId: input.threadId, - turnId, - ordinal: input.ordinals.ordinalFor(input.threadId, turnId, item.id) - } - } - return { provider: 'orca', clientMessageId: `codex-item:${input.threadId}:${item.id}` } -} - -function readString(source: Record, key: string): string | null { - const value = source[key] - return typeof value === 'string' && value.length > 0 ? value : null -} - -function readFirstString(source: Record, keys: readonly string[]): string | null { - for (const key of keys) { - const value = readString(source, key) - if (value !== null) { - return value - } - } - return null -} - -function readTextContent(source: Record, key: string): string | null { - const direct = readString(source, key) - if (direct) { - return direct - } - const value = source[key] - if (!Array.isArray(value)) { - return null - } - const parts = value.flatMap((part) => { - if (typeof part === 'string') { - return part.length > 0 ? [part] : [] - } - if (typeof part !== 'object' || part === null) { - return [] - } - const text = readString(part as Record, 'text') - return text ? [text] : [] - }) - return parts.length > 0 ? parts.join('\n') : null -} +// Codex thread items → journal item bodies. /** `userMessage` carries structured content parts; `agentMessage` a flat text. */ export function codexMessageBlocks(item: CodexThreadItem): NativeChatBlock[] { @@ -175,75 +83,6 @@ export type CodexJournalItem = { handled: boolean } -/** - * Codex's own classification of a shell call: the tool name to show, and the - * fields worth lifting into `input` for the shared label helper (a file target, - * a search term, a scanned root). A `Map`, not an object — an object index - * answers `__proto__` with a truthy non-string. Every other action type stays an - * unclassified `shell` row. - * - * Nothing is invented for a field Codex sends as null: a stand-in path is a - * claim about a target, and the label helper turns any path into a file link. - */ -type CommandActionClass = { - name: string - /** Action field to the `input` key it lifts to. A scan root and a listed - * directory lift to `directory`, never `path`: the label helper reads `path` - * as a file target, which mobile turns into a tappable open-file link. */ - keys: Readonly> -} - -const COMMAND_ACTION_CLASSES = new Map([ - ['read', { name: 'read', keys: { path: 'path' } }], - ['search', { name: 'search', keys: { query: 'query', path: 'directory' } }], - ['listFiles', { name: 'list', keys: { path: 'directory' } }] -]) - -/** The one class every classified `commandActions` entry agrees on, with the - * fields they all agree on; null leaves the row exactly as a Codex that sends no - * classification renders it. `cat a.txt && ls src` classifies as two different - * things, and naming that row after either would drop the other, so it stays a - * `shell` row that shows the whole command. */ -function commandActionFacts( - item: CodexThreadItem -): { name: string; fields: Record } | null { - const actions = item.commandActions - if (!Array.isArray(actions)) { - return null - } - let matched: { class: CommandActionClass; fields: Record } | null = null - for (const action of actions) { - const record = readRecord(action) - const type = readString(record, 'type') - const classified = type === null ? undefined : COMMAND_ACTION_CLASSES.get(type) - if (classified === undefined) { - continue - } - if (matched === null) { - const fields: Record = {} - for (const [source, lifted] of Object.entries(classified.keys)) { - const value = readString(record, source) - if (value !== null) { - fields[lifted] = value - } - } - matched = { class: classified, fields } - continue - } - if (matched.class.name !== classified.name) { - return null - } - // The same class twice keeps the class, but only a target both entries name. - for (const [source, lifted] of Object.entries(matched.class.keys)) { - const kept = matched.fields[lifted] - if (kept !== undefined && readString(record, source) !== kept) { - delete matched.fields[lifted] - } - } - } - return matched === null ? null : { name: matched.class.name, fields: matched.fields } -} - function commandItem(item: CodexThreadItem): CodexJournalItem { const output = readFirstString(item, ['aggregatedOutput', 'aggregated_output']) const bounded = output === null ? null : boundInlineText(output, DEFAULT_JOURNAL_PAYLOAD_LIMITS) @@ -296,6 +135,85 @@ function fileChangeItem(item: CodexThreadItem): CodexJournalItem { } } +/** The tool name reaches the row verbatim — downstream dispatch (diff renderer, + * question parsers, input previews) matches raw identifiers, so any casing + * transform would silently miss them. `server/` qualifies it so two servers + * exposing the same tool stay distinguishable and neither shadows a built-in. */ +function mcpToolCallName(item: CodexThreadItem): string { + const tool = readString(item, 'tool') + const server = readString(item, 'server') + return tool === null ? 'mcp' : server === null ? tool : `${server}/${tool}` +} + +/** Row-label derivation only reads top-level keys, so the call's own arguments + * have to be the input itself. `arguments` is arbitrary JSON upstream: a + * non-object stays addressable under a key rather than being dropped, while a + * no-argument call — `{}` on the wire, the shape every argument-less MCP tool + * sends — becomes null so the row reads as a bare `server/tool` instead of a + * literal `{}`. */ +function mcpToolArguments(value: unknown): unknown { + if (typeof value !== 'object' || value === null) { + return value === null || value === undefined ? null : { arguments: value } + } + return Array.isArray(value) ? { arguments: value } : Object.keys(value).length > 0 ? value : null +} + +function mcpToolCallItem(item: CodexThreadItem): CodexJournalItem { + const failure = readString(readRecord(item.error), 'message') + const text = failure ?? readTextContent(readRecord(item.result), 'content') + const bounded = text === null ? null : boundInlineText(text, DEFAULT_JOURNAL_PAYLOAD_LIMITS) + return { + body: { + kind: 'tool-call', + name: mcpToolCallName(item), + input: boundToolInput(mcpToolArguments(item.arguments), DEFAULT_JOURNAL_PAYLOAD_LIMITS), + state: failure === null ? commandState(item) : 'failed', + ...(bounded === null ? {} : { output: bounded.bounded }) + }, + handled: true + } +} + +/** A row label is read off top-level keys only, so the action's own labelable + * fields are hoisted beside the query while `action` stays whole for the + * expanded detail. The action `type` lands on `description`, the lowest-ranked + * label key, so it names only an action that carries nothing better. */ +function webSearchInput(item: CodexThreadItem): Record | null { + const action = readRecord(item.action) + const fields: [string, unknown][] = [ + ['url', readString(action, 'url')], + ['pattern', readString(action, 'pattern')], + ['description', readString(action, 'type')], + ['action', item.action ?? null] + ] + const query = readString(item, 'query') ?? readString(action, 'query') + const present = fields.filter(([, value]) => value !== null) + // A blank `query` is the run header's "this call has no brief argument" + // signal; drop the key and the header stands the row's raw JSON in for one. + return query === null && present.length === 0 + ? null + : { query: query ?? '', ...Object.fromEntries(present) } +} + +/** `webSearch` carries no status: Codex starts it with an empty query and a null + * action, then sends the action, so `action` is the completion signal — a + * completed item's own `query` is routinely still empty. The hits arrive on + * `results` and are the call's output. */ +function webSearchItem(item: CodexThreadItem): CodexJournalItem { + const hits = Array.isArray(item.results) && item.results.length > 0 ? item.results : null + const bounded = hits && boundInlineText(JSON.stringify(hits), DEFAULT_JOURNAL_PAYLOAD_LIMITS) + return { + body: { + kind: 'tool-call', + name: 'web_search', + input: boundToolInput(webSearchInput(item), DEFAULT_JOURNAL_PAYLOAD_LIMITS), + state: item.action === null || item.action === undefined ? 'running' : 'completed', + ...(bounded === null ? {} : { output: bounded.bounded }) + }, + handled: true + } +} + /** * Journal body for a Codex item, or null for one with nothing to render. * @@ -319,6 +237,12 @@ export function codexJournalItem(item: CodexThreadItem): CodexJournalItem { if (item.type === 'fileChange') { return fileChangeItem(item) } + if (item.type === 'mcpToolCall') { + return mcpToolCallItem(item) + } + if (item.type === 'webSearch') { + return webSearchItem(item) + } if (item.type === 'reasoning' || item.type === 'plan') { const text = readTextContent(item, 'text') ?? diff --git a/src/main/codex/codex-thread-item-identity.ts b/src/main/codex/codex-thread-item-identity.ts new file mode 100644 index 00000000000..0488e5c00e9 --- /dev/null +++ b/src/main/codex/codex-thread-item-identity.ts @@ -0,0 +1,65 @@ +import type { AgentJournalItemIdentity } from '../../shared/agent-session-journal-types' +import type { CodexTurnOrdinals } from './codex-turn-ordinals' + +// Codex thread items → durable journal identities. +// +// THE ORDINAL RULE, and why it is not "index within the turn". Codex renumbers +// item ids positionally on resume (`item-1`…`item-N` across the whole thread), +// and a resumed turn does NOT contain every item the live turn emitted — +// reasoning and command execution are dropped from persisted history. Numbering +// by live position would therefore shift every message after the first tool +// call and hand the user a duplicate of the assistant's answer after a resume. +// +// So the ordinal counts MESSAGE items only, and the same projection is applied +// to the live stream and to a resumed turn's item list. Any other item type — +// including ones this build does not model — is skipped identically on both +// sides, which is what makes the key survive a Codex release that adds one. + +/** Only these carry a durable `(threadId, turnId, ordinal)` identity. */ +const CODEX_MESSAGE_ITEM_TYPES = new Set(['userMessage', 'agentMessage']) + +export type CodexThreadItem = { + type: string + id: string + [key: string]: unknown +} + +export function isCodexMessageItemType(type: string): boolean { + return CODEX_MESSAGE_ITEM_TYPES.has(type) +} + +export function readCodexThreadItem(value: unknown): CodexThreadItem | null { + if (typeof value !== 'object' || value === null) { + return null + } + const record = value as Record + return typeof record.type === 'string' && typeof record.id === 'string' + ? (record as CodexThreadItem) + : null +} + +/** + * Durable identity for a Codex item, or null for one that has none. + * + * Non-message items fall back to the `orca` namespace keyed by the Codex item + * id. That id is unstable across resume, so those rows are live-session detail + * that a recovered journal simply will not contain — which is correct: Codex + * itself does not persist them either. + */ +export function codexItemIdentity(input: { + threadId: string + turnId: string | null + item: CodexThreadItem + ordinals: CodexTurnOrdinals +}): AgentJournalItemIdentity { + const { item, turnId } = input + if (turnId && isCodexMessageItemType(item.type)) { + return { + provider: 'codex', + threadId: input.threadId, + turnId, + ordinal: input.ordinals.ordinalFor(input.threadId, turnId, item.id) + } + } + return { provider: 'orca', clientMessageId: `codex-item:${input.threadId}:${item.id}` } +} diff --git a/src/main/git/repo-branch-conflict.test.ts b/src/main/git/repo-branch-conflict.test.ts index 873c8bd3340..82bd1e65c9d 100644 --- a/src/main/git/repo-branch-conflict.test.ts +++ b/src/main/git/repo-branch-conflict.test.ts @@ -21,7 +21,7 @@ describe('getBranchConflictKindViaExec', () => { await expect(getBranchConflictKindViaExec(exec, 'feature/fix')).resolves.toBe('remote') expect(calls).toEqual([ - ['rev-parse', '--verify', 'refs/heads/feature/fix'], + ['rev-parse', '--verify', '--quiet', 'refs/heads/feature/fix'], ['remote'], ['show-ref', '--verify', '--quiet', '--', 'refs/remotes/foo/bar/feature/fix'], ['show-ref', '--verify', '--quiet', '--', 'refs/remotes/origin/feature/fix'] @@ -41,7 +41,10 @@ describe('getBranchConflictKindViaExec', () => { await expect( getBranchConflictKindViaExec(exec, 'feature/fix', 'origin/feature/fix') ).resolves.toBeNull() - expect(calls).toEqual([['rev-parse', '--verify', 'refs/heads/feature/fix'], ['remote']]) + expect(calls).toEqual([ + ['rev-parse', '--verify', '--quiet', 'refs/heads/feature/fix'], + ['remote'] + ]) }) it('keeps longest configured remote-name matching semantics', async () => { @@ -176,7 +179,7 @@ describe('getBranchConflictKindViaExec batched remote probe', () => { getBranchConflictKindViaExec(exec, 'feature', undefined, {}, batched) ).resolves.toBeNull() expect(calls).toEqual([ - ['rev-parse', '--verify', 'refs/heads/feature'], + ['rev-parse', '--verify', '--quiet', 'refs/heads/feature'], ['remote'], ['cat-file', '--batch-check'] ]) @@ -254,3 +257,68 @@ describe('getBranchConflictKindViaExec batched remote probe', () => { expect(calls.filter((argv) => argv[0] === 'show-ref')).toHaveLength(3) }) }) + +describe('branch conflict with existing-branch adoption', () => { + const absent = () => Object.assign(new Error('missing'), { code: 1, stderr: '' }) + + it('skips adoption and its commit probe for a proven missing local ref', async () => { + const exec = vi.fn(async (argv: string[]) => { + if (argv[0] === 'rev-parse') { + throw absent() + } + return { stdout: '' } + }) + const adopt = vi.fn(async () => false) + await expect( + getBranchConflictKindViaExec(exec, 'new', undefined, {}, undefined, adopt) + ).resolves.toBeNull() + expect(adopt).not.toHaveBeenCalled() + expect(exec).toHaveBeenCalledTimes(2) + }) + + it('allows an existing branch without querying remote refs', async () => { + const exec = vi.fn(async () => ({ stdout: 'a'.repeat(40) })) + const adopt = vi.fn(async () => true) + await expect( + getBranchConflictKindViaExec(exec, 'existing', undefined, {}, undefined, adopt) + ).resolves.toBeNull() + expect(adopt).toHaveBeenCalledOnce() + expect(exec).toHaveBeenCalledOnce() + }) + + it('retains conflicts for refs whose objects cannot be adopted as commits', async () => { + const exec = vi.fn(async () => ({ stdout: 'a'.repeat(40) })) + const adopt = vi.fn(async () => false) + await expect( + getBranchConflictKindViaExec(exec, 'dangling', undefined, {}, undefined, adopt) + ).resolves.toBe('local') + expect(adopt).toHaveBeenCalledOnce() + expect(exec).toHaveBeenCalledTimes(2) + }) + + it.each([ + Object.assign(new Error('transport'), { code: 1, stderr: 'transport failed' }), + Object.assign(new Error('timeout'), { code: 'ETIMEDOUT' }) + ])('still attempts adoption after an undecided ref probe: %s', async (error) => { + const exec = vi.fn(async () => { + throw error + }) + const adopt = vi.fn(async () => true) + await expect( + getBranchConflictKindViaExec(exec, 'existing', undefined, {}, undefined, adopt) + ).resolves.toBeNull() + expect(adopt).toHaveBeenCalledOnce() + }) + + it('rechecks a ref that disappeared while adoption was running', async () => { + const exec = vi + .fn() + .mockResolvedValueOnce({ stdout: 'a'.repeat(40) }) + .mockRejectedValueOnce(absent()) + .mockResolvedValueOnce({ stdout: '' }) + await expect( + getBranchConflictKindViaExec(exec, 'removed', undefined, {}, undefined, async () => false) + ).resolves.toBeNull() + expect(exec).toHaveBeenCalledTimes(3) + }) +}) diff --git a/src/main/git/repo-branch-conflict.ts b/src/main/git/repo-branch-conflict.ts index 5c6e03b94b0..f22fd97f99e 100644 --- a/src/main/git/repo-branch-conflict.ts +++ b/src/main/git/repo-branch-conflict.ts @@ -3,6 +3,7 @@ import { gitExecOptions, type LocalGitExecOptions } from './repo-default-base-re import { gitExecFileAsync } from './runner' import { isSafeGitRefName } from '../../shared/git-status-upstream-ref' import { + isShowRefNoMatchError, probeAnyExactRef, probeAnyExactRefBatched, type ExactRefProbeExec, @@ -29,16 +30,17 @@ function canQueryRemoteBranchName(branchName: string): boolean { return !branchName.startsWith('-') && isSafeGitRefName(`refs/heads/${branchName}`) } -async function hasGitRefAsync( +async function probeLocalBranchRef( exec: ExactRefProbeExec, ref: string, options: ExactRefProbeExecOptions -): Promise { +): Promise<'present' | 'absent' | 'unknown'> { try { - const { stdout } = await runGit(exec, ['rev-parse', '--verify', ref], options) - return stdout.trim().length > 0 - } catch { - return false + // Quiet absence avoids retrying the WSL probe through a login shell. + const { stdout } = await runGit(exec, ['rev-parse', '--verify', '--quiet', ref], options) + return stdout.trim().length > 0 ? 'present' : 'unknown' + } catch (error) { + return isShowRefNoMatchError(error) ? 'absent' : 'unknown' } } @@ -105,7 +107,8 @@ export async function getBranchConflictKindViaExec( branchName: string, allowedBaseRef?: string, options: ExactRefProbeExecOptions = {}, - batchedExec?: ExactRefProbeStdinExec + batchedExec?: ExactRefProbeStdinExec, + allowLocalBranch?: () => Promise ): Promise { if (!canQueryRemoteBranchName(branchName)) { return null @@ -114,7 +117,16 @@ export async function getBranchConflictKindViaExec( // are quiet, so introducing a smaller implicit cap would only make a large // remote configuration look like a missing conflict. const probeOptions: ExactRefProbeExecOptions = options - if (await hasGitRefAsync(exec, `refs/heads/${branchName}`, probeOptions)) { + const localRef = `refs/heads/${branchName}` + let presence = await probeLocalBranchRef(exec, localRef, probeOptions) + if (allowLocalBranch && presence !== 'absent') { + if (await allowLocalBranch()) { + return null + } + // Adoption can span ref changes; preserve the fresh conflict check after it fails. + presence = await probeLocalBranchRef(exec, localRef, probeOptions) + } + if (presence === 'present') { return 'local' } @@ -142,7 +154,8 @@ export function getBranchConflictKind( path: string, branchName: string, allowedBaseRef?: string, - options: LocalGitExecOptions = {} + options: LocalGitExecOptions = {}, + allowLocalBranch?: () => Promise ): Promise { const execOptions = gitExecOptions(path, options) const runLocalGit = ( @@ -168,7 +181,8 @@ export function getBranchConflictKind( // one `show-ref` subprocess per remote -- the exact cost the batch exists to remove. // `show-ref --verify --quiet` prints nothing and is read by exit code, so it needs // no fence; the capture wrapper preserves the payload's exit status either way. - (argv, commandOptions) => runLocalGit(argv, commandOptions, true) + (argv, commandOptions) => runLocalGit(argv, commandOptions, true), + allowLocalBranch ) } diff --git a/src/main/git/runner-wsl-direct-read.test.ts b/src/main/git/runner-wsl-direct-read.test.ts index 284cda55718..e7ea423f78e 100644 --- a/src/main/git/runner-wsl-direct-read.test.ts +++ b/src/main/git/runner-wsl-direct-read.test.ts @@ -17,6 +17,7 @@ vi.mock('../observability/instrumentation', () => ({ })) vi.mock('../diagnostics/main-thread-churn-probe', () => ({ recordSubprocessSpawn: vi.fn() })) +import { getBranchConflictKind } from './repo-branch-conflict' import { pendingWslDirectGitReadEnvironment } from './command-runner/git-command-resolution' import { gitExecFileAsync, gitSpawn, gitStreamStdout } from './runner' import { @@ -584,6 +585,33 @@ describe('WSL direct Git reads', () => { }) }) + it('checks a missing branch conflict without retrying through a login shell', async () => { + await withPlatform('win32', async () => { + seedWslGitReadEnvironmentForTests(DISTRO, LOGIN_ENVIRONMENT) + execFileMock.mockImplementation((_command, args: string[], _options, callback) => { + const child = createMockChild() + queueMicrotask(() => { + const missingRef = args.join(' ').includes('rev-parse') + const quiet = args.includes('--quiet') + const code = missingRef ? (quiet ? 1 : 128) : 0 + callback?.( + code ? Object.assign(new Error('missing ref'), { code }) : null, + '', + missingRef && !quiet ? 'fatal: Needed a single revision' : '' + ) + child.emit('close', code, null) + }) + return child + }) + + await expect( + getBranchConflictKind(String.raw`\\wsl.localhost\Ubuntu\repo`, 'new-feature') + ).resolves.toBeNull() + expect(execFileMock).toHaveBeenCalledTimes(2) + expect(execFileMock.mock.calls[0]?.[1]).toContain('--quiet') + }) + }) + it('keeps the fast path when direct and login Git both report an expected failure', async () => { await withPlatform('win32', async () => { seedWslGitReadEnvironmentForTests(DISTRO, LOGIN_ENVIRONMENT) diff --git a/src/main/git/worktree-add-creation-config.test.ts b/src/main/git/worktree-add-creation-config.test.ts index 44394b7728d..ff44ca7ac6d 100644 --- a/src/main/git/worktree-add-creation-config.test.ts +++ b/src/main/git/worktree-add-creation-config.test.ts @@ -199,7 +199,7 @@ describe('addWorktree', () => { }) const worktreeAddCall = gitExecFileAsyncMock.mock.calls.find( - ([argv]) => Array.isArray(argv) && argv[0] === 'worktree' && argv[1] === 'add' + ([argv]) => Array.isArray(argv) && argv.includes('worktree') && argv.includes('add') ) expect(worktreeAddCall?.[1]).toMatchObject({ timeout: WORKTREE_ADD_TIMEOUT_MS }) expect(WORKTREE_ADD_TIMEOUT_MS).toBeGreaterThan(0) @@ -214,7 +214,7 @@ describe('addWorktree', () => { }) const worktreeAddCall = gitExecFileAsyncMock.mock.calls.find( - ([argv]) => Array.isArray(argv) && argv[0] === 'worktree' && argv[1] === 'add' + ([argv]) => Array.isArray(argv) && argv.includes('worktree') && argv.includes('add') ) expect(worktreeAddCall?.[1]).toMatchObject({ timeout: 600_000 }) }) diff --git a/src/main/git/worktree-add-local-base-refresh.test.ts b/src/main/git/worktree-add-local-base-refresh.test.ts index dba4303647c..1ba2e6c16b8 100644 --- a/src/main/git/worktree-add-local-base-refresh.test.ts +++ b/src/main/git/worktree-add-local-base-refresh.test.ts @@ -1,5 +1,5 @@ // addWorktree: fast-forwarding the local base ref (reset --hard / update-ref) and its safety bailouts. -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const { gitExecFileAsyncMock, @@ -32,11 +32,14 @@ import { registerWorktreeSuiteHooks } from './worktree-test-harness' registerWorktreeSuiteHooks() describe('addWorktree', () => { + afterEach(() => vi.restoreAllMocks()) const resolveCreationBaseConfigWrite = () => { gitExecFileAsyncMock.mockResolvedValueOnce({ stdout: '' }) // config --local --replace-all branch..base } beforeEach(() => { + // These branch-safety assertions use POSIX argv; Windows flags have separate coverage. + vi.spyOn(process, 'platform', 'get').mockReturnValue('darwin') gitExecFileAsyncMock.mockReset() gitExecFileSyncMock.mockReset() translateWslOutputPathsMock.mockClear() diff --git a/src/main/git/worktree-add-local-base-suggestion.test.ts b/src/main/git/worktree-add-local-base-suggestion.test.ts index a65f77dd77d..3b449c339f6 100644 --- a/src/main/git/worktree-add-local-base-suggestion.test.ts +++ b/src/main/git/worktree-add-local-base-suggestion.test.ts @@ -1,5 +1,5 @@ // addWorktree: advisory local-base-ref update suggestions when the refresh setting is off. -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const { gitExecFileAsyncMock, @@ -32,7 +32,10 @@ import { registerWorktreeSuiteHooks } from './worktree-test-harness' registerWorktreeSuiteHooks() describe('addWorktree', () => { + afterEach(() => vi.restoreAllMocks()) beforeEach(() => { + // These branch-safety assertions use POSIX argv; Windows flags have separate coverage. + vi.spyOn(process, 'platform', 'get').mockReturnValue('darwin') gitExecFileAsyncMock.mockReset() gitExecFileSyncMock.mockReset() translateWslOutputPathsMock.mockClear() diff --git a/src/main/git/worktree-add.ts b/src/main/git/worktree-add.ts index ea6ec704b46..380f3a3cc34 100644 --- a/src/main/git/worktree-add.ts +++ b/src/main/git/worktree-add.ts @@ -12,7 +12,7 @@ import { getLocalBaseRefUpdateSuggestionForWorktreeCreate, refreshLocalBaseRefForWorktreeCreate } from './worktree-base-refresh' -import { hasWorktreeBaseCommitRef } from './worktree-base-ref-probe' +import { resolveWorktreeBaseCommitOid } from './worktree-base-ref-probe' import type { AddWorktreeOptions, AddWorktreeResult, @@ -23,6 +23,7 @@ import { bumpWorktreeScanGeneration } from './worktree-scan-cache' export type WorktreeAddBaseContext = AddWorktreeResult & { effectiveBase: string + effectiveBaseOid?: string } export async function resolveWorktreeAddBaseContext( @@ -31,9 +32,11 @@ export async function resolveWorktreeAddBaseContext( refreshLocalBaseRef: boolean, options: AddWorktreeOptions ): Promise { - const effectiveBase = await resolveWorktreeAddBaseRef(baseBranch, (qualifiedRef) => - hasWorktreeBaseCommitRef(repoPath, qualifiedRef, options) - ) + let effectiveBaseOid: string | null = null + const effectiveBase = await resolveWorktreeAddBaseRef(baseBranch, async (qualifiedRef) => { + effectiveBaseOid = await resolveWorktreeBaseCommitOid(repoPath, qualifiedRef, options) + return effectiveBaseOid !== null + }) const localBaseRefRefresh = refreshLocalBaseRef ? await refreshLocalBaseRefForWorktreeCreate( repoPath, @@ -55,6 +58,10 @@ export async function resolveWorktreeAddBaseContext( : undefined return { effectiveBase, + // Refresh/suggestion work can span ref changes; only reuse the immediate resolution probe. + ...(!refreshLocalBaseRef && !options.suggestLocalBaseRefUpdate && effectiveBaseOid + ? { effectiveBaseOid } + : {}), ...(localBaseRefRefresh ? { localBaseRefRefresh } : {}), ...(localBaseRefUpdateSuggestion ? { localBaseRefUpdateSuggestion } : {}) } diff --git a/src/main/git/worktree-create-preparation-real-wsl.test.ts b/src/main/git/worktree-create-preparation-real-wsl.test.ts new file mode 100644 index 00000000000..8ccaf3c23bf --- /dev/null +++ b/src/main/git/worktree-create-preparation-real-wsl.test.ts @@ -0,0 +1,77 @@ +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { expect, it } from 'vitest' +import { createWorktreePreparationLockReason } from '../../shared/worktree/create-preparation' +import { gitExecFileAsync } from './runner' +import { + discardPreparedWorktree, + finalizePreparedWorktree, + prepareWorktreeCreateCheckout +} from './worktree-create-preparation' + +// Opt in on Windows with a running distro; all Git commands use the production WSL router. +const wslDistro = process.env.ORCA_TEST_WSL_DISTRO + +it.skipIf(process.platform !== 'win32' || !wslDistro)( + 'prepares, retargets, moves and cleans up a real WSL checkout from Windows', + async () => { + const fixtureParent = process.env.ORCA_TEST_WSL_ROOT ?? `\\\\wsl.localhost\\${wslDistro}\\tmp` + const root = await mkdtemp(join(fixtureParent, 'orca-create-route-')) + const repoPath = join(root, 'repo') + const preparedPath = join(root, 'prepared checkout') + const finalPath = join(root, 'final checkout') + const options = { wslDistro, timeout: 60_000 } + const git = async (cwd: string, args: string[]): Promise => + (await gitExecFileAsync(args, { cwd, ...options })).stdout.trim() + + try { + await mkdir(repoPath) + await git(repoPath, ['init', '--quiet']) + expect(await git(repoPath, ['rev-parse', '--show-toplevel'])).toMatch(/^\/(?!\/)/) + await git(repoPath, ['symbolic-ref', 'HEAD', 'refs/heads/main']) + await git(repoPath, ['config', 'user.name', 'Test User']) + await git(repoPath, ['config', 'user.email', 'test@example.com']) + await writeFile(join(repoPath, 'version.txt'), 'one\n') + await git(repoPath, ['add', 'version.txt']) + await git(repoPath, ['commit', '--quiet', '-m', 'initial']) + await prepareWorktreeCreateCheckout( + repoPath, + preparedPath, + 'main', + createWorktreePreparationLockReason('real-wsl-test'), + options + ) + expect(await git(repoPath, ['worktree', 'list', '--porcelain'])).toContain( + 'locked orca-create-preparation:v1:' + ) + + await writeFile(join(repoPath, 'version.txt'), 'two\n') + await git(repoPath, ['commit', '--quiet', '-am', 'advance base']) + const target = await git(repoPath, ['rev-parse', 'HEAD']) + await finalizePreparedWorktree( + repoPath, + preparedPath, + finalPath, + 'feature/routed', + 'main', + false, + options + ) + expect(await git(finalPath, ['rev-parse', 'HEAD'])).toBe(target) + expect(await git(finalPath, ['symbolic-ref', '--short', 'HEAD'])).toBe('feature/routed') + expect(await git(finalPath, ['status', '--porcelain'])).toBe('') + expect(await readFile(join(finalPath, 'version.txt'), 'utf8')).toBe('two\n') + expect(await git(finalPath, ['config', '--get', 'branch.feature/routed.base'])).toBe( + 'refs/heads/main' + ) + expect(await git(repoPath, ['worktree', 'list', '--porcelain'])).not.toContain('locked ') + await discardPreparedWorktree(repoPath, finalPath, options) + expect( + (await git(repoPath, ['worktree', 'list', '--porcelain'])).match(/^worktree /gm) + ).toHaveLength(1) + } finally { + await rm(root, { recursive: true, force: true }) + } + }, + 120_000 +) diff --git a/src/main/git/worktree-create-preparation.ts b/src/main/git/worktree-create-preparation.ts index b60dc01ec33..78713957690 100644 --- a/src/main/git/worktree-create-preparation.ts +++ b/src/main/git/worktree-create-preparation.ts @@ -195,25 +195,38 @@ export async function finalizePreparedWorktree( } try { return await runWithGitReadCacheInvalidation(async () => { - const baseContext = await resolveWorktreeAddBaseContext( - repoPath, - baseBranch, - refreshLocalBaseRef, - finalizeGitOptions - ) - const [targetHeadResult, preparedHeadResult] = await Promise.all([ - gitExecFileAsync( - ['rev-parse', '--verify', `${baseContext.effectiveBase}^{commit}`], - gitExecOptions(repoPath, finalizeGitOptions) - ), + const [targetResult, preparedResult] = await Promise.allSettled([ + (async () => { + const baseContext = await resolveWorktreeAddBaseContext( + repoPath, + baseBranch, + refreshLocalBaseRef, + finalizeGitOptions + ) + const targetHead = + baseContext.effectiveBaseOid ?? + ( + await gitExecFileAsync( + ['rev-parse', '--verify', `${baseContext.effectiveBase}^{commit}`], + gitExecOptions(repoPath, finalizeGitOptions) + ) + ).stdout.trim() + return { baseContext, targetHead } + })(), gitExecFileAsync( ['rev-parse', '--verify', 'HEAD'], gitExecOptions(preparedPath, finalizeGitOptions) ) ]) - const { stdout: targetHeadOutput } = targetHeadResult - const targetHead = targetHeadOutput.trim() - const { stdout: preparedHeadOutput } = preparedHeadResult + // Settle both reads before failure cleanup can remove the prepared checkout. + if (targetResult.status === 'rejected') { + throw targetResult.reason + } + if (preparedResult.status === 'rejected') { + throw preparedResult.reason + } + const { baseContext, targetHead } = targetResult.value + const preparedHeadOutput = preparedResult.value.stdout if (preparedHeadOutput.trim() !== targetHead) { await gitExecFileAsync( [...windowsLongPathGitArgs(preparedPath), 'reset', '--hard', targetHead], diff --git a/src/main/git/worktree-preparation-base-oid.test.ts b/src/main/git/worktree-preparation-base-oid.test.ts new file mode 100644 index 00000000000..5864b9d305a --- /dev/null +++ b/src/main/git/worktree-preparation-base-oid.test.ts @@ -0,0 +1,98 @@ +import { beforeEach, expect, it, vi } from 'vitest' + +const gitExec = vi.hoisted(() => vi.fn()) +vi.mock('./runner', () => ({ gitExecFileAsync: gitExec })) +vi.mock('./worktree-base-refresh', () => ({ + refreshLocalBaseRefForWorktreeCreate: vi.fn(), + getLocalBaseRefUpdateSuggestionForWorktreeCreate: vi.fn() +})) +vi.mock('./status', () => ({ runWithGitReadCacheInvalidation: (run: () => unknown) => run() })) +vi.mock('./wsl-linked-worktree-git-routing', () => ({ + invalidateWslLinkedWorktreeGitRouting: vi.fn() +})) + +import { finalizePreparedWorktree } from './worktree-create-preparation' + +const originalOid = '1'.repeat(40) +const refreshedOid = '2'.repeat(40) + +beforeEach(() => { + gitExec.mockReset().mockImplementation(async (args: string[]) => ({ + stdout: + args[0] === 'rev-parse' + ? args.includes('--quiet') || args.at(-1) === 'HEAD' + ? originalOid + : refreshedOid + : '' + })) +}) + +it('reuses the current base-resolution oid and preserves WSL routing', async () => { + await finalizePreparedWorktree('/repo', '/prepared', '/final', 'feature', 'main', false, { + wslDistro: 'Ubuntu', + timeout: 8000 + }) + const revisions = gitExec.mock.calls.filter(([args]) => args[0] === 'rev-parse') + expect(revisions.map(([args]) => args)).toEqual([ + ['rev-parse', '--verify', '--quiet', 'refs/heads/main^{commit}'], + ['rev-parse', '--verify', 'HEAD'] + ]) + expect(gitExec.mock.calls.find(([args]) => args.includes('checkout'))?.[0]).toContain(originalOid) + expect(gitExec.mock.calls.some(([args]) => args.includes('reset'))).toBe(false) + for (const [, options] of gitExec.mock.calls) { + expect(options).toMatchObject({ wslDistro: 'Ubuntu', timeout: 8000 }) + } +}) + +it.each([ + { base: 'refs/heads/main', refresh: false, options: {} }, + { base: 'main', refresh: true, options: {} }, + { base: 'main', refresh: false, options: { suggestLocalBaseRefUpdate: true } } +])('re-reads the target for $base, refresh=$refresh, options=$options', async (test) => { + await finalizePreparedWorktree( + '/repo', + '/prepared', + '/final', + 'feature', + test.base, + test.refresh, + test.options + ) + expect(gitExec).toHaveBeenCalledWith( + ['rev-parse', '--verify', 'refs/heads/main^{commit}'], + expect.objectContaining({ cwd: '/repo' }) + ) + expect(gitExec.mock.calls.find(([args]) => args.includes('reset'))?.[0]).toContain(refreshedOid) + expect(gitExec.mock.calls.find(([args]) => args.includes('checkout'))?.[0]).toContain( + refreshedOid + ) +}) + +it('starts both independent probes before either resolves and settles them before failure', async () => { + let resolveBase!: (value: { stdout: string }) => void + let rejectPrepared!: (reason: Error) => void + gitExec.mockImplementation((args: string[]) => { + if (args.includes('--quiet')) { + return new Promise((resolve) => (resolveBase = resolve)) + } + if (args.at(-1) === 'HEAD') { + return new Promise((_, reject) => (rejectPrepared = reject)) + } + return Promise.resolve({ stdout: '' }) + }) + let settled = false + const error = new Error('prepared HEAD unreadable') + const result = finalizePreparedWorktree('/repo', '/prepared', '/final', 'feature', 'main') + const checked = expect(result).rejects.toBe(error) + void result.then( + () => (settled = true), + () => (settled = true) + ) + await vi.waitFor(() => expect(gitExec).toHaveBeenCalledTimes(2)) + rejectPrepared(error) + await Promise.resolve() + expect(settled).toBe(false) + resolveBase({ stdout: originalOid }) + await checked + expect(gitExec.mock.calls.some(([args]) => args.includes('move'))).toBe(false) +}) diff --git a/src/main/ipc/worktree-logic-wsl.test.ts b/src/main/ipc/worktree-logic-wsl.test.ts index c30c387263e..c21c20e0050 100644 --- a/src/main/ipc/worktree-logic-wsl.test.ts +++ b/src/main/ipc/worktree-logic-wsl.test.ts @@ -16,6 +16,7 @@ vi.mock('../wsl', () => ({ import { computeWorktreePath, computeWorktreePathAsync, + computeWorkspaceRootAsync, getWorktreePathSettings } from './worktree-logic' import { @@ -32,6 +33,26 @@ describe('computeWorktreePath WSL layout', () => { parseWslPathMock.mockReset() }) + it('reuses an asynchronously resolved root for every name candidate without a sync probe', async () => { + parseWslPathMock.mockReturnValue({ distro: 'Ubuntu', linuxPath: '/home/jin/repo' }) + const repoPath = String.raw`\\wsl.localhost\Ubuntu\home\jin\repo` + const home = String.raw`\\wsl.localhost\Ubuntu\home\jin` + const settings = { workspaceDir: 'C:\\workspaces', nestWorkspaces: true } + let resolveHome!: (home: string) => void + getWslHomeAsyncMock.mockReturnValue(new Promise((resolve) => (resolveHome = resolve))) + const pendingRoot = computeWorkspaceRootAsync(repoPath, settings) + expect(getWslHomeMock).not.toHaveBeenCalled() + resolveHome(home) + const root = await pendingRoot + for (const name of ['feature', 'feature-2', 'feature-3']) { + expect(computeWorktreePath(name, repoPath, settings, root)).toBe( + win32.join(home, 'orca', 'workspaces', 'repo', name) + ) + } + expect(getWslHomeAsyncMock).toHaveBeenCalledExactlyOnceWith('Ubuntu') + expect(getWslHomeMock).not.toHaveBeenCalled() + }) + it('places WSL repo worktrees under the distro home workspace root', () => { parseWslPathMock.mockReturnValue({ distro: 'Ubuntu', diff --git a/src/main/ipc/worktree-logic.ts b/src/main/ipc/worktree-logic.ts index 7a8fe175c89..744572f6e37 100644 --- a/src/main/ipc/worktree-logic.ts +++ b/src/main/ipc/worktree-logic.ts @@ -103,12 +103,13 @@ export function ensurePathWithinWorkspace(targetPath: string, workspaceDir: stri export function computeWorktreePath( sanitizedName: string, repoPath: string, - settings: WorktreePathSettings + settings: WorktreePathSettings, + workspaceRoot?: string ): string { return computeWorktreePathFromWorkspaceRoot( sanitizedName, repoPath, - computeWorkspaceRoot(repoPath, settings), + workspaceRoot ?? computeWorkspaceRoot(repoPath, settings), settings.nestWorkspaces ) } @@ -130,7 +131,7 @@ function computeWorktreePathFromWorkspaceRoot( } /** Async twin of computeWorktreePath. Same result; resolves the WSL home without blocking the main - * thread, so callers off the create path never freeze the app on a stopped distro. */ + * thread, so callers never freeze the app on a stopped distro. */ export async function computeWorktreePathAsync( sanitizedName: string, repoPath: string, @@ -147,7 +148,7 @@ export async function computeWorktreePathAsync( /** Async twin of computeWorkspaceRoot. Same result; the WSL home probe spawns `wsl.exe`, so * background preparation uses this variant rather than blocking the Electron main thread for up * to the probe timeout. The sync twin below still serves callers that cannot await (allowed-roots - * resolution, the create click, CLI create, watch targets, worktree trash). */ + * resolution, CLI create, watch targets, worktree trash). */ export async function computeWorkspaceRootAsync( repoPath: string, settings: { workspaceDir: string; wslMirrorDistro?: string } diff --git a/src/main/ipc/worktree-remote.ts b/src/main/ipc/worktree-remote.ts index 27eaa9264db..ef65f2fcb53 100644 --- a/src/main/ipc/worktree-remote.ts +++ b/src/main/ipc/worktree-remote.ts @@ -87,7 +87,7 @@ import { computeValidatedBranchName, computeWorktreePath, computeRemoteWorktreePath, - computeWorkspaceRoot, + computeWorkspaceRootAsync, ensurePathWithinWorkspace, getWorktreeCreationLayout, getWorktreePathSettings, @@ -2444,7 +2444,7 @@ export async function createLocalWorktree( emitCreateWorktreeProgress(mainWindow, 'fetching', args.creationId) } } - const workspaceRoot = computeWorkspaceRoot(repo.path, worktreePathSettings) + const workspaceRoot = await computeWorkspaceRootAsync(repo.path, worktreePathSettings) // Why: this validation doesn't depend on remote refs, so it can overlap a required remote-tracking base refresh. const primarySetupScript = getEffectiveHooks(repo)?.scripts.setup @@ -2530,19 +2530,33 @@ export async function createLocalWorktree( username, localWorktreeGitOptions ) - checkoutExistingBranch = await canCheckoutExistingLocalBranch( - repo.path, - branchName, - baseBranch, - localWorktreeGitOptions - ) - if (checkoutExistingBranch && !selectedExistingLocalBranchName) { - // Why: suffix retries may need a new path, but an existing-branch checkout must keep the user-selected branch, not a sibling. - selectedExistingLocalBranchName = branchName + const tryExistingBranch = async (): Promise => { + checkoutExistingBranch = await canCheckoutExistingLocalBranch( + repo.path, + branchName, + baseBranch, + localWorktreeGitOptions + ) + return checkoutExistingBranch } + // Explicit branch selections retain the adoption-first path. + const preferExistingBranch = Boolean( + args.branchNameOverride || selectedExistingLocalBranchName + ) + checkoutExistingBranch = preferExistingBranch && (await tryExistingBranch()) lastBranchConflictKind = checkoutExistingBranch ? null - : await getBranchConflictKind(repo.path, branchName, baseBranch, localWorktreeGitOptions) + : await getBranchConflictKind( + repo.path, + branchName, + baseBranch, + localWorktreeGitOptions, + preferExistingBranch ? undefined : tryExistingBranch + ) + if (checkoutExistingBranch && !selectedExistingLocalBranchName) { + // Path retries must retain the adopted branch. + selectedExistingLocalBranchName = branchName + } const allowedPushTargetRemoteConflict = lastBranchConflictKind && isAllowedPushTargetRemoteConflict(lastBranchConflictKind, branchName, args) @@ -2604,7 +2618,7 @@ export async function createLocalWorktree( } worktreePath = ensurePathWithinWorkspace( - computeWorktreePath(effectiveSanitizedName, repo.path, worktreePathSettings), + computeWorktreePath(effectiveSanitizedName, repo.path, worktreePathSettings, workspaceRoot), workspaceRoot ) if (existsSync(worktreePath)) { @@ -3010,6 +3024,8 @@ export async function createLocalWorktree( } }) + // Startup resolves the new id before lifecycle notifications invalidate runtime caches. + runtime?.invalidateWorktreeCatalog?.(repo.id) const stagedStartup = await timing.time('spawn_startup_terminal', () => spawnLocalStartupAndSetupTerminals({ runtime, diff --git a/src/main/ipc/worktrees-local-create-flow.test.ts b/src/main/ipc/worktrees-local-create-flow.test.ts index abc711bbd9b..fc57c6969b6 100644 --- a/src/main/ipc/worktrees-local-create-flow.test.ts +++ b/src/main/ipc/worktrees-local-create-flow.test.ts @@ -2,6 +2,8 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { resolve } from 'node:path' import type { CreateWorktreeResult } from '../../shared/worktree/create-types' import { resolveRegisteredWorktreePath } from './registered-worktree-roots-cache' +import { computeWorkspaceRootAsync } from './worktree-logic' +import type * as WorktreeLogic from './worktree-logic' import { listWorktreesMock, describeCreatedWorktreeMock, @@ -78,11 +80,13 @@ vi.mock('../setup-hook-env-vars', async (importOriginal) => (await importOriginal()) as Record ) ) -vi.mock('./worktree-logic', async (importOriginal) => - (await import('./worktrees-test-module-mocks')).worktreeLogicModuleMock( - (await importOriginal()) as Record - ) -) +vi.mock('./worktree-logic', async (importOriginal) => { + const actual = await importOriginal() + return { + ...(await import('./worktrees-test-module-mocks')).worktreeLogicModuleMock(actual), + computeWorkspaceRootAsync: vi.fn(actual.computeWorkspaceRootAsync) + } +}) vi.mock('../terminal-history-deletion', async () => (await import('./worktrees-test-module-mocks')).terminalHistoryDeletionModuleMock() ) @@ -409,15 +413,23 @@ describe('registerWorktreeHandlers', () => { } ]) - await handlers['worktrees:create'](null, { + const root = Promise.withResolvers() + vi.mocked(computeWorkspaceRootAsync).mockReturnValueOnce(root.promise) + const create = handlers['worktrees:create'](null, { repoId: 'repo-1', name: 'feature' }) - expect(computeWorktreePathMock).toHaveBeenCalledWith('feature', '/workspace/repo', { - nestWorkspaces: false, - workspaceDir: '../worktrees' - }) + await vi.waitFor(() => expect(computeWorkspaceRootAsync).toHaveBeenCalled()) + expect(addWorktreeMock).not.toHaveBeenCalled() + root.resolve('/workspace/worktrees') + await create + expect(computeWorktreePathMock).toHaveBeenCalledWith( + 'feature', + '/workspace/repo', + { nestWorkspaces: false, workspaceDir: '../worktrees' }, + '/workspace/worktrees' + ) expect(addWorktreeMock).toHaveBeenCalledWith( '/workspace/repo', '../worktrees/feature', @@ -689,6 +701,10 @@ describe('registerWorktreeHandlers', () => { expect(setupCommand).toBe('bash /workspace/repo/.git/orca/setup-runner.sh') expect(result.setup).toBeUndefined() expect(result.startupTerminal).toEqual({ spawned: true, surface: 'visible' }) + expect(runtimeStub.invalidateWorktreeCatalog).toHaveBeenCalledWith('repo-1') + expect(runtimeStub.invalidateWorktreeCatalog.mock.invocationCallOrder[0]).toBeLessThan( + runtimeStub.createTerminal.mock.invocationCallOrder[0] + ) expect(result.timing?.phases.map((phase) => phase.phase)).toEqual( expect.arrayContaining([ 'git_worktree_add', diff --git a/src/main/ipc/worktrees-test-module-mocks.ts b/src/main/ipc/worktrees-test-module-mocks.ts index a1925d1ce72..8d2787fcf2f 100644 --- a/src/main/ipc/worktrees-test-module-mocks.ts +++ b/src/main/ipc/worktrees-test-module-mocks.ts @@ -1,4 +1,5 @@ import { type Mock, vi } from 'vitest' +import type { computeWorktreePath } from './worktree-logic' import type { HandlerMap } from './worktrees-test-ipc-surface' /** Loose signature: one mock stands in for many unrelated module exports. */ @@ -78,13 +79,7 @@ export const resolveSetupRunnerShellMock: ModuleMock = vi.fn() export const runHookMock: ModuleMock = vi.fn() export const hasHooksFileMock: ModuleMock = vi.fn() export const loadHooksMock: ModuleMock = vi.fn() -export const computeWorktreePathMock: Mock< - ( - sanitizedName: string, - repoPath: string, - settings: { nestWorkspaces: boolean; workspaceDir: string } - ) => string -> = vi.fn() +export const computeWorktreePathMock: Mock = vi.fn() export const ensurePathWithinWorkspaceMock: StringArgMock = vi.fn() export const gitExecFileAsyncMock: GitArgvMock = vi.fn() export const getSshGitProviderMock: StringArgMock = vi.fn() @@ -138,7 +133,19 @@ export const gitRepoModuleMock = () => ({ resolveDefaultBaseRefWithLocalGit: resolveDefaultBaseRefWithLocalGitMock, resolveDefaultBaseRefViaExec: resolveDefaultBaseRefViaExecMock, getDefaultRemote: getDefaultRemoteMock, - getBranchConflictKind: getBranchConflictKindMock + getBranchConflictKind: async ( + repoPath: string, + branch: string, + base?: string, + options?: { wslDistro?: string }, + allowLocalBranch?: () => Promise + ) => { + // These handler tests stub ref presence; policy tests cover the absent-ref fast path. + if (allowLocalBranch && (await allowLocalBranch())) { + return null + } + return getBranchConflictKindMock(repoPath, branch, base, options) + } }) export const githubClientModuleMock = () => ({ diff --git a/src/main/ipc/worktrees-test-runtime-stub.ts b/src/main/ipc/worktrees-test-runtime-stub.ts index 647d4801d13..bb2f33cb05e 100644 --- a/src/main/ipc/worktrees-test-runtime-stub.ts +++ b/src/main/ipc/worktrees-test-runtime-stub.ts @@ -12,6 +12,7 @@ export type WorktreeRuntimeStub = { clearOptimisticReconcileToken: ReturnType resolveManagedMrBase: ReturnType createTerminal: ReturnType + invalidateWorktreeCatalog: ReturnType splitTerminal: ReturnType notifyWorktreesChangedForRemoteClients: ReturnType closeFileWatchersForRemoval: ReturnType @@ -38,6 +39,7 @@ export function createWorktreeRuntimeStub(): WorktreeRuntimeStub { title: null, surface: 'visible' }), + invalidateWorktreeCatalog: vi.fn(), splitTerminal: vi.fn().mockResolvedValue({ handle: 'term-setup', tabId: 'tab-startup', diff --git a/src/main/native-chat/agent-session-journal/journal-file-format-remnant.test.ts b/src/main/native-chat/agent-session-journal/journal-file-format-remnant.test.ts new file mode 100644 index 00000000000..d6424ce7952 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-file-format-remnant.test.ts @@ -0,0 +1,196 @@ +// An empty chat beside a pre-SQLite journal explains itself. +// +// The SQLite move shipped no importer, so a session whose history is a +// `log.jsonl` founds a fresh empty journal beside it and looks exactly like a +// chat created seconds ago. One status row is the difference. + +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import Database from '../../sqlite/sync-database' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import type { AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' +import { projectStructuredItemsToNativeChat } from '../../../shared/structured-agent-session-projection' +import { openJournalDatabase } from './journal-database' +import { JOURNAL_DB_SCHEMA_VERSION } from './journal-database-schema' +import { JOURNAL_FILE_FORMAT_REMNANT_DISCLOSURE_IDENTITY } from './journal-file-format-remnant' +import { loadJournal } from './journal-open' +import { journalDatabaseFile } from './journal-paths' +import type { AgentSessionJournal } from './journal-store' +import type { openAgentSessionJournal } from './journal-store-factory' +import { createTrackedJournalOpener } from './journal-store-test-open' + +const IDENTITY: AgentSessionJournalIdentity = { + sessionId: 'session-1', + workspaceId: 'ws-1', + hostId: 'host-1', + agent: 'codex', + providerHandle: { kind: 'codex', threadId: 'thread-1' } +} + +const DISCLOSURE_ITEM_ID = agentJournalItemKey(JOURNAL_FILE_FORMAT_REMNANT_DISCLOSURE_IDENTITY) + +let root: string +let clock = 1_000 +const journals = createTrackedJournalOpener() + +function open(overrides: Partial[0]> = {}) { + return journals.open({ + identity: IDENTITY, + journalDir: root, + now: () => (clock += 1), + mintEpoch: () => `epoch-${clock}`, + ...overrides + }) +} + +function writeRemnant(name = 'log.jsonl'): Promise { + return writeFile(join(root, name), '{"kind":"epoch","v":1,"seq":1}\n', 'utf8') +} + +function disclosure(journal: AgentSessionJournal): string | null { + const row = journal.snapshot().items.find((entry) => entry.itemId === DISCLOSURE_ITEM_ID) + return row?.body.kind === 'status' ? row.body.text : null +} + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-journal-remnant-')) + clock = 1_000 +}) + +afterEach(async () => { + await journals.closeAll() + await rm(root, { recursive: true, force: true }) +}) + +describe('a chat whose history is still in the pre-SQLite format', () => { + it('says how to carry on, and where the transcript is', async () => { + await writeRemnant() + + const journal = await open() + + expect(disclosure(journal)).toContain('send a message to pick up where you left off') + expect(disclosure(journal)).toContain(join(root, 'log.jsonl')) + expect(disclosure(journal)).toContain('Codex') + }) + + // Both files is the normal shape of a pre-SQLite directory: every epoch roll + // staged a snapshot whether or not anything compacted into it, so preferring + // the snapshot would name an empty file for ~every affected chat. + it('names the log, not the snapshot staged beside it', async () => { + await writeRemnant('log.jsonl') + await writeRemnant('snapshot.json') + + const journal = await open() + + expect(disclosure(journal)).toContain(join(root, 'log.jsonl')) + expect(disclosure(journal)).not.toContain('snapshot.json') + }) + + it('falls back to the snapshot when a chat has no log beside it', async () => { + await writeRemnant('snapshot.json') + + const journal = await open() + + expect(disclosure(journal)).toContain(join(root, 'snapshot.json')) + }) + + it('says nothing to a chat that is genuinely new', async () => { + const journal = await open() + + expect(journal.snapshot().items).toEqual([]) + }) + + // Counting rows proves nothing here — the append upserts by identity, so a + // second append would still leave exactly one. The revision is what moves. + it('does not re-append the row on a later open', async () => { + await writeRemnant() + const first = await open() + const firstRevision = first + .snapshot() + .items.find((e) => e.itemId === DISCLOSURE_ITEM_ID)?.revision + await first.close() + + const reopened = await open() + + const row = reopened.snapshot().items.find((e) => e.itemId === DISCLOSURE_ITEM_ID) + expect(firstRevision).toBe(1) + expect(row?.revision).toBe(1) + expect(reopened.cursor().sequence).toBe(2) + }) + + // The epoch commit and this append are separate transactions; if the append is + // lost the epoch exists but holds nothing, and every later open takes the + // adopt branch. The offer has to survive that. + it('offers the message again when a committed epoch holds nothing', async () => { + const founded = await open() + await founded.close() + await writeRemnant() + + const reopened = await open() + + expect(disclosure(reopened)).toContain(join(root, 'log.jsonl')) + }) + + // A repair's epoch is the marker that history was deleted and never rebuilt, + // and any row that is not the repair's own disclosure retires it. Appending + // here would silently stop the session ever asking the provider for that + // history — with the journal still holding none. + it('stays out of a journal this open just repaired', async () => { + const journal = await open() + await journal.appendItem( + { provider: 'codex', threadId: 'thread-1', turnId: 'turn-1', ordinal: 0 }, + { kind: 'message', role: 'assistant', blocks: [{ type: 'text', text: 'history' }] }, + { fence: 1 } + ) + await journal.close() + // Deleting the anchor leaves every row unanchored: replay keeps nothing, so + // the repair publishes an empty `unreconcilable_prefix` epoch and — costing + // no malformed row — appends no disclosure of its own. That is the one state + // where this branch and a repair meet. + const opened = openJournalDatabase(journalDatabaseFile(root)) + try { + opened.db.prepare('DELETE FROM journal_rows WHERE seq = ?').run(1) + } finally { + opened.db.close() + } + await writeRemnant() + + const repaired = await open() + + expect(disclosure(repaired)).toBeNull() + // Still asking the provider for the history the repair dropped. + expect(loadJournal(root, IDENTITY.sessionId)).toMatchObject({ corrupt: true }) + }) + + // A latched journal loads empty, so it reaches the same branch — and an append + // into one throws, which would make the session unopenable rather than read-only. + it('writes nothing into a journal latched by a newer schema', async () => { + const founded = await open() + await founded.close() + const db = new Database(journalDatabaseFile(root)) + try { + db.pragma(`user_version = ${JOURNAL_DB_SCHEMA_VERSION + 1}`) + } finally { + db.close() + } + await writeRemnant() + + const latched = await open() + + expect(latched.isReadOnly).toBe(true) + expect(disclosure(latched)).toBeNull() + }) + + // A row nothing projects is a row nobody reads. + it('renders in the transcript as a system line', async () => { + await writeRemnant() + + const journal = await open() + + const messages = projectStructuredItemsToNativeChat(journal.snapshot().items) + expect(messages).toHaveLength(1) + expect(messages[0]?.role).toBe('system') + }) +}) diff --git a/src/main/native-chat/agent-session-journal/journal-file-format-remnant.ts b/src/main/native-chat/agent-session-journal/journal-file-format-remnant.ts new file mode 100644 index 00000000000..69dcc04bf64 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-file-format-remnant.ts @@ -0,0 +1,60 @@ +// A journal directory left behind by the pre-SQLite file format. +// +// Not `journal-legacy-import.ts`, which reads the PROVIDER's own transcript. +// This is Orca's own `log.jsonl`, which no build after the SQLite move reads. +// Nothing imports it, so the session it belonged to opens empty and is +// indistinguishable from a chat created seconds ago — same `session_created` +// epoch, same empty timeline. The remnant is the one durable fact that tells +// them apart, so the empty session says where its history went and how to carry +// on instead of silently claiming it never had any. + +import { existsSync } from 'node:fs' +import { join } from 'node:path' +import type { AgentJournalItemIdentity } from '../../../shared/agent-session-journal-types' +import { boundJournalStatusText } from './journal-prompt-body-bounds' +import { formatAgentTypeLabel } from '../../../shared/agent-type-label' +import type { AgentType } from '../../../shared/agent-status-types' + +/** The remnant's transcript, or null when the directory never held one. + * + * `log.jsonl` first, and the order matters: every epoch roll staged a + * `snapshot.json` whether or not anything was ever compacted into it, so the + * file's existence says nothing about where the history lives. Measured across + * a real profile, `compactedThrough` was 0 in all 80 — the log holds the + * transcript and the snapshot is the fallback for a session that has no log. */ +export function findJournalFileFormatRemnant(journalDir: string): string | null { + for (const name of ['log.jsonl', 'snapshot.json']) { + const path = join(journalDir, name) + if (existsSync(path)) { + return path + } + } + return null +} + +/** One stable identity, so a reopen upserts the same row instead of adding one. */ +export const JOURNAL_FILE_FORMAT_REMNANT_DISCLOSURE_IDENTITY: AgentJournalItemIdentity = { + provider: 'orca', + clientMessageId: 'journal-file-format-remnant' +} + +/** How to carry on. The session attaches on the record's own provider handle, so it + * still names the conversation the transcript no longer shows — whether the provider + * itself still holds that thread is its own business, hence "points at". */ +export function journalFileFormatRemnantDisclosure(input: { + transcriptPath: string + agent: AgentType +}): { identity: AgentJournalItemIdentity; body: { kind: 'status'; text: string } } { + return { + identity: JOURNAL_FILE_FORMAT_REMNANT_DISCLOSURE_IDENTITY, + body: { + kind: 'status', + text: boundJournalStatusText( + `This chat's history was saved in an older format Orca no longer reads, so it starts ` + + `empty. The session still points at the same ${formatAgentTypeLabel(input.agent)} ` + + `conversation — send a message to pick up where you left off. The original ` + + `transcript is on the session's host at \`${input.transcriptPath}\`` + ) + } + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-store-open.ts b/src/main/native-chat/agent-session-journal/journal-store-open.ts index e002048a094..721e5f4ba7f 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-open.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-open.ts @@ -1,7 +1,16 @@ import { mkdir } from 'node:fs/promises' +import type { AgentType } from '../../../shared/agent-status-types' +import { + findJournalFileFormatRemnant, + journalFileFormatRemnantDisclosure +} from './journal-file-format-remnant' import type { JournalLoad } from './journal-open' import { journalRepairDisclosure, type JournalRepairDisclosure } from './journal-repair-disclosure' +/** What any of this file's disclosures hands the store — a repair's, or the + * pre-SQLite notice's. Same shape, and neither is only a repair. */ +type JournalDisclosure = JournalRepairDisclosure + export async function ensureJournalDir(journalDir: string): Promise { await mkdir(journalDir, { recursive: true }) } @@ -32,6 +41,7 @@ export async function openJournalStoreState(input: { body: JournalRepairDisclosure['body'], fence: number ) => Promise + agent: AgentType highestFence: () => number malformedRows: () => number setMalformedRows: (count: number) => void @@ -40,6 +50,7 @@ export async function openJournalStoreState(input: { const loaded = input.loaded !== undefined ? input.loaded : input.replay() if (!loaded) { input.start() + await discloseFileFormatRemnant(input) return } input.adopt(loaded) @@ -59,4 +70,43 @@ export async function openJournalStoreState(input: { const disclosure = journalRepairDisclosure({ malformedRows: input.malformedRows() }) await input.appendDisclosure(disclosure.identity, disclosure.body, input.highestFence()) } + // Founding the epoch and appending the row are two transactions, and a + // committed epoch sends every later open down this branch instead. Anything + // that interrupts between them — a quit during startup restore, a failed + // append — would otherwise lose the message for good. An epoch holding nothing + // is exactly the state that append was owed, so offer it again. + // + // Never onto a repair, though: `loaded.state` is the PRE-repair load, so a + // journal this open just emptied looks identical. The repair's epoch is the + // marker that its history was deleted and never rebuilt, and any row that is + // not the repair's own disclosure retires it — this row would silently stop + // the session ever asking the provider for that history again. + if (!loaded.corrupt && loaded.state.items.size === 0 && loaded.state.submissions.size === 0) { + await discloseFileFormatRemnant(input) + } +} + +/** Says what happened to a chat whose history is in the abandoned file format. + * Upserts by a constant identity, so the offer above is exactly-once in effect: + * once the row exists the epoch is no longer empty. */ +async function discloseFileFormatRemnant(input: { + journalDir: string + agent: AgentType + appendDisclosure: ( + identity: JournalDisclosure['identity'], + body: JournalDisclosure['body'], + fence: number + ) => Promise + highestFence: () => number + readOnly: () => boolean +}): Promise { + if (input.readOnly()) { + return + } + const transcriptPath = findJournalFileFormatRemnant(input.journalDir) + if (!transcriptPath) { + return + } + const disclosure = journalFileFormatRemnantDisclosure({ transcriptPath, agent: input.agent }) + await input.appendDisclosure(disclosure.identity, disclosure.body, input.highestFence()) } diff --git a/src/main/native-chat/agent-session-journal/journal-store-restore.ts b/src/main/native-chat/agent-session-journal/journal-store-restore.ts index 53683a797d1..3a5d3c7ac6e 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-restore.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-restore.ts @@ -41,6 +41,7 @@ export function restoreJournalStore( adopt: host.adopt, appendDisclosure: (identity, body, fence) => host.journal().appendItem(identity, body, { fence }), + agent: host.identity.agent, highestFence: () => host.state().highestFence, malformedRows: host.malformedRows, setMalformedRows: host.setMalformedRows, diff --git a/src/main/native-chat/agent-session-wire/provider-frame-disposition.test.ts b/src/main/native-chat/agent-session-wire/provider-frame-disposition.test.ts index 22bd645d8a6..9860aaa81d8 100644 --- a/src/main/native-chat/agent-session-wire/provider-frame-disposition.test.ts +++ b/src/main/native-chat/agent-session-wire/provider-frame-disposition.test.ts @@ -112,4 +112,52 @@ describe('provider frame classification catalog', () => { // An item type nobody has dispositioned still falls through visibly. expect(classifyProviderFrame('codex', 'item:futureThing', {})).toBe('timeline-substantive') }) + + it('chromes the one unmodelled codex item type that carries no content', () => { + expect(classifyProviderFrame('codex', 'item:sleep', { id: 's', durationMs: 20_000 })).toBe( + 'status-chrome' + ) + // Payload inspection still outranks the item catalog, so chroming a type + // cannot swallow one that reports a failure. + expect(classifyProviderFrame('codex', 'item:sleep', { id: 's', status: 'failed' })).toBe( + 'error-surface' + ) + }) + + it('keeps subagent items visible — the only evidence a spawned agent is working', () => { + expect( + classifyProviderFrame('codex', 'item:subAgentActivity', { + id: 'a-1', + kind: 'started', + agentThreadId: 'thread-child', + agentPath: '/root/list_directory' + }) + ).toBe('timeline-substantive') + expect( + classifyProviderFrame('codex', 'item:collabAgentToolCall', { + id: 'c-1', + tool: 'spawn', + status: 'inProgress', + senderThreadId: 'thread-root', + receiverThreadIds: ['thread-child'], + agentsStates: {} + }) + ).toBe('timeline-substantive') + }) + + it('leaves content-bearing codex item types on the visible fallback', () => { + // Each carries text or a path a user would want: review output, the image + // the agent looked at or generated, injected hook prompt text. + for (const type of [ + 'imageView', + 'imageGeneration', + 'enteredReviewMode', + 'exitedReviewMode', + 'hookPrompt' + ]) { + expect(classifyProviderFrame('codex', `item:${type}`, { id: 'i' }), type).toBe( + 'timeline-substantive' + ) + } + }) }) diff --git a/src/main/native-chat/agent-session-wire/provider-frame-disposition.ts b/src/main/native-chat/agent-session-wire/provider-frame-disposition.ts index 474b1385a4f..f05f4cd4c6c 100644 --- a/src/main/native-chat/agent-session-wire/provider-frame-disposition.ts +++ b/src/main/native-chat/agent-session-wire/provider-frame-disposition.ts @@ -197,7 +197,12 @@ function hasProviderError(payload: unknown): boolean { const CODEX_ITEM_CLASSIFICATIONS: Record = { // The `thread/compacted` notification is already chrome; its item form is the // same event and must not read as a mysterious opcode row. - contextCompaction: 'status-chrome' + contextCompaction: 'status-chrome', + // `{id, durationMs}` and nothing else — Codex's own transcript renders it as + // nothing at all. Every other item type this build does not model carries text + // a user would want (review output, an image path, hook prompt text, subagent + // progress), so those keep their visible fallback row. + sleep: 'status-chrome' } function notificationKind(kind: string): string { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts index e44e8c39152..e6f8e478695 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-adapter.ts @@ -137,7 +137,11 @@ export type StructuredAgentSessionAdapter = { turnId: string fence: number }): Promise<{ cancelled: boolean }> - stopBackgroundTasks?(input: { sessionId: string; fence: number }): Promise<{ cancelled: boolean }> + stopBackgroundTasks?(input: { + sessionId: string + fence: number + taskId?: string + }): Promise<{ cancelled: boolean }> backgroundTaskState?(sessionId: string): AgentSessionBackgroundTaskState | null | undefined /** Fires the provider callback for an approval or a question. The wire calls * this only after the durable compare-and-set won, so it runs exactly once. */ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-history-result.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-history-result.ts index 70fc9a43ed2..b8e198c9b6b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-history-result.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-history-result.ts @@ -8,7 +8,7 @@ import type { import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { readAgentSessionHistory } from './agent-session-history-page' -function providerSessionMetadata( +export function structuredAgentSessionProviderSessionMetadata( record: AgentSessionRecord | null ): AgentProviderSessionMetadata | undefined { const head = record ? agentSessionProviderHandleChainHead(record.providerHandleChain) : null @@ -27,7 +27,7 @@ export function readStructuredAgentSessionHistoryResult(input: { }): AgentSessionHistoryResult { const result = readAgentSessionHistory(input.journal, input.request) const fence = input.record?.lease.runtimeFence - const providerSession = providerSessionMetadata(input.record) + const providerSession = structuredAgentSessionProviderSessionMetadata(input.record) if (fence === undefined) { return providerSession ? { ...result, providerSession } : result } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts index 2afd94ba128..ba62db1704e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts @@ -19,6 +19,8 @@ import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import { releaseStoredStructuredAgentSessionOwner } from './structured-agent-session-lease-release' +import { resumeHeldStructuredAgentSession } from './structured-agent-session-hold-resume' +import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire' export type StructuredAgentSessionLifetimeContext = { deps: StructuredAgentSessionHostDeps @@ -67,6 +69,27 @@ export async function evictHeldStructuredAgentSession( ) } +/** The first hold on a childless session: reconcile the lease, settle recovery, then attach. */ +export async function resumeStructuredAgentSessionForHold( + context: StructuredAgentSessionLifetimeContext & { + reconcileLeases: (sessionId: string) => Promise + }, + sessionId: string, + attach: Parameters[0]['attach'] +): Promise { + const unreconciled = await context.reconcileLeases(sessionId) + if (unreconciled) { + throw new Error(unreconciled.code) + } + await context.runtimeState.resolveRecovery(sessionId) + await resumeHeldStructuredAgentSession({ + sessionId, + deps: context.deps, + now: context.now, + attach + }) +} + export function createStructuredAgentSessionHolds( context: StructuredAgentSessionLifetimeContext, input: { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts index 7d2648930c4..f4a0244d0af 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-mutations.ts @@ -78,6 +78,7 @@ export function cancelStructuredAgentSessionTurn( envelope: AgentSessionMutationEnvelope turnId: string scope?: 'background-tasks' + taskId?: string } ): Promise> { return mutate(context, caller, params.envelope, cancelPlan(params)) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index 8989f5e4d72..aef76c16cdb 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -2,17 +2,7 @@ // Mutations share one durable admission path and serialize per session. import type { AgentSessionExecutionLocation } from '../../../shared/agent-session-record' -import type { - AgentSessionAttachResult, - AgentSessionHistoryRequest, - AgentSessionHistoryResult, - AgentSessionHandoffRequest, - AgentSessionHandoffResult, - AgentSessionHandoffStatus, - AgentSessionMutationResult, - AgentSessionOptionsResult, - AgentSessionWireRefusal -} from '../../../shared/agent-session-wire' +import type * as SessionWire from '../../../shared/agent-session-wire' import type { AgentSessionAttachParams } from './structured-agent-session-attach' import { AGENT_SESSION_NOT_ATTACHED } from './structured-agent-session-mutation-admission' import { createRestartReconciler } from './structured-agent-session-restart-reconcile' @@ -33,13 +23,13 @@ import { attachStructuredAgentSession } from './structured-agent-session-attach- import { createStructuredAgentSessionHolds, evictHeldStructuredAgentSession, + resumeStructuredAgentSessionForHold, type StructuredAgentSessionLifetimeContext } from './structured-agent-session-host-lifetime' import type { StructuredAgentSessionHolds, StructuredAgentSessionHoldOptions } from './structured-agent-session-holds' -import { resumeHeldStructuredAgentSession } from './structured-agent-session-hold-resume' import type { StructuredAgentSessionAttachContext } from './structured-agent-session-attach-context' import { listStructuredAgentSessionTabs } from './structured-agent-session-host-tabs' import { @@ -57,6 +47,7 @@ import type { StructuredAgentSessionHostDeps, StructuredAgentSessionHostSession } from './structured-agent-session-host-types' +import { StructuredAgentSessionStatusFeed } from './structured-agent-session-status-feed' import { StructuredAgentSessionEventRecovery } from './structured-agent-session-event-recovery' import { StructuredAgentSessionBackgroundTaskChannel } from './structured-agent-session-background-task-channel' import { withTimeout } from '../../../shared/promise-timeout-fallback' @@ -66,10 +57,19 @@ const HANDOFF_DRAIN_TIMEOUT_MS = 5_000 export class StructuredAgentSessionHost { private readonly sessions = new Map() - private readonly subscribers = new AgentSessionSubscribers() + private readonly statusFeed = new StructuredAgentSessionStatusFeed({ + sessions: this.sessions, + getRecord: (sessionId) => this.deps.store.getRecord(sessionId), + now: () => this.now() + }) + private readonly subscribers = new AgentSessionSubscribers({ + onJournalPublished: (sessionId, journal) => this.statusFeed.publish(sessionId, journal) + }) private readonly tasks = new StructuredAgentSessionTaskQueue() private readonly runtimeState: StructuredAgentSessionHostRuntimeState - private readonly reconcileLeases: (sessionId: string) => Promise + private readonly reconcileLeases: ( + sessionId: string + ) => Promise private readonly handoffs: StructuredAgentSessionHostHandoff private readonly readableRestorer: StructuredAgentSessionReadableRestorer private readonly restartRestore = new StructuredAgentSessionRestartRestoreGate() @@ -112,7 +112,12 @@ export class StructuredAgentSessionHost { now: this.now }) this.holds = createStructuredAgentSessionHolds(this.lifetimeContext(), { - resume: (sessionId) => this.resumeForHold(sessionId), + resume: (sessionId) => + resumeStructuredAgentSessionForHold( + { ...this.lifetimeContext(), reconcileLeases: this.reconcileLeases }, + sessionId, + (params) => this.attach({ callerKey: 'trusted-local:surface-hold' }, params) + ), evict: (sessionId) => this.close(sessionId) }) this.readableRestorer = new StructuredAgentSessionReadableRestorer({ @@ -125,7 +130,10 @@ export class StructuredAgentSessionHost { hasSession: (sessionId) => this.sessions.has(sessionId), // Site 10: cannot overwrite a live entry — the restorer returns early on // `hasSession` inside the same serialized step as this `set`. - onReadable: (sessionId, restored) => this.sessions.set(sessionId, restored), + onReadable: (sessionId, restored) => { + this.sessions.set(sessionId, restored) + this.statusFeed.publish(sessionId) + }, restoreHandoff: (sessionId) => this.handoffs.restore(sessionId) }) this.eventRecovery = new StructuredAgentSessionEventRecovery({ @@ -160,20 +168,6 @@ export class StructuredAgentSessionHost { /** That surface is gone. The child outlives it by the release grace, and by any running turn. */ release = (sessionId: string, holderId: string): void => this.holds.release(sessionId, holderId) - private async resumeForHold(sessionId: string): Promise { - const unreconciled = await this.reconcileLeases(sessionId) - if (unreconciled) { - throw new Error(unreconciled.code) - } - await this.runtimeState.resolveRecovery(sessionId) - await resumeHeldStructuredAgentSession({ - sessionId, - deps: this.deps, - now: () => this.now(), - attach: (params) => this.attach({ callerKey: 'trusted-local:surface-hold' }, params) - }) - } - handleAdapterEvent = (event: Parameters[0]) => this.eventRecovery.handle(event) @@ -250,7 +244,7 @@ export class StructuredAgentSessionHost { attach( caller: StructuredAgentSessionCaller, params: AgentSessionAttachParams - ): Promise> { + ): Promise> { return attachStructuredAgentSession(this.attachContext(), caller.callerKey, params) } @@ -316,22 +310,23 @@ export class StructuredAgentSessionHost { requestHandoff = ( caller: StructuredAgentSessionCaller, - params: AgentSessionHandoffRequest - ): Promise> => + params: SessionWire.AgentSessionHandoffRequest + ): Promise> => this.handoffs.request(caller.callerKey, params) - readOptions = (sessionId: string): Promise => + readOptions = (sessionId: string): Promise => readStructuredAgentSessionOptions(this.mutationContext(), sessionId) - async handoffStatus(sessionId: string): Promise { + async handoffStatus(sessionId: string): Promise { this.requireSession(sessionId) return this.serialize(sessionId, () => refreshRecoverableStructuredHandoffStatus(this.handoffs, this.deps.store, sessionId) ) } - history = (request: AgentSessionHistoryRequest): AgentSessionHistoryResult => - this.backgroundTasks.history(request) + history = ( + request: SessionWire.AgentSessionHistoryRequest + ): SessionWire.AgentSessionHistoryResult => this.backgroundTasks.history(request) subscribe = (input: AgentSessionSubscribeInput): (() => void) => this.backgroundTasks.subscribe(input) @@ -342,6 +337,11 @@ export class StructuredAgentSessionHost { ) => this.backgroundTasks.publish(sessionId, state) unsubscribe = (sessionId: string, id: string): void => this.subscribers.close(sessionId, id) + /** Every session's projected status for session lists; unlike `subscribe`, retains nothing. */ + subscribeStatus = ( + subscriber: Parameters[0] + ): (() => void) => this.statusFeed.subscribe(subscriber) + private requireSession(sessionId: string): StructuredAgentSessionHostSession { const session = this.sessions.get(sessionId) if (!session) { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-plans.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-plans.ts index d0eb905443c..1194f0c87ff 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-plans.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-plans.ts @@ -76,15 +76,21 @@ export function cancelPlan(params: { envelope: AgentSessionMutationEnvelope turnId: string scope?: 'background-tasks' + taskId?: string }): MutationPlan { return { method: 'agentSession.cancel', - fields: { turnId: params.turnId, ...(params.scope ? { scope: params.scope } : {}) }, + fields: { + turnId: params.turnId, + ...(params.scope ? { scope: params.scope } : {}), + ...(params.taskId ? { taskId: params.taskId } : {}) + }, run: (ctx) => performCancel(ctx, { clientOperationId: params.envelope.clientOperationId, turnId: params.turnId, - ...(params.scope ? { scope: params.scope } : {}) + ...(params.scope ? { scope: params.scope } : {}), + ...(params.taskId ? { taskId: params.taskId } : {}) }), // Interrupting twice would kill a turn the client never asked to stop, so a // replay reports the turn as already handled instead. diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.test.ts new file mode 100644 index 00000000000..34e3fb8a4cf --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.test.ts @@ -0,0 +1,110 @@ +// Read restore decides whether a session comes back at all. +// +// A chat still in the pre-SQLite format has no `journal.db`, so the probe that +// loads one reports nothing. Reading that as "no session" is what removed these +// chats: an unpublished session is also what prunes its tab out of the saved +// workspace, so the tab is gone before anything can explain itself. + +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { journalDirectoryFor } from '../agent-session-journal/journal-paths' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { restoreStructuredAgentSessionRead } from './structured-agent-session-read-restore' + +const SESSION_ID = 'codex_read_restore_fixture' +const WORKSPACE_ID = 'repo-1::/tmp/workspace' + +const RECORD = { + schemaVersion: 2, + sessionId: SESSION_ID, + location: { + executionHostId: 'local', + wslDistro: null, + workspaceId: WORKSPACE_ID, + workspaceKind: 'git-worktree' + }, + provider: 'codex', + providerHandleChain: [ + { + linkId: 'codex-1-thread-1', + handle: { provider: 'codex', threadId: 'thread-1' }, + origin: 'created', + mintedAtFence: 1, + observedAt: 1 + } + ], + accountHome: { variable: 'CODEX_HOME', path: '/tmp/codex-home' }, + createdAt: 1, + updatedAt: 2, + lease: { sessionId: SESSION_ID, runtimeKind: 'native', runtimeFence: 1 } +} as unknown as AgentSessionRecord + +const store = { + getRecord: (sessionId: string) => (sessionId === SESSION_ID ? RECORD : null) +} as unknown as AgentSessionRecordStore + +let journalRoot: string +const opened: AgentSessionJournal[] = [] + +async function writeRemnant(name: string): Promise { + const dir = journalDirectoryFor(journalRoot, { + workspaceId: WORKSPACE_ID, + sessionId: SESSION_ID + }) + await mkdir(dir, { recursive: true }) + await writeFile(join(dir, name), '{"kind":"epoch","v":1,"seq":1}\n', 'utf8') + return join(dir, name) +} + +beforeEach(async () => { + journalRoot = await mkdtemp(join(tmpdir(), 'orca-read-restore-')) +}) + +afterEach(async () => { + await Promise.allSettled(opened.splice(0).map((journal) => journal.close())) + await rm(journalRoot, { recursive: true, force: true }) +}) + +describe('a session whose journal is still the pre-SQLite format', () => { + it('is published, carrying the message that explains it', async () => { + const transcript = await writeRemnant('log.jsonl') + + const restored = await restoreStructuredAgentSessionRead(store, journalRoot, SESSION_ID) + + expect(restored).not.toBeNull() + opened.push(restored!.journal) + const disclosed = restored!.journal + .snapshot() + .items.map((entry) => (entry.body.kind === 'status' ? entry.body.text : '')) + expect(disclosed.join('')).toContain(transcript) + // Publishing it costs no agent process; acquisition still waits for the user. + expect(restored!.hasProviderChild).toBe(false) + }) + + it('is published for a remnant whose log is gone', async () => { + await writeRemnant('snapshot.json') + + const restored = await restoreStructuredAgentSessionRead(store, journalRoot, SESSION_ID) + + expect(restored).not.toBeNull() + opened.push(restored!.journal) + }) + + it('still drops a session with neither a journal nor a remnant', async () => { + const restored = await restoreStructuredAgentSessionRead(store, journalRoot, SESSION_ID) + + expect(restored).toBeNull() + }) + + it('still drops a session with no record', async () => { + await writeRemnant('log.jsonl') + + const restored = await restoreStructuredAgentSessionRead(store, journalRoot, 'unknown-session') + + expect(restored).toBeNull() + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.ts index 5bed8f2920e..34fd6452b08 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-read-restore.ts @@ -3,6 +3,7 @@ import type { AgentSessionRecord } from '../../../shared/agent-session-record' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { findJournalFileFormatRemnant } from '../agent-session-journal/journal-file-format-remnant' import { loadJournal } from '../agent-session-journal/journal-open' import { journalDirectoryFor } from '../agent-session-journal/journal-paths' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' @@ -40,15 +41,27 @@ export async function restoreStructuredAgentSessionRead( sessionId }) const loaded = loadJournal(journalDir, sessionId) - if (!loaded || loaded.corrupt) { + if (loaded?.corrupt) { + return null + } + // A session still in the pre-SQLite format has no `journal.db` to load. Dropping + // it here leaves it unpublished, which is also what prunes its tab out of the + // saved workspace — so the chat disappears with nowhere to explain itself. + if (!loaded && !findJournalFileFormatRemnant(journalDir)) { return null } const journal = await openAgentSessionJournal({ identity: journalIdentityFor(record, params), journalDir, - loaded + // Omitted, not `null`: the store reads `null` as "replay already ran and + // found nothing" and founds a fresh epoch. In process the probe above is the + // previous statement, so the window is zero-width; this holds the line for a + // database another process creates in between. + ...(loaded ? { loaded } : {}) }) - // Read restore opens the journal and nothing else: no adapter call, so no provider child. + // Read restore opens the journal and nothing else: no adapter call, so no + // provider child. Opening it can still write — a session whose history is in + // the old format founds its epoch and commits the row explaining that here. return { journal, params, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-status-publication.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-status-publication.test.ts new file mode 100644 index 00000000000..884fba78a4e --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-status-publication.test.ts @@ -0,0 +1,149 @@ +// Startup restore has to publish status, not just index the session. +// +// A tab nobody reopens after a restart still owes the sidebar a row. The host restores such a +// session read-only, without a provider child, so the only thing that can surface its state is +// the status publication the restore wiring makes. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import type { + AgentSessionMutationEnvelope, + AgentSessionStatusEvent +} from '../../../shared/agent-session-wire' +import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' +import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { + HOST_TEST_NOW as NOW, + HOST_TEST_SESSION as SESSION, + HOST_TEST_THREAD as THREAD, + hostTestAttachParams, + hostTestMessage, + hostTestOperationId, + resetHostTestOperationIds +} from './structured-agent-session-host-test-data' + +const CALLER = { callerKey: 'client-1' } + +const hosts: StructuredAgentSessionHost[] = [] +let root = '' + +function adapter(): StructuredAgentSessionAdapter { + return { + acquire: async ({ fence, spawnToken }) => ({ + process: { hostId: 'local', pid: 4242, processStartTimeMs: 1_700_000_000_000, spawnToken }, + link: { + linkId: `link-${fence}`, + handle: { provider: 'codex', threadId: THREAD }, + origin: 'created', + mintedAtFence: fence, + observedAt: NOW + } + }), + dispatch: async () => ({ + state: 'accepted', + providerIdentity: { provider: 'codex', threadId: THREAD, turnId: 'turn-1', ordinal: 1 } + }), + cancelTurn: async () => ({ cancelled: true }), + answerPrompt: async () => undefined, + setOption: async () => undefined + } +} + +function createHost(store: AgentSessionRecordStore): StructuredAgentSessionHost { + const host = new StructuredAgentSessionHost({ + store, + adapter: adapter(), + journalRoot: root, + claimKeyId: 'key-1', + mintSpawnToken: () => 'spawn-a', + probeOwner: async () => ({ + outcome: 'indeterminate', + reason: 'read does not need ownership' + }), + now: () => NOW + }) + hosts.push(host) + return host +} + +function sendEnvelope( + store: AgentSessionRecordStore, + fields: Record +): AgentSessionMutationEnvelope { + return { + sessionId: SESSION, + clientOperationId: hostTestOperationId(), + expectedRuntimeFence: store.getRecord(SESSION)?.lease.runtimeFence ?? 1, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.send', + sessionId: SESSION, + fields + }) + } +} + +/** Persists one turn, then hands back a restarted host over the same directories. */ +async function restartWithPersistedTurn(): Promise { + root = await mkdtemp(join(tmpdir(), 'orca-restart-status-')) + resetHostTestOperationIds() + const directory = join(root, 'store') + const store = await AgentSessionRecordStore.open({ directory, hostId: 'local' }) + const host = createHost(store) + expect(await host.attach(CALLER, hostTestAttachParams(null))).toMatchObject({ ok: true }) + const body = hostTestMessage('persisted conversation') + await host.send(CALLER, { envelope: sendEnvelope(store, { body }), body }) + await host.flushAllStreamedEvents() + return createHost(await AgentSessionRecordStore.open({ directory, hostId: 'local' })) +} + +afterEach(async () => { + await Promise.all(hosts.splice(0).map((host) => host.flushAllStreamedEvents())) + await rm(root, { recursive: true, force: true }) + root = '' +}) + +describe('structured session restart status publication', () => { + // Served by the subscribe-time re-projection rather than the restore's own publish, so this + // covers what a restored journal projects — not the restore wiring. The test below pins that. + it('projects the persisted turn of a session restored without a provider', async () => { + const restarted = await restartWithPersistedTurn() + + await restarted.restoreReadableSessions() + const events: AgentSessionStatusEvent[] = [] + restarted.subscribeStatus({ id: 'session-list', emit: (event) => events.push(event) }) + + expect(events).toEqual([ + { + type: 'snapshot', + sessions: [ + expect.objectContaining({ + sessionId: SESSION, + workspaceId: 'workspace-1', + agent: 'codex', + status: 'idle', + latestPrompt: 'persisted conversation' + }) + ] + } + ]) + }) + + it('publishes a restored session to a list already sitting on the stream', async () => { + const restarted = await restartWithPersistedTurn() + const events: AgentSessionStatusEvent[] = [] + restarted.subscribeStatus({ id: 'session-list', emit: (event) => events.push(event) }) + expect(events).toEqual([{ type: 'snapshot', sessions: [] }]) + + await restarted.restoreReadableSessions() + + // The restore wiring publishes; without it this list never hears about the session at all. + expect(events.at(-1)).toEqual({ + type: 'status', + session: expect.objectContaining({ sessionId: SESSION, status: 'idle' }) + }) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts new file mode 100644 index 00000000000..7efd147c420 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts @@ -0,0 +1,242 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import type { AgentSessionStatusEvent } from '../../../shared/agent-session-wire' +import { createTrackedJournalOpener } from '../agent-session-journal/journal-store-test-open' +import { StructuredAgentSessionStatusFeed } from './structured-agent-session-status-feed' + +const SESSION = 'status-session' +const TURN_IDENTITY = { + provider: 'codex', + threadId: 'thread-1', + turnId: 'turn-1', + ordinal: 0 +} as const +const USER_IDENTITY = { + provider: 'codex', + threadId: 'thread-1', + turnId: 'turn-1', + ordinal: 1 +} as const + +let root: string +const journals = createTrackedJournalOpener() + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-agent-status-feed-')) +}) + +afterEach(async () => { + await journals.closeAll() + await rm(root, { recursive: true, force: true }) +}) + +async function openJournal(sessionId = SESSION) { + return journals.open({ + identity: { + sessionId, + workspaceId: 'workspace-1', + hostId: 'local', + agent: 'codex', + providerHandle: { kind: 'codex', threadId: 'thread-1' } + }, + journalDir: join(root, sessionId) + }) +} + +function indexed(session: { journal: Awaited> }) { + return { + journal: session.journal, + params: { location: { workspaceId: 'workspace-1' }, provider: 'codex' as const } + } +} + +function feedFor(sessions: Map> }>) { + let now = 1_000 + const feed = new StructuredAgentSessionStatusFeed({ + sessions: { + get: (sessionId: string) => { + const session = sessions.get(sessionId) + return session ? indexed(session) : undefined + }, + [Symbol.iterator]: function* () { + for (const [sessionId, session] of sessions) { + yield [sessionId, indexed(session)] as const + } + } + } as unknown as ReadonlyMap>, + getRecord: () => null, + now: () => (now += 1) + }) + const events: AgentSessionStatusEvent[] = [] + const dispose = feed.subscribe({ id: 'list-1', emit: (event) => events.push(event) }) + return { feed, events, dispose } +} + +describe('StructuredAgentSessionStatusFeed', () => { + it('opens with every readable session and reports no status before a persisted turn', async () => { + const journal = await openJournal() + const { events } = feedFor(new Map([[SESSION, { journal }]])) + + expect(events).toEqual([ + { + type: 'snapshot', + sessions: [ + { + sessionId: SESSION, + workspaceId: 'workspace-1', + agent: 'codex', + status: null, + latestPrompt: '', + updatedAt: expect.any(Number) + } + ] + } + ]) + }) + + it('publishes working, then idle once the running marker is tombstoned, and never a repeat', async () => { + const journal = await openJournal() + const { feed, events } = feedFor(new Map([[SESSION, { journal }]])) + await journal.appendItem( + USER_IDENTITY, + { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'write a poem' }] }, + { fence: 1 } + ) + await journal.appendItem( + TURN_IDENTITY, + { kind: 'status', text: 'Working', turnLifecycle: { turnId: 'turn-1', state: 'running' } }, + { fence: 1 } + ) + + feed.publish(SESSION) + feed.publish(SESSION) + expect(events.slice(1)).toEqual([ + { + type: 'status', + session: expect.objectContaining({ + sessionId: SESSION, + status: 'working', + latestPrompt: 'write a poem' + }) + } + ]) + + await journal.appendTombstone(TURN_IDENTITY, { fence: 1 }) + feed.publish(SESSION) + expect(events.at(-1)).toEqual({ + type: 'status', + session: expect.objectContaining({ sessionId: SESSION, status: 'idle' }) + }) + expect(events).toHaveLength(3) + }) + + it('reports a pending approval as attention', async () => { + const journal = await openJournal() + const { feed, events } = feedFor(new Map([[SESSION, { journal }]])) + await journal.appendItem( + USER_IDENTITY, + { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'run it' }] }, + { fence: 1 } + ) + await journal.appendItem( + TURN_IDENTITY, + { + kind: 'approval', + title: 'Run command?', + detail: null, + options: [{ id: 'yes', label: 'Allow' }], + resolution: { state: 'pending', selectedOptionId: null, resolvedBy: null, resolvedAt: null } + }, + { fence: 1 } + ) + + feed.publish(SESSION) + expect(events.at(-1)).toEqual({ + type: 'status', + session: expect.objectContaining({ status: 'attention' }) + }) + }) + + it('keeps the last projection for an evicted session and serves it to a new subscriber', async () => { + const journal = await openJournal() + const sessions = new Map([[SESSION, { journal }]]) + const { feed, events } = feedFor(sessions) + await journal.appendItem( + USER_IDENTITY, + { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'hello' }] }, + { fence: 1 } + ) + feed.publish(SESSION) + expect(events.at(-1)).toEqual({ + type: 'status', + session: expect.objectContaining({ status: 'idle', latestPrompt: 'hello' }) + }) + + // Eviction drops the host's index entry; the projection it already made stays true. + sessions.delete(SESSION) + feed.publish(SESSION) + const late: AgentSessionStatusEvent[] = [] + feed.subscribe({ id: 'list-late', emit: (event) => late.push(event) }) + + expect(events).toHaveLength(2) + expect(late).toEqual([ + { + type: 'snapshot', + sessions: [expect.objectContaining({ sessionId: SESSION, status: 'idle' })] + } + ]) + }) + + it('tells the sitting subscribers about a change a new subscriber re-projected', async () => { + const journal = await openJournal() + const { feed, events } = feedFor(new Map([[SESSION, { journal }]])) + // Journal appends and the feed's publish are separate queue submissions, so the journal + // can already hold the turn when a second client connects and re-projects it. + await journal.appendItem( + USER_IDENTITY, + { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'hello' }] }, + { fence: 1 } + ) + + const late: AgentSessionStatusEvent[] = [] + feed.subscribe({ id: 'list-late', emit: (event) => late.push(event) }) + + expect(events.at(-1)).toEqual({ + type: 'status', + session: expect.objectContaining({ status: 'idle', latestPrompt: 'hello' }) + }) + // The arriving subscriber reads that same state once, from its snapshot. + expect(late).toEqual([ + { + type: 'snapshot', + sessions: [expect.objectContaining({ status: 'idle', latestPrompt: 'hello' })] + } + ]) + // The cache is not left holding a value nobody was told about. + feed.publish(SESSION) + expect(events).toHaveLength(2) + }) + + it('ends a closed subscriber and keeps publishing to the rest', async () => { + const journal = await openJournal() + const { feed, events, dispose } = feedFor(new Map([[SESSION, { journal }]])) + const others: AgentSessionStatusEvent[] = [] + feed.subscribe({ id: 'list-2', emit: (event) => others.push(event) }) + + dispose() + await journal.appendItem( + USER_IDENTITY, + { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'hello' }] }, + { fence: 1 } + ) + feed.publish(SESSION) + + expect(events.at(-1)).toEqual({ type: 'end' }) + expect(others.at(-1)).toEqual({ + type: 'status', + session: expect.objectContaining({ status: 'idle' }) + }) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts new file mode 100644 index 00000000000..5ad494cf830 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts @@ -0,0 +1,130 @@ +// The host's answer to "what is every structured session doing", fanned out to session lists. +// +// A client used to learn whether a turn was running by replaying the journal through its own +// reducer, which tied the answer to whichever surface happened to hold a reader open: hide the +// chat and the sidebar froze on the last thing it had heard. The host always has the journal, so +// it projects the status once per journal publication and sends only the changes. +// +// The last projection is kept after the session's provider child is evicted: an idle session is +// still idle without a process, and a renderer that reloads must not lose every settled row until +// each chat is reopened. Restart is the one boundary that forgets, and restoring readable sessions +// republishes them. + +import { agentProviderSessionsEqual } from '../../../shared/agent-session-resume' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import type { + AgentSessionStatusEvent, + AgentSessionStatusSummary +} from '../../../shared/agent-session-wire' +import { projectStructuredAgentSessionStatusSummary } from '../../../shared/structured-agent-session-projection' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { structuredAgentSessionProviderSessionMetadata } from './structured-agent-session-history-result' + +export type StructuredAgentSessionStatusSubscriber = { + id: string + emit: (event: AgentSessionStatusEvent) => void +} + +type StatusFeedSession = { + journal: AgentSessionJournal + params: { location: { workspaceId: string }; provider: AgentSessionRecord['provider'] } +} + +export type StructuredAgentSessionStatusFeedDeps = { + sessions: ReadonlyMap + getRecord: (sessionId: string) => AgentSessionRecord | null + now: () => number +} + +function summariesEqual(a: AgentSessionStatusSummary, b: AgentSessionStatusSummary): boolean { + return ( + a.workspaceId === b.workspaceId && + a.agent === b.agent && + a.status === b.status && + a.latestPrompt === b.latestPrompt && + agentProviderSessionsEqual(undefined, a.providerSession, b.providerSession) + ) +} + +export class StructuredAgentSessionStatusFeed { + private readonly subscribers = new Map() + private readonly published = new Map() + + constructor(private readonly deps: StructuredAgentSessionStatusFeedDeps) {} + + /** Opens with every session this host has projected, live ones re-read, then only changes. */ + subscribe(subscriber: StructuredAgentSessionStatusSubscriber): () => void { + // Re-project before registering: a change found here has to reach the subscribers that + // already read the old value, and the arriving one carries it in its snapshot instead. + for (const [sessionId] of this.deps.sessions) { + this.publish(sessionId) + } + this.subscribers.set(subscriber.id, subscriber) + this.emit(subscriber, { type: 'snapshot', sessions: [...this.published.values()] }) + return () => this.unsubscribe(subscriber.id) + } + + unsubscribe(id: string): void { + const subscriber = this.subscribers.get(id) + if (!subscriber) { + return + } + this.subscribers.delete(id) + try { + subscriber.emit({ type: 'end' }) + } catch { + // The transport is already gone; teardown must remain idempotent. + } + } + + /** Re-projects one session after its journal changed; equal projections are not re-sent. */ + publish(sessionId: string, journal?: AgentSessionJournal): void { + const session = this.deps.sessions.get(sessionId) + if (!session) { + return + } + const summary = this.summaryFor(sessionId, session, journal ?? session.journal) + const previous = this.published.get(sessionId) + if (previous && summariesEqual(previous, summary)) { + return + } + this.published.set(sessionId, summary) + this.broadcast({ type: 'status', session: summary }) + } + + private summaryFor( + sessionId: string, + session: StatusFeedSession, + journal: AgentSessionJournal + ): AgentSessionStatusSummary { + // An unreadable journal projects as "no turn": the chat itself shows the reset. + const items = journal.isReadOnly ? [] : journal.snapshot().items + const providerSession = structuredAgentSessionProviderSessionMetadata( + this.deps.getRecord(sessionId) + ) + return { + sessionId, + workspaceId: session.params.location.workspaceId, + agent: session.params.provider, + ...projectStructuredAgentSessionStatusSummary(items), + ...(providerSession ? { providerSession } : {}), + updatedAt: this.deps.now() + } + } + + private broadcast(event: AgentSessionStatusEvent): void { + // A Map skips entries deleted mid-iteration, so a failing subscriber can drop itself here. + for (const subscriber of this.subscribers.values()) { + this.emit(subscriber, event) + } + } + + /** A dead transport must not poison every later publication. */ + private emit(subscriber: StructuredAgentSessionStatusSubscriber, event: AgentSessionStatusEvent) { + try { + subscriber.emit(event) + } catch { + this.subscribers.delete(subscriber.id) + } + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.test.ts index 6e156d91532..81bcfa82b40 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.test.ts @@ -5,6 +5,7 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest' import { AGENT_SESSION_JOURNAL_SCHEMA_VERSION } from '../../../shared/agent-session-journal-types' import type { AgentSessionHandoffStatus, + AgentSessionStatusEvent, AgentSessionSubscribeEvent } from '../../../shared/agent-session-wire' import { @@ -16,6 +17,7 @@ import { journalDatabaseFile } from '../agent-session-journal/journal-paths' import { insertJournalRow } from '../agent-session-journal/journal-row-table' import type { JournalRow } from '../agent-session-journal/journal-row-schema' import { createTrackedJournalOpener } from '../agent-session-journal/journal-store-test-open' +import { StructuredAgentSessionStatusFeed } from './structured-agent-session-status-feed' import { AgentSessionSubscribers } from './structured-agent-session-subscribers' const SESSION = 'subscriber-session' @@ -70,6 +72,96 @@ describe('AgentSessionSubscribers', () => { ]) }) + it('reports every content publication to the journal hook, subscribed or not', async () => { + const journal = await journals.open({ + identity: { + sessionId: SESSION, + workspaceId: 'workspace-1', + hostId: 'local', + agent: 'codex', + providerHandle: { kind: 'codex', threadId: 'thread-1' } + }, + journalDir: join(root, 'hook-journal') + }) + const published: string[] = [] + const subscribers = new AgentSessionSubscribers({ + onJournalPublished: (sessionId, published_journal) => { + expect(published_journal).toBe(journal) + published.push(sessionId) + } + }) + + subscribers.publish(SESSION, journal) + subscribers.reset(SESSION, journal, 'epoch_changed', 1) + subscribers.snapshot(SESSION, journal, 1) + subscribers.handoff(SESSION, 1, { + owner: 'native', + direction: null, + phase: 'idle', + stage: null, + operationId: null + }) + + expect(published).toEqual([SESSION, SESSION, SESSION]) + }) + + it('settles a session nobody is reading, from running to idle', async () => { + // The defect this whole feed exists for: status used to come from a transcript reader, so a + // session with no open pane had no reader and froze on whatever it last said. Nothing here + // ever calls `subscribers.open`. + const journal = await journals.open({ + identity: { + sessionId: SESSION, + workspaceId: 'workspace-1', + hostId: 'local', + agent: 'codex', + providerHandle: { kind: 'codex', threadId: 'thread-1' } + }, + journalDir: join(root, 'unread-journal') + }) + const statusFeed = new StructuredAgentSessionStatusFeed({ + sessions: new Map([ + [ + SESSION, + { journal, params: { location: { workspaceId: 'workspace-1' }, provider: 'codex' } } + ] + ]), + getRecord: () => null, + now: () => 1_000 + }) + const subscribers = new AgentSessionSubscribers({ + onJournalPublished: (sessionId, published) => statusFeed.publish(sessionId, published) + }) + const statuses: AgentSessionStatusEvent[] = [] + statusFeed.subscribe({ id: 'session-list', emit: (event) => statuses.push(event) }) + const turn = { provider: 'codex', threadId: 'thread-1', turnId: 'turn-1', ordinal: 0 } as const + + await journal.appendItem( + { ...turn, ordinal: 1 }, + { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'write a poem' }] }, + { fence: 1 } + ) + await journal.appendItem( + turn, + { kind: 'status', text: 'Working', turnLifecycle: { turnId: 'turn-1', state: 'running' } }, + { fence: 1 } + ) + subscribers.publish(SESSION, journal) + + expect(statuses.at(-1)).toEqual({ + type: 'status', + session: expect.objectContaining({ status: 'working', latestPrompt: 'write a poem' }) + }) + + await journal.appendTombstone(turn, { fence: 1 }) + subscribers.publish(SESSION, journal) + + expect(statuses.at(-1)).toEqual({ + type: 'status', + session: expect.objectContaining({ status: 'idle' }) + }) + }) + it('publishes handoff-only changes without serializing a transcript snapshot', async () => { const journal = await journals.open({ identity: { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts index d3131505384..29dffa6a687 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts @@ -36,9 +36,17 @@ type Subscriber = { fence: number } +export type AgentSessionSubscribersHooks = { + /** Fires after any publication that can change journal content, whether or not anyone + * is subscribed to the transcript: session lists project status from this same edge. */ + onJournalPublished?: (sessionId: string, journal: AgentSessionJournal) => void +} + export class AgentSessionSubscribers { private readonly bySession = new Map>() + constructor(private readonly hooks: AgentSessionSubscribersHooks = {}) {} + /** Opens the stream with a bounded tail page or, when the client's cursor * still resolves, with the rows it missed. Returns the disposer. */ open(input: { @@ -99,6 +107,7 @@ export class AgentSessionSubscribers { for (const subscriber of this.subscribers(sessionId)) { this.deliver(subscriber, journal) } + this.hooks.onJournalPublished?.(sessionId, journal) } /** Force every subscriber back to a bounded tail page — recovery, epoch @@ -123,6 +132,7 @@ export class AgentSessionSubscribers { subscriber.cursor = page.liveCursor ?? page.window.nextCursor subscriber.fence = fence } + this.hooks.onJournalPublished?.(sessionId, journal) } snapshot( @@ -143,6 +153,7 @@ export class AgentSessionSubscribers { subscriber.cursor = page.liveCursor ?? page.window.nextCursor subscriber.fence = fence } + this.hooks.onJournalPublished?.(sessionId, journal) } handoff(sessionId: string, fence: number, handoff: AgentSessionHandoffStatus): void { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-turns.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-turns.test.ts index e8e6f998bdd..31df2c44551 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-turns.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-turns.test.ts @@ -104,4 +104,40 @@ describe('performCancel', () => { expect(cancelTurn).not.toHaveBeenCalled() expect(journal.snapshot().items).toEqual([]) }) + + it('routes one background task id without interrupting the foreground turn or writing a row', async () => { + root = await mkdtemp(join(tmpdir(), 'orca-background-task-targeted-cancel-')) + const journal = await journals.open({ identity: IDENTITY, journalDir: root }) + const cancelTurn = vi.fn(async () => ({ cancelled: true })) + const stopBackgroundTasks = vi.fn(async () => ({ cancelled: true })) + const ctx: AgentSessionTurnContext = { + sessionId: 'session-1', + journal, + fence: 1, + adapter: { cancelTurn, stopBackgroundTasks } as unknown as StructuredAgentSessionAdapter, + persistOptions: async () => undefined, + resolvedBy: 'client-1', + publish: vi.fn(), + now: () => 1 + } + + const result = await performCancel(ctx, { + clientOperationId: 'cancel-background-task-2', + turnId: 'background-tasks', + scope: 'background-tasks', + taskId: 'task-2' + }) + + expect(result).toEqual({ + ok: true, + value: { turnId: 'background-tasks', cancelled: true } + }) + expect(stopBackgroundTasks).toHaveBeenCalledWith({ + sessionId: 'session-1', + fence: 1, + taskId: 'task-2' + }) + expect(cancelTurn).not.toHaveBeenCalled() + expect(journal.snapshot().items).toEqual([]) + }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts index 76c4e8e89d6..3bd98a61735 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts @@ -150,6 +150,7 @@ export async function performCancel( clientOperationId: string turnId: string scope?: 'background-tasks' + taskId?: string } ): Promise> { let cancelled = false @@ -159,7 +160,8 @@ export async function performCancel( ? ( await ctx.adapter.stopBackgroundTasks?.({ sessionId: ctx.sessionId, - fence: ctx.fence + fence: ctx.fence, + ...(input.taskId ? { taskId: input.taskId } : {}) }) )?.cancelled === true : ( diff --git a/src/main/providers/local-pty-provider-spawn-session.test.ts b/src/main/providers/local-pty-provider-spawn-session.test.ts index 7513d9c72cb..9dfa08d81bf 100644 --- a/src/main/providers/local-pty-provider-spawn-session.test.ts +++ b/src/main/providers/local-pty-provider-spawn-session.test.ts @@ -172,6 +172,7 @@ describe('LocalPtyProvider', () => { expect(second).toEqual({ id: 'serve-session-1', + incarnationId: first.incarnationId, pid: 12345, isReattach: true, // Why published: this attach really moved the PTY, unlike daemon/relay attach, so main @@ -220,7 +221,11 @@ describe('LocalPtyProvider', () => { attachOnly: true }) - expect(result).toMatchObject({ id: first.id, isReattach: true }) + expect(result).toMatchObject({ + id: first.id, + incarnationId: first.incarnationId, + isReattach: true + }) expect(spawnMock).not.toHaveBeenCalled() }) diff --git a/src/main/providers/local-pty-spawn-state.ts b/src/main/providers/local-pty-spawn-state.ts index d41f858cf98..2ab145f6c39 100644 --- a/src/main/providers/local-pty-spawn-state.ts +++ b/src/main/providers/local-pty-spawn-state.ts @@ -1,6 +1,7 @@ import type { PtySpawnResult } from './types' import { pendingLocalPtySpawns, + ptyIncarnations, ptyProcesses, ptyWslDistroById, type PendingLocalPtySpawn @@ -60,6 +61,7 @@ export function reattachLocalPty(id: string, cols: number, rows: number): PtySpa } return { id, + ...(ptyIncarnations.has(id) ? { incarnationId: ptyIncarnations.get(id) } : {}), pid: existing.pid, ...(ptyWslDistroById.has(id) ? { wslDistro: ptyWslDistroById.get(id) ?? null } : {}), isReattach: true, diff --git a/src/main/runtime/claude-structured-session-integration.test.ts b/src/main/runtime/claude-structured-session-integration.test.ts index 2e42d595b04..e9cba45ffa9 100644 --- a/src/main/runtime/claude-structured-session-integration.test.ts +++ b/src/main/runtime/claude-structured-session-integration.test.ts @@ -599,6 +599,46 @@ describe('a structured Claude session over agentSession.*', () => { `claude:${PROVIDER_SESSION}:assistant-leaf` ) + claude.live().handlers.onMessage?.({ + type: 'system', + subtype: 'background_tasks_changed', + session_id: PROVIDER_SESSION, + uuid: 'background-roster', + tasks: [ + { task_id: 'task-one', task_type: 'local_agent', description: 'First task' }, + { task_id: 'task-two', task_type: 'local_bash', description: 'Second task' } + ] + }) + const itemsBeforeTaskStop = itemsOf(stream) + const targetedStopFields = { + turnId: 'background-tasks', + scope: 'background-tasks', + taskId: 'task-two' + } + await expect( + ok('agentSession.cancel', { + envelope: envelope('agentSession.cancel', targetedStopFields, created.fence), + ...targetedStopFields + }) + ).resolves.toMatchObject({ turnId: 'background-tasks', cancelled: true }) + expect(claude.live().calls.filter((entry) => entry.subtype === 'stop_task')).toEqual([ + { subtype: 'stop_task', params: { taskId: 'task-two' } } + ]) + expect(itemsOf(stream)).toEqual(itemsBeforeTaskStop) + + const staleStopFields = { + turnId: 'background-tasks', + scope: 'background-tasks', + taskId: 'task-stale' + } + await expect( + ok('agentSession.cancel', { + envelope: envelope('agentSession.cancel', staleStopFields, created.fence), + ...staleStopFields + }) + ).resolves.toMatchObject({ turnId: 'background-tasks', cancelled: false }) + expect(claude.live().calls.filter((entry) => entry.subtype === 'stop_task')).toHaveLength(1) + const answeredPermission = Promise.resolve( claude.live().handlers.canUseTool?.('Bash', { command: 'ls' }, { requestId: 'permission-1', diff --git a/src/main/runtime/fetch-remote-cache.test.ts b/src/main/runtime/fetch-remote-cache.test.ts index 11cd2e0260d..5f0b8e249d6 100644 --- a/src/main/runtime/fetch-remote-cache.test.ts +++ b/src/main/runtime/fetch-remote-cache.test.ts @@ -176,8 +176,17 @@ describe('OrcaRuntimeService.fetchRemoteWithCache', () => { expect(caches.fetchLastCompletedAt.has('/repo/cache-0::origin')).toBe(false) }) - it.each(['main', 'a'.repeat(40), 'refs/remotes/main', ''])( - 'does not launch Git for a base without a remote/branch separator: %s', + it.each([ + 'main', + 'a'.repeat(40), + 'refs/remotes/main', + '', + 'origin/', + '/main', + 'refs/remotes/origin/', + 'refs/remotes//main' + ])( + 'does not launch Git for a base without both remote and branch components: %s', async (base) => { const runtime = new OrcaRuntimeService(null) await expect(runtime.resolveRemoteTrackingBase('/repo/e', base)).resolves.toBeNull() diff --git a/src/main/runtime/orca-runtime-refresh-repo-worktree-scan.ts b/src/main/runtime/orca-runtime-refresh-repo-worktree-scan.ts index 1964b5fdd2f..2df3f7b9f17 100644 --- a/src/main/runtime/orca-runtime-refresh-repo-worktree-scan.ts +++ b/src/main/runtime/orca-runtime-refresh-repo-worktree-scan.ts @@ -153,6 +153,11 @@ export class OrcaRuntimeWithRefreshRepoWorktreeScan extends OrcaRuntimeWithListK } } + invalidateWorktreeCatalog(repoId: string): void { + this.invalidateResolvedWorktreeCache() + this.invalidateWorktreeScanCacheForRepo(repoId) + } + protected invalidateSshWorktreeScanCacheInternal(targetId: string): void { const repos = this.store?.getRepos() ?? [] const affectedRepos = repos.filter((repo) => getRepoSshConnectionId(repo) === targetId) diff --git a/src/main/runtime/orca-runtime-tests/local-worktree-creation-part-02.spec.ts b/src/main/runtime/orca-runtime-tests/local-worktree-creation-part-02.spec.ts index fe6e8bfe3da..a12ffabcd61 100644 --- a/src/main/runtime/orca-runtime-tests/local-worktree-creation-part-02.spec.ts +++ b/src/main/runtime/orca-runtime-tests/local-worktree-creation-part-02.spec.ts @@ -50,7 +50,13 @@ describe('OrcaRuntimeService', () => { pushTarget: { remoteName: 'origin', branchName: 'feature/fix' } }) - expect(getBranchConflictKind).toHaveBeenCalledWith(TEST_REPO_PATH, 'feature/fix', 'abc123') + expect(getBranchConflictKind).toHaveBeenCalledWith( + TEST_REPO_PATH, + 'feature/fix', + 'abc123', + {}, + undefined + ) expect(getPRForBranchMock).toHaveBeenCalledWith(TEST_REPO_PATH, 'feature/fix') expect(addWorktree).toHaveBeenCalledWith( TEST_REPO_PATH, @@ -165,7 +171,9 @@ describe('OrcaRuntimeService', () => { expect(getBranchConflictKind).toHaveBeenCalledWith( TEST_REPO_PATH, 'feature/bitbucket', - 'abc123' + 'abc123', + {}, + undefined ) expect(getHostedReviewForBranchMock).toHaveBeenCalledWith( expect.objectContaining({ diff --git a/src/main/runtime/orca-runtime-tests/local-worktree-creation.spec.ts b/src/main/runtime/orca-runtime-tests/local-worktree-creation.spec.ts index 17fe670a09c..3dcd2d0584c 100644 --- a/src/main/runtime/orca-runtime-tests/local-worktree-creation.spec.ts +++ b/src/main/runtime/orca-runtime-tests/local-worktree-creation.spec.ts @@ -533,10 +533,14 @@ describe('OrcaRuntimeService', () => { branchNameOverride: 'feature/something' }) + // Why: an explicit branch override adopts the local branch before the conflict + // probe, so no lazy adoption callback is handed to getBranchConflictKind. expect(getBranchConflictKind).toHaveBeenCalledWith( TEST_REPO_PATH, 'feature/something', - 'origin/feature/something' + 'origin/feature/something', + {}, + undefined ) expect(addWorktree).toHaveBeenCalledWith( TEST_REPO_PATH, diff --git a/src/main/runtime/orca-runtime-tests/worktree-removal-and-reconciliation.spec.ts b/src/main/runtime/orca-runtime-tests/worktree-removal-and-reconciliation.spec.ts index 65da9caccde..630eaebf007 100644 --- a/src/main/runtime/orca-runtime-tests/worktree-removal-and-reconciliation.spec.ts +++ b/src/main/runtime/orca-runtime-tests/worktree-removal-and-reconciliation.spec.ts @@ -376,7 +376,18 @@ describe('OrcaRuntimeService', () => { TEST_REPO_PATH, 'runtime-wsl', 'origin/main', - { wslDistro: 'Ubuntu' } + { wslDistro: 'Ubuntu' }, + expect.any(Function) + ) + // Why: the lazy adoption callback is only invoked when the conflict probe + // sees a local ref, so drive it here to prove adoption also routes via WSL. + const adoptLocalBranch = vi + .mocked(getBranchConflictKind) + .mock.calls.findLast((call) => call[1] === 'runtime-wsl')?.[4] + await expect(adoptLocalBranch?.()).resolves.toBe(false) + expect(gitSpy).toHaveBeenCalledWith( + ['rev-parse', '--verify', '--quiet', 'refs/heads/runtime-wsl^{commit}'], + { cwd: TEST_REPO_PATH, wslDistro: 'Ubuntu' } ) expect(getPRForBranchMock).toHaveBeenCalledWith( TEST_REPO_PATH, diff --git a/src/main/runtime/orchestration/db/dispatch-context/dispatch-context-store.ts b/src/main/runtime/orchestration/db/dispatch-context/dispatch-context-store.ts index 822e06896db..38905f30434 100644 --- a/src/main/runtime/orchestration/db/dispatch-context/dispatch-context-store.ts +++ b/src/main/runtime/orchestration/db/dispatch-context/dispatch-context-store.ts @@ -8,6 +8,7 @@ import { claimDispatchContextRow } from '../dispatch-row-writer' import { recordedCreatorIdentity, type DispatchCreator } from '../dispatch-depth' import type { OrchestrationDb } from '../orchestration-db' import { transitionLifecycleWithDb } from '../lifecycle-transition' +import { taskNotFoundError, taskNotStartableError } from '../../task-dispatch-refusal' export function createDispatchContext( this: OrchestrationDb, @@ -27,10 +28,14 @@ export function createDispatchContext( const depth = this.resolveChildDispatchDepth(params.creator, params.maxDepth) const task = this.getTask(taskId) if (!task) { - throw new Error(`Task not found: ${taskId}`) + throw taskNotFoundError(`Task not found: ${taskId}`, { taskId }) } if (task.status !== 'ready') { - throw new Error(`Task ${taskId} is ${task.status}; only ready tasks can be dispatched`) + throw taskNotStartableError( + this, + `Task ${taskId} is ${task.status}; only ready tasks can be dispatched`, + task + ) } // Why: lock on pane identity too, so a reminted handle can't open a second concurrent dispatch on the same pane. @@ -76,9 +81,12 @@ export function createDispatchContext( `Terminal ${assigneeHandle} already has an active dispatch (${occupied.id} for task ${occupied.task_id})` ) } - throw new Error( - `Task ${taskId} is ${current?.status ?? 'missing'}; only ready tasks can be dispatched` - ) + // Why: the atomic claim lost to a concurrent status change; report it with the same + // typed receipt as the precheck so the loser can recover instead of reading runtime_error. + const message = `Task ${taskId} is ${current?.status ?? 'missing'}; only ready tasks can be dispatched` + throw current + ? taskNotStartableError(this, message, current) + : taskNotFoundError(message, { taskId }) } transitionLifecycleWithDb(this.db, { entity: 'task', diff --git a/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-start.ts b/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-start.ts index 9013ad321f9..22e4a81403d 100644 --- a/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-start.ts +++ b/src/main/runtime/orchestration/db/worker-dispatch/worker-dispatch-start.ts @@ -7,6 +7,7 @@ import type { OrchestrationDb } from '../orchestration-db' import { insertStartingDispatchContextRow } from '../dispatch-row-writer' import { recordedCreatorIdentity, type DispatchCreator } from '../dispatch-depth' import { transitionLifecycleWithDb } from '../lifecycle-transition' +import { taskNotFoundError, taskNotStartableError } from '../../task-dispatch-refusal' export function createStartingWorkerDispatch( this: OrchestrationDb, @@ -84,7 +85,9 @@ export function createStartingWorkerDispatch( }) : undefined if (!task) { - throw new OrchestrationError('task_not_found', `Task ${params.taskId} was not found.`) + // Why: `--spec` creates the Task inline, so a missing row here always names an explicit id. + const taskId = params.taskId ?? '' + throw taskNotFoundError(`Task ${taskId} was not found.`, { taskId }) } if (params.retryOf) { const prior = this.getDispatchContextById(params.retryOf) @@ -101,15 +104,18 @@ export function createStartingWorkerDispatch( !priorSettled || !['failed', 'blocked'].includes(task.status) ) { - throw new OrchestrationError( - 'task_not_startable', - `Task ${task.id} cannot retry from Dispatch ${params.retryOf}.` + throw taskNotStartableError( + this, + `Task ${task.id} cannot retry from Dispatch ${params.retryOf}.`, + task, + params.retryOf ) } } else if (task.status !== 'ready') { - throw new OrchestrationError( - 'task_not_startable', - `Task ${task.id} is ${task.status}; only a ready Task can start.` + throw taskNotStartableError( + this, + `Task ${task.id} is ${task.status}; only a ready Task can start.`, + task ) } diff --git a/src/main/runtime/orchestration/orchestration-worker-dispatch-db.test.ts b/src/main/runtime/orchestration/orchestration-worker-dispatch-db.test.ts index 6676288520c..16a118008de 100644 --- a/src/main/runtime/orchestration/orchestration-worker-dispatch-db.test.ts +++ b/src/main/runtime/orchestration/orchestration-worker-dispatch-db.test.ts @@ -196,7 +196,12 @@ describe('OrchestrationDb worker Dispatch state', () => { payloadHash: 'payload_hash' } }) - ).toThrow('was not found') + ).toThrowError( + expect.objectContaining({ + code: 'task_not_found', + message: 'Task task_missing was not found.' + }) + ) expect(d.getMutationReceipt('caller_fingerprint', 'invalid_worker_start')).toBeUndefined() }) diff --git a/src/main/runtime/orchestration/task-dispatch-refusal.ts b/src/main/runtime/orchestration/task-dispatch-refusal.ts new file mode 100644 index 00000000000..f3e026d0f73 --- /dev/null +++ b/src/main/runtime/orchestration/task-dispatch-refusal.ts @@ -0,0 +1,61 @@ +import type { OrchestrationDb } from './db' +import { OrchestrationError } from './orchestration-error' +import type { TaskRow } from './types' +import { + injectRejectedRefusal, + taskNotFoundRefusal, + taskNotStartableRefusal, + type DispatchRefusalReceipt, + type InjectRejectionReason +} from '../../../shared/orchestration-dispatch-refusal-contract' + +// Why: each site keeps the exact message it published before; only the code and data are shared. + +export function taskNotFoundError( + message: string, + detail: { taskId: string; runId?: string } +): OrchestrationError { + return toError(taskNotFoundRefusal(message, detail)) +} + +export function taskNotStartableError( + db: OrchestrationDb, + message: string, + task: TaskRow, + retryOf?: string +): OrchestrationError { + return toError( + taskNotStartableRefusal(message, { + taskId: task.id, + status: task.status, + unmetDependencies: unmetTaskDependencies(db, task), + ...(retryOf ? { retryOf } : {}) + }) + ) +} + +export function injectRejectedError( + terminal: string, + reason: InjectRejectionReason +): OrchestrationError { + return toError(injectRejectedRefusal(terminal, reason)) +} + +function toError(receipt: DispatchRefusalReceipt): OrchestrationError { + return new OrchestrationError(receipt.code, receipt.message, receipt.data) +} + +function unmetTaskDependencies(db: OrchestrationDb, task: TaskRow): string[] { + let deps: unknown + try { + deps = JSON.parse(task.deps) + } catch { + return [] + } + if (!Array.isArray(deps)) { + return [] + } + return deps.filter( + (dep): dep is string => typeof dep === 'string' && db.getTask(dep)?.status !== 'completed' + ) +} diff --git a/src/main/runtime/rpc/errors.ts b/src/main/runtime/rpc/errors.ts index dcb235e8354..bbf918f4f55 100644 --- a/src/main/runtime/rpc/errors.ts +++ b/src/main/runtime/rpc/errors.ts @@ -86,6 +86,7 @@ const STRUCTURED_RUNTIME_PASSTHROUGH_CODES: ReadonlySet = new Set([ 'consumer_fenced', 'task_not_found', 'task_not_startable', + 'inject_rejected', 'dispatch_not_found', 'dispatch_run_mismatch', 'terminal_not_found', diff --git a/src/main/runtime/rpc/methods/orchestration-dispatch-error-codes.test.ts b/src/main/runtime/rpc/methods/orchestration-dispatch-error-codes.test.ts new file mode 100644 index 00000000000..918724e6aaf --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-dispatch-error-codes.test.ts @@ -0,0 +1,242 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { ORCHESTRATION_CONTRACT_VERSION } from '../../../../shared/protocol-version' +import { + buildInjectRejectionMessage, + injectRejectedRefusal, + taskNotFoundRefusal, + taskNotStartableRefusal +} from '../../../../shared/orchestration-dispatch-refusal-contract' +import { OrcaRuntimeService } from '../../orca-runtime' +import { OrchestrationDb } from '../../orchestration/db' +import type { RpcFailure, RpcRequest, RpcResponse } from '../core' +import { RpcDispatcher } from '../dispatcher' +import { ORCHESTRATION_METHODS } from './orchestration' + +const COORDINATOR_HANDLE = 'term_codes_coordinator' +const COORDINATOR_PANE = 'tab_coord:cccccccc-cccc-4ccc-8ccc-cccccccccccc' +const WORKER_HANDLE = 'term_codes_worker' +const WORKER_PANE = 'tab_worker:dddddddd-dddd-4ddd-8ddd-dddddddddddd' + +type Harness = { db: OrchestrationDb; runtime: OrcaRuntimeService; dispatcher: RpcDispatcher } + +const harnesses: Harness[] = [] +let requestSequence = 0 + +afterEach(() => { + for (const harness of harnesses.splice(0)) { + harness.db.close() + } + vi.restoreAllMocks() +}) + +// Why: an agent reads the receipt code to pick a recovery; every case is driven from the real +// RPC dispatcher and checked against the shared contract the CLI-side test formats. +describe('orchestration dispatch failure codes through RpcDispatcher', () => { + it('reports task_not_found for a task id that does not exist', async () => { + const harness = createHarness() + + const response = await dispatch(harness, { task: 'task_missing', to: WORKER_HANDLE }) + + expect(expectFailure(response).error).toEqual( + taskNotFoundRefusal('Task not found: task_missing', { taskId: 'task_missing' }) + ) + }) + + it('reports task_not_startable with the unmet dependencies for a pending task', async () => { + const harness = createHarness() + const parent = harness.db.createTask({ spec: 'parent' }) + const child = harness.db.createTask({ spec: 'child', deps: [parent.id] }) + + const response = await dispatch(harness, { task: child.id, to: WORKER_HANDLE }) + + expect(expectFailure(response).error).toEqual( + taskNotStartableRefusal(`Task ${child.id} is pending; only ready tasks can be dispatched`, { + taskId: child.id, + status: 'pending', + unmetDependencies: [parent.id] + }) + ) + expect(harness.db.getTask(child.id)?.status).toBe('pending') + }) + + it('reports task_not_startable with the status for a completed task', async () => { + const harness = createHarness() + const task = harness.db.createTask({ spec: 'done' }) + harness.db.updateTaskStatus(task.id, 'completed') + + const response = await dispatch(harness, { task: task.id, to: WORKER_HANDLE }) + + expect(expectFailure(response).error).toEqual( + taskNotStartableRefusal(`Task ${task.id} is completed; only ready tasks can be dispatched`, { + taskId: task.id, + status: 'completed', + unmetDependencies: [] + }) + ) + }) + + it('reports inject_rejected when the target terminal runs no recognized agent', async () => { + const harness = createHarness() + const task = harness.db.createTask({ spec: 'work' }) + vi.spyOn(harness.runtime, 'isTerminalRunningAgent').mockResolvedValue(false) + + const response = await dispatch(harness, { task: task.id, to: WORKER_HANDLE, inject: true }) + + expect(expectFailure(response).error).toEqual( + injectRejectedRefusal(WORKER_HANDLE, 'no_agent_detected') + ) + expect(expectFailure(response).error.message).toBe(buildInjectRejectionMessage(WORKER_HANDLE)) + expect(harness.db.getTask(task.id)?.status).toBe('ready') + expect(harness.db.getDispatchContext(task.id)).toBeUndefined() + }) + + it('reports task_not_startable with dependency detail from worker-start', async () => { + const harness = createHarness() + const parent = harness.db.createTask({ spec: 'parent' }) + const child = harness.db.createTask({ spec: 'child', deps: [parent.id] }) + mockWorkerStartTopology(harness.runtime) + + const response = await harness.dispatcher.dispatch( + request('orchestration.workerStart', { + task: child.id, + from: COORDINATOR_HANDLE, + agent: 'claude' + }) + ) + + expect(expectFailure(response).error).toEqual( + taskNotStartableRefusal(`Task ${child.id} is pending; only a ready Task can start.`, { + taskId: child.id, + status: 'pending', + unmetDependencies: [parent.id] + }) + ) + expect(harness.db.getTask(child.id)?.status).toBe('pending') + }) + + it('reports task_not_startable with retry detail for an invalid --retry-of', async () => { + const harness = createHarness() + const task = harness.db.createTask({ spec: 'work' }) + mockWorkerStartTopology(harness.runtime) + + const response = await harness.dispatcher.dispatch( + request('orchestration.workerStart', { + task: task.id, + from: COORDINATOR_HANDLE, + agent: 'claude', + retryOf: 'ctx_missing' + }) + ) + + expect(expectFailure(response).error).toEqual( + taskNotStartableRefusal(`Task ${task.id} cannot retry from Dispatch ctx_missing.`, { + taskId: task.id, + status: 'ready', + unmetDependencies: [], + retryOf: 'ctx_missing' + }) + ) + }) + + it('types the atomic claim loser when the task changes after the ready precheck', async () => { + const harness = createHarness() + const task = harness.db.createTask({ spec: 'raced' }) + // Why: the pane lookup runs after the ready precheck and before the DB claim, so failing the + // task there is the interleaving a concurrent status change produces. The loser's DB refusal + // must carry the same typed receipt instead of the bare Error it used to throw. + vi.mocked(harness.runtime.getTerminalPaneKey).mockImplementation((handle) => { + if (handle === WORKER_HANDLE) { + harness.db.updateTaskStatus(task.id, 'failed', 'raced out') + return WORKER_PANE + } + return handle === COORDINATOR_HANDLE ? COORDINATOR_PANE : null + }) + + const response = await dispatch(harness, { task: task.id, to: WORKER_HANDLE }) + + expect(expectFailure(response).error).toEqual( + taskNotStartableRefusal(`Task ${task.id} is failed; only ready tasks can be dispatched`, { + taskId: task.id, + status: 'failed', + unmetDependencies: [] + }) + ) + expect(harness.db.getDispatchContext(task.id)).toBeUndefined() + }) + + it('keeps runtime_error for a genuinely unexpected dispatch failure', async () => { + const harness = createHarness() + const task = harness.db.createTask({ spec: 'work' }) + vi.spyOn(harness.runtime, 'isTerminalRunningAgent').mockRejectedValue( + new Error('probe exploded') + ) + + const response = await dispatch(harness, { task: task.id, to: WORKER_HANDLE, inject: true }) + + expect(expectFailure(response).error).toMatchObject({ + code: 'runtime_error', + message: 'probe exploded' + }) + }) +}) + +function expectFailure(response: RpcResponse): RpcFailure { + if (response.ok) { + throw new Error(`Expected a failure, got ${JSON.stringify(response.result)}`) + } + return response +} + +function createHarness(): Harness { + const db = new OrchestrationDb(':memory:') + const runtime = new OrcaRuntimeService() + runtime.setOrchestrationDb(db) + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === COORDINATOR_HANDLE ? COORDINATOR_PANE : handle === WORKER_HANDLE ? WORKER_PANE : null + ) + vi.spyOn(runtime, 'getTerminalProcessIncarnation').mockImplementation((handle) => + handle === WORKER_HANDLE ? 'pty-worker:incarnation-1' : null + ) + const runId = db.createRun({ + objective: 'Typed dispatch failures', + coordinatorHandle: COORDINATOR_HANDLE, + coordinatorPaneKey: COORDINATOR_PANE + }).id + const createTask = db.createTask.bind(db) + db.createTask = (task) => createTask({ ...task, runId: task.runId ?? runId }) + const harness = { + db, + runtime, + dispatcher: new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS }) + } + harnesses.push(harness) + return harness +} + +function mockWorkerStartTopology(runtime: OrcaRuntimeService): void { + vi.spyOn(runtime, 'validateOrchestrationAgentLauncher').mockImplementation(() => {}) + vi.spyOn(runtime, 'showTerminal').mockImplementation( + async (handle) => ({ handle, worktreeId: 'repo::worktree', status: 'running' }) as never + ) + vi.spyOn(runtime, 'showManagedTerminalWorkspace').mockResolvedValue({ + id: 'repo::worktree' + } as never) +} + +function dispatch(harness: Harness, params: Record): Promise { + return harness.dispatcher.dispatch( + request('orchestration.dispatch', { from: COORDINATOR_HANDLE, ...params }) + ) +} + +function request(method: string, params: Record): RpcRequest { + requestSequence += 1 + return { + id: `rpc_dispatch_error_codes_${requestSequence}`, + authToken: 'test-token', + method, + params, + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationRequestId: `dispatch_error_codes_${requestSequence}` + } +} diff --git a/src/main/runtime/rpc/methods/orchestration/gates/gates.ts b/src/main/runtime/rpc/methods/orchestration/gates/gates.ts index 8ed88450bc2..76bfd23b76e 100644 --- a/src/main/runtime/rpc/methods/orchestration/gates/gates.ts +++ b/src/main/runtime/rpc/methods/orchestration/gates/gates.ts @@ -4,7 +4,7 @@ import { OptionalFiniteNumber, OptionalString, requiredString } from '../../../s import type { GateStatus } from '../../../../orchestration/db' import { Coordinator } from '../../../../orchestration/coordinator' import { resolveRunScope } from '../runs/run-scope' -import { OrchestrationError } from '../../../../orchestration/orchestration-error' +import { taskNotFoundError } from '../../../../orchestration/task-dispatch-refusal' // Why: the coordinator instance is stored at module scope so orchestration.runStop // can signal it to halt. Only one coordinator can run at a time (enforced by @@ -137,10 +137,10 @@ export const ORCHESTRATION_GATE_METHODS: RpcMethod[] = [ callerEvidence: orchestrationCompatibilityEvidence }) if (task.run_id !== run.id) { - throw new OrchestrationError( - 'task_not_found', - `Task ${params.task} was not found in Run ${run.id}.` - ) + throw taskNotFoundError(`Task ${params.task} was not found in Run ${run.id}.`, { + taskId: params.task, + runId: run.id + }) } const gate = db.createGate({ taskId: params.task, diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/inject-rejection-message.test.ts b/src/main/runtime/rpc/methods/orchestration/messaging/inject-rejection-message.test.ts deleted file mode 100644 index 3bb215ad0a9..00000000000 --- a/src/main/runtime/rpc/methods/orchestration/messaging/inject-rejection-message.test.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { buildInjectRejectionMessage } from './inject-rejection-message' -import { TUI_AGENT_CONFIG } from '../../../../../../shared/tui-agent-config' -import { recognizeAgentProcess } from '../../../../../../shared/agent-process-recognition' - -describe('buildInjectRejectionMessage', () => { - const message = buildInjectRejectionMessage('term_a') - - it('keeps the substring callers and scripts match on', () => { - expect(message).toContain('Cannot dispatch --inject to terminal term_a') - expect(message).toContain('no recognized agent detected') - }) - - it('names every agent Orca recognizes, including agy', () => { - expect(message).toMatch(/\bagy\b/) - for (const config of Object.values(TUI_AGENT_CONFIG)) { - expect(message).toContain(config.expectedProcess) - } - }) - - it('lists only names detection actually resolves, deduped and sorted', () => { - const listed = (/\(([^)]+)\)/.exec(message)?.[1] ?? '').split(', ') - - expect(listed.length).toBeGreaterThan(0) - expect(new Set(listed).size).toBe(listed.length) - expect([...listed].sort()).toEqual(listed) - for (const name of listed) { - expect(recognizeAgentProcess(name)).not.toBeNull() - } - }) -}) diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/inject-rejection-message.ts b/src/main/runtime/rpc/methods/orchestration/messaging/inject-rejection-message.ts deleted file mode 100644 index 9340d452b21..00000000000 --- a/src/main/runtime/rpc/methods/orchestration/messaging/inject-rejection-message.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { TUI_AGENT_CONFIG } from '../../../../../../shared/tui-agent-config' - -// Why: the old five-name example read as an allowlist (#15125); derive from the field detection keys on so it cannot drift. -// Not filtered by `disabledTuiAgents` — that gates Orca's launchers, not detection, so a hand-started disabled agent still injects. -const RECOGNIZED_AGENT_PROCESS_NAMES = [ - ...new Set(Object.values(TUI_AGENT_CONFIG).map((config) => config.expectedProcess)) -].sort() - -export function buildInjectRejectionMessage(terminal: string): string { - return ( - `Cannot dispatch --inject to terminal ${terminal}: no recognized agent detected. ` + - `Orca detects these agent CLIs (${RECOGNIZED_AGENT_PROCESS_NAMES.join(', ')}). ` + - 'Start one in the terminal and let it finish launching, ' + - 'or dispatch without --inject and send the prompt manually.' - ) -} diff --git a/src/main/runtime/rpc/methods/orchestration/runs/dispatch-methods.ts b/src/main/runtime/rpc/methods/orchestration/runs/dispatch-methods.ts index f836a5fc02d..abc941bf98c 100644 --- a/src/main/runtime/rpc/methods/orchestration/runs/dispatch-methods.ts +++ b/src/main/runtime/rpc/methods/orchestration/runs/dispatch-methods.ts @@ -2,7 +2,11 @@ import { defineMethod, type RpcMethod } from '../../../core' import { OrchestrationError } from '../../../../orchestration/orchestration-error' import { buildDispatchPreamble } from '../../../../orchestration/preamble' import { resolveDispatchCreator } from './dispatch-creator' -import { buildInjectRejectionMessage } from '../messaging/inject-rejection-message' +import { + injectRejectedError, + taskNotFoundError, + taskNotStartableError +} from '../../../../orchestration/task-dispatch-refusal' import { resolveRunScope } from './run-scope' import { DispatchParams, DispatchShowParams } from '../schemas' @@ -23,7 +27,7 @@ export const ORCHESTRATION_DISPATCH_METHODS: RpcMethod[] = [ const db = runtime.getOrchestrationDb() const task = db.getTask(params.task) if (!task) { - throw new Error(`Task not found: ${params.task}`) + throw taskNotFoundError(`Task not found: ${params.task}`, { taskId: params.task }) } const run = resolveRunScope(runtime, { runId: params.run, @@ -33,10 +37,10 @@ export const ORCHESTRATION_DISPATCH_METHODS: RpcMethod[] = [ callerEvidence: orchestrationCompatibilityEvidence }) if (task.run_id !== run.id) { - throw new OrchestrationError( - 'task_not_found', - `Task ${task.id} was not found in Run ${run.id}.` - ) + throw taskNotFoundError(`Task ${task.id} was not found in Run ${run.id}.`, { + taskId: task.id, + runId: run.id + }) } // Why: dry-run previews the preamble without mutating state, so it skips the ready-status check and uses a placeholder dispatchId. @@ -67,7 +71,11 @@ export const ORCHESTRATION_DISPATCH_METHODS: RpcMethod[] = [ const to = params.to if (task.status !== 'ready') { - throw new Error(`Task ${params.task} is ${task.status}; only ready tasks can be dispatched`) + throw taskNotStartableError( + db, + `Task ${params.task} is ${task.status}; only ready tasks can be dispatched`, + task + ) } const dispatchAuthority = runtime.getOrchestrationDispatchAuthority(to) @@ -102,7 +110,7 @@ export const ORCHESTRATION_DISPATCH_METHODS: RpcMethod[] = [ if (params.inject) { const hasAgent = await runtime.isTerminalRunningAgent(to) if (!hasAgent) { - throw new Error(buildInjectRejectionMessage(to)) + throw injectRejectedError(to, 'no_agent_detected') } } diff --git a/src/main/runtime/rpc/methods/orchestration/runs/tasks-dispatch.test.ts b/src/main/runtime/rpc/methods/orchestration/runs/tasks-dispatch.test.ts index 25db715b78e..f7418cee573 100644 --- a/src/main/runtime/rpc/methods/orchestration/runs/tasks-dispatch.test.ts +++ b/src/main/runtime/rpc/methods/orchestration/runs/tasks-dispatch.test.ts @@ -3,7 +3,7 @@ import type { RpcContext } from '../../../core' import { createOrchestrationRpcHarness } from '../rpc-test-harness' import type { OrchestrationDb } from '../../../../orchestration/db' import type { OrcaRuntimeService } from '../../../../orca-runtime' -import { buildInjectRejectionMessage } from '../messaging/inject-rejection-message' +import { buildInjectRejectionMessage } from '../../../../../../shared/orchestration-dispatch-refusal-contract' import { createRootDispatch } from '../../../../orchestration/db/root-dispatch-test-fixture' describe('orchestration RPC methods', () => { diff --git a/src/main/runtime/rpc/methods/structured-agent-session-schemas.ts b/src/main/runtime/rpc/methods/structured-agent-session-schemas.ts index da66923c731..58dece2256c 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-schemas.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-schemas.ts @@ -152,9 +152,13 @@ export const CancelParams = z .object({ envelope: MutationEnvelope, turnId: Identifier('Invalid turn id'), - scope: z.literal('background-tasks').optional() + scope: z.literal('background-tasks').optional(), + taskId: Identifier('Invalid task id').optional() }) .strict() + .refine((value) => value.taskId === undefined || value.scope === 'background-tasks', { + message: 'A task id requires background-task scope' + }) export const RespondParams = z .object({ diff --git a/src/main/runtime/rpc/methods/structured-agent-session-status-stream.ts b/src/main/runtime/rpc/methods/structured-agent-session-status-stream.ts new file mode 100644 index 00000000000..8637089c254 --- /dev/null +++ b/src/main/runtime/rpc/methods/structured-agent-session-status-stream.ts @@ -0,0 +1,62 @@ +// `agentSession.subscribeStatus` — every structured session's projected status on one stream. +// +// Session lists read turn state from here instead of replaying transcripts: one stream per client +// covers every session, and unlike a transcript subscription it retains none of them. + +import { defineStreamingMethod, type RpcAnyMethod, type RpcContext } from '../core' +import { requireStructuredHost as requireHost } from './structured-agent-session-gate' +import { structuredAgentSessionStatusSubscriptionId } from './structured-agent-session-subscription-id' + +/** Ties a stream to both ends that can close it — the runtime's subscription registry and the + * transport abort — so either one runs `onClose` exactly once. */ +export function bindStructuredAgentSessionStream( + ctx: RpcContext, + subscriptionId: string, + onClose: () => void +): { isClosed: () => boolean } { + let closed = false + let releaseTransportSubscription = (): void => {} + const onTransportAbort = (): void => releaseTransportSubscription() + const cleanup = (): void => { + closed = true + ctx.signal?.removeEventListener('abort', onTransportAbort) + onClose() + } + let registration: { releaseIfCurrent: () => void } + if (typeof ctx.runtime.registerOwnedSubscriptionCleanup === 'function') { + registration = ctx.runtime.registerOwnedSubscriptionCleanup( + subscriptionId, + cleanup, + ctx.connectionId + ) + } else { + ctx.runtime.registerSubscriptionCleanup(subscriptionId, cleanup, ctx.connectionId) + registration = { releaseIfCurrent: () => ctx.runtime.cleanupSubscription(subscriptionId) } + } + releaseTransportSubscription = registration.releaseIfCurrent + ctx.signal?.addEventListener('abort', onTransportAbort, { once: true }) + if (ctx.signal?.aborted) { + onTransportAbort() + } + return { isClosed: () => closed } +} + +export const STRUCTURED_AGENT_SESSION_STATUS_METHODS: RpcAnyMethod[] = [ + defineStreamingMethod({ + name: 'agentSession.subscribeStatus', + params: null, + handler: async (_params, ctx, emit) => { + const host = requireHost(ctx) + const subscriptionId = structuredAgentSessionStatusSubscriptionId(ctx) + let dispose = (): void => {} + const stream = bindStructuredAgentSessionStream(ctx, subscriptionId, () => dispose()) + if (stream.isClosed()) { + return + } + dispose = host.subscribeStatus({ id: subscriptionId, emit }) + if (stream.isClosed()) { + dispose() + } + } + }) +] diff --git a/src/main/runtime/rpc/methods/structured-agent-session-subscription-id.ts b/src/main/runtime/rpc/methods/structured-agent-session-subscription-id.ts new file mode 100644 index 00000000000..f3d00232359 --- /dev/null +++ b/src/main/runtime/rpc/methods/structured-agent-session-subscription-id.ts @@ -0,0 +1,29 @@ +// Subscription ids for the streaming `agentSession.*` methods. +// +// Shared control multiplexes several streams over one socket, so the frame id keeps one +// subscriber from evicting another. It is appended only when present: collapsing a missing +// frame id to a constant is the collision the rule exists to prevent. + +import type { RpcContext } from '../core' + +const SUBSCRIPTION_PREFIX = 'agentSession' + +function withFrameId(ctx: RpcContext, base: string): string { + return ctx.requestId ? `${base}:${ctx.requestId}` : base +} + +/** The id a session's streams share before the frame id. `unsubscribe` addresses this + * directly and sweeps `${base}:` to reach every frame under it. */ +export function structuredAgentSessionSubscriptionBase(ctx: RpcContext, sessionId: string): string { + return `${SUBSCRIPTION_PREFIX}:${ctx.connectionId ?? 'local'}:${sessionId}` +} + +/** One session's transcript stream. */ +export function structuredAgentSessionSubscriptionId(ctx: RpcContext, sessionId: string): string { + return withFrameId(ctx, structuredAgentSessionSubscriptionBase(ctx, sessionId)) +} + +/** The status feed, which is per connection rather than per session. */ +export function structuredAgentSessionStatusSubscriptionId(ctx: RpcContext): string { + return withFrameId(ctx, `${SUBSCRIPTION_PREFIX}.status:${ctx.connectionId ?? 'local'}`) +} diff --git a/src/main/runtime/rpc/methods/structured-agent-session.test.ts b/src/main/runtime/rpc/methods/structured-agent-session.test.ts index e4888e4a9df..5220b3a00fe 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session.test.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session.test.ts @@ -2,8 +2,14 @@ // accepts once they can. import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { AgentJournalRenderItem } from '../../../../shared/agent-session-journal-types' +import type { AgentSessionJournal } from '../../../native-chat/agent-session-journal/journal-store' import type { StructuredAgentSessionHost } from '../../../native-chat/agent-session-wire/structured-agent-session-host' import { setStructuredAgentSessionHost } from '../../../native-chat/agent-session-wire/structured-agent-session-registry' +import { + StructuredAgentSessionStatusFeed, + type StructuredAgentSessionStatusSubscriber +} from '../../../native-chat/agent-session-wire/structured-agent-session-status-feed' import { RUNTIME_CAPABILITIES, RUNTIME_PROTOCOL_VERSION, @@ -64,6 +70,44 @@ function request(method: string, params: unknown): RpcRequest { let hostCalls: Record> let runtimeCalls: Record> +const STATUS_SESSION = 'session-status' +const STATUS_ITEMS: AgentJournalRenderItem[] = [ + { + itemId: 'user-1', + sequence: 1, + revision: 1, + observedAt: 1, + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'write a poem' }] } + }, + { + itemId: 'turn-1', + sequence: 2, + revision: 1, + observedAt: 2, + body: { kind: 'status', text: 'Working', turnLifecycle: { turnId: 'turn-1', state: 'running' } } + } +] + +/** One indexed session over a journal that reads back fixed items; the projection is real. */ +function statusFeed(): StructuredAgentSessionStatusFeed { + return new StructuredAgentSessionStatusFeed({ + sessions: new Map([ + [ + STATUS_SESSION, + { + journal: { + isReadOnly: false, + snapshot: () => ({ items: STATUS_ITEMS }) + } as unknown as AgentSessionJournal, + params: { location: { workspaceId: 'workspace-1' }, provider: 'codex' as const } + } + ] + ]), + getRecord: () => null, + now: () => 1_000 + }) +} + function hostStub(): StructuredAgentSessionHost { hostCalls = { attach: vi.fn(async () => ({ @@ -122,6 +166,11 @@ function hostStub(): StructuredAgentSessionHost { })), history: vi.fn(() => ({ ok: true, page: { items: [] } })), subscribe: vi.fn(() => () => undefined), + // A real feed, so the snapshot this method hands back is a genuine projection rather + // than a shape the stub restated. + subscribeStatus: vi.fn((subscriber: StructuredAgentSessionStatusSubscriber) => + statusFeed().subscribe(subscriber) + ), unsubscribe: vi.fn() } return hostCalls as unknown as StructuredAgentSessionHost @@ -240,7 +289,7 @@ describe('capability gating', () => { } // Bump deliberately: the whole agentSession.* surface is behind the structured capability, // so an additive method is invisible to old clients and needs no protocol bump. - expect(STRUCTURED_AGENT_SESSION_METHODS).toHaveLength(17) + expect(STRUCTURED_AGENT_SESSION_METHODS).toHaveLength(18) }) it('hides the surface from a declared client that did not advertise it', async () => { @@ -480,6 +529,20 @@ describe('method routing', () => { ]) }) + it('routes an optional background task id through cancellation', async () => { + const params = { + envelope: envelope(), + turnId: 'background-tasks', + scope: 'background-tasks' as const, + taskId: 'task-2' + } + + const response = await call('agentSession.cancel', params, STRUCTURED_CLIENT) + + expect(response).toMatchObject({ ok: true }) + expect(hostCalls.cancel).toHaveBeenCalledWith(expect.anything(), params) + }) + it('routes the structured handoff mutation through the host', async () => { const response = await call('agentSession.requestHandoff', { envelope: envelope(), @@ -510,6 +573,21 @@ describe('parameter validation', () => { }) }) + it('rejects invalid or unscoped background task ids', async () => { + await rejects('agentSession.cancel', { + envelope: envelope(), + turnId: 'background-tasks', + scope: 'background-tasks', + taskId: ' task-2' + }) + await rejects('agentSession.cancel', { + envelope: envelope(), + turnId: 'turn-1', + taskId: 'task-2' + }) + expect(hostCalls.cancel).not.toHaveBeenCalled() + }) + it('refuses to let a client author anything but a user turn', async () => { await rejects( 'agentSession.send', @@ -583,3 +661,31 @@ describe('parameter validation', () => { expect(response).toMatchObject({ ok: true }) }) }) + +describe('agentSession.subscribeStatus', () => { + it('is invisible to a client without the structured capability', async () => { + const reply = await call('agentSession.subscribeStatus', null, { clientKind: 'runtime' }) + expect(reply.ok).toBe(false) + expect(hostCalls.subscribeStatus).not.toHaveBeenCalled() + }) + + it('opens the host status feed with a projected snapshot as its first reply', async () => { + const reply = await call('agentSession.subscribeStatus', null, STRUCTURED_CLIENT) + expect(reply).toMatchObject({ + ok: true, + result: { + type: 'snapshot', + sessions: [ + { + sessionId: STATUS_SESSION, + workspaceId: 'workspace-1', + agent: 'codex', + status: 'working', + latestPrompt: 'write a poem' + } + ] + } + }) + expect(hostCalls.subscribeStatus).toHaveBeenCalledOnce() + }) +}) diff --git a/src/main/runtime/rpc/methods/structured-agent-session.ts b/src/main/runtime/rpc/methods/structured-agent-session.ts index b69ff6fd628..61b0f4dcf4f 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session.ts @@ -21,6 +21,14 @@ import { import type { AgentSessionAttachParams } from '../../../native-chat/agent-session-wire/structured-agent-session-attach' import { STRUCTURED_AGENT_SESSION_HOLD_METHODS } from './structured-agent-session-hold' import { resolveUncommittedStructuredCreate } from './structured-agent-session-precommit-refusal' +import { + bindStructuredAgentSessionStream, + STRUCTURED_AGENT_SESSION_STATUS_METHODS +} from './structured-agent-session-status-stream' +import { + structuredAgentSessionSubscriptionBase as subscriptionBaseFor, + structuredAgentSessionSubscriptionId as subscriptionIdFor +} from './structured-agent-session-subscription-id' import { AttachParams, CancelParams, @@ -37,15 +45,6 @@ import { UnsubscribeParams } from './structured-agent-session-schemas' -const SUBSCRIPTION_PREFIX = 'agentSession' - -function subscriptionIdFor(ctx: RpcContext, sessionId: string): string { - const base = `${SUBSCRIPTION_PREFIX}:${ctx.connectionId ?? 'local'}:${sessionId}` - // Shared control multiplexes several streams over one socket; the frame id - // keeps one subscriber from evicting another on the same session. - return ctx.requestId ? `${base}:${ctx.requestId}` : base -} - /** * The attach-shaped entries take the location from the client instead of resolving it from a * worktree, so they never reach the worktree-resolving create-support check. Ask the executing @@ -245,33 +244,12 @@ export const STRUCTURED_AGENT_SESSION_METHODS: RpcAnyMethod[] = [ // Retain-only: reading history must never be what starts a provider process. Current clients // explicitly hold every open surface before subscribing. const streamHolder = `subscription:${subscriptionId}` - let closed = false let dispose = (): void => {} - let releaseTransportSubscription = (): void => {} - const onTransportAbort = (): void => releaseTransportSubscription() - const cleanup = () => { - closed = true - ctx.signal?.removeEventListener('abort', onTransportAbort) + const stream = bindStructuredAgentSessionStream(ctx, subscriptionId, () => { dispose() host.release(params.sessionId, streamHolder) - } - let registration: { releaseIfCurrent: () => void } - if (typeof ctx.runtime.registerOwnedSubscriptionCleanup === 'function') { - registration = ctx.runtime.registerOwnedSubscriptionCleanup( - subscriptionId, - cleanup, - ctx.connectionId - ) - } else { - ctx.runtime.registerSubscriptionCleanup(subscriptionId, cleanup, ctx.connectionId) - registration = { releaseIfCurrent: () => ctx.runtime.cleanupSubscription(subscriptionId) } - } - releaseTransportSubscription = registration.releaseIfCurrent - ctx.signal?.addEventListener('abort', onTransportAbort, { once: true }) - if (ctx.signal?.aborted) { - onTransportAbort() - } - if (closed) { + }) + if (stream.isClosed()) { return } // The host emits the opening snapshot (or the missed batch) synchronously @@ -282,7 +260,7 @@ export const STRUCTURED_AGENT_SESSION_METHODS: RpcAnyMethod[] = [ emit, ...(params.cursor ? { cursor: params.cursor } : {}) }) - if (closed) { + if (stream.isClosed()) { dispose() } else { // Fire-and-forget, but never unhandled: a resume that refuses leaves the stream holding a @@ -300,8 +278,7 @@ export const STRUCTURED_AGENT_SESSION_METHODS: RpcAnyMethod[] = [ params: UnsubscribeParams, handler: async (params, ctx) => { requireHost(ctx) - const connection = ctx.connectionId ?? 'local' - const base = `${SUBSCRIPTION_PREFIX}:${connection}:${params.sessionId}` + const base = subscriptionBaseFor(ctx, params.sessionId) if (params.subscriptionId) { ctx.runtime.cleanupSubscription(`${base}:${params.subscriptionId}`) return { unsubscribed: true } @@ -311,5 +288,6 @@ export const STRUCTURED_AGENT_SESSION_METHODS: RpcAnyMethod[] = [ return { unsubscribed: true } } }), - ...STRUCTURED_AGENT_SESSION_HOLD_METHODS + ...STRUCTURED_AGENT_SESSION_HOLD_METHODS, + ...STRUCTURED_AGENT_SESSION_STATUS_METHODS ] diff --git a/src/main/runtime/runtime-local-worktree-create-candidate.ts b/src/main/runtime/runtime-local-worktree-create-candidate.ts index 6c454666148..9de955c5cd1 100644 --- a/src/main/runtime/runtime-local-worktree-create-candidate.ts +++ b/src/main/runtime/runtime-local-worktree-create-candidate.ts @@ -110,23 +110,31 @@ export async function resolveRuntimeLocalWorktreeCreateCandidate(args: { args.username, args.localWorktreeGitOptions ) - checkoutExistingBranch = await canCheckoutExistingLocalBranch( - args.repo.path, - branchName, - args.baseBranch, - ...args.localWorktreeGitOptionArgs - ) - if (checkoutExistingBranch && !selectedExistingLocalBranchName) { - selectedExistingLocalBranchName = branchName + const tryExistingBranch = async (): Promise => { + checkoutExistingBranch = await canCheckoutExistingLocalBranch( + args.repo.path, + branchName, + args.baseBranch, + ...args.localWorktreeGitOptionArgs + ) + return checkoutExistingBranch } + const preferExistingBranch = Boolean( + args.request.branchNameOverride || selectedExistingLocalBranchName + ) + checkoutExistingBranch = preferExistingBranch && (await tryExistingBranch()) branchConflictKind = checkoutExistingBranch ? null : await getBranchConflictKind( args.repo.path, branchName, args.baseBranch, - ...args.localWorktreeGitOptionArgs + args.localWorktreeGitOptions, + preferExistingBranch ? undefined : tryExistingBranch ) + if (checkoutExistingBranch && !selectedExistingLocalBranchName) { + selectedExistingLocalBranchName = branchName + } const allowedPushTargetRemoteConflict = branchConflictKind && isAllowedPushTargetRemoteConflict(branchConflictKind, branchName, args.request) diff --git a/src/main/runtime/runtime-remote-fetch-controller.ts b/src/main/runtime/runtime-remote-fetch-controller.ts index f40f25f6c82..dbcc240525b 100644 --- a/src/main/runtime/runtime-remote-fetch-controller.ts +++ b/src/main/runtime/runtime-remote-fetch-controller.ts @@ -231,7 +231,7 @@ export class RuntimeRemoteFetchController { ? baseBranch.slice(remoteRefPrefix.length) : baseBranch // A remote-tracking base needs both a configured remote and a branch component. - if (!shortBaseBranch.includes('/')) { + if (shortBaseBranch.indexOf('/') <= 0 || shortBaseBranch.endsWith('/')) { return null } let remotes: string[] diff --git a/src/main/runtime/worktree-scan-admin-fingerprint-gate.test.ts b/src/main/runtime/worktree-scan-admin-fingerprint-gate.test.ts index 3750884f84f..7dfe0144406 100644 --- a/src/main/runtime/worktree-scan-admin-fingerprint-gate.test.ts +++ b/src/main/runtime/worktree-scan-admin-fingerprint-gate.test.ts @@ -273,6 +273,22 @@ describe('worktree scan admin-fingerprint gate', () => { } }) + it('resolves a just-created id after invalidation even within both cache TTLs', async () => { + const { runtime, list } = makeRuntime() + listWorktreesStrictMock.mockResolvedValueOnce([ + { path: REPO_PATH, head: 'abc', branch: 'main', isBare: false, isMainWorktree: true } + ]) + await list() + await expect(runtime.showManagedWorktree(`id:${WORKTREE_ID}`)).rejects.toThrow( + 'selector_not_found' + ) + runtime.invalidateWorktreeCatalog(REPO_ID) + await expect(runtime.showManagedWorktree(`id:${WORKTREE_ID}`)).resolves.toMatchObject({ + id: WORKTREE_ID + }) + expect(scanCount()).toBe(2) + }) + it('scans when the probe cannot describe the repo', async () => { vi.useFakeTimers() try { diff --git a/src/main/ssh/ssh-remote-cli-dispatch-refusal-passthrough.test.ts b/src/main/ssh/ssh-remote-cli-dispatch-refusal-passthrough.test.ts new file mode 100644 index 00000000000..cd0503348d0 --- /dev/null +++ b/src/main/ssh/ssh-remote-cli-dispatch-refusal-passthrough.test.ts @@ -0,0 +1,64 @@ +import { EventEmitter } from 'node:events' +import { expect, it, vi } from 'vitest' + +vi.mock('electron', () => ({ + app: { + isPackaged: false, + getAppPath: () => '/host/app' + } +})) +vi.mock('../persistence', () => ({ + getCanonicalUserDataPath: () => '/host/user-data' +})) + +import { OrcaRuntimeService } from '../runtime/orca-runtime' +import { runRemoteOrcaCli } from './ssh-remote-orca-cli' + +// Why: the SSH bridge captures the host CLI child's stdout and exit code without reparsing; this +// pins that a typed refusal envelope and its nonzero exit reach the remote agent unchanged. +it('relays typed dispatch refusal codes from the host CLI unchanged', async () => { + const child = new EventEmitter() as EventEmitter & { + stdout: EventEmitter + stderr: EventEmitter + stdin: { end: ReturnType; on: ReturnType } + kill: ReturnType + } + child.stdout = new EventEmitter() + child.stderr = new EventEmitter() + child.stdin = { end: vi.fn(), on: vi.fn() } + child.kill = vi.fn() + const spawn = vi.fn(() => child) + const refusal = { + id: 'rpc_1', + ok: false, + error: { + code: 'task_not_startable', + message: 'Task task_1 is pending; only ready tasks can be dispatched', + data: { taskId: 'task_1', status: 'pending', unmetDependencies: ['task_0'] } + }, + _meta: { runtimeId: 'runtime_1' } + } + + const resultPromise = runRemoteOrcaCli( + new OrcaRuntimeService(), + { + argv: ['orchestration', 'dispatch', '--task', 'task_1', '--to', 'term_w', '--json'], + cwd: '/home/alice/repo', + env: { ORCA_TERMINAL_HANDLE: 'term_ssh' } + }, + { + execPath: '/host/electron', + cliEntryPath: '/host/app/out/cli/index.js', + userDataPath: '/host/user-data', + entryExists: () => true, + spawn: spawn as never + } + ) + + const stdout = `${JSON.stringify(refusal, null, 2)}\n` + await Promise.resolve() + child.stdout.emit('data', Buffer.from(stdout)) + child.emit('close', 1) + + expect(await resultPromise).toEqual({ stdout, stderr: '', exitCode: 1 }) +}) diff --git a/src/main/window/createMainWindow-startup-reveal.test.ts b/src/main/window/createMainWindow-startup-reveal.test.ts index 1200132d319..f103881ea83 100644 --- a/src/main/window/createMainWindow-startup-reveal.test.ts +++ b/src/main/window/createMainWindow-startup-reveal.test.ts @@ -78,6 +78,34 @@ describe('createMainWindow', () => { } } + it.each(['darwin', 'linux', 'win32'] as const)( + 'keeps explicit background startup hidden through ready/load/fallback on %s', + (platform) => { + vi.useFakeTimers() + vi.stubEnv('ORCA_BACKGROUND_LAUNCH', '1') + const { browserWindowInstance, windowHandlers } = createStartupRevealWindowFixture() + const showInactive = vi.fn() + Object.assign(browserWindowInstance, { showInactive }) + try { + withPlatform(platform, () => { + createMainWindow(createStartupRevealStore(true) as never, { revealOnDidFinishLoad: true }) + const revealAfterLoad = browserWindowInstance.webContents.on.mock.calls.find( + ([event]) => event === 'did-finish-load' + )?.[1] + expect(revealAfterLoad).toBeTypeOf('function') + revealAfterLoad?.() + windowHandlers['ready-to-show']() + vi.advanceTimersByTime(10_000) + expect(browserWindowInstance.show).not.toHaveBeenCalled() + expect(showInactive).not.toHaveBeenCalled() + expect(browserWindowInstance.maximize).not.toHaveBeenCalled() + }) + } finally { + vi.unstubAllEnvs() + } + } + ) + it('ignores duplicate ready-to-show events after startup maximize has already run', () => { const { browserWindowInstance, windowHandlers } = createStartupRevealWindowFixture() diff --git a/src/main/window/focus-existing-window.test.ts b/src/main/window/focus-existing-window.test.ts index babb9a15490..9f5dc522150 100644 --- a/src/main/window/focus-existing-window.test.ts +++ b/src/main/window/focus-existing-window.test.ts @@ -1,5 +1,5 @@ import type { App, BrowserWindow } from 'electron' -import { describe, expect, it, vi } from 'vitest' +import { afterEach, describe, expect, it, vi } from 'vitest' import { focusExistingMainWindow } from './focus-existing-window' type FakeWindowOptions = { @@ -78,7 +78,32 @@ function makeTimer(): { } } +afterEach(() => vi.unstubAllEnvs()) + describe('focusExistingMainWindow', () => { + it.each(['darwin', 'linux', 'win32'] as const)( + 'never restores or activates a background window on %s', + (platform) => { + vi.stubEnv('ORCA_BACKGROUND_LAUNCH', '1') + vi.stubEnv('ORCA_E2E_FOREGROUND', '1') + const app = makeFakeApp() + const window = makeFakeWindow({ minimized: true }) + const timer = makeTimer() + focusExistingMainWindow({ + app, + getWindow: () => window, + openWindow: vi.fn(), + platform, + setTimeout: timer.setTimeout + }) + expect(app.focus).not.toHaveBeenCalled() + for (const call of Object.values(window.calls)) { + expect(call).not.toHaveBeenCalled() + } + expect(timer.scheduledMs()).toEqual([]) + } + ) + it('aggressively foregrounds an existing Windows window on second launch', () => { const app = makeFakeApp() const window = makeFakeWindow() diff --git a/src/main/window/focus-existing-window.ts b/src/main/window/focus-existing-window.ts index 4cadb49a743..903e6a8b321 100644 --- a/src/main/window/focus-existing-window.ts +++ b/src/main/window/focus-existing-window.ts @@ -1,5 +1,9 @@ import type { App, BrowserWindow } from 'electron' -import { isBackgroundLaunch, showWindowWithoutStealingFocus } from './foreground-activation-policy' +import { + isBackgroundLaunch, + isWindowlessLaunch, + showWindowWithoutStealingFocus +} from './foreground-activation-policy' type FocusTimer = (callback: () => void, ms: number) => unknown @@ -34,7 +38,7 @@ function safelyFocusApp(app: Pick): void { } export function safelyRevealWindow(window: BrowserWindow): void { - if (window.isDestroyed()) { + if (window.isDestroyed() || isWindowlessLaunch()) { return } if (window.isMinimized()) { diff --git a/src/main/window/foreground-activation-policy.test.ts b/src/main/window/foreground-activation-policy.test.ts index 0a45f00387e..3b33c9be881 100644 --- a/src/main/window/foreground-activation-policy.test.ts +++ b/src/main/window/foreground-activation-policy.test.ts @@ -32,6 +32,10 @@ describe('isBackgroundLaunch', () => { expect(isBackgroundLaunch({})).toBe(false) }) + it('keeps an explicit background request despite inherited foreground flags', () => { + expect(isBackgroundLaunch({ ORCA_BACKGROUND_LAUNCH: '1', ORCA_E2E_FOREGROUND: '1' })).toBe(true) + }) + it('lets native-focus specs opt back into the foreground', () => { expect(isBackgroundLaunch({ ORCA_E2E_HEADFUL: '1', ORCA_E2E_FOREGROUND: '1' })).toBe(false) expect(isWindowlessLaunch({ ORCA_E2E_HEADLESS: '1', ORCA_E2E_FOREGROUND: '1' })).toBe(false) @@ -39,10 +43,17 @@ describe('isBackgroundLaunch', () => { }) describe('isWindowlessLaunch', () => { - it('is headless-only; a headful run still paints', () => { + it('keeps explicit background launches hidden while headful E2E can paint', () => { expect(isWindowlessLaunch({ ORCA_E2E_HEADLESS: '1' })).toBe(true) expect(isWindowlessLaunch({ ORCA_E2E_HEADLESS: '1', ORCA_E2E_HEADFUL: '1' })).toBe(false) - expect(isWindowlessLaunch({ ORCA_BACKGROUND_LAUNCH: '1' })).toBe(false) + expect(isWindowlessLaunch({ ORCA_BACKGROUND_LAUNCH: '1' })).toBe(true) + expect( + isWindowlessLaunch({ + ORCA_BACKGROUND_LAUNCH: '1', + ORCA_E2E_HEADFUL: '1', + ORCA_E2E_FOREGROUND: '1' + }) + ).toBe(true) }) }) @@ -54,9 +65,16 @@ describe('showWindowWithoutStealingFocus', () => { expect(window.showInactive).not.toHaveBeenCalled() }) - it('shows a background window without activating it', () => { + it('never reveals an explicitly background window', () => { const window = makeWindow() showWindowWithoutStealingFocus(window, { ORCA_BACKGROUND_LAUNCH: '1' }) + expect(window.showInactive).not.toHaveBeenCalled() + expect(window.show).not.toHaveBeenCalled() + }) + + it('still reveals explicitly headful E2E without activation', () => { + const window = makeWindow() + showWindowWithoutStealingFocus(window, { ORCA_E2E_HEADFUL: '1' }) expect(window.showInactive).toHaveBeenCalledOnce() expect(window.show).not.toHaveBeenCalled() }) @@ -83,18 +101,21 @@ describe('applyBackgroundActivationPolicy', () => { } } - it('drops the macOS Dock tile and menu bar for headless runs', () => { - const app = makeApp() - expect( - applyBackgroundActivationPolicy({ - app, - env: { ORCA_E2E_HEADLESS: '1' }, - platform: 'darwin' - }) - ).toBe(true) - expect(app.dock.hide).toHaveBeenCalledOnce() - expect(app.setActivationPolicy).toHaveBeenCalledWith('accessory') - }) + it.each(['ORCA_E2E_HEADLESS', 'ORCA_BACKGROUND_LAUNCH'])( + 'drops the macOS Dock tile and menu bar for %s', + (flag) => { + const app = makeApp() + expect( + applyBackgroundActivationPolicy({ + app, + env: { [flag]: '1' }, + platform: 'darwin' + }) + ).toBe(true) + expect(app.dock.hide).toHaveBeenCalledOnce() + expect(app.setActivationPolicy).toHaveBeenCalledWith('accessory') + } + ) it('leaves a headful or user launch with its normal Dock presence', () => { const headful = makeApp() diff --git a/src/main/window/foreground-activation-policy.ts b/src/main/window/foreground-activation-policy.ts index c2ee6b19e73..5d51f50e487 100644 --- a/src/main/window/foreground-activation-policy.ts +++ b/src/main/window/foreground-activation-policy.ts @@ -5,8 +5,8 @@ import { app as electronApp, type BrowserWindow } from 'electron' * validation). These runs may use the machine, but must never take the OS * foreground away from whatever the developer is doing. * - * ORCA_BACKGROUND_LAUNCH=1 opts a normal launch in; ORCA_E2E_FOREGROUND=1 opts - * back out for the few specs whose subject *is* native focus (IME, key events). + * ORCA_BACKGROUND_LAUNCH=1 keeps automation off screen. Native-focus specs + * can use ORCA_E2E_FOREGROUND=1 only without an explicit background request. */ type ActivationPolicyApp = { @@ -19,19 +19,21 @@ type PolicyEnv = Readonly> /** True when this process must not steal focus, raise windows, or activate the app. */ export function isBackgroundLaunch(env: PolicyEnv = process.env): boolean { + if (env.ORCA_BACKGROUND_LAUNCH === '1') { + return true + } if (env.ORCA_E2E_FOREGROUND === '1') { return false } - return ( - env.ORCA_BACKGROUND_LAUNCH === '1' || - env.ORCA_E2E_HEADLESS === '1' || - env.ORCA_E2E_HEADFUL === '1' - ) + return env.ORCA_E2E_HEADLESS === '1' || env.ORCA_E2E_HEADFUL === '1' } -/** True when no window should reach the screen at all (headless E2E; Playwright drives via CDP). */ +/** True when no window should reach the screen at all (background or headless E2E; Playwright drives via CDP). */ export function isWindowlessLaunch(env: PolicyEnv = process.env): boolean { - return isBackgroundLaunch(env) && env.ORCA_E2E_HEADLESS === '1' && env.ORCA_E2E_HEADFUL !== '1' + return ( + env.ORCA_BACKGROUND_LAUNCH === '1' || + (isBackgroundLaunch(env) && env.ORCA_E2E_HEADLESS === '1' && env.ORCA_E2E_HEADFUL !== '1') + ) } /** @@ -63,7 +65,7 @@ export function applyBackgroundActivationPolicy( /** * Reveal a window without taking the foreground: hidden entirely when windowless, - * `showInactive()` (visible, not raised over the active app) in background launches. + * `showInactive()` for explicitly headful E2E runs. */ export function showWindowWithoutStealingFocus( window: BrowserWindow, diff --git a/src/renderer/src/components/automations/AutomationListExternalRow.tsx b/src/renderer/src/components/automations/AutomationListExternalRow.tsx new file mode 100644 index 00000000000..b26173467ab --- /dev/null +++ b/src/renderer/src/components/automations/AutomationListExternalRow.tsx @@ -0,0 +1,260 @@ +import React from 'react' +import { MoreHorizontal, Pause, Pencil, Play, Trash2 } from 'lucide-react' +import { + ContextMenu, + ContextMenuContent, + ContextMenuItem, + ContextMenuSeparator, + ContextMenuTrigger +} from '@/components/ui/context-menu' +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuSeparator, + DropdownMenuTrigger +} from '@/components/ui/dropdown-menu' +import { Button } from '@/components/ui/button' +import { cn } from '@/lib/utils' +import type { + ExternalAutomationAction, + ExternalAutomationJob, + ExternalAutomationManager +} from '../../../../shared/automations-types' +import type { SshConnectionState } from '../../../../shared/ssh-types' +import type { ExternalAutomationListEntry } from './external-automation-list-entries' +import type { ExternalAutomationScope } from './external-automation-scope-client' +import { + formatExternalDate, + getExternalProviderLabel, + getExternalTargetKindLabel +} from './external-automation-display' +import { getExternalAutomationScheduleDisplay } from './external-automation-schedule-display' +import { getExternalAutomationActionDisabledMessage } from './external-automation-source-availability' +import { AUTOMATIONS_TABLE_GRID_CLASS } from './automations-table-layout' +import { + LIST_TABLE_ROW_CLASS, + LIST_TABLE_ROW_SELECTED_CLASS, + LIST_TABLE_STICKY_ROW_CELL_CLASS +} from '@/lib/list-table-layout' +import { isPortaledRowMenuClick, isRowActivationKey } from '@/lib/list-row-interaction' +import { getExternalAutomationLastRunSnapshot } from './automation-list-last-run' +import { AutomationListLastRunCell } from './AutomationListLastRunCell' +import { AutomationListStatusCell } from './AutomationListStatusCell' +import { translate } from '@/i18n/i18n' + +export type AutomationListExternalRowProps = { + entry: ExternalAutomationListEntry + selectedExternalKey: string | null | undefined + relativeNow: number + sshConnectionStates: ReadonlyMap> + externalActionKey: string | null + onSelect: (entryKey: string) => void + onRequestAction: ( + manager: ExternalAutomationManager, + job: ExternalAutomationJob, + action: ExternalAutomationAction, + scope: ExternalAutomationScope + ) => void + onEdit: ( + manager: ExternalAutomationManager, + job: ExternalAutomationJob, + scope: ExternalAutomationScope + ) => void +} + +export function AutomationListExternalRow({ + entry, + selectedExternalKey, + relativeNow, + sshConnectionStates, + externalActionKey, + onSelect, + onRequestAction, + onEdit +}: AutomationListExternalRowProps): React.JSX.Element { + const providerLabel = getExternalProviderLabel(entry.manager) + const targetKindLabel = getExternalTargetKindLabel(entry.manager) + const isSelected = selectedExternalKey === entry.key + const sshStatus = + entry.manager.target.type === 'ssh' + ? sshConnectionStates.get(entry.manager.target.connectionId)?.status + : undefined + const disabledMessage = getExternalAutomationActionDisabledMessage({ + manager: entry.manager, + providerLabel, + targetKindLabel, + sshStatus, + actionInProgress: externalActionKey !== null + }) + const actionDisabled = disabledMessage !== null + const scheduleLabel = getExternalAutomationScheduleDisplay(entry.manager, entry.job).label + const hostLabel = entry.manager.targetLabel || entry.manager.label || 'Local' + const projectLabel = entry.job.workdir ?? providerLabel + const nextRunLabel = entry.job.enabled + ? formatExternalDate(entry.job.nextRunAt, relativeNow) + : translate('auto.components.automations.AutomationsPage.paused', 'Paused') + const lastRunSnapshot = getExternalAutomationLastRunSnapshot(entry.job) + + return ( + + +
{ + // Why: Radix portals menus out of the row DOM, but React still + // bubbles those clicks here — ignore so menu actions don't open detail. + if (isPortaledRowMenuClick(event)) { + return + } + onSelect(entry.key) + }} + onKeyDown={(event) => { + if (!isRowActivationKey(event)) { + return + } + event.preventDefault() + onSelect(entry.key) + }} + className={cn( + AUTOMATIONS_TABLE_GRID_CLASS, + LIST_TABLE_ROW_CLASS, + isSelected && LIST_TABLE_ROW_SELECTED_CLASS + )} + > + + {entry.job.name} + + + {scheduleLabel} + + + {projectLabel} + + + {hostLabel} + + + {nextRunLabel} + + + + + {providerLabel} + + + + + + + onRequestAction(entry.manager, entry.job, 'run', entry.scope)} + > + + + {disabledMessage ?? + translate('auto.components.automations.AutomationsPage.2faecab10b', 'Run Now')} + + + {entry.manager.provider === 'hermes' ? ( + onEdit(entry.manager, entry.job, entry.scope)} + > + + {translate('auto.components.automations.AutomationsPage.f4612e3f78', 'Edit')} + + ) : null} + + onRequestAction( + entry.manager, + entry.job, + entry.job.enabled ? 'pause' : 'resume', + entry.scope + ) + } + > + {entry.job.enabled ? : } + {entry.job.enabled + ? translate('auto.components.automations.AutomationsPage.b457436d6a', 'Pause') + : translate('auto.components.automations.AutomationsPage.376631ef2b', 'Resume')} + + + onRequestAction(entry.manager, entry.job, 'delete', entry.scope)} + > + + {translate('auto.components.automations.AutomationsPage.15e0bfb13b', 'Delete')} + + + +
+ + + onRequestAction(entry.manager, entry.job, 'run', entry.scope)} + > + + + {disabledMessage ?? + translate('auto.components.automations.AutomationsPage.2faecab10b', 'Run Now')} + + + {entry.manager.provider === 'hermes' ? ( + onEdit(entry.manager, entry.job, entry.scope)} + > + + {translate('auto.components.automations.AutomationsPage.f4612e3f78', 'Edit')} + + ) : null} + + onRequestAction( + entry.manager, + entry.job, + entry.job.enabled ? 'pause' : 'resume', + entry.scope + ) + } + > + {entry.job.enabled ? : } + {entry.job.enabled + ? translate('auto.components.automations.AutomationsPage.b457436d6a', 'Pause') + : translate('auto.components.automations.AutomationsPage.376631ef2b', 'Resume')} + + + onRequestAction(entry.manager, entry.job, 'delete', entry.scope)} + > + + {translate('auto.components.automations.AutomationsPage.15e0bfb13b', 'Delete')} + + + + ) +} diff --git a/src/renderer/src/components/automations/AutomationListExternalRows.tsx b/src/renderer/src/components/automations/AutomationListExternalRows.tsx index 976a93b2433..3ed78fc2a69 100644 --- a/src/renderer/src/components/automations/AutomationListExternalRows.tsx +++ b/src/renderer/src/components/automations/AutomationListExternalRows.tsx @@ -1,285 +1,23 @@ import React from 'react' -import { MoreHorizontal, Pause, Pencil, Play, Trash2 } from 'lucide-react' -import { - ContextMenu, - ContextMenuContent, - ContextMenuItem, - ContextMenuSeparator, - ContextMenuTrigger -} from '@/components/ui/context-menu' -import { - DropdownMenu, - DropdownMenuContent, - DropdownMenuItem, - DropdownMenuSeparator, - DropdownMenuTrigger -} from '@/components/ui/dropdown-menu' -import { Button } from '@/components/ui/button' -import { cn } from '@/lib/utils' -import type { - ExternalAutomationAction, - ExternalAutomationJob, - ExternalAutomationManager -} from '../../../../shared/automations-types' -import type { SshConnectionState } from '../../../../shared/ssh-types' import type { ExternalAutomationListEntry } from './external-automation-list-entries' -import type { ExternalAutomationScope } from './external-automation-scope-client' import { - formatExternalDate, - getExternalProviderLabel, - getExternalTargetKindLabel -} from './external-automation-display' -import { getExternalAutomationScheduleDisplay } from './external-automation-schedule-display' -import { getExternalAutomationActionDisabledMessage } from './external-automation-source-availability' -import { AUTOMATIONS_TABLE_GRID_CLASS } from './automations-table-layout' -import { - LIST_TABLE_ROW_CLASS, - LIST_TABLE_ROW_SELECTED_CLASS, - LIST_TABLE_STICKY_ROW_CELL_CLASS -} from '@/lib/list-table-layout' -import { isPortaledRowMenuClick, isRowActivationKey } from '@/lib/list-row-interaction' -import { getExternalAutomationLastRunSnapshot } from './automation-list-last-run' -import { AutomationListLastRunCell } from './AutomationListLastRunCell' -import { AutomationListStatusCell } from './AutomationListStatusCell' -import { translate } from '@/i18n/i18n' + AutomationListExternalRow, + type AutomationListExternalRowProps +} from './AutomationListExternalRow' + +export type AutomationListExternalRowsProps = Omit & { + entries: readonly ExternalAutomationListEntry[] +} export function AutomationListExternalRows({ entries, - selectedExternalKey, - relativeNow, - sshConnectionStates, - externalActionKey, - onSelect, - onRequestAction, - onEdit -}: { - entries: readonly ExternalAutomationListEntry[] - selectedExternalKey: string | null | undefined - relativeNow: number - sshConnectionStates: ReadonlyMap> - externalActionKey: string | null - onSelect: (entryKey: string) => void - onRequestAction: ( - manager: ExternalAutomationManager, - job: ExternalAutomationJob, - action: ExternalAutomationAction, - scope: ExternalAutomationScope - ) => void - onEdit: ( - manager: ExternalAutomationManager, - job: ExternalAutomationJob, - scope: ExternalAutomationScope - ) => void -}): React.JSX.Element { + ...rowProps +}: AutomationListExternalRowsProps): React.JSX.Element { return ( <> - {entries.map((entry) => { - const providerLabel = getExternalProviderLabel(entry.manager) - const targetKindLabel = getExternalTargetKindLabel(entry.manager) - const isSelected = selectedExternalKey === entry.key - const sshStatus = - entry.manager.target.type === 'ssh' - ? sshConnectionStates.get(entry.manager.target.connectionId)?.status - : undefined - const disabledMessage = getExternalAutomationActionDisabledMessage({ - manager: entry.manager, - providerLabel, - targetKindLabel, - sshStatus, - actionInProgress: externalActionKey !== null - }) - const actionDisabled = disabledMessage !== null - const scheduleLabel = getExternalAutomationScheduleDisplay(entry.manager, entry.job).label - const hostLabel = entry.manager.targetLabel || entry.manager.label || 'Local' - const projectLabel = entry.job.workdir ?? providerLabel - const nextRunLabel = entry.job.enabled - ? formatExternalDate(entry.job.nextRunAt, relativeNow) - : translate('auto.components.automations.AutomationsPage.paused', 'Paused') - const lastRunSnapshot = getExternalAutomationLastRunSnapshot(entry.job) - - return ( - - -
{ - // Why: Radix portals menus out of the row DOM, but React still - // bubbles those clicks here — ignore so menu actions don't open detail. - if (isPortaledRowMenuClick(event)) { - return - } - onSelect(entry.key) - }} - onKeyDown={(event) => { - if (!isRowActivationKey(event)) { - return - } - event.preventDefault() - onSelect(entry.key) - }} - className={cn( - AUTOMATIONS_TABLE_GRID_CLASS, - LIST_TABLE_ROW_CLASS, - isSelected && LIST_TABLE_ROW_SELECTED_CLASS - )} - > - - {entry.job.name} - - - {scheduleLabel} - - - {projectLabel} - - - {hostLabel} - - - {nextRunLabel} - - - - - {providerLabel} - - - - - - - onRequestAction(entry.manager, entry.job, 'run', entry.scope)} - > - - - {disabledMessage ?? - translate( - 'auto.components.automations.AutomationsPage.2faecab10b', - 'Run Now' - )} - - - {entry.manager.provider === 'hermes' ? ( - onEdit(entry.manager, entry.job, entry.scope)} - > - - {translate( - 'auto.components.automations.AutomationsPage.f4612e3f78', - 'Edit' - )} - - ) : null} - - onRequestAction( - entry.manager, - entry.job, - entry.job.enabled ? 'pause' : 'resume', - entry.scope - ) - } - > - {entry.job.enabled ? ( - - ) : ( - - )} - {entry.job.enabled - ? translate( - 'auto.components.automations.AutomationsPage.b457436d6a', - 'Pause' - ) - : translate( - 'auto.components.automations.AutomationsPage.376631ef2b', - 'Resume' - )} - - - - onRequestAction(entry.manager, entry.job, 'delete', entry.scope) - } - > - - {translate( - 'auto.components.automations.AutomationsPage.15e0bfb13b', - 'Delete' - )} - - - -
- - - onRequestAction(entry.manager, entry.job, 'run', entry.scope)} - > - - - {disabledMessage ?? - translate('auto.components.automations.AutomationsPage.2faecab10b', 'Run Now')} - - - {entry.manager.provider === 'hermes' ? ( - onEdit(entry.manager, entry.job, entry.scope)} - > - - {translate('auto.components.automations.AutomationsPage.f4612e3f78', 'Edit')} - - ) : null} - - onRequestAction( - entry.manager, - entry.job, - entry.job.enabled ? 'pause' : 'resume', - entry.scope - ) - } - > - {entry.job.enabled ? : } - {entry.job.enabled - ? translate('auto.components.automations.AutomationsPage.b457436d6a', 'Pause') - : translate('auto.components.automations.AutomationsPage.376631ef2b', 'Resume')} - - - onRequestAction(entry.manager, entry.job, 'delete', entry.scope)} - > - - {translate('auto.components.automations.AutomationsPage.15e0bfb13b', 'Delete')} - - - - ) - })} + {entries.map((entry) => ( + + ))} ) } diff --git a/src/renderer/src/components/automations/AutomationListLocalRow.tsx b/src/renderer/src/components/automations/AutomationListLocalRow.tsx new file mode 100644 index 00000000000..a9c1a5bc8b6 --- /dev/null +++ b/src/renderer/src/components/automations/AutomationListLocalRow.tsx @@ -0,0 +1,391 @@ +import React from 'react' +import { MoreHorizontal, Pause, Pencil, Play, Trash2 } from 'lucide-react' +import { + ContextMenu, + ContextMenuContent, + ContextMenuItem, + ContextMenuSeparator, + ContextMenuTrigger +} from '@/components/ui/context-menu' +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuSeparator, + DropdownMenuTrigger +} from '@/components/ui/dropdown-menu' +import { Button } from '@/components/ui/button' +import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip' +import { AgentIcon } from '@/lib/agent-catalog' +import { cn } from '@/lib/utils' +import type { AutomationRun } from '../../../../shared/automations-types' +import { getAutomationRunRepoId } from '../../../../shared/automation-run-identity' +import { formatUiAutomationSchedule } from './automation-schedule-label' +import { + getExecutionHostLabel, + getLocalExecutionHostLabel, + getRepoExecutionHostId +} from '../../../../shared/execution-host' +import type { SshConnectionState } from '../../../../shared/ssh-types' +import type { ProjectHostSetup } from '../../../../shared/project-types' +import type { Repo } from '../../../../shared/repo-types' +import type { Worktree } from '../../../../shared/worktree/types' +import type { RuntimeStatus } from '../../../../shared/runtime-types' +import type { TaskSourceHostAvailability } from '../task-source-context-summary' +import type { AutomationRowAction } from './automation-captured-owner' +import type { AutomationHostTarget } from './automation-host-client' +import { + getAutomationRowLastRunSnapshot, + getLocalAutomationLastRunSnapshot +} from './automation-list-last-run' +import { AutomationListLastRunCell } from './AutomationListLastRunCell' +import { formatAutomationDateTimeWithRelative } from './automation-page-parts' +import { getAutomationTargetAvailability } from './automation-target-availability' +import { getAgentLabel } from './automation-draft-model' +import type { AutomationListRow } from './automation-list-row-identity' +import { + formatAutomationCost, + formatAutomationTokens, + type AutomationUsageSummary +} from './automation-usage-model' +import { AUTOMATIONS_TABLE_GRID_CLASS } from './automations-table-layout' +import { + LIST_TABLE_ROW_CLASS, + LIST_TABLE_ROW_SELECTED_CLASS, + LIST_TABLE_STICKY_ROW_CELL_CLASS +} from '@/lib/list-table-layout' +import { isPortaledRowMenuClick, isRowActivationKey } from '@/lib/list-row-interaction' +import { AutomationListStatusCell } from './AutomationListStatusCell' +import { translate } from '@/i18n/i18n' + +export type AutomationListLocalRowProps = { + row: AutomationListRow + selectedRowKey: string | null | undefined + isSelectedLocal: boolean + lastRunByAutomationId: ReadonlyMap + relativeNow: number + repoMap: ReadonlyMap + worktreeMap: ReadonlyMap + repoForRow?: (row: AutomationListRow) => Repo | undefined + worktreeForRow?: (row: AutomationListRow, repo: Repo | undefined) => Worktree | undefined + projectHostSetups: readonly ProjectHostSetup[] + sshConnectionStates: ReadonlyMap> + runtimeStatusByEnvironmentId: ReadonlyMap< + string, + { status: RuntimeStatus | null; checkedAt: number } + > + hostTargetFor: (row: AutomationListRow) => AutomationHostTarget | null + automationSourceHostAvailabilityByRowKey: ReadonlyMap + hostLabelById?: ReadonlyMap + isActionEnabled?: (row: AutomationListRow, action: AutomationRowAction) => boolean + onSelect: (rowKey: string) => void + onRunNow: (row: AutomationListRow) => void + onEdit: (row: AutomationListRow) => void + onToggle: (row: AutomationListRow) => void + onDelete: (row: AutomationListRow) => void +} + +const EMPTY_HOST_LABELS: ReadonlyMap = new Map() + +function automationUsageText(summary: AutomationUsageSummary | undefined): string { + if (!summary || summary.unavailableRuns > 0) { + return summary?.knownRuns + ? usageAmountText(summary) + : translate( + 'auto.components.automations.AutomationsPage.usageUnavailable', + 'Usage unavailable' + ) + } + return summary.knownRuns > 0 + ? usageAmountText(summary) + : translate('auto.components.automations.AutomationsPage.noRunUsageYet', 'No run usage yet') +} + +function usageAmountText(summary: AutomationUsageSummary): string { + return translate( + 'auto.components.automations.AutomationsPage.runUsageSummary', + '{{cost}} est. · {{tokens}} tokens', + { + cost: formatAutomationCost(summary.estimatedCostUsd), + tokens: formatAutomationTokens(summary.totalTokens) + } + ) +} + +export function AutomationListLocalRow({ + row, + selectedRowKey, + isSelectedLocal, + lastRunByAutomationId, + relativeNow, + repoMap, + worktreeMap, + repoForRow, + worktreeForRow, + projectHostSetups, + sshConnectionStates, + runtimeStatusByEnvironmentId, + hostTargetFor, + automationSourceHostAvailabilityByRowKey, + hostLabelById = EMPTY_HOST_LABELS, + isActionEnabled, + onSelect, + onRunNow, + onEdit, + onToggle, + onDelete +}: AutomationListLocalRowProps): React.JSX.Element { + const allows = (row: AutomationListRow, action: AutomationRowAction): boolean => + isActionEnabled?.(row, action) ?? true + const { automation } = row + const automationRepo = repoForRow?.(row) ?? repoMap.get(getAutomationRunRepoId(automation)) + const automationWorktree = automation.workspaceId + ? (worktreeForRow?.(row, automationRepo) ?? worktreeMap.get(automation.workspaceId)) + : null + const automationRunAvailability = getAutomationTargetAvailability({ + automation, + repo: automationRepo, + workspace: automationWorktree, + projectHostSetups, + sshConnectionStates, + runtimeStatusByEnvironmentId, + automationHostTarget: hostTargetFor(row), + sourceHostAvailability: automationSourceHostAvailabilityByRowKey.get(row.key) + }) + const projectLabel = + automationRepo?.displayName ?? + translate('auto.components.automations.AutomationsPage.13118faadf', 'Unknown project') + const scheduleLabel = formatUiAutomationSchedule(automation.rrule) + const nextRunLabel = automation.enabled + ? formatAutomationDateTimeWithRelative(automation.nextRunAt, relativeNow) + : translate('auto.components.automations.enablement.paused', 'Paused') + const isSelected = isSelectedLocal && selectedRowKey === row.key + const agentLabel = getAgentLabel(automation.agentId) + const hostId = + automation.runContext?.hostId ?? + (automationRepo ? getRepoExecutionHostId(automationRepo) : null) + const hostLabel = + row.hostLabel || + (hostId + ? (hostLabelById.get(hostId) ?? getExecutionHostLabel(hostId)) + : getLocalExecutionHostLabel()) + const agentTooltipLabel = `${agentLabel} · ${hostLabel} · ${automationUsageText(row.usageSummary ?? undefined)}` + const canRunNow = automationRunAvailability.canRunNow && allows(row, 'run') + const lastRun = lastRunByAutomationId.get(automation.id) + // Without a fetched run, the row's projected summary carries the newest + // retained run's status — the list never downloads run history for this. + const lastRunSnapshot = lastRun + ? getLocalAutomationLastRunSnapshot(automation, lastRun) + : getAutomationRowLastRunSnapshot(row) + + const actionItems = ( + <> + onRunNow(row)} + /> + } + label={translate('auto.components.automations.AutomationsPage.f4612e3f78', 'Edit')} + onSelect={() => onEdit(row)} + /> + : } + label={ + automation.enabled + ? translate('auto.components.automations.AutomationsPage.b457436d6a', 'Pause') + : translate('auto.components.automations.AutomationsPage.376631ef2b', 'Resume') + } + onSelect={() => onToggle(row)} + /> + + } + label={translate('auto.components.automations.AutomationsPage.15e0bfb13b', 'Delete')} + variant="destructive" + onSelect={() => onDelete(row)} + /> + + ) + + return ( + + +
{ + // Why: Radix portals menus out of the row DOM, but React still + // bubbles those clicks here — ignore so menu actions don't open detail. + if (isPortaledRowMenuClick(event)) { + return + } + onSelect(row.key) + }} + onKeyDown={(event) => { + if (!isRowActivationKey(event)) { + return + } + event.preventDefault() + onSelect(row.key) + }} + className={cn( + AUTOMATIONS_TABLE_GRID_CLASS, + LIST_TABLE_ROW_CLASS, + isSelected && LIST_TABLE_ROW_SELECTED_CLASS + )} + > + + {automation.name} + + + {scheduleLabel} + + + {projectLabel} + + + {hostLabel} + + + {nextRunLabel} + + + + + + + + + + + {agentTooltipLabel} + + + + + + + + { + if (canRunNow) { + onRunNow(row) + } + }} + > + + + {automationRunAvailability.canRunNow + ? translate('auto.components.automations.AutomationsPage.2faecab10b', 'Run Now') + : automationRunAvailability.message} + + + onEdit(row)}> + + {translate('auto.components.automations.AutomationsPage.f4612e3f78', 'Edit')} + + onToggle(row)}> + {automation.enabled ? ( + + ) : ( + + )} + {automation.enabled + ? translate('auto.components.automations.AutomationsPage.b457436d6a', 'Pause') + : translate('auto.components.automations.AutomationsPage.376631ef2b', 'Resume')} + + + onDelete(row)} + > + + {translate('auto.components.automations.AutomationsPage.15e0bfb13b', 'Delete')} + + + +
+ + {actionItems} + + ) +} + +function MenuRunItem({ + disabled, + label, + onSelect +}: { + disabled: boolean + label: string + onSelect: () => void +}): React.JSX.Element { + return ( + { + if (disabled) { + event.preventDefault() + return + } + onSelect() + }} + > + + {label} + + ) +} + +function MenuItem({ + disabled, + icon, + label, + onSelect, + variant +}: { + disabled?: boolean + icon: React.ReactNode + label: string + onSelect: () => void + variant?: 'destructive' +}): React.JSX.Element { + return ( + + {icon} + {label} + + ) +} + +function MenuSeparator(): React.JSX.Element { + return +} diff --git a/src/renderer/src/components/automations/AutomationListLocalRows.tsx b/src/renderer/src/components/automations/AutomationListLocalRows.tsx index 292eb545b4d..3fa02cc1884 100644 --- a/src/renderer/src/components/automations/AutomationListLocalRows.tsx +++ b/src/renderer/src/components/automations/AutomationListLocalRows.tsx @@ -1,414 +1,20 @@ import React from 'react' -import { MoreHorizontal, Pause, Pencil, Play, Trash2 } from 'lucide-react' -import { - ContextMenu, - ContextMenuContent, - ContextMenuItem, - ContextMenuSeparator, - ContextMenuTrigger -} from '@/components/ui/context-menu' -import { - DropdownMenu, - DropdownMenuContent, - DropdownMenuItem, - DropdownMenuSeparator, - DropdownMenuTrigger -} from '@/components/ui/dropdown-menu' -import { Button } from '@/components/ui/button' -import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip' -import { AgentIcon } from '@/lib/agent-catalog' -import { cn } from '@/lib/utils' -import type { AutomationRun } from '../../../../shared/automations-types' -import { getAutomationRunRepoId } from '../../../../shared/automation-run-identity' -import { formatUiAutomationSchedule } from './automation-schedule-label' -import { - getExecutionHostLabel, - getLocalExecutionHostLabel, - getRepoExecutionHostId -} from '../../../../shared/execution-host' -import type { SshConnectionState } from '../../../../shared/ssh-types' -import type { ProjectHostSetup } from '../../../../shared/project-types' -import type { Repo } from '../../../../shared/repo-types' -import type { Worktree } from '../../../../shared/worktree/types' -import type { RuntimeStatus } from '../../../../shared/runtime-types' -import type { TaskSourceHostAvailability } from '../task-source-context-summary' -import type { AutomationRowAction } from './automation-captured-owner' -import type { AutomationHostTarget } from './automation-host-client' -import { - getAutomationRowLastRunSnapshot, - getLocalAutomationLastRunSnapshot -} from './automation-list-last-run' -import { AutomationListLastRunCell } from './AutomationListLastRunCell' -import { formatAutomationDateTimeWithRelative } from './automation-page-parts' -import { getAutomationTargetAvailability } from './automation-target-availability' -import { getAgentLabel } from './automation-draft-model' import type { AutomationListRow } from './automation-list-row-identity' -import { - formatAutomationCost, - formatAutomationTokens, - type AutomationUsageSummary -} from './automation-usage-model' -import { AUTOMATIONS_TABLE_GRID_CLASS } from './automations-table-layout' -import { - LIST_TABLE_ROW_CLASS, - LIST_TABLE_ROW_SELECTED_CLASS, - LIST_TABLE_STICKY_ROW_CELL_CLASS -} from '@/lib/list-table-layout' -import { isPortaledRowMenuClick, isRowActivationKey } from '@/lib/list-row-interaction' -import { AutomationListStatusCell } from './AutomationListStatusCell' -import { translate } from '@/i18n/i18n' +import { AutomationListLocalRow, type AutomationListLocalRowProps } from './AutomationListLocalRow' -export type AutomationListLocalRowsProps = { +export type AutomationListLocalRowsProps = Omit & { rows: readonly AutomationListRow[] - selectedRowKey: string | null | undefined - isSelectedLocal: boolean - lastRunByAutomationId: ReadonlyMap - relativeNow: number - repoMap: ReadonlyMap - worktreeMap: ReadonlyMap - repoForRow?: (row: AutomationListRow) => Repo | undefined - worktreeForRow?: (row: AutomationListRow, repo: Repo | undefined) => Worktree | undefined - projectHostSetups: readonly ProjectHostSetup[] - sshConnectionStates: ReadonlyMap> - runtimeStatusByEnvironmentId: ReadonlyMap< - string, - { status: RuntimeStatus | null; checkedAt: number } - > - hostTargetFor: (row: AutomationListRow) => AutomationHostTarget | null - automationSourceHostAvailabilityByRowKey: ReadonlyMap - hostLabelById?: ReadonlyMap - isActionEnabled?: (row: AutomationListRow, action: AutomationRowAction) => boolean - onSelect: (rowKey: string) => void - onRunNow: (row: AutomationListRow) => void - onEdit: (row: AutomationListRow) => void - onToggle: (row: AutomationListRow) => void - onDelete: (row: AutomationListRow) => void -} - -const EMPTY_HOST_LABELS: ReadonlyMap = new Map() - -function automationUsageText(summary: AutomationUsageSummary | undefined): string { - if (!summary || summary.unavailableRuns > 0) { - return summary?.knownRuns - ? usageAmountText(summary) - : translate( - 'auto.components.automations.AutomationsPage.usageUnavailable', - 'Usage unavailable' - ) - } - return summary.knownRuns > 0 - ? usageAmountText(summary) - : translate('auto.components.automations.AutomationsPage.noRunUsageYet', 'No run usage yet') -} - -function usageAmountText(summary: AutomationUsageSummary): string { - return translate( - 'auto.components.automations.AutomationsPage.runUsageSummary', - '{{cost}} est. · {{tokens}} tokens', - { - cost: formatAutomationCost(summary.estimatedCostUsd), - tokens: formatAutomationTokens(summary.totalTokens) - } - ) } export function AutomationListLocalRows({ rows, - selectedRowKey, - isSelectedLocal, - lastRunByAutomationId, - relativeNow, - repoMap, - worktreeMap, - repoForRow, - worktreeForRow, - projectHostSetups, - sshConnectionStates, - runtimeStatusByEnvironmentId, - hostTargetFor, - automationSourceHostAvailabilityByRowKey, - hostLabelById = EMPTY_HOST_LABELS, - isActionEnabled, - onSelect, - onRunNow, - onEdit, - onToggle, - onDelete + ...rowProps }: AutomationListLocalRowsProps): React.JSX.Element { - const allows = (row: AutomationListRow, action: AutomationRowAction): boolean => - isActionEnabled?.(row, action) ?? true return ( <> - {rows.map((row) => { - const { automation } = row - const automationRepo = repoForRow?.(row) ?? repoMap.get(getAutomationRunRepoId(automation)) - const automationWorktree = automation.workspaceId - ? (worktreeForRow?.(row, automationRepo) ?? worktreeMap.get(automation.workspaceId)) - : null - const automationRunAvailability = getAutomationTargetAvailability({ - automation, - repo: automationRepo, - workspace: automationWorktree, - projectHostSetups, - sshConnectionStates, - runtimeStatusByEnvironmentId, - automationHostTarget: hostTargetFor(row), - sourceHostAvailability: automationSourceHostAvailabilityByRowKey.get(row.key) - }) - const projectLabel = - automationRepo?.displayName ?? - translate('auto.components.automations.AutomationsPage.13118faadf', 'Unknown project') - const scheduleLabel = formatUiAutomationSchedule(automation.rrule) - const nextRunLabel = automation.enabled - ? formatAutomationDateTimeWithRelative(automation.nextRunAt, relativeNow) - : translate('auto.components.automations.enablement.paused', 'Paused') - const isSelected = isSelectedLocal && selectedRowKey === row.key - const agentLabel = getAgentLabel(automation.agentId) - const hostId = - automation.runContext?.hostId ?? - (automationRepo ? getRepoExecutionHostId(automationRepo) : null) - const hostLabel = - row.hostLabel || - (hostId - ? (hostLabelById.get(hostId) ?? getExecutionHostLabel(hostId)) - : getLocalExecutionHostLabel()) - const agentTooltipLabel = `${agentLabel} · ${hostLabel} · ${automationUsageText(row.usageSummary ?? undefined)}` - const canRunNow = automationRunAvailability.canRunNow && allows(row, 'run') - const lastRun = lastRunByAutomationId.get(automation.id) - // Without a fetched run, the row's projected summary carries the newest - // retained run's status — the list never downloads run history for this. - const lastRunSnapshot = lastRun - ? getLocalAutomationLastRunSnapshot(automation, lastRun) - : getAutomationRowLastRunSnapshot(row) - - const actionItems = ( - <> - onRunNow(row)} - /> - } - label={translate('auto.components.automations.AutomationsPage.f4612e3f78', 'Edit')} - onSelect={() => onEdit(row)} - /> - : - } - label={ - automation.enabled - ? translate('auto.components.automations.AutomationsPage.b457436d6a', 'Pause') - : translate('auto.components.automations.AutomationsPage.376631ef2b', 'Resume') - } - onSelect={() => onToggle(row)} - /> - - } - label={translate('auto.components.automations.AutomationsPage.15e0bfb13b', 'Delete')} - variant="destructive" - onSelect={() => onDelete(row)} - /> - - ) - - return ( - - -
{ - // Why: Radix portals menus out of the row DOM, but React still - // bubbles those clicks here — ignore so menu actions don't open detail. - if (isPortaledRowMenuClick(event)) { - return - } - onSelect(row.key) - }} - onKeyDown={(event) => { - if (!isRowActivationKey(event)) { - return - } - event.preventDefault() - onSelect(row.key) - }} - className={cn( - AUTOMATIONS_TABLE_GRID_CLASS, - LIST_TABLE_ROW_CLASS, - isSelected && LIST_TABLE_ROW_SELECTED_CLASS - )} - > - - {automation.name} - - - {scheduleLabel} - - - {projectLabel} - - - {hostLabel} - - - {nextRunLabel} - - - - - - - - - - - {agentTooltipLabel} - - - - - - - - { - if (canRunNow) { - onRunNow(row) - } - }} - > - - - {automationRunAvailability.canRunNow - ? translate( - 'auto.components.automations.AutomationsPage.2faecab10b', - 'Run Now' - ) - : automationRunAvailability.message} - - - onEdit(row)}> - - {translate('auto.components.automations.AutomationsPage.f4612e3f78', 'Edit')} - - onToggle(row)} - > - {automation.enabled ? ( - - ) : ( - - )} - {automation.enabled - ? translate( - 'auto.components.automations.AutomationsPage.b457436d6a', - 'Pause' - ) - : translate( - 'auto.components.automations.AutomationsPage.376631ef2b', - 'Resume' - )} - - - onDelete(row)} - > - - {translate( - 'auto.components.automations.AutomationsPage.15e0bfb13b', - 'Delete' - )} - - - -
- - {actionItems} - - ) - })} + {rows.map((row) => ( + + ))} ) } - -function MenuRunItem({ - disabled, - label, - onSelect -}: { - disabled: boolean - label: string - onSelect: () => void -}): React.JSX.Element { - return ( - { - if (disabled) { - event.preventDefault() - return - } - onSelect() - }} - > - - {label} - - ) -} - -function MenuItem({ - disabled, - icon, - label, - onSelect, - variant -}: { - disabled?: boolean - icon: React.ReactNode - label: string - onSelect: () => void - variant?: 'destructive' -}): React.JSX.Element { - return ( - - {icon} - {label} - - ) -} - -function MenuSeparator(): React.JSX.Element { - return -} diff --git a/src/renderer/src/components/automations/AutomationListSortHeader.tsx b/src/renderer/src/components/automations/AutomationListSortHeader.tsx new file mode 100644 index 00000000000..2c24a344328 --- /dev/null +++ b/src/renderer/src/components/automations/AutomationListSortHeader.tsx @@ -0,0 +1,51 @@ +import React from 'react' +import { ArrowDown, ArrowUp } from 'lucide-react' +import { cn } from '@/lib/utils' +import { translate } from '@/i18n/i18n' +import type { AutomationListSort, AutomationListSortField } from './automation-list-view' + +export function AutomationListSortHeader({ + field, + label, + sort, + onSort +}: { + field: AutomationListSortField + label: string + sort: AutomationListSort | null + onSort: (field: AutomationListSortField) => void +}): React.JSX.Element { + const active = sort?.field === field + const direction = active ? sort.direction : null + // Why: one interpolated key per direction — word order and punctuation around + // the column name differ per language. + const sortedLabel = + direction === 'asc' + ? translate( + 'auto.components.automations.AutomationListSortHeader.sortedAscending', + '{{value0}}, sorted ascending', + { value0: label } + ) + : direction === 'desc' + ? translate( + 'auto.components.automations.AutomationListSortHeader.sortedDescending', + '{{value0}}, sorted descending', + { value0: label } + ) + : null + return ( + + ) +} diff --git a/src/renderer/src/components/automations/AutomationListTableHeader.test.tsx b/src/renderer/src/components/automations/AutomationListTableHeader.test.tsx index 5c5bbe8e568..638e96a23bc 100644 --- a/src/renderer/src/components/automations/AutomationListTableHeader.test.tsx +++ b/src/renderer/src/components/automations/AutomationListTableHeader.test.tsx @@ -1,7 +1,8 @@ // @vitest-environment happy-dom import { cleanup, render, screen } from '@testing-library/react' -import { afterEach, describe, expect, it } from 'vitest' +import { afterEach, describe, expect, it, vi } from 'vitest' +import userEvent from '@testing-library/user-event' import { AutomationListTableHeader } from './AutomationListTableHeader' import { LIST_TABLE_HEADER_CLASS, @@ -43,3 +44,45 @@ describe('AutomationListTableHeader', () => { expect(nameCell.className).toBe(LIST_TABLE_STICKY_HEADER_CELL_CLASS) }) }) + +describe('AutomationListTableHeader sorting', () => { + afterEach(cleanup) + + it('exposes only the orderable columns as buttons', () => { + render( {}} />) + + expect(screen.getAllByRole('button').map((button) => button.textContent)).toEqual([ + 'Name', + 'Last run' + ]) + }) + + it('reports the sorted column and direction in the accessible name', () => { + const { rerender } = render( + {}} /> + ) + expect(screen.getByRole('button', { name: 'Name, sorted ascending' })).toBeDefined() + expect(screen.getByRole('button', { name: 'Last run' })).toBeDefined() + + rerender( + {}} /> + ) + expect(screen.getByRole('button', { name: 'Last run, sorted descending' })).toBeDefined() + expect(screen.getByRole('button', { name: 'Name' })).toBeDefined() + }) + + it('requests a sort for the clicked column', async () => { + const onSort = vi.fn() + render() + + await userEvent.click(screen.getByRole('button', { name: 'Last run' })) + + expect(onSort.mock.calls).toEqual([['lastRun']]) + }) + + it('stays non-interactive when the list cannot be sorted', () => { + render() + + expect(screen.queryAllByRole('button')).toEqual([]) + }) +}) diff --git a/src/renderer/src/components/automations/AutomationListTableHeader.tsx b/src/renderer/src/components/automations/AutomationListTableHeader.tsx index dcbd107fcbc..605baf8a945 100644 --- a/src/renderer/src/components/automations/AutomationListTableHeader.tsx +++ b/src/renderer/src/components/automations/AutomationListTableHeader.tsx @@ -5,34 +5,85 @@ import { LIST_TABLE_HEADER_CLASS, LIST_TABLE_STICKY_HEADER_CELL_CLASS } from '@/lib/list-table-layout' +import { AutomationListSortHeader } from './AutomationListSortHeader' +import type { AutomationListSort, AutomationListSortField } from './automation-list-view' -export function AutomationListTableHeader(): React.JSX.Element { - const labels = [ - ['auto.components.automations.AutomationsPage.tableName', 'Name'], - ['auto.components.automations.AutomationDetail.18763ded26', 'Schedule'], - ['auto.components.automations.AutomationsPage.tableProject', 'Project'], - ['auto.components.automations.AutomationsPage.tableHost', 'Host'], - ['auto.components.automations.AutomationDetail.578ff46987', 'Next run'], - ['auto.components.automations.AutomationsPage.tableLastRun', 'Last run'], - ['auto.components.automations.AutomationsPage.tableStatus', 'Status'], - ['auto.components.automations.AutomationDetail.2df8970cd5', 'Agent'] - ] as const +type HeaderColumn = { + key: string + fallback: string + /** Absent for columns the list cannot order by. */ + sortField?: AutomationListSortField +} + +const COLUMNS: readonly HeaderColumn[] = [ + { + key: 'auto.components.automations.AutomationsPage.tableName', + fallback: 'Name', + sortField: 'name' + }, + { + key: 'auto.components.automations.AutomationDetail.18763ded26', + fallback: 'Schedule' + }, + { + key: 'auto.components.automations.AutomationsPage.tableProject', + fallback: 'Project' + }, + { + key: 'auto.components.automations.AutomationsPage.tableHost', + fallback: 'Host' + }, + { + key: 'auto.components.automations.AutomationDetail.578ff46987', + fallback: 'Next run' + }, + { + key: 'auto.components.automations.AutomationsPage.tableLastRun', + fallback: 'Last run', + sortField: 'lastRun' + }, + { + key: 'auto.components.automations.AutomationsPage.tableStatus', + fallback: 'Status' + }, + { + key: 'auto.components.automations.AutomationDetail.2df8970cd5', + fallback: 'Agent' + } +] + +export function AutomationListTableHeader({ + sort = null, + onSort +}: { + sort?: AutomationListSort | null + onSort?: (field: AutomationListSortField) => void +} = {}): React.JSX.Element { return (
- {labels.map(([key, fallback], index) => ( - - {translate(key, fallback)} - - ))} + {COLUMNS.map((column, index) => { + const label = translate(column.key, column.fallback) + const className = + index === 0 + ? LIST_TABLE_STICKY_HEADER_CELL_CLASS + : index === COLUMNS.length - 1 + ? 'text-center' + : undefined + return ( + + {column.sortField && onSort ? ( + + ) : ( + label + )} + + ) + })} {translate('auto.components.automations.AutomationsPage.tableActions', 'Actions')} diff --git a/src/renderer/src/components/automations/AutomationsListPanel.test.tsx b/src/renderer/src/components/automations/AutomationsListPanel.test.tsx index 2f772d2a7ed..f2362b83d0f 100644 --- a/src/renderer/src/components/automations/AutomationsListPanel.test.tsx +++ b/src/renderer/src/components/automations/AutomationsListPanel.test.tsx @@ -11,7 +11,12 @@ import { createRoot, type Root } from 'react-dom/client' import { afterEach, beforeEach, describe, expect, it } from 'vitest' import { TooltipProvider } from '@/components/ui/tooltip' import { AutomationsListPanel } from './AutomationsListPanel' -import { EMPTY_AUTOMATION_LIST_FILTER } from './automation-list-view' +import { + buildAutomationListViewItems, + EMPTY_AUTOMATION_LIST_FILTER, + type AutomationListSort, + type AutomationListSortField +} from './automation-list-view' import type { AutomationHostCatalogView } from './use-automation-host-catalog' import { makeAutomation, @@ -49,7 +54,13 @@ const HOST_CATALOG = { status: 'all', announceFallback: false }, - rows: { rows: [], automations: [], capturedOwners: new Map(), groups: [], answered: true }, + rows: { + rows: [], + automations: [], + capturedOwners: new Map(), + groups: [], + answered: true + }, loadCounts: { failedHostCount: 0, totalHostCount: 1 }, selectHost: () => undefined, recover: () => undefined, @@ -70,6 +81,8 @@ function renderPanel( selectExternalKey?: (key: string | null) => void externalEntries?: readonly ExternalAutomationListEntry[] setActivePaneTab?: (tab: AutomationPaneTab) => void + listSort?: AutomationListSort | null + onListSortChange?: (field: AutomationListSortField) => void } = {} ): void { const externalEntries = options.externalEntries ?? [] @@ -95,8 +108,12 @@ function renderPanel( externalManagersUncheckedNotice={uncheckedNotice} onSelectHost={() => undefined} onRecoverHost={() => undefined} - filteredRows={rows} - filteredExternalAutomationEntries={externalEntries} + sortedListItems={buildAutomationListViewItems({ + rows, + externalEntries + })} + listSort={options.listSort ?? null} + onListSortChange={options.onListSortChange ?? (() => undefined)} selectedRowKey={options.selectedRowKey ?? null} selectedExternalKey={options.selectedExternalKey ?? null} relativeNow={0} @@ -221,7 +238,11 @@ describe('AutomationsListPanel enter key navigation', () => { const input = searchField() expect(input).not.toBeNull() - const enter = new KeyboardEvent('keydown', { key: 'Enter', bubbles: true, cancelable: true }) + const enter = new KeyboardEvent('keydown', { + key: 'Enter', + bubbles: true, + cancelable: true + }) input?.dispatchEvent(enter) expect(enter.defaultPrevented).toBe(true) @@ -252,7 +273,11 @@ describe('AutomationsListPanel enter key navigation', () => { const input = searchField() expect(input).not.toBeNull() - const enter = new KeyboardEvent('keydown', { key: 'Enter', bubbles: true, cancelable: true }) + const enter = new KeyboardEvent('keydown', { + key: 'Enter', + bubbles: true, + cancelable: true + }) input?.dispatchEvent(enter) expect(enter.defaultPrevented).toBe(true) @@ -272,7 +297,11 @@ describe('AutomationsListPanel enter key navigation', () => { const input = searchField() expect(input).not.toBeNull() - const enter = new KeyboardEvent('keydown', { key: 'Enter', bubbles: true, cancelable: true }) + const enter = new KeyboardEvent('keydown', { + key: 'Enter', + bubbles: true, + cancelable: true + }) input?.dispatchEvent(enter) expect(detailOpened).toBe(false) diff --git a/src/renderer/src/components/automations/AutomationsListPanel.tsx b/src/renderer/src/components/automations/AutomationsListPanel.tsx index 5943096756a..783eee57da6 100644 --- a/src/renderer/src/components/automations/AutomationsListPanel.tsx +++ b/src/renderer/src/components/automations/AutomationsListPanel.tsx @@ -23,13 +23,19 @@ import { import type { AutomationListRow } from './automation-list-row-identity' import type { AutomationPaneTab } from './automation-page-state' import { AutomationListFilterPills } from './AutomationListFilterMenu' -import { isAutomationListFilterActive, type AutomationListFilter } from './automation-list-view' +import { + isAutomationListFilterActive, + type AutomationListFilter, + type AutomationListSort, + type AutomationListSortField, + type AutomationListViewItem +} from './automation-list-view' import { automationHostFilterStableKey } from '../../../../shared/automation-host-filter' import type { AutomationTemplate } from './automation-templates' import type { ExternalAutomationListEntry } from './external-automation-list-entries' import type { ExternalAutomationScope } from './external-automation-scope-client' -import { AutomationListLocalRows } from './AutomationListLocalRows' -import { AutomationListExternalRows } from './AutomationListExternalRows' +import { AutomationListLocalRow } from './AutomationListLocalRow' +import { AutomationListExternalRow } from './AutomationListExternalRow' import { AutomationHostFilterNotice, AutomationHostLoadSummary } from './AutomationHostFilterNotice' import { AutomationListEmptyView } from './AutomationListEmptyView' import { resolveAutomationListEmptyState } from './automation-list-empty-state' @@ -63,8 +69,10 @@ type AutomationsListPanelProps = { action: AutomationHostRecoveryAction, entry?: AutomationHostCatalogEntry | null ) => void - filteredRows: readonly AutomationListRow[] - filteredExternalAutomationEntries: readonly ExternalAutomationListEntry[] + /** Both collections as one list in render order; the sort spans local and external rows. */ + sortedListItems: readonly AutomationListViewItem[] + listSort: AutomationListSort | null + onListSortChange: (field: AutomationListSortField) => void selectedRowKey: string | null selectedExternalKey: string | null selectedExternal?: ExternalAutomationListEntry | null @@ -124,8 +132,9 @@ export function AutomationsListPanel(props: AutomationsListPanelProps): React.JS externalManagersUncheckedNotice, onSelectHost, onRecoverHost, - filteredRows, - filteredExternalAutomationEntries, + sortedListItems, + listSort, + onListSortChange, selectedRowKey, selectedExternalKey, relativeNow, @@ -161,18 +170,20 @@ export function AutomationsListPanel(props: AutomationsListPanelProps): React.JS // Hosts moved into the Filters menu, so its toolbar row is the focus fallback now. const toolbarRef = useRef(null) const pendingKeyboardScrollRef = useRef(false) - const rowKeys = React.useMemo(() => filteredRows.map((row) => row.key), [filteredRows]) - const visibleItems = React.useMemo( - () => [ - ...filteredRows.map((row) => ({ kind: 'local' as const, id: row.key })), - ...filteredExternalAutomationEntries.map((entry) => ({ - kind: 'external' as const, - id: entry.key - })) - ], - [filteredExternalAutomationEntries, filteredRows] + // Why: keyboard traversal and focus recovery read render order, which the sort owns. + const rowKeys = React.useMemo( + () => sortedListItems.filter((item) => item.kind === 'local').map((item) => item.id), + [sortedListItems] ) - useAutomationListFocusRecovery({ rowKeys, containerRef: listRef, fallbackRef: toolbarRef }) + const visibleItems = React.useMemo( + () => sortedListItems.map((item) => ({ kind: item.kind, id: item.id })), + [sortedListItems] + ) + useAutomationListFocusRecovery({ + rowKeys, + containerRef: listRef, + fallbackRef: toolbarRef + }) const handleSearchArrowNavigate = React.useCallback( (key: AutomationListArrowKey) => { const next = getAutomationListArrowNavigationTarget({ @@ -331,24 +342,30 @@ export function AutomationsListPanel(props: AutomationsListPanelProps): React.JS > {hasFilteredListItems ? (
- +
- - { - selectAutomationRow(null) - selectExternalKey(entryKey) - setActivePaneTab('overview') - onOpenDetail() - }} - onRequestAction={requestExternalAction} - onEdit={openEditExternalDialog} - /> + {sortedListItems.map((item) => + item.kind === 'local' ? ( + + ) : ( + { + selectAutomationRow(null) + selectExternalKey(entryKey) + setActivePaneTab('overview') + onOpenDetail() + }} + onRequestAction={requestExternalAction} + onEdit={openEditExternalDialog} + /> + ) + )}
) : ( diff --git a/src/renderer/src/components/automations/AutomationsPage.create-destination.test.tsx b/src/renderer/src/components/automations/AutomationsPage.create-destination.test.tsx index a0778029ef9..0635ecdb6ff 100644 --- a/src/renderer/src/components/automations/AutomationsPage.create-destination.test.tsx +++ b/src/renderer/src/components/automations/AutomationsPage.create-destination.test.tsx @@ -19,7 +19,6 @@ import { addRuntimeProject, api, installAutomationsPageHarness, - listedRow, mocks, renderPage, runtimeHost, @@ -30,6 +29,7 @@ import { scopedList, settleHostQueries } from './automations-page-test-harness' +import { listedRow } from './automations-page-listed-items' import { makeAutomation, REPO_ID, WORKSPACE_ID } from './automations-page-fixtures' import type { Repo } from '../../../../shared/repo-types' import type { ProjectHostSetup } from '../../../../shared/project-types' diff --git a/src/renderer/src/components/automations/AutomationsPage.cross-authority-actions.test.tsx b/src/renderer/src/components/automations/AutomationsPage.cross-authority-actions.test.tsx index 618c092b45a..8193502cb36 100644 --- a/src/renderer/src/components/automations/AutomationsPage.cross-authority-actions.test.tsx +++ b/src/renderer/src/components/automations/AutomationsPage.cross-authority-actions.test.tsx @@ -22,6 +22,7 @@ import { SELF_PRECONDITION, settleHostQueries } from './automations-page-test-harness' +import { listedRows } from './automations-page-listed-items' import { makeAutomation } from './automations-page-fixtures' installAutomationsPageHarness() @@ -36,9 +37,7 @@ async function collidingHosts(): Promise { } function selectDesktopRow(): string { - const row = mocks.listPanel?.filteredRows.find( - (candidate) => candidate.automation.name === 'Desktop nightly' - ) + const row = listedRows().find((candidate) => candidate.automation.name === 'Desktop nightly') expect(row).toBeDefined() return row?.key ?? '' } @@ -58,9 +57,7 @@ describe('AutomationsPage row actions under a colliding automation id', () => { await renderPage() await settleHostQueries() - const remote = mocks.listPanel?.filteredRows.find( - (candidate) => candidate.automation.name === 'Remote nightly' - ) + const remote = listedRows().find((candidate) => candidate.automation.name === 'Remote nightly') await act(async () => { mocks.listPanel?.selectAutomationRow(remote?.key ?? '') }) diff --git a/src/renderer/src/components/automations/AutomationsPage.external-scope.test.tsx b/src/renderer/src/components/automations/AutomationsPage.external-scope.test.tsx index cd966dcbcd7..b4ea3413cb3 100644 --- a/src/renderer/src/components/automations/AutomationsPage.external-scope.test.tsx +++ b/src/renderer/src/components/automations/AutomationsPage.external-scope.test.tsx @@ -20,6 +20,7 @@ import { RUNTIME_SELF_FILTER, settleHostQueries } from './automations-page-test-harness' +import { listedExternalEntries } from './automations-page-listed-items' import { makeExternalManager } from './automations-page-fixtures' installAutomationsPageHarness() @@ -117,7 +118,7 @@ describe('AutomationsPage external manager probes', () => { await renderPage() await settleHostQueries() - expect(mocks.listPanel?.filteredExternalAutomationEntries).toEqual([]) + expect(listedExternalEntries()).toEqual([]) }) it('drops the previous host rows when the selection moves, not when the new probe lands', async () => { @@ -127,7 +128,7 @@ describe('AutomationsPage external manager probes', () => { const { rerender } = await renderPage() await settleHostQueries() - expect(mocks.listPanel?.filteredExternalAutomationEntries.length).toBeGreaterThan(0) + expect(listedExternalEntries().length).toBeGreaterThan(0) // The new host never answers, so anything still listed belongs to the old one. api.automations.listExternalManagerForOwner.mockImplementation( @@ -137,7 +138,7 @@ describe('AutomationsPage external manager probes', () => { await rerender() await settleHostQueries() - expect(mocks.listPanel?.filteredExternalAutomationEntries).toEqual([]) + expect(listedExternalEntries()).toEqual([]) }) it('reports a host it could not check rather than showing it as clean', async () => { diff --git a/src/renderer/src/components/automations/AutomationsPage.notice-recovery.test.tsx b/src/renderer/src/components/automations/AutomationsPage.notice-recovery.test.tsx index d99cfb91dac..69c27640be1 100644 --- a/src/renderer/src/components/automations/AutomationsPage.notice-recovery.test.tsx +++ b/src/renderer/src/components/automations/AutomationsPage.notice-recovery.test.tsx @@ -15,7 +15,6 @@ import { addRuntimeProject, api, installAutomationsPageHarness, - listedRow, mocks, renderPage, runtimeHost, @@ -26,6 +25,7 @@ import { scopedList, settleHostQueries } from './automations-page-test-harness' +import { listedRow } from './automations-page-listed-items' import { makeAutomation } from './automations-page-fixtures' installAutomationsPageHarness() diff --git a/src/renderer/src/components/automations/AutomationsPage.refresh-selection.test.tsx b/src/renderer/src/components/automations/AutomationsPage.refresh-selection.test.tsx index 523cc50df7d..f33c4d09d6c 100644 --- a/src/renderer/src/components/automations/AutomationsPage.refresh-selection.test.tsx +++ b/src/renderer/src/components/automations/AutomationsPage.refresh-selection.test.tsx @@ -22,6 +22,7 @@ import { SELF_PRECONDITION, settleHostQueries } from './automations-page-test-harness' +import { listedRows } from './automations-page-listed-items' import { makeAutomation, makeRun } from './automations-page-fixtures' installAutomationsPageHarness() @@ -69,7 +70,7 @@ describe('AutomationsPage refresh', () => { await renderPage() - expect(mocks.listPanel?.filteredRows[0]?.usageSummary).toEqual(usageSummary) + expect(listedRows()[0]?.usageSummary).toEqual(usageSummary) }) it('does not re-list through the active runtime just because one is selected', async () => { @@ -231,9 +232,7 @@ describe('AutomationsPage multi-host selection', () => { ) ).toEqual(['Desktop nightly', 'Remote nightly']) - const remote = mocks.listPanel?.filteredRows.find( - (row) => row.automation.name === 'Remote nightly' - ) + const remote = listedRows().find((row) => row.automation.name === 'Remote nightly') await act(async () => { mocks.listPanel?.selectAutomationRow(remote?.key ?? '') }) diff --git a/src/renderer/src/components/automations/AutomationsPage.run-visibility.test.tsx b/src/renderer/src/components/automations/AutomationsPage.run-visibility.test.tsx index f7ad0be65a7..5e605a4487a 100644 --- a/src/renderer/src/components/automations/AutomationsPage.run-visibility.test.tsx +++ b/src/renderer/src/components/automations/AutomationsPage.run-visibility.test.tsx @@ -16,12 +16,12 @@ import type { Automation } from '../../../../shared/automations-types' import { api, installAutomationsPageHarness, - listedRow, mocks, renderPage, scopedList, settleHostQueries } from './automations-page-test-harness' +import { listedRow, listedRows } from './automations-page-listed-items' import { makeAutomation } from './automations-page-fixtures' installAutomationsPageHarness() @@ -42,7 +42,7 @@ function desktopStoreHolds(automations: Automation[]): void { /** The next-run column reads this; the mocked list panel renders only names. */ function listedNextRunAt(): number | null | undefined { - return mocks.listPanel?.filteredRows[0]?.automation.nextRunAt + return listedRows()[0]?.automation.nextRunAt } describe('AutomationsPage run visibility', () => { diff --git a/src/renderer/src/components/automations/AutomationsPage.test.tsx b/src/renderer/src/components/automations/AutomationsPage.test.tsx index 768d3250d80..a62a433a4d1 100644 --- a/src/renderer/src/components/automations/AutomationsPage.test.tsx +++ b/src/renderer/src/components/automations/AutomationsPage.test.tsx @@ -24,13 +24,13 @@ import { api, DESKTOP_SELF_OWNER, installAutomationsPageHarness, - listedRow, mocks, renderPage, rows, scopedList, SELF_PRECONDITION } from './automations-page-test-harness' +import { listedRow, listedExternalEntries } from './automations-page-listed-items' import { makeAutomation, makeExternalManager, @@ -147,7 +147,7 @@ describe('AutomationsPage list rendering', () => { api.automations.updateExternalForOwner.mockResolvedValue(undefined) await renderPage() - const entry = mocks.listPanel?.filteredExternalAutomationEntries[0] + const entry = listedExternalEntries()[0] if (!entry) { throw new Error('no external entry to edit') } @@ -177,7 +177,7 @@ describe('AutomationsPage list rendering', () => { api.automations.runExternalActionForOwner.mockResolvedValue(undefined) await renderPage() - const entry = mocks.listPanel?.filteredExternalAutomationEntries[0] + const entry = listedExternalEntries()[0] if (!entry) { throw new Error('no external entry to act on') } @@ -217,7 +217,7 @@ describe('AutomationsPage list rendering', () => { api.automations.listExternalRunsForOwner.mockResolvedValue({ runs: [], total: 0 }) const { container } = await renderPage() - const entry = mocks.listPanel?.filteredExternalAutomationEntries[0] + const entry = listedExternalEntries()[0] if (!entry) { throw new Error('no external entry to read runs for') } diff --git a/src/renderer/src/components/automations/AutomationsPageListPanel.tsx b/src/renderer/src/components/automations/AutomationsPageListPanel.tsx index 25c7ff7b88c..7adea56c608 100644 --- a/src/renderer/src/components/automations/AutomationsPageListPanel.tsx +++ b/src/renderer/src/components/automations/AutomationsPageListPanel.tsx @@ -1,6 +1,7 @@ import React from 'react' import type { AutomationsPageController } from './use-automations-page-controller' import { AutomationsListPanel } from './AutomationsListPanel' +import { nextAutomationListSort } from './automation-list-view' export function AutomationsPageListPanel({ controller, @@ -45,8 +46,6 @@ export function AutomationsPageListPanel({ hasListItems, hasFilteredListItems, isListSearchQueryTooLarge, - filteredRows, - filteredExternalAutomationEntries, selectedRow, selectedExternal, searchCounts @@ -79,8 +78,9 @@ export function AutomationsPageListPanel({ void pageRefresh.refresh() } }} - filteredRows={filteredRows} - filteredExternalAutomationEntries={filteredExternalAutomationEntries} + sortedListItems={list.sortedListItems} + listSort={local.listSort} + onListSortChange={(field) => local.setListSort(nextAutomationListSort(local.listSort, field))} selectedRowKey={selectedRow?.key ?? null} selectedExternalKey={local.selectedExternalKey} selectedExternal={selectedExternal} diff --git a/src/renderer/src/components/automations/automation-list-view-sort.test.ts b/src/renderer/src/components/automations/automation-list-view-sort.test.ts index d3dfbe63133..8fbef8a0590 100644 --- a/src/renderer/src/components/automations/automation-list-view-sort.test.ts +++ b/src/renderer/src/components/automations/automation-list-view-sort.test.ts @@ -5,32 +5,34 @@ import { type AutomationListSort, type AutomationListViewItem } from './automation-list-view' +import { unscopedAutomationListRows } from './automation-list-row-identity' import { makeAutomation } from './automations-page-fixtures' -const locale = vi.hoisted(() => ({ value: 'en' })) -vi.mock('@/i18n/i18n', () => ({ getIntlLocale: () => locale.value })) - afterEach(() => { vi.restoreAllMocks() - locale.value = 'en' }) -function rows(count = 512): AutomationListViewItem[] { +function items(count = 512): AutomationListViewItem[] { const names = ['Alpha', 'álpha', 'Ångström', 'Zebra', 'Örebro', 'I', 'ı', 'İ', 'job 10', 'job 2'] return buildAutomationListViewItems({ - automations: Array.from({ length: count }, (_, index) => - makeAutomation({ id: `job-${index}`, name: names[(index * 7) % names.length] }) + rows: unscopedAutomationListRows( + Array.from({ length: count }, (_, index) => + makeAutomation({ + id: `job-${index}`, + name: names[(index * 7) % names.length] + }) + ) ), - externalEntries: [], - runs: [] + externalEntries: [] }) } -function previousOrder(items: AutomationListViewItem[], sort: AutomationListSort) { +/** The pre-collator comparator, resolving options on every comparison. */ +function previousOrder(list: AutomationListViewItem[], sort: AutomationListSort, locale: string) { function compare(left: AutomationListViewItem, right: AutomationListViewItem) { const value = sort.field === 'name' - ? left.name.localeCompare(right.name, locale.value, { sensitivity: 'base' }) + ? left.name.localeCompare(right.name, locale, { sensitivity: 'base' }) : (left.lastRunAt ?? 0) - (right.lastRunAt ?? 0) return value !== 0 ? sort.direction === 'asc' @@ -38,37 +40,35 @@ function previousOrder(items: AutomationListViewItem[], sort: AutomationListSort : -value : left.id.localeCompare(right.id) } - return [...items].sort(compare) + return [...list].sort(compare) } describe('automation list collation', () => { it.each(['en', 'sv', 'tr', 'ja'])( 'preserves %s ordering, tie-breaks and input identity', - (language) => { - locale.value = language - const items = rows() - const original = [...items] + (locale) => { + const list = items() + const original = [...list] for (const direction of ['asc', 'desc'] as const) { const sort = { field: 'name', direction } as const - const expected = previousOrder(items, sort) - const result = sortAutomationListViewItems(items, sort) + const expected = previousOrder(list, sort, locale) + const result = sortAutomationListViewItems(list, sort, locale) expect(result).toEqual(expected) expect(result.every((row, index) => row === expected[index])).toBe(true) } - expect(items).toEqual(original) + expect(list).toEqual(original) } ) - it('resolves collation once per name sort and responds to locale changes', () => { - const items = rows() + it('resolves collation once per name sort and follows the locale it is given', () => { + const list = items() const OriginalCollator = Intl.Collator const construct = vi.spyOn(Intl, 'Collator').mockImplementation(function (locales, options) { return new OriginalCollator(locales, options) }) const compare = vi.spyOn(String.prototype, 'localeCompare') - sortAutomationListViewItems(items, { field: 'name', direction: 'asc' }) - locale.value = 'sv' - sortAutomationListViewItems(items, { field: 'name', direction: 'desc' }) + sortAutomationListViewItems(list, { field: 'name', direction: 'asc' }, 'en') + sortAutomationListViewItems(list, { field: 'name', direction: 'desc' }, 'sv') expect(construct.mock.calls).toEqual([ ['en', { sensitivity: 'base' }], ['sv', { sensitivity: 'base' }] @@ -76,16 +76,39 @@ describe('automation list collation', () => { expect(compare.mock.calls.filter((args) => args.length >= 3)).toHaveLength(0) }) + it('orders by row key, not the bare automation ID, so hosts cannot collapse', () => { + const duplicate = makeAutomation({ id: 'shared', name: 'Same' }) + const list = buildAutomationListViewItems({ + rows: [ + { + key: 'row|host-b|shared', + automation: duplicate, + hostLabel: 'b', + usageSummary: null + }, + { + key: 'row|host-a|shared', + automation: duplicate, + hostLabel: 'a', + usageSummary: null + } + ], + externalEntries: [] + }) + const sorted = sortAutomationListViewItems(list, { field: 'name', direction: 'asc' }, 'en') + expect(sorted.map((item) => item.id)).toEqual(['row|host-a|shared', 'row|host-b|shared']) + }) + it('does not construct collation for unsorted, time-sorted or trivial lists', () => { - const items = rows() + const list = items() const construct = vi.spyOn(Intl, 'Collator') - expect(sortAutomationListViewItems(items, null)).toEqual(items) + expect(sortAutomationListViewItems(list, null, 'en')).toEqual(list) const sort = { field: 'lastRun', direction: 'desc' } as const - expect(sortAutomationListViewItems(items, sort)).toEqual(previousOrder(items, sort)) - expect(sortAutomationListViewItems([], { field: 'name', direction: 'asc' })).toEqual([]) + expect(sortAutomationListViewItems(list, sort, 'en')).toEqual(previousOrder(list, sort, 'en')) + expect(sortAutomationListViewItems([], { field: 'name', direction: 'asc' }, 'en')).toEqual([]) expect( - sortAutomationListViewItems(items.slice(0, 1), { field: 'name', direction: 'asc' }) - ).toEqual(items.slice(0, 1)) + sortAutomationListViewItems(list.slice(0, 1), { field: 'name', direction: 'asc' }, 'en') + ).toEqual(list.slice(0, 1)) expect(construct).not.toHaveBeenCalled() }) }) diff --git a/src/renderer/src/components/automations/automation-list-view.test.ts b/src/renderer/src/components/automations/automation-list-view.test.ts index 188c94f3db7..169fbdd360a 100644 --- a/src/renderer/src/components/automations/automation-list-view.test.ts +++ b/src/renderer/src/components/automations/automation-list-view.test.ts @@ -1,7 +1,6 @@ import { describe, expect, it } from 'vitest' import type { Automation, - AutomationRun, AutomationRunStatus, ExternalAutomationJob, ExternalAutomationManager @@ -48,31 +47,6 @@ function makeAutomation(overrides: Partial = {}): Automation { } } -function makeRun(overrides: Partial = {}): AutomationRun { - return { - id: 'run-1', - automationId: 'automation-1', - title: 'Zebra job', - scheduledFor: 10, - status: 'completed', - trigger: 'scheduled', - workspaceId: 'worktree-1', - sessionKind: 'terminal', - chatSessionId: null, - terminalSessionId: null, - terminalPaneKey: null, - terminalPtyId: null, - outputSnapshot: null, - precheckResult: null, - usage: null, - error: null, - startedAt: 20, - dispatchedAt: 30, - createdAt: 10, - ...overrides - } -} - function makeExternalEntry( overrides: Partial = {} ): ExternalAutomationListEntry { @@ -118,22 +92,69 @@ function makeExternalEntry( } } +/** A catalog row with an optional projected last-run status, keyed like a real host row. */ +function makeCatalogRow( + id: string, + overrides: Partial = {}, + lastRunStatus?: AutomationRunStatus +): AutomationListRow { + return { + key: `row|host|${id}`, + automation: makeAutomation({ id, ...overrides }), + hostLabel: 'This computer', + usageSummary: lastRunStatus + ? { + knownRuns: 1, + unavailableRuns: 0, + inputTokens: 0, + outputTokens: 0, + cacheTokens: 0, + reasoningOutputTokens: 0, + totalTokens: 0, + estimatedCostUsd: null, + lastRunStatus, + lastRunAt: 111 + } + : null + } +} + +const rowKey = (id: string): string => `row|host|${id}` + describe('automation-list-view', () => { it('counts and detects active filters', () => { - expect(isAutomationListFilterActive({ status: 'all', lastRun: 'all', agentIds: [] })).toBe( - false - ) - expect(isAutomationListFilterActive({ status: 'paused', lastRun: 'all', agentIds: [] })).toBe( - true - ) - expect(countAutomationListFilters({ status: 'paused', lastRun: 'failed', agentIds: [] })).toBe( - 2 - ) + expect( + isAutomationListFilterActive({ + status: 'all', + lastRun: 'all', + agentIds: [] + }) + ).toBe(false) + expect( + isAutomationListFilterActive({ + status: 'paused', + lastRun: 'all', + agentIds: [] + }) + ).toBe(true) + expect( + countAutomationListFilters({ + status: 'paused', + lastRun: 'failed', + agentIds: [] + }) + ).toBe(2) }) it('toggles sort direction and defaults last run to newest first', () => { - expect(nextAutomationListSort(null, 'name')).toEqual({ field: 'name', direction: 'asc' }) - expect(nextAutomationListSort(null, 'lastRun')).toEqual({ field: 'lastRun', direction: 'desc' }) + expect(nextAutomationListSort(null, 'name')).toEqual({ + field: 'name', + direction: 'asc' + }) + expect(nextAutomationListSort(null, 'lastRun')).toEqual({ + field: 'lastRun', + direction: 'desc' + }) expect(nextAutomationListSort({ field: 'name', direction: 'asc' }, 'name')).toEqual({ field: 'name', direction: 'desc' @@ -146,82 +167,62 @@ describe('automation-list-view', () => { it('filters by enabled state and last-run outcome', () => { const items = applyAutomationListView({ - automations: [ - makeAutomation({ id: 'paused', name: 'Paused', enabled: false }), - makeAutomation({ id: 'ok', name: 'Healthy' }) + rows: [ + makeCatalogRow('paused', { name: 'Paused', enabled: false }, 'completed'), + makeCatalogRow('ok', { name: 'Healthy' }, 'dispatch_failed') ], externalEntries: [makeExternalEntry()], - runs: [ - makeRun({ automationId: 'paused', status: 'completed' }), - makeRun({ automationId: 'ok', status: 'dispatch_failed' }) - ], filter: { status: 'enabled', lastRun: 'failed', agentIds: [] }, - sort: null + sort: null, + locale: 'en' }) - expect(items.map((item) => item.id)).toEqual(['ok', 'manager-1:job-1']) + expect(items.map((item) => item.id)).toEqual([rowKey('ok'), 'manager-1:job-1']) }) it('filters local rows by multiple agents and leaves external rows out of agent scopes', () => { const items = applyAutomationListView({ - automations: [ - makeAutomation({ id: 'codex-job', agentId: 'codex' }), - makeAutomation({ id: 'claude-job', agentId: 'claude' }) + rows: [ + makeCatalogRow('codex-job', { agentId: 'codex' }), + makeCatalogRow('claude-job', { agentId: 'claude' }) ], externalEntries: [makeExternalEntry()], - runs: [], filter: { status: 'all', lastRun: 'all', agentIds: ['codex', 'claude'] }, - sort: null + sort: null, + locale: 'en' }) - expect(items.map((item) => item.id)).toEqual(['codex-job', 'claude-job']) + expect(items.map((item) => item.id)).toEqual([rowKey('codex-job'), rowKey('claude-job')]) }) it('counts an agent filter alongside status and last-run filters', () => { - expect(isAutomationListFilterActive({ status: 'all', lastRun: 'all', agentIds: [] })).toBe( - false - ) expect( - countAutomationListFilters({ status: 'paused', lastRun: 'failed', agentIds: ['codex'] }) + isAutomationListFilterActive({ + status: 'all', + lastRun: 'all', + agentIds: [] + }) + ).toBe(false) + expect( + countAutomationListFilters({ + status: 'paused', + lastRun: 'failed', + agentIds: ['codex'] + }) ).toBe(3) }) it('sorts by name across local and external rows', () => { const items = applyAutomationListView({ - automations: [makeAutomation({ name: 'Zebra job' })], + rows: [makeCatalogRow('zebra', { name: 'Zebra job' })], externalEntries: [makeExternalEntry({ name: 'Alpha digest' })], - runs: [], filter: { status: 'all', lastRun: 'all', agentIds: [] }, - sort: { field: 'name', direction: 'asc' } + sort: { field: 'name', direction: 'asc' }, + locale: 'en' }) expect(items.map((item) => item.name)).toEqual(['Alpha digest', 'Zebra job']) }) it('filters catalog rows by status, agent, and the projected last-run status', () => { - function makeCatalogRow( - id: string, - overrides: Partial, - lastRunStatus?: AutomationRunStatus - ): AutomationListRow { - return { - key: `row|host|${id}`, - automation: makeAutomation({ id, ...overrides }), - hostLabel: 'This computer', - usageSummary: lastRunStatus - ? { - knownRuns: 1, - unavailableRuns: 0, - inputTokens: 0, - outputTokens: 0, - cacheTokens: 0, - reasoningOutputTokens: 0, - totalTokens: 0, - estimatedCostUsd: null, - lastRunStatus, - lastRunAt: 111 - } - : null - } - } const rows = [ makeCatalogRow('paused-codex', { enabled: false, agentId: 'codex' }), makeCatalogRow('failed-claude', { agentId: 'claude' }, 'dispatch_failed'), @@ -229,9 +230,10 @@ describe('automation-list-view', () => { makeCatalogRow('never-codex', { agentId: 'codex' }) ] const ids = (filter: Partial) => - filterAutomationListRows(rows, { ...EMPTY_AUTOMATION_LIST_FILTER, ...filter }).map( - (row) => row.automation.id - ) + filterAutomationListRows(rows, { + ...EMPTY_AUTOMATION_LIST_FILTER, + ...filter + }).map((row) => row.automation.id) expect(ids({ status: 'paused' })).toEqual(['paused-codex']) expect(ids({ agentIds: ['claude'] })).toEqual(['failed-claude']) @@ -249,7 +251,10 @@ describe('automation-list-view', () => { catalogRef: targetId === null ? null - : { authority: { kind: 'desktop' }, selector: { kind: 'ssh', targetId } }, + : { + authority: { kind: 'desktop' }, + selector: { kind: 'ssh', targetId } + }, hostLabel: targetId ?? '', usageSummary: null }) @@ -257,9 +262,10 @@ describe('automation-list-view', () => { const keyOf = (row: AutomationListRow): string => row.catalogRef ? hostStableKey(row.catalogRef) : '' const ids = (hostStableKeys: readonly string[]) => - filterAutomationListRows(rows, { ...EMPTY_AUTOMATION_LIST_FILTER, hostStableKeys }).map( - (row) => row.automation.id - ) + filterAutomationListRows(rows, { + ...EMPTY_AUTOMATION_LIST_FILTER, + hostStableKeys + }).map((row) => row.automation.id) // Multi-select is any-of; a pre-catalog row names no host and is excluded. expect(ids([keyOf(rows[0]), keyOf(rows[1])])).toEqual(['on-a', 'on-b']) @@ -290,15 +296,22 @@ describe('automation-list-view', () => { it('sorts by last run newest first and keeps never-run rows last', () => { const items = applyAutomationListView({ - automations: [ - makeAutomation({ id: 'old', name: 'Old' }), - makeAutomation({ id: 'never', name: 'Never' }) + rows: [ + makeCatalogRow('old', { + name: 'Old', + lastRunAt: Date.parse('2026-08-11T09:00:00Z') + }), + makeCatalogRow('never', { name: 'Never' }) ], externalEntries: [makeExternalEntry({ lastRunAt: '2026-08-12T09:00:00Z' })], - runs: [makeRun({ automationId: 'old', dispatchedAt: Date.parse('2026-08-11T09:00:00Z') })], filter: { status: 'all', lastRun: 'all', agentIds: [] }, - sort: { field: 'lastRun', direction: 'desc' } + sort: { field: 'lastRun', direction: 'desc' }, + locale: 'en' }) - expect(items.map((item) => item.id)).toEqual(['manager-1:job-1', 'old', 'never']) + expect(items.map((item) => item.id)).toEqual([ + 'manager-1:job-1', + rowKey('old'), + rowKey('never') + ]) }) }) diff --git a/src/renderer/src/components/automations/automation-list-view.ts b/src/renderer/src/components/automations/automation-list-view.ts index 459d4b0f588..cedd394ed23 100644 --- a/src/renderer/src/components/automations/automation-list-view.ts +++ b/src/renderer/src/components/automations/automation-list-view.ts @@ -1,5 +1,3 @@ -import { getIntlLocale } from '@/i18n/i18n' -import type { Automation, AutomationRun } from '../../../../shared/automations-types' import type { TuiAgent } from '../../../../shared/tui-agent' import { hostStableKey } from '../../../../shared/automation-owner-key' import type { AutomationListRow } from './automation-list-row-identity' @@ -7,8 +5,6 @@ import type { ExternalAutomationListEntry } from './external-automation-list-ent import { getAutomationRowLastRunSnapshot, getExternalAutomationLastRunSnapshot, - getLocalAutomationLastRunSnapshot, - indexLatestAutomationRuns, type AutomationLastRunSnapshot } from './automation-list-last-run' @@ -22,6 +18,13 @@ export type AutomationListSort = { direction: AutomationListSortDirection } +/** + * A row and an external job flattened to what the shared list renders and sorts. + * + * `id` is the row's own key, never the bare automation ID: under All hosts two + * authorities can return the same ID, and the sort tie-break decides render + * order, so a bare ID would collapse them. See `automation-list-row-identity`. + */ export type AutomationListViewItem = | { kind: 'local' @@ -31,7 +34,7 @@ export type AutomationListViewItem = lastRunAt: number | null lastRun: AutomationLastRunSnapshot agentId: TuiAgent - automation: Automation + row: AutomationListRow } | { kind: 'external' @@ -117,30 +120,26 @@ function matchesLastRunFilter( return snapshot.tone === filter } +/** Flattens the two rendered collections into one sortable list, preserving row identity. */ export function buildAutomationListViewItems({ - automations, - externalEntries, - runs + rows, + externalEntries }: { - automations: readonly Automation[] + rows: readonly AutomationListRow[] externalEntries: readonly ExternalAutomationListEntry[] - runs: readonly AutomationRun[] }): AutomationListViewItem[] { - const lastRunByAutomationId = indexLatestAutomationRuns(runs) - const locals: AutomationListViewItem[] = automations.map((automation) => { - const lastRun = getLocalAutomationLastRunSnapshot( - automation, - lastRunByAutomationId.get(automation.id) - ) + const locals: AutomationListViewItem[] = rows.map((row) => { + // Why: the same snapshot the row cell renders, so the sort matches the column. + const lastRun = getAutomationRowLastRunSnapshot(row) return { kind: 'local', - id: automation.id, - name: automation.name, - enabled: automation.enabled, + id: row.key, + name: row.automation.name, + enabled: row.automation.enabled, lastRunAt: lastRun.at, lastRun, - agentId: automation.agentId, - automation + agentId: row.automation.agentId, + row } }) const externals: AutomationListViewItem[] = externalEntries.map((entry) => { @@ -217,34 +216,21 @@ export function filterExternalAutomationListEntries( ) } -export function filterAutomationListViewItems( - items: readonly AutomationListViewItem[], - filter: AutomationListFilter -): AutomationListViewItem[] { - if (!isAutomationListFilterActive(filter)) { - return [...items] - } - return items.filter( - (item) => - matchesStatusFilter(item.enabled, filter.status) && - matchesLastRunFilter(item.lastRun, filter.lastRun) && - (filter.agentIds.length === 0 || - (item.agentId !== null && filter.agentIds.includes(item.agentId))) - ) -} - +/** + * `locale` is a parameter, not a `getIntlLocale()` read, so callers memoizing this + * can declare it — a hidden read is invisible to a dependency array. + */ export function sortAutomationListViewItems( items: readonly AutomationListViewItem[], - sort: AutomationListSort | null + sort: AutomationListSort | null, + locale: string ): AutomationListViewItem[] { if (!sort || items.length < 2) { return [...items] } const next = [...items] const compareNames = - sort.field === 'name' - ? new Intl.Collator(getIntlLocale(), { sensitivity: 'base' }).compare - : null + sort.field === 'name' ? new Intl.Collator(locale, { sensitivity: 'base' }).compare : null next.sort((left, right) => { const compared = compareNames ? compareNames(left.name, right.name) @@ -257,24 +243,26 @@ export function sortAutomationListViewItems( return next } +/** The rendered list: filter each collection with its own rules, then sort as one. */ export function applyAutomationListView({ - automations, + rows, externalEntries, - runs, filter, - sort + sort, + locale }: { - automations: readonly Automation[] + rows: readonly AutomationListRow[] externalEntries: readonly ExternalAutomationListEntry[] - runs: readonly AutomationRun[] filter: AutomationListFilter sort: AutomationListSort | null + locale: string }): AutomationListViewItem[] { return sortAutomationListViewItems( - filterAutomationListViewItems( - buildAutomationListViewItems({ automations, externalEntries, runs }), - filter - ), - sort + buildAutomationListViewItems({ + rows: filterAutomationListRows(rows, filter), + externalEntries: filterExternalAutomationListEntries(externalEntries, filter) + }), + sort, + locale ) } diff --git a/src/renderer/src/components/automations/automations-page-listed-items.ts b/src/renderer/src/components/automations/automations-page-listed-items.ts new file mode 100644 index 00000000000..d87ae62b4a8 --- /dev/null +++ b/src/renderer/src/components/automations/automations-page-listed-items.ts @@ -0,0 +1,32 @@ +/** + * What the page actually listed, read back from the mocked list panel. + * + * Tests act through the same authority-qualified keys and render order the + * user's click carries, rather than synthesizing either. + */ + +import type { AutomationListRow } from './automation-list-row-identity' +import type { ExternalAutomationListEntry } from './external-automation-list-entries' +import { mocks } from './automations-page-test-harness' + +function listedItems() { + return mocks.listPanel?.sortedListItems ?? [] +} + +/** Local rows the page listed, in render order. */ +export function listedRows(): readonly AutomationListRow[] { + return listedItems().flatMap((item) => (item.kind === 'local' ? [item.row] : [])) +} + +/** External entries the page listed, in render order. */ +export function listedExternalEntries(): readonly ExternalAutomationListEntry[] { + return listedItems().flatMap((item) => (item.kind === 'external' ? [item.entry] : [])) +} + +export function listedRow(automationId: string): AutomationListRow { + const row = listedRows().find((entry) => entry.automation.id === automationId) + if (!row) { + throw new Error(`no listed row for ${automationId}`) + } + return row +} diff --git a/src/renderer/src/components/automations/automations-page-test-harness.tsx b/src/renderer/src/components/automations/automations-page-test-harness.tsx index d9fd1088bf7..e34ae0640bc 100644 --- a/src/renderer/src/components/automations/automations-page-test-harness.tsx +++ b/src/renderer/src/components/automations/automations-page-test-harness.tsx @@ -27,6 +27,7 @@ import type { AutomationHostCatalogView } from './use-automation-host-catalog' import type { AutomationCreateDestinationControl } from './use-automation-create-destination' import type { ExternalAutomationListEntry } from './external-automation-list-entries' import type { AutomationListRow } from './automation-list-row-identity' +import type { AutomationListViewItem } from './automation-list-view' import { resetAutomationCapabilityProbes } from './automation-scoped-list-client' import { addRuntimeProject as addRuntimeProjectFixture, @@ -39,7 +40,7 @@ export const RUNTIME_REPO_ID = RUNTIME_REPO_ID_FIXTURE export const RUNTIME_WORKSPACE_ID = RUNTIME_WORKSPACE_ID_FIXTURE export type ListPanelProps = { - filteredExternalAutomationEntries: ExternalAutomationListEntry[] + sortedListItems: readonly AutomationListViewItem[] selectedExternal: ExternalAutomationListEntry | null openEditExternalDialog: ( manager: ExternalAutomationListEntry['manager'], @@ -55,7 +56,6 @@ export type ListPanelProps = { ) => void hasListItems: boolean hasFilteredListItems: boolean - filteredRows: readonly AutomationListRow[] selectedRowKey: string | null selectedExternalKey: string | null hostCatalog: AutomationHostCatalogView @@ -211,30 +211,31 @@ vi.mock('./AutomationsListPanel', () => ({ return (
- ))} - {props.filteredExternalAutomationEntries.map((entry) => ( - - ))} + {props.sortedListItems.map((item) => + item.kind === 'local' ? ( + + ) : ( + + ) + )} {props.hasListItems ? null :
}
) @@ -407,18 +408,6 @@ export async function refreshOnFocus(): Promise { }) } -/** - * The row the page actually listed for an ID, so tests act through the same - * authority-qualified key the user's click carries rather than a synthesized one. - */ -export function listedRow(automationId: string): AutomationListRow { - const row = mocks.listPanel?.filteredRows.find((entry) => entry.automation.id === automationId) - if (!row) { - throw new Error(`no listed row for ${automationId}`) - } - return row -} - export function rows(container: HTMLElement, testId: string): string[] { return [...container.querySelectorAll(`[data-testid="${testId}"]`)].map( (node) => node.textContent ?? '' diff --git a/src/renderer/src/components/automations/use-automations-page-list-state.ts b/src/renderer/src/components/automations/use-automations-page-list-state.ts index 65cce90ffc2..b13c8a689ec 100644 --- a/src/renderer/src/components/automations/use-automations-page-list-state.ts +++ b/src/renderer/src/components/automations/use-automations-page-list-state.ts @@ -4,9 +4,12 @@ import { buildExternalAutomationListEntries } from './external-automation-list-e import { externalAutomationScopeEntries } from './external-automation-scope-gating' import { externalAutomationUncheckedNotice } from './external-automation-unchecked-hosts' import { + buildAutomationListViewItems, filterAutomationListRows, - filterExternalAutomationListEntries + filterExternalAutomationListEntries, + sortAutomationListViewItems } from './automation-list-view' +import { getIntlLocale } from '@/i18n/i18n' import { unscopedAutomationListRows } from './automation-list-row-identity' import { useAutomationHostCatalog } from './use-automation-host-catalog' import { useAutomationListSearch } from './use-automation-list-search' @@ -28,6 +31,7 @@ export function useAutomationsPageListState({ failedAuthorityKeys, listSearchQuery, listFilter, + listSort, selectedRowKey, selectedExternalKey, selectedAutomationRuns, @@ -129,6 +133,21 @@ export function useAutomationsPageListState({ () => externalAutomationUncheckedNotice(scopedExternal.failures, hostCatalog.entries), [hostCatalog.entries, scopedExternal.failures] ) + // Why: a language switch changes collation without touching rows, so the locale + // has to reach the memo as a value. + const sortLocale = getIntlLocale() + const sortedListItems = useMemo( + () => + sortAutomationListViewItems( + buildAutomationListViewItems({ + rows: filteredRows, + externalEntries: filteredExternalAutomationEntries + }), + listSort, + sortLocale + ), + [filteredExternalAutomationEntries, filteredRows, listSort, sortLocale] + ) return { hostCatalog, @@ -146,6 +165,7 @@ export function useAutomationsPageListState({ isListSearchQueryTooLarge, filteredRows, filteredExternalAutomationEntries, + sortedListItems, hasListItems, hasFilteredListItems, searchCounts, diff --git a/src/renderer/src/components/automations/use-automations-page-local-state.ts b/src/renderer/src/components/automations/use-automations-page-local-state.ts index e92f666cb22..7a097b144c3 100644 --- a/src/renderer/src/components/automations/use-automations-page-local-state.ts +++ b/src/renderer/src/components/automations/use-automations-page-local-state.ts @@ -12,7 +12,11 @@ import type { AutomationActionNotice } from './automation-row-action-dispatch' import type { AutomationHostCatalogEntry } from './automation-host-catalog-types' import type { AutomationCreateDestination } from './automation-create-destination' import type { AutomationListRow } from './automation-list-row-identity' -import { EMPTY_AUTOMATION_LIST_FILTER, type AutomationListFilter } from './automation-list-view' +import { + EMPTY_AUTOMATION_LIST_FILTER, + type AutomationListFilter, + type AutomationListSort +} from './automation-list-view' import type { AutomationPaneTab, AutomationRunPageOrigin, @@ -54,6 +58,7 @@ export function useAutomationsPageLocalState(store: AutomationsPageStoreState) { const [isSaving, setIsSaving] = useState(false) const [listSearchQuery, setListSearchQuery] = useState('') const [listFilter, setListFilter] = useState(EMPTY_AUTOMATION_LIST_FILTER) + const [listSort, setListSort] = useState(null) const [createOpen, setCreateOpen] = useState(false) const [createTarget, setCreateTarget] = useState('orca') const [editingAutomationId, setEditingAutomationId] = useState(null) @@ -178,6 +183,8 @@ export function useAutomationsPageLocalState(store: AutomationsPageStoreState) { setListSearchQuery, listFilter, setListFilter, + listSort, + setListSort, createOpen, setCreateOpen, createTarget, diff --git a/src/renderer/src/components/native-chat/NativeChatBackgroundTasksStatus.tsx b/src/renderer/src/components/native-chat/NativeChatBackgroundTasksStatus.tsx index 5163d5f467b..9f8f08e1efd 100644 --- a/src/renderer/src/components/native-chat/NativeChatBackgroundTasksStatus.tsx +++ b/src/renderer/src/components/native-chat/NativeChatBackgroundTasksStatus.tsx @@ -25,8 +25,10 @@ function backgroundTaskLabel(task: AgentSessionBackgroundTask): string { export function NativeChatBackgroundTasksStatus(props: { tasks: readonly AgentSessionBackgroundTask[] - stopping: boolean - onStop: () => void + supportsTaskStop: boolean + stoppingTaskIds: ReadonlySet + stoppingAll: boolean + onStop: (taskId?: string) => void }): React.JSX.Element { const [expanded, setExpanded] = useState(false) const taskListId = useId() @@ -36,7 +38,7 @@ export function NativeChatBackgroundTasksStatus(props: { className="shrink-0 bg-background px-3 pt-2 sm:px-4" >
-
+
-
{expanded ? (
- {props.tasks.map((task) => ( -
  • -
    • - ))} + {props.tasks.map((task) => { + const label = backgroundTaskLabel(task) + return ( +
  • +
    • + ) + })} ) : (

    @@ -100,6 +115,23 @@ export function NativeChatBackgroundTasksStatus(props: { )}

    )} + {!props.supportsTaskStop ? ( +
    0 ? 'mt-2 border-t border-border pt-2' : 'mt-2'}> + +
    + ) : null}
    ) : null}
    diff --git a/src/renderer/src/components/native-chat/NativeChatStructuredSession.test.tsx b/src/renderer/src/components/native-chat/NativeChatStructuredSession.test.tsx index afdf5ace7c5..2b4a9686aaf 100644 --- a/src/renderer/src/components/native-chat/NativeChatStructuredSession.test.tsx +++ b/src/renderer/src/components/native-chat/NativeChatStructuredSession.test.tsx @@ -29,8 +29,9 @@ const mocks = vi.hoisted(() => ({ pasteFromClipboard: vi.fn(), submissions: [] as unknown[], monitoringBackgroundTasks: false, + supportsBackgroundTaskStop: false, backgroundTasks: [] as AgentSessionBackgroundTask[], - stopBackgroundTasks: vi.fn() + stopBackgroundTask: vi.fn() })) vi.mock('@/runtime/structured-agent-session-client', () => ({ @@ -75,10 +76,11 @@ vi.mock('./use-structured-agent-session', async () => { retry: outbox.retry, isWorking: false, isMonitoringBackgroundTasks: mocks.monitoringBackgroundTasks, + supportsBackgroundTaskStop: mocks.supportsBackgroundTaskStop, backgroundTasks: mocks.backgroundTasks, turnId: null, cancel: vi.fn(), - stopBackgroundTasks: mocks.stopBackgroundTasks, + stopBackgroundTask: (taskId?: string) => mocks.stopBackgroundTask(props.sessionId, taskId), respond: mocks.respond, optionSnapshot: [ { @@ -166,7 +168,8 @@ describe('NativeChatStructuredSession', () => { mocks.pasteFromClipboard.mockReset() mocks.submissions = [] mocks.monitoringBackgroundTasks = false - mocks.stopBackgroundTasks.mockReset() + mocks.supportsBackgroundTaskStop = false + mocks.stopBackgroundTask.mockReset() mocks.backgroundTasks = [] }) @@ -225,11 +228,12 @@ describe('NativeChatStructuredSession', () => { it('places background monitoring above the usable composer and stops without an active turn', async () => { mocks.monitoringBackgroundTasks = true + mocks.supportsBackgroundTaskStop = true mocks.backgroundTasks = [ { id: 'task-command', kind: 'command', description: 'sleep 180' }, { id: 'task-agent', kind: 'agent' } ] - mocks.stopBackgroundTasks.mockResolvedValue({ cancelled: true }) + mocks.stopBackgroundTask.mockResolvedValue({ cancelled: true }) render( { expect(status.compareDocumentPosition(composer) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy() expect(mocks.composerProps?.isWorking).toBe(false) expect(screen.queryByRole('list', { name: 'Running background tasks' })).toBeNull() + expect(screen.queryByRole('button', { name: /^Stop / })).toBeNull() const disclosure = screen.getByRole('button', { name: 'Monitoring background tasks' }) expect(disclosure.getAttribute('aria-expanded')).toBe('false') @@ -260,8 +265,130 @@ describe('NativeChatStructuredSession', () => { expect(screen.getByText('sleep 180')).toBeTruthy() expect(screen.getByText('Background agent')).toBeTruthy() - fireEvent.click(screen.getByRole('button', { name: 'Stop' })) - await waitFor(() => expect(mocks.stopBackgroundTasks).toHaveBeenCalledOnce()) + fireEvent.click(screen.getByRole('button', { name: 'Stop sleep 180' })) + await waitFor(() => + expect(mocks.stopBackgroundTask).toHaveBeenCalledWith('session-background', 'task-command') + ) + }) + + it('tracks concurrent task stops independently and clears each pending result', async () => { + mocks.monitoringBackgroundTasks = true + mocks.supportsBackgroundTaskStop = true + mocks.backgroundTasks = [ + { id: 'task-one', kind: 'command', description: 'First task' }, + { id: 'task-two', kind: 'command', description: 'Second task' } + ] + let finishFirst!: (value: unknown) => void + let finishSecond!: (value: unknown) => void + mocks.stopBackgroundTask.mockImplementation( + (_sessionId: string, taskId: string) => + new Promise((resolve) => { + if (taskId === 'task-one') { + finishFirst = resolve + } else { + finishSecond = resolve + } + }) + ) + + render( + + ) + fireEvent.click(screen.getByRole('button', { name: 'Monitoring background tasks' })) + const firstStop = screen.getByRole('button', { name: 'Stop First task' }) + const secondStop = screen.getByRole('button', { name: 'Stop Second task' }) + + fireEvent.click(firstStop) + fireEvent.click(secondStop) + expect((firstStop as HTMLButtonElement).disabled).toBe(true) + expect((secondStop as HTMLButtonElement).disabled).toBe(true) + + await act(async () => finishFirst({ cancelled: true })) + await waitFor(() => expect((firstStop as HTMLButtonElement).disabled).toBe(false)) + expect((secondStop as HTMLButtonElement).disabled).toBe(true) + + await act(async () => finishSecond(null)) + await waitFor(() => expect((secondStop as HTMLButtonElement).disabled).toBe(false)) + }) + + it('keeps a stale session stop result from clearing the current session pending state', async () => { + mocks.monitoringBackgroundTasks = true + mocks.supportsBackgroundTaskStop = true + mocks.backgroundTasks = [{ id: 'task-one', kind: 'command', description: 'Shared task' }] + let finishOld!: (value: unknown) => void + let finishCurrent!: (value: unknown) => void + mocks.stopBackgroundTask.mockImplementation( + (sessionId: string) => + new Promise((resolve) => { + if (sessionId === 'session-old') { + finishOld = resolve + } else { + finishCurrent = resolve + } + }) + ) + const { rerender } = render( + + ) + fireEvent.click(screen.getByRole('button', { name: 'Monitoring background tasks' })) + fireEvent.click(screen.getByRole('button', { name: 'Stop Shared task' })) + + rerender( + + ) + const currentStop = screen.getByRole('button', { name: 'Stop Shared task' }) + expect((currentStop as HTMLButtonElement).disabled).toBe(false) + fireEvent.click(currentStop) + expect((currentStop as HTMLButtonElement).disabled).toBe(true) + + await act(async () => finishOld({ cancelled: true })) + expect((currentStop as HTMLButtonElement).disabled).toBe(true) + await act(async () => finishCurrent({ cancelled: true })) + await waitFor(() => expect((currentStop as HTMLButtonElement).disabled).toBe(false)) + }) + + it('keeps the expanded all-task stop fallback for a taskless older host', async () => { + mocks.monitoringBackgroundTasks = true + mocks.stopBackgroundTask.mockResolvedValue({ cancelled: true }) + + render( + + ) + expect(screen.queryByRole('button', { name: 'Stop background tasks' })).toBeNull() + fireEvent.click(screen.getByRole('button', { name: 'Monitoring background tasks' })) + expect(screen.getByText('Task details are unavailable for this session.')).toBeTruthy() + fireEvent.click(screen.getByRole('button', { name: 'Stop background tasks' })) + + await waitFor(() => + expect(mocks.stopBackgroundTask).toHaveBeenCalledWith( + 'session-taskless-background', + undefined + ) + ) }) it('routes a bare model command to the native option picker', async () => { diff --git a/src/renderer/src/components/native-chat/NativeChatStructuredSession.tsx b/src/renderer/src/components/native-chat/NativeChatStructuredSession.tsx index 9ac354f8a73..87adb0bda73 100644 --- a/src/renderer/src/components/native-chat/NativeChatStructuredSession.tsx +++ b/src/renderer/src/components/native-chat/NativeChatStructuredSession.tsx @@ -22,6 +22,14 @@ import { useStructuredNativeChatPaneCommands } from './use-structured-native-cha import type { NativeChatStructuredViewProps } from './native-chat-view-types' import { NativeChatBackgroundTasksStatus } from './NativeChatBackgroundTasksStatus' +type StoppingBackgroundTasks = { + sessionId: string + taskIds: ReadonlySet + all: boolean +} + +const NO_STOPPING_TASKS: ReadonlySet = new Set() + function encodeQuestionAnswer(questionId: string, answer: string): string { return `${encodeURIComponent(questionId)}:${encodeURIComponent(answer)}` } @@ -31,7 +39,8 @@ export function NativeChatStructuredSession( ): React.JSX.Element { const controller = useStructuredAgentSession(props) const [composerError, setComposerError] = useState(null) - const [stoppingBackgroundTasks, setStoppingBackgroundTasks] = useState(false) + const [stoppingBackgroundTasks, setStoppingBackgroundTasks] = + useState(null) const [optionPickerRequest, setOptionPickerRequest] = useState<{ id: string sequence: number @@ -83,6 +92,8 @@ export function NativeChatStructuredSession( const fileLinkContext = useNativeChatFileLinkContext(props.tabId) const imageRuntimeContext = useNativeChatImageRuntimeContext(props.tabId) const fileLinkClick = useNativeChatFileLinkClick(fileLinkContext) + const activeStoppingBackgroundTasks = + stoppingBackgroundTasks?.sessionId === props.sessionId ? stoppingBackgroundTasks : null const prompt = controller.prompts[0] ?? null const questionBody = prompt?.body.kind === 'question' ? prompt.body : null const questions = @@ -277,10 +288,39 @@ export function NativeChatStructuredSession( {controller.isMonitoringBackgroundTasks ? ( { - setStoppingBackgroundTasks(true) - void controller.stopBackgroundTasks().finally(() => setStoppingBackgroundTasks(false)) + supportsTaskStop={controller.supportsBackgroundTaskStop} + stoppingTaskIds={activeStoppingBackgroundTasks?.taskIds ?? NO_STOPPING_TASKS} + stoppingAll={activeStoppingBackgroundTasks?.all ?? false} + onStop={(taskId) => { + const targetSessionId = props.sessionId + setStoppingBackgroundTasks((current) => { + const taskIds = new Set( + current?.sessionId === targetSessionId ? current.taskIds : NO_STOPPING_TASKS + ) + if (taskId) { + taskIds.add(taskId) + } + return { + sessionId: targetSessionId, + taskIds, + all: taskId ? current?.sessionId === targetSessionId && current.all : true + } + }) + void controller.stopBackgroundTask(taskId).finally(() => { + setStoppingBackgroundTasks((current) => { + if (current?.sessionId !== targetSessionId) { + return current + } + const taskIds = new Set(current.taskIds) + if (taskId) { + taskIds.delete(taskId) + } + const all = taskId ? current.all : false + return taskIds.size === 0 && !all + ? null + : { sessionId: targetSessionId, taskIds, all } + }) + }) }} /> ) : null} diff --git a/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.test.tsx b/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.test.tsx index 8a61399d669..a8eb22ae7a5 100644 --- a/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.test.tsx +++ b/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.test.tsx @@ -2,17 +2,23 @@ import { act, cleanup, render, waitFor } from '@testing-library/react' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { + AgentSessionStatusEvent, + AgentSessionStatusSummary +} from '../../../../shared/agent-session-wire' import type { Tab } from '../../../../shared/tab-types' +import type * as RuntimeRpcClientModule from '@/runtime/runtime-rpc-client' const mocks = vi.hoisted(() => ({ - call: vi.fn(), removeAgentStatus: vi.fn(), setAgentStatus: vi.fn(), store: null as null | { getState: () => Record setState: (state: Record) => void }, - subscribe: vi.fn(), + subscribeStatus: vi.fn(), + subscribeTranscript: vi.fn(), + supportsCapability: vi.fn(), unsubscribe: vi.fn() })) @@ -73,17 +79,22 @@ vi.mock('@/lib/worktree-runtime-owner', () => ({ state.testRuntimeOwner ?? null })) +vi.mock('@/runtime/runtime-rpc-client', async (importOriginal) => ({ + ...(await importOriginal()), + runtimeEnvironmentSupportsCapability: mocks.supportsCapability +})) + vi.mock('@/runtime/structured-agent-session-client', () => ({ - callStructuredAgentSession: mocks.call, - subscribeStructuredAgentSession: mocks.subscribe + callStructuredAgentSession: vi.fn(), + subscribeStructuredAgentSession: mocks.subscribeTranscript, + subscribeStructuredAgentSessionStatus: mocks.subscribeStatus })) import { getStructuredAgentSessionTabs, StructuredAgentSessionStatusBridge } from './StructuredAgentSessionStatusBridge' -import { resetStructuredAgentSessionReadOwnersForTests } from './structured-agent-session-read-owner' -import { useStructuredAgentSessionRead } from './use-structured-agent-session-read' +import { resetStructuredAgentSessionStatusFeedsForTests } from '@/runtime/structured-agent-session-status-feed' const structuredTab = { id: 'structured-tab-1', @@ -100,60 +111,40 @@ const structuredTab = { agentSessionAgent: 'codex' } satisfies Tab -const userItem = { - itemId: 'item-1', - revision: 1, - sequence: 1, - observedAt: 1, - body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'hello' }] } -} as const +const providerSession = { key: 'session_id', id: '01a002e9-9a1c-7d42-a642-e481f64446f1' } as const -const historyResult = { - ok: true, - providerSession: { key: 'session_id', id: '01a002e9-9a1c-7d42-a642-e481f64446f1' }, - page: { +function summary(overrides: Partial = {}): AgentSessionStatusSummary { + return { sessionId: 'session-1', - epoch: 'epoch-1', - fence: 1, - direction: 'tail', - items: [userItem], - removedItemIds: [], - submissions: [], - window: { - oldest: { epoch: 'epoch-1', sequence: 1 }, - newest: { epoch: 'epoch-1', sequence: 1 }, - nextCursor: { epoch: 'epoch-1', sequence: 1 } - }, - liveCursor: { epoch: 'epoch-1', sequence: 1 }, - hasOlder: false, - hasNewer: false + workspaceId: 'wt-1', + agent: 'codex', + status: 'working', + latestPrompt: 'hello', + providerSession, + updatedAt: 1, + ...overrides } } -function ActiveSessionRead(): null { - useStructuredAgentSessionRead({ - sessionId: structuredTab.entityId, - target: { kind: 'local' }, - isVisible: true - }) - return null +function statuses(): Record[] { + return Object.values(mocks.store?.getState().agentStatusByPaneKey ?? {}) } -function ActiveComposition(): React.JSX.Element { - return ( - <> - - - - ) +/** The host side of the most recent status subscription. */ +function feed(index = 0): { target: unknown; emit: (event: AgentSessionStatusEvent) => void } { + const call = mocks.subscribeStatus.mock.calls[index] + if (!call) { + throw new Error('status feed not subscribed') + } + return { target: call[0], emit: call[1] as (event: AgentSessionStatusEvent) => void } } describe('StructuredAgentSessionStatusBridge', () => { beforeEach(() => { vi.clearAllMocks() - resetStructuredAgentSessionReadOwnersForTests() - mocks.call.mockResolvedValue(historyResult) - mocks.subscribe.mockResolvedValue({ unsubscribe: mocks.unsubscribe }) + resetStructuredAgentSessionStatusFeedsForTests() + mocks.subscribeStatus.mockResolvedValue({ unsubscribe: mocks.unsubscribe }) + mocks.supportsCapability.mockResolvedValue(true) mocks.store?.setState({ agentStatusByPaneKey: {}, testRuntimeOwner: null, @@ -163,7 +154,7 @@ describe('StructuredAgentSessionStatusBridge', () => { afterEach(() => { cleanup() - resetStructuredAgentSessionReadOwnersForTests() + resetStructuredAgentSessionStatusFeedsForTests() }) it('reuses the structured-tab projection for an unchanged tab map', () => { @@ -194,65 +185,111 @@ describe('StructuredAgentSessionStatusBridge', () => { ]) }) - it('keeps restored inactive tabs transport-neutral', async () => { + it('projects the host status feed without opening a transcript reader', async () => { render() - await act(() => Promise.resolve()) + await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) + expect(feed().target).toEqual({ kind: 'local' }) + expect(mocks.subscribeTranscript).not.toHaveBeenCalled() - expect(mocks.call).not.toHaveBeenCalled() - expect(mocks.subscribe).not.toHaveBeenCalled() + act(() => feed().emit({ type: 'snapshot', sessions: [summary()] })) + + expect(statuses()).toEqual([ + expect.objectContaining({ + state: 'working', + prompt: 'hello', + agentType: 'codex', + sessionBoundary: false, + tabId: structuredTab.id, + worktreeId: 'wt-1', + terminalTitle: 'Codex Chat', + terminalResumeEligible: false, + providerSession + }) + ]) + }) + + // Hiddenness is the host's side of this: see structured-agent-session-subscribers.test.ts, + // which drives an unsubscribed journal through the feed. Here the transport is a mock, so + // only the summary-to-store mapping is under test. + it('maps each host status onto the sidebar agent state', async () => { + render() + await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) + act(() => feed().emit({ type: 'snapshot', sessions: [summary()] })) + expect(statuses()).toEqual([expect.objectContaining({ state: 'working' })]) + + act(() => feed().emit({ type: 'status', session: summary({ status: 'idle', updatedAt: 2 }) })) + expect(statuses()).toEqual([expect.objectContaining({ state: 'done', sessionBoundary: true })]) + + act(() => + feed().emit({ type: 'status', session: summary({ status: 'attention', updatedAt: 3 }) }) + ) + expect(statuses()).toEqual([expect.objectContaining({ state: 'blocked' })]) + }) + + it('shows no status before a persisted turn', async () => { + render() + await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) + + act(() => feed().emit({ type: 'snapshot', sessions: [summary({ status: null })] })) expect(mocks.setAgentStatus).not.toHaveBeenCalled() + + act(() => feed().emit({ type: 'status', session: summary({ updatedAt: 2 }) })) + expect(statuses()).toEqual([expect.objectContaining({ state: 'working' })]) }) - it('shares the visible pane subscriber with status projection', async () => { - render() - - await waitFor(() => expect(mocks.setAgentStatus).toHaveBeenCalledOnce()) - expect(mocks.call).toHaveBeenCalledOnce() - expect(mocks.subscribe).toHaveBeenCalledOnce() - expect(mocks.setAgentStatus.mock.calls[0]?.[5]).toEqual({ - providerSession: historyResult.providerSession, - terminalResumeEligible: false - }) - }) - - it('keeps the status map reference stable for coalesced assistant deltas', async () => { - render() - await waitFor(() => expect(mocks.setAgentStatus).toHaveBeenCalledOnce()) + it('keeps the status map reference stable for repeated equal summaries', async () => { + render() + await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) + act(() => feed().emit({ type: 'snapshot', sessions: [summary()] })) const before = mocks.store?.getState().agentStatusByPaneKey - const onEvent = mocks.subscribe.mock.calls[0]?.[2] as (event: unknown) => void act(() => { - for (let sequence = 2; sequence <= 12; sequence += 1) { - onEvent({ - type: 'batch', - sessionId: 'session-1', - batch: { - cursor: { epoch: 'epoch-1', sequence }, - items: [ - { - itemId: 'assistant-1', - revision: sequence, - sequence, - observedAt: sequence, - body: { - kind: 'message', - role: 'assistant', - blocks: [{ type: 'text', text: `delta-${sequence}` }] - } - } - ], - removedItemIds: [], - submissions: [] - } - }) + for (let updatedAt = 2; updatedAt <= 12; updatedAt += 1) { + feed().emit({ type: 'status', session: summary({ updatedAt }) }) } }) - await act(async () => new Promise((resolve) => setTimeout(resolve, 60))) expect(mocks.setAgentStatus).toHaveBeenCalledOnce() expect(mocks.store?.getState().agentStatusByPaneKey).toBe(before) }) + it('drops the status and the feed when the last structured tab closes', async () => { + render() + await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) + act(() => feed().emit({ type: 'snapshot', sessions: [summary()] })) + expect(statuses()).toHaveLength(1) + + act(() => mocks.store?.setState({ unifiedTabsByWorktree: { 'wt-1': [] } })) + + expect(statuses()).toEqual([]) + await waitFor(() => expect(mocks.unsubscribe).toHaveBeenCalledOnce()) + }) + + it('reconnects after the host ends the stream', async () => { + vi.useFakeTimers() + try { + render() + await act(() => Promise.resolve()) + expect(mocks.subscribeStatus).toHaveBeenCalledOnce() + + act(() => feed().emit({ type: 'end' })) + await act(() => vi.advanceTimersByTimeAsync(300)) + + expect(mocks.unsubscribe).toHaveBeenCalledOnce() + expect(mocks.subscribeStatus).toHaveBeenCalledTimes(2) + } finally { + vi.useRealTimers() + } + }) + + it('keys the feed by the worktree runtime environment', async () => { + mocks.store?.setState({ testRuntimeOwner: 'env-1' }) + render() + await waitFor(() => expect(mocks.subscribeStatus).toHaveBeenCalledOnce()) + + expect(feed().target).toEqual({ kind: 'environment', environmentId: 'env-1' }) + }) + it('does not project an unknown provider as Codex', async () => { mocks.store?.setState({ unifiedTabsByWorktree: { @@ -262,8 +299,7 @@ describe('StructuredAgentSessionStatusBridge', () => { render() await act(() => Promise.resolve()) - expect(mocks.call).not.toHaveBeenCalled() - expect(mocks.subscribe).not.toHaveBeenCalled() + expect(mocks.subscribeStatus).not.toHaveBeenCalled() expect(mocks.setAgentStatus).not.toHaveBeenCalled() }) }) diff --git a/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.tsx b/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.tsx index 8409858dd17..d4cb74ab93b 100644 --- a/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.tsx +++ b/src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.tsx @@ -1,19 +1,14 @@ -import { useEffect, useMemo } from 'react' +import { useEffect, useMemo, useSyncExternalStore } from 'react' import { useShallow } from 'zustand/react/shallow' -import type { AgentProviderSessionMetadata } from '../../../../shared/agent-session-resume' import { agentProviderSessionsEqual } from '../../../../shared/agent-session-resume' -import { - hasPersistedStructuredAgentSessionTurn, - projectStructuredAgentSessionStatus, - structuredAgentSessionPaneKey -} from '../../../../shared/structured-agent-session-projection' -import type { StructuredAgentSessionState } from '../../../../shared/structured-agent-session-reducer' +import type { AgentSessionStatusSummary } from '../../../../shared/agent-session-wire' +import { structuredAgentSessionPaneKey } from '../../../../shared/structured-agent-session-projection' import type { Tab } from '../../../../shared/tab-types' import { isAgentSessionHandleProvider } from '../../../../shared/agent-session-provider-handle' import { getRuntimeEnvironmentIdForWorktree } from '@/lib/worktree-runtime-owner' import { useAppStore } from '@/store' -import { getActiveRuntimeTarget } from '@/runtime/runtime-rpc-client' -import { useStructuredAgentSessionReadObservation } from './use-structured-agent-session-read' +import { getActiveRuntimeTarget, type RuntimeClientTarget } from '@/runtime/runtime-rpc-client' +import { getStructuredAgentSessionStatusFeed } from '@/runtime/structured-agent-session-status-feed' type StructuredTab = Tab & { contentType: 'agent-session' } @@ -47,35 +42,40 @@ export function getStructuredAgentSessionTabs( return tabs } -function latestPrompt(state: StructuredAgentSessionState): string { - for (let index = state.items.length - 1; index >= 0; index -= 1) { - const body = state.items[index]?.body - if (body?.kind === 'message' && body.role === 'user') { - return body.blocks.flatMap((block) => (block.type === 'text' ? [block.text] : [])).join('\n') - } - } - return '' +/** The host's projected status for one session, live while the caller is mounted. */ +function useStructuredAgentSessionStatusSummary( + sessionId: string, + target: RuntimeClientTarget +): AgentSessionStatusSummary | null { + const feed = useMemo(() => getStructuredAgentSessionStatusFeed(target), [target]) + useEffect(() => feed.activate(), [feed]) + return useSyncExternalStore( + feed.subscribe, + () => feed.getSnapshot().get(sessionId) ?? null, + () => null + ) } -function projectStatus( - tab: StructuredTab, - state: StructuredAgentSessionState, - providerSession: AgentProviderSessionMetadata | undefined -): void { +function projectStatus(tab: StructuredTab, summary: AgentSessionStatusSummary | null): void { const paneKey = structuredAgentSessionPaneKey(tab.id, tab.entityId) const store = useAppStore.getState() - if (!hasPersistedStructuredAgentSessionTurn(state.items)) { + // No persisted turn yet (or nothing known): the row shows no agent status at all. + if (!summary?.status) { if (store.agentStatusByPaneKey?.[paneKey]) { store.removeAgentStatus(paneKey) } return } - const projection = projectStructuredAgentSessionStatus(state.items) const desired = { - state: projection === 'working' ? 'working' : projection === 'attention' ? 'blocked' : 'done', - prompt: latestPrompt(state), + state: + summary.status === 'working' + ? 'working' + : summary.status === 'attention' + ? 'blocked' + : 'done', + prompt: summary.latestPrompt, agentType: tab.agentSessionAgent, - sessionBoundary: projection === 'idle' + sessionBoundary: summary.status === 'idle' } as const const current = store.agentStatusByPaneKey?.[paneKey] if ( @@ -87,7 +87,11 @@ function projectStatus( current.tabId === tab.id && current.worktreeId === tab.worktreeId && current.terminalResumeEligible === false && - agentProviderSessionsEqual(tab.agentSessionAgent, current.providerSession, providerSession) + agentProviderSessionsEqual( + tab.agentSessionAgent, + current.providerSession, + summary.providerSession + ) ) { return } @@ -98,7 +102,7 @@ function projectStatus( undefined, { tabId: tab.id, worktreeId: tab.worktreeId }, { - ...(providerSession ? { providerSession } : {}), + ...(summary.providerSession ? { providerSession: summary.providerSession } : {}), terminalResumeEligible: false } ) @@ -112,13 +116,10 @@ function StructuredAgentSessionStatusProjection({ tab }: { tab: StructuredTab }) () => getActiveRuntimeTarget({ activeRuntimeEnvironmentId: environmentId }), [environmentId] ) - const { providerSession, state } = useStructuredAgentSessionReadObservation({ - sessionId: tab.entityId, - target - }) + const summary = useStructuredAgentSessionStatusSummary(tab.entityId, target) useEffect(() => { - projectStatus(tab, state, providerSession) - }, [providerSession, state, tab]) + projectStatus(tab, summary) + }, [summary, tab]) useEffect( () => () => useAppStore.getState().removeAgentStatus(structuredAgentSessionPaneKey(tab.id, tab.entityId)), diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-read.test.tsx b/src/renderer/src/components/native-chat/use-structured-agent-session-read.test.tsx index 2d320694c9c..9e1ce8067b7 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session-read.test.tsx +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-read.test.tsx @@ -19,10 +19,7 @@ vi.mock('@/runtime/structured-agent-session-client', () => ({ subscribeStructuredAgentSession: mocks.subscribe })) -import { - useStructuredAgentSessionRead, - useStructuredAgentSessionReadObservation -} from './use-structured-agent-session-read' +import { useStructuredAgentSessionRead } from './use-structured-agent-session-read' import { resetStructuredAgentSessionReadOwnersForTests } from './structured-agent-session-read-owner' const LOCAL_TARGET = { kind: 'local' } as const @@ -357,32 +354,6 @@ describe('useStructuredAgentSessionRead history window', () => { second.unmount() }) - it('shares one subscriber when pane and projection observe the same visible session', async () => { - const unsubscribe = vi.fn() - mocks.call.mockResolvedValue({ ok: true, page: page('tail', [], false) }) - mocks.subscribe.mockResolvedValue({ unsubscribe }) - - const view = renderHook(() => { - const pane = useStructuredAgentSessionRead({ - sessionId: 'session-shared', - target: LOCAL_TARGET, - isVisible: true - }) - const projection = useStructuredAgentSessionReadObservation({ - sessionId: 'session-shared', - target: LOCAL_TARGET - }) - return { pane, projection } - }) - - await waitFor(() => expect(mocks.subscribe).toHaveBeenCalledOnce()) - expect(mocks.call).toHaveBeenCalledOnce() - expect(view.result.current.pane.state).toBe(view.result.current.projection.state) - - view.unmount() - expect(unsubscribe).toHaveBeenCalledOnce() - }) - it('preserves cached state while switching away and refreshes once on re-entry', async () => { const unsubscribe = vi.fn() mocks.call.mockImplementation((_target, _method, params) => { diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-read.ts b/src/renderer/src/components/native-chat/use-structured-agent-session-read.ts index 894730f5e65..834d11d3fa1 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session-read.ts +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-read.ts @@ -20,13 +20,6 @@ function useReadOwnerSnapshot( return { owner, snapshot } } -export function useStructuredAgentSessionReadObservation(args: { - sessionId: string - target: RuntimeClientTarget -}): StructuredAgentSessionReadSnapshot { - return useReadOwnerSnapshot(args.sessionId, args.target).snapshot -} - export function useStructuredAgentSessionRead(args: { sessionId: string target: RuntimeClientTarget diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session.test.tsx b/src/renderer/src/components/native-chat/use-structured-agent-session.test.tsx index 2e611e4c726..9a42ccb6da6 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session.test.tsx +++ b/src/renderer/src/components/native-chat/use-structured-agent-session.test.tsx @@ -296,4 +296,40 @@ describe('useStructuredAgentSession options', () => { expect(result.current.error).toBeNull() }) + + it('includes one background task id in the cancel fingerprint and payload', async () => { + mocks.call.mockImplementation((_target, method) => + method === 'agentSession.options' + ? Promise.resolve(OPTIONS) + : Promise.resolve({ + ok: true, + value: { turnId: 'background-tasks', cancelled: true } + }) + ) + const { result } = renderHook(() => + useStructuredAgentSession({ + sessionId: 'session-1', + target: LOCAL_TARGET, + agent: 'claude', + isVisible: true + }) + ) + + await act(async () => { + await expect(result.current.stopBackgroundTask('task-2')).resolves.toMatchObject({ + cancelled: true + }) + }) + + const mutation = mocks.call.mock.calls.find(([, method]) => method === 'agentSession.cancel') + expect(mutation?.[2]).toMatchObject({ + envelope: { + sessionId: 'session-1', + expectedRuntimeFence: 3 + }, + turnId: 'background-tasks', + scope: 'background-tasks', + taskId: 'task-2' + }) + }) }) diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session.ts b/src/renderer/src/components/native-chat/use-structured-agent-session.ts index 3ae45bc3f43..8af9a36e0c4 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session.ts +++ b/src/renderer/src/components/native-chat/use-structured-agent-session.ts @@ -245,12 +245,14 @@ export function useStructuredAgentSession(args: { isWorking: turnId !== null, isMonitoringBackgroundTasks, backgroundTasks: state.backgroundTasks?.tasks ?? [], + supportsBackgroundTaskStop: state.backgroundTasks?.supportsTaskStop === true, turnId, cancel: (turnId: string) => mutate('agentSession.cancel', 'agentSession.cancel', { turnId }), - stopBackgroundTasks: () => + stopBackgroundTask: (taskId?: string) => mutate('agentSession.cancel', 'agentSession.cancel', { turnId: 'background-tasks', - scope: 'background-tasks' + scope: 'background-tasks', + ...(taskId ? { taskId } : {}) }), respond: (item: StructuredPromptItem, optionId: string) => mutate( diff --git a/src/renderer/src/components/terminal-pane/ipc-pty-connect-result.ts b/src/renderer/src/components/terminal-pane/ipc-pty-connect-result.ts index 3dda2f83fec..bcfc0d3f67c 100644 --- a/src/renderer/src/components/terminal-pane/ipc-pty-connect-result.ts +++ b/src/renderer/src/components/terminal-pane/ipc-pty-connect-result.ts @@ -16,6 +16,10 @@ export function projectIpcPtyConnectResult( snapshot: spawnResult.snapshot, snapshotCols: spawnResult.snapshotCols, snapshotRows: spawnResult.snapshotRows, + ...(spawnResult.snapshotSeq !== undefined ? { snapshotSeq: spawnResult.snapshotSeq } : {}), + ...(spawnResult.snapshotKittyKeyboardFlags !== undefined + ? { snapshotKittyKeyboardFlags: spawnResult.snapshotKittyKeyboardFlags } + : {}), ...(spawnResult.snapshotPrefixAnsi !== undefined ? { snapshotPrefixAnsi: spawnResult.snapshotPrefixAnsi } : {}), diff --git a/src/renderer/src/components/terminal-pane/pty-connection-deferred-reattach-live-output.test.ts b/src/renderer/src/components/terminal-pane/pty-connection-deferred-reattach-live-output.test.ts index fa110cdfe0e..ed38f6a31a4 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection-deferred-reattach-live-output.test.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection-deferred-reattach-live-output.test.ts @@ -231,6 +231,41 @@ describe('connectPanePty', () => { expect(transport.sendInput).not.toHaveBeenCalled() }) + it.each([ + { kind: 'covered backlog', snapshot: 'startup\r\n', seq: 9, count: 1 }, + { kind: 'legacy unsequenced snapshot', snapshot: 'startup\r\n', seq: undefined, count: 2 }, + { kind: 'blank snapshot', snapshot: '\x1b[2J', seq: 9, count: 1 } + ])( + 'preserves output while reconciling $kind on daemon adoption', + async ({ snapshot, seq, count }) => { + const { connectPanePty } = await import('./pty-connection') + const transport = createMockTransport('tab-pty') + transport.connect.mockImplementation( + async ({ sessionId, callbacks }: { sessionId?: string; callbacks?: ConnectCallbacks }) => { + callbacks?.onData?.('startup\r\n', { seq: 9, rawLength: 9 }) + callbacks?.onData?.('new output\r\n', { seq: 21, rawLength: 12 }) + return { id: sessionId, snapshot, snapshotSeq: seq } + } + ) + transportFactoryQueue.push(transport) + const pane = createPane(1) + const { writes, parseCallbacks } = captureCallbackTerminalWrites(pane) + const deps = createDeps({ + isVisibleRef: { current: true }, + restoredLeafId: LEAF_1, + restoredPtyIdByLeafId: { [LEAF_1]: 'tab-pty' } + }) + connectPanePty(pane as never, createManager(1) as never, deps as never) + await flushAsyncTicks(20) + for (let step = 0; step < 40; step += 1) { + parseCallbacks.shift()?.() + await flushAsyncTicks(2) + } + expect(writes.join('').match(/startup/g)).toHaveLength(count) + expect(writes.join('')).toContain('new output') + } + ) + it('drains live bytes after transport confirms an explicit reattach', async () => { const { connectPanePty } = await import('./pty-connection') const { deliverTerminalDataWithDeferredCredit } = diff --git a/src/renderer/src/components/terminal-pane/pty-connection/apply-reattach-payload.ts b/src/renderer/src/components/terminal-pane/pty-connection/apply-reattach-payload.ts index cb0806900af..ad64d458711 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection/apply-reattach-payload.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection/apply-reattach-payload.ts @@ -100,6 +100,13 @@ export function createReattachPayloadHandlers( // Why last: re-arm the dangling mid-escape after the reset (whose ESC would abort it) so the live continuation completes it (#7329). session.writeReplayData(ctx.connectResult.pendingEscapeTailAnsi) } + // The initial attach backlog can contain bytes already painted by this snapshot. + session.setRestoredSnapshotBaseline( + ctx.ptyId, + { seq: ctx.connectResult.snapshotSeq }, + restoredSnapshotPaintsPrintableContent({ data: daemonSnapshotReplay }) + ) + session.recordRendererOrderedSeq({ seq: ctx.connectResult.snapshotSeq }) session.sendFocusedReattachFocusInAfterReplay(ctx.ptyId, ctx.attemptGeneration) if (ctx.connectResult.coldRestore) { // Snapshot superseded the cold-restore payload; ack so the daemon doesn't redeliver it. diff --git a/src/renderer/src/components/terminal-pane/pty-transport-connect-spawn.test.ts b/src/renderer/src/components/terminal-pane/pty-transport-connect-spawn.test.ts index 66dca98b360..138059f372b 100644 --- a/src/renderer/src/components/terminal-pane/pty-transport-connect-spawn.test.ts +++ b/src/renderer/src/components/terminal-pane/pty-transport-connect-spawn.test.ts @@ -26,6 +26,24 @@ describe('createIpcPtyTransport', () => { restorePtySpecWindow(originalWindow) }) + it.each([0, 420])( + 'preserves snapshot sequence and keyboard proof %s across IPC reattach', + async (seq) => { + const { createIpcPtyTransport } = await import('./pty-transport') + vi.mocked(window.api.pty.spawn).mockResolvedValue({ + id: 'existing', + isReattach: true, + snapshot: 'ready', + snapshotSeq: seq, + snapshotKittyKeyboardFlags: 0 + }) + const transport = createIpcPtyTransport({}) + const result = await transport.connect({ url: '', sessionId: 'existing', callbacks: {} }) + expect(result).toMatchObject({ snapshotSeq: seq, snapshotKittyKeyboardFlags: 0 }) + transport.detach?.() + } + ) + it('leaves title tracking to the PTY data stream (no OpenCode IPC channel)', async () => { // Why: the OpenCode status IPC channel is gone (now the agent-hooks server), so the transport has no per-agent status callback. const { createIpcPtyTransport } = await import('./pty-transport') diff --git a/src/renderer/src/components/terminal-pane/terminal-pane-manager-options.ts b/src/renderer/src/components/terminal-pane/terminal-pane-manager-options.ts index 08eab4c0cf5..128b715a3c5 100644 --- a/src/renderer/src/components/terminal-pane/terminal-pane-manager-options.ts +++ b/src/renderer/src/components/terminal-pane/terminal-pane-manager-options.ts @@ -1,6 +1,7 @@ import type { IDisposable } from '@xterm/xterm' import type { PaneManagerOptions } from '@/lib/pane-manager/pane-manager' import { useAppStore } from '@/store' +import { resolveTerminalLigaturesEnabled } from '../../../../shared/terminal-ligatures' import { resolveTerminalFontWeights } from '../../../../shared/terminal-fonts' import { normalizeTerminalLineHeight } from '../../../../shared/terminal-line-height-settings' import { normalizeDesktopTerminalScrollbackRows } from '../../../../shared/terminal-scrollback-policy' @@ -102,6 +103,11 @@ export function createTerminalPaneManagerOptions( }, resolveExternalPaneDropTarget, onExternalPaneDrop, + terminalLigaturesEnabled: () => + resolveTerminalLigaturesEnabled( + settingsRef.current?.terminalLigatures, + settingsRef.current?.terminalFontFamily + ), terminalOptions: () => { const currentSettings = settingsRef.current const terminalFontWeights = resolveTerminalFontWeights( diff --git a/src/renderer/src/i18n/locales/en.json b/src/renderer/src/i18n/locales/en.json index 4f9919db5c4..b91c89cfa5a 100644 --- a/src/renderer/src/i18n/locales/en.json +++ b/src/renderer/src/i18n/locales/en.json @@ -16989,6 +16989,8 @@ "backgroundTasks": { "monitoring": "Monitoring background tasks", "stop": "Stop", + "stopTask": "Stop {{value0}}", + "stopAll": "Stop background tasks", "agent": "Background agent", "workflow": "Background workflow", "command": "Background command", diff --git a/src/renderer/src/lib/launch-structured-agent-session.test.ts b/src/renderer/src/lib/launch-structured-agent-session.test.ts index e9f65f3477b..d9a75ee2827 100644 --- a/src/renderer/src/lib/launch-structured-agent-session.test.ts +++ b/src/renderer/src/lib/launch-structured-agent-session.test.ts @@ -5,7 +5,8 @@ import { createStructuredAgentSessionLaunchIntent, isDefinitiveStructuredAgentSessionCreateError, launchStructuredAgentSession, - StructuredAgentSessionCreateRefusalError + StructuredAgentSessionCreateRefusalError, + StructuredAgentSessionCreateUnknownOutcomeError } from './launch-structured-agent-session' vi.mock('@/runtime/structured-agent-session-client', () => ({ @@ -256,11 +257,31 @@ describe('structured agent session launch', () => { createStructuredAgentSessionLaunchIntent('workspace-unknown', 'codex') ).catch((caught: unknown) => caught) + expect(error).toBeInstanceOf(StructuredAgentSessionCreateUnknownOutcomeError) expect(error).not.toBeInstanceOf(StructuredAgentSessionCreateRefusalError) expect(error).toMatchObject({ code: 'agent_session_operation_unknown' }) expect(isDefinitiveStructuredAgentSessionCreateError(error)).toBe(false) }) + /** The class is the verdict, so a refusal message that happens to end in a definitive token + * must not be re-read into one by the transport-error matcher. */ + it('keeps an unknown outcome unknown even when its message ends in a definitive token', async () => { + vi.mocked(callStructuredAgentSession).mockResolvedValue({ + ok: false, + refusal: { + code: 'agent_session_ownership_unknown', + message: 'Owner check failed: method_not_found' + } + }) + + const error = await launchStructuredAgentSession( + createStructuredAgentSessionLaunchIntent('workspace-unknown-token', 'codex') + ).catch((caught: unknown) => caught) + + expect(error).toBeInstanceOf(StructuredAgentSessionCreateUnknownOutcomeError) + expect(isDefinitiveStructuredAgentSessionCreateError(error)).toBe(false) + }) + it('preserves a definitive refusal code for the fallback path', async () => { vi.mocked(callStructuredAgentSession).mockResolvedValue({ ok: false, diff --git a/src/renderer/src/lib/launch-structured-agent-session.ts b/src/renderer/src/lib/launch-structured-agent-session.ts index 6c2d1694437..3d7f94a1135 100644 --- a/src/renderer/src/lib/launch-structured-agent-session.ts +++ b/src/renderer/src/lib/launch-structured-agent-session.ts @@ -33,24 +33,52 @@ export type StructuredAgentSessionLaunchIntent = { params: StructuredAgentSessionCreateParams } -export class StructuredAgentSessionCreateRefusalError extends Error { +class StructuredAgentSessionCreateError extends Error { constructor( message: string, - readonly code: string = 'structured_agent_session_unsupported' + /** The wire refusal code, or the RPC error code when the create never reached a handler. */ + readonly code: string ) { super(message) + } +} + +/** + * The host proved it created nothing, so a caller may open a legacy terminal instead. The class + * itself is the verdict: `launchStructuredAgentSession` is the only place that decides it, against + * the shared allowlist, so no consumer has to remember to re-check a code. + */ +export class StructuredAgentSessionCreateRefusalError extends StructuredAgentSessionCreateError { + constructor(message: string, code: string = 'structured_agent_session_unsupported') { + super(message, code) this.name = 'StructuredAgentSessionCreateRefusalError' } } +/** + * Refused with a code that does not prove the session is absent. A sibling opened here would sit + * beside a session the host may already hold, so this deliberately is NOT a refusal error: it flows + * down the same path as a lost reply, which replays the intent and reconciles. + */ +export class StructuredAgentSessionCreateUnknownOutcomeError extends StructuredAgentSessionCreateError { + constructor(message: string, code: string) { + super(message, code) + this.name = 'StructuredAgentSessionCreateUnknownOutcomeError' + } +} + const DEFINITIVE_CREATE_FAILURE_CODES = [ 'structured_agent_session_unsupported', 'method_not_found' ] as const function definitiveStructuredAgentSessionCreateErrorCode(error: unknown): string | null { - if (error instanceof StructuredAgentSessionCreateRefusalError) { - return isDefinitiveAgentSessionCreateRefusal(error.code) ? error.code : null + if (error instanceof StructuredAgentSessionCreateError) { + // Our own classes already carry the verdict; message sniffing below could only invert it. + return error instanceof StructuredAgentSessionCreateRefusalError && + isDefinitiveAgentSessionCreateRefusal(error.code) + ? error.code + : null } for (const code of DEFINITIVE_CREATE_FAILURE_CODES) { if (hasRuntimeRpcErrorCode(error, code)) { @@ -200,15 +228,13 @@ export async function launchStructuredAgentSession( throw error } if (!result.ok) { - const error = new StructuredAgentSessionCreateRefusalError( - result.refusal.message, - result.refusal.code - ) - if (isDefinitiveStructuredAgentSessionCreateError(error)) { - abandonStructuredAgentSessionLaunchIntent(intent) - throw error + const { code, message } = result.refusal + if (!isDefinitiveAgentSessionCreateRefusal(code)) { + // Keep the focus intent: the session may exist, and recovery still has to adopt it. + throw new StructuredAgentSessionCreateUnknownOutcomeError(message, code) } - throw Object.assign(new Error(error.message), { code: error.code }) + abandonStructuredAgentSessionLaunchIntent(intent) + throw new StructuredAgentSessionCreateRefusalError(message, code) } return { sessionId: result.value.sessionId, fence: result.value.fence } } diff --git a/src/renderer/src/lib/pane-manager/pane-lifecycle.test.ts b/src/renderer/src/lib/pane-manager/pane-lifecycle.test.ts index 3a174253d9c..e84638c33ea 100644 --- a/src/renderer/src/lib/pane-manager/pane-lifecycle.test.ts +++ b/src/renderer/src/lib/pane-manager/pane-lifecycle.test.ts @@ -7,7 +7,7 @@ import { primeTerminalWebglAddon, resetTerminalWebglSuggestion } from './pane-webgl-renderer' -import { attachLigatures, disposePane, openTerminal } from './pane-lifecycle' +import { attachLigatures, disposePane, openTerminal, setLigaturesEnabled } from './pane-lifecycle' import { ensureArabicShapingJoinerForText } from './terminal-arabic-shaping-joiner' import { buildDefaultTerminalOptions, @@ -531,6 +531,7 @@ describe('openTerminal — addon and provider wiring', () => { }), attachCustomWheelEventHandler: vi.fn(), onWriteParsed: vi.fn(() => ({ dispose: vi.fn() })), + refresh: vi.fn(), write: vi.fn(() => { events.push('write') }), @@ -588,6 +589,31 @@ describe('openTerminal — addon and provider wiring', () => { // unicode v11 is activated (still on default v6 width tables), wide chars // lay out as single cells. The bug surfaces as the broken `?`-style glyphs // users saw on worktree switch. + it('builds one initial WebGL atlas with ligatures and still rebuilds on a live toggle', async () => { + await primeTerminalWebglAddon() + resetTerminalWebglSuggestion() + vi.mocked(WebglAddon).mockClear() + webglMock.dispose.mockClear() + vi.stubGlobal('navigator', { platform: 'MacIntel', userAgent: 'Macintosh' }) + const { pane } = createOpenTerminalHarness() + pane.terminalGpuAcceleration = 'auto' + pane.gpuRenderingEnabled = true + + openTerminal(pane, true) + expect(pane.ligaturesAddon).not.toBeNull() + expect(pane.webglAddon).not.toBeNull() + const addons = vi.mocked(pane.terminal.loadAddon).mock.calls.map(([addon]) => addon) + expect(addons.indexOf(pane.ligaturesAddon!)).toBeLessThan(addons.indexOf(pane.webglAddon!)) + setLigaturesEnabled(pane, true) + expect(WebglAddon).toHaveBeenCalledTimes(1) + expect(webglMock.dispose).not.toHaveBeenCalled() + + setLigaturesEnabled(pane, false) + expect(WebglAddon).toHaveBeenCalledTimes(2) + expect(webglMock.dispose).toHaveBeenCalledTimes(1) + expect(pane.ligaturesAddon).toBeNull() + }) + it('activates unicode 11 before any caller-driven write would be possible', () => { const { pane, events } = createOpenTerminalHarness() diff --git a/src/renderer/src/lib/pane-manager/pane-lifecycle.ts b/src/renderer/src/lib/pane-manager/pane-lifecycle.ts index 3c89a6ec723..cf4b50783d1 100644 --- a/src/renderer/src/lib/pane-manager/pane-lifecycle.ts +++ b/src/renderer/src/lib/pane-manager/pane-lifecycle.ts @@ -28,7 +28,7 @@ import { installTerminalImeCandidateAnchor } from './terminal-ime-candidate-anch export { createPaneDOM } from './pane-dom-creation' /** Open terminal into its container and load addons. Must be called after the container is in the DOM. */ -export function openTerminal(pane: ManagedPaneInternal): void { +export function openTerminal(pane: ManagedPaneInternal, ligaturesEnabled = false): void { const { terminal, container, @@ -100,6 +100,10 @@ export function openTerminal(pane: ManagedPaneInternal): void { pane.focusClassSyncCleanup = attachDomRendererFocusClassSync(terminal.element) + // Configure the first atlas with ligatures instead of immediately rebuilding it. + if (ligaturesEnabled) { + attachLigatures(pane) + } if (pane.gpuRenderingEnabled) { attachWebgl(pane) } diff --git a/src/renderer/src/lib/pane-manager/pane-manager-pane-creation.ts b/src/renderer/src/lib/pane-manager/pane-manager-pane-creation.ts index 483f49a1f55..c4c3a28c4ca 100644 --- a/src/renderer/src/lib/pane-manager/pane-manager-pane-creation.ts +++ b/src/renderer/src/lib/pane-manager/pane-manager-pane-creation.ts @@ -18,7 +18,7 @@ export function createInitialManagedPane( overflow: 'hidden' }) host.root.appendChild(pane.container) - openTerminal(pane) + openTerminal(pane, host.options.terminalLigaturesEnabled?.()) host.setActivePaneId(pane.id) applyPaneOpacity(host.panes.values(), host.getActivePaneId(), host.getStyleOptions()) diff --git a/src/renderer/src/lib/pane-manager/pane-manager-types.ts b/src/renderer/src/lib/pane-manager/pane-manager-types.ts index 7b23c177236..00637ae7f97 100644 --- a/src/renderer/src/lib/pane-manager/pane-manager-types.ts +++ b/src/renderer/src/lib/pane-manager/pane-manager-types.ts @@ -63,6 +63,7 @@ export type PaneManagerOptions = { resolveExternalPaneDropTarget?: PaneExternalDropResolver onExternalPaneDrop?: PaneExternalDropHandler terminalOptions?: (paneId: number) => Partial + terminalLigaturesEnabled?: () => boolean terminalTuiScrollSensitivity?: () => number | undefined onLinkClick?: (paneId: number, event: MouseEvent | undefined, url: string) => void /** Resolved per hover so link-routing setting changes apply without recreating panes. */ diff --git a/src/renderer/src/lib/pane-manager/pane-split-close.ts b/src/renderer/src/lib/pane-manager/pane-split-close.ts index df725156655..c5c71b03be6 100644 --- a/src/renderer/src/lib/pane-manager/pane-split-close.ts +++ b/src/renderer/src/lib/pane-manager/pane-split-close.ts @@ -141,7 +141,7 @@ function openSplitPane( newPane: ManagedPaneInternal, cwd?: string ): void { - openTerminal(newPane) + openTerminal(newPane, args.managerOptions.terminalLigaturesEnabled?.()) applyPaneOpacity(args.panes.values(), newPane.id, args.styleOptions) applyDividerStyles(args.root, args.styleOptions) newPane.terminal.focus() diff --git a/src/renderer/src/lib/structured-agent-session-launch-refusal-fallback.test.ts b/src/renderer/src/lib/structured-agent-session-launch-refusal-fallback.test.ts new file mode 100644 index 00000000000..45c41bd111e --- /dev/null +++ b/src/renderer/src/lib/structured-agent-session-launch-refusal-fallback.test.ts @@ -0,0 +1,208 @@ +// @vitest-environment happy-dom + +// The duplicate-session guard: which create refusals may open a legacy terminal beside the chat. +// Deliberately exercises the real `launch-structured-agent-session`, because the classification +// under test lives there — mocking it out would assert nothing. + +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { toast } from 'sonner' +import type { RuntimeMobileSessionTabsResult } from '../../../shared/runtime-session-contracts' +import { RuntimeRpcCallError } from '@/runtime/runtime-rpc-client' + +const mocks = vi.hoisted(() => ({ + call: vi.fn(), + refresh: vi.fn() +})) + +vi.mock('sonner', () => ({ + toast: { error: vi.fn(), message: vi.fn() } +})) + +vi.mock('@/i18n/i18n', () => ({ + translate: (_key: string, fallback: string, options?: { value0?: string }) => + fallback.replace('{{value0}}', options?.value0 ?? '') +})) + +vi.mock('@/lib/agent-catalog', () => ({ + getAgentCatalog: () => [{ id: 'codex', label: 'Codex' }] +})) + +vi.mock('@/runtime/structured-agent-session-client', () => ({ + callStructuredAgentSession: mocks.call +})) + +vi.mock('@/runtime/local-structured-session-tabs-sync', () => ({ + LOCAL_STRUCTURED_SESSION_OWNER: 'local', + refreshLocalStructuredSessionTabs: mocks.refresh +})) + +vi.mock('@/store', () => ({ + useAppStore: { + getState: () => ({ unifiedTabsByWorktree: {} }), + subscribe: () => () => {} + } +})) + +import { + StructuredAgentSessionCreateRefusalError, + StructuredAgentSessionCreateUnknownOutcomeError +} from '@/lib/launch-structured-agent-session' +import { + getStructuredAgentLaunchStatus, + startStructuredAgentLaunch +} from './structured-agent-session-launch' + +type CreateReply = { ok: boolean; refusal?: { code: string; message: string } } + +/** Replies to every `agentSession.create` in turn, repeating the last reply thereafter. */ +function replyToCreates(...replies: CreateReply[]): void { + let index = 0 + mocks.call.mockImplementation(async (_target: unknown, method: string, params: unknown) => { + if (method !== 'agentSession.create') { + return { ok: true, page: { fence: 1 } } + } + const reply = replies[Math.min(index, replies.length - 1)] + index += 1 + if (!reply.ok) { + return reply + } + const sessionId = (params as { envelope: { sessionId: string } }).envelope.sessionId + return { ok: true, replayed: index > 1, fence: 1, value: { sessionId, fence: 1 } } + }) +} + +function refused(code: string): CreateReply { + return { ok: false, refusal: { code, message: `create refused: ${code}` } } +} + +function publishedSnapshot(worktreeId: string, sessionId: string): RuntimeMobileSessionTabsResult { + return { + worktree: worktreeId, + publicationEpoch: 'epoch-1', + snapshotVersion: 1, + activeGroupId: null, + activeTabId: null, + activeTabType: null, + tabs: [ + { + type: 'agent-session', + id: 'tab-1', + title: 'Codex', + sessionId, + agent: 'codex', + isActive: true + } + ] + } +} + +async function flushLaunchSettlement(): Promise { + for (let i = 0; i < 20; i += 1) { + await Promise.resolve() + } +} + +describe('legacy terminal fallback after a refused structured create', () => { + beforeEach(() => { + vi.clearAllMocks() + localStorage.clear() + mocks.refresh.mockResolvedValue([]) + }) + + it.each(['agent_session_operation_unknown', 'agent_session_ownership_unknown'])( + 'opens no sibling terminal when the host answers %s', + async (code) => { + const worktreeId = `wt-${code}` + const legacyTerminals: string[] = [] + replyToCreates(refused(code)) + + const launch = startStructuredAgentLaunch(worktreeId, 'codex') + void launch.claimDefinitiveRefusalFallback(() => { + legacyTerminals.push('legacy-terminal') + }) + + await expect(launch.launchResult).rejects.toBeInstanceOf( + StructuredAgentSessionCreateUnknownOutcomeError + ) + await flushLaunchSettlement() + + // The host may already hold the session, so the user keeps exactly one thing: no chat it + // could confirm, and no terminal beside a session it could not rule out. + expect(legacyTerminals).toEqual([]) + expect(launch.isVisibilityUnknown()).toBe(true) + expect(toast.error).toHaveBeenCalledOnce() + } + ) + + it('adopts the session an unknown outcome had already created, without a sibling', async () => { + const worktreeId = 'wt-unknown-then-published' + const legacyTerminals: string[] = [] + replyToCreates(refused('agent_session_operation_unknown'), { ok: true }) + + const launch = startStructuredAgentLaunch(worktreeId, 'codex') + const fallbackRan = launch.claimDefinitiveRefusalFallback(() => { + legacyTerminals.push('legacy-terminal') + }) + mocks.refresh + .mockResolvedValueOnce([]) + .mockResolvedValue([publishedSnapshot(worktreeId, launch.sessionId)]) + + await expect(launch.launchResult).resolves.toEqual({ + sessionId: launch.sessionId, + fence: 1 + }) + await expect(fallbackRan).resolves.toBe(false) + await flushLaunchSettlement() + + expect(legacyTerminals).toEqual([]) + expect(toast.error).not.toHaveBeenCalled() + }) + + it('opens exactly one legacy terminal when the refusal is on the definitive allowlist', async () => { + const worktreeId = 'wt-unsupported' + const legacyTerminals: string[] = [] + replyToCreates(refused('structured_agent_session_unsupported')) + + const launch = startStructuredAgentLaunch(worktreeId, 'codex') + const fallbackRan = launch.claimDefinitiveRefusalFallback(() => { + legacyTerminals.push('legacy-terminal') + }) + + await expect(launch.launchResult).rejects.toBeInstanceOf( + StructuredAgentSessionCreateRefusalError + ) + await expect(fallbackRan).resolves.toBe(true) + await flushLaunchSettlement() + + expect(legacyTerminals).toEqual(['legacy-terminal']) + // A proven "nothing was created" needs no replay, so the terminal is the only surface open. + expect( + mocks.call.mock.calls.filter(([, method]) => method === 'agentSession.create') + ).toHaveLength(1) + expect(launch.isVisibilityUnknown()).toBe(false) + }) + + it('opens exactly one legacy terminal when an older runtime has no create method', async () => { + const legacyTerminals: string[] = [] + mocks.call.mockRejectedValue( + new RuntimeRpcCallError({ + id: 'rpc-old-runtime', + ok: false, + error: { code: 'method_not_found', message: 'Unknown method: agentSession.create' } + }) + ) + + const launch = startStructuredAgentLaunch('wt-old-runtime', 'codex') + const fallbackRan = launch.claimDefinitiveRefusalFallback(() => { + legacyTerminals.push('legacy-terminal') + }) + + await expect(launch.launchResult).rejects.toBeInstanceOf( + StructuredAgentSessionCreateRefusalError + ) + await expect(fallbackRan).resolves.toBe(true) + expect(legacyTerminals).toEqual(['legacy-terminal']) + expect(mocks.call).toHaveBeenCalledOnce() + expect(getStructuredAgentLaunchStatus('wt-old-runtime', 'codex')).toBe('idle') + }) +}) diff --git a/src/renderer/src/runtime/structured-agent-session-client.ts b/src/renderer/src/runtime/structured-agent-session-client.ts index 0e8d2ce16f2..71be3f3449d 100644 --- a/src/renderer/src/runtime/structured-agent-session-client.ts +++ b/src/renderer/src/runtime/structured-agent-session-client.ts @@ -1,5 +1,8 @@ import type { RuntimeRpcResponse } from '../../../shared/runtime-rpc-envelope' -import type { AgentSessionSubscribeEvent } from '../../../shared/agent-session-wire' +import type { + AgentSessionStatusEvent, + AgentSessionSubscribeEvent +} from '../../../shared/agent-session-wire' import { getRuntimeEnvironmentRevision } from './runtime-environment-revision' import { callRuntimeRpc, type RuntimeClientTarget } from './runtime-rpc-client' @@ -11,10 +14,11 @@ export function callStructuredAgentSession( return callRuntimeRpc(target, method, params) } -export async function subscribeStructuredAgentSession( +async function subscribeStructuredAgentSessionMethod( target: RuntimeClientTarget, + method: string, params: unknown, - onEvent: (event: AgentSessionSubscribeEvent) => void, + onEvent: (event: TEvent) => void, onError: (error: unknown) => void, onClose: () => void ): Promise<{ unsubscribe: () => void }> { @@ -23,15 +27,15 @@ export async function subscribeStructuredAgentSession( onError(response.error) return } - onEvent(response.result as AgentSessionSubscribeEvent) + onEvent(response.result as TEvent) } if (target.kind === 'local') { - return window.api.runtime.subscribe({ method: 'agentSession.subscribe', params }, onResponse) + return window.api.runtime.subscribe({ method, params }, onResponse) } return window.api.runtimeEnvironments.subscribe( { selector: target.environmentId, - method: 'agentSession.subscribe', + method, params, timeoutMs: 15_000, expectedEnvironmentPairingRevision: getRuntimeEnvironmentRevision(target.environmentId) @@ -39,3 +43,37 @@ export async function subscribeStructuredAgentSession( { onResponse, onError, onClose } ) } + +export function subscribeStructuredAgentSession( + target: RuntimeClientTarget, + params: unknown, + onEvent: (event: AgentSessionSubscribeEvent) => void, + onError: (error: unknown) => void, + onClose: () => void +): Promise<{ unsubscribe: () => void }> { + return subscribeStructuredAgentSessionMethod( + target, + 'agentSession.subscribe', + params, + onEvent, + onError, + onClose + ) +} + +/** Every structured session's projected status on one runtime, as the host publishes it. */ +export function subscribeStructuredAgentSessionStatus( + target: RuntimeClientTarget, + onEvent: (event: AgentSessionStatusEvent) => void, + onError: (error: unknown) => void, + onClose: () => void +): Promise<{ unsubscribe: () => void }> { + return subscribeStructuredAgentSessionMethod( + target, + 'agentSession.subscribeStatus', + {}, + onEvent, + onError, + onClose + ) +} diff --git a/src/renderer/src/runtime/structured-agent-session-status-feed.test.ts b/src/renderer/src/runtime/structured-agent-session-status-feed.test.ts new file mode 100644 index 00000000000..d9cb4dd3d72 --- /dev/null +++ b/src/renderer/src/runtime/structured-agent-session-status-feed.test.ts @@ -0,0 +1,140 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { + AgentSessionStatusEvent, + AgentSessionStatusSummary +} from '../../../shared/agent-session-wire' +import { AGENT_SESSION_STATUS_FEED_RUNTIME_CAPABILITY } from '../../../shared/protocol-version' + +const mocks = vi.hoisted(() => ({ + subscribeStatus: vi.fn(), + supportsCapability: vi.fn(), + unsubscribe: vi.fn() +})) + +vi.mock('./structured-agent-session-client', () => ({ + subscribeStructuredAgentSessionStatus: mocks.subscribeStatus +})) + +vi.mock('./runtime-rpc-client', () => ({ + runtimeEnvironmentSupportsCapability: mocks.supportsCapability +})) + +import { + getStructuredAgentSessionStatusFeed, + resetStructuredAgentSessionStatusFeedsForTests +} from './structured-agent-session-status-feed' + +const REMOTE = { kind: 'environment', environmentId: 'env-1' } as const +const LOCAL = { kind: 'local' } as const + +function summary( + sessionId: string, + status: AgentSessionStatusSummary['status'] = 'idle' +): AgentSessionStatusSummary { + return { + sessionId, + workspaceId: 'wt-1', + agent: 'codex', + status, + latestPrompt: 'hello', + updatedAt: 1 + } +} + +/** The event callback the feed handed to the most recent subscription. */ +function hostEmit(index = 0): (event: AgentSessionStatusEvent) => void { + const call = mocks.subscribeStatus.mock.calls[index] + if (!call) { + throw new Error('status feed not subscribed') + } + return call[1] as (event: AgentSessionStatusEvent) => void +} + +describe('structured agent session status feed', () => { + beforeEach(() => { + vi.useFakeTimers() + vi.clearAllMocks() + resetStructuredAgentSessionStatusFeedsForTests() + mocks.subscribeStatus.mockResolvedValue({ unsubscribe: mocks.unsubscribe }) + mocks.supportsCapability.mockResolvedValue(true) + }) + + afterEach(() => { + resetStructuredAgentSessionStatusFeedsForTests() + vi.useRealTimers() + }) + + it('never subscribes, and never retries, against a host without the status feed', async () => { + mocks.supportsCapability.mockResolvedValue(false) + getStructuredAgentSessionStatusFeed(REMOTE).activate() + + await vi.advanceTimersByTimeAsync(0) + expect(mocks.supportsCapability).toHaveBeenCalledWith( + 'env-1', + AGENT_SESSION_STATUS_FEED_RUNTIME_CAPABILITY + ) + expect(mocks.subscribeStatus).not.toHaveBeenCalled() + expect(vi.getTimerCount()).toBe(0) + + await vi.advanceTimersByTimeAsync(60_000) + expect(mocks.subscribeStatus).not.toHaveBeenCalled() + expect(mocks.supportsCapability).toHaveBeenCalledOnce() + }) + + it('subscribes once the remote host advertises the status feed', async () => { + getStructuredAgentSessionStatusFeed(REMOTE).activate() + + await vi.advanceTimersByTimeAsync(0) + expect(mocks.subscribeStatus).toHaveBeenCalledOnce() + expect(mocks.subscribeStatus.mock.calls[0]?.[0]).toEqual(REMOTE) + }) + + it('reconnects when the capability probe fails, which is not an answer', async () => { + mocks.supportsCapability.mockRejectedValue(new Error('relay unreachable')) + getStructuredAgentSessionStatusFeed(REMOTE).activate() + + await vi.advanceTimersByTimeAsync(0) + expect(mocks.supportsCapability).toHaveBeenCalledOnce() + + await vi.advanceTimersByTimeAsync(300) + expect(mocks.supportsCapability).toHaveBeenCalledTimes(2) + expect(mocks.subscribeStatus).not.toHaveBeenCalled() + }) + + it('probes nothing for a local host, which is this build', async () => { + getStructuredAgentSessionStatusFeed(LOCAL).activate() + + await vi.advanceTimersByTimeAsync(0) + expect(mocks.supportsCapability).not.toHaveBeenCalled() + expect(mocks.subscribeStatus).toHaveBeenCalledOnce() + }) + + it('merges a snapshot over the cached rows instead of retracting them', async () => { + const feed = getStructuredAgentSessionStatusFeed(LOCAL) + feed.activate() + await vi.advanceTimersByTimeAsync(0) + hostEmit()({ type: 'snapshot', sessions: [summary('session-1'), summary('session-2')] }) + expect([...feed.getSnapshot().keys()]).toEqual(['session-1', 'session-2']) + + // A restarted host restores its readable sessions after the stream reopens. + hostEmit()({ type: 'snapshot', sessions: [] }) + expect([...feed.getSnapshot().keys()]).toEqual(['session-1', 'session-2']) + + hostEmit()({ type: 'snapshot', sessions: [summary('session-1', 'working')] }) + expect(feed.getSnapshot().get('session-1')?.status).toBe('working') + expect(feed.getSnapshot().get('session-2')?.status).toBe('idle') + }) + + it('stops a pending reconnect when the feeds are reset between tests', async () => { + getStructuredAgentSessionStatusFeed(LOCAL).activate() + await vi.advanceTimersByTimeAsync(0) + hostEmit()({ type: 'end' }) + expect(vi.getTimerCount()).toBe(1) + + resetStructuredAgentSessionStatusFeedsForTests() + + expect(vi.getTimerCount()).toBe(0) + await vi.advanceTimersByTimeAsync(10_000) + expect(mocks.subscribeStatus).toHaveBeenCalledOnce() + }) +}) diff --git a/src/renderer/src/runtime/structured-agent-session-status-feed.ts b/src/renderer/src/runtime/structured-agent-session-status-feed.ts new file mode 100644 index 00000000000..2b550679315 --- /dev/null +++ b/src/renderer/src/runtime/structured-agent-session-status-feed.ts @@ -0,0 +1,211 @@ +// One host status stream per runtime target, shared by every session-list projection. +// +// The feed is a read-only mirror: the host projects each session's status from its journal and +// this owner keeps the latest summary per session while anyone is looking. Losing the stream +// keeps the cached summaries and reconnects; a fresh snapshot merges over them. +// Which sessions are listed is the tab map's decision, so the feed never retracts a summary. + +import type { + AgentSessionStatusEvent, + AgentSessionStatusSummary +} from '../../../shared/agent-session-wire' +import { AGENT_SESSION_STATUS_FEED_RUNTIME_CAPABILITY } from '../../../shared/protocol-version' +import { + runtimeEnvironmentSupportsCapability, + type RuntimeClientTarget +} from './runtime-rpc-client' +import { subscribeStructuredAgentSessionStatus } from './structured-agent-session-client' + +export type StructuredAgentSessionStatusSnapshot = ReadonlyMap + +export type StructuredAgentSessionStatusFeedOwner = { + activate: () => () => void + getSnapshot: () => StructuredAgentSessionStatusSnapshot + subscribe: (listener: () => void) => () => void +} + +const RECONNECT_MAX_DELAY_MS = 5_000 + +/** `stop` is the map's own teardown, not part of the owner contract callers hold. */ +type OwnedStatusFeed = StructuredAgentSessionStatusFeedOwner & { stop: () => void } + +const owners = new Map() + +export function structuredAgentSessionStatusFeedKey(target: RuntimeClientTarget): string { + return target.kind === 'local' ? 'local' : `environment:${target.environmentId}` +} + +function createOwner(target: RuntimeClientTarget): OwnedStatusFeed { + let snapshot: StructuredAgentSessionStatusSnapshot = new Map() + const listeners = new Set<() => void>() + const activations = new Set() + let generation = 0 + let handle: { unsubscribe: () => void } | null = null + let reconnectTimer: ReturnType | null = null + let reconnectAttempt = 0 + + const emit = (): void => { + for (const listener of listeners) { + listener() + } + } + const setSnapshot = (next: StructuredAgentSessionStatusSnapshot): void => { + snapshot = next + emit() + } + const applyEvent = (event: AgentSessionStatusEvent): void => { + if (event.type === 'snapshot') { + reconnectAttempt = 0 + // Merged, not replaced: a restarted host restores its readable sessions asynchronously, so + // the first snapshot can be empty and dropping those rows flickers every one to no-status. + const next = new Map(snapshot) + for (const session of event.sessions) { + next.set(session.sessionId, session) + } + setSnapshot(next) + return + } + if (event.type === 'status') { + const next = new Map(snapshot) + next.set(event.session.sessionId, event.session) + setSnapshot(next) + } + } + const active = (candidate: number): boolean => activations.size > 0 && candidate === generation + const clearReconnect = (): void => { + if (reconnectTimer) { + clearTimeout(reconnectTimer) + reconnectTimer = null + } + } + const dropHandle = (): void => { + handle?.unsubscribe() + handle = null + } + let open = (): void => {} + const scheduleReconnect = (candidate: number): void => { + if (!active(candidate) || reconnectTimer) { + return + } + const delay = Math.min(250 * 2 ** reconnectAttempt, RECONNECT_MAX_DELAY_MS) + reconnectAttempt += 1 + reconnectTimer = setTimeout(() => { + reconnectTimer = null + if (active(candidate)) { + open() + } + }, delay) + } + const subscribeToHost = (candidate: number): void => { + void subscribeStructuredAgentSessionStatus( + target, + (event) => { + if (!active(candidate)) { + return + } + if (event.type === 'end') { + dropHandle() + scheduleReconnect(candidate) + return + } + applyEvent(event) + }, + () => { + if (active(candidate)) { + dropHandle() + scheduleReconnect(candidate) + } + }, + () => { + if (active(candidate)) { + dropHandle() + scheduleReconnect(candidate) + } + } + ) + .then((opened) => { + if (active(candidate)) { + handle = opened + } else { + opened.unsubscribe() + } + }) + .catch(() => scheduleReconnect(candidate)) + } + open = (): void => { + const candidate = ++generation + dropHandle() + if (target.kind !== 'environment') { + // A local host is this build; only a remote one can predate the method. + subscribeToHost(candidate) + return + } + const environmentId = target.environmentId + void runtimeEnvironmentSupportsCapability( + environmentId, + AGENT_SESSION_STATUS_FEED_RUNTIME_CAPABILITY + ) + .then((supported) => { + if (!active(candidate)) { + return + } + // A host without the method is terminal, not a fault: retrying would relay-probe + // forever. A failed probe is not an answer, so that path still reconnects. + if (supported) { + subscribeToHost(candidate) + return + } + console.warn('[structured-session-status] host too old for the status feed', environmentId) + }) + .catch(() => scheduleReconnect(candidate)) + } + const stop = (): void => { + generation += 1 + clearReconnect() + dropHandle() + reconnectAttempt = 0 + } + + return { + activate: () => { + const token = Symbol('status-feed') + activations.add(token) + if (activations.size === 1) { + open() + } + return () => { + activations.delete(token) + if (activations.size === 0) { + stop() + } + } + }, + getSnapshot: () => snapshot, + subscribe: (listener) => { + listeners.add(listener) + return () => listeners.delete(listener) + }, + stop + } +} + +export function getStructuredAgentSessionStatusFeed( + target: RuntimeClientTarget +): StructuredAgentSessionStatusFeedOwner { + const key = structuredAgentSessionStatusFeedKey(target) + let owner = owners.get(key) + if (!owner) { + owner = createOwner(target) + owners.set(key, owner) + } + return owner +} + +export function resetStructuredAgentSessionStatusFeedsForTests(): void { + // Dropping the map alone leaves a live subscription and its pending reconnect running + // into the next test, where they reopen a stream nothing is holding. + for (const owner of owners.values()) { + owner.stop() + } + owners.clear() +} diff --git a/src/shared/agent-session-wire.ts b/src/shared/agent-session-wire.ts index 40a408df899..a0af962583d 100644 --- a/src/shared/agent-session-wire.ts +++ b/src/shared/agent-session-wire.ts @@ -12,8 +12,13 @@ import type { AgentJournalResolution, AgentJournalSubmission } from './agent-session-journal-types' -import type { AgentSessionHandoffStage, AgentSessionOwnerRuntimeKind } from './agent-session-record' +import type { + AgentSessionHandoffStage, + AgentSessionOwnerRuntimeKind, + AgentSessionRecord +} from './agent-session-record' import type { AgentProviderSessionMetadata } from './agent-session-resume' +import type { StructuredAgentSessionProjectedStatus } from './structured-agent-session-projection' export type AgentSessionHandoffDirection = 'to-tui' | 'to-native' export type AgentSessionHandoffMode = 'now' | 'after-turn' | 'stop-turn' @@ -59,6 +64,8 @@ export type AgentSessionBackgroundTaskState = { state: 'monitoring' /** Optional so mixed-version clients can consume state-only hosts. */ tasks?: AgentSessionBackgroundTask[] + /** Optional so clients only send targeted stops to hosts that accept them. */ + supportsTaskStop?: boolean } /** Backward paging is the client's normal read; 40 matches the page size the @@ -159,6 +166,29 @@ export type AgentSessionSubscribeEvent = } | { type: 'end' } +// ─── Status feed ──────────────────────────────────────────────────────────── + +/** What a session list needs to know about one session. The host projects it + * from the journal so no client has to replay a transcript to learn whether a + * turn is running. Additive surface: an older host has no such method. */ +export type AgentSessionStatusSummary = { + sessionId: string + workspaceId: string + agent: AgentSessionRecord['provider'] + /** Null until the journal holds a persisted user or assistant message. */ + status: StructuredAgentSessionProjectedStatus | null + latestPrompt: string + providerSession?: AgentProviderSessionMetadata + updatedAt: number +} + +/** A summary outlives its provider child: an evicted idle session is still idle, so the host + * keeps the last projection and never retracts one. Tabs, not this feed, decide what is listed. */ +export type AgentSessionStatusEvent = + | { type: 'snapshot'; sessions: AgentSessionStatusSummary[] } + | { type: 'status'; session: AgentSessionStatusSummary } + | { type: 'end' } + // ─── Mutation envelope ────────────────────────────────────────────────────── /** diff --git a/src/shared/git-binary-compatibility.test.ts b/src/shared/git-binary-compatibility.test.ts index 6387f200b4c..fb8161b9f90 100644 --- a/src/shared/git-binary-compatibility.test.ts +++ b/src/shared/git-binary-compatibility.test.ts @@ -111,6 +111,33 @@ describeBinaryCompatibility('real Git binary compatibility', () => { } }) + it('quietly distinguishes present and absent branch refs', async () => { + const head = (await runGit(['rev-parse', 'HEAD'])).stdout.trim() + await runGit(['branch', 'quiet-probe-present', head]) + await expect( + runGit(['rev-parse', '--verify', '--quiet', 'refs/heads/quiet-probe-present']) + ).resolves.toMatchObject({ stdout: `${head}\n`, stderr: '' }) + await expect( + runGit(['rev-parse', '--verify', '--quiet', 'refs/heads/quiet-probe-absent']) + ).rejects.toMatchObject({ code: 1, stdout: '', stderr: '' }) + }) + + it('distinguishes an absent branch from a ref pointing at a missing object', async () => { + const missingObject = 'a'.repeat(40) + const refPath = join(repoPath, '.git', 'refs', 'heads', 'quiet-probe-dangling') + await writeFile(refPath, `${missingObject}\n`) + try { + await expect( + runGit(['rev-parse', '--verify', '--quiet', 'refs/heads/quiet-probe-dangling']) + ).resolves.toMatchObject({ stdout: `${missingObject}\n`, stderr: '' }) + await expect( + runGit(['rev-parse', '--verify', '--quiet', 'refs/heads/quiet-probe-dangling^{commit}']) + ).rejects.toMatchObject({ code: 1, stdout: '', stderr: '' }) + } finally { + await rm(refPath) + } + }) + it('recognizes worktree-list and rev-parse compatibility boundaries', async () => { await expectPreferredOrRecognizedFallback( ['worktree', 'list', '--porcelain', '-z'], diff --git a/src/shared/orchestration-dispatch-refusal-contract.test.ts b/src/shared/orchestration-dispatch-refusal-contract.test.ts new file mode 100644 index 00000000000..4f9fa4bf617 --- /dev/null +++ b/src/shared/orchestration-dispatch-refusal-contract.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from 'vitest' +import { + buildInjectRejectionMessage, + taskNotFoundRefusal, + taskNotStartableRefusal +} from './orchestration-dispatch-refusal-contract' +import { TUI_AGENT_CONFIG } from './tui-agent-config' +import { recognizeAgentProcess } from './agent-process-recognition' + +describe('buildInjectRejectionMessage', () => { + const message = buildInjectRejectionMessage('term_a') + + it('keeps the substring callers and scripts match on', () => { + expect(message).toContain('Cannot dispatch --inject to terminal term_a') + expect(message).toContain('no recognized agent detected') + }) + + it('names every agent Orca recognizes, including agy', () => { + expect(message).toMatch(/\bagy\b/) + for (const config of Object.values(TUI_AGENT_CONFIG)) { + expect(message).toContain(config.expectedProcess) + } + }) + + it('lists only names detection actually resolves, deduped and sorted', () => { + const listed = (/\(([^)]+)\)/.exec(message)?.[1] ?? '').split(', ') + + expect(listed.length).toBeGreaterThan(0) + expect(new Set(listed).size).toBe(listed.length) + expect([...listed].sort()).toEqual(listed) + for (const name of listed) { + expect(recognizeAgentProcess(name)).not.toBeNull() + } + }) +}) + +// Why: these strings are published receipts; they are pinned as literals, independent of the +// builders, so a refactor cannot silently rewrite them together with the expectation. +describe('dispatch refusal receipts keep their published messages', () => { + it('leaves the message exactly as each call site supplies it', () => { + expect(taskNotFoundRefusal('Task not found: task_1', { taskId: 'task_1' }).message).toBe( + 'Task not found: task_1' + ) + expect( + taskNotStartableRefusal('Task task_1 is pending; only a ready Task can start.', { + taskId: 'task_1', + status: 'pending', + unmetDependencies: [] + }).message + ).toBe('Task task_1 is pending; only a ready Task can start.') + }) + + it('tailors nextSteps to retry, dependency, occupancy, and terminal-status refusals', () => { + const base = { taskId: 'task_1', status: 'failed', unmetDependencies: [] } + expect(taskNotStartableRefusal('m', { ...base, retryOf: 'ctx_1' }).data.nextSteps[0]).toMatch( + /--retry-of.*ctx_1/ + ) + expect( + taskNotStartableRefusal('m', { ...base, status: 'pending', unmetDependencies: ['task_0'] }) + .data.nextSteps[0] + ).toMatch(/task_0.*unblock failed/) + expect( + taskNotStartableRefusal('m', { ...base, status: 'dispatched' }).data.nextSteps[0] + ).toMatch(/dispatch-show --task task_1/) + expect(taskNotStartableRefusal('m', base).data.nextSteps[0]).toMatch(/failed Task cannot/) + }) +}) diff --git a/src/shared/orchestration-dispatch-refusal-contract.ts b/src/shared/orchestration-dispatch-refusal-contract.ts new file mode 100644 index 00000000000..b764067402b --- /dev/null +++ b/src/shared/orchestration-dispatch-refusal-contract.ts @@ -0,0 +1,102 @@ +import { TUI_AGENT_CONFIG } from './tui-agent-config' + +// Why: one source for each dispatch refusal's code, message, and data, so the runtime emits and +// the CLI test formats the identical envelope. Messages are supplied per call site because each +// existing string is a published receipt an old consumer may match on. + +export type DispatchRefusalReceipt = { + code: 'task_not_found' | 'task_not_startable' | 'inject_rejected' + message: string + data: Record & { nextSteps: string[] } +} + +export function taskNotFoundRefusal( + message: string, + detail: { taskId: string; runId?: string } +): DispatchRefusalReceipt { + return { + code: 'task_not_found', + message, + data: { + ...detail, + nextSteps: [ + 'Run orca orchestration task-list --json in the bound Run to find the intended Task id.', + 'If the Task does not exist yet, create it with orca orchestration task-create --spec --json.' + ] + } + } +} + +export type TaskNotStartableDetail = { + taskId: string + status: string + unmetDependencies: string[] + retryOf?: string +} + +export function taskNotStartableRefusal( + message: string, + detail: TaskNotStartableDetail +): DispatchRefusalReceipt { + return { + code: 'task_not_startable', + message, + data: { ...detail, nextSteps: taskNotStartableNextSteps(detail) } + } +} + +function taskNotStartableNextSteps(detail: TaskNotStartableDetail): string[] { + if (detail.retryOf) { + return [ + `--retry-of must name the latest settled Dispatch of a failed or blocked Task; check orca orchestration dispatch-show --task ${detail.taskId} --json and orca orchestration worker-show --dispatch ${detail.retryOf} --json.` + ] + } + if (detail.unmetDependencies.length > 0) { + return [ + `Dependencies ${detail.unmetDependencies.join(', ')} are not completed. Wait for running ones with orca orchestration check --wait --json; retry or unblock failed ones before dispatching again.` + ] + } + if (detail.status === 'dispatched') { + return [ + `The Task already has an active Dispatch; inspect it with orca orchestration dispatch-show --task ${detail.taskId} --json.` + ] + } + return [ + `A ${detail.status} Task cannot be dispatched; create a new Task or use worker-start --retry-of for a failed attempt.` + ] +} + +// Why: the old five-name example read as an allowlist (#15125); derive from the field detection keys on so it cannot drift. +// Not filtered by `disabledTuiAgents` — that gates Orca's launchers, not detection, so a hand-started disabled agent still injects. +const RECOGNIZED_AGENT_PROCESS_NAMES = [ + ...new Set(Object.values(TUI_AGENT_CONFIG).map((config) => config.expectedProcess)) +].sort() + +export function buildInjectRejectionMessage(terminal: string): string { + return ( + `Cannot dispatch --inject to terminal ${terminal}: no recognized agent detected. ` + + `Orca detects these agent CLIs (${RECOGNIZED_AGENT_PROCESS_NAMES.join(', ')}). ` + + 'Start one in the terminal and let it finish launching, ' + + 'or dispatch without --inject and send the prompt manually.' + ) +} + +export type InjectRejectionReason = 'no_agent_detected' + +export function injectRejectedRefusal( + terminal: string, + reason: InjectRejectionReason +): DispatchRefusalReceipt { + return { + code: 'inject_rejected', + message: buildInjectRejectionMessage(terminal), + data: { + terminal, + reason, + nextSteps: [ + 'Start a recognized agent CLI in that terminal and wait for it to finish launching, or pick a terminal that already runs one.', + 'Alternatively dispatch without --inject and deliver the prompt with orca terminal send --terminal --text --enter --json.' + ] + } + } +} diff --git a/src/shared/pane-agent-identity-inventory.test.ts b/src/shared/pane-agent-identity-inventory.test.ts index cd7c914ce09..7de6b14f1c7 100644 --- a/src/shared/pane-agent-identity-inventory.test.ts +++ b/src/shared/pane-agent-identity-inventory.test.ts @@ -56,7 +56,7 @@ const INVENTORY: readonly InventoryGroup[] = [ 'src/renderer/src/components/agent-session-continuation/AgentSessionContinuationDialog.tsx', 2 ], - ['src/renderer/src/components/automations/AutomationListLocalRows.tsx', 2], + ['src/renderer/src/components/automations/AutomationListLocalRow.tsx', 2], 'src/renderer/src/components/automations/automation-draft-model.ts', ['src/renderer/src/components/automations/automation-list-search-rows.ts', 2], ['src/renderer/src/components/dashboard-popout/AgentMapSnapshotWorkspaceMenu.tsx', 2], diff --git a/src/shared/protocol-version.ts b/src/shared/protocol-version.ts index baa791a38a4..f5546a6f6a0 100644 --- a/src/shared/protocol-version.ts +++ b/src/shared/protocol-version.ts @@ -141,6 +141,10 @@ export const CLAUDE_STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY = // to stop provider children after the last surface closes without tying lifetime to a transport. export const STRUCTURED_AGENT_SESSION_HOLD_RUNTIME_CAPABILITY = 'agent-session.structured.hold.v1' as const +// Why: agentSession.subscribeStatus is additive to a surface that already shipped, so a host +// advertising agent-session.structured.v1 may still answer it with method_not_found. Clients must +// probe before subscribing or they reconnect forever and never show any status at all. +export const AGENT_SESSION_STATUS_FEED_RUNTIME_CAPABILITY = 'agent-session.status-feed.v1' as const // Why: adding kimi to RESUMABLE_TUI_AGENTS grows terminal.ensureAgentSession's enum, and an // older host answers the unknown member with invalid_argument — a code the launch fallback does // not retry on — so clients must probe before taking the host-authority path. @@ -241,6 +245,7 @@ export const RUNTIME_CAPABILITIES = [ AGENT_SESSION_OMP_RESUME_PATH_RUNTIME_CAPABILITY, STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY, STRUCTURED_AGENT_SESSION_HOLD_RUNTIME_CAPABILITY, + AGENT_SESSION_STATUS_FEED_RUNTIME_CAPABILITY, AGENT_SESSION_KIMI_RESUME_RUNTIME_CAPABILITY, FILE_MUTATION_OWNERSHIP_RUNTIME_CAPABILITY, GITHUB_MARK_PR_READY_RUNTIME_CAPABILITY, diff --git a/src/shared/structured-agent-session-projection.test.ts b/src/shared/structured-agent-session-projection.test.ts index 048051e70a5..8bdce30577e 100644 --- a/src/shared/structured-agent-session-projection.test.ts +++ b/src/shared/structured-agent-session-projection.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from 'vitest' +import { AGENT_STATUS_MAX_FIELD_LENGTH } from './agent-status-field-normalization' import type { AgentJournalRenderItem } from './agent-session-journal-types' import { parsePaneKey } from './stable-pane-id' import { @@ -6,6 +7,7 @@ import { hasPersistedStructuredAgentSessionTurn, projectStructuredItemToNativeChat, projectStructuredAgentSessionStatus, + projectStructuredAgentSessionStatusSummary, structuredAgentSessionPaneKey } from './structured-agent-session-projection' @@ -44,6 +46,52 @@ describe('structured agent session status projection', () => { expect(projectStructuredAgentSessionStatus([running, completed])).toBe('idle') }) + it('summarizes status with the newest user prompt, and null before any persisted turn', () => { + const running = item('running', 3, { + kind: 'status', + text: 'Working', + turnLifecycle: { turnId: 'turn-1', state: 'running' } + }) + const first = item('first', 1, { + kind: 'message', + role: 'user', + blocks: [{ type: 'text', text: 'first' }] + }) + const second = item('second', 2, { + kind: 'message', + role: 'user', + blocks: [ + { type: 'text', text: 'second' }, + { type: 'text', text: 'line' } + ] + }) + + expect(projectStructuredAgentSessionStatusSummary([running])).toEqual({ + status: null, + latestPrompt: '' + }) + expect(projectStructuredAgentSessionStatusSummary([first, second, running])).toEqual({ + status: 'working', + latestPrompt: 'second line' + }) + expect(projectStructuredAgentSessionStatusSummary([first, second])).toEqual({ + status: 'idle', + latestPrompt: 'second line' + }) + }) + + it('bounds the wire prompt at the shared agent-status preview cap', () => { + const pasted = item('pasted', 1, { + kind: 'message', + role: 'user', + blocks: [{ type: 'text', text: 'x'.repeat(AGENT_STATUS_MAX_FIELD_LENGTH * 40) }] + }) + + expect(projectStructuredAgentSessionStatusSummary([pasted]).latestPrompt).toHaveLength( + AGENT_STATUS_MAX_FIELD_LENGTH + ) + }) + it('creates a deterministic pane identity for status stores', () => { const paneKey = structuredAgentSessionPaneKey('structured-agent-session-1', 'session-1') diff --git a/src/shared/structured-agent-session-projection.ts b/src/shared/structured-agent-session-projection.ts index 94938dc955f..6a5f01ba9ea 100644 --- a/src/shared/structured-agent-session-projection.ts +++ b/src/shared/structured-agent-session-projection.ts @@ -1,3 +1,4 @@ +import { normalizePromptField } from './agent-status-field-normalization' import type { AgentJournalRenderItem } from './agent-session-journal-types' import type { NativeChatBlock, NativeChatMessage } from './native-chat-types' import { sha256 } from './sha256' @@ -162,6 +163,34 @@ export function projectStructuredAgentSessionStatus( return activeStructuredAgentSessionTurnId(items) ? 'working' : 'idle' } +/** The newest user prompt, as the sidebar quotes it. */ +export function latestStructuredAgentSessionPrompt( + items: readonly AgentJournalRenderItem[] +): string { + for (let index = items.length - 1; index >= 0; index -= 1) { + const body = items[index]?.body + if (body?.kind === 'message' && body.role === 'user') { + return body.blocks.flatMap((block) => (block.type === 'text' ? [block.text] : [])).join('\n') + } + } + return '' +} + +/** One projection shared by host and client: null status means "no turn yet", not idle. + * The prompt is bounded to the same preview every other agent-status row carries — a send + * admits 256 KB, and one status frame carries every retained session at once. */ +export function projectStructuredAgentSessionStatusSummary( + items: readonly AgentJournalRenderItem[] +): { status: StructuredAgentSessionProjectedStatus | null; latestPrompt: string } { + if (!hasPersistedStructuredAgentSessionTurn(items)) { + return { status: null, latestPrompt: '' } + } + return { + status: projectStructuredAgentSessionStatus(items), + latestPrompt: normalizePromptField(latestStructuredAgentSessionPrompt(items)) + } +} + export function structuredAgentSessionPaneKey(tabId: string, sessionId: string): string { const bytes = sha256(new TextEncoder().encode(sessionId)) const hex = Array.from(bytes.slice(0, 16), (byte) => byte.toString(16).padStart(2, '0')).join('') diff --git a/src/shared/structured-agent-session-reducer.test.ts b/src/shared/structured-agent-session-reducer.test.ts index 99ced770641..d36b6717758 100644 --- a/src/shared/structured-agent-session-reducer.test.ts +++ b/src/shared/structured-agent-session-reducer.test.ts @@ -54,6 +54,39 @@ function hydrationPage( } describe('structured agent session reducer', () => { + it('applies an additive targeted-stop capability update without journal churn', () => { + const backgroundTasks = { + state: 'monitoring' as const, + tasks: [{ id: 'task-1', kind: 'agent' as const }] + } + const initial = reduceStructuredAgentSession(EMPTY_STRUCTURED_AGENT_SESSION, { + type: 'event', + event: { + type: 'snapshot', + sessionId: 'session-a', + fence: 1, + page: { ...hydrationPage([]), backgroundTasks } + } + }) + const updated = reduceStructuredAgentSession(initial, { + type: 'event', + event: { + type: 'batch', + sessionId: 'session-a', + batch: { + cursor: { epoch: 'epoch-a', sequence: 0 }, + items: [], + removedItemIds: [], + submissions: [] + }, + backgroundTasks: { ...backgroundTasks, supportsTaskStop: true } + } + }) + + expect(updated.backgroundTasks).toEqual({ ...backgroundTasks, supportsTaskStop: true }) + expect(updated.items).toBe(initial.items) + }) + it('uses the bounded hydration page pagination boundary', () => { const restored = reduceStructuredAgentSession(EMPTY_STRUCTURED_AGENT_SESSION, { type: 'event', diff --git a/src/shared/structured-agent-session-reducer.ts b/src/shared/structured-agent-session-reducer.ts index 0a330539548..88d41b2f8e5 100644 --- a/src/shared/structured-agent-session-reducer.ts +++ b/src/shared/structured-agent-session-reducer.ts @@ -51,7 +51,12 @@ function backgroundTaskStatesEqual( if (left === right) { return true } - if (!left || !right || left.state !== right.state) { + if ( + !left || + !right || + left.state !== right.state || + left.supportsTaskStop !== right.supportsTaskStop + ) { return false } if (left.tasks === right.tasks) { diff --git a/tests/AGENTS.md b/tests/AGENTS.md index f445415e7df..26987a87c25 100644 --- a/tests/AGENTS.md +++ b/tests/AGENTS.md @@ -6,16 +6,18 @@ take the foreground — no window raised over the editor, no focus stolen, no Do `src/main/window/foreground-activation-policy.ts` enforces this in the main process. It is on whenever `ORCA_E2E_HEADLESS=1`, `ORCA_E2E_HEADFUL=1`, or `ORCA_BACKGROUND_LAUNCH=1`: -- headless → the window never reaches the screen (Playwright drives it via CDP) -- headful / background → `showInactive()`, no `app.focus({ steal: true })`, no +- headless / explicit background → the window never reaches the screen (Playwright drives it via CDP) +- headful without explicit background → `showInactive()`, no `app.focus({ steal: true })`, no `moveTop()`/always-on-top reinforcement -- macOS headless → `accessory` activation policy, so no Dock tile and no menu-bar takeover +- macOS headless / explicit background → `accessory` activation policy, so no Dock tile and no menu-bar takeover Rules when adding tests or scripts: - Launch through `tests/e2e/helpers/orca-app.ts` (or `orca-restart.ts`) — they already set the env. - A raw `electron.launch()` outside those helpers must pass `ORCA_BACKGROUND_LAUNCH: '1'`. -- Call `showInactive()`, never `show()`, when an `app.evaluate()` block reveals a window. +- Do not reveal windows in explicit background or headless runs. Only an explicitly headful run + may call `showInactive()`; never call `show()` or `bringToFront()` in automated background checks. - Tag a spec `@headful` only when it needs real pixels; it still runs in the background. - `ORCA_E2E_FOREGROUND=1` is the only opt-out, for runs whose subject _is_ native focus (IME and - other OS-level key injection). Add a comment saying why. + other OS-level key injection). Clear `ORCA_BACKGROUND_LAUNCH` for that isolated run and add a + comment saying why; an explicit background request takes precedence. diff --git a/tests/e2e/cross-version-wire/cross-version-agent-session-wire.unit.test.ts b/tests/e2e/cross-version-wire/cross-version-agent-session-wire.unit.test.ts index a2a64897fcb..ecf3072b39a 100644 --- a/tests/e2e/cross-version-wire/cross-version-agent-session-wire.unit.test.ts +++ b/tests/e2e/cross-version-wire/cross-version-agent-session-wire.unit.test.ts @@ -23,7 +23,10 @@ import { setStructuredAgentSessionHost } from '../../../src/main/native-chat/age import { AgentSessionRecordStore } from '../../../src/main/runtime/agent-session-record-store' import { computeAgentSessionPayloadFingerprint } from '../../../src/shared/agent-session-mutation-envelope' import type { AgentSessionSubscribeEvent } from '../../../src/shared/agent-session-wire' -import { STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY } from '../../../src/shared/protocol-version' +import { + AGENT_SESSION_STATUS_FEED_RUNTIME_CAPABILITY, + STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY +} from '../../../src/shared/protocol-version' import { resolveBaselineReleaseRef } from './release-checkout' import { loadAgentSessionWireBuild, @@ -41,6 +44,7 @@ const WORKSPACE = 'workspace-1' const THREAD = '019fd532-7c11-7a90-b6de-4e1a2c3d5f60' const NOW = 1_800_000_000_000 const CLIENT_CAPABILITY_UPDATE_METHOD = 'runtime.clientCapabilities.update' +const STATUS_FEED_METHOD = 'agentSession.subscribeStatus' /** Every method the structured surface publishes: the host method it must reach, * and the result it must hand back. A gate that hides one method and leaks @@ -107,6 +111,12 @@ const STRUCTURED_CALLS: { // A subscription that opens with nothing to say answers with no reply at all, // so reaching the host is the only signal that the gate opened. { method: 'agentSession.subscribe', hostMethod: 'subscribe' }, + // The status feed opens with a snapshot of every session, so its first reply is the contract. + { + method: STATUS_FEED_METHOD, + hostMethod: 'subscribeStatus', + result: { type: 'snapshot', sessions: [] } + }, // Teardown runs through the runtime's subscription registry rather than the // host, so its reply is the only signal that the gate opened. { method: 'agentSession.unsubscribe', hostMethod: null, result: { unsubscribed: true } } @@ -320,6 +330,10 @@ function structuredHostStub(): Record> { readOptions: vi.fn(async () => ({ models: [], current: { model: 'gpt-live' } })), history: vi.fn(() => ({ ok: true, page: { items: [] } })), subscribe: vi.fn(() => () => undefined), + subscribeStatus: vi.fn((subscriber: { emit: (event: unknown) => void }) => { + subscriber.emit({ type: 'snapshot', sessions: [] }) + return () => undefined + }), unsubscribe: vi.fn() } } @@ -443,6 +457,14 @@ describe('cross-version structured agent sessions', () => { expect(baseline.capabilities.includes(STRUCTURED_AGENT_SESSION_RUNTIME_CAPABILITY)).toBe( baselineStructuredMethods().length > 0 ) + // The status feed is additive to a surface that already shipped, so it carries its own + // capability or a client cannot tell "host too old" from "the call failed" — and it + // would relay-retry a method_not_found forever instead of degrading once. + for (const build of [current, baseline]) { + expect(build.capabilities.includes(AGENT_SESSION_STATUS_FEED_RUNTIME_CAPABILITY)).toBe( + build.methodNames.includes(STATUS_FEED_METHOD) + ) + } // Additive surface: bumping the protocol number would strand every paired // device on this release rather than degrade one feature. expect(current.protocolVersion).toBe(baseline.protocolVersion) diff --git a/tests/e2e/helpers/electron-crashpad-cleanup.ts b/tests/e2e/helpers/electron-crashpad-cleanup.ts new file mode 100644 index 00000000000..8ec1a98b26b --- /dev/null +++ b/tests/e2e/helpers/electron-crashpad-cleanup.ts @@ -0,0 +1,46 @@ +import { execFileSync } from 'node:child_process' +import path from 'node:path' + +function ownsCrashpad(command: string, userDataDir: string): boolean { + return ( + command.includes('/chrome_crashpad_handler ') && + command.includes(` --database=${path.join(userDataDir, 'Crashpad')} `) + ) +} + +export function cleanupE2ECrashpad(userDataDir: string): void { + if (process.platform !== 'darwin') { + return + } + + // macOS reparents Crashpad before app exit; its inherited stderr can keep Playwright open. + try { + const table = execFileSync('ps', ['-axo', 'pid=,command='], { + encoding: 'utf8', + timeout: 5_000 + }) + for (const row of table.split('\n')) { + const match = row.match(/^\s*(\d+)\s+(.+)$/) + if (!match || !ownsCrashpad(match[2], userDataDir)) { + continue + } + const pid = Number(match[1]) + if (!Number.isSafeInteger(pid) || pid <= 1) { + continue + } + try { + const command = execFileSync('ps', ['-p', String(pid), '-o', 'command='], { + encoding: 'utf8', + timeout: 5_000 + }) + if (ownsCrashpad(command, userDataDir)) { + process.kill(pid, 'SIGTERM') + } + } catch { + // The test-owned reporter may already have exited. + } + } + } catch { + // Cleanup remains best-effort when process enumeration is unavailable. + } +} diff --git a/tests/e2e/helpers/electron-crashpad-cleanup.unit.test.ts b/tests/e2e/helpers/electron-crashpad-cleanup.unit.test.ts new file mode 100644 index 00000000000..cdf552e2548 --- /dev/null +++ b/tests/e2e/helpers/electron-crashpad-cleanup.unit.test.ts @@ -0,0 +1,45 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { execFileSync } from 'node:child_process' +import path from 'node:path' +import { cleanupE2ECrashpad } from './electron-crashpad-cleanup' + +vi.mock('node:child_process', () => ({ execFileSync: vi.fn() })) + +const profile = '/tmp/test profile' +const database = path.join(profile, 'Crashpad') +const reporter = `/Electron Framework/Helpers/chrome_crashpad_handler --database=${database} --annotation=prod=Electron` + +afterEach(() => vi.restoreAllMocks()) + +describe('test-owned macOS Crashpad cleanup', () => { + it('terminates only the reporter for the exact temporary profile after rechecking ownership', () => { + vi.spyOn(process, 'platform', 'get').mockReturnValue('darwin') + const kill = vi.spyOn(process, 'kill').mockReturnValue(true) + vi.mocked(execFileSync) + .mockReturnValueOnce( + `111 ${reporter}\n222 ${reporter.replace('Crashpad ', 'Crashpad-old ')}\n333 ${reporter.replace('test profile', 'another profile')}\n444 /bin/echo --database=${database} \n` + ) + .mockReturnValueOnce(reporter) + cleanupE2ECrashpad(profile) + expect(kill).toHaveBeenCalledExactlyOnceWith(111, 'SIGTERM') + expect(execFileSync).toHaveBeenLastCalledWith('ps', ['-p', '111', '-o', 'command='], { + encoding: 'utf8', + timeout: 5_000 + }) + }) + + it('does not signal a PID whose ownership changed after enumeration', () => { + vi.spyOn(process, 'platform', 'get').mockReturnValue('darwin') + const kill = vi.spyOn(process, 'kill').mockReturnValue(true) + vi.mocked(execFileSync).mockReturnValueOnce(`111 ${reporter}`).mockReturnValueOnce('/bin/sh') + cleanupE2ECrashpad(profile) + expect(kill).not.toHaveBeenCalled() + }) + + it.each(['win32', 'linux'] as const)('does not enumerate processes on %s', (platform) => { + vi.spyOn(process, 'platform', 'get').mockReturnValue(platform) + vi.mocked(execFileSync).mockClear() + cleanupE2ECrashpad(profile) + expect(execFileSync).not.toHaveBeenCalled() + }) +}) diff --git a/tests/e2e/helpers/electron-process-shutdown.ts b/tests/e2e/helpers/electron-process-shutdown.ts index 48ddb60bf43..f9b642a676e 100644 --- a/tests/e2e/helpers/electron-process-shutdown.ts +++ b/tests/e2e/helpers/electron-process-shutdown.ts @@ -2,6 +2,7 @@ import type { ChildProcess } from 'node:child_process' import { execFileSync } from 'node:child_process' import { existsSync, readFileSync, readdirSync } from 'node:fs' import path from 'node:path' +import { cleanupE2ECrashpad } from './electron-crashpad-cleanup' import type { ElectronApplication } from '@stablyai/playwright-test' const GRACEFUL_CLOSE_TIMEOUT_MS = 10_000 @@ -238,4 +239,5 @@ export async function cleanupE2EDaemons(userDataDir: string): Promise { for (const pid of readDaemonPidFiles(userDataDir)) { await forceKillPidTree(pid) } + cleanupE2ECrashpad(userDataDir) } diff --git a/tests/e2e/helpers/remote-skill-cloud-fixture.ts b/tests/e2e/helpers/remote-skill-cloud-fixture.ts index f1d28d926b7..8e1650947dd 100644 --- a/tests/e2e/helpers/remote-skill-cloud-fixture.ts +++ b/tests/e2e/helpers/remote-skill-cloud-fixture.ts @@ -8,13 +8,12 @@ import { } from '../../../src/main/skills/skill-package-creation' import { SKILL_PACKAGE_CONTENT_TYPE } from '../../../src/shared/skill-package-manifest' -export const REMOTE_SKILL_CLOUD_PORT = Number(process.env.ORCA_E2E_SKILL_CLOUD_PORT ?? '43961') -export const REMOTE_SKILL_CLOUD_ORIGIN = `http://127.0.0.1:${REMOTE_SKILL_CLOUD_PORT}` export const REMOTE_SKILL_PACKAGE_ID = 'package_remote_e2e' export const REMOTE_SKILL_VERSION_ID = 'version_remote_e2e' export const REMOTE_SKILL_NAME = 'remote-e2e-skill' export type RemoteSkillCloudFixture = { + origin: string archive: CreatedSkillPackage bytes: Buffer requests: { method: string; path: string; body: unknown }[] @@ -39,19 +38,30 @@ export async function startRemoteSkillCloudFixture(): Promise { - void handleRemoteSkillCloudRequest({ request, response, archive, bytes, requests }).catch( - (error) => { - response.writeHead(500, { 'content-type': 'application/json' }) - response.end(JSON.stringify({ code: 'fixture_failed', message: String(error) })) - } - ) + void handleRemoteSkillCloudRequest({ + request, + response, + archive, + bytes, + requests, + origin + }).catch((error) => { + response.writeHead(500, { 'content-type': 'application/json' }) + response.end(JSON.stringify({ code: 'fixture_failed', message: String(error) })) + }) }) await new Promise((resolve, reject) => { server.once('error', reject) - server.listen(REMOTE_SKILL_CLOUD_PORT, '127.0.0.1', resolve) + server.listen(Number(process.env.ORCA_E2E_SKILL_CLOUD_PORT ?? 0), '127.0.0.1', resolve) }) - return { archive, bytes, requests, root, server } + const address = server.address() + if (!address || typeof address === 'string') { + throw new Error('Skill fixture has no TCP address') + } + origin = `http://127.0.0.1:${address.port}` + return { archive, bytes, requests, root, server, origin } } export async function stopRemoteSkillCloudFixture(fixture: RemoteSkillCloudFixture): Promise { @@ -60,13 +70,14 @@ export async function stopRemoteSkillCloudFixture(fixture: RemoteSkillCloudFixtu } async function handleRemoteSkillCloudRequest(input: { + origin: string request: IncomingMessage response: ServerResponse archive: CreatedSkillPackage bytes: Buffer requests: RemoteSkillCloudFixture['requests'] }): Promise { - const path = new URL(input.request.url ?? '/', REMOTE_SKILL_CLOUD_ORIGIN).pathname + const path = new URL(input.request.url ?? '/', input.origin).pathname if (input.request.method === 'GET' && path === '/package.tar.gz') { input.requests.push({ method: 'GET', path, body: null }) input.response.writeHead(200, { @@ -84,17 +95,19 @@ async function handleRemoteSkillCloudRequest(input: { const body = JSON.parse(await readRequestBody(input.request)) as unknown input.requests.push({ method: 'POST', path, body }) input.response.writeHead(200, { 'content-type': 'application/json' }) - input.response.end(JSON.stringify(downloadGrant(input.archive, input.bytes.length))) + input.response.end( + JSON.stringify(downloadGrant(input.archive, input.bytes.length, input.origin)) + ) return } input.response.writeHead(404, { 'content-type': 'application/json' }) input.response.end(JSON.stringify({ code: 'not_found', message: 'Not found' })) } -function downloadGrant(archive: CreatedSkillPackage, compressedBytes: number) { +function downloadGrant(archive: CreatedSkillPackage, compressedBytes: number, origin: string) { return { grant: { - url: `${REMOTE_SKILL_CLOUD_ORIGIN}/package.tar.gz`, + url: `${origin}/package.tar.gz`, expiresAt: '2099-01-01T00:00:00.000Z' }, version: { diff --git a/tests/e2e/helpers/remote-skill-cloud-fixture.unit.test.ts b/tests/e2e/helpers/remote-skill-cloud-fixture.unit.test.ts new file mode 100644 index 00000000000..70291cf0e8f --- /dev/null +++ b/tests/e2e/helpers/remote-skill-cloud-fixture.unit.test.ts @@ -0,0 +1,41 @@ +import { expect, it, vi } from 'vitest' +import { + REMOTE_SKILL_PACKAGE_ID, + REMOTE_SKILL_VERSION_ID, + startRemoteSkillCloudFixture, + stopRemoteSkillCloudFixture +} from './remote-skill-cloud-fixture' + +it('serves concurrent skill fixtures from independent bound origins', async () => { + vi.stubEnv('ORCA_E2E_SKILL_CLOUD_PORT', undefined) + const results = await Promise.allSettled([ + startRemoteSkillCloudFixture(), + startRemoteSkillCloudFixture() + ]) + const fixtures = results.flatMap((result) => + result.status === 'fulfilled' ? [result.value] : [] + ) + try { + expect(results.every((result) => result.status === 'fulfilled')).toBe(true) + expect(new Set(fixtures.map((fixture) => fixture.origin)).size).toBe(2) + for (const fixture of fixtures) { + const response = await fetch( + `${fixture.origin}/v1/skill-packages/${REMOTE_SKILL_PACKAGE_ID}/versions/${REMOTE_SKILL_VERSION_ID}/download-grants`, + { + method: 'POST', + body: '{}', + headers: { 'content-type': 'application/json' } + } + ) + expect(response.status).toBe(200) + const result = (await response.json()) as { grant: { url: string } } + expect(result.grant.url).toBe(`${fixture.origin}/package.tar.gz`) + const archive = await fetch(result.grant.url) + expect(Buffer.from(await archive.arrayBuffer())).toEqual(fixture.bytes) + expect(fixture.requests).toHaveLength(2) + } + } finally { + await Promise.all(fixtures.map(stopRemoteSkillCloudFixture)) + vi.unstubAllEnvs() + } +}) diff --git a/tests/e2e/helpers/source-control-ai-generation.ts b/tests/e2e/helpers/source-control-ai-generation.ts index f6be16d7342..c93a20e37a1 100644 --- a/tests/e2e/helpers/source-control-ai-generation.ts +++ b/tests/e2e/helpers/source-control-ai-generation.ts @@ -67,7 +67,7 @@ export async function seedCreatePrComposer(page: Page): Promise<{ prWorktreePath: string primaryBranch: string }> { - return page.evaluate(async () => { + const seeded = await page.evaluate(async () => { const store = window.__store ?? (() => { @@ -101,6 +101,7 @@ export async function seedCreatePrComposer(page: Page): Promise<{ const eligibility = { provider: 'github' as const, review: null, + reviewLookupOutcome: 'not_found' as const, canCreate: true, blockedReason: null, nextAction: null, @@ -121,7 +122,7 @@ export async function seedCreatePrComposer(page: Page): Promise<{ ...current.remoteStatusesByWorktree, [prWorktree.id]: { hasUpstream: true, - upstreamName: `origin/${branch}`, + upstreamName: primaryBranch, ahead: 0, behind: 0 } @@ -130,6 +131,10 @@ export async function seedCreatePrComposer(page: Page): Promise<{ args.branch === branch ? eligibility : { ...eligibility, canCreate: false }, fetchHostedReviewForBranch: async () => null, fetchPRForBranch: async () => null, + enqueueGitHubPRRefresh: () => undefined, + // Ignore provider work queued before this generation-only fixture was installed. + getEffectiveGitHubPRRefreshState: () => undefined, + prRefreshStates: {}, fetchUpstreamStatus: async () => undefined, setUpstreamStatus: () => undefined })) @@ -141,6 +146,12 @@ export async function seedCreatePrComposer(page: Page): Promise<{ primaryBranch } }) + // Checks reads fresh Git state instead of the seeded store cache. + execFileSync('git', ['branch', '--set-upstream-to', seeded.primaryBranch], { + cwd: seeded.prWorktreePath, + stdio: 'pipe' + }) + return seeded } export async function seedCommitMessageComposer(page: Page): Promise<{ diff --git a/tests/e2e/helpers/source-control-ai-generators.ts b/tests/e2e/helpers/source-control-ai-generators.ts index be3f1b43247..8c09bb8b556 100644 --- a/tests/e2e/helpers/source-control-ai-generators.ts +++ b/tests/e2e/helpers/source-control-ai-generators.ts @@ -14,13 +14,13 @@ async function setCustomGenerator(page: Page, scriptPath: string): Promise } await store.getState().updateSettings({ activeRuntimeEnvironmentId: null, - commitMessageAi: { - ...currentSettings.commitMessageAi, + sourceControlAi: { enabled: true, agentId: 'custom' as const, selectedModelByAgent: {}, selectedThinkingByModel: {}, - customPrompt: '', + instructionsByOperation: {}, + actions: {}, customAgentCommand: `node ${JSON.stringify(scriptPath)}` } }) diff --git a/tests/e2e/helpers/ssh-recovery-input-observation.ts b/tests/e2e/helpers/ssh-recovery-input-observation.ts new file mode 100644 index 00000000000..06b4bd7de3e --- /dev/null +++ b/tests/e2e/helpers/ssh-recovery-input-observation.ts @@ -0,0 +1,53 @@ +import type { Page, TestInfo } from '@playwright/test' +import type { RuntimeTerminalListResult } from '../../../src/shared/runtime-types' + +export async function attachSshRecoveryInputObservation( + page: Page, + testInfo: TestInfo, + targetId: string, + originalPtyId: string, + label: string +): Promise { + const observation = await page.evaluate( + async ({ targetId, originalPtyId }) => { + const state = window.__store?.getState() + const panes = [...(window.__paneManagers?.entries() ?? [])].flatMap(([tabId, manager]) => + manager.getPanes().map((pane) => ({ + tabId, + leafId: pane.leafId, + ptyId: pane.container.dataset.ptyId, + active: manager.getActivePane()?.id === pane.id + })) + ) + let timer: ReturnType | undefined + try { + const runtime = await Promise.race([ + window.api.runtime + .call({ method: 'terminal.list', params: { limit: 50, includeVisualLayouts: false } }) + .then((response) => + response.ok + ? { terminals: (response.result as RuntimeTerminalListResult).terminals } + : { error: response.error } + ), + new Promise<{ error: string }>((resolve) => { + timer = setTimeout(() => resolve({ error: 'Observation timed out' }), 1000) + }) + ]) + return { + originalPtyId, + authority: state?.sshConnectionStates.get(targetId), + activeWorktreeId: state?.activeWorktreeId, + panes, + runtime + } + } finally { + clearTimeout(timer) + } + }, + { targetId, originalPtyId } + ) + await testInfo.attach(`ssh-input-${label}.json`, { + body: JSON.stringify(observation, null, 2), + contentType: 'application/json' + }) +} diff --git a/tests/e2e/live-background-terminal-mount-authority.spec.ts b/tests/e2e/live-background-terminal-mount-authority.spec.ts index ea6ffd1871a..a785454f695 100644 --- a/tests/e2e/live-background-terminal-mount-authority.spec.ts +++ b/tests/e2e/live-background-terminal-mount-authority.spec.ts @@ -23,6 +23,10 @@ import type { } from '../../src/shared/runtime-types' import { PROTOCOL_VERSION } from '../../src/main/daemon/types' import { makePaneKey } from '../../src/shared/stable-pane-id' +import { + buildFakeAgentCommandOverride, + FAKE_AGENT_WINDOWS_SHELL +} from './helpers/fake-agent-command-override' type SpawnEvent = { args: string[]; pid: number } type TerminalIdentity = Pick< @@ -70,6 +74,10 @@ if (process.platform === 'win32') { chmodSync(executable, 0o755) } +const fakeCodexCommand = buildFakeAgentCommandOverride( + path.join(fakeCliDir, process.platform === 'win32' ? 'codex.cmd' : 'codex') +) + const test = base.extend({ launchEnv: [ { @@ -535,23 +543,28 @@ test('adopts runtime-owned agent and Setup PTYs on first mount', async ({ const repoId = added.result.repo.id await expect .poll(() => - orcaPage.evaluate(async (repoId) => { - const state = window.__store?.getState() - await state?.fetchRepos() - const repo = window.__store?.getState().repos.find((candidate) => candidate.id === repoId) - if (!repo) { - return false - } - await window.__store?.getState().updateRepo(repoId, { - hookSettings: { ...repo.hookSettings, setupAgentStartupPolicy: 'start-immediately' } - }) - await window.__store?.getState().updateSettings({ - disabledTuiAgents: [], - setupScriptLaunchMode: 'new-tab', - terminalHiddenViewParking: false - }) - return true - }, repoId) + orcaPage.evaluate( + async ({ repoId, command, windowsShell }) => { + const state = window.__store?.getState() + await state?.fetchRepos() + const repo = window.__store?.getState().repos.find((candidate) => candidate.id === repoId) + if (!repo) { + return false + } + await window.__store?.getState().updateRepo(repoId, { + hookSettings: { ...repo.hookSettings, setupAgentStartupPolicy: 'start-immediately' } + }) + await window.__store?.getState().updateSettings({ + agentCmdOverrides: { codex: command }, + terminalWindowsShell: windowsShell, + disabledTuiAgents: [], + setupScriptLaunchMode: 'new-tab', + terminalHiddenViewParking: false + }) + return true + }, + { repoId, command: fakeCodexCommand, windowsShell: FAKE_AGENT_WINDOWS_SHELL } + ) ) .toBe(true) diff --git a/tests/e2e/paired-remote-terminal-host-restart-background-sync.spec.ts b/tests/e2e/paired-remote-terminal-host-restart-background-sync.spec.ts index 9855d8d92b0..cf2f96e9c84 100644 --- a/tests/e2e/paired-remote-terminal-host-restart-background-sync.spec.ts +++ b/tests/e2e/paired-remote-terminal-host-restart-background-sync.spec.ts @@ -283,6 +283,24 @@ async function expectTerminalInteractive( } async function moveHostAwayFromWorktree(page: Page, targetWorktreeId: string): Promise { + await expect + .poll( + () => + page.evaluate(async (targetId) => { + const state = window.__store?.getState() + const target = state?.allWorktrees().find((worktree) => worktree.id === targetId) + if (!state || !target) { + return false + } + await state.fetchWorktrees(target.repoId) + return window + .__store!.getState() + .allWorktrees() + .some((worktree) => worktree.repoId === target.repoId && worktree.id !== targetId) + }, targetWorktreeId), + { message: 'Seeded alternate host worktree never loaded' } + ) + .toBe(true) const alternateWorktreeId = await page.evaluate((targetId) => { const state = window.__store?.getState() const alternate = state?.allWorktrees().find((worktree) => worktree.id !== targetId) @@ -423,6 +441,9 @@ test('foregrounds a preserved daemon PTY after the paired host relaunches', asyn expect(reconnectControl.ptyId).not.toBe(target.ptyId) await openClientTab(client.page, worktreeId, reconnectControl.webTabId) await waitForPaneConnected(client.page, reconnectControl.webTabId) + await expect + .poll(() => readPaneContent(client!.page, reconnectControl.webTabId), { timeout: 30_000 }) + .toContain('READY') await expectTerminalInteractive(client, reconnectControl, 'y') } finally { if (client) { diff --git a/tests/e2e/paired-skill-installation.spec.ts b/tests/e2e/paired-skill-installation.spec.ts index ac31a81e2f1..0268dbb84a3 100644 --- a/tests/e2e/paired-skill-installation.spec.ts +++ b/tests/e2e/paired-skill-installation.spec.ts @@ -14,7 +14,6 @@ import { type HeadlessPairedRuntimeHost } from './helpers/headless-paired-runtime-host' import { - REMOTE_SKILL_CLOUD_ORIGIN, REMOTE_SKILL_NAME, REMOTE_SKILL_PACKAGE_ID, REMOTE_SKILL_VERSION_ID, @@ -119,13 +118,14 @@ test('installs on a headless serve runtime through the same contract', async ({ }) function cloudClientEnvironment(): Record { + const { origin } = requireCloudFixture() return { - ORCA_ARTIFACTS_API_URL: REMOTE_SKILL_CLOUD_ORIGIN, - ORCA_CLOUD_API_URL: REMOTE_SKILL_CLOUD_ORIGIN, + ORCA_ARTIFACTS_API_URL: origin, + ORCA_CLOUD_API_URL: origin, ORCA_CLOUD_CLIENT_ID: 'skills-e2e-client', ORCA_CLOUD_DEV_AUTH: '1', ORCA_CLOUD_ALLOW_PLAINTEXT_SESSION: '1', - ORCA_SKILL_PACKAGE_DOWNLOAD_ORIGINS: REMOTE_SKILL_CLOUD_ORIGIN + ORCA_SKILL_PACKAGE_DOWNLOAD_ORIGINS: origin } } diff --git a/tests/e2e/restart-restore-terminal-input.spec.ts b/tests/e2e/restart-restore-terminal-input.spec.ts index 1ceed4254c5..79528fabffe 100644 --- a/tests/e2e/restart-restore-terminal-input.spec.ts +++ b/tests/e2e/restart-restore-terminal-input.spec.ts @@ -239,7 +239,6 @@ test('restored pane recovers input after the daemon un-wedges', async (// oxlint const second = await session.launch() secondApp = second.app - await settleRestoredLaunch(second.page) // Field-fidelity check, not a hard gate: does the pane paint restored // content while its PTY attach cannot complete? That visible-but-dead @@ -258,6 +257,8 @@ test('restored pane recovers input after the daemon un-wedges', async (// oxlint } stoppedDaemonPid = null + // Session readiness requires a daemon response; resume it before waiting for restoration. + await settleRestoredLaunch(second.page) await expectRestoredPaneAcceptsInput( second.page, `daemon wedged during relaunch (painted while wedged: ${paintedWhileWedged}, ` + diff --git a/tests/e2e/right-sidebar-windows-titlebar.spec.ts b/tests/e2e/right-sidebar-windows-titlebar.spec.ts index 1d6d4b8981f..ce39d7de28d 100644 --- a/tests/e2e/right-sidebar-windows-titlebar.spec.ts +++ b/tests/e2e/right-sidebar-windows-titlebar.spec.ts @@ -6,41 +6,19 @@ type RightSidebarHeaderGeometry = { stripTop: number closeTop: number titlebarActivityButtonCount: number + activityButtonCount: number firstButtonCenterHitsFirst: boolean lastButtonCenterHitsLast: boolean } -test.describe('Right sidebar Windows titlebar spacing', () => { - test('top activity buttons render inside the sidebar instead of the titlebar', async ({ - orcaPage - }) => { - await orcaPage.addInitScript(() => { - const userAgent = - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/146 Safari/537.36' - Object.defineProperty(navigator, 'userAgent', { - get: () => userAgent, - configurable: true - }) - }) - await orcaPage.reload({ waitUntil: 'domcontentloaded' }) - await orcaPage.waitForFunction(() => Boolean(window.__store), null, { timeout: 30_000 }) +test.describe('Right sidebar native titlebar spacing', () => { + test('top activity buttons follow the native desktop chrome layout', async ({ orcaPage }) => { await waitForSessionReady(orcaPage) await waitForActiveWorktree(orcaPage) await ensureTerminalVisible(orcaPage) - await expect - .poll( - async () => - orcaPage.evaluate(() => ({ - hasWindowsUserAgent: navigator.userAgent.includes('Windows'), - hasWindowsTitlebarChrome: Boolean(document.querySelector('.window-controls')) - })), - { - timeout: 5_000, - message: 'Renderer did not switch to the Windows titlebar branch' - } - ) - .toEqual({ hasWindowsUserAgent: true, hasWindowsTitlebarChrome: true }) + const hasDesktopWindowChrome = process.platform !== 'darwin' + expect(await orcaPage.evaluate(() => window.api.platform.get().platform)).toBe(process.platform) await orcaPage.evaluate(() => { const store = window.__store @@ -95,6 +73,7 @@ test.describe('Right sidebar Windows titlebar spacing', () => { stripTop: stripRect.top, closeTop: closeRect.top, titlebarActivityButtonCount, + activityButtonCount: activityButtons.length, firstButtonCenterHitsFirst: elementAtFirstCenter !== null && firstButton.contains(elementAtFirstCenter), lastButtonCenterHitsLast: @@ -117,8 +96,13 @@ test.describe('Right sidebar Windows titlebar spacing', () => { .toBe(true) expect(headerGeometry).not.toBeNull() - expect(headerGeometry!.titlebarActivityButtonCount).toBe(0) - expect(headerGeometry!.stripTop).toBeGreaterThanOrEqual(headerGeometry!.headerBottom) + if (hasDesktopWindowChrome) { + expect(headerGeometry!.titlebarActivityButtonCount).toBe(0) + expect(headerGeometry!.stripTop).toBeGreaterThanOrEqual(headerGeometry!.headerBottom) + } else { + expect(headerGeometry!.titlebarActivityButtonCount).toBe(headerGeometry!.activityButtonCount) + expect(headerGeometry!.stripTop).toBeLessThan(headerGeometry!.headerBottom) + } expect(headerGeometry!.closeTop).toBeLessThan(headerGeometry!.headerBottom) expect(headerGeometry!.firstButtonCenterHitsFirst).toBe(true) expect(headerGeometry!.lastButtonCenterHitsLast).toBe(true) diff --git a/tests/e2e/settings-agent-awake.spec.ts b/tests/e2e/settings-agent-awake.spec.ts index 8a2ad840a14..ebea82a1241 100644 --- a/tests/e2e/settings-agent-awake.spec.ts +++ b/tests/e2e/settings-agent-awake.spec.ts @@ -1,4 +1,5 @@ import { randomUUID } from 'node:crypto' +import { runProcess } from '../../src/shared/child-process/run-process' import type { ElectronApplication, Page } from '@stablyai/playwright-test' import { test, expect } from './helpers/orca-app' import { waitForSessionReady } from './helpers/store' @@ -104,6 +105,19 @@ async function readPowerSaveBlockerProbe( }) } +async function readMacosSleepAssertionPids(electronApp: ElectronApplication): Promise { + const result = await runProcess({ + program: '/usr/bin/pgrep', + args: ['-P', String(electronApp.process().pid), '-f', '^/usr/bin/caffeinate -i -s$'], + maxOutputBytes: 4_096 + }) + if (result.code === 1) { + return [] + } + expect(result.code, result.stderr).toBe(0) + return result.stdout.trim().split(/\s+/).filter(Boolean).map(Number) +} + async function postCodexHookEvent( electronApp: ElectronApplication, options: { @@ -176,7 +190,9 @@ test.describe('Agent awake setting', () => { electronApp, orcaPage }) => { - await installPowerSaveBlockerProbe(electronApp) + if (process.platform !== 'darwin') { + await installPowerSaveBlockerProbe(electronApp) + } await setKeepAwake(orcaPage, true) const tabId = 'e2e-awake-tab' @@ -187,24 +203,33 @@ test.describe('Agent awake setting', () => { eventName: 'UserPromptSubmit' }) - await expect - .poll(async () => await readPowerSaveBlockerProbe(electronApp), { - timeout: 5_000, - message: 'powerSaveBlocker did not start for the working agent' - }) - .toEqual( - expect.objectContaining({ - activeIds: expect.arrayContaining([expect.any(Number)]), - starts: expect.arrayContaining([ - expect.objectContaining({ type: 'prevent-display-sleep' }) - ]) + await expect( + orcaPage.getByRole('button', { name: 'Keep computer awake, Agent · Active' }) + ).toBeVisible() + let startedIds: number[] = [] + if (process.platform === 'darwin') { + // macOS uses an app-owned caffeinate assertion instead of Electron's display blocker. + await expect + .poll(() => readMacosSleepAssertionPids(electronApp), { timeout: 5_000 }) + .not.toEqual([]) + } else { + await expect + .poll(async () => await readPowerSaveBlockerProbe(electronApp), { + timeout: 5_000, + message: 'powerSaveBlocker did not start for the working agent' }) - ) + .toEqual( + expect.objectContaining({ + activeIds: expect.arrayContaining([expect.any(Number)]), + starts: expect.arrayContaining([ + expect.objectContaining({ type: 'prevent-display-sleep' }) + ]) + }) + ) - const startedIds = (await readPowerSaveBlockerProbe(electronApp)).starts.map( - (start) => start.id - ) - expect(startedIds.length).toBeGreaterThan(0) + startedIds = (await readPowerSaveBlockerProbe(electronApp)).starts.map((start) => start.id) + expect(startedIds.length).toBeGreaterThan(0) + } await postCodexHookEvent(electronApp, { paneKey, @@ -212,6 +237,15 @@ test.describe('Agent awake setting', () => { eventName: 'Stop' }) + await expect( + orcaPage.getByRole('button', { name: 'Keep computer awake, Agent · Inactive' }) + ).toBeVisible() + if (process.platform === 'darwin') { + await expect + .poll(() => readMacosSleepAssertionPids(electronApp), { timeout: 5_000 }) + .toEqual([]) + return + } await expect .poll(async () => await readPowerSaveBlockerProbe(electronApp), { timeout: 5_000, diff --git a/tests/e2e/setup-script-import.spec.ts b/tests/e2e/setup-script-import.spec.ts index 180b34f85aa..340258c884b 100644 --- a/tests/e2e/setup-script-import.spec.ts +++ b/tests/e2e/setup-script-import.spec.ts @@ -1,5 +1,5 @@ import { execFileSync } from 'node:child_process' -import { mkdirSync, rmSync, writeFileSync } from 'node:fs' +import { mkdirSync, realpathSync, rmSync, writeFileSync } from 'node:fs' import path from 'node:path' import type { Locator, Page } from '@stablyai/playwright-test' import { test, expect } from './helpers/orca-app' @@ -108,7 +108,7 @@ async function addAndActivateRepo(page: Page, repoPath: string): Promise state.setActiveWorktree(worktree.id) state.setSidebarOpen(true) return addedRepo.id - }, repoPath) + }, realpathSync.native(repoPath)) } async function openRepoSettings(page: Page, repoId: string): Promise { diff --git a/tests/e2e/setup-script-prompt-unreadable-orca-yaml.spec.ts b/tests/e2e/setup-script-prompt-unreadable-orca-yaml.spec.ts index 151a4b02bbc..199694b961a 100644 --- a/tests/e2e/setup-script-prompt-unreadable-orca-yaml.spec.ts +++ b/tests/e2e/setup-script-prompt-unreadable-orca-yaml.spec.ts @@ -1,5 +1,5 @@ import { execFileSync } from 'node:child_process' -import { mkdirSync, rmSync, writeFileSync } from 'node:fs' +import { mkdirSync, realpathSync, rmSync, writeFileSync } from 'node:fs' import path from 'node:path' import type { ElectronApplication, Page } from '@stablyai/playwright-test' import { test, expect } from './helpers/orca-app' @@ -112,17 +112,10 @@ async function addRepoAndActivateMainWorktree( if (!store) { throw new Error('window.__store is not available') } - const normalize = (value: string): string => - value.startsWith('/private/var/') ? value.slice('/private'.length) : value - const state = store.getState() const worktrees = state.worktreesByRepo[targetRepoId] ?? [] - const mainWorktree = worktrees.find( - (entry) => normalize(entry.path) === normalize(targetRepoPath) - ) - const featureWorktree = worktrees.find( - (entry) => normalize(entry.path) === normalize(targetFeaturePath) - ) + const mainWorktree = worktrees.find((entry) => entry.path === targetRepoPath) + const featureWorktree = worktrees.find((entry) => entry.path === targetFeaturePath) if (!mainWorktree || !featureWorktree) { throw new Error( `Missing worktrees for ${targetRepoPath}: ${worktrees.map((entry) => entry.path).join(', ')}` @@ -145,7 +138,11 @@ async function addRepoAndActivateMainWorktree( featureWorktreeId: featureWorktree.id } }, - { targetRepoId: repoId, targetRepoPath: repoPath, targetFeaturePath: featureWorktreePath } + { + targetRepoId: repoId, + targetRepoPath: realpathSync.native(repoPath), + targetFeaturePath: realpathSync.native(featureWorktreePath) + } ) } diff --git a/tests/e2e/ssh-codex-display-artifacts-repro.spec.ts b/tests/e2e/ssh-codex-display-artifacts-repro.spec.ts index e19a090df4d..f4c02d04c94 100644 --- a/tests/e2e/ssh-codex-display-artifacts-repro.spec.ts +++ b/tests/e2e/ssh-codex-display-artifacts-repro.spec.ts @@ -48,7 +48,7 @@ import { resetWebglAndCaptureGraySlabAnalysis } from './terminal-webgl-reset-cap const RUN_DOCKER_SSH = process.env.ORCA_E2E_SSH_DOCKER === '1' const RUN_REAL_REMOTE_CODEX = process.env.ORCA_E2E_REAL_REMOTE_CODEX === '1' -const EXPECT_NO_ARTIFACTS = process.env.ORCA_E2E_EXPECT_NO_CODEX_ARTIFACTS === '1' +const EXPECT_NO_ARTIFACTS = process.env.ORCA_E2E_EXPECT_NO_CODEX_ARTIFACTS !== '0' const CAPTURE_WHILE_REMOTE_TUI_RUNNING = process.env.ORCA_E2E_CAPTURE_WHILE_REMOTE_TUI_RUNNING === '1' const HIDE_UNTIL_REMOTE_TUI_DONE = process.env.ORCA_E2E_HIDE_UNTIL_REMOTE_TUI_DONE === '1' diff --git a/tests/e2e/ssh-codex-repro-remote-fixtures.ts b/tests/e2e/ssh-codex-repro-remote-fixtures.ts index 3ee48187e7c..14597bc084a 100644 --- a/tests/e2e/ssh-codex-repro-remote-fixtures.ts +++ b/tests/e2e/ssh-codex-repro-remote-fixtures.ts @@ -135,7 +135,7 @@ async function insertCodexHistory(frame) { const phase = String(frame).padStart(4, '0') + '.' + index await write('\\r\\n') await write(\`\\x1b[48;2;72;72;72m\\x1b[K\`) - await write(\`\\x1b[38;2;220;220;220;48;2;72;72;72m\${pad('gpt-5.5 high · ~/code/pr-12250-migration-compare-move-baseprice-claim · /ps to view · /stop to close ' + phase, width)}\\x1b[0m\`) + await write(\`\\x1b[38;2;220;220;220;48;2;72;72;72m\${pad('gpt-5.5 high · ' + phase + ' · ~/code/pr-12250-migration-compare-move-baseprice-claim · /ps to view · /stop to close', width)}\\x1b[0m\`) } await write('\\x1b[r') await write(\`\\x1b[\${viewportBottom};1H\`) @@ -176,7 +176,7 @@ for (let frame = 0; frame < ${REMOTE_CODEX_FIXTURE_FRAMES}; frame += 1) { await reverseIndexCodexHistory(frame) } if (frame % 9 === 0) { - await grayScrollLine(\`gpt-5.5 high · ~/code/pr-12250-migration-compare-move-baseprice-claim · /ps to view · /stop to close \${frame}\`) + await grayScrollLine(\`gpt-5.5 high · \${frame} · ~/code/pr-12250-migration-compare-move-baseprice-claim · /ps to view · /stop to close\`) } await sleep(${REMOTE_CODEX_FIXTURE_FRAME_DELAY_MS}) } diff --git a/tests/e2e/ssh-docker-transport-drop-recovery.spec.ts b/tests/e2e/ssh-docker-transport-drop-recovery.spec.ts index 42ac316790c..c64761ede80 100644 --- a/tests/e2e/ssh-docker-transport-drop-recovery.spec.ts +++ b/tests/e2e/ssh-docker-transport-drop-recovery.spec.ts @@ -4,6 +4,7 @@ import type { ElectronApplication } from '@playwright/test' import { test, expect } from './helpers/orca-app' import { DEFAULT_LOCAL_ORCA_PROFILE_ID } from '../../src/shared/orca-profiles' import { sshRemotePtyLeaseAllowsReattach, type SshRemotePtyLease } from '../../src/shared/ssh-types' +import { toRelaySshPtyId } from '../../src/shared/ssh-pty-id' import { ensureTerminalVisible, waitForActiveWorktree, waitForSessionReady } from './helpers/store' import { execInTerminal, @@ -29,6 +30,8 @@ import { withStalledDockerSshRelayTarget } from './helpers/docker-ssh-relay-faults' +import { attachSshRecoveryInputObservation } from './helpers/ssh-recovery-input-observation' + const RUN_DOCKER_SSH = process.env.ORCA_E2E_SSH_DOCKER === '1' /** @@ -338,16 +341,21 @@ test.describe('SSH transport drop recovery', () => { const generations: string[][] = [] for (let generation = 1; generation <= 5; generation++) { - const predecessor = await waitForActivePanePtyId(orcaPage, 60_000) + const previousPtyId = await waitForActivePanePtyId(orcaPage, 60_000) await recoverDockerSshRelayAfterFault(orcaPage, remote.targetId, () => { - expect(killDockerSshRelayDaemon(target!)).toBeGreaterThan(0) + expect( + killDockerSshRelayDaemon(target!), + 'no relay process was found to kill' + ).toBeGreaterThan(0) }) - await expect - .poll(() => waitForActivePanePtyId(orcaPage, 60_000), { timeout: 120_000 }) - .not.toBe(predecessor) await waitForActiveTerminalManager(orcaPage, 120_000) - // The pane must be usable again before the count is meaningful: recovery is what mints the - // successor lease that retires the generation before it. + // Transport status can still be connected while the pane retains its old binding. + await expect + .poll(() => waitForActivePanePtyId(orcaPage, 60_000).catch(() => previousPtyId), { + timeout: 120_000, + message: `pane kept its old PTY binding after relay kill ${generation}` + }) + .not.toBe(previousPtyId) const ptyId = await waitForActivePanePtyId(orcaPage, 120_000) const markerSuffix = `${generation}_${Date.now()}` const marker = `LEASE_GEN_${markerSuffix}` @@ -356,15 +364,14 @@ test.describe('SSH transport drop recovery', () => { try { await expect - .poll(() => readReattachablePtyIds(userDataDir, remote.targetId).length, { + .poll(() => readReattachablePtyIds(userDataDir, remote.targetId), { timeout: 60_000 }) - .toBe(1) + .toEqual([toRelaySshPtyId(remote.targetId, ptyId)]) } catch (error) { - // Why re-thrown with the rows: the count alone cannot say WHICH predecessor stayed - // reattachable, and the user-data dir is torn down before the report is read. + // Preserve lease ownership diagnostics before the user-data directory is removed. throw new Error( - `reattachable lease count never settled at 1 in generation ${generation}; leases: ${describeSshLeases(userDataDir, remote.targetId)}`, + `reattachable leases never settled at the active PTY ${ptyId} in generation ${generation}; leases: ${describeSshLeases(userDataDir, remote.targetId)}`, { cause: error } ) } @@ -432,6 +439,7 @@ test.describe('SSH transport drop recovery', () => { test('accepts input again after a frozen host resumes', async ({ orcaPage }, testInfo) => { test.slow() let target: DockerSshRelayTarget | null = null + let observationTarget: { targetId: string; ptyId: string } | undefined try { target = startDockerSshRelayTarget(testInfo) enableDockerSshRelayTargetShellTitle(target) @@ -444,6 +452,18 @@ test.describe('SSH transport drop recovery', () => { await waitForActiveTerminalManager(orcaPage, 60_000) const ptyId = await waitForActivePanePtyId(orcaPage, 60_000) + observationTarget = { targetId: remote.targetId, ptyId } + const beforeSuffix = Date.now() + await execInTerminal(orcaPage, ptyId, `printf 'STALL_BEFORE_%s\\n' ${beforeSuffix}`) + await waitForTerminalOutput(orcaPage, `STALL_BEFORE_${beforeSuffix}`, 60_000) + await attachSshRecoveryInputObservation( + orcaPage, + testInfo, + remote.targetId, + ptyId, + 'before-freeze' + ) + await recoverDockerSshRelayAfterFault(orcaPage, remote.targetId, async () => { await withStalledDockerSshRelayTarget(target!, async () => { await orcaPage.waitForTimeout(30_000) @@ -454,7 +474,25 @@ test.describe('SSH transport drop recovery', () => { const afterSuffix = Date.now() const afterMarker = `STALL_AFTER_${afterSuffix}` await execInTerminal(orcaPage, ptyId, `printf 'STALL_AFTER_%s\\n' ${afterSuffix}`) + await attachSshRecoveryInputObservation( + orcaPage, + testInfo, + remote.targetId, + ptyId, + 'after-write' + ) await waitForTerminalOutput(orcaPage, afterMarker, 60_000) + } catch (error) { + if (observationTarget) { + await attachSshRecoveryInputObservation( + orcaPage, + testInfo, + observationTarget.targetId, + observationTarget.ptyId, + 'failure-before-cleanup' + ).catch(() => undefined) + } + throw error } finally { if (target) { clearDockerSshRelayFaults(target) diff --git a/tests/e2e/ssh-skill-installation.spec.ts b/tests/e2e/ssh-skill-installation.spec.ts index a102478fb62..794883b2cd1 100644 --- a/tests/e2e/ssh-skill-installation.spec.ts +++ b/tests/e2e/ssh-skill-installation.spec.ts @@ -10,7 +10,6 @@ import { import { connectDockerSshRelayTarget } from './helpers/docker-ssh-relay-connection' import { waitForActiveWorktree, waitForSessionReady } from './helpers/store' import { - REMOTE_SKILL_CLOUD_ORIGIN, REMOTE_SKILL_NAME, REMOTE_SKILL_PACKAGE_ID, REMOTE_SKILL_VERSION_ID, @@ -25,13 +24,19 @@ const REMOTE_FOLDER = '/tmp/orca-skill-folder-workspace' let cloud: RemoteSkillCloudFixture | null = null test.use({ - orcaAppExtraEnv: { - ORCA_ARTIFACTS_API_URL: REMOTE_SKILL_CLOUD_ORIGIN, - ORCA_CLOUD_API_URL: REMOTE_SKILL_CLOUD_ORIGIN, - ORCA_CLOUD_CLIENT_ID: 'skills-e2e-client', - ORCA_CLOUD_DEV_AUTH: '1', - ORCA_CLOUD_ALLOW_PLAINTEXT_SESSION: '1', - ORCA_SKILL_PACKAGE_DOWNLOAD_ORIGINS: REMOTE_SKILL_CLOUD_ORIGIN + // oxlint-disable-next-line no-empty-pattern -- The server starts in beforeAll before this test fixture runs. + orcaAppExtraEnv: async ({}, provideEnv) => { + if (!cloud) { + throw new Error('Skill cloud fixture unavailable') + } + await provideEnv({ + ORCA_ARTIFACTS_API_URL: cloud.origin, + ORCA_CLOUD_API_URL: cloud.origin, + ORCA_CLOUD_CLIENT_ID: 'skills-e2e-client', + ORCA_CLOUD_DEV_AUTH: '1', + ORCA_CLOUD_ALLOW_PLAINTEXT_SESSION: '1', + ORCA_SKILL_PACKAGE_DOWNLOAD_ORIGINS: cloud.origin + }) } }) diff --git a/tests/e2e/tab-rename.spec.ts b/tests/e2e/tab-rename.spec.ts index 6e7f0a7fdc1..30cdb9175a9 100644 --- a/tests/e2e/tab-rename.spec.ts +++ b/tests/e2e/tab-rename.spec.ts @@ -126,7 +126,7 @@ test.describe('Tab Rename (Inline)', () => { expect(originalTitle.length).toBeGreaterThan(0) await tabLocatorByTitle(orcaPage, originalTitle).click({ button: 'right' }) - await orcaPage.getByRole('menuitem', { name: 'Change Title', exact: true }).click() + await orcaPage.getByRole('menuitem', { name: /^Change Title(?:\s|$)/ }).click() const renameInput = orcaPage.getByRole('textbox', { name: `Rename tab ${originalTitle}`, diff --git a/tests/e2e/terminal-codex-home.spec.ts b/tests/e2e/terminal-codex-home.spec.ts index 1f85a4f4c9b..3364152d38c 100644 --- a/tests/e2e/terminal-codex-home.spec.ts +++ b/tests/e2e/terminal-codex-home.spec.ts @@ -1,3 +1,5 @@ +import { mkdirSync, writeFileSync } from 'node:fs' +import path from 'node:path' import { test, expect } from './helpers/orca-app' import { execInTerminal, @@ -27,7 +29,42 @@ test.describe('Terminal Codex runtime home', () => { await ensureTerminalVisible(orcaPage) }) - test('terminal process receives the Orca-managed Codex home', async ({ orcaPage }) => { + test('terminal process receives the selected account Codex home', async ({ + electronApp, + orcaPage + }) => { + const userData = await electronApp.evaluate(({ app }) => app.getPath('userData')) + const accountId = 'e2e-terminal-home' + const managedHomePath = path.join(userData, 'codex-accounts', accountId, 'home') + mkdirSync(managedHomePath, { recursive: true }) + writeFileSync(path.join(managedHomePath, '.orca-managed-home'), `${accountId}\n`) + writeFileSync( + path.join(managedHomePath, 'auth.json'), + JSON.stringify({ OPENAI_API_KEY: 'e2e-placeholder' }) + ) + await orcaPage.evaluate( + async ({ accountId, managedHomePath }) => { + const state = window.__store!.getState() + await state.updateSettings({ + codexManagedAccounts: [ + { + id: accountId, + email: 'terminal-home@example.invalid', + managedHomePath, + createdAt: 1, + updatedAt: 1, + lastAuthenticatedAt: 1 + } + ], + activeCodexManagedAccountId: accountId, + activeCodexManagedAccountIdsByRuntime: { host: accountId, wsl: {} } + }) + const tab = state.createTab(state.activeWorktreeId!) + state.setActiveTab(tab.id) + state.setActiveTabType('terminal') + }, + { accountId, managedHomePath } + ) await waitForActiveTerminalManager(orcaPage) const ptyId = await waitForActivePanePtyId(orcaPage) const marker = `__ORCA_CODEX_HOME_E2E_${Date.now()}__` @@ -43,17 +80,10 @@ test.describe('Terminal Codex runtime home', () => { .poll( async () => { probe = readCodexHomeProbe(await getTerminalContent(orcaPage), marker) - return Boolean( - probe?.codexHome && - probe.orcaCodexHome && - probe.codexHome === probe.orcaCodexHome && - /[\\/]codex-runtime-home[\\/]home$/.test(probe.codexHome) - ) + return probe }, - { timeout: 15_000, message: 'Terminal did not expose Orca-managed Codex home env' } + { timeout: 15_000, message: 'Terminal did not expose the selected Codex account home' } ) - .toBe(true) - - expect(probe?.codexHome).toBe(probe?.orcaCodexHome) + .toEqual({ codexHome: managedHomePath, orcaCodexHome: managedHomePath }) }) }) diff --git a/tests/e2e/terminal-scroll-intent-follow.spec.ts b/tests/e2e/terminal-scroll-intent-follow.spec.ts index c4d8b171fed..ae2dfa15466 100644 --- a/tests/e2e/terminal-scroll-intent-follow.spec.ts +++ b/tests/e2e/terminal-scroll-intent-follow.spec.ts @@ -168,6 +168,7 @@ async function injectQueuedWriteThenType(page: Page, paneKey: string): Promise { const injectionTarget = window as Window & { __terminalPtyDataInjection?: { inject: (paneKey: string, data: string) => boolean } + __releaseScrollIntentTestWrite?: () => void } const state = window.__store?.getState() const worktreeId = state?.activeWorktreeId @@ -184,38 +185,44 @@ async function injectQueuedWriteThenType(page: Page, paneKey: string): Promise void } | null } = { write: null } + const heldWrites: { data: string; callback?: () => void }[] = [] terminal.write = ((data: string, callback?: () => void) => { - holder.write = { data, callback } + heldWrites.push({ data, callback }) }) as typeof terminal.write + injectionTarget.__releaseScrollIntentTestWrite = () => { + terminal.write = originalWrite + delete injectionTarget.__releaseScrollIntentTestWrite + for (const held of heldWrites) { + originalWrite.call(terminal, held.data, held.callback) + } + } try { const payload = '\x1b[?2026h\r\x1b[2KWorking in-flight\x1b[?2026l' if (!injectionTarget.__terminalPtyDataInjection?.inject(targetPaneKey, payload)) { throw new Error('PTY injector unavailable') } + if (heldWrites.length === 0) { + throw new Error('Foreground terminal write was not captured') + } const textarea = pane.container.querySelector('.xterm-helper-textarea') if (!textarea) { throw new Error('xterm helper textarea unavailable') } textarea.focus() - const event = new KeyboardEvent('keydown', { - bubbles: true, - cancelable: true, - key: 'x', - code: 'KeyX' - }) - Object.defineProperty(event, 'keyCode', { configurable: true, value: 88 }) - Object.defineProperty(event, 'which', { configurable: true, value: 88 }) - textarea.dispatchEvent(event) - } finally { - terminal.write = originalWrite + } catch (error) { + injectionTarget.__releaseScrollIntentTestWrite() + throw error } - const heldWrite = holder.write - if (!heldWrite) { - throw new Error('Foreground terminal write was not captured') - } - originalWrite.call(terminal, heldWrite.data, heldWrite.callback) }, paneKey) + try { + await page.keyboard.press('x') + } finally { + await page.evaluate(() => { + ;( + window as Window & { __releaseScrollIntentTestWrite?: () => void } + ).__releaseScrollIntentTestWrite?.() + }) + } } async function startStreamingFixturePhase1(page: Page): Promise { @@ -308,5 +315,6 @@ test.describe('terminal scroll intent keeps following output', () => { { timeout: 5_000, intervals: [25] } ) .toBe(0) + await waitForMarkerAtBottom(orcaPage, 'STREAM_PHASE2_DONE') }) }) diff --git a/tests/e2e/terminal-send-agent-prompt-submit.spec.ts b/tests/e2e/terminal-send-agent-prompt-submit.spec.ts index 780f6c50710..35742e27f94 100644 --- a/tests/e2e/terminal-send-agent-prompt-submit.spec.ts +++ b/tests/e2e/terminal-send-agent-prompt-submit.spec.ts @@ -58,6 +58,7 @@ async function createFakeCodexTerminal( if (!worktree) { throw new Error(`runtime did not register ${testRepoPath}`) } + rmSync(fixtureReport, { force: true }) const created = await client.call<{ terminal: { handle: string } }>('terminal.create', { worktree: `id:${worktree.id}`, command: [fakeCodexCommand, ...args].join(' '), diff --git a/tests/e2e/worktree-scroll-to-current.spec.ts b/tests/e2e/worktree-scroll-to-current.spec.ts index 19c51005cfe..d61d96847a0 100644 --- a/tests/e2e/worktree-scroll-to-current.spec.ts +++ b/tests/e2e/worktree-scroll-to-current.spec.ts @@ -39,22 +39,30 @@ test.describe('Reveal active workspace button', () => { // the "outside the virtualized window" test below. test('clears sidebar filters before revealing a hidden current workspace', async ({ - orcaPage + orcaPage, + testRepoPath }) => { await prepareSidebarForScrollTest(orcaPage) - const renderedOptions = orcaPage.locator('[data-worktree-sidebar] [role="option"]') - await expect(renderedOptions).toHaveCount(2) - - const targetId = await renderedOptions.last().getAttribute('data-worktree-id') + // Other specs can add worktrees to the shared repository before this test runs. + const targetId = await orcaPage.evaluate((repoPath) => { + const state = window.__store!.getState() + const repo = state.repos.find((candidate) => candidate.path === repoPath) + return repo + ? state.worktreesByRepo[repo.id]?.find( + (worktree) => worktree.branch === 'refs/heads/e2e-secondary' + )?.id + : undefined + }, testRepoPath) if (!targetId) { - throw new Error('Bottom workspace row did not expose a data-worktree-id') + throw new Error('Seeded secondary worktree is missing') } const targetRows = orcaPage.locator( `[data-worktree-sidebar] [data-worktree-id=${JSON.stringify(targetId)}]` ) const targetRow = targetRows.first() + await expect(targetRows.and(orcaPage.getByRole('option'))).toHaveCount(1) const revealButton = orcaPage.getByRole('button', { name: 'Reveal active workspace' }) await orcaPage.evaluate((targetId) => { @@ -92,6 +100,10 @@ test.describe('Reveal active workspace button', () => { // contract under test is that reveal clears the filter (asserted below). await revealButton.click() + await orcaPage + .getByRole('dialog', { name: 'Reveal hidden workspace?' }) + .getByRole('button', { name: 'Clear filters and reveal' }) + .click() await expect(targetRow).toBeVisible() await expect(targetRow).toHaveAttribute('data-scroll-reveal-highlight', 'true') diff --git a/tests/tools/repro-terminal-send-submit.mjs b/tests/tools/repro-terminal-send-submit.mjs index df23161c8f9..e2179123e6f 100644 --- a/tests/tools/repro-terminal-send-submit.mjs +++ b/tests/tools/repro-terminal-send-submit.mjs @@ -186,7 +186,9 @@ async function parentMain() { const expectBlocked = hasFlag('expect-blocked') const providedHandle = argValue('terminal') await mkdir(tempDir, { recursive: true }) - await rm(reportPath, { force: true }) + if (!providedHandle) { + await rm(reportPath, { force: true }) + } let handle = providedHandle if (!handle) {