Merge origin/main (6d519f7835) into settle-split part 2

Main's #28333 (a "New" line above unseen messages) marks a catch-up page that
has more pages behind it with an optional `hasNewer` on the batch event. It
crossed this part's current-work fields on the same event: the batch type keeps
both `hasNewer` and `AgentSessionCurrentWorkFields`, and the subscriber
catch-up page sends `actionablePromptIds`, `working` and `hasNewer` together.

.github/actions/prepare-orcad-prebuilds/action.yml

@@ -63,5 +63,5 @@ runs:
           fi
           pnpm build:orcad-prebuilds
         fi
-        pnpm build:orcad-prebuilds --require-slots "$(node config/scripts/build-orcad-prebuilds.mjs --print-slot)"
-        pnpm build:orcad-prebuilds --smoke
+        node config/scripts/build-orcad-prebuilds.mjs --require-slots "$(node config/scripts/build-orcad-prebuilds.mjs --print-slot)"
+        node config/scripts/build-orcad-prebuilds.mjs --smoke

.github/actions/prepare-wayland-clipboard/action.yml

@@ -0,0 +1,53 @@
+name: Prepare Wayland clipboard helper
+description: Reuse the source-matched Ubuntu 20.04 helper, validating its architecture and glibc requirements every time.
+
+inputs:
+  save-cache:
+    description: Publish a validated helper from the default branch.
+    default: 'false'
+
+runs:
+  using: composite
+  steps:
+    - name: Resolve Wayland clipboard build identity
+      id: identity
+      shell: bash
+      run: |
+        node --input-type=module <<'NODE'
+        import { appendFileSync } from 'node:fs'
+        import { waylandClipboardFingerprint } from './config/scripts/build-wayland-clipboard.mjs'
+        const fingerprint = waylandClipboardFingerprint(process.cwd(), process.arch)
+        // Immutable invalid hits need a version bump here before publishing their repair.
+        appendFileSync(process.env.GITHUB_OUTPUT, `key=wayland-clipboard-v1-${process.arch}-${fingerprint}\n`)
+        NODE
+
+    - name: Restore Wayland clipboard helper
+      id: cache
+      uses: actions/cache/restore@v5
+      continue-on-error: true
+      with:
+        path: native/wayland-clipboard/.build
+        key: ${{ steps.identity.outputs.key }}
+
+    - name: Build or validate Wayland clipboard helper
+      shell: bash
+      env:
+        CACHE_HIT: ${{ steps.cache.outputs.cache-hit }}
+      run: |
+        if node config/scripts/build-wayland-clipboard.mjs; then
+          exit 0
+        fi
+        if [ "$CACHE_HIT" != true ]; then
+          exit 1
+        fi
+        echo 'Discarding an unusable Wayland clipboard cache and rebuilding from source.'
+        rm -rf -- native/wayland-clipboard/.build
+        node config/scripts/build-wayland-clipboard.mjs
+
+    - name: Save validated Wayland clipboard helper from main
+      if: inputs.save-cache == 'true' && steps.cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
+      uses: actions/cache/save@v5
+      continue-on-error: true
+      with:
+        path: native/wayland-clipboard/.build
+        key: ${{ steps.identity.outputs.key }}

.github/workflows/ci-cache-warmup-platform-jobs.yml

@@ -53,6 +53,29 @@ jobs:
       - name: Verify native cache is usable
         run: node config/scripts/ensure-native-runtime.mjs --check-only
 
+      - name: Cache Windows CLI launcher
+        uses: actions/cache@v5
+        with:
+          path: |
+            native/windows-cli-launcher/.build
+            native/windows-cli-launcher/target
+            native/windows-agent-hook/.build
+            native/windows-agent-hook/target
+            ~/.cargo/registry
+          key: windows-cli-launcher-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('native/windows-cli-launcher/**', 'native/windows-agent-hook/**', 'config/scripts/build-windows-cli-launcher.mjs', 'config/scripts/build-windows-agent-hook.mjs', 'config/scripts/windows-rust-executable-build.mjs', 'resources/build/icon.ico', 'package.json') }}
+
+      - name: Build the Windows native executables
+        shell: pwsh
+        run: |
+          if (-not (Get-Command cargo -ErrorAction SilentlyContinue)) {
+            rustup toolchain install stable --profile minimal
+            rustup default stable
+          }
+          node config/scripts/build-windows-agent-hook.mjs
+          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+          node config/scripts/build-windows-cli-launcher.mjs
+          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+
   warm-linux-package-fixtures:
     concurrency:
       group: ci-cache-warmup-linux-package-fixtures-${{ github.event.pull_request.number || github.ref }}

.github/workflows/ci-cache-warmup-platforms.yml

@@ -34,6 +34,12 @@ on:
       - 'config/scripts/space-sharing-copy.mjs'
       - 'config/patches/**'
       - 'native/windows-registry/**'
+      - 'native/windows-cli-launcher/**'
+      - 'native/windows-agent-hook/**'
+      - 'config/scripts/build-windows-cli-launcher.mjs'
+      - 'config/scripts/build-windows-agent-hook.mjs'
+      - 'config/scripts/windows-rust-executable-build.mjs'
+      - 'resources/build/icon.ico'
   pull_request:
     paths:
       - '.github/workflows/ci-cache-warmup.yml'

.github/workflows/ci-cache-warmup.yml

@@ -18,6 +18,9 @@ on:
       - '.github/actions/prepare-headless-compiler/**'
       - 'config/scripts/headless-detector-compiler-cache.mjs'
       - '.github/actions/prepare-linux-package-fixture/**'
+      - '.github/actions/prepare-wayland-clipboard/**'
+      - 'native/wayland-clipboard/**'
+      - 'config/scripts/build-wayland-clipboard.mjs'
       - 'config/docker/headless-serve-shutdown/**'
       - 'config/docker/cli-launch-contract/**'
       - 'config/docker/daemon-shutdown-descendants/**'
@@ -48,6 +51,7 @@ on:
       - '.github/workflows/ci-cache-warmup-platforms.yml'
       - '.github/actions/prepare-headless-compiler/**'
       - 'config/scripts/headless-detector-compiler-cache.mjs'
+      - '.github/actions/prepare-wayland-clipboard/**'
 
 permissions:
   contents: read
@@ -83,6 +87,10 @@ jobs:
 
       - uses: ./.github/actions/prepare-git-compatibility
 
+      - uses: ./.github/actions/prepare-wayland-clipboard
+        with:
+          save-cache: 'true'
+
   warm-platforms:
     if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
     uses: ./.github/workflows/ci-cache-warmup-platform-jobs.yml

.github/workflows/cloud-deploy-relay-production-capacity-job.yml

@@ -47,8 +47,8 @@ jobs:
     timeout-minutes: 75
     environment: production
     env:
-      # These drains are unpaced, but restart-safe still needs a quiet runtime for the production
-      # pace window, the same one the same-cap drain uses.
+      # These drains are unpaced, but restart-safe still waits out the production pace window
+      # from the drain, the same one the same-cap drain uses.
       RESTART_SAFE_PACE_WINDOW_MS: '300000'
       GCP_PROJECT_ID: onorca-cloud
       GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}

.github/workflows/cloud-deploy-relay-production-same-cap-job.yml

@@ -54,8 +54,7 @@ jobs:
       # /v1/assign at hosts / window and is re-placed through the director's drain-return
       # lane. The wave validator admits only its closed set, and anything below the 300000
       # default only on US general cells: Asia drains are bound by their targets' accept
-      # rate. The transition wait below is the 15-min migration lease plus this window, plus
-      # its excess over 300000, since restart-safe waits a window of quiet after the drain. A
+      # rate. The transition wait below is the 15-min migration lease plus this window. A
       # cell on an older image drains at once, or refuses a window above 300000.
       DRAIN_PACE_WINDOW_MS: ${{ inputs.drain-pace-window-ms }}
     steps:
@@ -504,7 +503,7 @@ jobs:
         run: echo 'Exact same-cap rollback point verified.'
 
       # The sample can age the first one-hour token by up to 20 min; the drain's restart-safe
-      # wait (20 min at the default pace, 50 at the slowest) gets a fresh one.
+      # wait (20 min at the default pace, 35 at the slowest) gets a fresh one.
       - id: drain-auth
         if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME != 'true' }}
         uses: google-github-actions/auth@v2
@@ -555,7 +554,7 @@ jobs:
           # What the cell accepted, which is 0 when an older image fell back to an unpaced drain.
           echo "drain-applied-pace-window-ms=$(jq -er '.paceWindowMs' <<< "${DRAIN_RESULT}")" \
             >> "${GITHUB_OUTPUT}"
-          # Restart-safe means an empty runtime for a whole pace window after the drain; leases
+          # Restart-safe means an empty runtime that lasts past the drain's pace window; leases
           # held by unplaceable hosts are reported, not waited on (c28, 2026-10-01).
           node dev/scripts/verify-relay-capacity-transition.mjs \
             --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
@@ -564,7 +563,7 @@ jobs:
             --heartbeat either --admission migration-only --draining required \
             --activity restart-safe --expected-image-digests "${PREDECESSOR_IMAGE_DIGEST}" \
             --pace-window-ms "${DRAIN_PACE_WINDOW_MS}" \
-            --timeout-ms "$((900000 + DRAIN_PACE_WINDOW_MS + (DRAIN_PACE_WINDOW_MS > 300000 ? DRAIN_PACE_WINDOW_MS - 300000 : 0)))"
+            --timeout-ms "$((900000 + DRAIN_PACE_WINDOW_MS))"
           echo "drain-settled-at=$(date -u +%FT%TZ)" >> "${GITHUB_OUTPUT}"
 
       - id: capacity-auth

.github/workflows/cloud-operate-relay-production-reserve-watch.yml

@@ -0,0 +1,327 @@
+name: Operate Relay Production Reserve Watch
+
+# Watches one cell already switched to reserve mode and flips it back to db on the first stop
+# rule it trips (step 5 design §9). Manual only. dry-run evaluates and logs, writes nothing.
+# An operator watches the run. A refused watch or a flip back that did not land raises an ::error::
+# annotation, as the clock-skew monitor does, and in write mode logs the line the reserve-watch
+# alert pages on. Refuses to start while a rollout holds the Cloud SQL rollout lease.
+on:
+  workflow_dispatch:
+    inputs:
+      mode:
+        description: dry-run logs what a trip would write; write flips the cell back on a trip
+        required: true
+        default: dry-run
+        type: choice
+        options: [dry-run, write]
+      cell-id:
+        description: Exact cell id already in reserve mode, e.g. production-gce-c26
+        required: true
+        type: string
+      duration-minutes:
+        description: How long to watch, 5 to 300
+        required: true
+        default: '120'
+        type: string
+      flipped-at:
+        description: >-
+          When the reserve write applied, ISO with a zone, e.g. 2026-10-14T12:00:00Z. The 503
+          background is the median of the 10 minutes before it
+        required: false
+        default: ''
+        type: string
+      background-503-per-minute:
+        description: >-
+          Director 503s per minute before the flip; overrides the one measured before flipped-at.
+          One of the two is required
+        required: false
+        default: ''
+        type: string
+      unverifiable-limit:
+        description: Consecutive samples with any unreadable signal that count as a trip
+        required: true
+        default: '3'
+        type: string
+      confirmation:
+        description: Required for write, exactly "WRITE_RELAY_CELL_FLAGS <cell-id>"
+        required: false
+        type: string
+
+permissions:
+  contents: read
+  id-token: write
+
+# One watch per cell; why not the flag workflow's group: cloud/docs/relay-workflows.md.
+concurrency:
+  group: relay-production-reserve-watch-${{ inputs.cell-id }}
+  cancel-in-progress: false
+
+defaults:
+  run:
+    # Explicit bash runs with pipefail, so the script's exit code survives the tee.
+    shell: bash
+    working-directory: cloud
+
+env:
+  MODE: ${{ inputs.mode }}
+  CELL_ID: ${{ inputs.cell-id }}
+  CONFIRMATION: ${{ inputs.confirmation }}
+  GCP_PROJECT_ID: onorca-cloud
+
+jobs:
+  watch:
+    if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
+    runs-on: ubuntu-latest
+    # Six 55-minute segments cover the longest watch; under the 360-minute job limit.
+    timeout-minutes: 350
+    environment: production
+    outputs:
+      state: ${{ steps.outcome.outputs.state }}
+      verdict: ${{ steps.outcome.outputs.verdict }}
+    env:
+      DURATION_MINUTES: ${{ inputs.duration-minutes }}
+      FLIPPED_AT: ${{ inputs.flipped-at }}
+      BACKGROUND_503_PER_MINUTE: ${{ inputs.background-503-per-minute }}
+      UNVERIFIABLE_LIMIT: ${{ inputs.unverifiable-limit }}
+      # One segment, after the auth step before it: fixed text, the inputs arrive as env vars.
+      WATCH_SEGMENT: |
+        set -euo pipefail
+        [[ "${CELL_ID}" =~ ^production-gce-c[0-9]{1,3}$ ]]
+        node -e "if (!/^[^.]+[.][^.]+[.][^.]+$/.test(process.env.ORCA_RELAY_ADMIN_ID_TOKEN ?? '')) process.exit(1)"
+        node dev/scripts/watch-relay-reserve-cell.mjs watch \
+          --mode "${MODE}" --cell-id "${CELL_ID}" \
+          --cell-origin "https://${CELL_ID#production-gce-}.relay.onorca.dev" \
+          --project-id "${GCP_PROJECT_ID}" --director-origin "https://relay.onorca.dev" \
+          --director-service orca-cloud-relay --confirmation "${CONFIRMATION}" \
+          --duration-minutes "${DURATION_MINUTES}" --segment-minutes 55 \
+          --flipped-at "${FLIPPED_AT}" --background-503-per-minute "${BACKGROUND_503_PER_MINUTE}" \
+          --unverifiable-limit "${UNVERIFIABLE_LIMIT}" \
+          --state-file "${RUNNER_TEMP}/relay-reserve-watch.json" --github-output "${GITHUB_OUTPUT}" \
+          | tee -a "${RUNNER_TEMP}/relay-reserve-watch.jsonl"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 24
+
+      - id: google-auth-1
+        uses: google-github-actions/auth@v2
+        with:
+          workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+          token_format: id_token
+          id_token_audience: https://relay.onorca.dev/v1/admin/drain
+          id_token_include_email: true
+
+      - uses: google-github-actions/setup-gcloud@v2
+        with:
+          skip_install: true
+
+      - id: watch-1
+        name: Watch segment 1 of 6
+        env:
+          ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-1.outputs.id_token }}
+        run: bash -c "${WATCH_SEGMENT}"
+
+      - id: google-auth-2
+        if: ${{ steps.watch-1.outputs.state == 'watching' }}
+        uses: google-github-actions/auth@v2
+        with:
+          workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+          token_format: id_token
+          id_token_audience: https://relay.onorca.dev/v1/admin/drain
+          id_token_include_email: true
+
+      - id: watch-2
+        name: Watch segment 2 of 6
+        if: ${{ steps.watch-1.outputs.state == 'watching' }}
+        env:
+          ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-2.outputs.id_token }}
+        run: bash -c "${WATCH_SEGMENT}"
+
+      - id: google-auth-3
+        if: ${{ steps.watch-2.outputs.state == 'watching' }}
+        uses: google-github-actions/auth@v2
+        with:
+          workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+          token_format: id_token
+          id_token_audience: https://relay.onorca.dev/v1/admin/drain
+          id_token_include_email: true
+
+      - id: watch-3
+        name: Watch segment 3 of 6
+        if: ${{ steps.watch-2.outputs.state == 'watching' }}
+        env:
+          ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-3.outputs.id_token }}
+        run: bash -c "${WATCH_SEGMENT}"
+
+      - id: google-auth-4
+        if: ${{ steps.watch-3.outputs.state == 'watching' }}
+        uses: google-github-actions/auth@v2
+        with:
+          workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+          token_format: id_token
+          id_token_audience: https://relay.onorca.dev/v1/admin/drain
+          id_token_include_email: true
+
+      - id: watch-4
+        name: Watch segment 4 of 6
+        if: ${{ steps.watch-3.outputs.state == 'watching' }}
+        env:
+          ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-4.outputs.id_token }}
+        run: bash -c "${WATCH_SEGMENT}"
+
+      - id: google-auth-5
+        if: ${{ steps.watch-4.outputs.state == 'watching' }}
+        uses: google-github-actions/auth@v2
+        with:
+          workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+          token_format: id_token
+          id_token_audience: https://relay.onorca.dev/v1/admin/drain
+          id_token_include_email: true
+
+      - id: watch-5
+        name: Watch segment 5 of 6
+        if: ${{ steps.watch-4.outputs.state == 'watching' }}
+        env:
+          ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-5.outputs.id_token }}
+        run: bash -c "${WATCH_SEGMENT}"
+
+      - id: google-auth-6
+        if: ${{ steps.watch-5.outputs.state == 'watching' }}
+        uses: google-github-actions/auth@v2
+        with:
+          workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+          token_format: id_token
+          id_token_audience: https://relay.onorca.dev/v1/admin/drain
+          id_token_include_email: true
+
+      - id: watch-6
+        name: Watch segment 6 of 6
+        if: ${{ steps.watch-5.outputs.state == 'watching' }}
+        env:
+          ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-6.outputs.id_token }}
+        run: bash -c "${WATCH_SEGMENT}"
+
+      - id: outcome
+        name: Record the watch outcome
+        if: ${{ always() }}
+        run: |
+          node dev/scripts/watch-relay-reserve-cell.mjs outcome \
+            --state-file "${RUNNER_TEMP}/relay-reserve-watch.json" --github-output "${GITHUB_OUTPUT}"
+
+      # The documented flow writes reserve before dispatching, so a refusal can leave it unwatched.
+      - name: Alarm on a refused watch
+        if: ${{ always() && steps.outcome.outputs.state == 'refused' }}
+        env:
+          WATCH_VERDICT: ${{ steps.outcome.outputs.verdict }}
+        run: |
+          node dev/scripts/watch-relay-reserve-cell.mjs alarm --kind refused \
+            --mode "${MODE}" --cell-id "${CELL_ID}" --project-id "${GCP_PROJECT_ID}" --verdict "${WATCH_VERDICT}"
+
+      - name: Audit summary
+        if: ${{ always() }}
+        env:
+          WATCH_STATE: ${{ steps.outcome.outputs.state }}
+          WATCH_VERDICT: ${{ steps.outcome.outputs.verdict }}
+        run: |
+          {
+            echo "### Relay reserve watch: ${MODE} ${CELL_ID}"
+            echo "Triggered by: ${GITHUB_TRIGGERING_ACTOR}, run ${GITHUB_RUN_ID} attempt ${GITHUB_RUN_ATTEMPT}"
+            echo "Outcome: ${WATCH_STATE}: ${WATCH_VERDICT}"
+            echo '```json'
+            tail -n 20 "${RUNNER_TEMP}/relay-reserve-watch.jsonl" 2>/dev/null || echo '(no output)'
+            echo '```'
+          } >> "${GITHUB_STEP_SUMMARY}"
+
+  flip-back:
+    needs: watch
+    # Missing data is never healthy: a watch that crashed, timed out or was cancelled (by an
+    # operator too) flips back like a trip. In dry-run this only plans the write.
+    if: ${{ always() && vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && needs.watch.result != 'skipped' && !contains(fromJSON('["completed","stopped","refused"]'), needs.watch.outputs.state) }}
+    runs-on: ubuntu-latest
+    # A flip back waits for the cell to lease every control: 8-50 minutes, scaled to its controls.
+    timeout-minutes: 60
+    environment: production
+    # Only the run's per-cell group above, which a running run never yields (cancel-in-progress off).
+    # Every step runs on always() too, so an ordinary cancel cannot skip the switch-file write; a force-cancel
+    # can, and leaves the cell in reserve for the operator to flip back by hand. A cancel during the
+    # lease wait can still kill the process before Postgres records db: the file says db, Postgres reserve.
+    env:
+      VERDICT: ${{ needs.watch.outputs.verdict || format('the watch ended {0} without a verdict', needs.watch.result) }}
+    steps:
+      - uses: actions/checkout@v4
+        if: ${{ always() }}
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-node@v4
+        if: ${{ always() }}
+        with:
+          node-version: 24
+
+      - id: google-auth
+        if: ${{ always() }}
+        uses: google-github-actions/auth@v2
+        with:
+          workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+          token_format: id_token
+          id_token_audience: https://relay.onorca.dev/v1/admin/drain
+          id_token_include_email: true
+
+      - uses: google-github-actions/setup-gcloud@v2
+        if: ${{ always() }}
+        with:
+          skip_install: true
+
+      - name: Flip the cell back to db, or preview it
+        if: ${{ always() }}
+        env:
+          ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+        run: |
+          [[ "${CELL_ID}" =~ ^production-gce-c[0-9]{1,3}$ ]]
+          ORCA_RELAY_CONTROL_ACCESS_TOKEN="$(gcloud auth print-access-token)"
+          echo "::add-mask::${ORCA_RELAY_CONTROL_ACCESS_TOKEN}"
+          export ORCA_RELAY_CONTROL_ACCESS_TOKEN
+          node dev/scripts/watch-relay-reserve-cell.mjs flip-back \
+            --mode "${MODE}" --cell-id "${CELL_ID}" \
+            --cell-origin "https://${CELL_ID#production-gce-}.relay.onorca.dev" \
+            --project-id "${GCP_PROJECT_ID}" --director-origin "https://relay.onorca.dev" \
+            --confirmation "${CONFIRMATION}" --verdict "${VERDICT}" \
+            | tee "${RUNNER_TEMP}/relay-reserve-flip-back.jsonl"
+
+      # The flip-back line is logged only once the write (or its dry-run plan) finished; without
+      # it a missing token, a failed login or a silent cell left the cell in reserve, unwatched.
+      - name: Alarm when the flip back did not land
+        if: ${{ always() }}
+        run: |
+          if grep -qs '"event":"orca_relay_reserve_watch_flip_back"' "${RUNNER_TEMP}/relay-reserve-flip-back.jsonl"; then
+            exit 0
+          fi
+          {
+            echo "## FLIP BACK FAILED: ${CELL_ID} may still be in reserve mode"
+            echo "Flip it to db by hand with Operate Relay Production Cell Flags."
+          } >> "${GITHUB_STEP_SUMMARY}"
+          node dev/scripts/watch-relay-reserve-cell.mjs alarm --kind flip-back-failed \
+            --mode "${MODE}" --cell-id "${CELL_ID}" --project-id "${GCP_PROJECT_ID}" --verdict "${VERDICT}"
+
+      - name: Audit summary
+        if: ${{ always() }}
+        run: |
+          {
+            echo "### Relay reserve watch flip back: ${MODE} ${CELL_ID}"
+            echo "Triggered by: ${GITHUB_TRIGGERING_ACTOR}, run ${GITHUB_RUN_ID} attempt ${GITHUB_RUN_ATTEMPT}"
+            echo "Verdict: ${VERDICT}"
+            echo '```json'
+            cat "${RUNNER_TEMP}/relay-reserve-flip-back.jsonl" 2>/dev/null || echo '(no output)'
+            echo '```'
+          } >> "${GITHUB_STEP_SUMMARY}"

.github/workflows/cloud-prove-relay-staging-capacity.yml

@@ -69,8 +69,8 @@ jobs:
       DIRECTOR_SERVICE_NAME: orca-cloud-relay-staging
       DIRECTOR_ORIGIN: https://relay-staging.onorca.dev
       CELL_ORIGIN: https://c3.relay-staging.onorca.dev
-      # These drains are unpaced, but restart-safe still needs a quiet runtime for the production
-      # pace window, the same one the same-cap drain uses.
+      # These drains are unpaced, but restart-safe still waits out the production pace window
+      # from the drain, the same one the same-cap drain uses.
       RESTART_SAFE_PACE_WINDOW_MS: '300000'
       TARGET_CELL_ID: staging-gce-c3
       FALLBACK_CELL_ORIGIN: https://c2.relay-staging.onorca.dev

.github/workflows/cloud-verify.yml

@@ -4,12 +4,14 @@ on:
   pull_request:
     paths:
       - cloud/**
+      - src/packages/process-host/**
       - .github/workflows/cloud-*.yml
       - .github/actions/cloud-sql-rollout-lease/**
   push:
     branches: [main]
     paths:
       - cloud/**
+      - src/packages/process-host/**
       - .github/workflows/cloud-*.yml
       - .github/actions/cloud-sql-rollout-lease/**
 
@@ -93,8 +95,8 @@ jobs:
       - run: pnpm build
       - run: pnpm typecheck
 
-  # Runs in parallel with build. `pnpm test` compiles the one workspace
-  # package it needs through the relay pretest hook, so it does not depend on
+  # Runs in parallel with build. `pnpm test` compiles the workspace
+  # packages it needs through pretest hooks, so it does not depend on
   # `pnpm build` having run.
   test:
     runs-on: ubuntu-22.04
@@ -113,6 +115,7 @@ jobs:
           --health-timeout 5s
           --health-retries 10
     env:
+      ORCA_BACKGROUND_LAUNCH: '1'
       ORCA_PUSH_TEST_DATABASE_URL: postgres://relay_test:relay_test@127.0.0.1:5432/orca_relay_test
       ORCA_RELAY_TEST_POSTGRES_URL: postgres://relay_test:relay_test@127.0.0.1:5432/orca_relay_test
     steps:

.github/workflows/e2e.yml

@@ -52,7 +52,9 @@ jobs:
     name: build e2e app
     if: inputs.test_files == '' || github.event_name != 'pull_request' || inputs.needs_build
     runs-on: ubuntu-latest
-    timeout-minutes: 10
+    timeout-minutes: 15
+    env:
+      ORCA_BACKGROUND_LAUNCH: '1'
 
     steps:
       - name: Checkout
@@ -60,10 +62,44 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.ref }}
 
+      - name: Install native build tools
+        env:
+          ORCA_E2E_APT_PACKAGES: build-essential python3
+        run: &install_e2e_tools |
+          if [ "$ORCA_E2E_APT_PACKAGES" = 'build-essential python3' ] &&
+            command -v make >/dev/null && command -v g++ >/dev/null && command -v python3 >/dev/null; then
+            exit 0
+          fi
+          read -r -a packages <<< "$ORCA_E2E_APT_PACKAGES"
+          # Why apt-mirrors.txt: runner images resolve mirror+file: sources through it, so editing the lists alone leaves Azure in use.
+          for source in /etc/apt/apt-mirrors.txt /etc/apt/sources.list /etc/apt/sources.list.d/*.list /etc/apt/sources.list.d/*.sources; do
+            if [ -f "$source" ]; then
+              sudo sed -i 's|https*://azure\.archive\.ubuntu\.com/ubuntu|https://archive.ubuntu.com/ubuntu|g' "$source"
+            fi
+          done
+          sudo tee /etc/apt/apt.conf.d/99-orca-e2e >/dev/null <<'APTCONF'
+          Acquire::http::Timeout "15";
+          Acquire::https::Timeout "15";
+          Acquire::Retries "1";
+          APTCONF
+          timeout 120 sudo apt-get update
+          timeout 300 sudo apt-get install -y "${packages[@]}" || timeout 300 sudo apt-get install -y --fix-missing "${packages[@]}"
+
       # Why: the build's plain-Node daemon smoke load resolves node-pty.
       - uses: ./.github/actions/install-node-dependencies
+        id: deps
         with:
           native-runtime: node
+          persist-native-cache: 'false'
+          cache-electron-package: 'true'
+
+      # Save Node's ABI before the Electron preparation replaces the same build paths.
+      - name: Save compiled Node native modules
+        if: steps.deps.outputs.native-cache-key != '' && steps.deps.outputs.native-cache-hit != 'true'
+        uses: actions/cache/save@v5
+        with:
+          path: ${{ steps.deps.outputs.native-cache-path }}
+          key: ${{ steps.deps.outputs.native-cache-key }}
 
       # Why: building here avoids parallel builds inside Playwright globalSetup;
       # paired-browser specs also need the standalone web bundle.
@@ -88,7 +124,19 @@ jobs:
 
       - wait: e2e-cli
 
+      - name: Check shared native runtime action
+        id: native-runtime-action
+        run: |
+          if [ -f .github/actions/prepare-native-runtime/action.yml ]; then
+            echo 'available=true' >> "$GITHUB_OUTPUT"
+          else
+            echo 'available=false' >> "$GITHUB_OUTPUT"
+          fi
+
+      # Only the completed output upload overlaps the change from Node to Electron ABI.
       - name: Upload E2E build output
+        id: e2e-build-upload
+        background: true
         uses: actions/upload-artifact@v7
         with:
           name: e2e-build-out${{ inputs.artifact_suffix }}
@@ -97,46 +145,25 @@ jobs:
           retention-days: 1
           if-no-files-found: error
 
-  # Build Electron-native dependencies once per workflow. Consumer shards restore
-  # this immutable cache instead of compiling the same ABI concurrently.
-  prepare-native-cache:
-    name: prepare Electron native cache
-    if: inputs.test_files == '' || github.event_name != 'pull_request' || inputs.needs_build
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v6
+      - name: Prepare shared Electron native runtime
+        id: electron-native-runtime
+        if: steps.native-runtime-action.outputs.available == 'true' && steps.deps.outputs.node-version != ''
+        uses: ./.github/actions/prepare-native-runtime
         with:
-          ref: ${{ inputs.ref || github.ref }}
+          native-runtime: electron
+          node-version: ${{ steps.deps.outputs.node-version }}
 
-      - name: Install native build tools
-        env:
-          ORCA_E2E_APT_PACKAGES: build-essential python3
-        run: &install_e2e_tools |
-          read -r -a packages <<< "$ORCA_E2E_APT_PACKAGES"
-          # Why apt-mirrors.txt: runner images resolve mirror+file: sources through it, so editing the lists alone leaves Azure in use.
-          for source in /etc/apt/apt-mirrors.txt /etc/apt/sources.list /etc/apt/sources.list.d/*.list /etc/apt/sources.list.d/*.sources; do
-            if [ -f "$source" ]; then
-              sudo sed -i 's|https*://azure\.archive\.ubuntu\.com/ubuntu|https://archive.ubuntu.com/ubuntu|g' "$source"
-            fi
-          done
-          sudo tee /etc/apt/apt.conf.d/99-orca-e2e >/dev/null <<'APTCONF'
-          Acquire::http::Timeout "15";
-          Acquire::https::Timeout "15";
-          Acquire::Retries "1";
-          APTCONF
-          timeout 120 sudo apt-get update
-          timeout 300 sudo apt-get install -y "${packages[@]}" || timeout 300 sudo apt-get install -y --fix-missing "${packages[@]}"
-
-      - uses: ./.github/actions/install-node-dependencies
+      - name: Prepare Electron natives for older refs
+        if: steps.native-runtime-action.outputs.available != 'true' || steps.deps.outputs.node-version == ''
+        uses: ./.github/actions/install-node-dependencies
         with:
           native-runtime: electron
 
+      - wait: e2e-build-upload
+
   e2e:
     name: e2e ${{ matrix.shard_name }}
-    needs: [build, prepare-native-cache]
+    needs: [build]
     if: inputs.test_files == ''
     runs-on: ubuntu-latest
     timeout-minutes: 30
@@ -263,7 +290,7 @@ jobs:
 
   changed-e2e:
     name: changed e2e specs
-    needs: [build, prepare-native-cache]
+    needs: [build]
     if: inputs.test_files != '' && (github.event_name != 'pull_request' || inputs.run_changed_e2e)
     runs-on: ubuntu-latest
     timeout-minutes: 45
@@ -351,7 +378,7 @@ jobs:
 
   orcad-serve-mode-switch:
     name: orca serve Electron/orcad mode switch (D7)
-    needs: [build, prepare-native-cache]
+    needs: [build]
     if: >-
       inputs.test_files == '' ||
       contains(inputs.test_files, 'tests/e2e/orcad-serve-mode-switch.spec.ts') ||
@@ -411,7 +438,7 @@ jobs:
   # orcad template for the fixture's target (Debian, linux-x64-glibc), which only this job builds.
   orcad-ssh-docker:
     name: ssh on managed orcad (Docker ${{ matrix.shard }})
-    needs: [build, prepare-native-cache]
+    needs: [build]
     if: inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/ssh-orcad-')
     runs-on: ubuntu-latest
     timeout-minutes: 60

.github/workflows/mobile.yml

@@ -36,6 +36,7 @@ on:
       - 'config/scripts/mobile-release-check-scope*'
       - 'config/scripts/mobile-test-change-scope*'
       - 'config/scripts/pr-code-change-scope.mjs'
+      - 'config/scripts/pr-package-test-paths.mjs'
       - 'config/scripts/pr-layout-oracle-route.mjs'
   # Why main too: two pull requests can each pass against their own base and disagree once both
   # land, and `verify` never runs on main. The same source paths as above, less the CI inputs only

.github/workflows/node-server-tests.yml

@@ -27,7 +27,7 @@ on:
         description: Also merge every lane's slot into the desktop orcad template artifact
         type: boolean
         default: false
-  # Release packaging calls this to build the orcad template it ships (design D2).
+  # Release and dev builds qualify every platform for the template they ship.
   workflow_call:
     inputs:
       ref:
@@ -38,11 +38,14 @@ on:
         description: Upload each lane's release slot and merge them into the orcad-template artifact
         type: boolean
         default: false
+  # Requalify fresh builds daily; publish missing immutable Windows cache keys.
+  schedule:
+    - cron: '30 11 * * *'
 
 permissions:
   contents: read
 
-# Main pushes must finish detection before they can supersede relevant qualification.
+# Keep template callers independent while replacing superseded PR qualification.
 concurrency:
   group: node-server-${{ (inputs.build_template || github.event_name == 'push') && format('run-{0}', github.run_id) || github.event.pull_request.number || github.ref }}
   cancel-in-progress: ${{ !inputs.build_template && github.event_name != 'push' }}
@@ -117,9 +120,8 @@ jobs:
     concurrency:
       group: node-server-persistence-${{ matrix.os }}-${{ github.event_name == 'push' && !inputs.build_template && inputs.ref == '' && github.ref || github.run_id }}
       cancel-in-progress: ${{ github.event_name == 'push' && !inputs.build_template && inputs.ref == '' }}
-    # Missing/failed detection runs the full matrix; manual runs remain unconditional.
-    # A draft carries no platform verdict; readiness re-triggers this workflow. Spelled against
-    # the event name so the push and schedule paths do not rest on a null property comparison.
+    # Missing/failed detection runs the full matrix; scheduled and manual runs stay unconditional.
+    # Drafts re-trigger at readiness; inherited caller events must not depend on PR fields.
     if: >-
       ${{ !cancelled() && needs.changes.outputs.should_run != 'false' &&
       (github.event_name != 'pull_request' || github.event.pull_request.draft != true) }}

.github/workflows/pr.yml

@@ -79,6 +79,7 @@ jobs:
             /config/.bun-version
             /config/scripts/check-readme-local-links.mjs
             /config/scripts/pr-code-change-scope.mjs
+            /config/scripts/pr-package-test-paths.mjs
             /config/scripts/pr-layout-oracle-route.mjs
             /config/scripts/pr-e2e-source-routing.mjs
             /config/scripts/ci-e2e-job-selection.mjs
@@ -219,6 +220,7 @@ jobs:
         with:
           bun-version-file: config/.bun-version
 
+      # Run gate scripts directly to avoid repeated package-manager startup.
       - name: Check process-host imports
         if: '!cancelled()'
         id: process-host-imports
@@ -239,7 +241,7 @@ jobs:
           PREFLIGHT_PRIOR_SUCCESS: ${{ job.status == 'success' }}
         run: |
           if [ "$PREFLIGHT_PHASE_SELECTED" != true ] || [ "$PREFLIGHT_PRIOR_SUCCESS" != true ]; then exit 0; fi
-          bun --bun run check:main-sync-spawn-ratchet
+          bun config/scripts/check-main-sync-spawn-ratchet.mjs
 
       # Keep each check in its own log while sharing this runner.
       - name: Lint
@@ -251,7 +253,7 @@ jobs:
           PREFLIGHT_PRIOR_SUCCESS: ${{ job.status == 'success' }}
         run: |
           if [ "$PREFLIGHT_PHASE_SELECTED" != true ] || [ "$PREFLIGHT_PRIOR_SUCCESS" != true ]; then exit 0; fi
-          pnpm exec oxlint --format github
+          node node_modules/oxlint/bin/oxlint --format github
 
       - name: Reject low-evidence patterns
         if: needs.code_paths.outputs.static_analysis == 'true'
@@ -270,11 +272,11 @@ jobs:
           PREFLIGHT_PRIOR_SUCCESS: ${{ job.status == 'success' }}
         run: |
           if [ "$PREFLIGHT_PHASE_SELECTED" != true ] || [ "$PREFLIGHT_PRIOR_SUCCESS" != true ]; then exit 0; fi
-          pnpm run audit:code-quality:native
+          node node_modules/oxlint/bin/oxlint --config config/oxlint-code-quality-native-plugins.json src config tests mobile --deny-warnings
 
       - name: Enforce type-aware code-quality baseline
         if: needs.code_paths.outputs.static_analysis == 'true'
-        run: pnpm run audit:code-quality:type-aware
+        run: node node_modules/oxlint/bin/oxlint --type-aware --config config/oxlint-code-quality-type-aware.json src config tests --deny-warnings
 
       # Mobile installation changes import resolution for the native cycle check.
       - wait: native-code-quality
@@ -294,11 +296,11 @@ jobs:
           PREFLIGHT_PRIOR_SUCCESS: ${{ job.status == 'success' }}
         run: |
           if [ "$PREFLIGHT_PHASE_SELECTED" != true ] || [ "$PREFLIGHT_PRIOR_SUCCESS" != true ]; then exit 0; fi
-          pnpm run check:code-quality:changed -- "${{ github.event.pull_request.base.sha }}"
+          node config/scripts/check-changed-code-quality.mjs "${{ github.event.pull_request.base.sha }}"
 
       - name: Enforce React Doctor on changed lines
         if: needs.code_paths.outputs.static_analysis == 'true'
-        run: pnpm run check:react-doctor:changed -- "${{ github.event.pull_request.base.sha }}"
+        run: node config/scripts/check-react-doctor-changed.mjs "${{ github.event.pull_request.base.sha }}"
 
       - wait: changed-code-quality
 
@@ -316,15 +318,15 @@ jobs:
 
       - name: Check Zustand selector fan-out budget
         if: needs.code_paths.outputs.static_analysis == 'true'
-        run: pnpm run check:zustand-selector-fanout
+        run: node config/scripts/zustand-selector-fanout-benchmark.mjs --check
 
       - name: Check reliability gate manifest
         if: needs.code_paths.outputs.static_analysis == 'true'
-        run: pnpm run check:reliability-gates
+        run: node config/scripts/check-reliability-gates.mjs
 
       - name: Enforce dead design-system classes
         if: needs.code_paths.outputs.static_analysis == 'true'
-        run: pnpm run check:dead-classes
+        run: node node_modules/oxlint/bin/oxlint --config config/oxlint-dead-classes.json src/renderer
 
       - name: Check VM runtime rollback compatibility
         if: needs.code_paths.outputs.static_analysis == 'true'
@@ -350,19 +352,19 @@ jobs:
 
       - name: Enforce max-lines ratchet
         if: needs.code_paths.outputs.static_analysis == 'true'
-        run: pnpm run check:max-lines-ratchet
+        run: node config/scripts/check-max-lines-ratchet.mjs
 
       - name: Enforce ts-nocheck ratchet
         if: needs.code_paths.outputs.static_analysis == 'true'
-        run: pnpm run check:ts-nocheck-ratchet
+        run: node config/scripts/check-ts-nocheck-ratchet.mjs
 
       - name: Enforce runtime Electron-import ratchet
         if: needs.code_paths.outputs.static_analysis == 'true'
-        run: pnpm run check:runtime-electron-ratchet
+        run: node config/scripts/check-runtime-electron-ratchet.mjs
 
       - name: Enforce focus-read ratchet
         if: needs.code_paths.outputs.static_analysis == 'true'
-        run: pnpm run check:focus-read-ratchet
+        run: node config/scripts/check-focus-read-ratchet.mjs
 
       - name: Enforce layout-writer ratchets
         if: needs.code_paths.outputs.static_analysis == 'true'
@@ -370,7 +372,7 @@ jobs:
 
       - name: Check Node runtime pin
         if: needs.code_paths.outputs.static_analysis == 'true'
-        run: pnpm run check:node-runtime-pin
+        run: node config/scripts/check-node-runtime-pin.mjs
 
       # Why: extraction writes sorted evidence to an isolated temporary path,
       # so feature PRs need one normalized AST pass rather than a three-OS matrix.
@@ -414,19 +416,19 @@ jobs:
 
       - name: Verify the generated RPC params catalog
         if: needs.code_paths.outputs.static_analysis == 'true'
-        run: pnpm run verify:rpc-params-catalog
+        run: node config/scripts/generate-rpc-params-catalog.mjs --check
 
       - name: Verify the generated ACP protocol schema
         if: needs.code_paths.outputs.static_analysis == 'true'
-        run: pnpm run verify:acp-protocol
+        run: node config/scripts/acp/generate-protocol.mjs --check
 
       - name: Verify bundled skill guides
         if: needs.code_paths.outputs.static_analysis == 'true'
-        run: pnpm run verify:bundled-skill-guides
+        run: node config/scripts/generate-bundled-skill-guides.mjs --check
 
       - name: Verify skill freshness manifest
         if: needs.code_paths.outputs.static_analysis == 'true'
-        run: pnpm run verify:skill-bundle-manifest
+        run: node config/scripts/generate-skill-bundle-manifest.mjs
 
       - name: Verify localization catalogs
         if: '!cancelled()'
@@ -437,7 +439,7 @@ jobs:
           PREFLIGHT_PRIOR_SUCCESS: ${{ job.status == 'success' }}
         run: |
           if [ "$PREFLIGHT_PHASE_SELECTED" != true ] || [ "$PREFLIGHT_PRIOR_SUCCESS" != true ]; then exit 0; fi
-          pnpm run verify:localization-catalogs
+          bun config/scripts/verify-localization-catalogs.mjs
 
       - name: Verify localization coverage
         if: needs.code_paths.outputs.static_analysis == 'true'
@@ -465,11 +467,11 @@ jobs:
 
       - name: Check feature wall asset budget
         if: needs.code_paths.outputs.static_analysis == 'true'
-        run: pnpm check:feature-wall-assets
+        run: node config/scripts/check-feature-wall-assets.mjs
 
       - name: Verify macOS entitlements
         if: needs.code_paths.outputs.static_analysis == 'true'
-        run: pnpm verify:macos-entitlements
+        run: node config/scripts/verify-macos-entitlements.mjs
 
       # Planning shares setup and stays off the compiler's critical path.
       - name: Plan unit selection
@@ -1099,6 +1101,15 @@ jobs:
           src/main/browser/browser-route-h3-egress.electron.test.ts
           src/main/browser/browser-route-dns-prefetch.electron.test.ts
           src/main/persistence/profile-state/profile-state-writer-stall.electron.test.ts
+          src/main/browser/browser-popup-compatibility.electron.test.ts
+          src/main/browser/browser-popup-native-compatibility.electron.test.ts
+          src/main/browser/browser-popup-background-compatibility.electron.test.ts
+
+      # The isolated glibc-floor build writes only the Wayland helper, so JavaScript can overlap.
+      - name: Build native components
+        id: linux-native
+        background: true
+        uses: ./.github/actions/prepare-wayland-clipboard
 
       # Restore independent fixtures during builds; keep native lifecycle probes serial.
       - uses: ./.github/actions/prepare-linux-package-fixture
@@ -1154,10 +1165,7 @@ jobs:
       - name: Build mobile web bundle
         run: pnpm run build:mobile-web
 
-      - name: Build native components
-        run: pnpm run build:native
-
-      - wait: [linux-package-tools, web-client]
+      - wait: [linux-native, linux-package-tools, web-client]
 
       - name: Package unpacked app
         env:

.github/workflows/unit-module-cache-benchmark.yml

@@ -0,0 +1,49 @@
+name: Unit transform cache comparison
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: unit-module-cache-comparison-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  baseline:
+    uses: ./.github/workflows/unit-tests.yml
+    with:
+      node_versions: '["24.21.0"]'
+      runner: ubuntu-24.04-arm
+      shards: '[{"index":1,"count":5},{"index":2,"count":5},{"index":3,"count":5},{"index":4,"count":5},{"index":5,"count":5}]'
+      artifact_prefix: baseline-unit
+      phase_diagnostics: true
+      run_relay_integration: false
+
+  producer:
+    needs: baseline
+    uses: ./.github/workflows/unit-tests.yml
+    with:
+      node_versions: '["24.21.0"]'
+      runner: ubuntu-24.04-arm
+      shards: '[{"index":1,"count":5},{"index":2,"count":5},{"index":3,"count":5},{"index":4,"count":5},{"index":5,"count":5}]'
+      artifact_prefix: producer-unit
+      module_cache: cold
+      module_cache_namespace: ${{ github.run_id }}-${{ github.run_attempt }}
+      module_cache_seed: true
+      phase_diagnostics: true
+      run_relay_integration: false
+
+  warm:
+    needs: producer
+    uses: ./.github/workflows/unit-tests.yml
+    with:
+      node_versions: '["24.21.0"]'
+      runner: ubuntu-24.04-arm
+      shards: '[{"index":1,"count":5},{"index":2,"count":5},{"index":3,"count":5},{"index":4,"count":5},{"index":5,"count":5}]'
+      artifact_prefix: warm-unit
+      module_cache: warm
+      module_cache_namespace: ${{ github.run_id }}-${{ github.run_attempt }}
+      phase_diagnostics: true
+      run_relay_integration: false

.github/workflows/unit-tests.yml

@@ -31,13 +31,43 @@ on:
         required: true
         type: string
 
+      module_cache:
+        description: Transform cache policy (off, cold producer, or warm exact restore).
+        default: 'off'
+        type: string
+
+      module_cache_namespace:
+        description: Isolate a same-source cache comparison from existing archives.
+        default: ''
+        type: string
+
+      module_cache_seed:
+        description: Publish comparison caches from a manually dispatched workflow.
+        default: false
+        type: boolean
+
+      phase_diagnostics:
+        description: Attach bounded per-file worker phase diagnostics.
+        default: false
+        type: boolean
+
+      artifact_prefix:
+        description: Distinguish evidence from repeated same-source comparison lanes.
+        default: unit
+        type: string
+
+      run_relay_integration:
+        description: Run relay contracts alongside normal unit checks.
+        default: true
+        type: boolean
+
 permissions:
   contents: read
 
 jobs:
   test:
     if: inputs.run_shards
-    name: tests node ${{ matrix.node }} ${{ matrix.shard.index }}/${{ matrix.shard.count }}
+    name: ${{ inputs.artifact_prefix != 'unit' && format('{0} ', inputs.artifact_prefix) || '' }}tests node ${{ matrix.node }} ${{ matrix.shard.index }}/${{ matrix.shard.count }}
     runs-on: ${{ inputs.runner }}
     timeout-minutes: 60
     strategy:
@@ -65,17 +95,51 @@ jobs:
         with:
           bun-version-file: config/.bun-version
 
+      - name: Resolve unit module cache
+        id: module-cache
+        if: inputs.module_cache == 'cold' || inputs.module_cache == 'warm'
+        env:
+          ORCA_UNIT_SHARD_INDEX: ${{ matrix.shard.index }}
+          ORCA_UNIT_SHARD_COUNT: ${{ matrix.shard.count }}
+          ORCA_UNIT_CACHE_NAMESPACE: ${{ inputs.module_cache_namespace }}
+        run: node config/scripts/ci-unit-module-cache.mjs identity
+
+      - name: Restore exact unit module cache
+        id: module-cache-restore
+        if: inputs.module_cache == 'warm' && steps.module-cache.outputs.key != ''
+        continue-on-error: true
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ steps.module-cache.outputs.path }}
+          key: ${{ steps.module-cache.outputs.key }}
+
+      - name: Validate restored unit module cache or use cold transforms
+        if: steps.module-cache.outputs.key != ''
+        env:
+          ORCA_UNIT_CACHE_RESTORE_HIT: ${{ steps.module-cache-restore.outputs.cache-hit }}
+        run: node config/scripts/ci-unit-module-cache.mjs prepare
+
       - uses: actions/download-artifact@v8
         continue-on-error: true
         with:
           name: unit-selection-attempt-${{ github.run_attempt }}
           path: ci-shards/
 
+      - name: Record full unit discovery
+        if: inputs.phase_diagnostics
+        continue-on-error: true
+        run: >-
+          node --input-type=module -e '
+          import { discoverUnitFiles } from "./config/scripts/ci-unit-files.mjs";
+          import { writeAssignment } from "./config/scripts/ci-shard-assignment.mjs";
+          writeAssignment("ci-shards/unit-discovery.json", { files: discoverUnitFiles() });'
+
       - name: Test shard
         env:
           NODE_COMPILE_CACHE: ${{ runner.temp }}/orca-node-compile-cache-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.node }}-${{ matrix.shard.index }}
           ORCA_BALANCE_UNIT_SHARDS: '1'
           ORCA_BACKGROUND_LAUNCH: '1'
+          ORCA_UNIT_PHASE_REPORT: ${{ inputs.phase_diagnostics && 'ci-shards/unit-phases.json' || '' }}
         run: |
           ORCA_SHARD_SOURCE_SHA="$(git rev-parse HEAD)"
           export ORCA_SHARD_SOURCE_SHA
@@ -83,18 +147,37 @@ jobs:
             --shard=${{ matrix.shard.index }}/${{ matrix.shard.count }} \
             ${{ inputs.runner == 'ubuntu-24.04-arm' && '--maxWorkers=4' || '' }}
 
+      - name: Seal unit module cache
+        id: module-cache-seal
+        if: >-
+          success() && steps.module-cache.outputs.key != '' &&
+          steps.module-cache-restore.outputs.cache-hit != 'true' &&
+          (github.event_name == 'pull_request' ||
+          github.ref == 'refs/heads/main' && github.event_name != 'pull_request' ||
+          inputs.module_cache_seed && github.event_name == 'workflow_dispatch')
+        run: node config/scripts/ci-unit-module-cache.mjs seal
+
+      - name: Save unit module cache
+        if: steps.module-cache-seal.outputs.ready == 'true'
+        continue-on-error: true
+        uses: actions/cache/save@v5
+        with:
+          path: ${{ steps.module-cache.outputs.path }}
+          key: ${{ steps.module-cache.outputs.key }}
+
       - name: Upload unit shard assignment
         if: always()
         # Diagnostic upload outages must not change the test verdict.
         continue-on-error: true
         uses: actions/upload-artifact@v7
         with:
-          name: unit-shard-node-${{ matrix.node }}-${{ matrix.shard.index }}-attempt-${{ github.run_attempt }}
+          name: ${{ inputs.artifact_prefix }}-shard-node-${{ matrix.node }}-${{ matrix.shard.index }}-attempt-${{ github.run_attempt }}
           path: ci-shards/
           retention-days: 14
           if-no-files-found: warn
 
   relay_integration:
+    if: inputs.run_relay_integration
     name: relay integration node ${{ matrix.node }}
     strategy:
       fail-fast: false

cloud/apps/push/src/durable-push-fixture-lock-isolation.test.ts

@@ -0,0 +1,42 @@
+import { afterEach, expect, it } from 'vitest'
+import { DurablePushStore } from './durable-push-store.js'
+import {
+  cleanupDurablePushFixtures,
+  DEVICE_HEAD_SQL,
+  durablePushTestDatabaseUrl,
+  fixture,
+  notification,
+  pauseAfter,
+  within
+} from './durable-push-store.test-fixture.js'
+
+afterEach(cleanupDurablePushFixtures)
+
+it.skipIf(!durablePushTestDatabaseUrl)(
+  'claims the same device in another schema while a fixture holds its device lock',
+  async () => {
+    const first = await fixture()
+    const second = await fixture()
+    const [firstScope] = await first.db.query(
+      'SELECT current_database() AS database, current_schema() AS schema'
+    )
+    const [secondScope] = await second.db.query(
+      'SELECT current_database() AS database, current_schema() AS schema'
+    )
+    expect(firstScope?.database).toBe(secondScope?.database)
+    expect(firstScope?.schema).not.toBe(secondScope?.schema)
+    await first.store.accept('host', 'phone', notification(1))
+    await second.store.accept('host', 'phone', notification(2))
+    const paused = pauseAfter(first.db, DEVICE_HEAD_SQL)
+    const held = new DurablePushStore(paused.wrapped, first.clock).claim()
+    await paused.atCandidate
+    try {
+      expect(await within(first.store.claim(), 2_000)).toBeNull()
+      expect((await within(second.store.claim(), 2_000))?.notification.notificationSeq).toBe(2)
+    } finally {
+      paused.release()
+      await held
+    }
+    expect((await held)?.notification.notificationSeq).toBe(1)
+  }
+)

cloud/apps/push/src/durable-push-store.test-fixture.ts

@@ -11,7 +11,10 @@ const cleanups: (() => Promise<void>)[] = []
 export async function cleanupDurablePushFixtures(): Promise<void> {
   await Promise.all(cleanups.splice(0).map((cleanup) => cleanup()))
 }
-export const notification = (seq: number, kind: 'alert' | 'dismiss' = 'alert'): PushNotification => ({
+export const notification = (
+  seq: number,
+  kind: 'alert' | 'dismiss' = 'alert'
+): PushNotification => ({
   notificationId: `notification-${seq}`,
   notificationEpoch: 'epoch',
   notificationSeq: seq,
@@ -21,11 +24,26 @@ export const notification = (seq: number, kind: 'alert' | 'dismiss' = 'alert'):
   body: '',
   kind
 })
-// ownDatabase: advisory locks are database-wide, so a test that holds a global key must not share one.
+
+// Schemas isolate rows, but advisory locks need the same fixture boundary.
+function scopeFixtureLocks(database: PushDatabase, scope: string): PushDatabase {
+  return {
+    dialect: database.dialect,
+    query: (sql, params) => database.query(sql, params),
+    transaction: (run) => database.transaction((tx) => run(scopeFixtureLocks(tx, scope))),
+    lockQuotaScope: (key) => database.lockQuotaScope(`${scope}:${key}`),
+    tryLockScope: (key) => database.tryLockScope(`${scope}:${key}`),
+    tryLockSharedScope: (key) => database.tryLockSharedScope(`${scope}:${key}`),
+    close: () => database.close()
+  }
+}
+
+// ownDatabase keeps production lock keys for mixed-revision tests.
 export async function fixture({ ownDatabase = false } = {}) {
   const databaseUrl = durablePushTestDatabaseUrl
-  if (databaseUrl && !process.env.CI && new URL(databaseUrl).port !== '55440')
+  if (databaseUrl && !process.env.CI && new URL(databaseUrl).port !== '55440') {
     throw new Error('isolated_postgres_port_required')
+  }
   let db: PushDatabase
   if (databaseUrl) {
     const admin = new pg.Client({ connectionString: databaseUrl })
@@ -38,7 +56,9 @@ export async function fixture({ ownDatabase = false } = {}) {
       } finally {
         try {
           await admin.query(
-            ownDatabase ? `DROP DATABASE IF EXISTS ${name}` : `DROP SCHEMA IF EXISTS ${name} CASCADE`
+            ownDatabase
+              ? `DROP DATABASE IF EXISTS ${name}`
+              : `DROP SCHEMA IF EXISTS ${name} CASCADE`
           )
         } finally {
           await admin.end()
@@ -53,7 +73,8 @@ export async function fixture({ ownDatabase = false } = {}) {
       await admin.query(`CREATE SCHEMA ${name}`)
       url.searchParams.set('options', `-c search_path=${name}`)
     }
-    db = scoped = await openPushDatabase({ databaseUrl: url.toString(), dataDir: '', poolMax: 4 })
+    scoped = await openPushDatabase({ databaseUrl: url.toString(), dataDir: '', poolMax: 4 })
+    db = ownDatabase ? scoped : scopeFixtureLocks(scoped, name)
   } else {
     db = await openInMemoryPushDatabase()
     cleanups.push(() => db.close())
@@ -117,7 +138,9 @@ export async function within<T>(operation: Promise<T>, ms: number): Promise<T> {
   try {
     return await Promise.race([
       operation,
-      new Promise<T>((_, reject) => (timer = setTimeout(() => reject(new Error('claim_blocked')), ms)))
+      new Promise<T>(
+        (_, reject) => (timer = setTimeout(() => reject(new Error('claim_blocked')), ms))
+      )
     ])
   } finally {
     clearTimeout(timer)

cloud/apps/relay/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@orca-cloud/relay",
-  "private": true,
   "version": "0.0.0",
+  "private": true,
   "type": "module",
   "main": "dist/index.js",
   "scripts": {
@@ -11,7 +11,7 @@
     "lint": "tsc -p tsconfig.json --noEmit",
     "pretest": "pnpm --filter @orca-cloud/postgres-schema build && pnpm --filter @orca-cloud/relay-contract build",
     "start": "node dist/index.js",
-    "test": "vitest run",
+    "test": "node --import tsx --conditions=orca-source ../../dev/scripts/run-relay-tests.mjs",
     "typecheck": "tsc -p tsconfig.json --noEmit"
   },
   "dependencies": {

cloud/apps/relay/src/app.ts

@@ -81,6 +81,11 @@ import {
 import { createRelayTokenVerifier, readBearer } from './relay-token-verifier.js'
 import type { RelayJWKSet } from './stale-if-error-jwks.js'
 import { stagingAsiaProofMembership } from './staging-asia-proof-admission.js'
+import {
+  runInStickySlot,
+  StickySlotStatements,
+  type StickySlotSample
+} from './sticky-slot-statements.js'
 
 const RelayCellConnectionHardCapSchema = z.custom<RelayCellConnectionHardCap>(
   isRelayCellConnectionHardCap
@@ -159,6 +164,7 @@ export function createRelayApp(
     ) => void
     recordDrainReturnRetryAfter?: (seconds: number) => void
     recordAdmissionServiceMs?: (lane: 'sticky' | 'drain-return', durationMs: number) => void
+    recordStickySlot?: (sample: StickySlotSample) => void
     recordAssignmentUnavailable?: (cause: AssignmentUnavailableCause) => void
     recordRegionRequest?: (region: RelayRegion | undefined) => void
     // Step 3 shadow directory; absent unless the director polls seat feeds.
@@ -269,16 +275,25 @@ export function createRelayApp(
     return context.json({ error: 'assignments_temporarily_unavailable' }, 503)
   }
   // The slot's hold, not the request's latency: herd recovery is slots / this.
-  const timeStickySlot = (lease: { release(): void }): { release(): void } => {
+  const timeStickySlot = (
+    lease: { release(): void },
+    queueWaitMs: number
+  ): { release(): void; measure<T>(work: () => Promise<T>): Promise<T> } => {
     const startedAt = performance.now()
+    const statements = new StickySlotStatements()
     let released = false
     return {
       release: () => {
         if (released) return
         released = true
-        operations.recordAdmissionServiceMs?.('sticky', performance.now() - startedAt)
-        lease.release()
-      }
+        try {
+          operations.recordAdmissionServiceMs?.('sticky', performance.now() - startedAt)
+          operations.recordStickySlot?.({ queueWaitMs, ...statements.close() })
+        } finally {
+          lease.release()
+        }
+      },
+      measure: (work) => runInStickySlot(statements, work)
     }
   }
   // Classified server-side from the host's own assignment row, never from a
@@ -480,8 +495,11 @@ export function createRelayApp(
     }
     let admission: { release(): void } | null = null
     let lane: AssignmentAdmissionLane = 'placement'
+    // Charges the sticky slot's SQL to its request; any other lane runs work as is.
+    let measureHeld = <T>(work: () => Promise<T>): Promise<T> => work()
     if (body.data.reconnect) {
       let rejection: AssignmentAdmissionRejection | undefined
+      const queuedAt = performance.now()
       const stickySlot = await stickyAssignmentAdmission.acquire(claims.relayHostId, (reason) => {
         rejection = reason
       })
@@ -497,12 +515,12 @@ export function createRelayApp(
         })
         return rejectStickyAssignment(context)
       }
-      const fastLane = timeStickySlot(stickySlot)
+      const fastLane = timeStickySlot(stickySlot, performance.now() - queuedAt)
       let verified: ResolvedRelayAssignment | null = null
       try {
-        verified = await operations.assignments.resolve(identity, {
-          classifyHomeRollIsolation: true
-        })
+        verified = await fastLane.measure(() =>
+          operations.assignments.resolve(identity, { classifyHomeRollIsolation: true })
+        )
       } catch (error) {
         fastLane.release()
         operations.recordAssignmentAdmission?.('sticky-rejected')
@@ -546,6 +564,7 @@ export function createRelayApp(
       } else if (verified) {
         lane = 'sticky'
         admission = fastLane
+        measureHeld = fastLane.measure
         operations.recordAssignmentAdmission?.('sticky')
       } else {
         // An unverified hint joins the placement lane with today's semantics.
@@ -574,25 +593,27 @@ export function createRelayApp(
     let regionCorrection: RegionCorrectionResponse | undefined
     try {
       if (body.data.regionCorrection?.action === 'report') {
-        const current = await operations.assignments.resolve(identity)
+        const current = await measureHeld(() => operations.assignments.resolve(identity))
         if (!current) return context.json({ error: 'assignment_not_found' }, 409)
         assignment = current
       } else {
         // The host asked: it may leave a reserve cell the director proves dead.
-        assignment = await operations.assignments.assign(
-          identity,
-          requestedRegion,
-          requestedRegion ? targetRegion : undefined,
-          undefined,
-          { ...step5, hostRequested: true }
+        assignment = await measureHeld(() =>
+          operations.assignments.assign(
+            identity,
+            requestedRegion,
+            requestedRegion ? targetRegion : undefined,
+            undefined,
+            { ...step5, hostRequested: true }
+          )
         )
       }
       if (body.data.regionCorrection) {
         try {
-          regionCorrection = await operations.assignments.exchangeRegionCorrection(
-            identity,
-            body.data.regionCorrection,
-            assignment.assignmentEpoch
+          const correction = body.data.regionCorrection
+          const epoch = assignment.assignmentEpoch
+          regionCorrection = await measureHeld(() =>
+            operations.assignments.exchangeRegionCorrection(identity, correction, epoch)
           )
         } catch (error) {
           if (body.data.regionCorrection.action === 'report') throw error

cloud/apps/relay/src/cell-reserve-admission.blackbox.test.ts

@@ -330,6 +330,19 @@ describe('a reserve-mode cell admits from memory with its database wedged', () =
     expect(cell.book(identity, 9)).toEqual([{ outcome: 'ok' }])
   })
 
+  it('leaves a drained seat to its drain grace instead of the shorter demotion close', async () => {
+    const cell = await startCell({ admitMode: 'reserve' })
+    const identity = await cell.host()
+    cell.book(identity, 9)
+    await cell.connect(identity, 9)
+    const page = cell.relay.sessions.seatFeed(null)
+    const joinedAt = 'full' in page ? page.full[0]!.joinedAt : -1
+    cell.relay.sessions.drain(120_000)
+    const seat = { v: 1 as const, userId: 'user-1', relayHostId: identity.hostId, epoch: 9, joinedAt }
+    expect(cell.relay.sessions.demote(seat)).toBe('draining')
+    expect(cell.relay.sessions.demote(seat)).toBe('draining')
+  })
+
   it('gives every booking unit back when the cell starts to drain', async () => {
     const cell = await startCell({ admitMode: 'reserve' })
     const identity = await cell.host()

cloud/apps/relay/src/database.ts

@@ -14,6 +14,7 @@ import {
 import { applyPostgresSchema } from './postgres-schema-startup.js'
 import { POSTGRES_STATEMENT_STATS_MIGRATION } from './postgres-statement-stats.js'
 import { reportPostgresQueryFailure } from './postgres-query-failure.js'
+import { noteStickySlotPoolWait, noteStickySlotStatement } from './sticky-slot-statements.js'
 import {
   isPostgresReadTimeout,
   isRelayDatabaseLayerError,
@@ -910,19 +911,25 @@ type PostgresQueryTimeout = () => number | undefined
 
 async function timedQuery(
   client: pg.PoolClient,
+  connectionId: number,
   text: string,
   values: unknown[] | undefined,
   queryTimeout: PostgresQueryTimeout | undefined
 ): Promise<pg.QueryResult> {
-  const timeout = queryTimeout?.()
-  if (timeout === undefined) return values === undefined ? await client.query(text) : await client.query(text, values)
-  // pg@8.22 reads it at client.js:660; @types/pg does not declare it on QueryConfig.
-  const config: pg.QueryConfig & { query_timeout: number } = {
-    text,
-    ...(values === undefined ? {} : { values }),
-    query_timeout: timeout
+  const startedAt = performance.now()
+  try {
+    const timeout = queryTimeout?.()
+    if (timeout === undefined) return values === undefined ? await client.query(text) : await client.query(text, values)
+    // pg@8.22 reads it at client.js:660; @types/pg does not declare it on QueryConfig.
+    const config: pg.QueryConfig & { query_timeout: number } = {
+      text,
+      ...(values === undefined ? {} : { values }),
+      query_timeout: timeout
+    }
+    return await client.query(config)
+  } finally {
+    noteStickySlotStatement(performance.now() - startedAt, connectionId)
   }
-  return await client.query(config)
 }
 
 class PostgresTransaction implements RelayDatabase {
@@ -938,6 +945,7 @@ class PostgresTransaction implements RelayDatabase {
 
   constructor(
     protected readonly client: pg.PoolClient,
+    private readonly connectionId: number,
     private readonly queryTimeout?: PostgresQueryTimeout
   ) {}
 
@@ -966,7 +974,7 @@ class PostgresTransaction implements RelayDatabase {
       `WITH final_write AS (${inlinePostgresParameters(sql, params, this.client)}) ` +
       'SELECT 1 / (SELECT count(*)::int FROM final_write); COMMIT'
     try {
-      await timedQuery(this.client, message, undefined, this.queryTimeout)
+      await timedQuery(this.client, this.connectionId, message, undefined, this.queryTimeout)
     } catch (error) {
       // Simple query stops at the first error, so any server error means COMMIT never ran:
       // retryable codes take the caller's normal rollback-and-retry path. A lost connection
@@ -1017,7 +1025,13 @@ class PostgresTransaction implements RelayDatabase {
     this.assertNotCommitted()
     this.failIfPoisoned()
     try {
-      const result = await timedQuery(this.client, postgresSql(sql), params, this.queryTimeout)
+      const result = await timedQuery(
+        this.client,
+        this.connectionId,
+        postgresSql(sql),
+        params,
+        this.queryTimeout
+      )
       return returnsRows(sql) ? (result.rows as SqlRow[]) : [{ changes: result.rowCount ?? 0 }]
     } catch (error) {
       this.noteFailure(error)
@@ -1177,6 +1191,9 @@ export class PostgresDatabase implements RelayDatabase {
   readonly dialect = 'postgres' as const
   private readonly pressure: PostgresPoolPressure
   private readonly holds = new CellInventoryHoldSamples()
+  // pg-pool hands the same client object back for the same connection.
+  private readonly connectionIds = new WeakMap<pg.PoolClient, number>()
+  private nextConnectionId = 1
 
   consumeHoldCounts(): CellInventoryHoldCounts {
     return this.holds.consumeCounts()
@@ -1207,9 +1224,15 @@ export class PostgresDatabase implements RelayDatabase {
     let client: pg.PoolClient | undefined
     let lostReply: Error | undefined
     try {
-      client = await this.pressure.connect(lane)
+      client = await this.connect(lane)
       phase = 'execute'
-      const result = await timedQuery(client, postgresSql(sql), params, this.queryTimeout)
+      const result = await timedQuery(
+        client,
+        this.idOfConnection(client),
+        postgresSql(sql),
+        params,
+        this.queryTimeout
+      )
       return returnsRows(sql) ? (result.rows as SqlRow[]) : [{ changes: result.rowCount ?? 0 }]
     } catch (error) {
       markRelayDatabaseError(error)
@@ -1257,15 +1280,18 @@ export class PostgresDatabase implements RelayDatabase {
     options: RelayTransactionOptions = {}
   ): Promise<T> {
     for (let attempt = 1; attempt <= POSTGRES_TRANSACTION_ATTEMPTS; attempt++) {
-      const client = await this.pressure.connect()
-      const transaction = new PostgresTransaction(client, this.queryTimeout)
+      const client = await this.connect()
+      const connectionId = this.idOfConnection(client)
+      const transaction = new PostgresTransaction(client, connectionId, this.queryTimeout)
       let lostReply: Error | undefined
       try {
-        await timedQuery(client, 'BEGIN', undefined, this.queryTimeout)
+        await timedQuery(client, connectionId, 'BEGIN', undefined, this.queryTimeout)
         const result = await operation(transaction)
         // Even when the operation caught it: COMMIT must not run after a lost reply.
         if (transaction.poisonedBy) throw transaction.poisonedBy
-        if (transaction.open) await timedQuery(client, 'COMMIT', undefined, this.queryTimeout)
+        if (transaction.open) {
+          await timedQuery(client, connectionId, 'COMMIT', undefined, this.queryTimeout)
+        }
         recordMeasuredHold(this.holds, transaction)
         this.holds.recordUnavailable(transaction.consumeLockUnavailable())
         this.holds.recordLockTimeout(transaction.consumeLockTimeouts())
@@ -1322,6 +1348,24 @@ export class PostgresDatabase implements RelayDatabase {
     await this.pool.end()
   }
 
+  private async connect(lane?: PostgresPoolLane): Promise<pg.PoolClient> {
+    const startedAt = performance.now()
+    try {
+      return await this.pressure.connect(lane)
+    } finally {
+      noteStickySlotPoolWait(performance.now() - startedAt)
+    }
+  }
+
+  private idOfConnection(client: pg.PoolClient): number {
+    let id = this.connectionIds.get(client)
+    if (id === undefined) {
+      id = this.nextConnectionId++
+      this.connectionIds.set(client, id)
+    }
+    return id
+  }
+
   consumePoolPressure(): PostgresPoolPressureCounts {
     return this.pressure.consumeCounts()
   }

cloud/apps/relay/src/drain-release-cell-row-contention-postgres.test.ts

@@ -32,7 +32,8 @@ const describePostgres = databaseUrl ? describe : describe.skip
 // asia-east2 cell to the us-central1 database, measured in production.
 const ASIA_ROUND_TRIP_MS = 171
 const LOCAL_ROUND_TRIP_MS = 1
-const WINDOW_MS = 10_000
+// Keep the measured contention rates; repeating them for longer only repeats the same locks.
+const WINDOW_MS = 5_000
 // Reconnecting hosts dialling the directors during the window. Above the
 // ~6-15/s the production tail sustained, so the director side is never idle.
 const DIAL_RATE_PER_SECOND = 20

cloud/apps/relay/src/host-session-preauth-latency.test.ts

@@ -0,0 +1,321 @@
+import { createHash, createHmac } from 'node:crypto'
+import { EventEmitter } from 'node:events'
+import {
+  buildHostProofMacInput,
+  HOST_CHALLENGE_PLAINTEXT_DOMAIN,
+  RELAY_CLOSE_CODE
+} from '@orca-cloud/relay-contract'
+import nacl from 'tweetnacl'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import type WebSocket from 'ws'
+import type { RelayAssignmentStore } from './assignment-store.js'
+import type { CellReserveBook } from './cell-reserve-book.js'
+import type { RelayConfig } from './config.js'
+import type { RelayCredentialStore } from './credential-store.js'
+import { HostSessionRegistry, type CellReserveAdmission } from './host-session-registry.js'
+import type { RelayRuntimeObserver } from './relay-observability.js'
+import type { RelayTokenClaims } from './relay-token-verifier.js'
+import { ProcessQueuedByteBudget } from './splice-forwarder.js'
+
+const CELL_ID = 'staging-gce-c1'
+
+class FakeSocket extends EventEmitter {
+  readonly CLOSING = 2
+  readonly OPEN = 1
+  readonly CLOSED = 3
+  readyState = this.OPEN
+  readonly send = vi.fn()
+  readonly close = vi.fn((code?: number) => {
+    this.readyState = this.CLOSED
+    this.emit('close', code, Buffer.from(''))
+  })
+
+  asWebSocket(): WebSocket {
+    // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: FakeSocket implements the registry's WebSocket event and lifecycle surface.
+    return this as unknown as WebSocket
+  }
+
+  // The peer's close frame: ws leaves OPEN at once and emits 'close' later.
+  hangUp(): void {
+    this.readyState = this.CLOSING
+  }
+}
+
+function hostKey(): { keyPair: nacl.BoxKeyPair; identity: RelayTokenClaims } {
+  const keyPair = nacl.box.keyPair()
+  const relayHostId = createHash('sha256')
+    .update(keyPair.publicKey)
+    .digest('base64url')
+    .slice(0, 16)
+  return {
+    keyPair,
+    identity: {
+      sub: 'user-1',
+      prof: 'profile-1',
+      relayHostId,
+      purpose: 'host-control',
+      exp: 4_102_444_800
+    }
+  }
+}
+
+function setup(input: {
+  admitMode: 'db' | 'reserve'
+  databaseSays: boolean
+  databaseShedding?: boolean
+}) {
+  let clock = 1_000_000
+  const recordHostPreAuth = vi.fn<NonNullable<RelayRuntimeObserver['recordHostPreAuth']>>()
+  const observer = {
+    recordAuth: vi.fn(),
+    recordForwardedBytes: vi.fn(),
+    recordHttp: vi.fn(),
+    recordReconnect: vi.fn(),
+    recordSql: vi.fn(),
+    recordHostPreAuth
+  } satisfies RelayRuntimeObserver
+  const take = vi.fn<CellReserveBook['take']>((userId, relayHostId, epoch) => ({
+    userId,
+    relayHostId,
+    epoch,
+    directorId: 'director-1',
+    expiresAt: clock + 60_000
+  }))
+  const reserveAdmission: CellReserveAdmission = {
+    mode: () => input.admitMode,
+    ticketEnforce: () => false,
+    dryRunEnabled: () => false,
+    databaseShedding: () => input.databaseShedding ?? false,
+    // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: beginProof calls only take() on the reserve book.
+    book: { take } as unknown as CellReserveBook,
+    verifyLease: vi.fn(async () => 'absent' as const)
+  }
+  let answerDatabase: (valid: boolean) => void = () => {}
+  const databaseAnswered = new Promise<boolean>((resolve) => {
+    answerDatabase = resolve
+  })
+  // Answers at once unless the test holds it with holdDatabase().
+  let held = false
+  const verifyCellAssignment = vi.fn(async () => (held ? databaseAnswered : input.databaseSays))
+  const registry = new HostSessionRegistry(
+    config(),
+    vi.fn(),
+    // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the host hello path never reads credentials.
+    {} as RelayCredentialStore,
+    // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the host hello path calls only verifyCellAssignment().
+    { verifyCellAssignment } as unknown as RelayAssignmentStore,
+    new ProcessQueuedByteBudget(),
+    observer,
+    () => clock,
+    Math.random,
+    'incarnation-1',
+    undefined,
+    reserveAdmission
+  )
+  return {
+    registry,
+    recordHostPreAuth,
+    verifyCellAssignment,
+    take,
+    holdDatabase: () => {
+      held = true
+    },
+    answerDatabase: (valid: boolean) => answerDatabase(valid),
+    advance: (ms: number) => {
+      clock += ms
+    }
+  }
+}
+
+function sendHello(socket: FakeSocket, keyPair: nacl.BoxKeyPair, relayHostId: string): void {
+  socket.emit(
+    'message',
+    Buffer.from(
+      JSON.stringify({
+        type: 'host-hello',
+        v: 1,
+        relayHostId,
+        assignmentEpoch: 4,
+        hostPublicKeyB64: Buffer.from(keyPair.publicKey).toString('base64'),
+        appVersion: '1.4.200'
+      })
+    ),
+    false
+  )
+}
+
+function answerChallenge(socket: FakeSocket, keyPair: nacl.BoxKeyPair): void {
+  const challenge: unknown = JSON.parse(String(socket.send.mock.calls[0]?.[0]))
+  const field = (name: string): string => {
+    const value: unknown =
+      typeof challenge === 'object' && challenge !== null ? Reflect.get(challenge, name) : null
+    if (typeof value !== 'string') throw new Error(`host-challenge has no ${name}`)
+    return value
+  }
+  const plaintext = nacl.box.open(
+    Buffer.from(field('ciphertextB64'), 'base64'),
+    Buffer.from(field('nonceB64'), 'base64'),
+    Buffer.from(field('relayEphemeralPublicKeyB64'), 'base64'),
+    keyPair.secretKey
+  )
+  if (!plaintext) throw new Error('host-challenge did not open')
+  const domainLength = new TextEncoder().encode(`${HOST_CHALLENGE_PLAINTEXT_DOMAIN}\0`).length
+  const transcriptLength = new DataView(
+    plaintext.buffer,
+    plaintext.byteOffset + domainLength,
+    4
+  ).getUint32(0, false)
+  const transcriptStart = domainLength + 4
+  const transcript = plaintext.slice(transcriptStart, transcriptStart + transcriptLength)
+  const secret = plaintext.slice(transcriptStart + transcriptLength)
+  socket.emit(
+    'message',
+    Buffer.from(
+      JSON.stringify({
+        type: 'host-challenge-ack',
+        challengeId: field('challengeId'),
+        proofB64: createHmac('sha256', secret)
+          .update(buildHostProofMacInput(transcript))
+          .digest('base64')
+      })
+    ),
+    false
+  )
+}
+
+describe('host pre-auth latency', () => {
+  afterEach(() => vi.useRealTimers())
+
+  it('records a booked hello on the reserve path, from accept to a verified proof', async () => {
+    const h = setup({ admitMode: 'reserve', databaseSays: false })
+    const { keyPair, identity } = hostKey()
+    const socket = new FakeSocket()
+    h.registry.acceptControl(socket.asWebSocket(), identity)
+    h.advance(30)
+    sendHello(socket, keyPair, identity.relayHostId)
+    await vi.waitFor(() => expect(socket.send).toHaveBeenCalledTimes(1))
+    expect(h.recordHostPreAuth).not.toHaveBeenCalled()
+    h.advance(45)
+    answerChallenge(socket, keyPair)
+
+    expect(h.take).toHaveBeenCalledOnce()
+    expect(h.verifyCellAssignment).not.toHaveBeenCalled()
+    expect(h.recordHostPreAuth).toHaveBeenCalledExactlyOnceWith('reserve', 'admitted', 75)
+  })
+
+  it('records a hello the database refuses on the db path', async () => {
+    const h = setup({ admitMode: 'db', databaseSays: false })
+    const { keyPair, identity } = hostKey()
+    const socket = new FakeSocket()
+    h.registry.acceptControl(socket.asWebSocket(), identity)
+    h.advance(120)
+    sendHello(socket, keyPair, identity.relayHostId)
+    await vi.waitFor(() => expect(socket.close).toHaveBeenCalled())
+
+    expect(socket.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.WRONG_CELL, 'wrong assignment epoch')
+    expect(h.take).not.toHaveBeenCalled()
+    expect(h.recordHostPreAuth).toHaveBeenCalledExactlyOnceWith('db', 'refused', 120)
+  })
+
+  it('records a hello timeout as a refusal before any check, once', () => {
+    vi.useFakeTimers()
+    const h = setup({ admitMode: 'db', databaseSays: true })
+    const { identity } = hostKey()
+    const socket = new FakeSocket()
+    h.registry.acceptControl(socket.asWebSocket(), identity)
+    h.advance(2_000)
+    vi.advanceTimersByTime(2_000)
+
+    expect(socket.close).toHaveBeenCalledWith(
+      RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL,
+      'host hello timeout'
+    )
+    expect(h.recordHostPreAuth).toHaveBeenCalledExactlyOnceWith('beforeCheck', 'refused', 2_000)
+  })
+
+  it('records a host that hangs up before its hello as abandoned', () => {
+    const h = setup({ admitMode: 'db', databaseSays: true })
+    const { identity } = hostKey()
+    const socket = new FakeSocket()
+    h.registry.acceptControl(socket.asWebSocket(), identity)
+    h.advance(300)
+    socket.close()
+
+    expect(h.recordHostPreAuth).toHaveBeenCalledExactlyOnceWith('beforeCheck', 'abandoned', 300)
+  })
+
+  it.each([true, false])(
+    'records a database answer (%s) that arrives after the host hung up as abandoned',
+    async (valid) => {
+      const h = setup({ admitMode: 'db', databaseSays: valid })
+      h.holdDatabase()
+      const { keyPair, identity } = hostKey()
+      const socket = new FakeSocket()
+      h.registry.acceptControl(socket.asWebSocket(), identity)
+      sendHello(socket, keyPair, identity.relayHostId)
+      await vi.waitFor(() => expect(h.verifyCellAssignment).toHaveBeenCalled())
+      socket.hangUp()
+      h.advance(900)
+      h.answerDatabase(valid)
+      await vi.waitFor(() => expect(h.recordHostPreAuth).toHaveBeenCalled())
+
+      expect(socket.send).not.toHaveBeenCalled()
+      expect(h.recordHostPreAuth).toHaveBeenCalledExactlyOnceWith('db', 'abandoned', 900)
+    }
+  )
+
+  it('records a host that hangs up during its proof as abandoned on its path', async () => {
+    const h = setup({ admitMode: 'reserve', databaseSays: false })
+    const { keyPair, identity } = hostKey()
+    const socket = new FakeSocket()
+    h.registry.acceptControl(socket.asWebSocket(), identity)
+    sendHello(socket, keyPair, identity.relayHostId)
+    await vi.waitFor(() => expect(socket.send).toHaveBeenCalledTimes(1))
+    h.advance(700)
+    socket.close()
+
+    expect(h.recordHostPreAuth).toHaveBeenCalledExactlyOnceWith('reserve', 'abandoned', 700)
+  })
+
+  it('records the pool-pressure shed as a refusal before any check', async () => {
+    const h = setup({ admitMode: 'reserve', databaseSays: true, databaseShedding: true })
+    h.take.mockReturnValue(null)
+    const { keyPair, identity } = hostKey()
+    const socket = new FakeSocket()
+    h.registry.acceptControl(socket.asWebSocket(), identity)
+    h.advance(40)
+    sendHello(socket, keyPair, identity.relayHostId)
+    await vi.waitFor(() => expect(socket.close).toHaveBeenCalled())
+
+    expect(h.verifyCellAssignment).not.toHaveBeenCalled()
+    expect(h.recordHostPreAuth).toHaveBeenCalledExactlyOnceWith('beforeCheck', 'refused', 40)
+  })
+})
+
+function config(): RelayConfig {
+  return {
+    port: 8080,
+    publicUrl: 'https://c1.relay.example.test',
+    cellUrl: 'https://c1.relay.example.test',
+    authIssuer: 'https://auth.example.test',
+    authAudience: 'orca-relay',
+    jwksUrl: 'https://auth.example.test/jwks',
+    assignmentSigningKey: new TextEncoder().encode('0123456789abcdef0123456789abcdef'),
+    role: 'cell',
+    cellId: CELL_ID,
+    cells: [],
+    adminAudience: 'https://relay.example.test/v1/admin/drain',
+    deployServiceAccount: 'deploy@example.test',
+    runtimeServiceAccount: 'relay-cell@example.test',
+    adminJwksUrl: 'https://auth.example.test/jwks',
+    databasePoolMax: 10,
+    publicAssignmentsEnabled: true,
+    publicAssignmentConcurrency: 2,
+    publicAssignmentQueueMax: 128,
+    publicAssignmentWaitMs: 4_000,
+    publicResolveConcurrency: 1,
+    publicResolveWaitMs: 5_000,
+    publicAssignmentRetryAfterSeconds: 5,
+    dataDir: './data'
+  }
+}

cloud/apps/relay/src/host-session-registry.ts

@@ -42,6 +42,8 @@ import { relayHostLogDigest } from './relay-host-log-digest.js'
 import type { RelayTokenClaims } from './relay-token-verifier.js'
 import {
   percentile,
+  type HostPreAuthOutcome,
+  type HostPreAuthPath,
   type RelayClientAcceptStage,
   type RelayClientAcceptTimedStage,
   type RelayRuntimeObserver
@@ -186,26 +188,44 @@ function readControlFrame(
   socket: WebSocket,
   timeoutMs: number,
   timeoutReason: string,
-  receive: (raw: RawData, isBinary: boolean) => void
+  receive: (raw: RawData, isBinary: boolean) => void,
+  ended?: (how: 'timeout' | 'closed') => void
 ): void {
-  if (socket.readyState !== socket.OPEN) return
+  if (socket.readyState !== socket.OPEN) {
+    ended?.('closed')
+    return
+  }
   const timer = setTimeout(() => {
     finish()
+    ended?.('timeout')
     socket.close(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, timeoutReason)
   }, timeoutMs)
   function finish(): void {
     clearTimeout(timer)
     socket.off('message', onMessage)
-    socket.off('close', finish)
+    socket.off('close', onClose)
+  }
+  function onClose(): void {
+    finish()
+    ended?.('closed')
   }
   function onMessage(raw: RawData, isBinary: boolean): void {
     finish()
     receive(raw, isBinary)
   }
   socket.once('message', onMessage)
-  socket.once('close', finish)
+  socket.once('close', onClose)
 }
 
+// Records one host control's pre-auth time, from the cell accepting the WebSocket to the
+// hello's admission and proof being decided, exactly once.
+type SettlePreAuth = (outcome: HostPreAuthOutcome, path: HostPreAuthPath) => void
+
+const preAuthEnded =
+  (settle: SettlePreAuth, path: HostPreAuthPath) =>
+  (how: 'timeout' | 'closed'): void =>
+    settle(how === 'timeout' ? 'refused' : 'abandoned', path)
+
 // Hosts abandon connects after 15s; waiting much longer than that behind a
 // stalled predecessor only accumulates doomed sockets.
 const ACTIVATION_QUEUE_WAIT_MS = 30_000
@@ -699,7 +719,7 @@ export class HostSessionRegistry {
 
   // A director names this seat the loser of a duplicate: stop taking phones for it now, and
   // close it once its splices have had time to finish. Repeats are one demotion.
-  demote(request: DemoteRequest): 'demoted' | 'already-demoted' | 'not-seated' | 'off' {
+  demote(request: DemoteRequest): 'demoted' | 'already-demoted' | 'draining' | 'not-seated' | 'off' {
     // Only a reserve-mode cell takes demotions, so flipping admitMode back reverts the route.
     if (!this.reserveMode()) return 'off'
     const session = this.get(request)
@@ -715,6 +735,8 @@ export class HostSessionRegistry {
     ) {
       return 'not-seated'
     }
+    // A drain already told this host to go: its own grace stands, never cut to the demotion's.
+    if (this.drainSentHosts.has(session.relayHostId) || session.regionalDrainAttemptId) return 'draining'
     const socket = session.socket
     if (this.demotedSeats.get(key) === socket) return 'already-demoted'
     this.demotedSeats.set(key, socket)
@@ -1212,35 +1234,64 @@ export class HostSessionRegistry {
     hostCapabilities?: ReadonlySet<string>,
     assignmentLease?: string
   ): void {
+    const settlePreAuth = this.startPreAuth(socket)
+    const refuse = (code: number, reason: string): void => {
+      settlePreAuth('refused', 'beforeCheck')
+      socket.close(code, reason)
+    }
     if (this.idleAttempts.has(identity.relayHostId)) {
-      socket.close(RELAY_CLOSE_CODE.WRONG_CELL, 'idle cutover in progress')
+      refuse(RELAY_CLOSE_CODE.WRONG_CELL, 'idle cutover in progress')
       return
     }
     // Keyed by socket, not session: a rebind swaps the session's socket, and the
     // successor's own advertisement is the only one that describes its decoder.
     if (hostCapabilities?.size) this.hostCapabilities.set(socket, hostCapabilities)
     if (this.draining) {
-      socket.close(RELAY_CLOSE_CODE.DRAINING, 'relay draining')
+      refuse(RELAY_CLOSE_CODE.DRAINING, 'relay draining')
       return
     }
-    readControlFrame(socket, 2_000, 'host hello timeout', (raw, isBinary) => {
-      if (isBinary) {
-        socket.close(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'host hello must be text')
-        return
-      }
-      this.guardSessionTask(
-        () =>
-          this.beginProof(
-            socket,
-            identity,
-            payload(raw, 'host-hello'),
-            connectionInclusionWatermark,
-            assignmentLease
-          ),
-        socket,
-        'host hello proof'
-      )
-    })
+    readControlFrame(
+      socket,
+      2_000,
+      'host hello timeout',
+      (raw, isBinary) => {
+        if (isBinary) {
+          refuse(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'host hello must be text')
+          return
+        }
+        this.guardSessionTask(
+          () =>
+            this.beginProof(
+              socket,
+              identity,
+              payload(raw, 'host-hello'),
+              settlePreAuth,
+              connectionInclusionWatermark,
+              assignmentLease
+            ).catch((error: unknown) => {
+              // guardSessionTask closes the socket, so any throw refuses the hello as beforeCheck,
+              // including one after the database answered (e.g. in send or the session lookup).
+              settlePreAuth('refused', 'beforeCheck')
+              throw error
+            }),
+          socket,
+          'host hello proof'
+        )
+      },
+      preAuthEnded(settlePreAuth, 'beforeCheck')
+    )
+  }
+
+  private startPreAuth(socket: WebSocket): SettlePreAuth {
+    const startedAt = this.now()
+    let settled = false
+    return (outcome, path) => {
+      if (settled) return
+      settled = true
+      // A decision made after the host hung up never reached it, so it counts as abandoned.
+      const recorded = socket.readyState === socket.OPEN ? outcome : 'abandoned'
+      this.observer.recordHostPreAuth?.(path, recorded, nonNegativeMs(this.now() - startedAt))
+    }
   }
 
   // A dependency failure (e.g. a database connect timeout) must cost one
@@ -1379,29 +1430,34 @@ export class HostSessionRegistry {
     socket: WebSocket,
     identity: RelayTokenClaims,
     candidate: unknown,
+    settlePreAuth: SettlePreAuth,
     connectionInclusionWatermark?: number,
     assignmentLease?: string
   ): Promise<void> {
+    // Which check decided the hello; refusals before one answers stay 'beforeCheck'.
+    let path: HostPreAuthPath = 'beforeCheck'
+    const refuse = (code: number, reason: string): void => {
+      settlePreAuth('refused', path)
+      this.observer.recordAuth(false)
+      socket.close(code, reason)
+    }
     const hello = HostHelloSchema.safeParse(candidate)
     const hostPublicKey = hello.success
       ? decodeCanonicalBase64(hello.data.hostPublicKeyB64, 32)
       : null
     if (!hello.success) {
-      this.observer.recordAuth(false)
-      socket.close(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'invalid host hello')
+      refuse(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'invalid host hello')
       return
     }
     if (!hostPublicKey) {
-      this.observer.recordAuth(false)
-      socket.close(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'invalid host public key')
+      refuse(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'invalid host public key')
       return
     }
     if (
       hello.data.relayHostId !== identity.relayHostId ||
       relayHostId(hostPublicKey) !== identity.relayHostId
     ) {
-      this.observer.recordAuth(false)
-      socket.close(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'host key binding mismatch')
+      refuse(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'host key binding mismatch')
       return
     }
     const leaseShadow = this.config.role === 'cell' ? this.assignmentLeaseShadow : undefined
@@ -1434,12 +1490,12 @@ export class HostSessionRegistry {
             relayHostId: identity.relayHostId,
             helloEpoch: hello.data.assignmentEpoch
           })) === 'valid')
+      if (admittedFromMemory) path = 'reserve'
     }
     if (reserveMode && !admittedFromMemory && this.reserveAdmission!.databaseShedding()) {
       // Memory cannot admit it (e.g. seated here, but at another epoch), and the wedged pool
       // would only time out its database check: shed it now, as the upgrade shed does.
-      this.observer.recordAuth(false)
-      socket.close(RELAY_CLOSE_CODE.LIMIT_EXCEEDED, 'relay temporarily unavailable')
+      refuse(RELAY_CLOSE_CODE.LIMIT_EXCEEDED, 'relay temporarily unavailable')
       return
     }
     const assignmentReadStartedAt = performance.now()
@@ -1462,12 +1518,16 @@ export class HostSessionRegistry {
         performance.now() - assignmentReadStartedAt
       )
     }
+    // The `combined` role compares the epoch without querying but is still labelled db.
+    if (!admittedFromMemory) path = 'db'
     if (!assignmentValid) {
-      this.observer.recordAuth(false)
-      socket.close(RELAY_CLOSE_CODE.WRONG_CELL, 'wrong assignment epoch')
+      refuse(RELAY_CLOSE_CODE.WRONG_CELL, 'wrong assignment epoch')
+      return
+    }
+    if (socket.readyState !== socket.OPEN) {
+      settlePreAuth('abandoned', path)
       return
     }
-    if (socket.readyState !== socket.OPEN) return
 
     const key = this.key(identity.sub, identity.relayHostId)
     const existing = this.sessions.get(key)
@@ -1513,40 +1573,46 @@ export class HostSessionRegistry {
       ciphertextB64: Buffer.from(ciphertext).toString('base64'),
       expiresAt
     })
-    readControlFrame(socket, 10_000, 'host proof timeout', (raw, isBinary) => {
-      const ack = isBinary
-        ? null
-        : HostChallengeAckSchema.safeParse(payload(raw, 'host-challenge-ack'))
-      const proof = ack?.success ? decodeCanonicalBase64(ack.data.proofB64, 32) : null
-      if (
-        !ack?.success ||
-        ack.data.challengeId !== challengeId ||
-        !proof ||
-        this.now() > expiresAt ||
-        !timingSafeEqual(proof, expectedProof)
-      ) {
-        this.observer.recordAuth(false)
-        socket.close(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'invalid host proof')
-        return
-      }
-      this.observer.recordAuth(true)
-      this.guardSessionTask(
-        () =>
-          this.activate(
-            socket,
-            identity,
-            existing ?? null,
-            generation,
-            rebind,
-            hello.data.assignmentEpoch,
-            hello.data.appVersion,
-            connectionInclusionWatermark,
-            admittedFromMemory ? { reservedBy } : undefined
-          ),
-        socket,
-        'host activation'
-      )
-    })
+    readControlFrame(
+      socket,
+      10_000,
+      'host proof timeout',
+      (raw, isBinary) => {
+        const ack = isBinary
+          ? null
+          : HostChallengeAckSchema.safeParse(payload(raw, 'host-challenge-ack'))
+        const proof = ack?.success ? decodeCanonicalBase64(ack.data.proofB64, 32) : null
+        if (
+          !ack?.success ||
+          ack.data.challengeId !== challengeId ||
+          !proof ||
+          this.now() > expiresAt ||
+          !timingSafeEqual(proof, expectedProof)
+        ) {
+          refuse(RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'invalid host proof')
+          return
+        }
+        settlePreAuth('admitted', path)
+        this.observer.recordAuth(true)
+        this.guardSessionTask(
+          () =>
+            this.activate(
+              socket,
+              identity,
+              existing ?? null,
+              generation,
+              rebind,
+              hello.data.assignmentEpoch,
+              hello.data.appVersion,
+              connectionInclusionWatermark,
+              admittedFromMemory ? { reservedBy } : undefined
+            ),
+          socket,
+          'host activation'
+        )
+      },
+      preAuthEnded(settlePreAuth, path)
+    )
   }
 
   private activate(

cloud/apps/relay/src/postgres-lost-reply-postgres.test.ts

@@ -22,6 +22,7 @@ const PROBE_TABLE = 'relay_lost_reply_probe'
 async function blackholeProxy(target: URL, trigger: string, delayMs?: number) {
   const sockets = new Set<Socket>()
   const afterTrigger: Buffer[] = []
+  let forwardingResumed = false
   const server: Server = createServer((client) => {
     const upstream = connect(Number(target.port || 5432), target.hostname)
     sockets.add(client)
@@ -33,7 +34,7 @@ async function blackholeProxy(target: URL, trigger: string, delayMs?: number) {
       upstream.write(chunk)
     })
     upstream.on('data', (chunk: Buffer) => {
-      if (!swallowing) client.write(chunk)
+      if (!swallowing || forwardingResumed) client.write(chunk)
       else if (delayMs !== undefined) setTimeout(() => client.write(chunk), delayMs)
     })
     for (const socket of [client, upstream]) {
@@ -52,6 +53,9 @@ async function blackholeProxy(target: URL, trigger: string, delayMs?: number) {
   url.port = String(address.port)
   return {
     url: url.toString(),
+    resumeForwarding: () => {
+      forwardingResumed = true
+    },
     sentAfterTrigger: () => Buffer.concat(afterTrigger).toString('latin1'),
     close: async () => {
       for (const socket of sockets) socket.destroy()
@@ -174,6 +178,7 @@ describePostgres('a lost PostgreSQL reply against a real server', () => {
       { probe: 'stalled_reply_probe' }
     ])
     expect(performance.now() - startedAt).toBeGreaterThan(6_000)
+    proxy.resumeForwarding()
     // Not destroyed: the one pooled connection is still the same backend.
     expect(await pid()).toBe(before)
   }, 30_000)

cloud/apps/relay/src/public-assignment-drain-return-lane.test.ts

@@ -233,17 +233,17 @@ describe('drain-return lane', () => {
     expect(redial.headers.get('retry-after')).toBe('2')
   })
 
-  it('admits a 2,500-host drain across five directors without a sticky or placement rejection', async () => {
+  it('admits a paced drain across five directors without a sticky or placement rejection', async () => {
     vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] })
     vi.spyOn(console, 'warn').mockImplementation(() => {})
     // Five directors behind a load balancer that sends each request to any of
-    // them. 2,500 drained hosts over the 300 s pace window, beside ordinary
-    // reconnects (c29 baseline ~13/s) and a trickle of new placements.
+    // them. A shorter cohort preserves the production arrival rates: ~8 drained
+    // hosts/s, ~13 ordinary reconnects/s, and a trickle of new placements.
     const DIRECTORS = 5
-    const DRAINED = 2_500
-    const PACE_WINDOW_MS = 300_000
-    const ORDINARY = 3_900
-    const PLACEMENTS = 30
+    const DRAINED = 250
+    const PACE_WINDOW_MS = 30_000
+    const ORDINARY = 390
+    const PLACEMENTS = 3
     // Measured c29 2026-10-01: ~5.8 re-placements/s fleet-wide, bound by the
     // region's inventory lock, so ~172 ms of lock per re-placement.
     const INVENTORY_LOCK_MS = 172
@@ -348,7 +348,7 @@ describe('drain-return lane', () => {
     }
 
     let elapsedMs = 0
-    while (inFlight > 0 && elapsedMs < 1_200_000) {
+    while (inFlight > 0 && elapsedMs < 120_000) {
       await vi.advanceTimersByTimeAsync(20)
       elapsedMs += 20
       // Signing the lease completes off the timer queue.
@@ -363,10 +363,11 @@ describe('drain-return lane', () => {
     expect(outcomes.get('sticky')).toBe(ORDINARY)
     expect(outcomes.get('placement')).toBe(PLACEMENTS)
     expect(outcomes.get('drain-return')).toBe(DRAINED)
+    expect(outcomes.get('drain-return-deferred') ?? 0).toBeGreaterThan(0)
     // Paced returns: fewer than two deferrals per drained host, none past the cap.
     expect(outcomes.get('drain-return-deferred') ?? 0).toBeLessThan(2 * DRAINED)
     expect(Math.max(...retryAfters)).toBeLessThanOrEqual(300)
-    // The cohort finishes at the inventory lock's rate, not minutes behind it.
+    // Retry rounding and off-thread signing keep their original fixed allowance.
     expect(elapsedMs).toBeLessThan(DRAINED * INVENTORY_LOCK_MS + 60_000)
   }, 300_000)
 })

cloud/apps/relay/src/public-assignment-reconnect-lane.test.ts

@@ -1,6 +1,11 @@
 import { describe, expect, it, vi } from 'vitest'
 import { RelayAssignmentRowBusyError, type RelayAssignment } from './assignment-store.js'
 import type { RelayConfig } from './config.js'
+import type { StickySlotSample } from './sticky-slot-statements.js'
+import {
+  ScriptedPoolClient,
+  scriptedPostgresDatabase
+} from './test-fixtures/scripted-postgres-pool.js'
 
 const fakes = vi.hoisted(() => ({
   verifyRelayToken: vi.fn(async (token: string) => ({
@@ -199,6 +204,64 @@ describe('public assignment reconnect lane', () => {
     warn.mockRestore()
   })
 
+  it('reports what each sticky slot spent its hold on', async () => {
+    const first = 'ffffffffffffffff'
+    const queued = 'qqqqqqqqqqqqqqqq'
+    const newcomer = 'nnnnnnnnnnnnnnnn'
+    const firstHost = new ScriptedPoolClient()
+    const queuedHost = new ScriptedPoolClient()
+    // first: verify, assign; newcomer: assign; queued: verify, assign.
+    const database = scriptedPostgresDatabase([
+      firstHost,
+      firstHost,
+      new ScriptedPoolClient(),
+      queuedHost,
+      queuedHost
+    ])
+    const firstAssign = deferred<RelayAssignment>()
+    const resolve = vi.fn(async ({ relayHostId }: { relayHostId: string }) => {
+      await database.query('SELECT verify')
+      return assignment('cell-r', relayHostId)
+    })
+    const assign = vi.fn(async ({ relayHostId }: { relayHostId: string }) => {
+      await database.transaction(async (transaction) => {
+        await transaction.query('SELECT pin')
+        await transaction.query('UPDATE lease')
+      })
+      if (relayHostId === first) return await firstAssign.promise
+      return assignment('cell-r', relayHostId)
+    })
+    const samples: StickySlotSample[] = []
+    const app = createRelayApp(config(), {
+      store: {} as never,
+      assignments: { assign, resolve } as never,
+      drain: vi.fn(),
+      ready: vi.fn(async () => true),
+      recordStickySlot: (sample) => samples.push(sample)
+    })
+
+    const holding = app.request('/v1/assign', assignmentRequest(first, { reconnect: true }))
+    await vi.waitFor(() => expect(firstHost.statements).toHaveLength(5))
+    const waiting = app.request('/v1/assign', assignmentRequest(queued, { reconnect: true }))
+    // A placement request's statements are its own, even while a sticky slot is held.
+    expect((await app.request('/v1/assign', assignmentRequest(newcomer))).status).toBe(200)
+    await new Promise((resolveDelay) => setTimeout(resolveDelay, 60))
+    firstAssign.resolve(assignment('cell-r', first))
+    expect((await holding).status).toBe(200)
+    expect((await waiting).status).toBe(200)
+
+    // Verify, then BEGIN, two statements and COMMIT, each on that host's own connection.
+    const roundTrips = (sample: StickySlotSample | undefined) =>
+      [...(sample?.statementMsByConnection ?? [])].map(([id, ms]) => [id, ms.length])
+    expect(samples).toHaveLength(2)
+    expect(samples[0]).toMatchObject({ statements: 5 })
+    expect(roundTrips(samples[0])).toEqual([[1, 5]])
+    expect(samples[0]?.queueWaitMs).toBeLessThan(40)
+    expect(samples[1]).toMatchObject({ statements: 5 })
+    expect(roundTrips(samples[1])).toEqual([[3, 5]])
+    expect(samples[1]?.queueWaitMs).toBeGreaterThanOrEqual(40)
+  })
+
   it('never probes for unhinted requests', async () => {
     const host = 'uuuuuuuuuuuuuuuu'
     const assign = vi.fn(async () => assignment('cell-u', host))

cloud/apps/relay/src/relay-observability.test.ts

@@ -5,8 +5,10 @@ import { observeRelayDatabase } from './observed-relay-database.js'
 import {
   CONTROL_RTT_RESERVOIR_LIMIT,
   observedRelayRequests,
+  PRE_AUTH_RESERVOIR_LIMIT,
   percentile,
   RelayObservability,
+  STICKY_CONNECTION_LIMIT,
   type RelayProcessCounts
 } from './relay-observability.js'
 import { RELAY_FIX_LEVEL } from './relay-fix-level.js'
@@ -283,6 +285,75 @@ describe('relay observability', () => {
     expect(entries[1]).not.toHaveProperty('drainReturnServiceMsP50')
   })
 
+  it('summarises sticky slots per window and per pooled connection', () => {
+    const entries: Array<Record<string, unknown>> = []
+    const observability = new RelayObservability(
+      { role: 'director', cellId: 'director', region: 'us-central1' },
+      (entry) => entries.push(entry)
+    )
+    for (let ms = 1; ms <= 100; ms++) {
+      observability.recordStickySlot({
+        queueWaitMs: ms * 20,
+        poolWaitMs: ms,
+        sqlMs: ms * 2,
+        statements: 14,
+        // The slow connection serves every other request.
+        statementMsByConnection: new Map([ms % 2 === 0 ? [1, [300]] : [2, [5, 6]]])
+      })
+    }
+    observability.flush(counts)
+    observability.flush(counts)
+
+    expect(entries[0]).toMatchObject({
+      stickySlotsDelta: 100,
+      stickyQueueWaitMsP50: 1_000,
+      stickyQueueWaitMsP99: 1_980,
+      stickyQueueWaitMsMax: 2_000,
+      stickyPoolWaitMsP50: 50,
+      stickyPoolWaitMsP99: 99,
+      stickySqlMsP50: 100,
+      stickySqlMsP99: 198,
+      stickyStatementsP50: 14,
+      stickyStatementsMax: 14,
+      stickySqlByConnection: {
+        '1': {
+          requests: 50,
+          statements: 50,
+          statementMsP50: 300,
+          statementMsP99: 300,
+          statementMsMax: 300
+        },
+        '2': { requests: 50, statements: 100, statementMsP50: 5, statementMsP99: 6 }
+      },
+      stickySqlConnectionSamplesDroppedDelta: 0
+    })
+    expect(entries[1]).not.toHaveProperty('stickySlotsDelta')
+    expect(entries[1]).not.toHaveProperty('stickySqlByConnection')
+  })
+
+  it('bounds the per-connection sticky summary', () => {
+    const entries: Array<Record<string, unknown>> = []
+    const observability = new RelayObservability(
+      { role: 'director', cellId: 'director', region: 'us-central1' },
+      (entry) => entries.push(entry)
+    )
+    for (let id = 1; id <= STICKY_CONNECTION_LIMIT + 3; id++) {
+      observability.recordStickySlot({
+        queueWaitMs: 0,
+        poolWaitMs: 0,
+        sqlMs: 1,
+        statements: 1,
+        statementMsByConnection: new Map([[id, [1]]])
+      })
+    }
+    observability.flush(counts)
+
+    expect(Object.keys(entries[0]?.stickySqlByConnection as object)).toHaveLength(
+      STICKY_CONNECTION_LIMIT
+    )
+    expect(entries[0]).toMatchObject({ stickySqlConnectionSamplesDroppedDelta: 3 })
+  })
+
   it('sums lock waiters per role and table over the samples taken', () => {
     const entries: Array<Record<string, unknown>> = []
     const observability = new RelayObservability(
@@ -570,6 +641,109 @@ describe('relay observability', () => {
     expect(entries[0]!.controlRttMsMax).toBe(900)
   })
 
+  it('summarises host pre-auth latency per path and outcome', () => {
+    const entries: Array<Record<string, unknown>> = []
+    const observability = new RelayObservability(
+      { role: 'cell', cellId: 'staging-gce-c1', region: 'us-central1' },
+      (entry) => entries.push(entry)
+    )
+    for (let ms = 1; ms <= 100; ms++) observability.recordHostPreAuth('db', 'admitted', ms)
+    observability.recordHostPreAuth('db', 'abandoned', 6_000)
+    observability.recordHostPreAuth('beforeCheck', 'refused', 2_000)
+    observability.recordHostPreAuth('reserve', 'admitted', 12.3456)
+    observability.flush(counts)
+    observability.flush(counts)
+
+    const none = { admitted: { count: 0 }, refused: { count: 0 }, abandoned: { count: 0 } }
+    const buckets = (filled: Partial<Record<string, number>>): Record<string, number> => ({
+      '50': 0,
+      '100': 0,
+      '200': 0,
+      '300': 0,
+      '450': 0,
+      '700': 0,
+      '1000': 0,
+      '2000': 0,
+      '5000': 0,
+      '+Inf': 0,
+      ...filled
+    })
+    expect(entries[0]!.preAuthLatencyMs).toEqual({
+      beforeCheck: {
+        ...none,
+        refused: {
+          count: 1,
+          p50: 2_000,
+          p95: 2_000,
+          p99: 2_000,
+          max: 2_000,
+          bucketsLeMs: buckets({ '2000': 1 })
+        }
+      },
+      db: {
+        ...none,
+        admitted: {
+          count: 100,
+          p50: 50,
+          p95: 95,
+          p99: 99,
+          max: 100,
+          bucketsLeMs: buckets({ '50': 50, '100': 50 })
+        },
+        abandoned: {
+          count: 1,
+          p50: 6_000,
+          p95: 6_000,
+          p99: 6_000,
+          max: 6_000,
+          bucketsLeMs: buckets({ '+Inf': 1 })
+        }
+      },
+      reserve: {
+        ...none,
+        admitted: {
+          count: 1,
+          p50: 12.346,
+          p95: 12.346,
+          p99: 12.346,
+          max: 12.346,
+          bucketsLeMs: buckets({ '50': 1 })
+        }
+      }
+    })
+    // The window resets, and an empty window publishes counts only.
+    expect(entries[1]!.preAuthLatencyMs).toEqual({ beforeCheck: none, db: none, reserve: none })
+  })
+
+  it('counts every hello in the pre-auth buckets, inclusive of each bound', () => {
+    const entries: Array<Record<string, unknown>> = []
+    const observability = new RelayObservability(
+      { role: 'cell', cellId: 'staging-gce-c1', region: 'us-central1' },
+      (entry) => entries.push(entry)
+    )
+    const flooded = PRE_AUTH_RESERVOIR_LIMIT * 5
+    for (let sample = 0; sample < flooded; sample++) {
+      observability.recordHostPreAuth('reserve', 'admitted', 450)
+    }
+    observability.recordHostPreAuth('reserve', 'admitted', 450.001)
+    observability.recordHostPreAuth('reserve', 'admitted', 9_000)
+    observability.flush(counts)
+
+    expect(entries[0]).toMatchObject({
+      preAuthLatencyMs: {
+        reserve: {
+          admitted: {
+            // Exact past the reservoir cap, even when it declined the slowest hello.
+            count: flooded + 2,
+            max: 9_000,
+            p50: 450,
+            bucketsLeMs: { '300': 0, '450': flooded, '700': 1, '5000': 0, '+Inf': 1 }
+          }
+        }
+      }
+    })
+  })
+
   it('observes successful and failed database calls including transactions', async () => {
     const recordSql = vi.fn()
     const underlying: RelayDatabase = {

cloud/apps/relay/src/relay-observability.ts

@@ -6,6 +6,7 @@ import type { CellInventoryHoldCounts } from './cell-inventory-hold-samples.js'
 import type { PostgresPoolPressureCounts } from './postgres-pool-pressure.js'
 import type { RelayReadinessGraceEvent, RelayReadinessObservation } from './relay-readiness.js'
 import { RELAY_FIX_LEVEL } from './relay-fix-level.js'
+import type { StickySlotSample } from './sticky-slot-statements.js'
 
 export type RelayRuntimeCounts = {
   totalConnections: number
@@ -90,6 +91,7 @@ export interface RelayRuntimeObserver {
   recordAssignmentRejectionReason?(lane: AssignmentAdmissionLane, reason: string): void
   recordDrainReturnRetryAfter?(seconds: number): void
   recordAdmissionServiceMs?(lane: 'sticky' | 'drain-return', durationMs: number): void
+  recordStickySlot?(sample: StickySlotSample): void
   recordAssignmentUnavailable?(cause: AssignmentUnavailableCause): void
   recordDatabaseLockWaitSample?(sample: DatabaseLockWaitSample): void
   recordRegionRequest?(region: RelayRegion | undefined): void
@@ -103,6 +105,42 @@ export interface RelayRuntimeObserver {
   recordClientAcceptAbandoned?(stage: RelayClientAcceptStage, elapsedMs: number): void
   recordClientAcceptCompleted?(sample: RelayClientAcceptSample): void
   recordControlRtt?(rttMs: number): void
+  recordHostPreAuth?(path: HostPreAuthPath, outcome: HostPreAuthOutcome, durationMs: number): void
+}
+
+// Which check decided a host hello: the cell's memory in reserve mode (a booking, a seat it
+// remembers, or a lease), the database assignment check, or neither (beforeCheck: draining,
+// idle cutover, a bad or late hello, the pool shed, or a check that threw).
+export type HostPreAuthPath = 'beforeCheck' | 'db' | 'reserve'
+// Abandoned: the host hung up before a decision, or before the cell could act on one.
+export type HostPreAuthOutcome = 'admitted' | 'refused' | 'abandoned'
+
+// Upper bounds in ms; the last bucket is unbounded. 450 is the step-5 t_preauth gate.
+const PRE_AUTH_BUCKET_BOUNDS_MS = [50, 100, 200, 300, 450, 700, 1_000, 2_000, 5_000] as const
+
+type PreAuthLatencyGroup = {
+  samplesMs: number[]
+  observed: number
+  maxMs: number
+  // Non-cumulative; one more slot than there are bounds, for the unbounded bucket.
+  bucketCounts: number[]
+}
+type PreAuthLatency = Record<HostPreAuthPath, Record<HostPreAuthOutcome, PreAuthLatencyGroup>>
+
+const emptyPreAuthLatencyGroup = (): PreAuthLatencyGroup => ({
+  samplesMs: [],
+  observed: 0,
+  maxMs: 0,
+  bucketCounts: Array.from({ length: PRE_AUTH_BUCKET_BOUNDS_MS.length + 1 }, () => 0)
+})
+
+const emptyPreAuthLatency = (): PreAuthLatency => {
+  const byOutcome = (): Record<HostPreAuthOutcome, PreAuthLatencyGroup> => ({
+    admitted: emptyPreAuthLatencyGroup(),
+    refused: emptyPreAuthLatencyGroup(),
+    abandoned: emptyPreAuthLatencyGroup()
+  })
+  return { beforeCheck: byOutcome(), db: byOutcome(), reserve: byOutcome() }
 }
 
 // Which serialized accept step the phone had already hung up behind.
@@ -148,6 +186,12 @@ type RelayMetricDeltas = {
   drainReturnRetryAfterSecondsSum: number
   drainReturnServiceMs: number[]
   stickyServiceMs: number[]
+  stickySlotQueueWaitMs: number[]
+  stickySlotPoolWaitMs: number[]
+  stickySlotSqlMs: number[]
+  stickySlotStatements: number[]
+  stickySlotByConnection: Map<number, StickyConnectionStatements>
+  stickySlotConnectionSamplesDropped: number
   assign503sByCause: Record<string, number>
   dbLockWaitSamples: number
   dbLockWaitersByKey: Record<string, number>
@@ -170,11 +214,41 @@ type RelayMetricDeltas = {
   controlActivityRecoveries: number
   controlActivityRecoveryFailures: number
   hostHellosShed: number
+  preAuthLatency: PreAuthLatency
 }
 
+type StickyConnectionStatements = {
+  requests: number
+  statements: number
+  maxMs: number
+  samplesMs: number[]
+}
+
+// A window sees each pooled connection the sticky lane used; a pool that keeps
+// replacing connections must not grow the line without bound.
+export const STICKY_CONNECTION_LIMIT = 64
+export const STICKY_STATEMENT_RESERVOIR_LIMIT = 1024
+
 // A host chooses how often it answers a ping, so the process-wide window is a
 // reservoir: the heap cost of a flood is capped and the percentiles stay unbiased.
 export const CONTROL_RTT_RESERVOIR_LIMIT = 1024
+// Per path and outcome; a reconnect burst can put thousands of hellos in one window.
+export const PRE_AUTH_RESERVOIR_LIMIT = 1024
+
+// Algorithm R: every sample in the window keeps an equal chance of being kept.
+function keepInReservoir(
+  samples: number[],
+  observedBefore: number,
+  value: number,
+  limit: number
+): void {
+  if (samples.length < limit) {
+    samples.push(value)
+    return
+  }
+  const slot = Math.floor(Math.random() * (observedBefore + 1))
+  if (slot < limit) samples[slot] = value
+}
 
 type MetricWriter = (entry: Record<string, unknown>) => void
 
@@ -200,6 +274,12 @@ const emptyDeltas = (): RelayMetricDeltas => ({
   drainReturnRetryAfterSecondsSum: 0,
   drainReturnServiceMs: [],
   stickyServiceMs: [],
+  stickySlotQueueWaitMs: [],
+  stickySlotPoolWaitMs: [],
+  stickySlotSqlMs: [],
+  stickySlotStatements: [],
+  stickySlotByConnection: new Map(),
+  stickySlotConnectionSamplesDropped: 0,
   assign503sByCause: {},
   dbLockWaitSamples: 0,
   dbLockWaitersByKey: {},
@@ -227,7 +307,8 @@ const emptyDeltas = (): RelayMetricDeltas => ({
   controlRenewalFlushRowsMax: 0,
   controlActivityRecoveries: 0,
   controlActivityRecoveryFailures: 0,
-  hostHellosShed: 0
+  hostHellosShed: 0,
+  preAuthLatency: emptyPreAuthLatency()
 })
 
 function ascending(values: number[]): number[] {
@@ -252,12 +333,18 @@ function roundMs(value: number): number {
 // enough of it, so the maximum is folded instead. The fold is also not
 // interchangeable with the sorted last element: it is seeded with zero, so an
 // all-negative or NaN window reads differently.
-function latencySummary(samples: number[]): { p50: number; p95: number; max: number } {
-  // One sorted copy serves both ranks.
+function latencySummary(samples: number[]): {
+  p50: number
+  p95: number
+  p99: number
+  max: number
+} {
+  // One sorted copy serves every rank.
   const sorted = samples.length === 0 ? samples : ascending(samples)
   return {
     p50: roundMs(nearestRank(sorted, 0.5)),
     p95: roundMs(nearestRank(sorted, 0.95)),
+    p99: roundMs(nearestRank(sorted, 0.99)),
     max: roundMs(samples.reduce((highest, sample) => Math.max(highest, sample), 0))
   }
 }
@@ -331,6 +418,36 @@ export class RelayObservability implements RelayRuntimeObserver {
     else this.deltas.drainReturnServiceMs.push(durationMs)
   }
 
+  recordStickySlot(sample: StickySlotSample): void {
+    const deltas = this.deltas
+    deltas.stickySlotQueueWaitMs.push(sample.queueWaitMs)
+    deltas.stickySlotPoolWaitMs.push(sample.poolWaitMs)
+    deltas.stickySlotSqlMs.push(sample.sqlMs)
+    deltas.stickySlotStatements.push(sample.statements)
+    for (const [connectionId, samplesMs] of sample.statementMsByConnection) {
+      let connection = deltas.stickySlotByConnection.get(connectionId)
+      if (!connection) {
+        if (deltas.stickySlotByConnection.size >= STICKY_CONNECTION_LIMIT) {
+          deltas.stickySlotConnectionSamplesDropped++
+          continue
+        }
+        connection = { requests: 0, statements: 0, maxMs: 0, samplesMs: [] }
+        deltas.stickySlotByConnection.set(connectionId, connection)
+      }
+      connection.requests++
+      for (const durationMs of samplesMs) {
+        const observedBefore = connection.statements++
+        connection.maxMs = Math.max(connection.maxMs, durationMs)
+        keepInReservoir(
+          connection.samplesMs,
+          observedBefore,
+          durationMs,
+          STICKY_STATEMENT_RESERVOIR_LIMIT
+        )
+      }
+    }
+  }
+
   recordAssignmentUnavailable(cause: AssignmentUnavailableCause): void {
     increment(this.deltas.assign503sByCause, cause)
   }
@@ -433,15 +550,23 @@ export class RelayObservability implements RelayRuntimeObserver {
   }
 
   recordControlRtt(rttMs: number): void {
-    const samples = this.deltas.controlRttSamplesMs
     const observedBefore = this.deltas.controlRttObserved++
-    if (samples.length < CONTROL_RTT_RESERVOIR_LIMIT) {
-      samples.push(rttMs)
-      return
-    }
-    // Algorithm R: every round trip in the window keeps an equal chance of being kept.
-    const slot = Math.floor(Math.random() * (observedBefore + 1))
-    if (slot < CONTROL_RTT_RESERVOIR_LIMIT) samples[slot] = rttMs
+    keepInReservoir(
+      this.deltas.controlRttSamplesMs,
+      observedBefore,
+      rttMs,
+      CONTROL_RTT_RESERVOIR_LIMIT
+    )
+  }
+
+  recordHostPreAuth(path: HostPreAuthPath, outcome: HostPreAuthOutcome, durationMs: number): void {
+    const group = this.deltas.preAuthLatency[path][outcome]
+    const observedBefore = group.observed++
+    // Folded outside the reservoir so a dropped sample cannot hide the worst hello.
+    group.maxMs = Math.max(group.maxMs, durationMs)
+    const bucket = PRE_AUTH_BUCKET_BOUNDS_MS.findIndex((bound) => durationMs <= bound)
+    group.bucketCounts[bucket === -1 ? PRE_AUTH_BUCKET_BOUNDS_MS.length : bucket]! += 1
+    keepInReservoir(group.samplesMs, observedBefore, durationMs, PRE_AUTH_RESERVOIR_LIMIT)
   }
 
   start(readCounts: () => RelayProcessCounts, intervalMs = 30_000): void {
@@ -527,6 +652,7 @@ export class RelayObservability implements RelayRuntimeObserver {
             stickyServiceMsP50: roundMs(percentile(deltas.stickyServiceMs, 0.5)),
             stickyServiceMsP99: roundMs(percentile(deltas.stickyServiceMs, 0.99))
           }),
+      ...stickySlotFields(deltas),
       assign503sByCauseDelta: deltas.assign503sByCause,
       assignNonDrain503sDelta: Object.entries(deltas.assign503sByCause)
         .filter(([cause]) => cause !== 'drain-return-deferred')
@@ -584,6 +710,7 @@ export class RelayObservability implements RelayRuntimeObserver {
       controlActivityRecoveriesDelta: deltas.controlActivityRecoveries,
       controlActivityRecoveryFailuresDelta: deltas.controlActivityRecoveryFailures,
       hostHellosShedDelta: deltas.hostHellosShed,
+      preAuthLatencyMs: preAuthLatencyFields(deltas.preAuthLatency),
       // Meaning changed when renewals began batching: for a batched row this is
       // the flush's duration, not that row's own statement latency. The
       // per-flush fields below are the ones to read for statement cost.
@@ -619,6 +746,71 @@ function regionCounterFields(
   )
 }
 
+// Count and buckets cover every hello in the window, so buckets sum exactly across windows and
+// cells; the percentiles summarise the reservoir and are omitted when empty.
+function preAuthLatencyFields(
+  latency: PreAuthLatency
+): Record<HostPreAuthPath, Record<HostPreAuthOutcome, Record<string, unknown>>> {
+  const summarize = (group: PreAuthLatencyGroup): Record<string, unknown> => {
+    if (group.observed === 0) return { count: 0 }
+    const { p50, p95, p99 } = latencySummary(group.samplesMs)
+    const bucketsLeMs: Record<string, number> = {}
+    PRE_AUTH_BUCKET_BOUNDS_MS.forEach((bound, index) => {
+      bucketsLeMs[String(bound)] = group.bucketCounts[index]!
+    })
+    bucketsLeMs['+Inf'] = group.bucketCounts[PRE_AUTH_BUCKET_BOUNDS_MS.length]!
+    return { count: group.observed, p50, p95, p99, max: roundMs(group.maxMs), bucketsLeMs }
+  }
+  const byOutcome = (
+    groups: Record<HostPreAuthOutcome, PreAuthLatencyGroup>
+  ): Record<HostPreAuthOutcome, Record<string, unknown>> => ({
+    admitted: summarize(groups.admitted),
+    refused: summarize(groups.refused),
+    abandoned: summarize(groups.abandoned)
+  })
+  return {
+    beforeCheck: byOutcome(latency.beforeCheck),
+    db: byOutcome(latency.db),
+    reserve: byOutcome(latency.reserve)
+  }
+}
+
+// What the sticky slot spent its hold on: the queue before it, then pool waits and
+// round trips inside it; per connection, whether one slow connection keeps serving it.
+function stickySlotFields(deltas: RelayMetricDeltas): Record<string, unknown> {
+  if (deltas.stickySlotQueueWaitMs.length === 0) return {}
+  const queueWait = latencySummary(deltas.stickySlotQueueWaitMs)
+  const poolWait = latencySummary(deltas.stickySlotPoolWaitMs)
+  const sql = latencySummary(deltas.stickySlotSqlMs)
+  const statements = latencySummary(deltas.stickySlotStatements)
+  const byConnection: Record<string, Record<string, number>> = {}
+  for (const [connectionId, connection] of deltas.stickySlotByConnection) {
+    const { p50, p99 } = latencySummary(connection.samplesMs)
+    byConnection[String(connectionId)] = {
+      requests: connection.requests,
+      statements: connection.statements,
+      statementMsP50: p50,
+      statementMsP99: p99,
+      statementMsMax: roundMs(connection.maxMs)
+    }
+  }
+  return {
+    stickySlotsDelta: deltas.stickySlotQueueWaitMs.length,
+    stickyQueueWaitMsP50: queueWait.p50,
+    stickyQueueWaitMsP99: queueWait.p99,
+    stickyQueueWaitMsMax: queueWait.max,
+    stickyPoolWaitMsP50: poolWait.p50,
+    stickyPoolWaitMsP99: poolWait.p99,
+    stickySqlMsP50: sql.p50,
+    stickySqlMsP99: sql.p99,
+    stickyStatementsP50: statements.p50,
+    stickyStatementsMax: statements.max,
+    stickySqlByConnection: byConnection,
+    // One per request-connection pair that found the window's table full.
+    stickySqlConnectionSamplesDroppedDelta: deltas.stickySlotConnectionSamplesDropped
+  }
+}
+
 function lockWaiters(byKey: Record<string, number>, waiterRole: string): number {
   return Object.entries(byKey)
     .filter(([key]) => key.startsWith(`${waiterRole}:relay_cells:`))

cloud/apps/relay/src/relay-server.ts

@@ -255,6 +255,7 @@ export function createRelayServer(
         epoch: join.epoch
       })
     }
+    directory.onOutranked = (seat) => reservePlacement?.supersedeOutranked(seat)
     reserveLedger.start()
     const reconcile = (): void => {
       const now = (options.now ?? Date.now)()
@@ -469,6 +470,7 @@ export function createRelayServer(
     recordDrainReturnRetryAfter: (seconds) => observability.recordDrainReturnRetryAfter?.(seconds),
     recordAdmissionServiceMs: (lane, durationMs) =>
       observability.recordAdmissionServiceMs?.(lane, durationMs),
+    recordStickySlot: (sample) => observability.recordStickySlot?.(sample),
     recordAssignmentUnavailable: (cause) => observability.recordAssignmentUnavailable?.(cause),
     recordRegionRequest: (region) => observability.recordRegionRequest?.(region),
     shadowSeats: shadowSeatPoller?.directory,

cloud/apps/relay/src/reserve-assignment.test.ts

@@ -52,6 +52,7 @@ function setup(
     databaseBusy?: () => boolean
     mode?: 'off' | 'dry-run' | 'on'
     startedAt?: number
+    log?: (line: string) => void
   } = {}
 ) {
   const now = { value: 100_000 }
@@ -88,7 +89,7 @@ function setup(
     ...(options.databaseBusy ? { databaseBusy: options.databaseBusy } : {}),
     now: () => now.value,
     random: () => 0.25,
-    log: () => undefined
+    log: options.log ?? (() => undefined)
   })
   // Starts each cell's refill clock, as a director's first look does.
   for (const cell of assignment.placementCells()) placer.hasBudget(cell)
@@ -350,6 +351,17 @@ describe('reserve assignment on a director', () => {
     expect(demoted).toEqual([{ cellId: 'c1', epoch: 5, joinedAt: 1_000 }])
   })
 
+  // The booking map had not seen the c1 seat (half-open, joined within a poll lag): the feed
+  // demotes it once both joins are in the map, instead of the 75 s silence close.
+  it('tells an old reserve seat the booking missed to go once a newer join shows in the feed', () => {
+    const { assignment, directory, demoted, now } = setup()
+    directory.onOutranked = (seat) => assignment.supersedeOutranked(seat)
+    directory.apply('c2', feed('c2', { full: [seated(6, 3_000)] }), now.value, now.value)
+    expect(demoted).toEqual([])
+    directory.apply('c1', feed('c1', { full: [seated(5, 1_000)] }), now.value, now.value)
+    expect(demoted).toEqual([{ cellId: 'c1', epoch: 5, joinedAt: 1_000 }])
+  })
+
   it('answers a reconnect only at the newest epoch, newest join first', async () => {
     const { assignment, reserved } = setup({
       feeds: [feed('c1', { full: [seated(6, 1_000)] }), feed('c2', { full: [seated(5, 3_000)] })]
@@ -370,4 +382,38 @@ describe('reserve assignment on a director', () => {
     expect(plan.kind === 'database' && (await plan.placeFresh?.())).toBeNull()
     expect(reserved).toMatchObject([{ dryRun: true }])
   })
+
+  it('in dry run counts outcomes per picked cell, and a `full` refusal while the map showed room', async () => {
+    const summaries: Array<{ dryRun: Record<string, unknown> }> = []
+    const dryRunOn = async (units: number, outcome: string) => {
+      const { assignment, now } = setup({
+        mode: 'dry-run',
+        cells: cellList(['c1']),
+        feeds: [feed('c1', { counts: { seats: units, controls: 0, bookings: 0, units, ceiling: 500 } })],
+        reserve: () => ({ outcome }),
+        log: (line) => summaries.push(JSON.parse(line))
+      })
+      for (let index = 0; index < 2; index += 1) {
+        const plan = await assignment.plan(HOST, { reconnect: false, region: US })
+        if (plan.kind === 'database') await plan.placeFresh?.()
+      }
+      // Lets the fire-and-forget dry-run reserves settle before the summary flushes.
+      await Promise.resolve()
+      now.value += 60_000
+      await assignment.plan(HOST, { reconnect: false, region: US })
+    }
+    // 1 free seat of 500 is under the 1% bound; 250 free is far over it.
+    await dryRunOn(499, 'full')
+    await dryRunOn(250, 'full')
+    await dryRunOn(250, 'ok')
+    expect(summaries.map((summary) => summary.dryRun)).toMatchObject([
+      {
+        outcomes: { full: 2 },
+        count: 2,
+        cells: { c1: { outcomes: { full: 2 }, fullWithRoom: 0, minFullFreeShare: 0.002 } }
+      },
+      { outcomes: { full: 2 }, cells: { c1: { outcomes: { full: 2 }, fullWithRoom: 2, minFullFreeShare: 0.5 } } },
+      { outcomes: { ok: 2 }, cells: { c1: { outcomes: { ok: 2 }, fullWithRoom: 0, minFullFreeShare: null } } }
+    ])
+  })
 })

cloud/apps/relay/src/reserve-assignment.ts

@@ -13,7 +13,7 @@ import {
   type PlacementCell,
   type ReservePlacer
 } from './reserve-placement.js'
-import type { SeatFeedCell, ShadowSeat, ShadowSeatDirectory } from './shadow-seat-directory.js'
+import type { OutrankedSeat, SeatFeedCell, ShadowSeat, ShadowSeatDirectory } from './shadow-seat-directory.js'
 
 // Step 5 on a director: answer a desktop from the in-memory map for cells that report
 // admitMode=reserve, and fall back to today's database path for everything else. The cell
@@ -24,6 +24,8 @@ export const RESERVE_STICKY_RECENT_MS = 10 * 60_000
 // A starting director waits this long for every live cell before it answers.
 export const RESERVE_STARTUP_GATE_MS = 30_000
 const SUMMARY_INTERVAL_MS = 60_000
+// A dry-run `full` while the map showed more free room than this share of the ceiling is `fullWithRoom`.
+export const RESERVE_DRY_RUN_ROOM_BOUND = 0.01
 // A leaver for these reasons was moved off its cell; it is not sent back there.
 const NO_REJOIN_CLOSE_CODES: ReadonlySet<number> = new Set([
   RELAY_CLOSE_CODE.DRAINING,
@@ -342,13 +344,23 @@ export class ReserveAssignment {
     const cell = this.input.placer.pick(placements, region, new Set())
     const target = cell && this.input.cells().find((entry) => entry.cellId === cell.cellId)
     if (!target) return null
+    // The map's free room at pick time; pick only returns a cell with a ceiling.
+    const freeShare = cell.ceiling ? this.input.placer.free(cell) / cell.ceiling : 0
     const startedAt = performance.now()
     void this.input.client
       .reserve(target, { ...identity, epoch: 1, ttlMs: CELL_RESERVE_TTL_MS }, true)
       .then((answer) => {
         const elapsedMs = performance.now() - startedAt
-        this.summary.dryRun.outcomes[answer.outcome] = (this.summary.dryRun.outcomes[answer.outcome] ?? 0) + 1
-        this.summary.dryRun.latenciesMs.push(elapsedMs)
+        const { dryRun } = this.summary
+        dryRun.outcomes[answer.outcome] = (dryRun.outcomes[answer.outcome] ?? 0) + 1
+        dryRun.latenciesMs.push(elapsedMs)
+        const perCell = dryRun.cells[cell.cellId] ?? { outcomes: {}, fullWithRoom: 0, minFullFreeShare: null }
+        dryRun.cells[cell.cellId] = perCell
+        perCell.outcomes[answer.outcome] = (perCell.outcomes[answer.outcome] ?? 0) + 1
+        if (answer.outcome === 'full') {
+          if (freeShare > RESERVE_DRY_RUN_ROOM_BOUND) perCell.fullWithRoom += 1
+          perCell.minFullFreeShare = Math.min(perCell.minFullFreeShare ?? 1, freeShare)
+        }
       })
       .catch(() => undefined)
     return null
@@ -377,10 +389,10 @@ export class ReserveAssignment {
     return undefined
   }
 
-  // Fire-and-forget; the cell checks the seat's epoch and join itself.
-  demoteSeat(seat: { userId: string; relayHostId: string; cellId: string; epoch: number; joinedAt: number }): void {
+  // Fire-and-forget; the cell checks the seat's epoch and join itself. False: no such cell listed.
+  demoteSeat(seat: { userId: string; relayHostId: string; cellId: string; epoch: number; joinedAt: number }): boolean {
     const cell = this.input.cells().find((entry) => entry.cellId === seat.cellId)
-    if (!cell) return
+    if (!cell) return false
     void this.input.client.demote(cell, {
       v: 1,
       userId: seat.userId,
@@ -388,6 +400,7 @@ export class ReserveAssignment {
       epoch: seat.epoch,
       joinedAt: seat.joinedAt
     })
+    return true
   }
 
   // Supersede at placement: the old seats this director's map holds on reserve cells are told
@@ -396,32 +409,39 @@ export class ReserveAssignment {
     for (const seat of seats) {
       if (seat.cellId === cellId || seat.epoch >= epoch) continue
       if (this.input.directory.admitModeOf(seat.cellId) !== 'reserve') continue
-      const cell = this.input.cells().find((entry) => entry.cellId === seat.cellId)
-      if (!cell) continue
-      this.summary.supersedes += 1
-      void this.input.client.demote(cell, {
-        v: 1,
-        userId: identity.userId,
-        relayHostId: identity.relayHostId,
-        epoch: seat.epoch,
-        joinedAt: seat.joinedAt
-      })
-      this.input.log?.(
-        JSON.stringify({
-          event: 'orca_relay_reserve_supersede',
-          relayHostIdDigest: relayHostLogDigest(identity.relayHostId),
-          fromCellId: seat.cellId,
-          toCellId: cellId
-        })
-      )
+      this.demoteSuperseded({ ...identity, ...seat }, cellId, 'placement')
     }
   }
 
+  // The feed's half of supersede: a seat the booking map did not hold, found once both joins
+  // are in this map. Without it that seat holds its cell until the 75 s silence close.
+  supersedeOutranked(seat: OutrankedSeat): void {
+    this.demoteSuperseded(seat, seat.byCellId, 'feed')
+  }
+
+  private demoteSuperseded(
+    seat: { userId: string; relayHostId: string; cellId: string; epoch: number; joinedAt: number },
+    toCellId: string,
+    via: 'placement' | 'feed'
+  ): void {
+    if (!this.demoteSeat(seat)) return
+    this.summary.supersedes += 1
+    this.input.log?.(
+      JSON.stringify({
+        event: 'orca_relay_reserve_supersede',
+        relayHostIdDigest: relayHostLogDigest(seat.relayHostId),
+        fromCellId: seat.cellId,
+        toCellId,
+        via
+      })
+    )
+  }
+
   private flushSummary(): void {
     const now = this.now()
     if (now - this.summaryAt < SUMMARY_INTERVAL_MS) return
     this.summaryAt = now
-    const { latenciesMs, outcomes } = this.summary.dryRun
+    const { latenciesMs, outcomes, cells } = this.summary.dryRun
     latenciesMs.sort((left, right) => left - right)
     const percentile = (share: number) =>
       latenciesMs.length === 0
@@ -432,7 +452,7 @@ export class ReserveAssignment {
         event: 'orca_relay_reserve_placement_summary',
         mode: this.input.mode,
         ...this.summary,
-        dryRun: { outcomes, count: latenciesMs.length, p50Ms: percentile(0.5), p99Ms: percentile(0.99) }
+        dryRun: { outcomes, count: latenciesMs.length, p50Ms: percentile(0.5), p99Ms: percentile(0.99), cells }
       })
     )
     this.summary = emptySummary()
@@ -448,7 +468,19 @@ type PlacementSummary = {
   supersedes: number
   outcomes: Record<string, number>
   retries: { mapIncomplete: number; database: number; paced: number }
-  dryRun: { outcomes: Record<string, number>; latenciesMs: number[] }
+  dryRun: {
+    outcomes: Record<string, number>
+    latenciesMs: number[]
+    // Keyed by the picked cell, so bounded by the fleet.
+    cells: Record<string, DryRunCellSummary>
+  }
+}
+
+type DryRunCellSummary = {
+  outcomes: Record<string, number>
+  fullWithRoom: number
+  // Lowest map free share (of the ceiling) among this cell's `full` refusals.
+  minFullFreeShare: number | null
 }
 
 function emptySummary(): PlacementSummary {
@@ -461,6 +493,6 @@ function emptySummary(): PlacementSummary {
     supersedes: 0,
     outcomes: {},
     retries: { mapIncomplete: 0, database: 0, paced: 0 },
-    dryRun: { outcomes: {}, latenciesMs: [] }
+    dryRun: { outcomes: {}, latenciesMs: [], cells: Object.create(null) }
   }
 }

cloud/apps/relay/src/reserve-placement-simulation.test.ts

@@ -294,6 +294,8 @@ describe('step 5 deterministic simulation', () => {
     [1, nearFull({ bookPastCap: true })],
     [2, drain({ intakeBurst: 200, directorShare: 0.05 })],
     [3, drainedWithoutSupersede()],
+    // Half-open seats the booking map had not seen hold their cell until the silence close.
+    [3, levelSeats(11, { skipFeedSupersede: true })],
     [4, levelSeats(11, { localTieBreak: true })],
     [5, mixed({ unguardedLedger: true })],
     // A director restart mid-queue loses its booked joins' rows until its boot reconcile.
@@ -315,13 +317,6 @@ describe('step 5 deterministic simulation', () => {
     // A director whose map has not seen the leave yet may re-place level with the row; the
     // WRONG_CELL row read keeps that a small share (the rest are repaired by one more reconcile).
     expect(report.badFirstReseats / report.rowAheadDemotions).toBeLessThan(0.1)
-    const withoutRead = await runReservePlacementSimulation({ ...config, faults: { noWrongCellRowRead: true } })
-    process.stderr.write(
-      `row_ahead_repair ${JSON.stringify({
-        withRead: { demotions: report.rowAheadDemotions, badFirstReseats: report.badFirstReseats, perHost: report.rowAheadDemotionsPerHost },
-        withoutRead: { demotions: withoutRead.rowAheadDemotions, badFirstReseats: withoutRead.badFirstReseats, perHost: withoutRead.rowAheadDemotionsPerHost }
-      })}\n`
-    )
   }, 240_000)
 
   // Every director misses its feed polls for 150 s: each reserve cell's dead-man trips (60-120 s)

cloud/apps/relay/src/reserve-placement-simulation.ts

@@ -9,6 +9,7 @@ import {
   ReservePlacer,
   type PlacementCell
 } from './reserve-placement.js'
+import { seatOutranks } from './shadow-seat-directory.js'
 
 // A seeded discrete-event model of step 5 that drives the real CellReserveBook and
 // ReservePlacer. Everything else (sockets, the database, desktops) is modelled. The checks
@@ -111,8 +112,10 @@ export type SimulationFaults = {
   intakeBurst?: number
   // Each director assumes this share of every cell's intake, instead of a fifth.
   directorShare?: number
-  // Placement does not supersede the old seat: a duplicate then outlives its bound.
+  // Nothing supersedes the old seat, at placement or from the feed: a duplicate outlives its bound.
   skipSupersede?: boolean
+  // Only placement supersedes: a half-open seat the booking map had not seen lasts to its silence close.
+  skipFeedSupersede?: boolean
   // A restarted director forgets its ledger queue and never reconciles.
   noBootReconcile?: boolean
   // The reconcile leaves a seat whose row is ahead of it alone.
@@ -666,10 +669,15 @@ export async function runReservePlacementSimulation(
           placement: placementCell(cell, snapshot, requestedAt)
         }
         director.views.set(cell.cellId, current)
+        supersedeOutranked(director, seats!.keys())
       } else if (current && current.incarnation === snapshot.incarnation) {
         for (const change of changes) applyChange(director, cell, current, change)
         current.cursor = snapshot.cursor
         current.placement = placementCell(cell, snapshot, requestedAt)
+        supersedeOutranked(
+          director,
+          changes.filter((change) => change.kind === 'join').map((change) => change.host)
+        )
       } else {
         // A restarted director's (or a stale) delta: the next poll asks for a full snapshot.
         return
@@ -758,6 +766,26 @@ export async function runReservePlacementSimulation(
     }
   }
 
+  // As shadow-seat-directory.ts onOutranked: once both joins are in this map, a reserve seat behind
+  // a later join at a newer epoch is told to go. A lower epoch that joined later is the row's.
+  function supersedeOutranked(director: SimDirector, hosts: Iterable<string>): void {
+    if (faults.skipSupersede || faults.skipFeedSupersede) return
+    for (const host of new Set(hosts)) {
+      const seats = [...director.views].flatMap(([cellId, view]) => {
+        const seat = view.seats.get(host)
+        return seat ? [{ cellId, view, seat }] : []
+      })
+      if (seats.length < 2) continue
+      for (const { cellId, view, seat } of seats) {
+        if (seat.demoted || !view.placement.reserve) continue
+        const by = seats.find((other) => seatOutranks(other.seat, seat))
+        if (!by) continue
+        report.supersedes += 1
+        demoteFor(director, host, cellId, seat, { cellId: by.cellId, epoch: by.seat.epoch })
+      }
+    }
+  }
+
   // The seats the booking director's map held when it placed each booking: only those it supersedes.
   const heldAtBooking = new Map<string, Set<string>>()
 
@@ -1258,9 +1286,9 @@ export async function runReservePlacementSimulation(
       report.maxOccupancyShare[cell.cellId] = Math.max(report.maxOccupancyShare[cell.cellId] ?? 0, share)
     }
   }
-  // A booking supersedes the old seats its director's map held within a round trip. Any other
-  // duplicate (a half-open seat today's path placed past, or one the booking map had not yet
-  // seen) has no lease to re-check and lasts until that seat's silence close.
+  // A booking supersedes the old seats its director's map held within a round trip, and the feed
+  // any seat a later join outranks. Any other duplicate (a half-open seat today's path placed past
+  // at a lower epoch than it) has no lease to re-check and lasts until that seat's silence close.
   const supersedeBound =
     Math.max(...directors.map((director) => config.pollLagMs(director.index))) +
     2 * Math.max(...cells.map((cell) => config.rttMs(cell.region))) +
@@ -1293,13 +1321,20 @@ export async function runReservePlacementSimulation(
       for (const { cell, seat } of seats) {
         if (cell.mode !== 'reserve' || seat.epoch === newest) continue
         // A leased seat goes by its renewal, as today.
-        const bound = !seat.leased && supersededBy(cell.cellId, seat.joinedAt) ? supersedeBound : duplicateBound
+        const tight =
+          !seat.leased &&
+          (supersededBy(cell.cellId, seat.joinedAt) || seats.some((other) => seatOutranks(other.seat, seat)))
+        // The loose clock runs from the first outranking; the tight one from when supersede applies.
         const key = `${host}\u0000${cell.cellId}\u0000${seat.joinedAt}`
-        live.add(key)
-        const since = outrankedSince.get(key) ?? clock.now
-        outrankedSince.set(key, since)
-        if (clock.now - since > bound) {
-          violate(3, `${host}@${seat.epoch} on ${cell.cellId} outranked for ${clock.now - since} ms`)
+        const clocks: Array<[string, number]> = [[key, duplicateBound]]
+        if (tight) clocks.push([`${key}\u0000tight`, supersedeBound])
+        for (const [clockKey, bound] of clocks) {
+          live.add(clockKey)
+          const since = outrankedSince.get(clockKey) ?? clock.now
+          outrankedSince.set(clockKey, since)
+          if (clock.now - since > bound) {
+            violate(3, `${host}@${seat.epoch} on ${cell.cellId} outranked for ${clock.now - since} ms`)
+          }
         }
       }
     }

cloud/apps/relay/src/reserve-placement.test.ts

@@ -29,6 +29,13 @@ function warmPlacer(now: { value: number }, random = () => 0.3) {
 }
 
 describe('reserve placer', () => {
+  it('reports free seats under the ceiling, never below zero or without a ceiling', () => {
+    const placer = warmPlacer({ value: 0 })
+    expect(placer.free(cell('c1', { seats: 60, bookings: 10 }))).toBe(30)
+    expect(placer.free(cell('c1', { seats: 120 }))).toBe(0)
+    expect(placer.free(cell('c1', { ceiling: undefined }))).toBe(0)
+  })
+
   it('places on the first cell that answers ok and counts one call', async () => {
     const now = { value: 0 }
     const placer = warmPlacer(now)

cloud/apps/relay/src/reserve-placement.ts

@@ -68,6 +68,11 @@ export class ReservePlacer {
     return cell.seats + cell.bookings + this.ownBookingsSince(cell)
   }
 
+  // Estimated free seats under the ceiling; 0 when the cell reports none.
+  free(cell: PlacementCell): number {
+    return cell.ceiling === undefined ? 0 : Math.max(0, cell.ceiling - this.load(cell))
+  }
+
   hasSeat(cell: PlacementCell): boolean {
     return cell.ceiling !== undefined && this.load(cell) < cell.ceiling
   }
@@ -158,7 +163,7 @@ export class ReservePlacer {
       if (cell.region !== region || !cell.general || cell.draining) continue
       if (cell.reserve) {
         if (this.isCandidate(cell, region, now)) {
-          reserveFree += Math.max(0, cell.ceiling! - this.load(cell))
+          reserveFree += this.free(cell)
         }
         continue
       }

cloud/apps/relay/src/shadow-directory-compare.test.ts

@@ -5,6 +5,7 @@ import {
   classifyShadowSeat,
   SHADOW_COMPARE_FLUSH_MS,
   SHADOW_COMPARE_SAMPLES_PER_CLASS,
+  SHADOW_COMPARE_SUPERSEDED_CLOSE_MS,
   ShadowDirectoryCompare
 } from './shadow-directory-compare.js'
 import { ShadowSeatDirectory, type SeatFeedResponse } from './shadow-seat-directory.js'
@@ -27,8 +28,8 @@ function feed(cellId: string, overrides: Partial<SeatFeedResponse> = {}): SeatFe
   return { v: 1, cellId, incarnation: `inc-${cellId}`, seq: 1, at: 1, full: [], ...overrides }
 }
 
-function seat(epoch: number, state = 'active', relayHostId = HOST) {
-  return { userId: 'user-a', relayHostId, epoch, generation: 1, state, joinedAt: 1 }
+function seat(epoch: number, state = 'active', relayHostId = HOST, joinedAt = 1) {
+  return { userId: 'user-a', relayHostId, epoch, generation: 1, state, joinedAt }
 }
 
 function directoryWith(
@@ -84,7 +85,7 @@ describe('classifyShadowSeat', () => {
         true
       ],
       [
-        directoryWith({ 'cell-a': [seat(3)], 'cell-b': [seat(2)] }),
+        directoryWith({ 'cell-a': [seat(3)], 'cell-b': [seat(3)] }),
         at('cell-a', 3),
         'duplicate-seat',
         false
@@ -191,6 +192,178 @@ describe('classifyShadowSeat', () => {
     })
   })
 
+  describe('a duplicate seat left by a superseded half-open control', () => {
+    // Seated on cell-b at epoch 2 long ago; the director first saw epoch 3 on cell-a at firstSeen.
+    const firstSeen = 1_000_000
+    const directoryAt = (seats: Record<string, ReturnType<typeof seat>[]>) => {
+      const directory = directoryWith(seats)
+      directory.observeDatabaseEpoch('user-a', HOST, 3, firstSeen)
+      return directory
+    }
+    const superseded = (joinedAt = firstSeen) =>
+      directoryAt({
+        'cell-a': [seat(3, 'active', HOST, joinedAt)],
+        'cell-b': [seat(2, 'active', HOST, 1)]
+      })
+
+    it('is explained while the old control may still await its silence close', () => {
+      for (const now of [firstSeen + 1_700, firstSeen + SHADOW_COMPARE_SUPERSEDED_CLOSE_MS]) {
+        expect(classifyShadowSeat(superseded(), IDENTITY, at('cell-a', 3), now)).toEqual({
+          class: 'duplicate-seat',
+          explained: true
+        })
+      }
+    })
+
+    it('is unexplained just past the close bound', () => {
+      const now = firstSeen + SHADOW_COMPARE_SUPERSEDED_CLOSE_MS + 1
+      expect(classifyShadowSeat(superseded(), IDENTITY, at('cell-a', 3), now)).toEqual({
+        class: 'duplicate-seat',
+        explained: false
+      })
+    })
+
+    it('stays unexplained past the bound after a same-epoch rejoin resets joinedAt', () => {
+      const now = firstSeen + SHADOW_COMPARE_SUPERSEDED_CLOSE_MS + 30_000
+      expect(classifyShadowSeat(superseded(now - 1_000), IDENTITY, at('cell-a', 3), now)).toEqual({
+        class: 'duplicate-seat',
+        explained: false
+      })
+    })
+
+    const unseen = () =>
+      directoryWith({
+        'cell-a': [seat(3, 'active', HOST, firstSeen)],
+        'cell-b': [seat(2, 'active', HOST, 1)]
+      })
+
+    it('is its own unexplained class with no sighting in the instance\'s first 100 s', () => {
+      const now = firstSeen + 1_700
+      const startedAt = now - SHADOW_COMPARE_SUPERSEDED_CLOSE_MS
+      expect(classifyShadowSeat(unseen(), IDENTITY, at('cell-a', 3), now, startedAt)).toEqual({
+        class: 'superseded-unknown-age',
+        explained: false
+      })
+    })
+
+    it('is a plain unexplained duplicate with no sighting after the first 100 s', () => {
+      const now = firstSeen + 1_700
+      const startedAt = now - SHADOW_COMPARE_SUPERSEDED_CLOSE_MS - 1
+      for (const classified of [
+        classifyShadowSeat(unseen(), IDENTITY, at('cell-a', 3), now, startedAt),
+        classifyShadowSeat(unseen(), IDENTITY, at('cell-a', 3), now)
+      ]) {
+        expect(classified).toEqual({ class: 'duplicate-seat', explained: false })
+      }
+    })
+
+    it('stays a plain duplicate with no sighting when the current seat joined long ago', () => {
+      const now = firstSeen + SHADOW_COMPARE_SUPERSEDED_CLOSE_MS + 1
+      const directory = directoryWith({
+        'cell-a': [seat(3, 'active', HOST, firstSeen)],
+        'cell-b': [seat(2, 'active', HOST, 1)]
+      })
+      expect(classifyShadowSeat(directory, IDENTITY, at('cell-a', 3), now)).toEqual({
+        class: 'duplicate-seat',
+        explained: false
+      })
+    })
+
+    it('classifies before recording, so a later epoch bump is explained', () => {
+      // Seated on cell-a at epoch 2 and on cell-b at epoch 1; the database names cell-a at 2.
+      const directory = directoryWith({
+        'cell-a': [seat(2, 'active', HOST, firstSeen)],
+        'cell-b': [seat(1, 'active', HOST, 1)]
+      })
+      let now = firstSeen
+      const compare = new ShadowDirectoryCompare(directory, () => now, () => {})
+      expect(compare.compare('sticky-verify', IDENTITY, at('cell-a', 2))).toEqual({
+        class: 'superseded-unknown-age',
+        explained: false
+      })
+      // A re-assign to cell-b at epoch 3 supersedes cell-a's epoch-2 seat.
+      now = firstSeen + 1_700
+      directory.apply(
+        'cell-b',
+        feed('cell-b', { seq: 2, full: [seat(3, 'active', HOST, now)] }),
+        now
+      )
+      expect(compare.compare('sticky-verify', IDENTITY, at('cell-b', 3))).toEqual({
+        class: 'duplicate-seat',
+        explained: true
+      })
+    })
+
+    it('is explained when the director last saw the host at a lower epoch', () => {
+      const directory = directoryWith({
+        'cell-a': [seat(3, 'active', HOST, firstSeen)],
+        'cell-b': [seat(2, 'active', HOST, 1)]
+      })
+      directory.observeDatabaseEpoch('user-a', HOST, 2, 1)
+      expect(classifyShadowSeat(directory, IDENTITY, at('cell-a', 3), firstSeen + 1_700)).toEqual({
+        class: 'duplicate-seat',
+        explained: true
+      })
+    })
+
+    it('is unexplained after a lower-epoch sighting when the current seat joined long ago', () => {
+      const now = firstSeen + SHADOW_COMPARE_SUPERSEDED_CLOSE_MS + 1
+      const directory = directoryWith({
+        'cell-a': [seat(3, 'active', HOST, firstSeen)],
+        'cell-b': [seat(2, 'active', HOST, 1)]
+      })
+      directory.observeDatabaseEpoch('user-a', HOST, 2, 1)
+      expect(classifyShadowSeat(directory, IDENTITY, at('cell-a', 3), now)).toEqual({
+        class: 'duplicate-seat',
+        explained: false
+      })
+    })
+
+    it('is unexplained when the database answer regresses below the recorded epoch', () => {
+      const directory = directoryWith({
+        'cell-a': [seat(3, 'active', HOST, firstSeen)],
+        'cell-b': [seat(2, 'active', HOST, 1)]
+      })
+      directory.observeDatabaseEpoch('user-a', HOST, 4, firstSeen)
+      expect(classifyShadowSeat(directory, IDENTITY, at('cell-a', 3), firstSeen + 1_700)).toEqual({
+        class: 'duplicate-seat',
+        explained: false
+      })
+    })
+
+    it('is unexplained when the database names a newer epoch than the seat on its cell', () => {
+      const directory = directoryWith({
+        'cell-a': [seat(3, 'active', HOST, firstSeen)],
+        'cell-b': [seat(2, 'active', HOST, 1)]
+      })
+      directory.observeDatabaseEpoch('user-a', HOST, 4, firstSeen)
+      expect(classifyShadowSeat(directory, IDENTITY, at('cell-a', 4), firstSeen + 1_700)).toEqual({
+        class: 'duplicate-seat',
+        explained: false
+      })
+    })
+
+    it('is unexplained when the database names the lower epoch, or matches no seat', () => {
+      for (const answer of [at('cell-b', 2), at('cell-a', 4), at('cell-b', 3)]) {
+        expect(classifyShadowSeat(superseded(), IDENTITY, answer, firstSeen + 1_700)).toEqual({
+          class: 'duplicate-seat',
+          explained: false
+        })
+      }
+    })
+
+    it('is unexplained on a level epoch, which only the row can break', () => {
+      const directory = directoryAt({
+        'cell-a': [seat(3, 'active', HOST, firstSeen)],
+        'cell-b': [seat(3, 'active', HOST, 1)]
+      })
+      expect(classifyShadowSeat(directory, IDENTITY, at('cell-a', 3), firstSeen + 1_700)).toEqual({
+        class: 'duplicate-seat',
+        explained: false
+      })
+    })
+  })
+
   it('answers map-incomplete until every listed cell has answered', () => {
     const directory = new ShadowSeatDirectory()
     directory.setCells(['cell-a', 'cell-b'])
@@ -225,7 +398,8 @@ describe('ShadowDirectoryCompare', () => {
         class: 'map-only',
         explained: false,
         count: SHADOW_COMPARE_SAMPLES_PER_CLASS + 5,
-        windowMs: SHADOW_COMPARE_FLUSH_MS
+        windowMs: SHADOW_COMPARE_FLUSH_MS,
+        countByCell: { 'cell-a': SHADOW_COMPARE_SAMPLES_PER_CLASS + 5 }
       },
       {
         event: 'orca_relay_shadow_compare',
@@ -233,10 +407,55 @@ describe('ShadowDirectoryCompare', () => {
         class: 'agree',
         explained: true,
         count: 2,
-        windowMs: SHADOW_COMPARE_FLUSH_MS
+        windowMs: SHADOW_COMPARE_FLUSH_MS,
+        countByCell: { 'cell-a': 2 }
       }
     ])
   })
+
+  it('counts a compare once on every cell it involves, and on none for a host neither side has', () => {
+    const lines: Array<Record<string, unknown>> = []
+    const compareOn = (seats: Record<string, ReturnType<typeof seat>[]>) =>
+      new ShadowDirectoryCompare(directoryWith(seats), () => 0, (line) => lines.push(JSON.parse(line)))
+    const unseen = compareOn({ 'cell-a': [], 'cell-b': [] })
+    unseen.compare('sticky-verify', IDENTITY, assignment('cell-a'))
+    unseen.compare('sticky-verify', IDENTITY, assignment('cell-a'))
+    unseen.compare('sticky-verify', IDENTITY, assignment('cell-b'))
+    unseen.compare('resolve', IDENTITY, null)
+    unseen.flush(0)
+    // The database says cell-a at epoch 3; the map seats the host on cell-b at a newer epoch.
+    const mismatch = compareOn({ 'cell-a': [], 'cell-b': [seat(4)] })
+    mismatch.compare('sticky-verify', IDENTITY, assignment('cell-a'))
+    mismatch.flush(0)
+    // A level-epoch duplicate counts on the database's own cell once, and on the other seat cell.
+    const duplicate = compareOn({ 'cell-a': [seat(3)], 'cell-b': [seat(3)] })
+    duplicate.compare('sticky-verify', IDENTITY, assignment('cell-a'))
+    duplicate.flush(0)
+    // A drain-only second seat is explained, so it counts only on the database's cell.
+    const draining = compareOn({ 'cell-a': [seat(3)], 'cell-b': [seat(2, 'drain-only')] })
+    draining.compare('sticky-verify', IDENTITY, assignment('cell-a'))
+    draining.flush(0)
+    // A superseded older seat: the first-ever sighting has no age, so it is its own unexplained
+    // class on both cells; seen before, it is explained and counts only on the database's cell.
+    const superseded = compareOn({ 'cell-a': [seat(3)], 'cell-b': [seat(2)] })
+    superseded.compare('sticky-verify', IDENTITY, assignment('cell-a'))
+    superseded.compare('sticky-verify', IDENTITY, assignment('cell-a'))
+    superseded.flush(0)
+    const mapOnly = compareOn({ 'cell-a': [seat(3)], 'cell-b': [seat(3)] })
+    mapOnly.compare('resolve', IDENTITY, null)
+    mapOnly.flush(0)
+    const counts = lines.filter((line) => line.event === 'orca_relay_shadow_compare')
+    expect(counts.map((line) => [line.class, line.explained, line.count, line.countByCell])).toEqual([
+      ['db-only-unseen', true, 3, { 'cell-a': 2, 'cell-b': 1 }],
+      ['agree-absent', true, 1, {}],
+      ['cell-mismatch', false, 1, { 'cell-a': 1, 'cell-b': 1 }],
+      ['duplicate-seat', false, 1, { 'cell-a': 1, 'cell-b': 1 }],
+      ['duplicate-seat', true, 1, { 'cell-a': 1 }],
+      ['superseded-unknown-age', false, 1, { 'cell-a': 1, 'cell-b': 1 }],
+      ['duplicate-seat', true, 1, { 'cell-a': 1 }],
+      ['map-only', false, 1, { 'cell-a': 1, 'cell-b': 1 }]
+    ])
+  })
 })
 
 describe('unexplained samples', () => {

cloud/apps/relay/src/shadow-directory-compare.ts

@@ -1,3 +1,4 @@
+import { RELAY_PROTOCOL_LIMITS } from '@orca-cloud/relay-contract'
 import { relayHostLogDigest } from './relay-host-log-digest.js'
 import type { ShadowSeatDirectory } from './shadow-seat-directory.js'
 
@@ -20,6 +21,8 @@ export type ShadowCompareClass =
   | 'cell-mismatch'
   | 'epoch-mismatch'
   | 'duplicate-seat'
+  // A superseded-shaped duplicate with no prior sighting, in the instance's first 100 s.
+  | 'superseded-unknown-age'
   | 'map-only'
   // The map seats the host on a cell the database does not call live, so resolve answered null.
   | 'map-only-cell-unlive'
@@ -34,16 +37,28 @@ export const SHADOW_COMPARE_FLUSH_MS = 60_000
 // One poll, its timeout, and a reconnect (p95 ~6 s on 10-02). A move older than
 // this is not "the map catching up": a lost leave must not read as explained forever.
 export const SHADOW_COMPARE_LAG_BOUND_MS = 10_000
+// A superseded half-open control lingers until the cell's silence close (the timeout plus up
+// to one ping interval), then the map trails that close by a poll.
+export const SHADOW_COMPARE_SUPERSEDED_CLOSE_MS =
+  RELAY_PROTOCOL_LIMITS.controlSilenceTimeoutMs +
+  RELAY_PROTOCOL_LIMITS.controlPingIntervalMs +
+  SHADOW_COMPARE_LAG_BOUND_MS
 // Per class per flush window, so a systemic fault cannot flood the log.
 export const SHADOW_COMPARE_SAMPLES_PER_CLASS = 10
 
+// `countByCell` is keyed by cell, so bounded by the fleet; prototype-free, so any cell id is a key.
+type ShadowCompareCount = { count: number; countByCell: Record<string, number> }
+
 export function classifyShadowSeat(
   directory: ShadowSeatDirectory,
   identity: { userId: string; relayHostId: string },
   answer: ShadowCompareDatabaseAnswer,
-  now: number
+  now: number,
+  // When this instance began comparing; omitted, a missing sighting is never excused.
+  startedAt = -Infinity
 ): ShadowCompareVerdict {
-  // Pure: only ShadowDirectoryCompare records sightings. An epoch never seen is new.
+  // Pure: only ShadowDirectoryCompare records sightings. An epoch never seen is new; unlike the
+  // superseded rule, a wrong guess here only explains a mismatch for one lag bound.
   const movedAt = answer
     ? (directory.databaseEpochFirstSeen(
         identity.userId,
@@ -69,7 +84,31 @@ export function classifyShadowSeat(
   }
   if (seats.length > 1) {
     // Inside a drain or rehome grace the old seat is drain-only until it closes.
-    return { class: 'duplicate-seat', explained: seats.some((seat) => seat.state === 'drain-only') }
+    if (seats.some((seat) => seat.state === 'drain-only')) {
+      return { class: 'duplicate-seat', explained: true }
+    }
+    // A newer DB seat supersedes older-epoch seats until their silence close;
+    // a level epoch is a real tie.
+    const current = seats.find(
+      (seat) => seat.cellId === answer.cellId && seat.epoch === answer.assignmentEpoch
+    )
+    const supersededShape =
+      current !== undefined &&
+      seats.every((seat) => seat === current || seat.epoch < current.epoch) &&
+      // A long-ago lower-epoch sighting dates the move to now; the join bounds the seat itself.
+      now - current.joinedAt <= SHADOW_COMPARE_SUPERSEDED_CLOSE_MS
+    if (!supersededShape) return { class: 'duplicate-seat', explained: false }
+    const known = directory.databaseEpochOf(identity.userId, identity.relayHostId)
+    if (!known) {
+      // Later, no sighting means another instance served the earlier compares, or the map evicted
+      // the host, so a harmless supersede still reads as an unexplained duplicate-seat.
+      const starting = now - startedAt <= SHADOW_COMPARE_SUPERSEDED_CLOSE_MS
+      return { class: starting ? 'superseded-unknown-age' : 'duplicate-seat', explained: false }
+    }
+    const supersededAt = movedSince(known, answer, now)
+    const superseded =
+      supersededAt !== undefined && now - supersededAt <= SHADOW_COMPARE_SUPERSEDED_CLOSE_MS
+    return { class: 'duplicate-seat', explained: superseded }
   }
   const seat = seats[0]
   if (!seat) {
@@ -99,17 +138,54 @@ export function classifyShadowSeat(
   return { class: 'agree', explained: true }
 }
 
+// When the move to the answer's epoch happened, from the sighting recorded before this compare.
+// A regressed epoch means unknown, not new.
+function movedSince(
+  known: { epoch: number; firstSeenAt: number },
+  answer: { assignmentEpoch: number },
+  now: number
+): number | undefined {
+  if (known.epoch > answer.assignmentEpoch) return undefined
+  // A lower recorded epoch: the move is this sighting.
+  return known.epoch < answer.assignmentEpoch ? now : known.firstSeenAt
+}
+
+// The database's cell, or with no row every seat's cell; an unexplained cell-mismatch or
+// duplicate-seat also counts on every seat cell, so a stale seat feed shows on its own cell.
+// A compare with neither (agree-absent, or map-incomplete with no row and no seat) has none.
+function comparedCells(
+  directory: ShadowSeatDirectory,
+  identity: { userId: string; relayHostId: string },
+  answer: ShadowCompareDatabaseAnswer,
+  verdict: ShadowCompareVerdict
+): Set<string> {
+  const seats = directory.seatsOf(identity.userId, identity.relayHostId)
+  if (!answer) return new Set(seats.map((seat) => seat.cellId))
+  const cells = new Set([answer.cellId])
+  if (
+    !verdict.explained &&
+    (verdict.class === 'cell-mismatch' ||
+      verdict.class === 'duplicate-seat' ||
+      verdict.class === 'superseded-unknown-age')
+  ) {
+    for (const seat of seats) cells.add(seat.cellId)
+  }
+  return cells
+}
+
 export class ShadowDirectoryCompare {
-  private counts = new Map<string, number>()
+  private counts = new Map<string, ShadowCompareCount>()
   private samples = new Map<ShadowCompareClass, number>()
   private windowStartedAt: number
+  private readonly startedAt: number
 
   constructor(
     private readonly directory: ShadowSeatDirectory,
     private readonly now: () => number = Date.now,
     private readonly log: (line: string) => void = (line) => console.warn(line)
   ) {
-    this.windowStartedAt = now()
+    this.startedAt = now()
+    this.windowStartedAt = this.startedAt
   }
 
   compare(
@@ -118,6 +194,8 @@ export class ShadowDirectoryCompare {
     answer: ShadowCompareDatabaseAnswer
   ): ShadowCompareVerdict {
     const now = this.now()
+    // Classify first, so it can tell a first-ever sighting from one seen before.
+    const verdict = classifyShadowSeat(this.directory, identity, answer, now, this.startedAt)
     // Every answer, so a move's age counts from its first sighting on either route.
     if (answer) {
       this.directory.observeDatabaseEpoch(
@@ -127,17 +205,22 @@ export class ShadowDirectoryCompare {
         now
       )
     }
-    const verdict = classifyShadowSeat(this.directory, identity, answer, now)
     const key = `${route}\u0000${verdict.class}\u0000${verdict.explained}`
-    this.counts.set(key, (this.counts.get(key) ?? 0) + 1)
+    const entry = this.counts.get(key) ?? { count: 0, countByCell: Object.create(null) }
+    this.counts.set(key, entry)
+    entry.count += 1
+    for (const cellId of comparedCells(this.directory, identity, answer, verdict)) {
+      entry.countByCell[cellId] = (entry.countByCell[cellId] ?? 0) + 1
+    }
     if (!verdict.explained) this.sample(route, identity, answer, verdict, now)
     if (now - this.windowStartedAt >= SHADOW_COMPARE_FLUSH_MS) this.flush(now)
     return verdict
   }
 
-  // One line per route and class per window; the log metric sums `count`.
+  // One line per route and class per window; the log metric sums `count`. `countByCell`
+  // counts each compare once on every cell it involves (see comparedCells).
   flush(now: number = this.now()): void {
-    for (const [key, count] of this.counts) {
+    for (const [key, { count, countByCell }] of this.counts) {
       const [route, verdictClass, explained] = key.split('\u0000')
       this.log(
         JSON.stringify({
@@ -146,7 +229,8 @@ export class ShadowDirectoryCompare {
           class: verdictClass,
           explained: explained === 'true',
           count,
-          windowMs: now - this.windowStartedAt
+          windowMs: now - this.windowStartedAt,
+          countByCell
         })
       )
     }

cloud/apps/relay/src/shadow-seat-directory.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, it, vi } from 'vitest'
 import { RelayAssignmentStore } from './assignment-store.js'
 import { openInMemoryRelayDatabase } from './database.js'
 import {
+  SHADOW_SEAT_OUTRANK_SKEW_MS,
   SHADOW_SEAT_RECENTLY_LEFT_MAX_HOSTS,
   ShadowSeatDirectory,
   startShadowSeatPoller,
@@ -577,3 +578,76 @@ describe('ShadowSeatDirectory.placementModeOf', () => {
     expect(directory.placementModeOf('cell-a')).toBe('reserve')
   })
 })
+
+describe('ShadowSeatDirectory.onOutranked', () => {
+  const reserve = { generation: 3, flags: { admitMode: 'reserve' } }
+  const db = { generation: 3, flags: { admitMode: 'db' } }
+  const full = (cellId: string, epoch: number, joinedAt: number, flags = reserve, state = 'active') =>
+    feed({
+      cellId,
+      seq: 1,
+      flagsApplied: flags,
+      full: [{ userId: 'user-a', relayHostId: 'host-1', epoch, generation: 1, state, joinedAt }]
+    })
+  const directory = () => {
+    const map = new ShadowSeatDirectory()
+    map.setCells(['cell-a', 'cell-b'])
+    const outranked: unknown[] = []
+    map.onOutranked = (seat) => outranked.push(seat)
+    return { map, outranked }
+  }
+  const halfOpen = {
+    cellId: 'cell-a',
+    userId: 'user-a',
+    relayHostId: 'host-1',
+    epoch: 3,
+    joinedAt: 10,
+    byCellId: 'cell-b'
+  }
+
+  it('reports a reserve seat a later, newer join elsewhere outranks, in either feed order', () => {
+    const first = directory()
+    first.map.apply('cell-a', full('cell-a', 3, 10), 100)
+    first.map.apply('cell-b', full('cell-b', 4, 2_000), 101)
+    expect(first.outranked).toEqual([halfOpen])
+    const reversed = directory()
+    reversed.map.apply('cell-b', full('cell-b', 4, 2_000), 100)
+    reversed.map.apply('cell-a', full('cell-a', 3, 10), 101)
+    expect(reversed.outranked).toEqual([halfOpen])
+    const justPastSkew = directory()
+    justPastSkew.map.apply('cell-a', full('cell-a', 3, 10), 100)
+    justPastSkew.map.apply('cell-b', full('cell-b', 4, 11 + SHADOW_SEAT_OUTRANK_SKEW_MS), 101)
+    expect(justPastSkew.outranked).toEqual([halfOpen])
+  })
+
+  it('reports a seat once, even when a full resync re-judges its host', () => {
+    const { map, outranked } = directory()
+    map.apply('cell-a', full('cell-a', 3, 10), 100)
+    map.apply('cell-b', full('cell-b', 4, 2_000), 101)
+    map.apply('cell-b', { ...full('cell-b', 4, 2_000), incarnation: '22222222-2222-4222-8222-222222222222' }, 102)
+    map.apply('cell-a', { ...full('cell-a', 3, 10), incarnation: '22222222-2222-4222-8222-222222222222' }, 103)
+    expect(outranked).toEqual([halfOpen])
+  })
+
+  it('leaves a database-cell seat, a drain-only seat, a later lower epoch, and a join within the skew', () => {
+    const onDatabaseCell = directory()
+    onDatabaseCell.map.apply('cell-a', full('cell-a', 3, 10, db), 100)
+    onDatabaseCell.map.apply('cell-b', full('cell-b', 4, 2_000), 101)
+    const drainOnly = directory()
+    drainOnly.map.apply('cell-a', full('cell-a', 3, 10, reserve, 'drain-only'), 100)
+    drainOnly.map.apply('cell-b', full('cell-b', 4, 2_000), 101)
+    // Answered the old seat from a stale map after the newer join: the row decides that one.
+    const staleSticky = directory()
+    staleSticky.map.apply('cell-a', full('cell-a', 3, 3_000), 100)
+    staleSticky.map.apply('cell-b', full('cell-b', 4, 2_000), 101)
+    const withinSkew = directory()
+    withinSkew.map.apply('cell-a', full('cell-a', 3, 10), 100)
+    withinSkew.map.apply('cell-b', full('cell-b', 4, 10 + SHADOW_SEAT_OUTRANK_SKEW_MS), 101)
+    expect([onDatabaseCell, drainOnly, staleSticky, withinSkew].map(({ outranked }) => outranked)).toEqual([
+      [],
+      [],
+      [],
+      []
+    ])
+  })
+})

cloud/apps/relay/src/shadow-seat-directory.ts

@@ -16,6 +16,16 @@ export const SHADOW_SEAT_POLL_TIMEOUT_MS = 2_000
 export const SHADOW_SEAT_CELL_LIST_REFRESH_MS = 30_000
 export const SHADOW_SEAT_RECENTLY_LEFT_TTL_MS = 24 * 60 * 60 * 1_000
 export const SHADOW_SEAT_RECENTLY_LEFT_MAX_HOSTS = 100_000
+// Cells stamp joinedAt by their own clocks: a later join must clear this to outrank across cells.
+export const SHADOW_SEAT_OUTRANK_SKEW_MS = 1_000
+const SHADOW_SEAT_OUTRANKED_REPORTS_MAX = 100_000
+
+export function seatOutranks(
+  newer: { epoch: number; joinedAt: number },
+  seat: { epoch: number; joinedAt: number }
+): boolean {
+  return newer.epoch > seat.epoch && newer.joinedAt > seat.joinedAt + SHADOW_SEAT_OUTRANK_SKEW_MS
+}
 const RECENTLY_LEFT_PER_HOST = 4
 // Google identity tokens live an hour; one per poll would be 26 metadata reads a second.
 const IDENTITY_TOKEN_REUSE_MS = 10 * 60_000
@@ -172,6 +182,16 @@ export type BookedJoin = {
   reservedBy: string
 }
 
+// A reserve seat behind a later, newer-epoch join of its host elsewhere that placement supersede missed.
+export type OutrankedSeat = {
+  cellId: string
+  userId: string
+  relayHostId: string
+  epoch: number
+  joinedAt: number
+  byCellId: string
+}
+
 type CellCursor = SeatFeedCellState & { seats: Map<string, ShadowSeat> }
 
 export type HeartbeatSnapshot = { readAt: number; expiresAt: ReadonlyMap<string, number> }
@@ -180,6 +200,11 @@ function hostKey(userId: string, relayHostId: string): string {
   return `${userId}\u0000${relayHostId}`
 }
 
+function parseHostKey(key: string): { userId: string; relayHostId: string } {
+  const separator = key.indexOf('\u0000')
+  return { userId: key.slice(0, separator), relayHostId: key.slice(separator + 1) }
+}
+
 export class ShadowSeatDirectory {
   private readonly cells = new Map<string, CellCursor>()
   private readonly hosts = new Map<string, Map<string, ShadowSeat>>()
@@ -193,6 +218,10 @@ export class ShadowSeatDirectory {
   // Per host, the newest booking any cell reported (step 5 epoch mint). Age-ordered.
   private readonly bookings = new Map<string, { cellId: string; epoch: number; at: number }>()
   onBookedJoin?: (join: BookedJoin) => void
+  onOutranked?: (seat: OutrankedSeat) => void
+  private readonly joinedSinceApply = new Set<string>()
+  // Seats already reported, insertion-ordered: a full resync re-judges a host without a second demote.
+  private readonly reportedOutranked = new Set<string>()
 
   // Cells no longer listed are forgotten with their seats; new ones start pending.
   // `required` defaults to every listed cell.
@@ -225,6 +254,11 @@ export class ShadowSeatDirectory {
 
   // `polledAt` is when the request left; the reply holds every change before it.
   apply(cellId: string, response: SeatFeedResponse, now: number, polledAt = now): void {
+    this.applyResponse(cellId, response, now, polledAt)
+    this.reportOutranked()
+  }
+
+  private applyResponse(cellId: string, response: SeatFeedResponse, now: number, polledAt: number): void {
     const cursor = this.cells.get(cellId)
     if (!cursor) return
     if (response.cellId !== cellId) {
@@ -307,10 +341,7 @@ export class ShadowSeatDirectory {
   seatsOnCell(cellId: string): Array<{ userId: string; relayHostId: string; seat: ShadowSeat }> {
     const cursor = this.cells.get(cellId)
     if (!cursor) return []
-    return [...cursor.seats].map(([key, seat]) => {
-      const separator = key.indexOf('\u0000')
-      return { userId: key.slice(0, separator), relayHostId: key.slice(separator + 1), seat }
-    })
+    return [...cursor.seats].map(([key, seat]) => ({ ...parseHostKey(key), seat }))
   }
 
   // An old cell (404) has no feed; its last known seats stay, marked by status.
@@ -373,6 +404,15 @@ export class ShadowSeatDirectory {
     return known && known.epoch >= epoch ? known.firstSeenAt : undefined
   }
 
+  // The newest epoch this director has seen the database name for the host, if still kept.
+  databaseEpochOf(
+    userId: string,
+    relayHostId: string
+  ): { epoch: number; firstSeenAt: number } | undefined {
+    const known = this.databaseEpochs.get(hostKey(userId, relayHostId))
+    return known ? { ...known } : undefined
+  }
+
   // Records a sighting; returns when this director first saw the host at this epoch.
   observeDatabaseEpoch(userId: string, relayHostId: string, epoch: number, now: number): number {
     const key = hostKey(userId, relayHostId)
@@ -558,6 +598,34 @@ export class ShadowSeatDirectory {
       this.hosts.set(key, seats)
     }
     seats.set(cursor.cellId, seat)
+    if (this.onOutranked && seats.size > 1) this.joinedSinceApply.add(key)
+  }
+
+  // After the whole response, so a cell's first snapshot is judged by the mode it carries. Only a
+  // join triggers it, once per seat: a cell flipped to reserve later, or a lost demote, falls to
+  // the silence close or the ledger reconcile, as does a lower epoch that joined later.
+  private reportOutranked(): void {
+    const report = this.onOutranked
+    const keys = [...this.joinedSinceApply]
+    this.joinedSinceApply.clear()
+    if (!report) return
+    for (const key of keys) {
+      const seats = [...(this.hosts.get(key)?.values() ?? [])]
+      for (const seat of seats) {
+        if (seat.state !== 'active' || this.admitModeOf(seat.cellId) !== 'reserve') continue
+        const by = seats.find((other) => seatOutranks(other, seat))
+        if (!by) continue
+        const { cellId, epoch, joinedAt } = seat
+        const reported = `${key}\u0000${cellId}\u0000${epoch}\u0000${joinedAt}`
+        if (this.reportedOutranked.has(reported)) continue
+        this.reportedOutranked.add(reported)
+        if (this.reportedOutranked.size > SHADOW_SEAT_OUTRANKED_REPORTS_MAX) {
+          const oldest = this.reportedOutranked.values().next().value
+          if (oldest !== undefined) this.reportedOutranked.delete(oldest)
+        }
+        report({ ...parseHostKey(key), cellId, epoch, joinedAt, byCellId: by.cellId })
+      }
+    }
   }
 
   private unseat(cursor: CellCursor, key: string): void {

cloud/apps/relay/src/sticky-slot-statements.test.ts

@@ -0,0 +1,103 @@
+import { describe, expect, it } from 'vitest'
+import { runInStickySlot, StickySlotStatements } from './sticky-slot-statements.js'
+import {
+  ScriptedPoolClient,
+  scriptedPostgresDatabase
+} from './test-fixtures/scripted-postgres-pool.js'
+
+function gated() {
+  let open!: () => void
+  const opened = new Promise<void>((resolve) => (open = resolve))
+  return { open, wait: async () => await opened }
+}
+
+describe('sticky slot statements', () => {
+  it('charges the slot every round trip it made, by pooled connection', async () => {
+    const first = new ScriptedPoolClient()
+    const second = new ScriptedPoolClient()
+    const database = scriptedPostgresDatabase([first, second, first])
+    const slot = new StickySlotStatements()
+    await runInStickySlot(slot, async () => {
+      await database.query('SELECT verify')
+      await database.transaction(async (transaction) => {
+        await transaction.query('SELECT pin')
+        await transaction.queryLocked('SELECT row', [], { lockTimeoutMs: 200 })
+      })
+    })
+    // Outside the slot: not this request's work.
+    await database.query('SELECT other')
+
+    const sample = slot.close()
+    // BEGIN, the statement, both lock_timeout SETs, the locked read and COMMIT.
+    expect(second.statements).toHaveLength(6)
+    expect(sample.statements).toBe(7)
+    expect(sample.sqlMs).toBeGreaterThanOrEqual(0)
+    expect(
+      [...sample.statementMsByConnection].map(([id, samples]) => [id, samples.length])
+    ).toEqual([
+      [1, 1],
+      [2, 6]
+    ])
+  })
+
+  it('leaves out statements another request runs while the slot is held', async () => {
+    const otherGate = gated()
+    const database = scriptedPostgresDatabase([
+      new ScriptedPoolClient(otherGate.wait),
+      new ScriptedPoolClient()
+    ])
+    const slot = new StickySlotStatements()
+    const other = database.query('SELECT other')
+    await runInStickySlot(slot, async () => {
+      await database.query('SELECT mine')
+      otherGate.open()
+      await other
+    })
+
+    const sample = slot.close()
+    expect(sample.statements).toBe(1)
+    expect([...sample.statementMsByConnection.keys()]).toEqual([2])
+  })
+
+  it('ignores work the request left running after it released the slot', async () => {
+    const late = gated()
+    const database = scriptedPostgresDatabase([new ScriptedPoolClient(), new ScriptedPoolClient()])
+    const slot = new StickySlotStatements()
+    let leftRunning: Promise<unknown> | undefined
+    await runInStickySlot(slot, async () => {
+      await database.query('SELECT 1')
+      leftRunning = late.wait().then(async () => await database.query('SELECT late'))
+    })
+    const sample = slot.close()
+    late.open()
+    await leftRunning
+    expect(sample.statements).toBe(1)
+  })
+
+  it('reports the pool wait apart from the statements', async () => {
+    const database = scriptedPostgresDatabase([new ScriptedPoolClient()], 30)
+    const slot = new StickySlotStatements()
+    await runInStickySlot(slot, async () => await database.query('SELECT 1'))
+
+    const sample = slot.close()
+    expect(sample.poolWaitMs).toBeGreaterThanOrEqual(20)
+    expect(sample.sqlMs).toBeLessThan(20)
+  })
+
+  it('names each pooled connection once, however often it is checked out', async () => {
+    const first = new ScriptedPoolClient()
+    const second = new ScriptedPoolClient()
+    const database = scriptedPostgresDatabase([first, second, first])
+    const slot = new StickySlotStatements()
+    await runInStickySlot(slot, async () => {
+      for (let index = 0; index < 3; index++) await database.query('SELECT 1')
+    })
+
+    expect(
+      [...slot.close().statementMsByConnection].map(([id, samples]) => [id, samples.length])
+    ).toEqual([
+      [1, 2],
+      [2, 1]
+    ])
+  })
+})

cloud/apps/relay/src/sticky-slot-statements.ts

@@ -0,0 +1,62 @@
+import { AsyncLocalStorage } from 'node:async_hooks'
+
+export type StickySlotSample = {
+  queueWaitMs: number
+  // Waiting for a pooled connection, apart from the statements run on one.
+  poolWaitMs: number
+  sqlMs: number
+  statements: number
+  statementMsByConnection: ReadonlyMap<number, readonly number[]>
+}
+
+type StickySlotStatementTotals = Omit<StickySlotSample, 'queueWaitMs'>
+
+// PostgreSQL round trips (all but ROLLBACK) made while one request holds the sticky
+// slot. Closed at release, so work the request left running is not charged to it.
+export class StickySlotStatements {
+  private closed = false
+  private poolWaitMs = 0
+  private sqlMs = 0
+  private statements = 0
+  private readonly statementMsByConnection = new Map<number, number[]>()
+
+  notePoolWait(durationMs: number): void {
+    if (!this.closed) this.poolWaitMs += durationMs
+  }
+
+  noteStatement(durationMs: number, connectionId: number): void {
+    if (this.closed) return
+    this.statements++
+    this.sqlMs += durationMs
+    const samples = this.statementMsByConnection.get(connectionId)
+    if (samples) samples.push(durationMs)
+    else this.statementMsByConnection.set(connectionId, [durationMs])
+  }
+
+  close(): StickySlotStatementTotals {
+    this.closed = true
+    return {
+      poolWaitMs: this.poolWaitMs,
+      sqlMs: this.sqlMs,
+      statements: this.statements,
+      statementMsByConnection: this.statementMsByConnection
+    }
+  }
+}
+
+const activeStickySlot = new AsyncLocalStorage<StickySlotStatements>()
+
+export function runInStickySlot<T>(
+  statements: StickySlotStatements,
+  work: () => Promise<T>
+): Promise<T> {
+  return activeStickySlot.run(statements, work)
+}
+
+export function noteStickySlotPoolWait(durationMs: number): void {
+  activeStickySlot.getStore()?.notePoolWait(durationMs)
+}
+
+export function noteStickySlotStatement(durationMs: number, connectionId: number): void {
+  activeStickySlot.getStore()?.noteStatement(durationMs, connectionId)
+}

cloud/apps/relay/src/test-fixtures/scripted-postgres-pool.ts

@@ -0,0 +1,40 @@
+import { EventEmitter } from 'node:events'
+import { PostgresDatabase } from '../database.js'
+
+export class ScriptedPoolClient extends EventEmitter {
+  readonly statements: string[] = []
+  release: () => void = () => undefined
+
+  // Resolves each statement only once `gate` does, so a test can hold one mid-flight.
+  constructor(private readonly gate: () => Promise<void> = async () => undefined) {
+    super()
+  }
+
+  query = async (sql: string | { text: string }) => {
+    this.statements.push(typeof sql === 'string' ? sql : sql.text)
+    await this.gate()
+    return { rows: [], rowCount: 0 }
+  }
+}
+
+// Hands out the listed clients in order, reusing an object the way pg-pool reuses a connection.
+export function scriptedPostgresDatabase(
+  checkouts: ScriptedPoolClient[],
+  connectDelayMs = 0
+): PostgresDatabase {
+  const pool = {
+    totalCount: checkouts.length,
+    idleCount: 0,
+    waitingCount: 0,
+    connect: async () => {
+      if (connectDelayMs > 0) await new Promise((resolve) => setTimeout(resolve, connectDelayMs))
+      const client = checkouts.shift()
+      if (!client) throw new Error('scripted_pool_exhausted')
+      // pg-pool assigns a fresh release on every checkout.
+      client.release = () => undefined
+      return client
+    }
+  }
+  // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: PostgresDatabase and its pool pressure read only connect, these counts and the optional options.
+  return new PostgresDatabase(pool as never)
+}

cloud/apps/relay/vitest.config.ts

@@ -1,26 +1,22 @@
 import { readdirSync, readFileSync } from 'node:fs'
+import { join } from 'node:path'
 import { fileURLToPath } from 'node:url'
 import { defaultExclude, defineConfig } from 'vitest/config'
 
-// Every test file that opens ORCA_RELAY_TEST_POSTGRES_URL shares one CI
-// database, and those tests take session-level locks with a 1s lock_timeout.
-// Running them alongside anything else collides into lock timeouts, capacity
-// exhaustion, and afterAll hangs. Keep them in their own serialized project so
-// only they give up file parallelism; the rest of the suite opens SQLite data
-// directories and stays fully parallel.
+// PostgreSQL files share session locks and a small connection budget.
+// The runner overlaps this serialized project with SQLite tests after other packages finish.
 const sourceDirectory = fileURLToPath(new URL('src', import.meta.url))
 const sharedPostgresTests = readdirSync(sourceDirectory)
   .filter((entry) => entry.endsWith('.test.ts'))
   .filter((entry) =>
-    readFileSync(`${sourceDirectory}/${entry}`, 'utf8').includes(
-      'ORCA_RELAY_TEST_POSTGRES_URL'
-    )
+    readFileSync(join(sourceDirectory, entry), 'utf8').includes('ORCA_RELAY_TEST_POSTGRES_URL')
   )
   .map((entry) => `src/${entry}`)
 
 const timeouts = { testTimeout: 15_000, hookTimeout: 15_000 }
 
 export default defineConfig({
+  cacheDir: process.env.ORCA_RELAY_TEST_CACHE_DIR,
   test: {
     projects: [
       {

cloud/dev/fixtures/terraform-root-partition/families.json

@@ -146,6 +146,7 @@
     "google_logging_metric.relay_event_loop_stall",
     "google_logging_metric.relay_long_cell_inventory_hold",
     "google_logging_metric.relay_reserve_placement_off_with_reserve_cells",
+    "google_logging_metric.relay_reserve_watch_alarm",
     "google_logging_metric.relay_snapshot",
     "google_monitoring_alert_policy.relay_assignment_5xx",
     "google_monitoring_alert_policy.relay_assignment_edge_429",
@@ -176,6 +177,7 @@
     "google_monitoring_alert_policy.relay_process_fatal",
     "google_monitoring_alert_policy.relay_region_hint_skew",
     "google_monitoring_alert_policy.relay_reserve_placement_off_with_reserve_cells",
+    "google_monitoring_alert_policy.relay_reserve_watch_alarm",
     "google_monitoring_dashboard.relay_incident",
     "google_project_iam_custom_role.github_production_relay_capacity_mutation",
     "google_project_iam_custom_role.github_relay_asia_topology_mutation",
@@ -304,6 +306,10 @@
       "production": "relay",
       "staging": "apps"
     },
+    "google_project_iam_member.github_log_writer": {
+      "production": "relay",
+      "staging": "apps"
+    },
     "google_project_iam_member.github_logging_viewer": {
       "production": "relay",
       "staging": "apps"

cloud/dev/scripts/operate-relay-cell-flags.mjs

@@ -244,7 +244,7 @@ function objectUrl(request, suffix = '') {
 }
 
 // One transient 5xx or network failure is retried once, as every relay admin step does.
-async function readRuntime(fetchImpl, request, idToken, wait) {
+export async function readRuntime(fetchImpl, request, idToken, wait) {
   const response = await fetchAdminOnceMore(
     fetchImpl,
     `${request.cellOrigin}/v1/admin/runtime-status`,
@@ -262,7 +262,7 @@ async function readRuntime(fetchImpl, request, idToken, wait) {
   return await response.json()
 }
 
-async function readCurrentObject(fetchImpl, request, accessToken) {
+export async function readCurrentObject(fetchImpl, request, accessToken) {
   const headers = { authorization: `Bearer ${accessToken}` }
   const metadata = await fetchImpl(objectUrl(request), { headers })
   if (metadata.status === 404) {
@@ -278,6 +278,12 @@ async function readCurrentObject(fetchImpl, request, accessToken) {
   return { generation: String(generation), object: await media.json().catch(() => null) }
 }
 
+function generationConflict(current, expected) {
+  return Object.assign(new Error(`current generation is ${current}, not expected ${expected}`), {
+    generationConflict: true
+  })
+}
+
 async function writeObject(fetchImpl, request, object, accessToken, audit) {
   const boundary = `relay-cell-flags-${Date.now()}`
   const metadata = {
@@ -315,7 +321,8 @@ async function writeObject(fetchImpl, request, object, accessToken, audit) {
   if (response.status === 412) {
     await response.body?.cancel().catch(() => undefined)
     throw Object.assign(new Error('the object changed since expected-generation; re-read and retry'), {
-      notWritten: true
+      notWritten: true,
+      generationConflict: true
     })
   }
   if (!response.ok) {
@@ -409,7 +416,7 @@ async function breakGlassToDatabase(request, dependencies, { fetchImpl, log, sle
   }
   const current = await readCurrentObject(fetchImpl, request, accessToken)
   if (current.generation !== request.expectedGeneration) {
-    throw new Error(`current generation is ${current.generation}, not expected ${request.expectedGeneration}`)
+    throw generationConflict(current.generation, request.expectedGeneration)
   }
   const object = desiredObject(request, current.object)
   const generation = await writeObject(fetchImpl, request, object, accessToken, audit)
@@ -459,9 +466,7 @@ export async function operateCellFlags(request, dependencies) {
   // Before the generation check, so a dry run always shows the generation to pass next.
   log(JSON.stringify(plan))
   if (current.generation !== request.expectedGeneration) {
-    throw new Error(
-      `current generation is ${current.generation}, not expected ${request.expectedGeneration}`
-    )
+    throw generationConflict(current.generation, request.expectedGeneration)
   }
   // The cell applies every readable write; a generation it did not apply was voided there. A
   // write of an object this image accepts whole (checked above) repairs it.

cloud/dev/scripts/relay-production-identity-boundaries.test.mjs

@@ -33,6 +33,7 @@ test('no workflow names the retired generic production deploy identity', async (
     'operate-relay-asia-admission.yml',
     'operate-relay-production-cell-flags.yml',
     'operate-relay-production-rehome-job.yml',
+    'operate-relay-production-reserve-watch.yml',
     'publish-relay-production.yml'
   ].map((name) => relayWorkflowFile(name)).sort())
 })

cloud/dev/scripts/relay-production-same-cap-wave.test.mjs

@@ -922,18 +922,14 @@ test('the workflows offer exactly the closed set and scale the drain wait with i
   assert.match(job, /\n      DRAIN_PACE_WINDOW_MS: \$\{\{ inputs\.drain-pace-window-ms \}\}\n/)
   assert.doesNotMatch(job, /DRAIN_PACE_WINDOW_MS: '/)
   assert.match(job, /cell-class \\\n\s+--cell-id "\$\{TARGET_CELL_ID\}" --drain-pace-window-ms "\$\{DRAIN_PACE_WINDOW_MS\}"/)
-  // The 15-min migration lease plus the window, plus its excess over the default: restart-safe
-  // waits a whole window of quiet after the last send, so a slow drain needs about twice it.
+  // The 15-min migration lease plus the window: restart-safe passes once the window has ended.
   const timeout = /--timeout-ms "\$\(\((.+)\)\)"\n/.exec(job)[1]
   const timeoutMs = (pace) => Number(execFileSync(
     'bash', ['-c', `DRAIN_PACE_WINDOW_MS=${pace}; echo $((${timeout}))`], { encoding: 'utf8' }
   ))
-  // Exactly the timeouts every pace had before the slower ones existed.
-  for (const pace of [300_000, 60_000, 30_000]) assert.equal(timeoutMs(pace), 900_000 + pace)
   for (const pace of SAME_CAP_DRAIN_PACE_WINDOWS_MS) {
-    assert.ok(timeoutMs(pace) >= 2 * pace + 600_000, String(pace))
+    assert.equal(timeoutMs(pace), 900_000 + pace, String(pace))
   }
-  assert.equal(timeoutMs(1_200_000), 3_000_000)
   const slowest = timeoutMs(Math.max(...SAME_CAP_DRAIN_PACE_WINDOWS_MS))
   // The rest of the job keeps the 70 min it had beside the old 20-min drain step in 90.
   const jobMinutes = Number(/\n  rollout:\n(?:    [^\n]*\n)*?    timeout-minutes: (\d+)\n/.exec(job)[1])

cloud/dev/scripts/relay-public-workflow-contract.test.mjs

@@ -20,7 +20,7 @@ const UNGATED = relayWorkflowFile('verify.yml')
 const relayWorkflows = () => workflowFiles().filter((file) => file !== UNGATED)
 
 test('the copy carries every relay workflow', () => {
-  assert.equal(relayWorkflows().length, 27)
+  assert.equal(relayWorkflows().length, 28)
 })
 
 // Why: workflow_run chains match by display name, not filename. Renaming a file is safe; renaming

cloud/dev/scripts/relay-reserve-dead-man-alert.test.mjs

@@ -12,6 +12,8 @@ const deadMan = read('../../apps/relay/src/cell-reserve-dead-man.ts')
 const registry = read('../../apps/relay/src/host-session-registry.ts')
 const server = read('../../apps/relay/src/relay-server.ts')
 const poller = read('../../apps/relay/src/shadow-seat-directory.ts')
+const watch = read('./watch-relay-reserve-cell.mjs')
+const githubActions = read('../../infra/terraform/relay-github-actions.tf')
 
 const block = (kind, name) => {
   const body = new RegExp(`resource "${kind}" "${name}" \\{([\\s\\S]*?)\\n\\}`).exec(terraform)?.[1]
@@ -84,3 +86,24 @@ test('the withheld dead-cell verdict alert counts every director withheld line,
   assert.match(read('../../infra/terraform/variables.tf'), /variable "relay_dead_cell_verdict_alert" \{[\s\S]*?default     = false/)
   assert.match(read('../../infra/terraform/environments/staging.tfvars'), /relay_dead_cell_verdict_alert\s+= true/)
 })
+
+test('the reserve watch alarm pages on any line the watch runner writes, per cell', () => {
+  // The runner writes a global-resource entry under its own log name.
+  assert.match(watch, /export const ALARM_EVENT = 'orca_relay_reserve_watch_alarm'/)
+  assert.match(watch, /export const ALARM_LOG_NAME = 'orca-relay-reserve-watch'/)
+  assert.match(watch, /JSON\.stringify\(\{ event: ALARM_EVENT, cellId, kind, reason \}\)/)
+  const metric = block('google_logging_metric', 'relay_reserve_watch_alarm')
+  assert.ok(
+    metric.includes(
+      'resource.type=\\"global\\" AND logName=\\"projects/${var.project_id}/logs/orca-relay-reserve-watch\\" AND jsonPayload.event=\\"orca_relay_reserve_watch_alarm\\"'
+    )
+  )
+  assert.ok(metric.includes('cell_id = "EXTRACT(jsonPayload.cellId)"'))
+  notifyingOnAnyLine('relay_reserve_watch_alarm', 'orca_relay_reserve_watch_alarm', 'global', 'cell_id')
+  // The deploy identity the workflow runs as can write that line, and only in production.
+  const grant = /resource "google_project_iam_member" "github_log_writer" \{([\s\S]*?)\n\}/.exec(githubActions)?.[1]
+  assert.ok(grant, 'github_log_writer grant not found')
+  assert.match(grant, /count = local\.relay_create_production_ops_identity \? 1 : 0/)
+  assert.match(grant, /role    = "roles\/logging\.logWriter"/)
+  assert.match(grant, /member  = google_service_account\.github_deploy\[0\]\.member/)
+})

cloud/dev/scripts/relay-reserve-watch-rules.mjs

@@ -0,0 +1,258 @@
+// Stop rules for a cell watched in reserve mode (step 5 design §9, "Auto-abort per flip"). Pure:
+// it takes one already-read sample and the watch state, and returns the next state and whether
+// to flip the cell back. The reader lives in watch-relay-reserve-cell.mjs.
+import { backgroundOf, SHADOW_GATE_THRESHOLDS } from './relay-same-cap-shadow-gate-verdict.mjs'
+
+const MINUTE_MS = 60_000
+
+export const RESERVE_WATCH_RULES = {
+  // The roll ladder's warn line: max(1.5x, background + 20), so a zero background is not tripped
+  // by one 503. Three complete minutes in a row, not two. The read spans more minutes than that so
+  // a late publishing front still leaves three published minutes to judge.
+  director503: {
+    multiple: SHADOW_GATE_THRESHOLDS.nonDrain503Budget.warnMultiple,
+    marginPerMinute: SHADOW_GATE_THRESHOLDS.nonDrain503Budget.warnMarginPerMinute,
+    consecutiveMinutes: 3,
+    readMinutes: 10,
+    // Cloud Run publishes a minute's request count up to 3 minutes late.
+    publishDelayMs: 3 * MINUTE_MS
+  },
+  // The cell closes a demoted seat after 60 s (DEMOTION_CLOSE_MS); 30 s more covers a director's
+  // map still holding the closed seat.
+  duplicateSeat: { graceMs: 60_000 + 30_000, lookbackMs: 15 * MINUTE_MS },
+  // The floor keeps an almost empty cell's ordinary churn from reading as a 5% drop.
+  hostDrop: { share: 0.05, windowMs: 5 * MINUTE_MS, minHosts: 10 },
+  ledgerDrift: { share: 0.005, sustainMs: 2 * MINUTE_MS },
+  // A placing director logs its ledger summary every 60 s; none in this long is no evidence.
+  ledgerSummaryMaxAgeMs: 150_000,
+  // A watch dispatched before the reserve write applies waits this long for the cell to report it.
+  reserveApplyWaitMs: 10 * MINUTE_MS
+}
+
+// Precedence: the first tripped rule names the verdict, so exact signals come before proxies. The
+// sample line's `rules` keys follow this order too.
+export const RULE_NAMES = ['deadMan', 'overrun', 'director503', 'duplicateSeat', 'hostDrop', 'ledgerDrift']
+
+export function initialWatchState({ cellId, startedAt, durationMinutes, background503PerMinute, flippedAt }) {
+  return {
+    v: 1,
+    cellId,
+    startedAt,
+    deadlineAt: startedAt + durationMinutes * MINUTE_MS,
+    state: 'watching',
+    verdict: null,
+    samples: 0,
+    unverifiableStreak: 0,
+    // The background is measured in the minutes before the flip, unless the operator supplied it.
+    flippedAt: flippedAt ?? null,
+    background:
+      background503PerMinute === null ? null : { medianPerMinute: background503PerMinute, source: 'input' },
+    hosts: [],
+    reserveSeenAt: null,
+    ledgerAboveSince: null,
+    seatBehindRowFirstSeen: {}
+  }
+}
+
+// The background from the per-minute 503 counts before the flip. `publishedThrough` is the newest
+// minute any response code has a point for: a 503 minute at or before it with no point had none,
+// and the background waits (null) until it covers the last minute.
+export function measuredBackground(perMinute, minutes, publishedThrough) {
+  if (minutes.length === 0 || publishedThrough === null || publishedThrough < minutes.at(-1)) return null
+  const background = backgroundOf({ minutes, series: minutes.map((minute) => perMinute[minute] ?? 0), unverified: false })
+  return { medianPerMinute: background.medianPerMinute, minutes: background.minutes, source: 'measured' }
+}
+
+const pass = (detail = {}) => ({ status: 'pass', ...detail })
+const trip = (reason, detail = {}) => ({ status: 'trip', reason, ...detail })
+const unverifiable = (reason) => ({ status: 'unverifiable', reason })
+// Not readable yet by design, so it does not count toward the unverifiable streak.
+const pending = (reason) => ({ status: 'pending', reason })
+
+function judgeDeadMan(cell) {
+  if (!cell.ok) return unverifiable(cell.error)
+  if (cell.admitModeEffective !== 'db' && cell.admitModeEffective !== 'reserve') {
+    return unverifiable('runtime-status reports no admitModeEffective')
+  }
+  if (cell.admitModeEffective === 'db') {
+    return trip('the cell tripped its dead-man (reserve configured, db effective); Postgres still records reserve')
+  }
+  return pass()
+}
+
+// The cell ledger's total (seats, bookings, in-flight and phone units) against its hard cap. The
+// ledger refuses past the cap, so this is an invariant check: it trips only if that refusal broke.
+function judgeOverrun(cell) {
+  if (!cell.ok) return unverifiable(cell.error)
+  if (!Number.isInteger(cell.hardCap) || !Number.isInteger(cell.enforcedUnits)) {
+    return unverifiable('runtime-status reports no hard cap or ledger units')
+  }
+  if (cell.enforcedUnits > cell.hardCap) {
+    return trip(`ledger units ${cell.enforcedUnits} over hard cap ${cell.hardCap}`, {
+      enforcedUnits: cell.enforcedUnits,
+      hardCap: cell.hardCap
+    })
+  }
+  return pass({ enforcedUnits: cell.enforcedUnits, hardCap: cell.hardCap })
+}
+
+// Fleet-wide director 503s: no existing signal attributes a 503 to the cell a host was bound for,
+// so a problem local to one cell can stay under the fleet's line. This proxy can miss.
+function judgeDirector503(read, state, at) {
+  const { multiple, marginPerMinute, consecutiveMinutes, publishDelayMs } = RESERVE_WATCH_RULES.director503
+  const { background } = state
+  if (!background) {
+    const publishesAt = state.flippedAt === null ? null : Math.floor(state.flippedAt / MINUTE_MS) * MINUTE_MS + publishDelayMs
+    if (publishesAt !== null && at < publishesAt) return pending('the 503 background before the flip is not published yet')
+    return unverifiable('no 503 background yet')
+  }
+  if (!read.ok) return unverifiable(read.error)
+  const perMinute = background.medianPerMinute
+  const above = Math.max(perMinute * multiple, perMinute + marginPerMinute)
+  // Minute keys sort as time. Only minutes the publishing front has reached are judged: a 503
+  // minute there with no point had none, one past it is not published yet and is left out.
+  if (read.publishedThrough === null) return unverifiable('the director request series is empty')
+  const recent = read.minutes.filter((minute) => minute <= read.publishedThrough).slice(-consecutiveMinutes)
+  if (recent.length < consecutiveMinutes) {
+    return unverifiable(`director requests are published only through ${read.publishedThrough}`)
+  }
+  const counts = recent.map((minute) => read.perMinute[minute] ?? 0)
+  const detail = { backgroundPerMinute: perMinute, above, recent: Object.fromEntries(recent.map((m, i) => [m, counts[i]])) }
+  if (counts.every((count) => count > above)) {
+    return trip(`director 503s above ${above}/min for ${consecutiveMinutes} min (${counts.join(', ')})`, detail)
+  }
+  return pass(detail)
+}
+
+// A seat the ledger reconcile found behind its row and found again after the demotion should
+// have closed it. The line names no host, so (director instance, cell, epoch, rowEpoch) stands in
+// for one: a proxy that can only over-count, when two hosts share that key.
+function judgeDuplicateSeat(read, firstSeen, at) {
+  if (!read.ok) return { verdict: unverifiable(read.error), firstSeen }
+  const { graceMs, lookbackMs } = RESERVE_WATCH_RULES.duplicateSeat
+  const next = Object.fromEntries(
+    Object.entries(firstSeen).filter(([, seenAt]) => seenAt >= at - 2 * lookbackMs)
+  )
+  for (const report of [...read.reports].sort((left, right) => left.at - right.at)) {
+    next[report.key] = Math.min(next[report.key] ?? report.at, report.at)
+  }
+  for (const report of read.reports) {
+    const ageMs = report.at - next[report.key]
+    if (ageMs > graceMs) {
+      return {
+        verdict: trip(`a seat behind its ledger row (${report.key}) is still seated ${Math.round(ageMs / 1000)} s after first report`),
+        firstSeen: next
+      }
+    }
+  }
+  return { verdict: pass({ reportsInLookback: read.reports.length }), firstSeen: next }
+}
+
+function judgeHostDrop(hosts) {
+  const { share, minHosts, windowMs } = RESERVE_WATCH_RULES.hostDrop
+  const current = hosts.at(-1)
+  if (!current) return unverifiable('no host count')
+  if (hosts.some((entry) => entry.draining)) return pass({ controls: current.controls, draining: true })
+  const peak = Math.max(...hosts.map((entry) => entry.controls))
+  const drop = peak - current.controls
+  const detail = { controls: current.controls, peakInWindow: peak }
+  if (peak > 0 && drop >= minHosts && drop / peak > share) {
+    return trip(`connected hosts fell ${peak} -> ${current.controls} within ${windowMs / MINUTE_MS} min outside a drain`, detail)
+  }
+  return pass(detail)
+}
+
+// Rows the placing directors have not written (queued) or gave up on (dropped), per seat. The
+// summaries are fleet-wide: exact while this is the only reserve cell, an over-count otherwise.
+function judgeLedgerDrift(cell, ledger, aboveSince, at) {
+  if (!cell.ok) return { verdict: unverifiable(cell.error), aboveSince }
+  if (!ledger.ok) return { verdict: unverifiable(ledger.error), aboveSince }
+  const { share, sustainMs } = RESERVE_WATCH_RULES.ledgerDrift
+  const ratio = ledger.unwrittenRows / Math.max(cell.controls, 1)
+  const detail = { unwrittenRows: ledger.unwrittenRows, directors: ledger.directors, ratio }
+  if (ratio <= share) return { verdict: pass(detail), aboveSince: null }
+  const since = aboveSince ?? at
+  if (at - since >= sustainMs) {
+    return {
+      verdict: trip(`ledger drift ${(ratio * 100).toFixed(2)}% for ${Math.round((at - since) / 1000)} s`, detail),
+      aboveSince: since
+    }
+  }
+  return { verdict: pass({ ...detail, aboveForMs: at - since }), aboveSince: since }
+}
+
+/**
+ * One sample. `sample.cell` is the cell's runtime-status, `director503` the complete minutes of
+ * director 503s with the request series' publishing front (`publishedThrough`), `ledger` the placing directors' newest ledger summaries, `seatBehindRow` this
+ * cell's reconcile reports in the lookback. Each is `{ ok: false, error }` when it could not be read.
+ */
+export function evaluateWatchSample(state, sample, { unverifiableLimit }) {
+  const at = sample.at
+  const next = { ...state, samples: state.samples + 1 }
+  const { cell } = sample
+  if (cell.ok && cell.admitModeConfigured !== 'reserve') {
+    const seen = state.reserveSeenAt !== null
+    if (!seen && at < state.startedAt + RESERVE_WATCH_RULES.reserveApplyWaitMs) {
+      // Not judged, and the streak holds: the reserve write may not have applied yet.
+      const rules = { reserve: pending(`switch file says ${cell.admitModeConfigured}; waiting for reserve to apply`) }
+      return { state: next, rules, line: sampleLine(next, sample, rules) }
+    }
+    // Someone else flipped it back (or it never became reserve): nothing left to watch.
+    next.state = seen ? 'stopped' : 'refused'
+    next.verdict = seen
+      ? `${state.cellId} is not in reserve mode (switch file says ${cell.admitModeConfigured})`
+      : `${state.cellId} did not report reserve mode within ${RESERVE_WATCH_RULES.reserveApplyWaitMs / MINUTE_MS} min (switch file says ${cell.admitModeConfigured})`
+    return { state: next, rules: {}, line: sampleLine(next, sample, {}) }
+  }
+  if (cell.ok) {
+    next.reserveSeenAt = state.reserveSeenAt ?? at
+    next.hosts = [...state.hosts, { at, controls: cell.controls, draining: cell.draining }].filter(
+      (entry) => entry.at > at - RESERVE_WATCH_RULES.hostDrop.windowMs
+    )
+  }
+  const duplicate = judgeDuplicateSeat(sample.seatBehindRow, state.seatBehindRowFirstSeen, at)
+  next.seatBehindRowFirstSeen = duplicate.firstSeen
+  const ledger = judgeLedgerDrift(cell, sample.ledger, state.ledgerAboveSince, at)
+  next.ledgerAboveSince = ledger.aboveSince
+  const rules = {
+    deadMan: judgeDeadMan(cell),
+    overrun: judgeOverrun(cell),
+    director503: judgeDirector503(sample.director503, state, at),
+    duplicateSeat: duplicate.verdict,
+    hostDrop: cell.ok ? judgeHostDrop(next.hosts) : unverifiable(cell.error),
+    ledgerDrift: ledger.verdict
+  }
+  const tripped = RULE_NAMES.find((name) => rules[name].status === 'trip')
+  if (tripped) {
+    next.state = 'tripped'
+    next.verdict = `flip back ${state.cellId}: ${tripped}: ${rules[tripped].reason}`
+    return { state: next, rules, line: sampleLine(next, sample, rules) }
+  }
+  const missing = RULE_NAMES.filter((name) => rules[name].status === 'unverifiable')
+  const waiting = RULE_NAMES.some((name) => rules[name].status === 'pending')
+  // A pending rule holds the streak where it was: neither evidence of health nor a gap.
+  next.unverifiableStreak = missing.length > 0 ? state.unverifiableStreak + 1 : waiting ? state.unverifiableStreak : 0
+  if (next.unverifiableStreak >= unverifiableLimit) {
+    next.state = 'tripped'
+    next.verdict =
+      `flip back ${state.cellId}: unverifiable: ${next.unverifiableStreak} samples in a row could not ` +
+      `read ${missing.join(', ')}`
+  } else if (at >= state.deadlineAt) {
+    next.state = 'completed'
+    next.verdict = `${state.cellId} stayed healthy in reserve mode for the whole watch`
+  }
+  return { state: next, rules, line: sampleLine(next, sample, rules) }
+}
+
+function sampleLine(state, sample, rules) {
+  return JSON.stringify({
+    event: 'orca_relay_reserve_watch_sample',
+    cellId: state.cellId,
+    at: new Date(sample.at).toISOString(),
+    sample: state.samples,
+    state: state.state,
+    unverifiableStreak: state.unverifiableStreak,
+    rules,
+    ...(state.verdict ? { verdict: state.verdict } : {})
+  })
+}

cloud/dev/scripts/relay-same-cap-shadow-gate.mjs

@@ -122,7 +122,7 @@ async function retryingRead(reader, read) {
   return { failed: true, error: String(lastError?.message ?? lastError) }
 }
 
-async function readLogEntries(reader, { filter, projection, limit = ENTRY_LIMIT }) {
+export async function readLogEntries(reader, { filter, projection, limit = ENTRY_LIMIT }) {
   const args = [
     'logging', 'read', filter,
     '--project', reader.projectId,
@@ -154,21 +154,22 @@ async function readTimestampsOverWindow(reader, { filter, window }) {
 
 // Cloud Run's own request counter, aligned per minute server-side: a brownout's tens of thousands
 // of 503s are counted, where a log read of them stops at its entry limit (10-01 c29).
-async function readDirector503PerMinute(reader, { config, span }) {
+async function readDirectorRequestCountSeries(reader, { config, span, filters = [], groupByFields }) {
   const url = new URL(`https://monitoring.googleapis.com/v3/projects/${config.projectId}/timeSeries`)
   url.searchParams.set('filter', [
     'metric.type="run.googleapis.com/request_count"',
     'resource.type="cloud_run_revision"',
     `resource.label."service_name"="${config.directorService}"`,
-    'metric.label."response_code"="503"'
+    ...filters
   ].join(' AND '))
   url.searchParams.set('interval.startTime', span.startedAt.toISOString())
   url.searchParams.set('interval.endTime', span.endedAt.toISOString())
   url.searchParams.set('aggregation.alignmentPeriod', '60s')
   url.searchParams.set('aggregation.perSeriesAligner', 'ALIGN_DELTA')
   url.searchParams.set('aggregation.crossSeriesReducer', 'REDUCE_SUM')
+  if (groupByFields) url.searchParams.set('aggregation.groupByFields', groupByFields)
   url.searchParams.set('pageSize', '1000')
-  const read = await retryingRead(reader, async (timeoutMs) => {
+  return await retryingRead(reader, async (timeoutMs) => {
     const { stdout } = await reader.runGcloud(['auth', 'print-access-token'], { timeoutMs })
     const response = await reader.fetch(url, {
       headers: { authorization: `Bearer ${stdout.trim()}` },
@@ -179,17 +180,44 @@ async function readDirector503PerMinute(reader, { config, span }) {
     if (body.nextPageToken) throw new Error('Cloud Monitoring pagination is incomplete')
     return { timeSeries: body.timeSeries ?? [] }
   })
-  const perMinute = {}
-  for (const series of read.timeSeries ?? []) {
-    for (const point of series.points ?? []) {
-      // A delta point covers the minute that ends at its end time; a minute with no point had none.
-      const minute = minuteKey(Date.parse(point.interval.endTime) - 60_000)
-      perMinute[minute] = (perMinute[minute] ?? 0) + Number(point.value.int64Value ?? 0)
-    }
+}
+
+function addPointsPerMinute(perMinute, series) {
+  for (const point of series.points ?? []) {
+    // A delta point covers the minute that ends at its end time; a minute with no point had none.
+    const minute = minuteKey(Date.parse(point.interval.endTime) - 60_000)
+    perMinute[minute] = (perMinute[minute] ?? 0) + Number(point.value.int64Value ?? 0)
   }
+  return perMinute
+}
+
+export async function readDirector503PerMinute(reader, { config, span }) {
+  const read = await readDirectorRequestCountSeries(reader, {
+    config,
+    span,
+    filters: ['metric.label."response_code"="503"']
+  })
+  const perMinute = {}
+  for (const series of read.timeSeries ?? []) addPointsPerMinute(perMinute, series)
   return { perMinute, failed: read.failed }
 }
 
+// Every response code in one read, so a minute missing from the 503 series can be told apart from
+// one not published yet: the other codes' series carry the same publishing front.
+export async function readDirectorRequestsByCodePerMinute(reader, { config, span }) {
+  const read = await readDirectorRequestCountSeries(reader, {
+    config,
+    span,
+    groupByFields: 'metric.label."response_code"'
+  })
+  const byCode = {}
+  for (const series of read.timeSeries ?? []) {
+    const code = series.metric?.labels?.response_code ?? 'unknown'
+    byCode[code] = addPointsPerMinute(byCode[code] ?? {}, series)
+  }
+  return { byCode, failed: read.failed }
+}
+
 // Cells log through the COS container agent, so the text lives in jsonPayload.message; a
 // textPayload filter matches nothing here and returns zero without saying so.
 const CELL_LOG_SCOPE = 'resource.type="gce_instance" AND logName:"cos_containers"'
@@ -419,8 +447,8 @@ export async function evaluateShadowGate(config, {
       appliedPaceWindowMs: config.drainAppliedPaceWindowMs,
       targetHosts: config.targetHosts,
       settledAt: config.drainSettledAt || null,
-      // Isolate to restart-safe, which includes the quiet the restart wait holds after the cell
-      // empties: (ceil(pace / 5 s) + 1) samples of 5 s.
+      // Isolate to restart-safe: the later of the pace window's end and the cell emptying, plus
+      // one 5 s sample.
       settledAfterSeconds: config.drainStartedAt && config.drainSettledAt
         ? Math.round((Date.parse(config.drainSettledAt) - Date.parse(config.drainStartedAt)) / 1000)
         : null

cloud/dev/scripts/relay-staging-c4-refresh-workflow.test.mjs

@@ -42,9 +42,9 @@ const asiaCells = ['production-gce-c27', 'production-gce-c28', 'production-gce-c
 const c30Digest = '4158d8a2e18e9caec439d257f0c1e45d92ffea8c0262f057b2f08c76a134bcf0'
 // C31 launches on the digest the director served when it was declared.
 const c31Digest = 'f30b5cb1ec52b6b6145efecfa1b8be9e3d309403beffd8abcc64197a2087e269'
-// Staging C4 is declared on image F (main 3c1141f2, the six-cell dead-cell rerun); the staging
+// Staging C4 is declared on image G (main 2de73812, E-pre with the t_preauth metric); the staging
 // director deploy binds its digest to this line.
-const stagingC4Digest = '41cba995b288cab70579882ebef23e74c5ee426b6f8924cea7c986aba23bcbfe'
+const stagingC4Digest = '07b5bf67a7706b2a61ef5e9a6d8ce85ee37c2a08609496a94eb5254881b5b87d'
 
 function cellBlock(tfvars, cellId) {
   const start = tfvars.indexOf(`"${cellId}"`)

cloud/dev/scripts/run-relay-tests.mjs

@@ -0,0 +1,126 @@
+import { runProcess } from '@orca/process-host'
+import {
+  forceTerminateProcessTree,
+  signalProcessTree
+} from '@orca/process-host/process-tree-termination'
+import { mkdtempSync, mkdirSync, realpathSync, rmSync } from 'node:fs'
+import { createRequire } from 'node:module'
+import { availableParallelism, tmpdir } from 'node:os'
+import { dirname, join } from 'node:path'
+import { fileURLToPath, pathToFileURL } from 'node:url'
+
+const require = createRequire(import.meta.url)
+const vitest = join(dirname(require.resolve('vitest/package.json')), 'vitest.mjs')
+const relayDirectory = fileURLToPath(new URL('../../apps/relay', import.meta.url))
+
+export async function runRelayTests({
+  args = [],
+  cwd = relayDirectory,
+  env = process.env,
+  parallelism = availableParallelism(),
+  cli = vitest,
+  run = runProcess,
+  events = process,
+  report = console.error,
+  scratchParent = tmpdir()
+} = {}) {
+  const workerBudget = Math.max(parallelism - 1, 1)
+  // Explicit selectors and alternate database fixtures retain their existing execution path.
+  const serial =
+    args.length > 0 || workerBudget <= 1 || env.ORCA_IDLE_REHOME_POSTGRES_URL || env.DATABASE_URL
+  const roles = serial
+    ? [{ name: 'all', args }]
+    : [
+        { name: 'relay', args: ['--project=relay', `--maxWorkers=${workerBudget - 1}`] },
+        { name: 'relay-postgres', args: ['--project=relay-postgres', '--maxWorkers=1'] }
+      ]
+  const scratch = realpathSync(mkdtempSync(join(scratchParent, 'orca-relay-tests-')))
+  const abort = new AbortController()
+  const closed = new Set()
+  const terminatedTrees = new Set()
+  let signalExitCode
+  const onInterrupt = () => {
+    signalExitCode ??= 130
+    abort.abort()
+  }
+  const onTerminate = () => {
+    signalExitCode ??= 143
+    abort.abort()
+  }
+  events.on('SIGINT', onInterrupt)
+  events.on('SIGTERM', onTerminate)
+
+  async function runRole(role) {
+    const owned = join(scratch, role.name)
+    try {
+      mkdirSync(owned)
+      return await run({
+        program: process.execPath,
+        args: [cli, 'run', ...role.args],
+        cwd,
+        env: {
+          ...env,
+          TMPDIR: owned,
+          TMP: owned,
+          TEMP: owned,
+          ORCA_BACKGROUND_LAUNCH: '1',
+          ORCA_RELAY_TEST_CACHE_DIR: join(owned, 'cache')
+        },
+        stdio: 'inherit',
+        timeoutMs: null,
+        signal: abort.signal,
+        terminationBarrier: {
+          async signal(child, signal) {
+            const terminated = await signalProcessTree(child, signal)
+            if (process.platform === 'win32' && terminated) {
+              terminatedTrees.add(role.name)
+            }
+            return terminated
+          },
+          async force(child) {
+            const terminated = await forceTerminateProcessTree(child)
+            if (terminated) {
+              terminatedTrees.add(role.name)
+            }
+            return terminated
+          }
+        },
+        onChildTerminated: () => closed.add(role.name)
+      })
+    } catch (error) {
+      abort.abort()
+      throw error
+    }
+  }
+
+  let exitCode = 0
+  try {
+    const results = await Promise.allSettled(roles.map(runRole))
+    for (const result of results) {
+      if (result.status === 'rejected') {
+        report(result.reason)
+        exitCode ||= 1
+      } else if (result.value.code !== 0 || result.value.signal || result.value.timedOut) {
+        exitCode ||= result.value.code || 1
+      }
+    }
+  } finally {
+    events.removeListener('SIGINT', onInterrupt)
+    events.removeListener('SIGTERM', onTerminate)
+    // A cancelled root's close event alone does not prove its descendants stopped using scratch.
+    if (
+      closed.size === roles.length &&
+      (!abort.signal.aborted || terminatedTrees.size === roles.length)
+    ) {
+      rmSync(scratch, { recursive: true, force: true })
+    } else {
+      report(`Relay test process closure is unverifiable; retained ${scratch}`)
+      exitCode ||= 1
+    }
+  }
+  return signalExitCode ?? exitCode
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  process.exitCode = await runRelayTests({ args: process.argv.slice(2) })
+}

cloud/dev/scripts/run-relay-tests.test.mjs

@@ -0,0 +1,274 @@
+import assert from 'node:assert/strict'
+import { EventEmitter } from 'node:events'
+import {
+  cpSync,
+  existsSync,
+  mkdirSync,
+  mkdtempSync,
+  readFileSync,
+  readdirSync,
+  realpathSync,
+  rmSync,
+  symlinkSync,
+  writeFileSync
+} from 'node:fs'
+import { createRequire } from 'node:module'
+import { tmpdir } from 'node:os'
+import { join, resolve } from 'node:path'
+import { setImmediate } from 'node:timers/promises'
+import { pathToFileURL } from 'node:url'
+import test from 'node:test'
+import { runProcess } from '@orca/process-host'
+import { runRelayTests } from './run-relay-tests.mjs'
+
+const success = { code: 0, signal: null, timedOut: false }
+const databaseUrl = 'postgres://relay_test:relay_test@127.0.0.1:5432/orca_relay_test'
+const require = createRequire(import.meta.url)
+const cloud = resolve(import.meta.dirname, '../..')
+
+function temporary(t) {
+  const directory = realpathSync(mkdtempSync(join(tmpdir(), 'orca-relay-runner-contract-')))
+  t.after(() => rmSync(directory, { recursive: true, force: true }))
+  return directory
+}
+
+function start(t, options = {}) {
+  const scratchParent = temporary(t)
+  const calls = []
+  const reports = []
+  const events = new EventEmitter()
+  const env = { ORCA_RELAY_TEST_POSTGRES_URL: databaseUrl, PRESERVED: 'yes', ...options.env }
+  const pending = runRelayTests({
+    cwd: join(scratchParent, 'relay'),
+    cli: join(scratchParent, 'vitest.mjs'),
+    parallelism: 4,
+    ...options,
+    env,
+    events,
+    scratchParent,
+    report: (message) => reports.push(message),
+    run: (spec) => {
+      const deferred = Promise.withResolvers()
+      calls.push({ spec, ...deferred })
+      return deferred.promise
+    }
+  })
+  return { pending, calls, reports, events, scratchParent, env }
+}
+
+function finish(call, result = success, closed = true) {
+  if (closed) {
+    call.spec.onChildTerminated()
+  }
+  call.resolve(result)
+}
+
+function assertClean(harness) {
+  assert.deepEqual(readdirSync(harness.scratchParent), [])
+  assert.equal(harness.events.listenerCount('SIGINT'), 0)
+  assert.equal(harness.events.listenerCount('SIGTERM'), 0)
+}
+
+test('overlaps both projects within the worker budget and isolates their scratch and caches', async (t) => {
+  const h = start(t)
+  assert.equal(h.calls.length, 2)
+  const [regular, postgres] = h.calls.map((call) => call.spec)
+  assert.deepEqual(regular.args.slice(1), ['run', '--project=relay', '--maxWorkers=2'])
+  assert.deepEqual(postgres.args.slice(1), ['run', '--project=relay-postgres', '--maxWorkers=1'])
+  assert.notEqual(regular.env.TMPDIR, postgres.env.TMPDIR)
+  assert.notEqual(regular.env.ORCA_RELAY_TEST_CACHE_DIR, postgres.env.ORCA_RELAY_TEST_CACHE_DIR)
+  for (const spec of [regular, postgres]) {
+    assert.equal(spec.env.TMP, spec.env.TMPDIR)
+    assert.equal(spec.env.TEMP, spec.env.TMPDIR)
+    assert.equal(spec.env.ORCA_RELAY_TEST_CACHE_DIR, join(spec.env.TMPDIR, 'cache'))
+    assert.equal(spec.env.ORCA_RELAY_TEST_POSTGRES_URL, databaseUrl)
+    assert.equal(spec.env.PRESERVED, 'yes')
+    assert.equal(spec.env.ORCA_BACKGROUND_LAUNCH, '1')
+    assert.ok(existsSync(spec.env.TMPDIR))
+    assert.equal(spec.timeoutMs, null)
+    assert.equal(typeof spec.terminationBarrier.force, 'function')
+  }
+  assert.equal(regular.signal, postgres.signal)
+  let settled = false
+  h.pending.then(() => {
+    settled = true
+  })
+  finish(h.calls[0], { ...success, code: 7 })
+  await setImmediate()
+  assert.equal(settled, false)
+  assert.equal(postgres.signal.aborted, false)
+  assert.ok(existsSync(postgres.env.TMPDIR))
+  finish(h.calls[1])
+  assert.equal(await h.pending, 7)
+  assertClean(h)
+})
+
+for (const [name, options] of [
+  ['selectors', { args: ['src/assignment.test.ts', '-t', 'requested behavior'] }],
+  [
+    'alternate PostgreSQL',
+    { env: { ORCA_IDLE_REHOME_POSTGRES_URL: 'postgres://localhost:55440/relay' } }
+  ],
+  ['application database', { env: { DATABASE_URL: databaseUrl } }],
+  ['one CPU', { parallelism: 1 }],
+  ['two CPUs', { parallelism: 2 }]
+]) {
+  test(`retains the original single run for ${name}`, async (t) => {
+    const h = start(t, options)
+    assert.equal(h.calls.length, 1)
+    const { spec } = h.calls[0]
+    assert.deepEqual(spec.args.slice(1), ['run', ...(options.args ?? [])])
+    for (const [key, value] of Object.entries(h.env)) {
+      assert.equal(spec.env[key], value)
+    }
+    finish(h.calls[0])
+    assert.equal(await h.pending, 0)
+    assertClean(h)
+  })
+}
+
+for (const [name, result] of [
+  ['null exit code', { ...success, code: null }],
+  ['termination signal', { ...success, signal: 'SIGTERM' }],
+  ['timeout', { ...success, timedOut: true }]
+]) {
+  test(`fails a ${name} even when other result fields indicate success`, async (t) => {
+    const h = start(t)
+    finish(h.calls[0], result)
+    finish(h.calls[1])
+    assert.equal(await h.pending, 1)
+    assertClean(h)
+  })
+}
+
+test('fails and retains scratch when a child never reports closure', async (t) => {
+  const h = start(t)
+  finish(h.calls[0], success, false)
+  finish(h.calls[1])
+  assert.equal(await h.pending, 1)
+  assert.ok(existsSync(h.calls[0].spec.env.TMPDIR))
+  assert.match(String(h.reports[0]), /closure is unverifiable; retained/)
+  assert.equal(h.events.listenerCount('SIGINT'), 0)
+  assert.equal(h.events.listenerCount('SIGTERM'), 0)
+})
+
+for (const [signal, exitCode, laterSignal] of [
+  ['SIGINT', 130, 'SIGTERM'],
+  ['SIGTERM', 143, 'SIGINT']
+]) {
+  test(`${signal} cancels every child and retains scratch despite root close callbacks`, async (t) => {
+    const h = start(t)
+    h.events.emit(signal)
+    h.events.emit(laterSignal)
+    for (const call of h.calls) {
+      assert.equal(call.spec.signal.aborted, true)
+      finish(call)
+    }
+    assert.equal(await h.pending, exitCode)
+    for (const call of h.calls) {
+      assert.ok(existsSync(call.spec.env.TMPDIR))
+    }
+    assert.match(String(h.reports[0]), /closure is unverifiable; retained/)
+    assert.equal(h.events.listenerCount('SIGINT'), 0)
+    assert.equal(h.events.listenerCount('SIGTERM'), 0)
+  })
+}
+
+for (const verified of [1, 2]) {
+  test(`requires verification of both cancelled trees (${verified} verified)`, async (t) => {
+    const h = start(t)
+    h.events.emit('SIGINT')
+    for (const call of h.calls.slice(0, verified)) {
+      // An unspawned handle proves absence without signalling a real process.
+      assert.equal(await call.spec.terminationBarrier.force({ kill: () => true }), true)
+    }
+    for (const call of h.calls) {
+      finish(call)
+    }
+    assert.equal(await h.pending, 130)
+    if (verified === 2) {
+      assertClean(h)
+    } else {
+      assert.ok(existsSync(h.calls[0].spec.env.TMPDIR))
+    }
+  })
+}
+
+test('reports a spawn failure and waits for its cancelled peer before returning', async (t) => {
+  const h = start(t)
+  const failure = new Error('fixture spawn failed')
+  h.calls[0].reject(failure)
+  let settled = false
+  h.pending.then(() => {
+    settled = true
+  })
+  await setImmediate()
+  assert.equal(h.calls[1].spec.signal.aborted, true)
+  assert.equal(settled, false)
+  finish(h.calls[1])
+  assert.equal(await h.pending, 1)
+  assert.equal(h.reports[0], failure)
+  assert.ok(existsSync(h.calls[1].spec.env.TMPDIR))
+})
+
+test('launches with only the Cloud dependency link and no root install or compiled host package', async (t) => {
+  const root = temporary(t)
+  const standalone = join(root, 'cloud')
+  const manifest = JSON.parse(readFileSync(join(cloud, 'package.json'), 'utf8'))
+  const dependency = manifest.devDependencies['@orca/process-host']
+  assert.equal(dependency, 'link:../src/packages/process-host')
+  const packageDir = resolve(standalone, dependency.slice('link:'.length))
+  mkdirSync(packageDir, { recursive: true })
+  for (const file of ['package.json', 'src']) {
+    cpSync(join(cloud, '..', 'src', 'packages', 'process-host', file), join(packageDir, file), {
+      recursive: true
+    })
+  }
+  const scripts = join(standalone, 'dev', 'scripts')
+  mkdirSync(scripts, { recursive: true })
+  cpSync(join(cloud, 'dev', 'scripts', 'run-relay-tests.mjs'), join(scripts, 'run-relay-tests.mjs'))
+  writeFileSync(join(standalone, 'package.json'), JSON.stringify(manifest))
+  mkdirSync(join(standalone, 'apps', 'relay'), { recursive: true })
+  mkdirSync(join(standalone, 'node_modules', '@orca'), { recursive: true })
+  symlinkSync(
+    packageDir,
+    join(standalone, 'node_modules', '@orca', 'process-host'),
+    process.platform === 'win32' ? 'junction' : 'dir'
+  )
+  const fakeVitest = join(standalone, 'node_modules', 'vitest')
+  mkdirSync(fakeVitest)
+  writeFileSync(join(fakeVitest, 'package.json'), '{"name":"vitest","version":"0.0.0"}')
+  writeFileSync(
+    join(fakeVitest, 'vitest.mjs'),
+    'console.log(JSON.stringify({args:process.argv.slice(2), background:process.env.ORCA_BACKGROUND_LAUNCH, database:process.env.ORCA_RELAY_TEST_POSTGRES_URL}))'
+  )
+  assert.equal(existsSync(join(root, 'node_modules')), false)
+  assert.equal(existsSync(join(packageDir, 'dist')), false)
+  const result = await runProcess({
+    program: process.execPath,
+    args: [
+      '--import',
+      pathToFileURL(require.resolve('tsx')).href,
+      '--conditions=orca-source',
+      join(scripts, 'run-relay-tests.mjs'),
+      'src/requested.test.ts'
+    ],
+    cwd: standalone,
+    timeoutMs: 15_000,
+    terminationBarrier: true,
+    env: {
+      ...process.env,
+      NODE_PATH: '',
+      NODE_OPTIONS: '',
+      ORCA_BACKGROUND_LAUNCH: '1',
+      ORCA_RELAY_TEST_POSTGRES_URL: databaseUrl
+    }
+  })
+  assert.equal(result.code, 0, result.stderr)
+  assert.deepEqual(JSON.parse(result.stdout), {
+    args: ['run', 'src/requested.test.ts'],
+    background: '1',
+    database: databaseUrl
+  })
+  assert.equal(existsSync(join(packageDir, 'dist')), false)
+})

cloud/dev/scripts/verify-relay-capacity-transition.mjs

@@ -420,14 +420,17 @@ export async function verifyCapacityTransition(config, overrides = {}) {
   if (health.ok !== true || health.connectionCapacityProtocol !== CAPACITY_PROTOCOL) {
     throw new Error('director is not capacity-protocol compatible')
   }
-  const deadline = now() + config.timeoutMs
-  // A paced drain can refill the runtime until its window ends, so the quiet must outlast it.
-  // Each poll waits a full interval, so consecutive samples bound the quiet time from below.
-  const requiredRestartSafeSamples =
-    Math.max(Math.ceil((config.paceWindowMs ?? 0) / POLL_INTERVAL_MS) + 1, 2)
+  const startedAt = now()
+  const deadline = startedAt + config.timeoutMs
+  // A paced drain can refill the runtime until its window ends, so the second of two consecutive
+  // quiet samples must be taken at or after that end. This runs after the cell accepted the drain,
+  // so the cell's window ends no later than this one.
+  const paceWindowEndsAt = startedAt + (config.paceWindowMs ?? 0)
   let restartSafeSamples = 0
   let lastObservation = { runtimeAvailable: false }
   for (;;) {
+    // Before the read, so a sample taken inside the window cannot pass.
+    const sampledAt = now()
     const runtime = await cellRuntime(fetchImpl, config, token)
     lastObservation = { runtimeAvailable: runtime !== null }
     if ((runtime === null) === (config.runtime === 'unavailable')) {
@@ -469,7 +472,7 @@ export async function verifyCapacityTransition(config, overrides = {}) {
       if (
         matches &&
         (config.activity !== 'restart-safe' ||
-          restartSafeSamples + 1 >= requiredRestartSafeSamples)
+          (restartSafeSamples >= 1 && sampledAt >= paceWindowEndsAt))
       ) {
         return {
           cellId: status.cellId,
@@ -491,15 +494,16 @@ export async function verifyCapacityTransition(config, overrides = {}) {
       restartSafeSamples = 0
     }
     if (config.activity === 'restart-safe') {
+      const paceWindowRemainingMs = Math.max(paceWindowEndsAt - sampledAt, 0)
       lastObservation = {
         ...lastObservation,
         restartSafeSamples,
-        requiredRestartSafeSamples
+        paceWindowRemainingMs
       }
       progress({
         event: 'relay_capacity_transition_restart_progress',
         restartSafeSamples,
-        requiredRestartSafeSamples,
+        paceWindowRemainingMs,
         totalConnections: lastObservation.runtime?.totalConnections ?? null,
         preAuthConnections: lastObservation.runtime?.preAuthConnections ?? null,
         runtimeAvailable: lastObservation.runtimeAvailable,

cloud/dev/scripts/verify-relay-capacity-transition.test.mjs

@@ -856,8 +856,7 @@ test('restart timeout reports one of two safe settling samples', async () => {
       { fetch: base, token: 'masked-token' }
     ),
     (error) => {
-      assert.match(error.message, /"restartSafeSamples":1/)
-      assert.match(error.message, /"requiredRestartSafeSamples":2/)
+      assert.match(error.message, /"restartSafeSamples":1,"paceWindowRemainingMs":0/)
       return true
     }
   )
@@ -1177,8 +1176,8 @@ const pacedRestartConfig = {
   timeoutMs: 1_200_000
 }
 
-function fakeClock() {
-  let now = 0
+function fakeClock(start = 0) {
+  let now = start
   return {
     now: () => now,
     wait: async (milliseconds) => {
@@ -1231,7 +1230,8 @@ test('passes after one pace window of an empty runtime and reports stranded leas
     },
     token: 'masked-token',
     progress: (line) => lines.push(line),
-    ...fakeClock()
+    // A late start catches a window measured from zero instead of from the verifier's start.
+    ...fakeClock(1_000_000)
   })
   // One sample at each end of a full five-minute pace window.
   assert.equal(runtimeReads, 61)
@@ -1246,12 +1246,15 @@ test('passes after one pace window of an empty runtime and reports stranded leas
   assert.equal(lines.length, 60)
   assert.ok(lines.every((line) =>
     line.event === 'relay_capacity_transition_restart_progress' &&
-    line.requiredRestartSafeSamples === 61 &&
     line.stranded.restartBlockingActivityLeases === 4))
   assert.deepEqual(lines.slice(0, 2).map((line) => line.restartSafeSamples), [1, 2])
+  assert.deepEqual(
+    lines.slice(0, 2).map((line) => line.paceWindowRemainingMs),
+    [300_000, 295_000]
+  )
 })
 
-test('the pace window resets on any non-zero runtime sample', async () => {
+test('a non-zero runtime sample restarts the quiet count', async () => {
   const quiet = harness(strandedCell)
   const busy = harness({ ...strandedCell, controls: 1 })
   let runtimeReads = 0
@@ -1265,17 +1268,37 @@ test('the pace window resets on any non-zero runtime sample', async () => {
     progress: (line) => lines.push(line),
     ...fakeClock()
   })
-  assert.equal(runtimeReads, 91)
+  // A mid-window reset adds no time; the quiet after it still reaches the window's end.
+  assert.equal(runtimeReads, 61)
   assert.deepEqual(lines.slice(28, 31).map((line) => line.restartSafeSamples), [29, 0, 1])
   await assert.rejects(
     verifyCapacityTransition(
       { ...pacedRestartConfig, timeoutMs: 295_000 },
       { fetch: quiet, token: 'masked-token', progress: () => {}, ...fakeClock() }
     ),
-    /"restartSafeSamples":60,"requiredRestartSafeSamples":61/
+    /"restartSafeSamples":60,"paceWindowRemainingMs":5000/
   )
 })
 
+// c27 on 2026-10-13: the 20-minute paced drain emptied the runtime as its window closed, and the
+// old gate then waited a second full window of quiet samples before the restart.
+test('a runtime that empties as the pace window ends passes after two quiet samples', async () => {
+  let runtimeReads = 0
+  const lines = []
+  await verifyCapacityTransition(pacedRestartConfig, {
+    fetch: async (url, options) => {
+      if (new URL(url).pathname === '/v1/admin/runtime-status') runtimeReads += 1
+      return await harness({ ...strandedCell, controls: runtimeReads <= 60 ? 1 : 0 })(url, options)
+    },
+    token: 'masked-token',
+    progress: (line) => lines.push(line),
+    ...fakeClock()
+  })
+  // Busy through 295 s; quiet at 300 s and 305 s, the first eligible pair.
+  assert.equal(runtimeReads, 62)
+  assert.deepEqual(lines.slice(59).map((line) => line.restartSafeSamples), [0, 1])
+})
+
 test('restart-safe fails on live runtime work or an open migration regardless of leases', async (t) => {
   for (const blocker of [
     { controls: 1 },
@@ -1299,7 +1322,7 @@ test('restart-safe fails on live runtime work or an open migration regardless of
   }
 })
 
-test('unauthenticated redials do not reset the pace window', async () => {
+test('unauthenticated redials do not restart the quiet count', async () => {
   let runtimeReads = 0
   const lines = []
   const result = await verifyCapacityTransition(pacedRestartConfig, {
@@ -1439,7 +1462,7 @@ test('restart progress prints every counter the gate reads', async () => {
   assert.deepEqual(lines[0], {
     event: 'relay_capacity_transition_restart_progress',
     restartSafeSamples: 0,
-    requiredRestartSafeSamples: 61,
+    paceWindowRemainingMs: 300_000,
     totalConnections: 7,
     preAuthConnections: 1,
     runtimeAvailable: true,

cloud/dev/scripts/watch-relay-reserve-cell.mjs

@@ -0,0 +1,581 @@
+#!/usr/bin/env node
+// Watches one cell already in reserve mode and, on the first stop rule it trips, flips it back to
+// db through the flag tool's own write (CAS, read-back, re-registration wait). Step 5 design §9.
+//
+// watch:     one segment of the watch (the workflow re-authenticates between segments, since an
+//            ID token lasts an hour). State carries over in --state-file. Writes nothing.
+// outcome:   the job's state and verdict, from the state file.
+// flip-back: the flag tool's admitMode=db write (or its dry run), then exits non-zero with the
+//            watch's verdict. Why it holds no flag-workflow group: docs/relay-workflows.md.
+// alarm:     a refused watch or a flip back that did not land: an error annotation, and in write
+//            mode a log line the reserve-watch alert pages on.
+import { execFile } from 'node:child_process'
+import { appendFile, readFile, writeFile } from 'node:fs/promises'
+import { parseArgs, promisify } from 'node:util'
+import { pathToFileURL } from 'node:url'
+import {
+  BREAK_GLASS_CONFIRMATION,
+  operateCellFlags,
+  parseCellFlagsRequest,
+  readCurrentObject,
+  readRuntime,
+  WRITE_CONFIRMATION
+} from './operate-relay-cell-flags.mjs'
+import { readDirectorRequestsByCodePerMinute, readLogEntries } from './relay-same-cap-shadow-gate.mjs'
+import { minutesOf } from './relay-same-cap-shadow-gate-verdict.mjs'
+import { PRODUCTION_LEASE } from './cloud-sql-rollout-lock-census.mjs'
+import { RELAY_REPOSITORY_ROOT } from './relay-repository.mjs'
+import {
+  evaluateWatchSample,
+  initialWatchState,
+  measuredBackground,
+  RESERVE_WATCH_RULES
+} from './relay-reserve-watch-rules.mjs'
+
+const execFileAsync = promisify(execFile)
+const MINUTE_MS = 60_000
+export const SAMPLE_INTERVAL_MS = MINUTE_MS
+export const MAX_WATCH_MINUTES = 300
+// Below the hour an ID token lasts, with room for the segment's last sample.
+export const MAX_SEGMENT_MINUTES = 55
+const METRIC_DELAY_MS = RESERVE_WATCH_RULES.director503.publishDelayMs
+const BACKGROUND_MINUTES = 10
+// A flipped-at older than this is a typo or a stale dispatch, not this flip.
+const MAX_FLIPPED_AT_AGE_MS = 24 * 60 * MINUTE_MS
+export const FLIP_BACK_ATTEMPTS = 5
+const FLIP_BACK_BACKOFF_MS = 2_000
+// One slow runtime-status reply is not a dead cell: retried this often before break glass.
+export const FLIP_BACK_RUNTIME_ATTEMPTS = 4
+const SAMPLE_READ_BUDGET_MS = 40_000
+const LOG_ENTRY_LIMIT = 1_000
+
+function integerIn(name, raw, min, max) {
+  const value = Number(raw)
+  if (!/^[0-9]+$/.test(raw ?? '') || value < min || value > max) {
+    throw new Error(`--${name} must be an integer from ${min} to ${max}`)
+  }
+  return value
+}
+
+// The flip back's request. Its generation is a placeholder: the flip back re-reads the current one.
+export function dbFlipRequest(values) {
+  return parseCellFlagsRequest({ ...values, set: 'admitMode=db', 'expected-generation': '0' })
+}
+
+// `now` is the watch's start: a later segment of a long watch must not age flipped-at out.
+function parseFlippedAt(raw, now) {
+  if (raw === '') return null
+  const at = Date.parse(raw)
+  if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(:\d{2}(\.\d{1,3})?)?(Z|[+-]\d{2}:\d{2})$/.test(raw) || !Number.isFinite(at)) {
+    throw new Error('--flipped-at must be an ISO time with a zone, e.g. 2026-10-14T12:00:00Z')
+  }
+  if (at > now) throw new Error('--flipped-at is in the future')
+  if (at < now - MAX_FLIPPED_AT_AGE_MS) throw new Error('--flipped-at is more than 24 hours ago')
+  return at
+}
+
+export function parseWatchArguments(values, now = Date.now()) {
+  // The flag tool's own checks, so the watch is of exactly the cell the flip back would write;
+  // a write-mode watch refuses a missing confirmation now, not hours later.
+  const request = dbFlipRequest(values)
+  if (!/^[a-z][a-z0-9-]{0,62}$/.test(values['director-service'] ?? '')) {
+    throw new Error('--director-service is not a Cloud Run service name')
+  }
+  const background = values['background-503-per-minute'] ?? ''
+  if (background !== '' && !/^(0|[1-9][0-9]{0,6})(\.[0-9]+)?$/.test(background)) {
+    throw new Error('--background-503-per-minute must be empty (measure it) or a non-negative number')
+  }
+  // The background must predate the flip: a watch starts after it, so its own past is not one.
+  const flippedAt = parseFlippedAt(values['flipped-at'] ?? '', now)
+  if (background === '' && flippedAt === null) {
+    throw new Error('--flipped-at (the background is the 10 minutes before it) or --background-503-per-minute is required')
+  }
+  if (!values['state-file']) throw new Error('--state-file is required')
+  return {
+    request,
+    directorService: values['director-service'],
+    durationMinutes: integerIn('duration-minutes', values['duration-minutes'], 5, MAX_WATCH_MINUTES),
+    segmentMinutes: integerIn('segment-minutes', values['segment-minutes'], 1, MAX_SEGMENT_MINUTES),
+    unverifiableLimit: integerIn('unverifiable-limit', values['unverifiable-limit'], 1, 30),
+    background503PerMinute: background === '' ? null : Number(background),
+    flippedAt,
+    stateFile: values['state-file'],
+    githubOutput: values['github-output'] || null
+  }
+}
+
+const failedRead = (error) => ({ ok: false, error: error instanceof Error ? error.message : String(error) })
+
+export async function readCellSample(fetchImpl, request, idToken, wait) {
+  try {
+    const runtime = await readRuntime(fetchImpl, request, idToken, wait)
+    if (runtime.role !== 'cell' || runtime.cellId !== request.cellId) {
+      return failedRead(`origin answers as ${runtime.role}/${runtime.cellId}`)
+    }
+    if (!runtime.flagsApplied) return failedRead('runtime-status reports no flagsApplied')
+    const controls = runtime.runtime?.controls
+    if (!Number.isInteger(controls)) return failedRead('runtime-status reports no control count')
+    return {
+      ok: true,
+      admitModeConfigured: runtime.flagsApplied.flags?.admitMode ?? 'db',
+      admitModeEffective: runtime.admitModeEffective,
+      draining: runtime.draining === true,
+      controls,
+      enforcedUnits: runtime.runtime?.enforcedConnectionUnits,
+      hardCap: runtime.connectionCapacity?.hardCap
+    }
+  } catch (error) {
+    return failedRead(error)
+  }
+}
+
+const minuteFloor = (at) => Math.floor(at / MINUTE_MS) * MINUTE_MS
+const failed503Read = (error) => ({ ok: false, error, perMinute: {}, minutes: [], publishedThrough: null })
+
+// The `minutes` whole minutes that end at the minute boundary `endedAt`: the 503 counts, and the
+// newest minute any response code has a point for. The director always serves something, so that
+// front is how far the 503 series is published; no front at all is no evidence. A code publishing
+// late only delays a trip: a later read sees its points.
+export async function readDirector503Minutes(reader, { projectId, directorService, endedAt, minutes }) {
+  const span = { startedAt: new Date(endedAt - minutes * MINUTE_MS), endedAt: new Date(endedAt) }
+  let read
+  try {
+    read = await readDirectorRequestsByCodePerMinute(reader, { config: { projectId, directorService }, span })
+  } catch (error) {
+    // A point with an unparseable end time has no minute.
+    return failed503Read(`director 503 metric unreadable: ${error instanceof Error ? error.message : error}`)
+  }
+  if (read.failed) return failed503Read('director 503 metric read failed')
+  const published = Object.values(read.byCode).flatMap((perMinute) => Object.keys(perMinute)).sort()
+  return {
+    ok: true,
+    perMinute: read.byCode['503'] ?? {},
+    minutes: minutesOf(span),
+    publishedThrough: published.at(-1) ?? null
+  }
+}
+
+const LEDGER_SUMMARY_EVENT = 'orca_relay_reserve_ledger_summary'
+const SEAT_BEHIND_ROW_EVENT = 'orca_relay_reserve_ledger_seat_behind_row'
+
+function directorLogFilter(directorService, eventClause, sinceAt) {
+  return [
+    'resource.type="cloud_run_revision"',
+    `resource.labels.service_name="${directorService}"`,
+    eventClause,
+    `timestamp>="${new Date(sinceAt).toISOString()}"`
+  ].join(' AND ')
+}
+
+// Newest summary per director instance; queued plus dropped rows are the ones not in Postgres.
+export async function readLedgerSample(reader, { directorService, at }) {
+  const read = await readLogEntries(reader, {
+    filter: directorLogFilter(directorService, `jsonPayload.event="${LEDGER_SUMMARY_EVENT}"`, at - RESERVE_WATCH_RULES.ledgerSummaryMaxAgeMs),
+    projection: 'json(timestamp,labels.instanceId,jsonPayload.pending,jsonPayload.dropped)',
+    limit: LOG_ENTRY_LIMIT
+  })
+  if (read.failed) return failedRead(read.error)
+  if (read.entries.length >= LOG_ENTRY_LIMIT) return failedRead('ledger summary read truncated')
+  if (read.entries.length === 0) return failedRead('no placing director logged a ledger summary')
+  if (read.entries.some((entry) => !Number.isFinite(Date.parse(entry.timestamp)))) {
+    return failedRead('ledger summary with an unparseable timestamp')
+  }
+  const newest = new Map()
+  for (const entry of read.entries) {
+    const instance = entry.labels?.instanceId ?? 'unknown'
+    const previous = newest.get(instance)
+    if (!previous || Date.parse(entry.timestamp) > Date.parse(previous.timestamp)) newest.set(instance, entry)
+  }
+  let unwrittenRows = 0
+  for (const entry of newest.values()) {
+    const pending = Number(entry.jsonPayload?.pending)
+    const dropped = Number(entry.jsonPayload?.dropped)
+    if (!Number.isFinite(pending) || !Number.isFinite(dropped)) return failedRead('ledger summary without counts')
+    unwrittenRows += pending + dropped
+  }
+  return { ok: true, unwrittenRows, directors: newest.size }
+}
+
+// The same read also takes the ledger summary every placing director logs each minute, as a
+// positive control: with no fresh summary in it, no seat-behind-row line is blindness, not zero.
+export async function readSeatBehindRowSample(reader, { directorService, cellId, at }) {
+  const read = await readLogEntries(reader, {
+    filter: directorLogFilter(
+      directorService,
+      `((jsonPayload.event="${SEAT_BEHIND_ROW_EVENT}" AND jsonPayload.cellId="${cellId}") OR jsonPayload.event="${LEDGER_SUMMARY_EVENT}")`,
+      at - RESERVE_WATCH_RULES.duplicateSeat.lookbackMs
+    ),
+    projection: 'json(timestamp,labels.instanceId,jsonPayload.event,jsonPayload.epoch,jsonPayload.rowEpoch)',
+    limit: LOG_ENTRY_LIMIT
+  })
+  if (read.failed) return failedRead(read.error)
+  if (read.entries.length >= LOG_ENTRY_LIMIT) return failedRead('seat-behind-row read truncated')
+  const reports = []
+  let controlSeen = false
+  for (const entry of read.entries) {
+    const reportedAt = Date.parse(entry.timestamp)
+    if (!Number.isFinite(reportedAt)) return failedRead('seat-behind-row line with an unparseable timestamp')
+    if (entry.jsonPayload?.event === LEDGER_SUMMARY_EVENT) {
+      controlSeen ||= reportedAt >= at - RESERVE_WATCH_RULES.ledgerSummaryMaxAgeMs
+      continue
+    }
+    const { epoch, rowEpoch } = entry.jsonPayload ?? {}
+    if (epoch === undefined || rowEpoch === undefined) return failedRead('seat-behind-row line without epochs')
+    // Everything the line carries but its time, which differs on every report of one seat. A
+    // proxy for the host: two hosts sharing it read as one seat, so it can only over-count.
+    reports.push({ at: reportedAt, key: `${entry.labels?.instanceId ?? 'unknown'}/${cellId}/${epoch}:${rowEpoch}` })
+  }
+  if (!controlSeen) return failedRead('seat-behind-row read holds no fresh ledger summary (positive control)')
+  return { ok: true, reports }
+}
+
+function createReader(dependencies, projectId) {
+  const now = dependencies.now
+  return {
+    runGcloud: dependencies.runGcloud,
+    fetch: dependencies.fetchImpl,
+    retryDelayMs: dependencies.retryDelayMs ?? 5_000,
+    readTimeoutMs: 20_000,
+    now,
+    // Every sample gets its own budget, so a slow read costs one sample, not the watch.
+    deadlineAt: now() + SAMPLE_READ_BUDGET_MS,
+    projectId
+  }
+}
+
+/**
+ * Same-cap and capacity rolls hold the Cloud SQL rollout lease for the whole run; it names no
+ * cell, so any live holder refuses the watch's start. Read only: holding it for hours would block
+ * every rollout. An unreadable lease refuses too (fail closed, as the lease action does).
+ */
+export async function rolloutLeaseRefusal(readRolloutLease, now) {
+  try {
+    const existing = await readRolloutLease()
+    if (!existing || existing.record.expires_at <= now) return null
+    return (
+      `a rollout holds the Cloud SQL rollout lease until ${new Date(existing.record.expires_at).toISOString()} ` +
+      `(${existing.holder}); start the watch after it ends`
+    )
+  } catch (error) {
+    return `the Cloud SQL rollout lease is unreadable (${error instanceof Error ? error.message : error}); refusing to start`
+  }
+}
+
+// The lease action's own reader, resolved from the repository root because this tree's depth
+// differs between the two repositories' copies.
+async function readProductionRolloutLease() {
+  const action = new URL('.github/actions/cloud-sql-rollout-lease/', RELAY_REPOSITORY_ROOT)
+  const [{ CloudSqlRolloutLease, describeHolder }, { createAccessTokenSource }] = await Promise.all([
+    import(new URL('storage-lease.mjs', action).href),
+    import(new URL('gcloud-access-token.mjs', action).href)
+  ])
+  const lease = new CloudSqlRolloutLease({
+    bucket: PRODUCTION_LEASE.bucket,
+    objectName: PRODUCTION_LEASE.object,
+    accessToken: createAccessTokenSource()
+  })
+  const existing = await lease.read()
+  return existing && { record: existing.record, holder: describeHolder(existing.record) }
+}
+
+// Only a missing file is a fresh watch. Anything else that is not a state fails the segment, which
+// the outcome step reads as failed and flips back: a corrupt file must not restart or refuse the watch.
+export async function loadState(path) {
+  const text = await readFile(path, 'utf8').catch((error) => (error?.code === 'ENOENT' ? null : Promise.reject(error)))
+  if (text === null) return null
+  const state = JSON.parse(text)
+  const isObject = state !== null && typeof state === 'object' && !Array.isArray(state)
+  if (!isObject || typeof state.state !== 'string') throw new Error(`state file ${path} holds no watch state`)
+  if (state.state === 'watching' && (typeof state.cellId !== 'string' || !Number.isFinite(state.startedAt))) {
+    throw new Error(`state file ${path} has no cell or start`)
+  }
+  return state
+}
+
+// A segment's config. Later segments check flipped-at against the watch's start; bad inputs before
+// the first sample are refused, so no flip back follows a typo.
+export async function parseWatchSegment(values, now = Date.now()) {
+  const existing = values['state-file'] ? await loadState(values['state-file']) : null
+  try {
+    return parseWatchArguments(values, Number.isFinite(existing?.startedAt) ? existing.startedAt : now)
+  } catch (error) {
+    if (values['state-file'] && existing === null) {
+      await writeFile(values['state-file'], JSON.stringify({ state: 'refused', verdict: error.message }))
+    }
+    throw error
+  }
+}
+
+export async function watchSegment(config, dependencies) {
+  const now = dependencies.now ?? Date.now
+  const sleep = dependencies.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)))
+  const log = dependencies.log ?? console.log
+  const deps = { ...dependencies, now, sleep, fetchImpl: dependencies.fetchImpl ?? fetch }
+  const { request, directorService } = config
+  let state = await loadState(config.stateFile)
+  if (state === null) {
+    const refusal = await rolloutLeaseRefusal(dependencies.readRolloutLease, now())
+    if (refusal) {
+      state = { state: 'refused', verdict: refusal }
+      await writeFile(config.stateFile, JSON.stringify(state))
+      if (config.githubOutput) await writeOutcome(config.githubOutput, state)
+      return state
+    }
+    state = initialWatchState({
+      cellId: request.cellId,
+      startedAt: now(),
+      durationMinutes: config.durationMinutes,
+      background503PerMinute: config.background503PerMinute,
+      flippedAt: config.flippedAt
+    })
+  }
+  if (state.cellId !== request.cellId) throw new Error(`state file is for ${state.cellId}, not ${request.cellId}`)
+  const segmentEndsAt = now() + config.segmentMinutes * MINUTE_MS
+  while (state.state === 'watching' && now() < segmentEndsAt) {
+    const at = now()
+    const baselineEndsAt = state.flippedAt === null ? null : minuteFloor(state.flippedAt)
+    if (!state.background && baselineEndsAt !== null && at >= baselineEndsAt + METRIC_DELAY_MS) {
+      // The minutes just before the flip, once published; retried every sample until one reads.
+      const read = await readDirector503Minutes(createReader(deps, request.projectId), {
+        projectId: request.projectId,
+        directorService,
+        endedAt: baselineEndsAt,
+        minutes: BACKGROUND_MINUTES
+      })
+      state = { ...state, background: read.ok ? measuredBackground(read.perMinute, read.minutes, read.publishedThrough) : null }
+    }
+    const reader = createReader(deps, request.projectId)
+    // Serialised: concurrent Logging reads are what earn a 429.
+    const sample = {
+      at,
+      cell: await readCellSample(deps.fetchImpl, request, dependencies.idToken, sleep),
+      director503: await readDirector503Minutes(reader, {
+        projectId: request.projectId,
+        directorService,
+        // Complete minutes only, held back past the publishing delay.
+        endedAt: minuteFloor(at - METRIC_DELAY_MS),
+        minutes: RESERVE_WATCH_RULES.director503.readMinutes
+      }),
+      ledger: await readLedgerSample(reader, { directorService, at }),
+      seatBehindRow: await readSeatBehindRowSample(reader, { directorService, cellId: request.cellId, at })
+    }
+    const result = evaluateWatchSample(state, sample, { unverifiableLimit: config.unverifiableLimit })
+    state = result.state
+    log(result.line)
+    await writeFile(config.stateFile, JSON.stringify(state))
+    if (state.state !== 'watching') break
+    await sleep(Math.max(0, at + SAMPLE_INTERVAL_MS - now()))
+  }
+  if (config.githubOutput) await writeOutcome(config.githubOutput, state)
+  return state
+}
+
+function writeOutcome(githubOutput, { state, verdict }) {
+  return appendFile(githubOutput, `state=${state}\nverdict=${(verdict ?? '').replace(/[\r\n]+/g, ' ')}\n`)
+}
+
+// The job's outcome. A watch that ended mid-segment (a crash, a timeout) records failed, which the
+// workflow flips back in write mode: missing data is never healthy.
+export async function recordOutcome(stateFile, githubOutput) {
+  const state = await loadState(stateFile).catch(() => null)
+  const outcome =
+    state && state.state !== 'watching' ? state : { state: 'failed', verdict: 'the watch stopped before a verdict' }
+  await writeOutcome(githubOutput, outcome)
+  return outcome
+}
+
+function breakGlassInputs(request, generation) {
+  return {
+    mode: 'write',
+    'cell-id': request.cellId,
+    set: 'admitMode=default',
+    'break-glass': `${BREAK_GLASS_CONFIRMATION} ${request.cellId}`,
+    'expected-generation': generation,
+    confirmation: `${WRITE_CONFIRMATION} ${request.cellId}`
+  }
+}
+
+async function readRuntimeRetrying(fetchImpl, request, idToken, sleep, log) {
+  for (let attempt = 1; ; attempt += 1) {
+    try {
+      return await readRuntime(fetchImpl, request, idToken, sleep)
+    } catch (error) {
+      if (attempt >= FLIP_BACK_RUNTIME_ATTEMPTS) throw error
+      log(JSON.stringify({ event: 'orca_relay_reserve_watch_runtime_retry', cellId: request.cellId, attempt, message: error instanceof Error ? error.message : String(error) }))
+      await sleep(FLIP_BACK_BACKOFF_MS * 2 ** (attempt - 1))
+    }
+  }
+}
+
+/**
+ * The flag tool's admitMode=db write at the generation read just now. A generation conflict (some
+ * other write landed first) is re-read and retried while the switch file still says reserve; a
+ * file that already says db is done. A cell that still does not answer after the runtime retries
+ * needs break glass, which only an operator may type.
+ */
+export async function flipBack(values, dependencies) {
+  const log = dependencies.log ?? console.log
+  const fetchImpl = dependencies.fetchImpl ?? fetch
+  const sleep = dependencies.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)))
+  const base = dbFlipRequest(values)
+  const record = (result) =>
+    log(
+      JSON.stringify({
+        event: 'orca_relay_reserve_watch_flip_back',
+        mode: base.mode,
+        cellId: base.cellId,
+        written: result.written,
+        generation: result.generation ?? null,
+        ...(result.alreadyDb ? { alreadyDb: true } : {})
+      })
+    )
+  for (let attempt = 1; ; attempt += 1) {
+    const current = await readCurrentObject(fetchImpl, base, dependencies.accessToken)
+    if ((current.object?.flags?.admitMode ?? 'db') !== 'reserve') {
+      const result = { written: false, alreadyDb: true, generation: current.generation }
+      record(result)
+      return result
+    }
+    try {
+      await readRuntimeRetrying(fetchImpl, base, dependencies.idToken, sleep, log)
+    } catch (error) {
+      const inputs = breakGlassInputs(base, current.generation)
+      log(JSON.stringify({ event: 'orca_relay_reserve_watch_break_glass_required', cellId: base.cellId, inputs }))
+      throw new Error(
+        `${base.cellId} does not answer runtime-status (${error instanceof Error ? error.message : error}), so ` +
+          'the flag tool cannot flip it back. Run Operate Relay Production Cell Flags with break glass: ' +
+          Object.entries(inputs).map(([key, value]) => `${key}=${value}`).join(', ')
+      )
+    }
+    try {
+      const result = await operateCellFlags(
+        { ...base, expectedGeneration: current.generation },
+        { ...dependencies, fetchImpl, log, sleep }
+      )
+      record(result)
+      return result
+    } catch (error) {
+      if (!error?.generationConflict || attempt >= FLIP_BACK_ATTEMPTS) throw error
+      log(JSON.stringify({ event: 'orca_relay_reserve_watch_flip_back_conflict', cellId: base.cellId, attempt, message: error.message }))
+      await sleep(FLIP_BACK_BACKOFF_MS * 2 ** (attempt - 1))
+    }
+  }
+}
+
+export const ALARM_EVENT = 'orca_relay_reserve_watch_alarm'
+export const ALARM_LOG_NAME = 'orca-relay-reserve-watch'
+export const ALARM_KINDS = ['refused', 'flip-back-failed']
+
+// GitHub's workflow-command escaping, so a verdict cannot end the annotation or start another.
+export const escapeWorkflowCommand = (text) => text.replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A')
+const escapeWorkflowProperty = (text) => escapeWorkflowCommand(text).replace(/:/g, '%3A').replace(/,/g, '%2C')
+
+/**
+ * Both kinds can leave the cell in reserve with nothing watching it. Only write mode pages: a
+ * dry-run watch never flips back, so its refusal or failure changes nothing for the cell.
+ */
+export async function raiseAlarm({ mode, cellId, projectId, kind, reason }, { runGcloud, log = console.log }) {
+  if (!ALARM_KINDS.includes(kind)) throw new Error(`--kind must be one of ${ALARM_KINDS.join(', ')}`)
+  const message =
+    kind === 'refused'
+      ? `${cellId} may be in reserve mode and unwatched: the watch was refused (${reason}). Rerun it, or flip the cell to db with Operate Relay Production Cell Flags.`
+      : `${cellId} may still be in reserve mode with no watch: the flip back did not land (${reason}). Flip it to db by hand with Operate Relay Production Cell Flags (set=admitMode=db), or with the break-glass inputs logged above if the cell does not answer.`
+  log(`::error title=${escapeWorkflowProperty(`Reserve watch ${kind} for ${cellId}`)}::${escapeWorkflowCommand(message)}`)
+  if (mode !== 'write') return { paged: false }
+  const entry = JSON.stringify({ event: ALARM_EVENT, cellId, kind, reason })
+  try {
+    // The flag tool's project check: a missing --project-id must not reach gcloud as "undefined".
+    if (!/^[a-z][a-z0-9-]{4,62}$/.test(projectId ?? '')) throw new Error('--project-id is not a GCP project id')
+    await runGcloud(
+      ['logging', 'write', ALARM_LOG_NAME, entry, '--payload-type=json', '--severity=ERROR', '--project', projectId],
+      { timeoutMs: 30_000 }
+    )
+  } catch (error) {
+    log(`::error title=Reserve watch page failed::${escapeWorkflowCommand(`the alarm log line was not written, so nothing pages: ${error instanceof Error ? error.message : error}`)}`)
+    return { paged: false }
+  }
+  return { paged: true }
+}
+
+if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) {
+  const [command, ...rest] = process.argv.slice(2)
+  const { values } = parseArgs({
+    args: rest,
+    options: Object.fromEntries(
+      [
+        'mode',
+        'cell-id',
+        'cell-origin',
+        'project-id',
+        'director-origin',
+        'director-service',
+        'confirmation',
+        'duration-minutes',
+        'segment-minutes',
+        'unverifiable-limit',
+        'background-503-per-minute',
+        'flipped-at',
+        'state-file',
+        'github-output',
+        'verdict',
+        'kind'
+      ].map((name) => [name, { type: 'string' }])
+    )
+  })
+  const idToken = process.env.ORCA_RELAY_ADMIN_ID_TOKEN
+  const runGcloud = (args, { timeoutMs }) =>
+    execFileAsync('gcloud', args, { maxBuffer: 64 * 1024 * 1024, timeout: timeoutMs, killSignal: 'SIGKILL' })
+  try {
+    if (command === 'alarm') {
+      await raiseAlarm(
+        { mode: values.mode, cellId: values['cell-id'] ?? '', projectId: values['project-id'], kind: values.kind, reason: values.verdict ?? '' },
+        { runGcloud }
+      )
+      // The alarm step is red, whatever the page did.
+      process.exitCode = 1
+    } else if (command === 'outcome') {
+      if (!values['state-file'] || !values['github-output']) throw new Error('--state-file and --github-output are required')
+      console.log(JSON.stringify({ event: 'orca_relay_reserve_watch_outcome', ...(await recordOutcome(values['state-file'], values['github-output'])) }))
+    } else if (!idToken) {
+      throw new Error('ORCA_RELAY_ADMIN_ID_TOKEN is required')
+    } else if (command === 'watch') {
+      const config = await parseWatchSegment(values)
+      const state = await watchSegment(config, {
+        idToken,
+        readRolloutLease: readProductionRolloutLease,
+        runGcloud
+      })
+      // A cell that was never reserve is the operator's mistake; one flipped back by hand is not.
+      if (state.state === 'refused') {
+        console.log(JSON.stringify({ event: 'orca_relay_reserve_watch_refused', verdict: state.verdict }))
+        process.exitCode = 1
+      }
+    } else if (command === 'flip-back') {
+      const accessToken = process.env.ORCA_RELAY_CONTROL_ACCESS_TOKEN
+      if (!accessToken) throw new Error('ORCA_RELAY_CONTROL_ACCESS_TOKEN is required')
+      if (!values.verdict) throw new Error('--verdict is required')
+      await flipBack(values, {
+        accessToken,
+        idToken,
+        audit: {
+          writtenBy: `reserve-watch/${process.env.GITHUB_TRIGGERING_ACTOR ?? process.env.GITHUB_ACTOR ?? 'unknown'}`,
+          runId: process.env.GITHUB_RUN_ID ?? 'local',
+          runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? '0'
+        }
+      })
+      // A trip is a failed watch even once the cell is safely back on db.
+      console.log(JSON.stringify({ event: 'orca_relay_reserve_watch_verdict', mode: values.mode, verdict: values.verdict }))
+      process.exitCode = 1
+    } else {
+      throw new Error('command must be watch, outcome, flip-back or alarm')
+    }
+  } catch (error) {
+    console.log(
+      JSON.stringify({
+        event: 'orca_relay_reserve_watch_failed',
+        message: error instanceof Error ? error.message : String(error)
+      })
+    )
+    process.exitCode = 1
+  }
+}

cloud/dev/scripts/watch-relay-reserve-cell.test.mjs

@@ -0,0 +1,958 @@
+import assert from 'node:assert/strict'
+import { readFileSync } from 'node:fs'
+import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { test } from 'node:test'
+import {
+  evaluateWatchSample,
+  initialWatchState,
+  measuredBackground
+} from './relay-reserve-watch-rules.mjs'
+import {
+  ALARM_EVENT,
+  ALARM_LOG_NAME,
+  FLIP_BACK_ATTEMPTS,
+  FLIP_BACK_RUNTIME_ATTEMPTS,
+  flipBack,
+  parseWatchArguments,
+  parseWatchSegment,
+  raiseAlarm,
+  readDirector503Minutes,
+  readSeatBehindRowSample,
+  recordOutcome,
+  rolloutLeaseRefusal,
+  watchSegment
+} from './watch-relay-reserve-cell.mjs'
+import { readRelayWorkflow } from './relay-repository.mjs'
+
+const CELL = 'production-gce-c26'
+const START = Date.parse('2026-10-14T12:00:00Z')
+const MINUTE = 60_000
+const LIMIT = { unverifiableLimit: 3 }
+
+const healthyCell = (overrides = {}) => ({
+  ok: true,
+  admitModeConfigured: 'reserve',
+  admitModeEffective: 'reserve',
+  draining: false,
+  controls: 1_000,
+  enforcedUnits: 1_100,
+  hardCap: 2_000,
+  ...overrides
+})
+const minutesBefore = (at, count) =>
+  Array.from({ length: count }, (_, index) =>
+    new Date(Math.floor(at / MINUTE) * MINUTE - (count + 3 - index) * MINUTE).toISOString().slice(0, 16)
+  )
+const quiet503 = (at, perMinute = 10) => {
+  const minutes = minutesBefore(at, 3)
+  return { ok: true, minutes, perMinute: Object.fromEntries(minutes.map((minute) => [minute, perMinute])), publishedThrough: minutes.at(-1) }
+}
+const sampleAt = (at, overrides = {}) => ({
+  at,
+  cell: healthyCell(),
+  director503: quiet503(at),
+  ledger: { ok: true, unwrittenRows: 0, directors: 5 },
+  seatBehindRow: { ok: true, reports: [] },
+  ...overrides
+})
+const freshState = (overrides = {}) => ({
+  ...initialWatchState({ cellId: CELL, startedAt: START, durationMinutes: 60, background503PerMinute: 10 }),
+  ...overrides
+})
+
+// Runs samples one minute apart; returns every result.
+function run(samples, state = freshState()) {
+  const results = []
+  for (const sample of samples) {
+    const result = evaluateWatchSample(state, sample, LIMIT)
+    results.push(result)
+    state = result.state
+    if (state.state !== 'watching') break
+  }
+  return results
+}
+
+test('a healthy cell keeps watching and completes at the deadline', () => {
+  const results = run(Array.from({ length: 61 }, (_, minute) => sampleAt(START + minute * MINUTE)))
+  assert.equal(results.length, 61)
+  assert.ok(results.slice(0, 60).every((result) => result.state.state === 'watching'))
+  assert.equal(results.at(-1).state.state, 'completed')
+  const line = JSON.parse(results[0].line)
+  assert.equal(line.event, 'orca_relay_reserve_watch_sample')
+  assert.deepEqual(Object.keys(line.rules), ['deadMan', 'overrun', 'director503', 'duplicateSeat', 'hostDrop', 'ledgerDrift'])
+})
+
+test('503s trip only after three complete minutes over max(1.5x background, background + 20)', () => {
+  // Background 100: the line is 150. Two minutes over and one under is not a trip.
+  const state = freshState({ background: { medianPerMinute: 100, source: 'input' } })
+  const series = (counts) => {
+    const minutes = minutesBefore(START, 3)
+    return {
+      ok: true,
+      minutes,
+      perMinute: Object.fromEntries(minutes.map((minute, index) => [minute, counts[index]])),
+      publishedThrough: minutes.at(-1)
+    }
+  }
+  const twoOver = evaluateWatchSample(state, sampleAt(START, { director503: series([151, 140, 200]) }), LIMIT)
+  assert.equal(twoOver.state.state, 'watching')
+  const atLine = evaluateWatchSample(state, sampleAt(START, { director503: series([150, 150, 150]) }), LIMIT)
+  assert.equal(atLine.state.state, 'watching')
+  const threeOver = evaluateWatchSample(state, sampleAt(START, { director503: series([151, 160, 170]) }), LIMIT)
+  assert.equal(threeOver.state.state, 'tripped')
+  assert.match(threeOver.state.verdict, /^flip back production-gce-c26: director503: /)
+  // A zero background still needs 20 a minute, not one 503.
+  const zero = freshState({ background: { medianPerMinute: 0, source: 'input' } })
+  assert.equal(evaluateWatchSample(zero, sampleAt(START, { director503: series([5, 5, 5]) }), LIMIT).state.state, 'watching')
+  assert.equal(evaluateWatchSample(zero, sampleAt(START, { director503: series([21, 21, 21]) }), LIMIT).state.state, 'tripped')
+  // A published minute with no 503 point had none.
+  const gap = series([151, 160, 170])
+  delete gap.perMinute[gap.minutes[1]]
+  assert.equal(evaluateWatchSample(state, sampleAt(START, { director503: gap }), LIMIT).state.state, 'watching')
+})
+
+test('a quiet fleet with no 503 series at all reads zero, not unverifiable', () => {
+  const minutes = minutesBefore(START, 10)
+  const quiet = { ok: true, minutes, perMinute: {}, publishedThrough: minutes.at(-1) }
+  const [result] = run([sampleAt(START, { director503: quiet })], freshState({ background: { medianPerMinute: 0, source: 'measured' } }))
+  assert.equal(result.rules.director503.status, 'pass')
+  assert.deepEqual(Object.values(result.rules.director503.recent), [0, 0, 0])
+  assert.equal(result.state.unverifiableStreak, 0)
+  assert.deepEqual(measuredBackground({}, minutes, minutes.at(-1)), { medianPerMinute: 0, minutes: 10, source: 'measured' })
+})
+
+test('503 minutes past the publishing front are left out, and only no front at all is unverifiable', () => {
+  const state = freshState({ background: { medianPerMinute: 100, source: 'input' } })
+  const minutes = minutesBefore(START, 10)
+  const judge = (director503) => evaluateWatchSample(state, sampleAt(START, { director503 }), LIMIT)
+  // The front is two minutes behind: the three minutes before it are judged, the two after are not.
+  const over = Object.fromEntries(minutes.map((minute) => [minute, 500]))
+  const behind = judge({ ok: true, minutes, perMinute: over, publishedThrough: minutes[7] })
+  assert.equal(behind.rules.director503.status, 'trip')
+  assert.deepEqual(Object.keys(behind.rules.director503.recent), minutes.slice(5, 8))
+  // A sparse 503 series: minutes after its last point but within the front are zero, not late.
+  const sparse = judge({ ok: true, minutes, perMinute: { [minutes[2]]: 500 }, publishedThrough: minutes.at(-1) })
+  assert.equal(sparse.rules.director503.status, 'pass')
+  assert.equal(sparse.state.unverifiableStreak, 0)
+  // No request series at all.
+  const empty = judge({ ok: true, minutes, perMinute: {}, publishedThrough: null })
+  assert.equal(empty.rules.director503.status, 'unverifiable')
+  assert.match(empty.rules.director503.reason, /request series is empty/)
+  assert.equal(empty.state.unverifiableStreak, 1)
+  // A front too far behind to leave three published minutes in the read.
+  const stale = judge({ ok: true, minutes, perMinute: {}, publishedThrough: minutes[1] })
+  assert.equal(stale.rules.director503.status, 'unverifiable')
+  assert.match(stale.rules.director503.reason, new RegExp(`published only through ${minutes[1]}`))
+})
+
+const SEAT_EVENT = 'orca_relay_reserve_ledger_seat_behind_row'
+const SUMMARY_EVENT = 'orca_relay_reserve_ledger_summary'
+const summaryLine = (at, instanceId = 'a') => ({
+  timestamp: new Date(at).toISOString(),
+  labels: { instanceId },
+  jsonPayload: { event: SUMMARY_EVENT, pending: 0, dropped: 0 }
+})
+
+// A Cloud Monitoring or Cloud Logging reader over fakes, as the watch builds one.
+const fakeReader = ({ fetch: fetchImpl = async () => new Response('{}'), runGcloud = async () => ({ stdout: 'token' }) } = {}) => ({
+  fetch: fetchImpl,
+  runGcloud,
+  retryDelayMs: 0,
+  readTimeoutMs: 1_000,
+  now: () => START,
+  deadlineAt: START + 10_000,
+  projectId: 'onorca-cloud'
+})
+
+test('an unparseable timestamp makes the read unverifiable', async () => {
+  const badPoint = {
+    timeSeries: [{ points: [{ interval: { endTime: 'not a time' }, value: { int64Value: '3' } }] }]
+  }
+  const read503 = await readDirector503Minutes(
+    fakeReader({ fetch: async () => new Response(JSON.stringify(badPoint)) }),
+    { projectId: 'onorca-cloud', directorService: 'orca-cloud-relay', endedAt: START, minutes: 3 }
+  )
+  assert.equal(read503.ok, false)
+  assert.match(read503.error, /unreadable/)
+  const seats = await readSeatBehindRowSample(
+    fakeReader({ runGcloud: async () => ({ stdout: JSON.stringify([summaryLine(START), { timestamp: 'garbage', jsonPayload: { epoch: 1, rowEpoch: 2 } }]) }) }),
+    { directorService: 'orca-cloud-relay', cellId: CELL, at: START }
+  )
+  assert.equal(seats.ok, false)
+  assert.match(seats.error, /unparseable timestamp/)
+})
+
+test('a seat-behind-row key carries the director instance and cell beside the epochs', async () => {
+  const entry = (instanceId) => ({ timestamp: new Date(START).toISOString(), labels: { instanceId }, jsonPayload: { event: SEAT_EVENT, epoch: 7, rowEpoch: 8 } })
+  const read = await readSeatBehindRowSample(
+    fakeReader({ runGcloud: async () => ({ stdout: JSON.stringify([entry('a'), summaryLine(START), entry('b')]) }) }),
+    { directorService: 'orca-cloud-relay', cellId: CELL, at: START }
+  )
+  assert.deepEqual(read.reports.map((report) => report.key), [`a/${CELL}/7:8`, `b/${CELL}/7:8`])
+})
+
+test('the duplicate read carries a positive control: no fresh ledger summary in it is unverifiable, not zero', async () => {
+  const filters = []
+  const readWith = (entries) =>
+    readSeatBehindRowSample(
+      fakeReader({ runGcloud: async (args) => (filters.push(args[2]), { stdout: JSON.stringify(entries) }) }),
+      { directorService: 'orca-cloud-relay', cellId: CELL, at: START }
+    )
+  // One read, one window: the control rides the same filter as the seat lines.
+  assert.deepEqual(await readWith([summaryLine(START - 30_000)]), { ok: true, reports: [] })
+  assert.match(filters[0], new RegExp(`\\(\\(jsonPayload\\.event="${SEAT_EVENT}" AND jsonPayload\\.cellId="${CELL}"\\) OR jsonPayload\\.event="${SUMMARY_EVENT}"\\)`))
+  assert.match(filters[0], /resource\.labels\.service_name="orca-cloud-relay"/)
+  for (const entries of [[], [summaryLine(START - 10 * MINUTE)]]) {
+    const blind = await readWith(entries)
+    assert.equal(blind.ok, false)
+    assert.match(blind.error, /no fresh ledger summary \(positive control\)/)
+  }
+  // Blind is a gap in the sample, which counts toward the unverifiable streak.
+  const [gap] = run([sampleAt(START, { seatBehindRow: await readWith([]) })])
+  assert.equal(gap.rules.duplicateSeat.status, 'unverifiable')
+  assert.equal(gap.state.unverifiableStreak, 1)
+})
+
+test('the measured background is the median of the minutes before the flip, published gaps as zero', () => {
+  const minutes = ['2026-10-14T11:47', '2026-10-14T11:48', '2026-10-14T11:49']
+  const front = minutes[2]
+  assert.equal(measuredBackground({ '2026-10-14T11:48': 30 }, minutes, front).medianPerMinute, 0)
+  assert.equal(measuredBackground({ [minutes[0]]: 10, [minutes[1]]: 30, [minutes[2]]: 20 }, minutes, front).medianPerMinute, 20)
+  assert.equal(measuredBackground({}, minutes, '2026-10-14T11:52').medianPerMinute, 0)
+  // Not published through the last minute yet, or no request series: no background yet.
+  assert.equal(measuredBackground({ [minutes[0]]: 10 }, minutes, minutes[1]), null)
+  assert.equal(measuredBackground({}, minutes, null), null)
+  assert.equal(measuredBackground({}, [], front), null)
+})
+
+test('any overrun trips at once', () => {
+  const [result] = run([sampleAt(START, { cell: healthyCell({ enforcedUnits: 2_001 }) })])
+  assert.equal(result.state.state, 'tripped')
+  assert.match(result.state.verdict, /overrun: ledger units 2001 over hard cap 2000/)
+  const [atCap] = run([sampleAt(START, { cell: healthyCell({ enforcedUnits: 2_000 }) })])
+  assert.equal(atCap.state.state, 'watching')
+})
+
+test('a seat reported behind its row again past the grace trips; inside the grace it does not', () => {
+  const report = (at) => ({ at, key: '7:8' })
+  const within = run([
+    sampleAt(START, { seatBehindRow: { ok: true, reports: [report(START - 30_000)] } }),
+    sampleAt(START + MINUTE, { seatBehindRow: { ok: true, reports: [report(START - 30_000), report(START + 50_000)] } })
+  ])
+  assert.equal(within.at(-1).state.state, 'watching')
+  // Carried across samples: the second read no longer holds the first report.
+  const past = run([
+    sampleAt(START, { seatBehindRow: { ok: true, reports: [report(START - 30_000)] } }),
+    sampleAt(START + 2 * MINUTE, { seatBehindRow: { ok: true, reports: [report(START + 70_000)] } })
+  ])
+  assert.equal(past.at(-1).state.state, 'tripped')
+  assert.match(past.at(-1).state.verdict, /duplicateSeat: a seat behind its ledger row \(7:8\)/)
+  // Another seat's report is not the same seat.
+  const other = run([
+    sampleAt(START, { seatBehindRow: { ok: true, reports: [report(START - 30_000)] } }),
+    sampleAt(START + 2 * MINUTE, { seatBehindRow: { ok: true, reports: [{ at: START + 70_000, key: '3:4' }] } })
+  ])
+  assert.equal(other.at(-1).state.state, 'watching')
+})
+
+test('a host drop over 5% in 5 minutes trips outside a drain only', () => {
+  const controls = [1_000, 990, 980, 960, 949]
+  const samples = (draining) =>
+    controls.map((count, minute) =>
+      sampleAt(START + minute * MINUTE, { cell: healthyCell({ controls: count, draining: draining && minute === 2 }) })
+    )
+  assert.equal(run(samples(false)).at(-1).state.state, 'tripped')
+  assert.match(run(samples(false)).at(-1).state.verdict, /hostDrop: connected hosts fell 1000 -> 949/)
+  assert.equal(run(samples(true)).at(-1).state.state, 'watching')
+  // A drop spread over more than 5 minutes is not one.
+  const slow = [1_000, 990, 980, 970, 960, 955, 950].map((count, minute) =>
+    sampleAt(START + minute * MINUTE, { cell: healthyCell({ controls: count }) })
+  )
+  assert.equal(run(slow).at(-1).state.state, 'watching')
+  // Under the floor on a nearly empty cell: 100 -> 92 is 8%, but only 8 hosts.
+  const small = [100, 92].map((count, minute) => sampleAt(START + minute * MINUTE, { cell: healthyCell({ controls: count }) }))
+  assert.equal(run(small).at(-1).state.state, 'watching')
+})
+
+test('ledger drift over 0.5% trips once it has lasted 2 minutes', () => {
+  const drift = (rows) => ({ ok: true, unwrittenRows: rows, directors: 5 })
+  const brief = run([6, 6, 0, 6].map((rows, minute) => sampleAt(START + minute * MINUTE, { ledger: drift(rows) })))
+  assert.equal(brief.at(-1).state.state, 'watching')
+  const sustained = run([6, 6, 6].map((rows, minute) => sampleAt(START + minute * MINUTE, { ledger: drift(rows) })))
+  assert.equal(sustained.length, 3)
+  assert.equal(sustained.at(-1).state.state, 'tripped')
+  assert.match(sustained.at(-1).state.verdict, /ledgerDrift: ledger drift 0\.60% for 120 s/)
+  const atLine = run([5, 5, 5, 5].map((rows, minute) => sampleAt(START + minute * MINUTE, { ledger: drift(rows) })))
+  assert.equal(atLine.at(-1).state.state, 'watching')
+})
+
+test('a tripped dead-man is a trip: Postgres still records reserve until a db write', () => {
+  const [result] = run([sampleAt(START, { cell: healthyCell({ admitModeEffective: 'db' }) })])
+  assert.equal(result.state.state, 'tripped')
+  assert.match(result.state.verdict, /deadMan: the cell tripped its dead-man/)
+})
+
+test('unreadable data is never healthy: N samples in a row with any gap trip', () => {
+  const unreadable = { ok: false, error: 'cell runtime status returned 502' }
+  const gaps = [
+    { cell: unreadable },
+    { ledger: { ok: false, error: 'no placing director logged a ledger summary' } },
+    { director503: { ok: false, error: 'director 503 metric read failed', perMinute: {}, minutes: [], publishedThrough: null } }
+  ]
+  const results = run(gaps.map((gap, minute) => sampleAt(START + minute * MINUTE, gap)))
+  assert.deepEqual(results.map((result) => result.state.unverifiableStreak), [1, 2, 3])
+  assert.equal(results.at(-1).state.state, 'tripped')
+  assert.match(results.at(-1).state.verdict, /unverifiable: 3 samples in a row could not read director503/)
+  // One whole sample resets the streak.
+  const reset = run(
+    [gaps[0], {}, gaps[0], gaps[0]].map((gap, minute) => sampleAt(START + minute * MINUTE, gap))
+  )
+  assert.deepEqual(reset.map((result) => result.state.unverifiableStreak), [1, 0, 1, 2])
+  assert.equal(reset.at(-1).state.state, 'watching')
+  // No background yet is a gap too.
+  const noBackground = freshState({ background: null })
+  assert.equal(evaluateWatchSample(noBackground, sampleAt(START), LIMIT).state.unverifiableStreak, 1)
+  // A measured trip still wins over a gap elsewhere.
+  const [both] = run([sampleAt(START, { cell: healthyCell({ enforcedUnits: 2_500 }), ledger: gaps[1].ledger })])
+  assert.equal(both.state.state, 'tripped')
+  assert.match(both.state.verdict, /overrun/)
+})
+
+test('a cell no longer in reserve stops the watch without a write; one never in reserve is refused', () => {
+  const dbCell = healthyCell({ admitModeConfigured: 'db', admitModeEffective: 'reserve' })
+  const later = run([sampleAt(START), sampleAt(START + MINUTE, { cell: dbCell })])
+  assert.equal(later.at(-1).state.state, 'stopped')
+  assert.match(later.at(-1).state.verdict, /is not in reserve mode \(switch file says db\)/)
+  // Never reserve within the wait: refused, which the workflow does not flip back.
+  const never = run(Array.from({ length: 12 }, (_, minute) => sampleAt(START + minute * MINUTE, { cell: dbCell })))
+  assert.equal(never.at(-1).state.state, 'refused')
+  assert.equal(never.length, 11)
+  assert.match(never.at(-1).state.verdict, /did not report reserve mode within 10 min \(switch file says db\)/)
+})
+
+test('a watch dispatched before the reserve write applied waits for it, then judges as usual', () => {
+  const dbCell = healthyCell({ admitModeConfigured: 'db', admitModeEffective: 'db' })
+  const results = run([
+    sampleAt(START, { cell: dbCell, ledger: { ok: false, error: 'not yet' } }),
+    sampleAt(START + MINUTE, { cell: dbCell }),
+    sampleAt(START + 2 * MINUTE),
+    sampleAt(START + 3 * MINUTE, { cell: healthyCell({ enforcedUnits: 2_500 }) })
+  ])
+  // Waiting samples judge nothing: not a dead-man trip on a db cell, not a gap.
+  assert.deepEqual(results.slice(0, 2).map((result) => result.state.state), ['watching', 'watching'])
+  assert.equal(results[0].rules.reserve.status, 'pending')
+  assert.deepEqual(Object.keys(results[0].rules), ['reserve'])
+  assert.deepEqual(results.map((result) => result.state.unverifiableStreak), [0, 0, 0, 0])
+  assert.equal(results[2].state.reserveSeenAt, START + 2 * MINUTE)
+  assert.equal(results[2].rules.overrun.status, 'pass')
+  assert.equal(results[3].state.state, 'tripped')
+  // Once seen in reserve, a later db reads as a hand flip back, not a refusal.
+  const after = run([sampleAt(START + 2 * MINUTE), sampleAt(START + 3 * MINUTE, { cell: dbCell })], results[0].state)
+  assert.equal(after.at(-1).state.state, 'stopped')
+})
+
+const watchValues = (overrides = {}) => ({
+  mode: 'dry-run',
+  'cell-id': CELL,
+  'cell-origin': 'https://c26.relay.onorca.dev',
+  'project-id': 'onorca-cloud',
+  'director-origin': 'https://relay.onorca.dev',
+  'director-service': 'orca-cloud-relay',
+  'duration-minutes': '120',
+  'segment-minutes': '55',
+  'unverifiable-limit': '3',
+  'background-503-per-minute': '',
+  'flipped-at': new Date(START - 5 * MINUTE).toISOString(),
+  'state-file': '/unused',
+  ...overrides
+})
+const parseAt = (values) => parseWatchArguments(values, START)
+
+test('watch arguments reuse the flag tool checks, so a write watch needs its confirmation up front', () => {
+  assert.equal(parseAt(watchValues()).background503PerMinute, null)
+  assert.equal(parseAt(watchValues()).flippedAt, START - 5 * MINUTE)
+  assert.equal(parseAt(watchValues({ 'background-503-per-minute': '12.5', 'flipped-at': '' })).background503PerMinute, 12.5)
+  assert.throws(() => parseAt(watchValues({ mode: 'write' })), /WRITE_RELAY_CELL_FLAGS production-gce-c26/)
+  assert.doesNotThrow(() => parseAt(watchValues({ mode: 'write', confirmation: `WRITE_RELAY_CELL_FLAGS ${CELL}` })))
+  assert.throws(() => parseAt(watchValues({ 'cell-origin': 'https://c27.relay.onorca.dev' })), /names another cell/)
+  assert.throws(() => parseAt(watchValues({ 'duration-minutes': '301' })), /duration-minutes/)
+  assert.throws(() => parseAt(watchValues({ 'segment-minutes': '56' })), /segment-minutes/)
+  assert.throws(() => parseAt(watchValues({ 'background-503-per-minute': '-1' })), /background/)
+})
+
+test('the 503 background needs flipped-at or an explicit value; flipped-at must be a recent zoned time', () => {
+  assert.throws(() => parseAt(watchValues({ 'flipped-at': '' })), /--flipped-at .* or --background-503-per-minute is required/)
+  assert.throws(() => parseAt(watchValues({ 'flipped-at': '2026-10-14T11:55:00' })), /ISO time with a zone/)
+  assert.throws(() => parseAt(watchValues({ 'flipped-at': '2026-10-14T12:01:00Z' })), /in the future/)
+  assert.throws(() => parseAt(watchValues({ 'flipped-at': '2026-10-13T11:00:00Z' })), /more than 24 hours/)
+  assert.equal(parseAt(watchValues({ 'flipped-at': '2026-10-14T07:55:00-04:00' })).flippedAt, START - 5 * MINUTE)
+})
+
+// A cell, Cloud Monitoring and Cloud Logging, answering from `script` per sample.
+function fakeSources({
+  cell = () => ({}),
+  // Logged at each sample's minute, so it stays fresh for the positive control.
+  ledger = (sample) => [{ timestamp: new Date(START + (sample - 1) * MINUTE).toISOString(), labels: { instanceId: 'a' }, jsonPayload: { pending: 0, dropped: 0 } }],
+  per503 = () => 10,
+  // Requests of every other code per minute; zero is a minute with no traffic at all.
+  perOther = () => 1_000,
+  seatControl = true,
+  rolloutLease = null
+} = {}) {
+  let samples = 0
+  const monitoringSpans = []
+  const fetchImpl = async (url) => {
+    const text = String(url)
+    if (text.endsWith('/v1/admin/runtime-status')) {
+      samples += 1
+      const overrides = cell(samples)
+      return new Response(
+        JSON.stringify({
+          role: 'cell',
+          cellId: CELL,
+          draining: false,
+          flagsApplied: { generation: 4, flags: { admitMode: 'reserve', readinessLocal: true } },
+          admitModeEffective: 'reserve',
+          connectionCapacity: { hardCap: 2_000 },
+          runtime: { controls: 1_000, enforcedConnectionUnits: 1_100 },
+          ...overrides
+        })
+      )
+    }
+    if (text.startsWith('https://monitoring.googleapis.com/')) {
+      const params = new URL(text).searchParams
+      const span = { startedAt: Date.parse(params.get('interval.startTime')), endedAt: Date.parse(params.get('interval.endTime')) }
+      monitoringSpans.push(span)
+      assert.equal(params.get('aggregation.groupByFields'), 'metric.label."response_code"')
+      assert.doesNotMatch(params.get('filter'), /response_code/)
+      // Cloud Monitoring returns no point for a minute with no requests of that code.
+      const seriesOf = (code, count) => {
+        const points = []
+        for (let minute = span.startedAt; minute < span.endedAt; minute += MINUTE) {
+          if (count(minute) > 0) {
+            points.push({ interval: { endTime: new Date(minute + MINUTE).toISOString() }, value: { int64Value: String(count(minute)) } })
+          }
+        }
+        return points.length === 0 ? [] : [{ metric: { labels: { response_code: code } }, points }]
+      }
+      return new Response(JSON.stringify({ timeSeries: [...seriesOf('200', perOther), ...seriesOf('503', per503)] }))
+    }
+    throw new Error(`unexpected fetch ${text}`)
+  }
+  const runGcloud = async (args) => {
+    if (args[0] === 'auth') return { stdout: 'token' }
+    const filter = args[2]
+    const summaries = ledger(samples).map((entry) => ({ ...entry, jsonPayload: { ...entry.jsonPayload, event: SUMMARY_EVENT } }))
+    if (filter.includes(SEAT_EVENT)) return { stdout: JSON.stringify(seatControl ? summaries : []) }
+    if (filter.includes(SUMMARY_EVENT)) return { stdout: JSON.stringify(ledger(samples)) }
+    return { stdout: '[]' }
+  }
+  return { fetchImpl, runGcloud, monitoringSpans, readRolloutLease: async () => rolloutLease }
+}
+
+test('a segment samples once a minute, carries state to the next segment, and stops on a trip', async () => {
+  const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-'))
+  try {
+    let clock = START
+    const lines = []
+    const config = parseAt(
+      watchValues({ 'state-file': join(directory, 'state.json'), 'github-output': join(directory, 'out'), 'segment-minutes': '3' })
+    )
+    const sources = fakeSources({ cell: (sample) => (sample === 5 ? { runtime: { controls: 1_000, enforcedConnectionUnits: 2_001 } } : {}) })
+    const dependencies = {
+      ...sources,
+      idToken: 'id',
+      retryDelayMs: 0,
+      now: () => clock,
+      sleep: async (ms) => { clock += ms },
+      log: (line) => lines.push(JSON.parse(line))
+    }
+    const first = await watchSegment(config, dependencies)
+    assert.equal(first.state, 'watching')
+    assert.equal(lines.length, 3)
+    assert.equal(first.background.source, 'measured')
+    assert.equal(await readFile(join(directory, 'out'), 'utf8'), 'state=watching\nverdict=\n')
+    const second = await watchSegment(config, dependencies)
+    assert.equal(second.state, 'tripped')
+    assert.equal(second.samples, 5)
+    assert.deepEqual(lines.map((line) => line.sample), [1, 2, 3, 4, 5])
+    assert.equal(lines.at(-1).rules.overrun.status, 'trip')
+    assert.match(await readFile(join(directory, 'out'), 'utf8'), /state=tripped\nverdict=flip back production-gce-c26: overrun: /)
+  } finally {
+    await rm(directory, { recursive: true, force: true })
+  }
+})
+
+test('a sample that cannot read its sources counts as unverifiable rather than crashing the watch', async () => {
+  const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-'))
+  try {
+    let clock = START
+    const config = parseAt(watchValues({ 'state-file': join(directory, 'state.json'), 'segment-minutes': '10' }))
+    const sources = fakeSources({ ledger: () => [] })
+    const state = await watchSegment(config, {
+      ...sources,
+      idToken: 'id',
+      retryDelayMs: 0,
+      now: () => clock,
+      sleep: async (ms) => { clock += ms },
+      log: () => {}
+    })
+    assert.equal(state.state, 'tripped')
+    assert.equal(state.samples, 3)
+    // No summary also blinds the duplicate rule, whose positive control it is.
+    assert.match(state.verdict, /unverifiable: 3 samples in a row could not read duplicateSeat, ledgerDrift/)
+  } finally {
+    await rm(directory, { recursive: true, force: true })
+  }
+})
+
+test('a watch that ended without a verdict records failed, which the workflow flips back', async () => {
+  const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-'))
+  try {
+    const output = join(directory, 'out')
+    assert.equal((await recordOutcome(join(directory, 'missing.json'), output)).state, 'failed')
+    await writeFile(join(directory, 'watching.json'), JSON.stringify(freshState()))
+    assert.equal((await recordOutcome(join(directory, 'watching.json'), output)).state, 'failed')
+    await writeFile(join(directory, 'tripped.json'), JSON.stringify(freshState({ state: 'tripped', verdict: 'flip back x:\ny' })))
+    assert.equal((await recordOutcome(join(directory, 'tripped.json'), output)).state, 'tripped')
+    assert.match(await readFile(output, 'utf8'), /state=tripped\nverdict=flip back x: y\n$/)
+  } finally {
+    await rm(directory, { recursive: true, force: true })
+  }
+})
+
+// Google Storage and the director, as far as the flag tool's flip back reaches them. `onUpload`
+// returns 'conflict' to play another writer landing first, optionally with its own admitMode.
+function fakeFlipBack({ generation = '7', onUpload = () => 'ok', cellAnswers = true } = {}) {
+  const calls = []
+  let uploads = 0
+  let object = { generation: Number(generation), flags: { admitMode: 'reserve', readinessLocal: true } }
+  let applied = object
+  const fetchImpl = async (url, init = {}) => {
+    const text = String(url)
+    calls.push({ url: text, method: init.method ?? 'GET', body: init.body })
+    if (text.includes('/upload/storage/v1/')) {
+      uploads += 1
+      assert.match(text, new RegExp(`ifGenerationMatch=${object.generation}$`))
+      const outcome = onUpload(uploads)
+      if (outcome !== 'ok') {
+        object = { generation: object.generation + 1, flags: { ...object.flags, admitMode: outcome.admitMode ?? 'reserve' } }
+        applied = object
+        return new Response('', { status: 412 })
+      }
+      object = { generation: object.generation + 1, flags: JSON.parse(init.body.split('\r\n')[7]).flags }
+      applied = object
+      return new Response(JSON.stringify({ generation: String(object.generation) }))
+    }
+    if (text.includes('?alt=media')) {
+      return new Response(JSON.stringify({ v: 1, cellId: CELL, flags: object.flags }))
+    }
+    if (text.includes('/storage/v1/b/')) return new Response(JSON.stringify({ generation: String(object.generation) }))
+    if (text.endsWith('/v1/admin/runtime-status')) {
+      if (!cellAnswers) return new Response('', { status: 502 })
+      return new Response(
+        JSON.stringify({
+          role: 'cell',
+          cellId: CELL,
+          flagsApplied: applied,
+          supportedFlags: {
+            admitMode: { type: 'enum', values: ['db', 'reserve'] },
+            readinessLocal: { type: 'boolean' }
+          },
+          admitModeEffective: applied.flags.admitMode,
+          runtime: { controls: 10 }
+        })
+      )
+    }
+    if (text.endsWith('/v1/admin/cell-admit-mode')) {
+      const admitMode = JSON.parse(init.body).admitMode
+      return new Response(JSON.stringify({ v: 1, cellId: CELL, admitMode: admitMode ?? 'reserve', heartbeatFresh: true }))
+    }
+    throw new Error(`unexpected fetch ${text}`)
+  }
+  return { fetchImpl, calls, uploads: () => uploads }
+}
+
+const flipDependencies = (fetchImpl, lines, sleeps = []) => {
+  let clock = START
+  return {
+    fetchImpl,
+    accessToken: 'access',
+    idToken: 'id',
+    audit: { writtenBy: 'reserve-watch/test', runId: '1', runAttempt: '1' },
+    now: () => clock,
+    sleep: async (ms) => { sleeps.push(ms); clock += ms },
+    log: (line) => lines.push(JSON.parse(line))
+  }
+}
+
+const flipValues = (overrides = {}) => ({
+  mode: 'write',
+  'cell-id': CELL,
+  'cell-origin': 'https://c26.relay.onorca.dev',
+  'project-id': 'onorca-cloud',
+  'director-origin': 'https://relay.onorca.dev',
+  confirmation: `WRITE_RELAY_CELL_FLAGS ${CELL}`,
+  ...overrides
+})
+
+test('the flip back is the flag tool write: admitMode=db at the generation just read, then Postgres db', async () => {
+  const { fetchImpl, calls } = fakeFlipBack()
+  const lines = []
+  const result = await flipBack(flipValues(), flipDependencies(fetchImpl, lines))
+  assert.equal(result.written, true)
+  const upload = calls.find((call) => call.url.includes('/upload/storage/v1/'))
+  assert.match(upload.body, /"flags":\{"admitMode":"db","readinessLocal":true\}/)
+  assert.ok(calls.some((call) => call.url.endsWith('/v1/admin/cell-admit-mode') && JSON.parse(call.body).admitMode === 'db'))
+  assert.deepEqual(lines.at(-1), {
+    event: 'orca_relay_reserve_watch_flip_back',
+    mode: 'write',
+    cellId: CELL,
+    written: true,
+    generation: '8'
+  })
+})
+
+test('a dry-run flip back plans the write and writes nothing', async () => {
+  const { fetchImpl, calls } = fakeFlipBack()
+  const lines = []
+  const result = await flipBack(flipValues({ mode: 'dry-run', confirmation: '' }), {
+    fetchImpl,
+    accessToken: 'access',
+    idToken: 'id',
+    audit: {},
+    sleep: async () => {},
+    log: (line) => lines.push(JSON.parse(line))
+  })
+  assert.equal(result.written, false)
+  assert.ok(!calls.some((call) => call.url.includes('/upload/')))
+  assert.ok(!calls.some((call) => call.url.endsWith('/v1/admin/cell-admit-mode') && JSON.parse(call.body).admitMode))
+  assert.equal(lines.find((line) => line.event === 'orca_relay_cell_flags_plan').desired.flags.admitMode, 'db')
+})
+
+test('a write flip back without the typed confirmation is refused before any read', async () => {
+  const { fetchImpl, calls } = fakeFlipBack()
+  await assert.rejects(
+    flipBack(flipValues({ confirmation: '' }), { fetchImpl, accessToken: 'a', idToken: 'i', audit: {}, log: () => {} }),
+    /WRITE_RELAY_CELL_FLAGS production-gce-c26/
+  )
+  assert.equal(calls.length, 0)
+})
+
+test('a generation conflict is re-read and retried while the switch file still says reserve', async () => {
+  const { fetchImpl, uploads } = fakeFlipBack({ onUpload: (n) => (n < 3 ? { admitMode: 'reserve' } : 'ok') })
+  const lines = []
+  const sleeps = []
+  const result = await flipBack(flipValues(), flipDependencies(fetchImpl, lines, sleeps))
+  assert.equal(result.written, true)
+  assert.equal(uploads(), 3)
+  assert.deepEqual(lines.filter((line) => line.event === 'orca_relay_reserve_watch_flip_back_conflict').map((line) => line.attempt), [1, 2])
+  assert.deepEqual(sleeps.slice(0, 1), [2_000])
+  assert.equal(lines.at(-1).generation, '10')
+})
+
+test('a conflict whose writer already flipped the cell to db ends the flip back without a write', async () => {
+  const { fetchImpl, uploads } = fakeFlipBack({ onUpload: () => ({ admitMode: 'db' }) })
+  const lines = []
+  const result = await flipBack(flipValues(), flipDependencies(fetchImpl, lines))
+  assert.equal(uploads(), 1)
+  assert.equal(result.written, false)
+  assert.equal(result.alreadyDb, true)
+  assert.deepEqual(lines.at(-1), { event: 'orca_relay_reserve_watch_flip_back', mode: 'write', cellId: CELL, written: false, generation: '8', alreadyDb: true })
+})
+
+test('conflicts past the retry bound fail the flip back', async () => {
+  const { fetchImpl, uploads } = fakeFlipBack({ onUpload: () => ({ admitMode: 'reserve' }) })
+  await assert.rejects(flipBack(flipValues(), flipDependencies(fetchImpl, [])), /object changed since expected-generation/)
+  assert.equal(uploads(), FLIP_BACK_ATTEMPTS)
+})
+
+test('an unreachable cell fails with the exact break-glass inputs and writes nothing', async () => {
+  const { fetchImpl, uploads } = fakeFlipBack({ cellAnswers: false })
+  const lines = []
+  await assert.rejects(
+    flipBack(flipValues(), flipDependencies(fetchImpl, lines)),
+    (error) => {
+      assert.match(error.message, /production-gce-c26 does not answer runtime-status \(cell runtime status returned 502\)/)
+      assert.match(error.message, /set=admitMode=default, break-glass=BREAK GLASS production-gce-c26, expected-generation=7/)
+      return true
+    }
+  )
+  assert.equal(uploads(), 0)
+  assert.deepEqual(lines.at(-1).inputs, {
+    mode: 'write',
+    'cell-id': CELL,
+    set: 'admitMode=default',
+    'break-glass': `BREAK GLASS ${CELL}`,
+    'expected-generation': '7',
+    confirmation: `WRITE_RELAY_CELL_FLAGS ${CELL}`
+  })
+})
+
+test('the 503 background is the 10 minutes before flipped-at, read once they are published', async () => {
+  const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-'))
+  try {
+    let clock = START
+    // Flipped one minute ago: the minutes before it are not published until 3 minutes after it.
+    const flippedAt = START - MINUTE + 20_000
+    const config = parseAt(
+      watchValues({ 'state-file': join(directory, 'state.json'), 'segment-minutes': '3', 'flipped-at': new Date(flippedAt).toISOString() })
+    )
+    const sources = fakeSources({ per503: (minute) => (minute < START - MINUTE ? 4 : 40) })
+    const lines = []
+    const state = await watchSegment(config, {
+      ...sources,
+      idToken: 'id',
+      retryDelayMs: 0,
+      now: () => clock,
+      sleep: async (ms) => { clock += ms },
+      log: (line) => lines.push(JSON.parse(line))
+    })
+    // Pending until then, and pending is not a gap: the streak stays at zero.
+    assert.deepEqual(lines.map((line) => line.rules.director503.status), ['pending', 'pending', 'pass'])
+    assert.match(lines[0].rules.director503.reason, /background before the flip is not published yet/)
+    assert.deepEqual(lines.map((line) => line.unverifiableStreak), [0, 0, 0])
+    const baseline = sources.monitoringSpans.find((span) => span.endedAt === START - MINUTE)
+    assert.equal(baseline.startedAt, START - 11 * MINUTE)
+    assert.deepEqual(state.background, { medianPerMinute: 4, minutes: 10, source: 'measured' })
+    // An explicit value overrides the measurement.
+    const explicit = initialWatchState({ cellId: CELL, startedAt: START, durationMinutes: 60, background503PerMinute: 9, flippedAt })
+    assert.deepEqual(explicit.background, { medianPerMinute: 9, source: 'input' })
+  } finally {
+    await rm(directory, { recursive: true, force: true })
+  }
+})
+
+test('a quiet fleet with zero director 503s watches on and never trips', async () => {
+  const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-'))
+  try {
+    let clock = START
+    const config = parseAt(
+      watchValues({
+        'state-file': join(directory, 'state.json'),
+        'segment-minutes': '20',
+        'flipped-at': new Date(START - 2 * MINUTE).toISOString()
+      })
+    )
+    const lines = []
+    const state = await watchSegment(config, {
+      ...fakeSources({ per503: () => 0 }),
+      idToken: 'id',
+      retryDelayMs: 0,
+      now: () => clock,
+      sleep: async (ms) => { clock += ms },
+      log: (line) => lines.push(JSON.parse(line))
+    })
+    assert.equal(state.state, 'watching')
+    assert.equal(lines.length, 20)
+    assert.deepEqual(state.background, { medianPerMinute: 0, minutes: 10, source: 'measured' })
+    assert.ok(lines.slice(1).every((line) => line.rules.director503.status === 'pass'))
+    assert.ok(lines.every((line) => line.unverifiableStreak === 0))
+  } finally {
+    await rm(directory, { recursive: true, force: true })
+  }
+})
+
+test('no request series at all is unverifiable and trips at the limit', async () => {
+  const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-'))
+  try {
+    let clock = START
+    const config = parseAt(watchValues({ 'state-file': join(directory, 'state.json'), 'segment-minutes': '10', 'background-503-per-minute': '0', 'flipped-at': '' }))
+    const state = await watchSegment(config, {
+      ...fakeSources({ per503: () => 0, perOther: () => 0 }),
+      idToken: 'id',
+      retryDelayMs: 0,
+      now: () => clock,
+      sleep: async (ms) => { clock += ms },
+      log: () => {}
+    })
+    assert.equal(state.state, 'tripped')
+    assert.match(state.verdict, /unverifiable: 3 samples in a row could not read director503/)
+  } finally {
+    await rm(directory, { recursive: true, force: true })
+  }
+})
+
+test('a later segment judges flipped-at against the watch start, so a long watch does not age it out', async () => {
+  const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-'))
+  try {
+    const stateFile = join(directory, 'state.json')
+    const values = watchValues({ 'state-file': stateFile, 'flipped-at': new Date(START - 23 * 60 * MINUTE).toISOString() })
+    const later = START + 4 * 60 * MINUTE
+    await writeFile(stateFile, JSON.stringify(freshState()))
+    assert.equal((await parseWatchSegment(values, later)).flippedAt, START - 23 * 60 * MINUTE)
+    // The first segment (no state file yet) still refuses it, and records refused.
+    await rm(stateFile)
+    await assert.rejects(parseWatchSegment(values, later), /more than 24 hours/)
+    assert.equal(JSON.parse(await readFile(stateFile, 'utf8')).state, 'refused')
+  } finally {
+    await rm(directory, { recursive: true, force: true })
+  }
+})
+
+test('a corrupt state file fails the segment, never refuses or restarts it, and the outcome reads failed', async () => {
+  const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-'))
+  try {
+    const stateFile = join(directory, 'state.json')
+    const output = join(directory, 'out')
+    const values = watchValues({ 'state-file': stateFile })
+    for (const corrupt of ['null', '[]', '7', JSON.stringify({ state: 'watching', cellId: 'c7' })]) {
+      await writeFile(stateFile, corrupt)
+      await assert.rejects(parseWatchSegment(values, START), /state file/)
+      assert.equal(await readFile(stateFile, 'utf8'), corrupt)
+      assert.equal((await recordOutcome(stateFile, output)).state, 'failed')
+    }
+  } finally {
+    await rm(directory, { recursive: true, force: true })
+  }
+})
+
+test('a slow runtime-status reply is retried before the flip back calls for break glass', async () => {
+  const { fetchImpl: answering, uploads } = fakeFlipBack()
+  let runtimeCalls = 0
+  // fetchAdminOnceMore makes two tries per read; the first read's two both fail.
+  const fetchImpl = async (url, init) => {
+    if (String(url).endsWith('/v1/admin/runtime-status') && (runtimeCalls += 1) <= 2) return new Response('', { status: 503 })
+    return answering(url, init)
+  }
+  const lines = []
+  const sleeps = []
+  const result = await flipBack(flipValues(), flipDependencies(fetchImpl, lines, sleeps))
+  assert.equal(result.written, true)
+  assert.equal(uploads(), 1)
+  assert.deepEqual(lines.filter((line) => line.event === 'orca_relay_reserve_watch_runtime_retry').map((line) => line.attempt), [1])
+  assert.ok(!lines.some((line) => line.event === 'orca_relay_reserve_watch_break_glass_required'))
+  assert.ok(sleeps.includes(2_000))
+  assert.ok(FLIP_BACK_RUNTIME_ATTEMPTS > 1)
+})
+
+const watchWorkflow = readRelayWorkflow('operate-relay-production-reserve-watch.yml')
+const jobOf = (name) => {
+  const start = watchWorkflow.indexOf(`\n  ${name}:\n`)
+  assert.notEqual(start, -1, `job ${name} is missing`)
+  const next = watchWorkflow.slice(start + 1).search(/\n  [a-z-]+:\n/)
+  return watchWorkflow.slice(start, next === -1 ? undefined : start + 1 + next)
+}
+
+test('workflow: neither job takes the flag workflow group, which would let a newer run cancel a pending flip back', () => {
+  assert.doesNotMatch(watchWorkflow, /group: relay-production-cell-flags/)
+  assert.doesNotMatch(jobOf('flip-back'), /^\s+concurrency:/m)
+  assert.doesNotMatch(jobOf('watch'), /^\s+concurrency:/m)
+})
+
+test('workflow: a watch that failed or was cancelled without a verdict flips back too', () => {
+  const flip = jobOf('flip-back')
+  // A cancel re-evaluates each step: every flip-back step must run on always() too.
+  const steps = flip.slice(flip.indexOf('\n    steps:\n')).split(/\n      - /).slice(1)
+  assert.equal(steps.length, 7)
+  for (const step of steps) assert.match(step, /\n\s+if: \$\{\{ always\(\) \}\}/, step.split('\n')[0])
+  assert.match(flip, /if: \$\{\{ always\(\) && vars\.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && needs\.watch\.result != 'skipped'/)
+  assert.match(flip, /!contains\(fromJSON\('\["completed","stopped","refused"\]'\), needs\.watch\.outputs\.state\)/)
+  // Every segment checks its ID token's shape, and runs the one shared command.
+  const segments = watchWorkflow.match(/run: bash -c "\$\{WATCH_SEGMENT\}"/g) ?? []
+  assert.equal(segments.length, 6)
+  assert.match(jobOf('watch'), /process\.env\.ORCA_RELAY_ADMIN_ID_TOKEN/)
+})
+
+test('a live rollout lease refuses the watch before its first sample; an expired one does not', async () => {
+  const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-'))
+  try {
+    const stateFile = join(directory, 'state.json')
+    const output = join(directory, 'out')
+    const config = parseAt(watchValues({ 'state-file': stateFile, 'github-output': output, 'segment-minutes': '1' }))
+    const lease = (expiresAt) => ({ record: { expires_at: expiresAt }, holder: 'Deploy Relay Production Same-Cap (run 9)' })
+    let clock = START
+    const dependencies = (rolloutLease) => ({
+      ...fakeSources({ rolloutLease }),
+      idToken: 'id',
+      retryDelayMs: 0,
+      now: () => clock,
+      sleep: async (ms) => { clock += ms },
+      log: () => {}
+    })
+    const held = await watchSegment(config, dependencies(lease(START + MINUTE)))
+    assert.equal(held.state, 'refused')
+    assert.match(held.verdict, /a rollout holds the Cloud SQL rollout lease until .*Same-Cap \(run 9\)/)
+    assert.equal(JSON.parse(await readFile(stateFile, 'utf8')).state, 'refused')
+    assert.match(await readFile(output, 'utf8'), /^state=refused\n/)
+    await rm(stateFile)
+    assert.equal((await watchSegment(config, dependencies(lease(START - 1)))).state, 'watching')
+    // Only the first segment checks: a later one carries on from its state file.
+    assert.equal((await watchSegment(config, dependencies(lease(clock + MINUTE)))).state, 'watching')
+    // A lease it cannot read refuses too, as the lease action fails closed.
+    assert.match(await rolloutLeaseRefusal(async () => { throw new Error('403') }, START), /lease is unreadable \(403\)/)
+    assert.equal(await rolloutLeaseRefusal(async () => null, START), null)
+  } finally {
+    await rm(directory, { recursive: true, force: true })
+  }
+})
+
+const stepOf = (job, name) => {
+  const text = jobOf(job)
+  const start = text.indexOf(`- name: ${name}\n`)
+  assert.notEqual(start, -1, `step ${name} is missing`)
+  const end = text.indexOf('\n      - ', start + 1)
+  return text.slice(start, end === -1 ? undefined : end)
+}
+const ALARM_COMMAND = /node dev\/scripts\/watch-relay-reserve-cell\.mjs alarm --kind KIND \\\n\s+--mode "\$\{MODE\}" --cell-id "\$\{CELL_ID\}" --project-id "\$\{GCP_PROJECT_ID\}" --verdict "\$\{VERDICT_VAR\}"/
+const alarmCommand = (kind, verdict) => new RegExp(ALARM_COMMAND.source.replace('KIND', kind).replace('VERDICT_VAR', verdict))
+
+test('workflow: a flip back that did not land and a refused watch both raise the alarm', () => {
+  const alarm = stepOf('flip-back', 'Alarm when the flip back did not land')
+  assert.match(alarm, /if: \$\{\{ always\(\) \}\}/)
+  assert.match(alarm, /grep -qs '"event":"orca_relay_reserve_watch_flip_back"' "\$\{RUNNER_TEMP\}\/relay-reserve-flip-back\.jsonl"/)
+  assert.match(alarm, alarmCommand('flip-back-failed', 'VERDICT'))
+  const refused = stepOf('watch', 'Alarm on a refused watch')
+  assert.match(refused, /if: \$\{\{ always\(\) && steps\.outcome\.outputs\.state == 'refused' \}\}/)
+  assert.match(refused, /WATCH_VERDICT: \$\{\{ steps\.outcome\.outputs\.verdict \}\}/)
+  assert.match(refused, alarmCommand('refused', 'WATCH_VERDICT'))
+  // After the outcome step, so it reads the recorded state.
+  assert.ok(jobOf('watch').indexOf('- name: Alarm on a refused watch') > jobOf('watch').indexOf('- id: outcome'))
+  // The flip-back line names only the write's own event, never its conflict retries.
+  assert.match(readFileSync(new URL('./watch-relay-reserve-cell.mjs', import.meta.url), 'utf8'), /event: 'orca_relay_reserve_watch_flip_back',\n/)
+})
+
+test('an alarm is an escaped error annotation everywhere, and in write mode a paging log line', async () => {
+  const calls = []
+  const lines = []
+  const runGcloud = async (args, options) => (calls.push({ args, options }), { stdout: '' })
+  const alarm = (overrides) =>
+    raiseAlarm(
+      { mode: 'write', cellId: CELL, projectId: 'onorca-cloud', kind: 'refused', reason: '50%\r\nlease held', ...overrides },
+      { runGcloud, log: (line) => lines.push(line) }
+    )
+  assert.deepEqual(await alarm(), { paged: true })
+  assert.equal(
+    lines[0],
+    `::error title=Reserve watch refused for ${CELL}::${CELL} may be in reserve mode and unwatched: the watch was refused (50%25%0D%0Alease held). ` +
+      'Rerun it, or flip the cell to db with Operate Relay Production Cell Flags.'
+  )
+  assert.deepEqual(calls[0].args, [
+    'logging', 'write', ALARM_LOG_NAME,
+    JSON.stringify({ event: ALARM_EVENT, cellId: CELL, kind: 'refused', reason: '50%\r\nlease held' }),
+    '--payload-type=json', '--severity=ERROR', '--project', 'onorca-cloud'
+  ])
+  // A dry run never flips back, so its alarm annotates but does not page.
+  assert.deepEqual(await alarm({ mode: 'dry-run', kind: 'flip-back-failed' }), { paged: false })
+  assert.equal(calls.length, 1)
+  assert.match(lines[1], /^::error title=Reserve watch flip-back-failed for production-gce-c26::.*did not land/)
+  // A title property escapes its own separators too.
+  await alarm({ mode: 'dry-run', cellId: 'a:b,c' })
+  assert.match(lines[2], /^::error title=Reserve watch refused for a%3Ab%2Cc::/)
+  // A failed log write is named, never swallowed silently.
+  const failed = await raiseAlarm(
+    { mode: 'write', cellId: CELL, projectId: 'onorca-cloud', kind: 'flip-back-failed', reason: 'x' },
+    { runGcloud: async () => { throw new Error('PERMISSION_DENIED') }, log: (line) => lines.push(line) }
+  )
+  assert.deepEqual(failed, { paged: false })
+  assert.match(lines.at(-1), /^::error title=Reserve watch page failed::.*nothing pages: PERMISSION_DENIED/)
+  await assert.rejects(alarm({ kind: 'other' }), /--kind must be one of/)
+  // A missing project still annotates, then names the page it could not send.
+  const before = calls.length
+  assert.deepEqual(await alarm({ projectId: undefined }), { paged: false })
+  assert.equal(calls.length, before)
+  assert.match(lines.at(-2), /^::error title=Reserve watch refused for /)
+  assert.match(lines.at(-1), /nothing pages: --project-id is not a GCP project id/)
+})

cloud/dev/scripts/workload-identity-attribute-conditions.test.mjs

@@ -34,7 +34,7 @@ const EXPECTED_CONDITIONS = {
         "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && assertion.event_name == 'workflow_dispatch' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-push-deploy.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-push-deploy.yml@refs/heads/main'",
 
       github:
-        "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && ((assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-fence-broker.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-director.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-multi-target.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-asia-admission.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-cell-flags.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-publish-relay-production.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-rehome.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-rehome-job.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main' && (assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml@refs/heads/main' || assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main')))",
+        "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && ((assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-fence-broker.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-director.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-multi-target.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-asia-admission.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-cell-flags.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-reserve-watch.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-publish-relay-production.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-rehome.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-rehome-job.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main' && (assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml@refs/heads/main' || assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main')))",
       github_monitor:
         "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-monitor-relay-production.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-monitor-relay-production-job.yml@refs/heads/main'",
       github_fence:

cloud/docs/relay-workflows.md

@@ -109,7 +109,7 @@ identity bootstrap apply. The providers require their exact workflows on
 `refs/heads/main` with the `production` environment. Missing values fail
 closed; no dedicated operations identity falls back to the shared deploy identity.
 
-The shared production identity is restricted to seven named direct Relay callers plus the exact
+The shared production identity is restricted to nine named direct Relay callers plus the exact
 regional-rehome and same-cap reusable wrapper/job pairs on `main` in the `production` environment.
 Its Artifact Registry and Cloud Run mutation permissions are scoped to the Orca repository, Relay
 director, and Relay fence broker; it cannot mutate the API or auth services.
@@ -461,8 +461,9 @@ margin covers. Each cell is isolated, drained until restart-safe, replaced
 from a targeted saved plan, and restored only after a new incarnation reports the exact digest, cap,
 heartbeat, and rehome protocol. Restart-safe means the cell runtime itself carries nothing live (no
 controls, in-flight connections, reserved connection units, splices, or queued bytes) and no
-migration is open, for a whole drain pace window, which every live restart-safe call must pass.
-Pre-auth and total connections are printed but do not reset the window: they include
+migration is open, on two consecutive samples the second of which is taken at or after the end
+of the drain pace window. Every live restart-safe call supplies that window (`--pace-window-ms`).
+Pre-auth and total connections are printed but do not restart the count: they include
 unauthenticated redials that lose nothing on a restart. Director activity leases left by hosts that already went or cannot
 be placed do not hold the restart; every `relay_capacity_transition_restart_progress` sample and
 the final verified line report them under `stranded`. The durable
@@ -539,11 +540,10 @@ drained. Then read the cell's live runtime image from
 The `drain-pace-window-ms` input sets the window the cell spreads its drain sends over. Each
 drained host re-dials the director as soon as it reads `drain`, so the window sets the re-placement
 arrival rate: hosts / window, about 1.8-2.7 hosts/s for a US cell at the default. The window also
-sets two waits: restart-safe needs an empty runtime for (ceil(window / 5 s) + 1) consecutive
-5-second samples, and the drain step's overall timeout is the 15-minute migration lease plus the
-window, plus the window's excess over `300000` (20 minutes at the default, 50 at `1200000`).
-Because the quiet starts only once the last host has left, a drain step takes about twice the
-window: about 30 minutes at `900000` and 40 at `1200000`.
+sets two waits: restart-safe needs an empty runtime on two consecutive 5-second samples, the
+second at or after the window's end, and the drain step's overall timeout is the 15-minute
+migration lease plus the window (20 minutes at the default, 35 at `1200000`). The runtime empties
+as the window closes, so a drain step takes about the window plus a few seconds to half a minute.
 
 - Allowed values are `300000` (the default, and every wave before this input), `60000`,
   `30000`, `900000`, and `1200000`. The wave validator refuses anything else, and each cell job
@@ -634,7 +634,7 @@ monitor already excludes 503s from its director 5xx rule.
    - its `paceVerdict` is PASS, sealed only from a report on that cell that drained at least 400
      hosts at that pace;
    - time to empty is within window + hosts / 50 s + 10 s. Take `settledAfterSeconds` minus the
-     restart-safe quiet; this is an upper bound, since it includes the isolate.
+     one 5-second confirming sample; this is an upper bound, since it includes the isolate.
 
    The batch check enforces the PASS. A canary that sealed anything else authorizes only
    `300000` batches. The 400-host floor exists because a pace is an arrival rate (hosts /
@@ -801,6 +801,76 @@ It ends with the single command that finishes from where it stopped. That comman
 state. Each run writes a timestamped log under `~/.orca/relay-director-deploy/`
 (`--log-directory` overrides it).
 
+## Reserve watch
+
+`Operate Relay Production Reserve Watch` watches one cell already switched to reserve mode and, on
+the first step 5 stop rule it trips, flips it back to db through the flag tool's own write
+(`operate-relay-cell-flags.mjs`: CAS, read-back, re-registration wait, Postgres record). It is
+manual only and inert until dispatched; `dry-run` logs the flag tool's plan instead of writing.
+`write` takes the flag workflow's confirmation, `WRITE_RELAY_CELL_FLAGS <cell-id>`.
+
+It samples once a minute for up to 300 minutes, re-authenticating every 55-minute segment because
+an ID token lasts an hour, and logs one JSON line per sample. The 503 background must predate the
+flip: give `flipped-at` (when the reserve write applied; the background is the median of the 10
+minutes before it) or `background-503-per-minute`, which overrides it. With neither, the watch is
+refused. A watch dispatched before the reserve write applied waits up to 10 minutes for the cell to
+report reserve, judging nothing meanwhile; a cell that never does ends the watch refused, without a
+write. It also refuses to start while any run holds the Cloud SQL rollout lease (same-cap and
+capacity rolls hold it for their whole run, and it names no cell), or when that lease cannot be
+read. It only reads the lease: holding it for hours would block every rollout. Pausing on a roll
+that starts mid-watch was not built, since a pause blinds the stop rules for the roll's length and
+a flip back mid-roll races its drain, so do not start a roll on a watched cell. It trips on, in
+this order:
+
+| Rule | Signal | Exact or proxy |
+|---|---|---|
+| Dead-man tripped | runtime-status `admitModeEffective=db` under a reserve switch file | Exact |
+| Overrun | runtime-status `enforcedConnectionUnits` over `connectionCapacity.hardCap` | Invariant check: the ledger already refuses past its cap, so this trips only if that refusal broke. It counts seats, bookings, in-flight and phone units |
+| 503s | Director `request_count` 503s above max(1.5x background, background + 20) for 3 complete minutes | Proxy that can miss: fleet-wide, since no signal attributes a 503 to a cell, so a problem on one cell can stay under the fleet's line |
+| Duplicate past grace | The ledger reconcile's `orca_relay_reserve_ledger_seat_behind_row` for the cell, reported again more than 90 s (60 s demotion close + 30 s map lag) after it was first seen, keyed by director instance, cell, epoch and row epoch | Proxy that can only over-count: the line names no host, so two hosts sharing that key read as one seat |
+| Host drop | runtime-status `controls` fell more than 5% (and at least 10 hosts) within 5 minutes with no `draining` sample | Exact |
+| Ledger drift | Queued plus dropped rows in each director's newest `orca_relay_reserve_ledger_summary`, over the cell's controls, above 0.5% for 2 minutes | Proxy that can only over-count: the summaries are fleet-wide, so exact only while this is the only reserve cell |
+
+Any rule it cannot read in a sample makes that sample unverifiable; `unverifiable-limit` such
+samples in a row is a trip of its own. The 503 read takes every response code in one query, and the
+newest minute any code has a point for is how far the series is published: a 503 minute at or
+before it with no point had no 503s, so a quiet fleet reads zero. Minutes past it are left out, not
+counted; fewer than 3 published minutes in the read, no request series at all, or an unparseable
+timestamp is unverifiable. This assumes the codes publish together; one code lagging the others
+delays a trip by its lag but cannot hide one, because a later read sees its points. Until the
+minutes before `flipped-at` are published (3 minutes past it) the 503 rule is pending, which does
+not add to the streak. The duplicate read also takes `orca_relay_reserve_ledger_summary`, which
+every placing director logs each minute, in the same query and window: with no summary from the
+last 150 s in it, the read is blind and the sample unverifiable, not zero duplicates. The duplicate
+rule keys a seat by director instance, so a director restart (a new instance id) starts its
+first-seen clock over. A cell whose switch file no longer says reserve ends the watch without a
+write.
+
+In `write` mode a watch that fails, times out or is cancelled before a verdict flips the cell back
+as a trip would; the flip-back job and each of its steps run on `always()`, so a cancel during the
+lease wait does not skip the switch-file write. That cancel can still kill the process before
+Postgres records db, leaving the file at db and Postgres at reserve; a force-cancel can stop it
+outright. The workflow's only group is
+its own per-cell one, which a running run never yields. It stays out of the flag workflow's group
+because GitHub keeps one pending run per group and a newer one cancels it, so a flip back queued
+there could be dropped by any later flag dispatch, and a watch held there would block manual writes
+for hours. The flag tool's generation check (CAS) orders it against other writes instead; on a
+conflict it re-reads and retries, up to 5 times, while the switch file still says reserve, and stops
+if it already says db. A cell that still does not answer runtime-status after 4 tries with backoff
+cannot take the ordinary flip back: the run fails and prints the exact break-glass inputs for
+`Operate Relay Production Cell Flags` (`set=admitMode=default`, `break-glass=BREAK GLASS <cell-id>`).
+
+An operator watches the run. A refused watch and a flip back that did not land (no ID token, a
+failed login, a cell that does not answer) can each leave the cell in reserve with nothing watching
+it, so both raise an `::error::` annotation naming the cell, as the clock-skew monitor does. In
+`write` mode both also log `orca_relay_reserve_watch_alarm` (log `orca-relay-reserve-watch`,
+resource `global`) with the deploy identity, and the `Orca Relay: reserve cell may be unwatched`
+alert pages on any such line. A failed login cannot page: the runner then has no credentials for
+`gcloud logging write`, so only the `::error::` and the operator watching the run report it. Apply
+the root Terraform that adds the workflow to the shared deploy identity's provider condition and
+grants that identity `roles/logging.logWriter` before the first `write` run; until then the
+workflow cannot log in and no alarm line can be written.
+
 ## Mobile push gateway
 
 `Deploy Push Gateway Production` (`.github/workflows/cloud-push-deploy.yml`) is the deploy path

cloud/infra/terraform/environments/staging.tfvars

@@ -48,8 +48,8 @@ relay_gce_cells = {
     boot_disk_gb      = 30
     boot_image        = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-7"
     capacity_requests = 4000
-    # Step-5 image F (main 3c1141f2). Never 2d0f6e6: it cannot report capped telemetry.
-    image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:41cba995b288cab70579882ebef23e74c5ee426b6f8924cea7c986aba23bcbfe"
+    # Step-5 image G (main 2de73812, reports preAuthLatencyMs). Never 2d0f6e6: it cannot report capped telemetry.
+    image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:07b5bf67a7706b2a61ef5e9a6d8ce85ee37c2a08609496a94eb5254881b5b87d"
     # Step-5 staging e2e: c1 is the second US reserve cell, so it needs a cap and rehome trust.
     connection_hard_cap         = 600
     connection_unobserved_bound = 60
@@ -86,7 +86,7 @@ relay_gce_cells = {
     boot_image                  = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21"
     capacity_requests           = 6000
     database_pool_max           = 10
-    image                       = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:41cba995b288cab70579882ebef23e74c5ee426b6f8924cea7c986aba23bcbfe"
+    image                       = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:07b5bf67a7706b2a61ef5e9a6d8ce85ee37c2a08609496a94eb5254881b5b87d"
     initially_enabled           = false
     connection_hard_cap         = 3000
     connection_unobserved_bound = 60
@@ -100,7 +100,7 @@ relay_gce_cells = {
     boot_disk_gb                = 30
     boot_image                  = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-7"
     capacity_requests           = 4000
-    image                       = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:41cba995b288cab70579882ebef23e74c5ee426b6f8924cea7c986aba23bcbfe"
+    image                       = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:07b5bf67a7706b2a61ef5e9a6d8ce85ee37c2a08609496a94eb5254881b5b87d"
     connection_hard_cap         = 600
     connection_unobserved_bound = 60
   }
@@ -111,7 +111,7 @@ relay_gce_cells = {
     boot_disk_gb                = 30
     boot_image                  = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-7"
     capacity_requests           = 4000
-    image                       = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:41cba995b288cab70579882ebef23e74c5ee426b6f8924cea7c986aba23bcbfe"
+    image                       = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:07b5bf67a7706b2a61ef5e9a6d8ce85ee37c2a08609496a94eb5254881b5b87d"
     connection_hard_cap         = 600
     connection_unobserved_bound = 60
   }
@@ -122,7 +122,7 @@ relay_gce_cells = {
     boot_disk_gb                = 30
     boot_image                  = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-7"
     capacity_requests           = 4000
-    image                       = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:41cba995b288cab70579882ebef23e74c5ee426b6f8924cea7c986aba23bcbfe"
+    image                       = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:07b5bf67a7706b2a61ef5e9a6d8ce85ee37c2a08609496a94eb5254881b5b87d"
     connection_hard_cap         = 600
     connection_unobserved_bound = 60
   }

cloud/infra/terraform/relay-github-actions.tf

@@ -20,6 +20,7 @@ locals {
     "deploy-relay-production.yml",
     "operate-relay-asia-admission.yml",
     "operate-relay-production-cell-flags.yml",
+    "operate-relay-production-reserve-watch.yml",
     "publish-relay-production.yml"
 
   ]
@@ -380,6 +381,16 @@ resource "google_project_iam_member" "github_logging_viewer" {
   member  = google_service_account.github_deploy[0].member
 }
 
+# The reserve watch logs its paging alarm line (orca_relay_reserve_watch_alarm) from the runner.
+# Accepted risk: Cloud Logging cannot narrow writes to one log, so this identity can write any line.
+resource "google_project_iam_member" "github_log_writer" {
+  count = local.relay_create_production_ops_identity ? 1 : 0
+
+  project = var.project_id
+  role    = "roles/logging.logWriter"
+  member  = google_service_account.github_deploy[0].member
+}
+
 resource "google_project_iam_member" "github_cloudsql_viewer" {
   count = local.relay_create_production_ops_identity ? 1 : 0
 

cloud/infra/terraform/relay-observability.tf

@@ -1748,6 +1748,74 @@ resource "google_monitoring_alert_policy" "relay_cell_reserve_dead_man_tripped"
   depends_on = [google_logging_metric.relay_cell_reserve_dead_man_tripped]
 }
 
+# The reserve watch workflow logs this from its runner (resource global) when a watch is refused or
+# a flip back did not land in write mode: either can leave a cell in reserve unwatched.
+resource "google_logging_metric" "relay_reserve_watch_alarm" {
+  project     = var.project_id
+  name        = "orca_relay_reserve_watch_alarm"
+  description = "Reserve watch runs that were refused or whose flip back to db did not land, so the cell may be in reserve mode with nothing watching it."
+  filter      = "resource.type=\"global\" AND logName=\"projects/${var.project_id}/logs/orca-relay-reserve-watch\" AND jsonPayload.event=\"orca_relay_reserve_watch_alarm\""
+  label_extractors = {
+    cell_id = "EXTRACT(jsonPayload.cellId)"
+    kind    = "EXTRACT(jsonPayload.kind)"
+  }
+
+  metric_descriptor {
+    metric_kind = "DELTA"
+    value_type  = "INT64"
+    unit        = "1"
+
+    labels {
+      key         = "cell_id"
+      value_type  = "STRING"
+      description = "Durable relay cell identifier."
+    }
+
+    labels {
+      key         = "kind"
+      value_type  = "STRING"
+      description = "refused or flip-back-failed."
+    }
+  }
+}
+
+resource "google_monitoring_alert_policy" "relay_reserve_watch_alarm" {
+  project               = var.project_id
+  display_name          = "Orca Relay: reserve cell may be unwatched"
+  combiner              = "OR"
+  enabled               = true
+  notification_channels = var.relay_alert_notification_channels
+
+  conditions {
+    display_name = "A reserve watch was refused or its flip back did not land"
+
+    condition_threshold {
+      filter          = "resource.type=\"global\" AND metric.type=\"logging.googleapis.com/user/orca_relay_reserve_watch_alarm\""
+      comparison      = "COMPARISON_GT"
+      threshold_value = 0
+      duration        = "0s"
+
+      aggregations {
+        alignment_period     = "300s"
+        per_series_aligner   = "ALIGN_SUM"
+        cross_series_reducer = "REDUCE_SUM"
+        group_by_fields      = ["metric.label.\"cell_id\""]
+      }
+
+      trigger {
+        count = 1
+      }
+    }
+  }
+
+  documentation {
+    content   = "A write-mode `Operate Relay Production Reserve Watch` run was refused (a rollout held the Cloud SQL rollout lease, the lease was unreadable, bad inputs, or the cell never reported reserve within 10 minutes) or its flip back to db did not land (no ID token, a failed login, or a cell that does not answer). The cell may be in reserve mode with nothing watching it. Read the cell's `runtime-status`; if its switch file says reserve, either rerun the watch or run `cloud-operate-relay-production-cell-flags.yml` with `admitMode=db` for that cell, using the break-glass inputs the failed run printed if the cell does not answer."
+    mime_type = "text/markdown"
+  }
+
+  depends_on = [google_logging_metric.relay_reserve_watch_alarm]
+}
+
 # host-session-registry.ts: a flip back closed a control it could not re-register (4409), after 20 refusals.
 resource "google_logging_metric" "relay_cell_reregistration_gave_up" {
   project     = var.project_id

cloud/package.json

@@ -1,12 +1,7 @@
 {
   "name": "orca-cloud",
-  "private": true,
   "version": "0.0.0",
-  "packageManager": "pnpm@10.34.6",
-  "engines": {
-    "node": ">=24 <27",
-    "pnpm": ">=10"
-  },
+  "private": true,
   "scripts": {
     "build": "pnpm -r build",
     "dev": "pnpm --filter @orca-cloud/relay dev",
@@ -21,14 +16,21 @@
     "load:relay:model": "node dev/scripts/run-relay-load-model.mjs",
     "load:relay:recovery-gate": "node dev/scripts/run-relay-recovery-wave-gate.mjs",
     "ops:relay": "pnpm --filter @orca-cloud/relay-ops dev",
-    "pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-control-flag-alerts.test.mjs dev/scripts/relay-db-fault-alerts.test.mjs dev/scripts/relay-lock-contention-alerts.test.mjs dev/scripts/relay-reconnect-herd-alerts.test.mjs dev/scripts/relay-reserve-dead-man-alert.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs",
-    "test": "pnpm -r test && node --test dev/scripts/check-relay-same-cap-headroom.test.mjs dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/drive-relay-director-deploy.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/measure-relay-drain-disconnect-gap.test.mjs dev/scripts/operate-relay-cell-flags.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/push-gateway-workflow.test.mjs dev/scripts/push-gateway-recovery.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-duplicate-assign.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-load-splice-retry.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-job-mode-conditions.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-same-cap-shadow-gate.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs",
-    "typecheck": "pnpm -r typecheck"
+    "pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-control-flag-alerts.test.mjs dev/scripts/relay-db-fault-alerts.test.mjs dev/scripts/relay-lock-contention-alerts.test.mjs dev/scripts/relay-reconnect-herd-alerts.test.mjs dev/scripts/relay-reserve-dead-man-alert.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs && pnpm test:runner",
+    "test": "pnpm -r --filter \"!@orca-cloud/relay\" test && pnpm --filter @orca-cloud/relay test && node --test dev/scripts/check-relay-same-cap-headroom.test.mjs dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/drive-relay-director-deploy.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/measure-relay-drain-disconnect-gap.test.mjs dev/scripts/operate-relay-cell-flags.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/push-gateway-workflow.test.mjs dev/scripts/push-gateway-recovery.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-duplicate-assign.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-load-splice-retry.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-job-mode-conditions.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-same-cap-shadow-gate.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/watch-relay-reserve-cell.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs",
+    "typecheck": "pnpm -r typecheck",
+    "test:runner": "node --import tsx --conditions=orca-source --test dev/scripts/run-relay-tests.test.mjs"
   },
   "devDependencies": {
+    "@orca/process-host": "link:../src/packages/process-host",
     "@types/node": "^24.19.0",
     "tsx": "^4.23.15",
     "typescript": "^5.9.3",
     "vitest": "^4.1.11"
-  }
+  },
+  "engines": {
+    "node": ">=24 <27",
+    "pnpm": ">=10"
+  },
+  "packageManager": "pnpm@10.34.6"
 }

cloud/pnpm-lock.yaml

@@ -8,6 +8,9 @@ importers:
 
   .:
     devDependencies:
+      '@orca/process-host':
+        specifier: link:../src/packages/process-host
+        version: link:../src/packages/process-host
       '@types/node':
         specifier: ^24.19.0
         version: 24.19.0

config/patches/i18next-cli@1.74.2.patch

@@ -1,5 +1,5 @@
 diff --git a/dist/cjs/extractor/core/extractor.js b/dist/cjs/extractor/core/extractor.js
-index 6d61ade2471c20bf1a253e3784bfadcb01440545..c1b001e78a215f89a8d4da1756a87562a5a6e723 100644
+index 6d61ade2471..23ad9f5ba49 100644
 --- a/dist/cjs/extractor/core/extractor.js
 +++ b/dist/cjs/extractor/core/extractor.js
 @@ -143,6 +143,8 @@ const extractionSiteRegexes = new WeakMap();
@@ -20,8 +20,185 @@ index 6d61ade2471c20bf1a253e3784bfadcb01440545..c1b001e78a215f89a8d4da1756a87562
          extractionSiteRegexes.set(config, re);
      }
      return re.test(code);
+@@ -185,13 +187,15 @@ function mayContainExtractionSite(code, config) {
+  *
+  * @internal
+  */
+-async function processFile(file, plugins, astVisitors, pluginContext, config, logger$1 = new logger.ConsoleLogger(), fileErrors) {
++async function processFile(file, plugins, astVisitors, pluginContext, config, logger$1 = new logger.ConsoleLogger(), fileErrors, parsedFiles) {
+     try {
+         // Skip directories that happen to match file-extension globs (e.g. a directory named "Foo.tsx")
+         const fileStat = await promises.stat(file);
+         if (fileStat.isDirectory())
+             return;
+-        let code = await promises.readFile(file, 'utf-8');
++        const cached = parsedFiles?.get(file);
++        parsedFiles?.delete(file);
++        let code = cached?.code ?? await promises.readFile(file, 'utf-8');
+         // Run onLoad hooks from plugins with error handling.
+         // Track whether any plugin actually transformed the code so we can make
+         // the right parse decision for non-native file types (e.g. .svelte, .vue).
+@@ -241,59 +245,61 @@ async function processFile(file, plugins, astVisitors, pluginContext, config, lo
+         const isTypeScriptFile = effectiveExt === '.ts' || effectiveExt === '.tsx' || effectiveExt === '.mts' || effectiveExt === '.cts';
+         const isTSX = effectiveExt === '.tsx';
+         const isJSX = effectiveExt === '.jsx';
+-        let ast;
+-        try {
+-            ast = await core.parse(code, {
+-                syntax: isTypeScriptFile ? 'typescript' : 'ecmascript',
+-                tsx: isTSX,
+-                jsx: isJSX,
+-                decorators: true,
+-                dynamicImport: true,
+-                comments: true,
+-            });
+-        }
+-        catch (err) {
+-            // Fallback for .ts files with JSX (already present)
+-            if (effectiveExt === '.ts' && !isTSX) {
+-                try {
+-                    ast = await core.parse(code, {
+-                        syntax: 'typescript',
+-                        tsx: true,
+-                        decorators: true,
+-                        dynamicImport: true,
+-                        comments: true,
+-                    });
+-                    logger$1.info?.(`Parsed ${file} using TSX fallback`);
+-                }
+-                catch (err2) {
+-                    throw new validation.ExtractorError('Failed to process file', file, err2);
+-                }
+-                // Fallback for .js files with JSX
++        let ast = cached?.ast;
++        if (!cached) {
++            try {
++                ast = await core.parse(code, {
++                    syntax: isTypeScriptFile ? 'typescript' : 'ecmascript',
++                    tsx: isTSX,
++                    jsx: isJSX,
++                    decorators: true,
++                    dynamicImport: true,
++                    comments: true,
++                });
+             }
+-            else if (effectiveExt === '.js' && !isJSX) {
+-                try {
+-                    ast = await core.parse(code, {
+-                        syntax: 'ecmascript',
+-                        jsx: true,
+-                        decorators: true,
+-                        dynamicImport: true,
+-                        comments: true,
+-                    });
+-                    logger$1.info?.(`Parsed ${file} using JSX fallback`);
++            catch (err) {
++                // Fallback for .ts files with JSX (already present)
++                if (effectiveExt === '.ts' && !isTSX) {
++                    try {
++                        ast = await core.parse(code, {
++                            syntax: 'typescript',
++                            tsx: true,
++                            decorators: true,
++                            dynamicImport: true,
++                            comments: true,
++                        });
++                        logger$1.info?.(`Parsed ${file} using TSX fallback`);
++                    }
++                    catch (err2) {
++                        throw new validation.ExtractorError('Failed to process file', file, err2);
++                    }
++                    // Fallback for .js files with JSX
+                 }
+-                catch (err2) {
+-                    throw new validation.ExtractorError('Failed to process file', file, err2);
++                else if (effectiveExt === '.js' && !isJSX) {
++                    try {
++                        ast = await core.parse(code, {
++                            syntax: 'ecmascript',
++                            jsx: true,
++                            decorators: true,
++                            dynamicImport: true,
++                            comments: true,
++                        });
++                        logger$1.info?.(`Parsed ${file} using JSX fallback`);
++                    }
++                    catch (err2) {
++                        throw new validation.ExtractorError('Failed to process file', file, err2);
++                    }
++                }
++                else {
++                    throw new validation.ExtractorError('Failed to process file', file, err);
+                 }
+             }
+-            else {
+-                throw new validation.ExtractorError('Failed to process file', file, err);
+-            }
++            // Normalize SWC span offsets so every span is a file-relative (0-based)
++            // character index.  SWC accumulates byte offsets across successive parse()
++            // calls and Module.span.start points to the first token, NOT to byte 0 of
++            // the source.
++            astUtils.normalizeSpansToCharIndices(ast, code);
+         }
+-        // Normalize SWC span offsets so every span is a file-relative (0-based)
+-        // character index.  SWC accumulates byte offsets across successive parse()
+-        // calls and Module.span.start points to the first token, NOT to byte 0 of
+-        // the source.
+-        astUtils.normalizeSpansToCharIndices(ast, code);
+         // "Wire up" the visitor's scope method to the context.
+         // This avoids a circular dependency while giving plugins access to the scope.
+         pluginContext.getVarFromScope = astVisitors.getVarFromScope.bind(astVisitors);
+@@ -344,7 +350,7 @@ async function processFile(file, plugins, astVisitors, pluginContext, config, lo
+  * @param logger - Logger for warnings/errors
+  * @param fileErrors - Optional array to collect per-file error messages
+  */
+-async function preScanFile(file, astVisitors, config, logger$1 = new logger.ConsoleLogger(), fileErrors) {
++async function preScanFile(file, astVisitors, config, logger$1 = new logger.ConsoleLogger(), fileErrors, parsedFiles) {
+     try {
+         // Skip directories that happen to match file-extension globs (e.g. a directory named "Foo.tsx")
+         const fileStat = await promises.stat(file);
+@@ -396,6 +402,8 @@ async function preScanFile(file, astVisitors, config, logger$1 = new logger.Cons
+         astUtils.normalizeSpansToCharIndices(ast, code);
+         astVisitors.setCurrentFile(file, code);
+         astVisitors.preScanForConstants(ast);
++        if (parsedFiles && mayContainExtractionSite(code, config))
++            parsedFiles.set(file, { code, ast });
+     }
+     catch (error) {
+         if (error instanceof pluginManager.ConflictError)
+diff --git a/dist/cjs/extractor/core/key-finder.js b/dist/cjs/extractor/core/key-finder.js
+index 1411f287157..200b6b974b6 100644
+--- a/dist/cjs/extractor/core/key-finder.js
++++ b/dist/cjs/extractor/core/key-finder.js
+@@ -40,6 +40,8 @@ async function findKeys(config, logger$1 = new logger.ConsoleLogger(), fileError
+     const plugins = pluginsOrUndefined || [];
+     const sourceFiles = await processSourceFiles(config);
+     const allKeys = new Map();
++    // Retain only translation-site trees; plugins can transform code between passes.
++    const parsedFiles = plugins.length === 0 ? new Map() : undefined;
+     // 1. Create the base context with config and logger.
+     const pluginContext = pluginManager.createPluginContext(allKeys, plugins, otherConfig, logger$1);
+     // 2. Create hooks for plugins to hook into AST
+@@ -92,7 +94,7 @@ async function findKeys(config, logger$1 = new logger.ConsoleLogger(), fileError
+     //    correctly even when the defining constants file is processed after the file that
+     //    uses the identifier.
+     for (const file of sourceFiles) {
+-        await extractor.preScanFile(file, astVisitors$1, otherConfig, logger$1, fileErrors);
++        await extractor.preScanFile(file, astVisitors$1, otherConfig, logger$1, fileErrors, parsedFiles);
+     }
+     // 6b. Mirror declarations onto the local names they were imported as, so that
+     //     `import { ResourceStatus as Status }` resolves like the unaliased import.
+@@ -100,7 +102,7 @@ async function findKeys(config, logger$1 = new logger.ConsoleLogger(), fileError
+     // 7. Extraction pass: all shared tables are now fully populated, so every
+     //    identifier reference can be resolved regardless of file order.
+     for (const file of sourceFiles) {
+-        await extractor.processFile(file, plugins, astVisitors$1, pluginContext, otherConfig, logger$1, fileErrors);
++        await extractor.processFile(file, plugins, astVisitors$1, pluginContext, otherConfig, logger$1, fileErrors, parsedFiles);
+     }
+     // 7. Run onEnd hooks
+     for (const plugin of plugins) {
 diff --git a/dist/esm/extractor/core/extractor.js b/dist/esm/extractor/core/extractor.js
-index 910bdb6e25e26c01ea6127191f1a00c7089693c1..00da4a6a7acb6c3da8d4b6fe7be106c409542f8d 100644
+index 910bdb6e25e..a3868d29950 100644
 --- a/dist/esm/extractor/core/extractor.js
 +++ b/dist/esm/extractor/core/extractor.js
 @@ -141,6 +141,8 @@ const extractionSiteRegexes = new WeakMap();
@@ -42,3 +219,180 @@ index 910bdb6e25e26c01ea6127191f1a00c7089693c1..00da4a6a7acb6c3da8d4b6fe7be106c4
          extractionSiteRegexes.set(config, re);
      }
      return re.test(code);
+@@ -183,13 +185,15 @@ function mayContainExtractionSite(code, config) {
+  *
+  * @internal
+  */
+-async function processFile(file, plugins, astVisitors, pluginContext, config, logger = new ConsoleLogger(), fileErrors) {
++async function processFile(file, plugins, astVisitors, pluginContext, config, logger = new ConsoleLogger(), fileErrors, parsedFiles) {
+     try {
+         // Skip directories that happen to match file-extension globs (e.g. a directory named "Foo.tsx")
+         const fileStat = await stat(file);
+         if (fileStat.isDirectory())
+             return;
+-        let code = await readFile(file, 'utf-8');
++        const cached = parsedFiles?.get(file);
++        parsedFiles?.delete(file);
++        let code = cached?.code ?? await readFile(file, 'utf-8');
+         // Run onLoad hooks from plugins with error handling.
+         // Track whether any plugin actually transformed the code so we can make
+         // the right parse decision for non-native file types (e.g. .svelte, .vue).
+@@ -239,59 +243,61 @@ async function processFile(file, plugins, astVisitors, pluginContext, config, lo
+         const isTypeScriptFile = effectiveExt === '.ts' || effectiveExt === '.tsx' || effectiveExt === '.mts' || effectiveExt === '.cts';
+         const isTSX = effectiveExt === '.tsx';
+         const isJSX = effectiveExt === '.jsx';
+-        let ast;
+-        try {
+-            ast = await parse(code, {
+-                syntax: isTypeScriptFile ? 'typescript' : 'ecmascript',
+-                tsx: isTSX,
+-                jsx: isJSX,
+-                decorators: true,
+-                dynamicImport: true,
+-                comments: true,
+-            });
+-        }
+-        catch (err) {
+-            // Fallback for .ts files with JSX (already present)
+-            if (effectiveExt === '.ts' && !isTSX) {
+-                try {
+-                    ast = await parse(code, {
+-                        syntax: 'typescript',
+-                        tsx: true,
+-                        decorators: true,
+-                        dynamicImport: true,
+-                        comments: true,
+-                    });
+-                    logger.info?.(`Parsed ${file} using TSX fallback`);
+-                }
+-                catch (err2) {
+-                    throw new ExtractorError('Failed to process file', file, err2);
+-                }
+-                // Fallback for .js files with JSX
++        let ast = cached?.ast;
++        if (!cached) {
++            try {
++                ast = await parse(code, {
++                    syntax: isTypeScriptFile ? 'typescript' : 'ecmascript',
++                    tsx: isTSX,
++                    jsx: isJSX,
++                    decorators: true,
++                    dynamicImport: true,
++                    comments: true,
++                });
+             }
+-            else if (effectiveExt === '.js' && !isJSX) {
+-                try {
+-                    ast = await parse(code, {
+-                        syntax: 'ecmascript',
+-                        jsx: true,
+-                        decorators: true,
+-                        dynamicImport: true,
+-                        comments: true,
+-                    });
+-                    logger.info?.(`Parsed ${file} using JSX fallback`);
++            catch (err) {
++                // Fallback for .ts files with JSX (already present)
++                if (effectiveExt === '.ts' && !isTSX) {
++                    try {
++                        ast = await parse(code, {
++                            syntax: 'typescript',
++                            tsx: true,
++                            decorators: true,
++                            dynamicImport: true,
++                            comments: true,
++                        });
++                        logger.info?.(`Parsed ${file} using TSX fallback`);
++                    }
++                    catch (err2) {
++                        throw new ExtractorError('Failed to process file', file, err2);
++                    }
++                    // Fallback for .js files with JSX
+                 }
+-                catch (err2) {
+-                    throw new ExtractorError('Failed to process file', file, err2);
++                else if (effectiveExt === '.js' && !isJSX) {
++                    try {
++                        ast = await parse(code, {
++                            syntax: 'ecmascript',
++                            jsx: true,
++                            decorators: true,
++                            dynamicImport: true,
++                            comments: true,
++                        });
++                        logger.info?.(`Parsed ${file} using JSX fallback`);
++                    }
++                    catch (err2) {
++                        throw new ExtractorError('Failed to process file', file, err2);
++                    }
++                }
++                else {
++                    throw new ExtractorError('Failed to process file', file, err);
+                 }
+             }
+-            else {
+-                throw new ExtractorError('Failed to process file', file, err);
+-            }
++            // Normalize SWC span offsets so every span is a file-relative (0-based)
++            // character index.  SWC accumulates byte offsets across successive parse()
++            // calls and Module.span.start points to the first token, NOT to byte 0 of
++            // the source.
++            normalizeSpansToCharIndices(ast, code);
+         }
+-        // Normalize SWC span offsets so every span is a file-relative (0-based)
+-        // character index.  SWC accumulates byte offsets across successive parse()
+-        // calls and Module.span.start points to the first token, NOT to byte 0 of
+-        // the source.
+-        normalizeSpansToCharIndices(ast, code);
+         // "Wire up" the visitor's scope method to the context.
+         // This avoids a circular dependency while giving plugins access to the scope.
+         pluginContext.getVarFromScope = astVisitors.getVarFromScope.bind(astVisitors);
+@@ -342,7 +348,7 @@ async function processFile(file, plugins, astVisitors, pluginContext, config, lo
+  * @param logger - Logger for warnings/errors
+  * @param fileErrors - Optional array to collect per-file error messages
+  */
+-async function preScanFile(file, astVisitors, config, logger = new ConsoleLogger(), fileErrors) {
++async function preScanFile(file, astVisitors, config, logger = new ConsoleLogger(), fileErrors, parsedFiles) {
+     try {
+         // Skip directories that happen to match file-extension globs (e.g. a directory named "Foo.tsx")
+         const fileStat = await stat(file);
+@@ -394,6 +400,8 @@ async function preScanFile(file, astVisitors, config, logger = new ConsoleLogger
+         normalizeSpansToCharIndices(ast, code);
+         astVisitors.setCurrentFile(file, code);
+         astVisitors.preScanForConstants(ast);
++        if (parsedFiles && mayContainExtractionSite(code, config))
++            parsedFiles.set(file, { code, ast });
+     }
+     catch (error) {
+         if (error instanceof ConflictError)
+diff --git a/dist/esm/extractor/core/key-finder.js b/dist/esm/extractor/core/key-finder.js
+index 86612731b2d..4e724e84538 100644
+--- a/dist/esm/extractor/core/key-finder.js
++++ b/dist/esm/extractor/core/key-finder.js
+@@ -38,6 +38,8 @@ async function findKeys(config, logger = new ConsoleLogger(), fileErrors) {
+     const plugins = pluginsOrUndefined || [];
+     const sourceFiles = await processSourceFiles(config);
+     const allKeys = new Map();
++    // Retain only translation-site trees; plugins can transform code between passes.
++    const parsedFiles = plugins.length === 0 ? new Map() : undefined;
+     // 1. Create the base context with config and logger.
+     const pluginContext = createPluginContext(allKeys, plugins, otherConfig, logger);
+     // 2. Create hooks for plugins to hook into AST
+@@ -90,7 +92,7 @@ async function findKeys(config, logger = new ConsoleLogger(), fileErrors) {
+     //    correctly even when the defining constants file is processed after the file that
+     //    uses the identifier.
+     for (const file of sourceFiles) {
+-        await preScanFile(file, astVisitors, otherConfig, logger, fileErrors);
++        await preScanFile(file, astVisitors, otherConfig, logger, fileErrors, parsedFiles);
+     }
+     // 6b. Mirror declarations onto the local names they were imported as, so that
+     //     `import { ResourceStatus as Status }` resolves like the unaliased import.
+@@ -98,7 +100,7 @@ async function findKeys(config, logger = new ConsoleLogger(), fileErrors) {
+     // 7. Extraction pass: all shared tables are now fully populated, so every
+     //    identifier reference can be resolved regardless of file order.
+     for (const file of sourceFiles) {
+-        await processFile(file, plugins, astVisitors, pluginContext, otherConfig, logger, fileErrors);
++        await processFile(file, plugins, astVisitors, pluginContext, otherConfig, logger, fileErrors, parsedFiles);
+     }
+     // 7. Run onEnd hooks
+     for (const plugin of plugins) {

config/renderer-layout-writer-baseline.txt

@@ -15,7 +15,6 @@
 1 src/renderer/src/components/status-bar/workspace-space-decision-inputs.ts
 2 src/renderer/src/components/tab-bar/terminal-tab-activity-status.ts
 3 src/renderer/src/components/tab-bar/use-tab-bar-runtime-model-worktree-write-probe.ts
-23 src/renderer/src/components/tab-group/tab-drag-preview-activation.ts
 1 src/renderer/src/components/tab-group/useTabGroupWorkspaceModel.ts
 1 src/renderer/src/components/terminal-pane/pty-connection-test-deps.ts
 4 src/renderer/src/components/terminal-pane/terminal-recovery-ledger-test-store.ts
@@ -51,7 +50,6 @@
 1 src/renderer/src/runtime/web-session-terminal-orphan-recovery-surface.ts
 2 src/renderer/src/runtime/web-session-terminal-orphan-topology.ts
 3 src/renderer/src/runtime/web-session-terminal-recovery-snapshot-validation.ts
-9 src/renderer/src/store/repos/repo-removal.ts
 10 src/renderer/src/store/slices/browser/browser-close-actions.ts
 6 src/renderer/src/store/slices/browser/browser-hydration-actions.ts
 5 src/renderer/src/store/slices/browser/browser-page-focus-actions.ts
@@ -60,7 +58,6 @@
 4 src/renderer/src/store/slices/browser/browser-tab-focus-actions.ts
 2 src/renderer/src/store/slices/browser/create-browser-slice.ts
 1 src/renderer/src/store/slices/direct-ssh-terminal-recovery.ts
-1 src/renderer/src/store/slices/editor/actions/chat-visual-tab-actions.ts
 9 src/renderer/src/store/slices/editor/actions/check-run-details-actions.ts
 7 src/renderer/src/store/slices/editor/actions/close-file-action.ts
 1 src/renderer/src/store/slices/editor/actions/git-status-actions.ts
@@ -82,13 +79,13 @@
 2 src/renderer/src/store/slices/editor/file-ids/hydrated-editor-file-selection.ts
 1 src/renderer/src/store/slices/editor/file-ids/open-file-path-rekey.ts
 4 src/renderer/src/store/slices/editor/tabs/editor-open-target-group.ts
-2 src/renderer/src/store/slices/ssh-target-cleanup.ts
 2 src/renderer/src/store/slices/tab-group-reference-repair.ts
 4 src/renderer/src/store/slices/tab-group-state.ts
 2 src/renderer/src/store/slices/tabs-hydration.ts
 4 src/renderer/src/store/slices/tabs/create-tabs-slice.ts
 17 src/renderer/src/store/slices/tabs/tabs-close-actions.ts
 14 src/renderer/src/store/slices/tabs/tabs-create-actions.ts
+23 src/renderer/src/store/slices/tabs/tabs-drag-preview-actions.ts
 11 src/renderer/src/store/slices/tabs/tabs-drop-actions.ts
 4 src/renderer/src/store/slices/tabs/tabs-focus-actions.ts
 13 src/renderer/src/store/slices/tabs/tabs-group-actions.ts
@@ -96,17 +93,15 @@
 8 src/renderer/src/store/slices/tabs/tabs-label-actions.ts
 10 src/renderer/src/store/slices/tabs/tabs-move-actions.ts
 9 src/renderer/src/store/slices/tabs/tabs-reconciliation.ts
-1 src/renderer/src/store/slices/tabs/tabs-secondary-actions.ts
 4 src/renderer/src/store/slices/tabs/tabs-session-actions.ts
+1 src/renderer/src/store/slices/tabs/tabs-split-ratio-actions.ts
 5 src/renderer/src/store/slices/tabs/tabs-surface.ts
+33 src/renderer/src/store/slices/tabs/worktree-layout-lifecycle.ts
 6 src/renderer/src/store/slices/terminal-orphan-helpers.ts
 4 src/renderer/src/store/slices/terminal-tab-title-batch.ts
 6 src/renderer/src/store/slices/terminals.ts
-7 src/renderer/src/store/slices/worktrees/session/set-active-worktree.ts
-15 src/renderer/src/store/slices/worktrees/session/worktree-identity-rename-state.ts
+5 src/renderer/src/store/slices/worktrees/session/set-active-worktree.ts
 2 src/renderer/src/store/slices/worktrees/session/worktree-slice-lookups.ts
-18 src/renderer/src/store/slices/worktrees/teardown/remove-worktree-store-cleanup.ts
-18 src/renderer/src/store/slices/worktrees/teardown/worktree-purge-state.ts
 5 src/renderer/src/store/terminals/terminal-layout-state.ts
 2 src/renderer/src/store/terminals/terminal-pane-hibernation.ts
 3 src/renderer/src/store/terminals/terminal-pty-bindings.ts

config/scripts/build-wayland-clipboard.test.mjs

@@ -12,6 +12,8 @@ import { createRequire } from 'node:module'
 import { tmpdir } from 'node:os'
 import { dirname, join } from 'node:path'
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { parse } from 'yaml'
+import { runProcessSync } from '@orca/process-host'
 import { buildWaylandClipboard } from './build-wayland-clipboard.mjs'
 import glibcVerification from './verify-linux-glibc-floor.cjs'
 
@@ -148,6 +150,50 @@ describe('Wayland clipboard build', () => {
   })
 })
 
+describe('Wayland clipboard CI cache recovery', () => {
+  it.skipIf(process.platform === 'win32').each([
+    ['healthy cache', true, 0, 0, 1],
+    ['invalid cache', true, 1, 0, 2],
+    ['failed replacement build', true, 1, 1, 2],
+    ['failed uncached build', false, 1, 1, 1]
+  ])('%s preserves build validation and failures', (_label, hit, initial, rebuild, calls) => {
+    const action = parse(
+      readFileSync('.github/actions/prepare-wayland-clipboard/action.yml', 'utf8')
+    )
+    const build = action.runs.steps.find(
+      (step) => step.name === 'Build or validate Wayland clipboard helper'
+    )
+    mkdirSync(join(root, 'native', 'wayland-clipboard', '.build'), { recursive: true })
+    writeFileSync(join(root, 'native', 'wayland-clipboard', '.build', 'restored-cache'), '')
+    const script = `
+      node() {
+        echo attempt >> calls
+        if [ -e native/wayland-clipboard/.build/restored-cache ]; then
+          return "$INITIAL_STATUS"
+        fi
+        return "$REBUILD_STATUS"
+      }
+      ${build.run}
+    `
+    const result = runProcessSync({
+      program: 'bash',
+      args: ['-e', '-c', script],
+      cwd: root,
+      env: {
+        ...process.env,
+        CACHE_HIT: String(hit),
+        INITIAL_STATUS: String(initial),
+        REBUILD_STATUS: String(rebuild)
+      }
+    })
+    expect(result.code).toBe(initial === 0 ? 0 : hit ? rebuild : initial)
+    expect(readFileSync(join(root, 'calls'), 'utf8').trim().split('\n')).toHaveLength(calls)
+    expect(existsSync(join(root, 'native', 'wayland-clipboard', '.build', 'restored-cache'))).toBe(
+      calls === 1
+    )
+  })
+})
+
 describe('Wayland clipboard packaging', () => {
   it.each([
     [1, 'x64'],

config/scripts/check-node-runtime-pin.test.mjs

@@ -189,7 +189,8 @@ describe('committed pin', () => {
 
   it('runs in the static analysis job', () => {
     const workflow = parse(readFileSync(path.join(projectDir, '.github/workflows/pr.yml'), 'utf8'))
+    const packageJson = JSON.parse(readFileSync(path.join(projectDir, 'package.json'), 'utf8'))
     const commands = workflow.jobs.preflight.steps.map((step) => step.run ?? '')
-    expect(commands).toContain('pnpm run check:node-runtime-pin')
+    expect(commands).toContain(packageJson.scripts['check:node-runtime-pin'])
   })
 })

config/scripts/ci-e2e-job-selection.test.mjs

@@ -97,7 +97,7 @@ it('keeps Electron build and native prerequisites for every consumer requiring t
   }
   expect(workflow.jobs['ssh-browser-network-route'].needs).toBeUndefined()
   for (const name of ['e2e', 'changed-e2e']) {
-    expect(workflow.jobs[name].needs, name).toEqual(['build', 'prepare-native-cache'])
+    expect(workflow.jobs[name].needs, name).toEqual(['build'])
   }
 })
 
@@ -132,11 +132,9 @@ it('applies allocation hints only to PRs and retains other callers and full refe
       default: true
     })
   }
-  for (const name of ['build', 'prepare-native-cache']) {
-    expect(workflow.jobs[name].if).toBe(
-      "inputs.test_files == '' || github.event_name != 'pull_request' || inputs.needs_build"
-    )
-  }
+  expect(workflow.jobs.build.if).toBe(
+    "inputs.test_files == '' || github.event_name != 'pull_request' || inputs.needs_build"
+  )
   expect(workflow.jobs.e2e.if).toBe("inputs.test_files == ''")
   const changed = workflow.jobs['changed-e2e']
   expect(changed.if).toContain("github.event_name != 'pull_request' || inputs.run_changed_e2e")

config/scripts/ci-native-cache.test.mjs

@@ -1,4 +1,4 @@
-import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { parse } from 'yaml'
@@ -10,6 +10,7 @@ const installerPath = '.github/actions/install-node-dependencies/action.yml'
 const nativePath = '.github/actions/prepare-native-runtime/action.yml'
 const installer = parse(readFileSync(installerPath, 'utf8'))
 const native = parse(readFileSync(nativePath, 'utf8'))
+const e2e = parse(readFileSync('.github/workflows/e2e.yml', 'utf8'))
 const identity = native.runs.steps.find((step) => step.id === 'native-cache-scope')
 const inputs = [...identity.env.NATIVE_SOURCE_HASH.matchAll(/'([^']+)'/g)].map((match) => match[1])
 const paths = [
@@ -176,4 +177,138 @@ describe('CI native cache ownership', () => {
       }
     }
   )
+
+  it('prepares both native ABIs in the E2E build before releasing its consumers', () => {
+    expect(e2e.jobs['prepare-native-cache']).toBeUndefined()
+    const { steps } = e2e.jobs.build
+    const install = steps.find((step) => step.id === 'deps')
+    const save = steps.find((step) => step.name === 'Save compiled Node native modules')
+    const build = steps.find((step) => step.name === 'Build E2E outputs')
+    const cliWait = steps.find((step) => step.wait === 'e2e-cli')
+    const electron = steps.find((step) => step.id === 'electron-native-runtime')
+    const upload = steps.find((step) => step.name === 'Upload E2E build output')
+    const uploadWait = steps.find((step) => step.wait === upload.id)
+    expect(install.with).toMatchObject({
+      'native-runtime': 'node',
+      'persist-native-cache': 'false',
+      'cache-electron-package': 'true'
+    })
+    expect(save.if).toContain("steps.deps.outputs.native-cache-key != ''")
+    expect(save.if).toContain("steps.deps.outputs.native-cache-hit != 'true'")
+    expect(save.with).toEqual({
+      key: '${{ steps.deps.outputs.native-cache-key }}',
+      path: '${{ steps.deps.outputs.native-cache-path }}'
+    })
+    const ordered = [install, save, build, cliWait, upload, electron, uploadWait]
+    expect(ordered.map((step) => steps.indexOf(step))).toEqual(
+      ordered.map((step) => steps.indexOf(step)).sort((left, right) => left - right)
+    )
+    expect(upload.background).toBe(true)
+    expect(upload.if).toBeUndefined()
+    expect(electron.background).toBeUndefined()
+    expect(electron.uses).toBe('./.github/actions/prepare-native-runtime')
+    expect(electron.with).toEqual({
+      'native-runtime': 'electron',
+      'node-version': '${{ steps.deps.outputs.node-version }}'
+    })
+    expect(uploadWait).toEqual({ wait: upload.id })
+    for (const name of ['e2e', 'changed-e2e', 'orcad-serve-mode-switch', 'orcad-ssh-docker']) {
+      expect(e2e.jobs[name].needs, name).toEqual(['build'])
+      expect(
+        e2e.jobs[name].steps.find(
+          (step) => step.uses === './.github/actions/install-node-dependencies'
+        ).with['native-runtime'],
+        name
+      ).toBe('electron')
+    }
+  })
+
+  it.skipIf(process.platform === 'win32').each([false, true])(
+    'detects shared native action availability=%s for older dispatch refs',
+    (available) => {
+      const directory = mkdtempSync(join(tmpdir(), 'orca-e2e-native-action-'))
+      try {
+        const output = join(directory, 'output')
+        if (available) {
+          mkdirSync(join(directory, '.github/actions/prepare-native-runtime'), { recursive: true })
+          writeFileSync(join(directory, nativePath), '')
+        }
+        const step = e2e.jobs.build.steps.find(
+          (candidate) => candidate.id === 'native-runtime-action'
+        )
+        const execution = runProcessSync({
+          program: 'bash',
+          args: ['-e', '-o', 'pipefail', '-c', step.run],
+          cwd: directory,
+          env: { ...process.env, GITHUB_OUTPUT: output },
+          timeoutMs: 10_000
+        })
+        expect(execution.code, execution.stderr).toBe(0)
+        expect(readFileSync(output, 'utf8')).toBe(`available=${available}\n`)
+        const fallback = e2e.jobs.build.steps.find(
+          (candidate) => candidate.name === 'Prepare Electron natives for older refs'
+        )
+        expect(fallback.if).toBe(
+          "steps.native-runtime-action.outputs.available != 'true' || steps.deps.outputs.node-version == ''"
+        )
+        expect(fallback.uses).toBe('./.github/actions/install-node-dependencies')
+        expect(fallback.with['native-runtime']).toBe('electron')
+      } finally {
+        rmSync(directory, { recursive: true, force: true })
+      }
+    }
+  )
+
+  it.skipIf(process.platform === 'win32').each([
+    ['true', 'build-essential python3', '', 0, false],
+    ['false', 'build-essential python3', '', 0, true],
+    ['true', 'build-essential python3 xvfb', '', 0, true],
+    ['false', 'build-essential python3', 'update', 7, true],
+    ['false', 'build-essential python3', 'install', 7, true]
+  ])(
+    'tools=%s packages=%s failure=%s retains bounded setup',
+    (present, packages, failure, status, installs) => {
+      const directory = mkdtempSync(join(tmpdir(), 'orca-e2e-native-tools-'))
+      try {
+        const log = join(directory, 'commands')
+        writeFileSync(log, '')
+        const step = e2e.jobs.build.steps.find(
+          (candidate) => candidate.name === 'Install native build tools'
+        )
+        const execution = runProcessSync({
+          program: 'bash',
+          args: [
+            '-e',
+            '-o',
+            'pipefail',
+            '-c',
+            `
+            command() { [ "$TOOLS_PRESENT" = true ]; }
+            sudo() { printf 'sudo %s\\n' "$*" >> "$COMMAND_LOG"; cat >/dev/null; }
+            timeout() {
+              printf 'timeout %s\\n' "$*" >> "$COMMAND_LOG"
+              if [ "$4" = "$FAIL_OPERATION" ]; then return 7; fi
+            }
+            ${step.run}`
+          ],
+          env: {
+            ...process.env,
+            TOOLS_PRESENT: present,
+            ORCA_E2E_APT_PACKAGES: packages,
+            FAIL_OPERATION: failure,
+            COMMAND_LOG: log
+          },
+          timeoutMs: 10_000
+        })
+        expect(execution.code, execution.stderr).toBe(status)
+        const commands = readFileSync(log, 'utf8')
+        expect(commands.includes('timeout 120 sudo apt-get update')).toBe(installs)
+        expect(commands.includes(`timeout 300 sudo apt-get install -y ${packages}`)).toBe(
+          installs && failure !== 'update'
+        )
+      } finally {
+        rmSync(directory, { recursive: true, force: true })
+      }
+    }
+  )
 })

config/scripts/ci-shard-timings.md

@@ -1,6 +1,6 @@
 # Timing-based CI shards
 
-The eight unit shards and fourteen general E2E shards use longest-processing-time
+Unit shards and general E2E shards use longest-processing-time
 assignment of whole files to the currently lightest shard. Ties use file path and
 then shard index, independent of filesystem enumeration and locale. Unknown,
 zero, or invalid durations use the baseline's positive median (1 second when no
@@ -46,7 +46,32 @@ age as specs change. Full CI runs on the existing runner classes are required to
 measure elapsed-time and occupancy improvements, including discovery overhead.
 No retries, assertions, coverage exclusions, runner classes or shard counts changed.
 
-## Current unit refresh: September 27, 2026
+## Current unit refresh: October 14, 2026
+
+The baseline imports all five successful ARM unit shards from
+[run 38646706136, attempt 1](https://github.com/stablyai/orca/actions/runs/38646706136/attempts/1).
+The tested merge was `8d7f46198b99432efd104ac7803174173b367c66`, using Node
+24.21.0, Bun 1.4.3, `ubuntu-24.04-arm`, and four workers per shard. All 12,314
+files appear exactly once: 12,163 passed and 151 skipped, with zero unhandled
+errors. Artifact digests, all 20 report hashes, source/runtime provenance, full
+discovery, and each saved assignment were checked before importing. The E2E
+weights remain unchanged.
+
+| Shard | Job ID | Measured worker-seconds | Job seconds |
+| --- | --- | ---: | ---: |
+| 1 | 116137259336 | 1,966.470 | 568 |
+| 2 | 116137259460 | 2,080.624 | 604 |
+| 3 | 116137259318 | 2,121.495 | 611 |
+| 4 | 116137259783 | 2,094.487 | 614 |
+| 5 | 116137259540 | 2,190.636 | 641 |
+
+Retrospectively assigning those measurements projects 2,090.726–2,090.747
+worker-seconds per shard, instead of 1,966.470–2,190.636. That is a scheduling
+projection, not a hosted speedup: module times overlap across workers, and total
+runner occupancy was 50.63 minutes. Runner capacity, selection, workers, and the
+assignment algorithm are unchanged. Later source changes need fresh validation.
+
+## Historical unit refresh: September 27, 2026
 
 The baseline imports every successful Node 24 shard from
 [run 36294142683, attempt 1](https://github.com/stablyai/orca/actions/runs/36294142683/attempts/1).

config/scripts/ci-unit-files.mjs

@@ -35,6 +35,16 @@ export const UNIT_EXCLUDE = [
   'src/shared/posix-command-path-lookup.test.ts',
   'tests/e2e/relay-region-compatibility.unit.test.ts',
   'tests/e2e/relay-region-correction.unit.test.ts',
+  // Electron probes with in-process deadlines; the package job runs them serially.
+  'src/main/browser/browser-client-page-renderer-lifecycle.electron.test.ts',
+  'src/main/browser/browser-route-tcp-egress.electron.test.ts',
+  'src/main/browser/browser-route-webrtc-egress.electron.test.ts',
+  'src/main/browser/browser-route-h3-egress.electron.test.ts',
+  'src/main/browser/browser-route-dns-prefetch.electron.test.ts',
+  'src/main/persistence/profile-state/profile-state-writer-stall.electron.test.ts',
+  'src/main/browser/browser-popup-compatibility.electron.test.ts',
+  'src/main/browser/browser-popup-native-compatibility.electron.test.ts',
+  'src/main/browser/browser-popup-background-compatibility.electron.test.ts',
   `${CROSS_VERSION_WIRE_DIR}**`
 ]
 

config/scripts/ci-unit-module-cache-contents.mjs

@@ -0,0 +1,217 @@
+import { createHash } from 'node:crypto'
+import { constants } from 'node:fs'
+import { lstat, open, readdir } from 'node:fs/promises'
+import { join } from 'node:path'
+import { checkModuleTrailer } from './ci-unit-module-cache-trailer.mjs'
+
+export const MODULE_CACHE_MANIFEST = '_orca_manifest.json'
+export const MODULE_CACHE_LIMITS = Object.freeze({
+  entries: 50_000,
+  totalBytes: 2 ** 30,
+  entryBytes: 32 * 2 ** 20
+})
+const METADATA = '_metadata.json'
+const MODULE_NAME = /^[a-f0-9]{40}$/
+const isRecord = (value) => value !== null && typeof value === 'object' && !Array.isArray(value)
+const fail = (message) => {
+  throw new Error(`Unit module cache: ${message}`)
+}
+
+async function readRegularFile(path) {
+  const before = await lstat(path)
+  if (!before.isFile()) {
+    fail(`not a regular file: ${path}`)
+  }
+  const file = await open(
+    path,
+    constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0) | (constants.O_NONBLOCK ?? 0)
+  )
+  try {
+    const info = await file.stat()
+    if (!info.isFile() || info.size > MODULE_CACHE_LIMITS.entryBytes) {
+      fail(`invalid file size or type: ${path}`)
+    }
+    const buffer = Buffer.alloc(info.size + 1)
+    let bytes = 0
+    while (bytes < buffer.length) {
+      const result = await file.read(buffer, bytes, buffer.length - bytes, bytes)
+      if (!result.bytesRead) {
+        break
+      }
+      bytes += result.bytesRead
+    }
+    if (bytes !== info.size) {
+      fail(`file changed while reading: ${path}`)
+    }
+    return buffer.subarray(0, bytes)
+  } finally {
+    await file.close()
+  }
+}
+
+function checkLockfileHash(lockfileHash) {
+  if (typeof lockfileHash !== 'string' || !/^[a-f0-9]{8}$/.test(lockfileHash)) {
+    fail('invalid expected lockfile hash')
+  }
+}
+
+export async function inventoryModuleCache({ path, lockfileHash }) {
+  checkLockfileHash(lockfileHash)
+  if (!(await lstat(path)).isDirectory()) {
+    fail('cache root is not a regular directory')
+  }
+  const names = (await readdir(path)).sort()
+  if (names.length > MODULE_CACHE_LIMITS.entries) {
+    fail('too many cache entries')
+  }
+  if (!names.includes(METADATA)) {
+    fail('cache metadata is missing')
+  }
+  let totalBytes = 0
+  let manifestBytes = 0
+  const entries = Array.from({ length: names.length })
+  let next = 0
+  const workers = await Promise.allSettled(
+    Array.from({ length: Math.min(4, names.length) }, async () => {
+      while (next < names.length) {
+        const index = next++
+        const name = names[index]
+        if (name !== METADATA && name !== MODULE_CACHE_MANIFEST && !MODULE_NAME.test(name)) {
+          fail(`invalid cache entry name: ${name}`)
+        }
+        const buffer = await readRegularFile(join(path, name))
+        if (name === MODULE_CACHE_MANIFEST) {
+          manifestBytes = buffer.length
+        } else {
+          totalBytes += buffer.length
+        }
+        if (totalBytes + manifestBytes > MODULE_CACHE_LIMITS.totalBytes) {
+          fail('cache byte limit exceeded')
+        }
+        if (name === MODULE_CACHE_MANIFEST) {
+          continue
+        }
+        if (name === METADATA) {
+          const metadata = JSON.parse(buffer.toString('utf8'))
+          if (!isRecord(metadata) || metadata.lockfileHash !== lockfileHash) {
+            fail('cache lockfile hash mismatch')
+          }
+        } else {
+          checkModuleTrailer(buffer)
+        }
+        entries[index] = {
+          name,
+          bytes: buffer.length,
+          sha256: createHash('sha256').update(buffer).digest('hex')
+        }
+      }
+    })
+  )
+  const failure = workers.find((result) => result.status === 'rejected')
+  if (failure) {
+    throw failure.reason
+  }
+  return { entries: entries.filter(Boolean), totalBytes }
+}
+
+function checkIdentity({ key, format, versions, sourceSha, lockfileHash }) {
+  checkLockfileHash(lockfileHash)
+  if (typeof key !== 'string' || !key || typeof format !== 'string' || !format) {
+    fail('invalid cache identity')
+  }
+  if (
+    !isRecord(versions) ||
+    !['node', 'bun', 'vitest'].every((name) => typeof versions[name] === 'string' && versions[name])
+  ) {
+    fail('invalid cache versions')
+  }
+  if (sourceSha !== undefined && (typeof sourceSha !== 'string' || !MODULE_NAME.test(sourceSha))) {
+    fail('invalid source SHA')
+  }
+}
+
+export async function sealModuleCache(options) {
+  checkIdentity(options)
+  if (!options.sourceSha) {
+    fail('source SHA is required when sealing')
+  }
+  const inventory = await inventoryModuleCache(options)
+  const manifest = {
+    version: 1,
+    key: options.key,
+    format: options.format,
+    versions: options.versions,
+    sourceSha: options.sourceSha,
+    lockfileHash: options.lockfileHash,
+    ...inventory
+  }
+  const content = `${JSON.stringify(manifest)}\n`
+  if (
+    Buffer.byteLength(content) > MODULE_CACHE_LIMITS.entryBytes ||
+    inventory.totalBytes + Buffer.byteLength(content) > MODULE_CACHE_LIMITS.totalBytes ||
+    inventory.entries.length + 1 > MODULE_CACHE_LIMITS.entries
+  ) {
+    fail('sealed cache exceeds limits')
+  }
+  const file = await open(
+    join(options.path, MODULE_CACHE_MANIFEST),
+    constants.O_WRONLY |
+      constants.O_CREAT |
+      (constants.O_NOFOLLOW ?? 0) |
+      (constants.O_NONBLOCK ?? 0),
+    0o600
+  )
+  try {
+    if (!(await file.stat()).isFile()) {
+      fail('manifest is not a regular file')
+    }
+    await file.truncate(0)
+    await file.writeFile(content)
+  } finally {
+    await file.close()
+  }
+  return manifest
+}
+
+export async function validateModuleCache(options) {
+  checkIdentity(options)
+  const manifest = JSON.parse(
+    (await readRegularFile(join(options.path, MODULE_CACHE_MANIFEST))).toString('utf8')
+  )
+  if (!isRecord(manifest)) {
+    fail('invalid cache manifest')
+  }
+  checkIdentity(manifest)
+  if (
+    !manifest.sourceSha ||
+    manifest.version !== 1 ||
+    manifest.key !== options.key ||
+    manifest.format !== options.format ||
+    manifest.lockfileHash !== options.lockfileHash ||
+    (options.sourceSha !== undefined && manifest.sourceSha !== options.sourceSha) ||
+    !['node', 'bun', 'vitest'].every((name) => manifest.versions[name] === options.versions[name])
+  ) {
+    fail('cache manifest identity mismatch')
+  }
+  const inventory = await inventoryModuleCache(options)
+  if (
+    manifest.totalBytes !== inventory.totalBytes ||
+    !Array.isArray(manifest.entries) ||
+    manifest.entries.length !== inventory.entries.length
+  ) {
+    fail('cache inventory mismatch')
+  }
+  for (let index = 0; index < inventory.entries.length; index++) {
+    const expected = manifest.entries[index]
+    const actual = inventory.entries[index]
+    if (
+      !isRecord(expected) ||
+      expected.name !== actual.name ||
+      expected.bytes !== actual.bytes ||
+      expected.sha256 !== actual.sha256
+    ) {
+      fail(`cache content mismatch: ${actual.name}`)
+    }
+  }
+  return manifest
+}

config/scripts/ci-unit-module-cache-contents.test.mjs

@@ -0,0 +1,313 @@
+import {
+  mkdtemp,
+  mkdir,
+  readFile,
+  rename,
+  rm,
+  symlink,
+  truncate,
+  writeFile
+} from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, describe, expect, it } from 'vitest'
+import {
+  inventoryModuleCache,
+  MODULE_CACHE_LIMITS,
+  MODULE_CACHE_MANIFEST,
+  sealModuleCache,
+  validateModuleCache
+} from './ci-unit-module-cache-contents.mjs'
+
+const roots = []
+const filename = 'a'.repeat(40)
+const identity = {
+  key: 'unit-module-v1-Linux-ARM64-runtime-config-lock',
+  format: 'vitest-fs-1.0.0-beta.8',
+  versions: { node: '24.21.0', bun: '1.4.3', vitest: '5.0.3' },
+  sourceSha: 'b'.repeat(40),
+  lockfileHash: '0028fbb2'
+}
+
+function moduleTable() {
+  return [
+    {
+      file: '1',
+      id: '1',
+      url: '2',
+      imports: '3',
+      mappings: false,
+      deps: '4',
+      dynamicDeps: '5',
+      staticMocks: '6',
+      moduleType: '7'
+    },
+    '/workspace/src/main.ts',
+    '/src/main.ts',
+    { urls: '8', ids: '9' },
+    ['10'],
+    [],
+    ['11'],
+    'esm',
+    ['10'],
+    { '/dependency.ts': '12' },
+    '/dependency.ts',
+    { method: '13', specifier: '10', hasFactory: true, factoryLoadsOriginal: false },
+    '/workspace/dependency.ts',
+    'mock'
+  ]
+}
+
+const moduleCode = (table = moduleTable()) =>
+  `export const value = 1\n//# vitestCache=${Buffer.from(JSON.stringify(table)).toString('base64')}`
+
+async function fixture() {
+  const path = await mkdtemp(join(tmpdir(), 'orca-unit-module-cache-'))
+  roots.push(path)
+  await writeFile(join(path, filename), moduleCode())
+  await writeFile(
+    join(path, '_metadata.json'),
+    JSON.stringify({ lockfileHash: identity.lockfileHash })
+  )
+  return { path, ...identity }
+}
+
+async function changeManifest(options, change) {
+  const path = join(options.path, MODULE_CACHE_MANIFEST)
+  const manifest = JSON.parse(await readFile(path, 'utf8'))
+  change(manifest)
+  await writeFile(path, JSON.stringify(manifest))
+}
+
+afterEach(async () => {
+  await Promise.all(roots.splice(0).map((path) => rm(path, { recursive: true, force: true })))
+})
+
+describe('unit module cache contents', () => {
+  it('seals all transformed files and Vitest metadata with exact content hashes', async () => {
+    const options = await fixture()
+    const inventory = await inventoryModuleCache(options)
+    expect(inventory.entries.map((entry) => entry.name)).toEqual(['_metadata.json', filename])
+    expect(inventory.entries.every((entry) => /^[a-f0-9]{64}$/.test(entry.sha256))).toBe(true)
+    expect(inventory.totalBytes).toBe(
+      inventory.entries.reduce((sum, entry) => sum + entry.bytes, 0)
+    )
+    const manifest = await sealModuleCache(options)
+    expect(manifest).toMatchObject({ version: 1, ...identity, ...inventory })
+    expect(await validateModuleCache(options)).toEqual(manifest)
+    const restoreOptions = { ...options, sourceSha: undefined }
+    expect(await validateModuleCache(restoreOptions)).toEqual(manifest)
+  })
+
+  it('accepts absent optional module type, dependency lists and mocks', async () => {
+    const options = await fixture()
+    await writeFile(
+      join(options.path, filename),
+      moduleCode([
+        { id: '1', url: '2', file: null, imports: '3', mappings: false },
+        '\u0000virtual-module',
+        '/@id/virtual-module',
+        { urls: '4', ids: '5' },
+        [],
+        {}
+      ])
+    )
+    await sealModuleCache(options)
+    await expect(validateModuleCache(options)).resolves.toMatchObject({ key: identity.key })
+  })
+
+  it.each(['missing', 'modified', 'extra', 'renamed', 'metadata'])(
+    'rejects %s sealed contents',
+    async (change) => {
+      const options = await fixture()
+      await sealModuleCache(options)
+      const path = join(options.path, filename)
+      if (change === 'missing') {
+        await rm(path)
+      }
+      if (change === 'modified') {
+        await writeFile(path, moduleCode().replace('value = 1', 'value = 2'))
+      }
+      if (change === 'extra') {
+        await writeFile(join(options.path, 'c'.repeat(40)), moduleCode())
+      }
+      if (change === 'renamed') {
+        await rename(path, join(options.path, 'c'.repeat(40)))
+      }
+      if (change === 'metadata') {
+        await writeFile(
+          join(options.path, '_metadata.json'),
+          JSON.stringify({ lockfileHash: identity.lockfileHash, changed: true })
+        )
+      }
+      await expect(validateModuleCache(options)).rejects.toThrow(
+        /cache (inventory|content) mismatch/
+      )
+    }
+  )
+
+  it.each([
+    ['key', 'different-key'],
+    ['format', 'different-format'],
+    ['sourceSha', 'd'.repeat(40)],
+    ['lockfileHash', '12345678'],
+    ['version', 2],
+    ['versions', { ...identity.versions, bun: '1.4.4' }]
+  ])('rejects mismatched manifest %s', async (field, value) => {
+    const options = await fixture()
+    await sealModuleCache(options)
+    await changeManifest(options, (manifest) => {
+      manifest[field] = value
+    })
+    await expect(validateModuleCache(options)).rejects.toThrow(/identity mismatch/)
+  })
+
+  it.each(['missing', 'malformed', 'mismatched'])(
+    'rejects %s Vitest metadata before sealing',
+    async (kind) => {
+      const options = await fixture()
+      const path = join(options.path, '_metadata.json')
+      if (kind === 'missing') {
+        await rm(path)
+      }
+      if (kind === 'malformed') {
+        await writeFile(path, '{')
+      }
+      if (kind === 'mismatched') {
+        await writeFile(path, '{"lockfileHash":"12345678"}')
+      }
+      await expect(sealModuleCache(options)).rejects.toThrow()
+    }
+  )
+
+  it.each(['.tmp-partial', 'invalid-module.js', 'B'.repeat(40)])(
+    'rejects invalid filename %s',
+    async (name) => {
+      const options = await fixture()
+      await writeFile(join(options.path, name), moduleCode())
+      await expect(inventoryModuleCache(options)).rejects.toThrow('invalid cache entry name')
+    }
+  )
+
+  it('rejects a directory masquerading as a transformed file', async () => {
+    const options = await fixture()
+    await mkdir(join(options.path, 'c'.repeat(40)))
+    await expect(inventoryModuleCache(options)).rejects.toThrow('not a regular file')
+  })
+
+  it.skipIf(process.platform === 'win32')(
+    'rejects symlinked modules, manifests and cache roots',
+    async () => {
+      const options = await fixture()
+      const outside = await fixture()
+      await symlink(join(outside.path, filename), join(options.path, 'c'.repeat(40)))
+      await expect(inventoryModuleCache(options)).rejects.toThrow('not a regular file')
+      await rm(join(options.path, 'c'.repeat(40)))
+      await symlink(join(outside.path, filename), join(options.path, MODULE_CACHE_MANIFEST))
+      await expect(sealModuleCache(options)).rejects.toThrow('not a regular file')
+      await symlink(options.path, join(outside.path, 'linked-root'))
+      await expect(
+        inventoryModuleCache({ ...options, path: join(outside.path, 'linked-root') })
+      ).rejects.toThrow('cache root')
+    }
+  )
+
+  it('rejects oversized entries before allocating their contents', async () => {
+    const options = await fixture()
+    await truncate(join(options.path, filename), MODULE_CACHE_LIMITS.entryBytes + 1)
+    await expect(inventoryModuleCache(options)).rejects.toThrow('invalid file size')
+  })
+
+  it.each([
+    ['missing trailer', 'export const value = 1'],
+    ['invalid base64', 'x\n//# vitestCache=***'],
+    ['noncanonical base64', 'x\n//# vitestCache=W10'],
+    ['invalid JSON', 'x\n//# vitestCache=ew=='],
+    ['plain JSON object', `x\n//# vitestCache=${Buffer.from('{}').toString('base64')}`]
+  ])('rejects %s', async (_label, code) => {
+    const options = await fixture()
+    await writeFile(join(options.path, filename), code)
+    await expect(sealModuleCache(options)).rejects.toThrow()
+  })
+
+  it.each([
+    [
+      'out-of-range reference',
+      (table) => {
+        table[0].id = '999'
+      }
+    ],
+    [
+      'noncanonical reference',
+      (table) => {
+        table[0].id = '01'
+      }
+    ],
+    [
+      'numeric id',
+      (table) => {
+        table[0].id = 1
+      }
+    ],
+    [
+      'cyclic imports',
+      (table) => {
+        table[0].imports = '0'
+      }
+    ],
+    [
+      'invalid mappings',
+      (table) => {
+        table[0].mappings = null
+      }
+    ],
+    [
+      'invalid module type',
+      (table) => {
+        table[7] = 'unknown'
+      }
+    ],
+    [
+      'invalid import URL',
+      (table) => {
+        table[8][0] = false
+      }
+    ],
+    [
+      'invalid import ids',
+      (table) => {
+        table[9]['/dependency.ts'] = 4
+      }
+    ],
+    [
+      'invalid dependencies',
+      (table) => {
+        table[4][0] = false
+      }
+    ],
+    [
+      'invalid static mocks',
+      (table) => {
+        table[11].hasFactory = 1
+      }
+    ],
+    [
+      'inline object',
+      (table) => {
+        table[0].imports = {}
+      }
+    ],
+    [
+      'unsafe property',
+      (table) => {
+        table[0] = JSON.parse('{"__proto__":"0"}')
+      }
+    ]
+  ])('rejects %s in module metadata', async (_label, mutate) => {
+    const options = await fixture()
+    const table = moduleTable()
+    mutate(table)
+    await writeFile(join(options.path, filename), moduleCode(table))
+    await expect(sealModuleCache(options)).rejects.toThrow()
+  })
+})

config/scripts/ci-unit-module-cache-trailer.mjs

@@ -0,0 +1,118 @@
+const TRAILER = '\n//# vitestCache='
+const isRecord = (value) => value !== null && typeof value === 'object' && !Array.isArray(value)
+const fail = (message) => {
+  throw new Error(`Unit module cache: ${message}`)
+}
+
+export function checkModuleTrailer(buffer) {
+  const text = buffer.toString('utf8')
+  if (!Buffer.from(text).equals(buffer)) {
+    fail('module is not valid UTF-8')
+  }
+  const marker = text.lastIndexOf(TRAILER)
+  if (marker === -1) {
+    fail('module trailer is missing')
+  }
+  const encoded = text.slice(marker + TRAILER.length)
+  const decoded = Buffer.from(encoded, 'base64')
+  if (!encoded || decoded.toString('base64') !== encoded) {
+    fail('module trailer is not canonical base64')
+  }
+  const json = decoded.toString('utf8')
+  if (!Buffer.from(json).equals(decoded)) {
+    fail('module metadata is not valid UTF-8')
+  }
+  const table = JSON.parse(json)
+  if (!Array.isArray(table) || !isRecord(table[0])) {
+    fail('module metadata is not a flatted object table')
+  }
+  const resolve = (value) => {
+    if (typeof value !== 'string') {
+      return value
+    }
+    if (
+      !/^(0|[1-9][0-9]*)$/.test(value) ||
+      !Number.isSafeInteger(Number(value)) ||
+      Number(value) >= table.length
+    ) {
+      fail('invalid flatted reference')
+    }
+    return table[Number(value)]
+  }
+  // Check table references directly; the cache only needs the documented metadata fields.
+  for (const slot of table) {
+    if (typeof slot === 'string') {
+      continue
+    }
+    if (!slot || typeof slot !== 'object') {
+      fail('invalid flatted table slot')
+    }
+    if (Object.hasOwn(slot, '__proto__')) {
+      fail('unsafe flatted property')
+    }
+    for (const value of Object.values(slot)) {
+      if (value !== null && typeof value === 'object') {
+        fail('inline object in flatted table')
+      }
+      if (typeof value === 'number' && !Number.isFinite(value)) {
+        fail('invalid flatted number')
+      }
+      resolve(value)
+    }
+  }
+  const fields = Object.fromEntries(
+    Object.entries(table[0]).map(([name, value]) => [name, resolve(value)])
+  )
+  const strings = (value) =>
+    Array.isArray(value) && value.every((item) => typeof resolve(item) === 'string')
+  if (typeof fields.id !== 'string' || typeof fields.url !== 'string') {
+    fail('invalid module id or url')
+  }
+  if (fields.file != null && typeof fields.file !== 'string') {
+    fail('invalid module file')
+  }
+  if (typeof fields.mappings !== 'boolean') {
+    fail('invalid module mappings')
+  }
+  if (
+    fields.moduleType !== undefined &&
+    fields.moduleType !== 'cjs' &&
+    fields.moduleType !== 'esm'
+  ) {
+    fail('invalid module type')
+  }
+  if (!isRecord(fields.imports)) {
+    fail('invalid module imports')
+  }
+  const urls = resolve(fields.imports.urls)
+  const ids = resolve(fields.imports.ids)
+  if (
+    !strings(urls) ||
+    !isRecord(ids) ||
+    !Object.values(ids).every((id) => typeof resolve(id) === 'string')
+  ) {
+    fail('invalid module import references')
+  }
+  for (const name of ['deps', 'dynamicDeps']) {
+    if (fields[name] !== undefined && !strings(fields[name])) {
+      fail(`invalid module ${name}`)
+    }
+  }
+  if (fields.staticMocks != null) {
+    if (!Array.isArray(fields.staticMocks)) {
+      fail('invalid module static mocks')
+    }
+    for (const reference of fields.staticMocks) {
+      const mock = resolve(reference)
+      if (
+        !isRecord(mock) ||
+        typeof resolve(mock.method) !== 'string' ||
+        typeof resolve(mock.specifier) !== 'string' ||
+        typeof mock.hasFactory !== 'boolean' ||
+        typeof mock.factoryLoadsOriginal !== 'boolean'
+      ) {
+        fail('invalid module static mock')
+      }
+    }
+  }
+}

config/scripts/ci-unit-module-cache.mjs

@@ -0,0 +1,239 @@
+import { createHash } from 'node:crypto'
+import {
+  appendFileSync,
+  existsSync,
+  globSync,
+  readFileSync,
+  realpathSync,
+  rmSync,
+  statSync
+} from 'node:fs'
+import { createRequire } from 'node:module'
+import { join, resolve } from 'node:path'
+import { performance } from 'node:perf_hooks'
+import { pathToFileURL } from 'node:url'
+import { runProcessSync } from '@orca/process-host'
+import { collectNodeServerInputs } from './node-server-change-scope.mjs'
+import { workspacePackageManifests } from './workspace-source-exports.mjs'
+import { writeAssignment } from './ci-shard-assignment.mjs'
+import { sealModuleCache, validateModuleCache } from './ci-unit-module-cache-contents.mjs'
+
+const ROOT = resolve(import.meta.dirname, '../..')
+const IDENTITY_REPORT = 'ci-shards/unit-module-cache-identity.json'
+const CACHE_REPORT = 'ci-shards/unit-module-cache.json'
+const CACHE_FORMAT = '1.0.0-beta.8'
+
+function digestInputs(root, files) {
+  const hash = createHash('sha256')
+  for (const file of [...new Set(files)].sort()) {
+    if (!existsSync(join(root, file))) {
+      hash.update(`${file}\0missing\0`)
+      continue
+    }
+    const bytes = readFileSync(join(root, file))
+    hash.update(`${file}\0${bytes.length}\0`).update(bytes)
+  }
+  return hash.digest('hex')
+}
+
+export async function moduleCachePolicyHash(root = ROOT) {
+  const runnerScripts = globSync('config/scripts/vitest*.{mjs,ts}', { cwd: root }).filter(
+    (file) => !file.includes('.test.')
+  )
+  const configInputs = await collectNodeServerInputs({
+    root,
+    entryPoints: [
+      'config/vitest.config.ts',
+      'config/scripts/run-vitest.mjs',
+      'config/scripts/ci-unit-timing-reporter.mjs',
+      ...runnerScripts
+    ]
+  })
+  return digestInputs(root, [
+    ...configInputs,
+    ...workspacePackageManifests(root),
+    ...globSync(['config/patches/**', '**/tsconfig*.json'], {
+      cwd: root,
+      exclude: ['**/node_modules/**', '**/.git/**', '**/dist/**', '**/out/**', 'mobile/vendor/**']
+    }).filter((file) => statSync(join(root, file)).isFile()),
+    'package.json',
+    'pnpm-lock.yaml',
+    'pnpm-workspace.yaml',
+    '.npmrc',
+    '.pnpmfile.cjs',
+    'config/.bun-version',
+    'config/scripts/ci-unit-module-cache.mjs',
+    'config/scripts/ci-unit-module-cache-contents.mjs',
+    'config/scripts/ci-unit-module-cache-trailer.mjs'
+  ])
+}
+
+export function moduleCacheIdentity({
+  root = ROOT,
+  cacheRoot = process.env.RUNNER_TEMP,
+  policyHash,
+  installedLockHash,
+  versions,
+  shard,
+  platform = process.platform,
+  arch = process.arch,
+  nodeEnv = process.env.NODE_ENV ?? 'test',
+  namespace = ''
+}) {
+  if (
+    !cacheRoot ||
+    !/^[a-f0-9]{64}$/.test(policyHash ?? '') ||
+    !/^[a-f0-9]{64}$/.test(installedLockHash ?? '') ||
+    ![versions?.node, versions?.bun, versions?.vitest].every((version) =>
+      /^\d+\.\d+\.\d+(?:[-+][\w.-]+)?$/.test(version ?? '')
+    ) ||
+    !Number.isInteger(shard?.index) ||
+    !Number.isInteger(shard?.count) ||
+    shard.index < 1 ||
+    shard.index > shard.count
+  ) {
+    throw new Error('Unit module cache identity is incomplete')
+  }
+  const identity = {
+    format: CACHE_FORMAT,
+    root: realpathSync(root),
+    platform,
+    arch,
+    versions,
+    policyHash,
+    installedLockHash,
+    nodeEnv,
+    shard,
+    namespace
+  }
+  const digest = createHash('sha256').update(JSON.stringify(identity)).digest('hex')
+  return {
+    ...identity,
+    key: `unit-module-v1-${platform}-${arch}-n${versions.node}-b${versions.bun}-v${versions.vitest}-s${shard.index}of${shard.count}-${digest}`,
+    path: join(resolve(cacheRoot), 'orca-vitest-module-cache')
+  }
+}
+
+export async function resolveModuleCacheIdentity({
+  root = ROOT,
+  cacheRoot = process.env.RUNNER_TEMP,
+  shard,
+  namespace = ''
+} = {}) {
+  const require = createRequire(join(root, 'package.json'))
+  const bun = runProcessSync({ program: 'bun', args: ['--version'], cwd: root, timeoutMs: 10_000 })
+  if (bun.code !== 0) {
+    throw new Error('Bun version is unavailable')
+  }
+  return moduleCacheIdentity({
+    root,
+    cacheRoot,
+    shard,
+    namespace,
+    policyHash: await moduleCachePolicyHash(root),
+    installedLockHash: createHash('sha256')
+      .update(readFileSync(join(root, 'node_modules', '.pnpm', 'lock.yaml')))
+      .digest('hex'),
+    versions: {
+      node: process.versions.node,
+      bun: bun.stdout.trim(),
+      vitest: require('vitest/package.json').version
+    }
+  })
+}
+
+export async function prepareModuleCache(identity, { restoreHit = false } = {}) {
+  const started = performance.now()
+  try {
+    if (!restoreHit) {
+      throw new Error('No exact cache was restored')
+    }
+    const manifest = await validateModuleCache({
+      path: identity.path,
+      key: identity.key,
+      format: identity.format,
+      versions: identity.versions,
+      lockfileHash: identity.installedLockHash.slice(0, 8)
+    })
+    return {
+      available: true,
+      entries: manifest.entries.length,
+      bytes: manifest.totalBytes,
+      producerSourceSha: manifest.sourceSha,
+      validationMs: performance.now() - started
+    }
+  } catch (error) {
+    rmSync(identity.path, { recursive: true, force: true })
+    return {
+      available: false,
+      reason: String(error),
+      validationMs: performance.now() - started
+    }
+  }
+}
+
+export async function finishModuleCache(identity, { sourceSha } = {}) {
+  const started = performance.now()
+  const manifest = await sealModuleCache({
+    ...identity,
+    sourceSha,
+    lockfileHash: identity.installedLockHash.slice(0, 8)
+  })
+  return {
+    entries: manifest.entries.length,
+    bytes: manifest.totalBytes,
+    sealMs: performance.now() - started
+  }
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  const phase = process.argv[2]
+  const output = (values) => {
+    if (process.env.GITHUB_OUTPUT) {
+      for (const [key, value] of Object.entries(values)) {
+        appendFileSync(process.env.GITHUB_OUTPUT, `${key}=${value}\n`)
+      }
+    }
+  }
+  if (phase === 'identity') {
+    try {
+      const identity = await resolveModuleCacheIdentity({
+        shard: {
+          index: Number(process.env.ORCA_UNIT_SHARD_INDEX),
+          count: Number(process.env.ORCA_UNIT_SHARD_COUNT)
+        },
+        namespace: process.env.ORCA_UNIT_CACHE_NAMESPACE ?? ''
+      })
+      writeAssignment(IDENTITY_REPORT, identity)
+      output({ key: identity.key, path: identity.path })
+      if (process.env.GITHUB_ENV) {
+        appendFileSync(process.env.GITHUB_ENV, `ORCA_UNIT_MODULE_CACHE_PATH=${identity.path}\n`)
+      }
+    } catch (error) {
+      console.warn(`Unit module cache unavailable: ${String(error)}`)
+    }
+  } else {
+    const identity = JSON.parse(readFileSync(IDENTITY_REPORT, 'utf8'))
+    if (phase === 'prepare') {
+      const result = await prepareModuleCache(identity, {
+        restoreHit: process.env.ORCA_UNIT_CACHE_RESTORE_HIT === 'true'
+      })
+      writeAssignment(CACHE_REPORT, { key: identity.key, mode: 'restore', ...result })
+      console.log(JSON.stringify(result))
+    } else if (phase === 'seal') {
+      try {
+        const result = await finishModuleCache(identity, {
+          sourceSha: process.env.ORCA_SHARD_SOURCE_SHA ?? process.env.GITHUB_SHA
+        })
+        const previous = JSON.parse(readFileSync(CACHE_REPORT, 'utf8'))
+        writeAssignment(CACHE_REPORT, { ...previous, produced: result })
+        output({ ready: true })
+        console.log(JSON.stringify(result))
+      } catch (error) {
+        console.warn(`Unit module cache was not saved: ${String(error)}`)
+      }
+    } else {
+      throw new Error(`Unknown unit module cache phase: ${phase}`)
+    }
+  }
+}

config/scripts/ci-unit-module-cache.test.mjs

@@ -0,0 +1,167 @@
+import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, describe, expect, it } from 'vitest'
+import { parse } from 'yaml'
+import {
+  finishModuleCache,
+  moduleCacheIdentity,
+  prepareModuleCache
+} from './ci-unit-module-cache.mjs'
+
+const temporary = []
+afterEach(() => {
+  for (const directory of temporary.splice(0)) {
+    rmSync(directory, { recursive: true, force: true })
+  }
+})
+
+function options() {
+  const directory = mkdtempSync(join(tmpdir(), 'orca-unit-cache-identity-'))
+  temporary.push(directory)
+  return {
+    root: directory,
+    cacheRoot: directory,
+    policyHash: 'a'.repeat(64),
+    installedLockHash: 'b'.repeat(64),
+    versions: { node: '24.21.0', bun: '1.4.3', vitest: '5.0.3' },
+    shard: { index: 1, count: 5 },
+    platform: 'linux',
+    arch: 'arm64',
+    nodeEnv: 'test',
+    namespace: ''
+  }
+}
+
+describe('persistent unit module cache boundary', () => {
+  it('separates hosts, runtimes, config, lockfiles, roots, shards and experiments', () => {
+    const original = options()
+    const identity = moduleCacheIdentity(original)
+    const anotherRoot = options().root
+    for (const update of [
+      { root: anotherRoot },
+      { policyHash: 'c'.repeat(64) },
+      { installedLockHash: 'd'.repeat(64) },
+      { platform: 'win32' },
+      { arch: 'x64' },
+      { nodeEnv: 'production' },
+      { namespace: 'comparison' },
+      { shard: { index: 2, count: 5 } },
+      { shard: { index: 1, count: 10 } },
+      { versions: { ...original.versions, node: '26.6.0' } },
+      { versions: { ...original.versions, bun: '1.4.4' } },
+      { versions: { ...original.versions, vitest: '5.0.4' } }
+    ]) {
+      const next = moduleCacheIdentity({ ...original, ...update })
+      expect(next.key).not.toBe(identity.key)
+      expect(next.path).toBe(identity.path)
+    }
+    expect(moduleCacheIdentity(original)).toEqual(identity)
+  })
+
+  it.each([
+    { cacheRoot: '' },
+    { policyHash: '' },
+    { installedLockHash: '' },
+    { versions: { node: '24', bun: '1.4.3', vitest: '5.0.3' } },
+    { shard: { index: 0, count: 5 } },
+    { shard: { index: 6, count: 5 } }
+  ])('refuses an incomplete identity %j', (update) => {
+    expect(() => moduleCacheIdentity({ ...options(), ...update })).toThrow('incomplete')
+  })
+
+  it.each([false, true])(
+    'starts cold and discards unusable restored bytes (hit %s)',
+    async (hit) => {
+      const identity = moduleCacheIdentity(options())
+      mkdirSync(identity.path)
+      writeFileSync(join(identity.path, 'untrusted'), 'throw Error("must not execute")')
+      const prepared = await prepareModuleCache(identity, { restoreHit: hit })
+      expect(prepared.available).toBe(false)
+      expect(prepared.validationMs).toBeGreaterThanOrEqual(0)
+      expect(() => readFileSync(join(identity.path, 'untrusted'))).toThrow()
+    }
+  )
+  it('restores a matching cache from an earlier source without copying evaluated state', async () => {
+    const identity = moduleCacheIdentity(options())
+    mkdirSync(identity.path)
+    writeFileSync(
+      join(identity.path, '_metadata.json'),
+      JSON.stringify({ lockfileHash: identity.installedLockHash.slice(0, 8) })
+    )
+    const metadata = [
+      { id: '1', url: '2', file: '1', imports: '3', mappings: false },
+      join(identity.root, 'source.ts'),
+      '/source.ts',
+      { urls: '4', ids: '5' },
+      [],
+      {}
+    ]
+    writeFileSync(
+      join(identity.path, 'a'.repeat(40)),
+      `export const value = 1\n//# vitestCache=${Buffer.from(JSON.stringify(metadata)).toString('base64')}`
+    )
+    const produced = await finishModuleCache(identity, { sourceSha: 'a'.repeat(40) })
+    expect(produced.entries).toBe(2)
+    const prepared = await prepareModuleCache(
+      { ...identity, sourceSha: 'b'.repeat(40) },
+      { restoreHit: true }
+    )
+    expect(prepared).toMatchObject({
+      available: true,
+      entries: produced.entries,
+      bytes: produced.bytes,
+      producerSourceSha: 'a'.repeat(40)
+    })
+  })
+})
+
+it('uses an optional exact restore after install and saves successful scoped producers', () => {
+  const workflow = parse(readFileSync('.github/workflows/unit-tests.yml', 'utf8'))
+  const steps = workflow.jobs.test.steps
+  const dependencies = steps.findIndex(
+    (step) => step.uses === './.github/actions/install-node-dependencies'
+  )
+  const identity = steps.findIndex((step) => step.id === 'module-cache')
+  const restore = steps.findIndex((step) => step.id === 'module-cache-restore')
+  const prepare = steps.findIndex((step) => step.run?.endsWith('ci-unit-module-cache.mjs prepare'))
+  const tests = steps.findIndex((step) => step.name === 'Test shard')
+  const seal = steps.find((step) => step.id === 'module-cache-seal')
+  const save = steps.find((step) => step.uses === 'actions/cache/save@v5')
+  expect(identity).toBeGreaterThan(dependencies)
+  expect(restore).toBeGreaterThan(identity)
+  expect(prepare).toBeGreaterThan(restore)
+  expect(tests).toBeGreaterThan(prepare)
+  expect(steps[restore].uses).toBe('actions/cache/restore@v5')
+  expect(steps[restore].with['restore-keys']).toBeUndefined()
+  expect(steps[restore]['continue-on-error']).toBe(true)
+  expect(seal.if).toContain('success()')
+  expect(seal.if).toContain("github.event_name == 'pull_request'")
+  expect(seal.if).toContain("github.ref == 'refs/heads/main'")
+  expect(seal.if).toContain("github.event_name != 'pull_request'")
+  expect(seal.if).toContain("inputs.module_cache_seed && github.event_name == 'workflow_dispatch'")
+  expect(save.if).toBe("steps.module-cache-seal.outputs.ready == 'true'")
+  expect(save.with.key).toBe(steps[restore].with.key)
+  expect(workflow.on.workflow_call.inputs.module_cache.default).toBe('off')
+  expect(workflow.on.workflow_call.inputs.run_relay_integration.default).toBe(true)
+})
+
+it('runs full same-source controls serially with five four-worker shards and included producer costs', () => {
+  const workflow = parse(readFileSync('.github/workflows/unit-module-cache-benchmark.yml', 'utf8'))
+  const { baseline, producer, warm } = workflow.jobs
+  expect(producer.needs).toBe('baseline')
+  expect(warm.needs).toBe('producer')
+  for (const job of [baseline, producer, warm]) {
+    expect(job.uses).toBe('./.github/workflows/unit-tests.yml')
+    expect(job.with.node_versions).toBe('["24.21.0"]')
+    expect(job.with.runner).toBe('ubuntu-24.04-arm')
+    expect(JSON.parse(job.with.shards)).toEqual(
+      Array.from({ length: 5 }, (_, index) => ({ index: index + 1, count: 5 }))
+    )
+    expect(job.with.phase_diagnostics).toBe(true)
+  }
+  expect(producer.with.module_cache).toBe('cold')
+  expect(producer.with.module_cache_seed).toBe(true)
+  expect(warm.with.module_cache).toBe('warm')
+  expect(warm.with.module_cache_namespace).toBe(producer.with.module_cache_namespace)
+})

config/scripts/ci-unit-timing-reporter.mjs

@@ -1,8 +1,114 @@
-import { relative } from 'node:path'
+import { relative, resolve } from 'node:path'
 import { appendFileSync, readFileSync } from 'node:fs'
 import { writeAssignment } from './ci-shard-assignment.mjs'
 import { auditUnitSelection } from './ci-unit-selection.mjs'
 
+export const UNIT_PHASE_FILE_LIMIT = 50_000
+const phaseNames = [
+  'environmentSetupDuration',
+  'prepareDuration',
+  'collectDuration',
+  'setupDuration',
+  'duration'
+]
+
+const nonnegativeNumber = (value) => (Number.isFinite(value) && value >= 0 ? value : null)
+const emptyPhases = () => Object.fromEntries(phaseNames.map((name) => [name, 0]))
+const emptyImportSample = () => ({ entries: 0, selfTimeMs: 0, totalTimeMs: 0 })
+const workerCounters = (state) => ({
+  workersSpawned: Number.isInteger(state?.workersSpawned)
+    ? nonnegativeNumber(state.workersSpawned)
+    : null,
+  startupTimeMs: nonnegativeNumber(state?.startupTime)
+})
+
+export function buildUnitPhaseReport(ctx, modules, errors, reason, baseline) {
+  const files = []
+  const projects = new Map()
+  const totals = emptyPhases()
+  const phaseSamples = emptyPhases()
+  for (const module of modules) {
+    const diagnostic = module.diagnostic()
+    const project = module.project?.name ?? 'unknown'
+    if (!projects.has(project)) {
+      projects.set(project, {
+        project,
+        files: 0,
+        phases: emptyPhases(),
+        phaseSamples: emptyPhases(),
+        sampledImports: emptyImportSample()
+      })
+    }
+    const aggregate = projects.get(project)
+    aggregate.files += 1
+    const phases = {}
+    for (const name of phaseNames) {
+      const value = nonnegativeNumber(diagnostic?.[name])
+      phases[name] = value
+      if (value !== null) {
+        aggregate.phases[name] += value
+        aggregate.phaseSamples[name] += 1
+        totals[name] += value
+        phaseSamples[name] += 1
+      }
+    }
+    const imports = diagnostic?.importDurations
+    if (imports && typeof imports === 'object') {
+      for (const id in imports) {
+        if (!Object.hasOwn(imports, id)) {
+          continue
+        }
+        const selfTime = nonnegativeNumber(imports[id]?.selfTime)
+        const totalTime = nonnegativeNumber(imports[id]?.totalTime)
+        if (selfTime !== null && totalTime !== null) {
+          aggregate.sampledImports.entries += 1
+          aggregate.sampledImports.selfTimeMs += selfTime
+          aggregate.sampledImports.totalTimeMs += totalTime
+        }
+      }
+    }
+    if (files.length < UNIT_PHASE_FILE_LIMIT) {
+      files.push({
+        file: relative(ctx.config.root, module.moduleId).replaceAll('\\', '/'),
+        project,
+        workerId: nonnegativeNumber(diagnostic?.workerId),
+        concurrencyId: nonnegativeNumber(diagnostic?.concurrencyId),
+        phases
+      })
+    }
+  }
+  const counters = workerCounters(ctx.state)
+  return {
+    metric: 'module-phase-diagnostics-v1',
+    nodeVersion: process.env.ORCA_TEST_NODE_VERSION ?? process.versions.node,
+    ...(process.versions.bun ? { bunVersion: process.versions.bun } : {}),
+    vitestVersion: ctx.version ?? null,
+    shard: ctx.config.shard ?? { index: 1, count: 1 },
+    status: reason,
+    unhandledErrors: errors.length,
+    units: 'milliseconds',
+    phaseTime: 'summed-worker-time',
+    fileLimit: UNIT_PHASE_FILE_LIMIT,
+    filesSeen: modules.length,
+    filesOmitted: modules.length - files.length,
+    workerTotals: {
+      scope: baseline ? 'test-run' : 'vitest-session',
+      ...Object.fromEntries(
+        Object.entries(counters).map(([name, value]) => [
+          name,
+          value === null || (baseline && baseline[name] === null)
+            ? null
+            : nonnegativeNumber(value - (baseline?.[name] ?? 0))
+        ])
+      )
+    },
+    phases: totals,
+    phaseSamples,
+    projects: [...projects.values()],
+    files
+  }
+}
+
 export function moduleDuration(diagnostic) {
   return Math.max(
     1,
@@ -21,6 +127,12 @@ export default class UnitTimingReporter {
     this.ctx = ctx
   }
 
+  onTestRunStart() {
+    if (process.env.ORCA_UNIT_PHASE_REPORT) {
+      this.workerBaseline = workerCounters(this.ctx.state)
+    }
+  }
+
   onTestRunEnd(modules, errors, reason) {
     const results = Object.fromEntries(
       modules.map((module) => [
@@ -61,5 +173,17 @@ export default class UnitTimingReporter {
     } catch {
       // Timing evidence remains useful when selection planning was unavailable.
     }
+    const phaseReport = process.env.ORCA_UNIT_PHASE_REPORT
+    const timingReport = process.env.ORCA_UNIT_TIMING_REPORT ?? 'ci-shards/unit-timings.json'
+    if (phaseReport && resolve(phaseReport) !== resolve(timingReport)) {
+      try {
+        writeAssignment(
+          phaseReport,
+          buildUnitPhaseReport(this.ctx, modules, errors, reason, this.workerBaseline)
+        )
+      } catch {
+        // Optional diagnostics must not change the test verdict.
+      }
+    }
   }
 }

config/scripts/ci-unit-timing-reporter.test.mjs

@@ -1,9 +1,13 @@
-import { mkdtempSync, readFileSync, rmSync } from 'node:fs'
+import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import { join, resolve } from 'node:path'
 import { describe, expect, it, vi } from 'vitest'
 import { importUnitTimingReports } from './ci-unit-timing-import.mjs'
-import UnitTimingReporter, { moduleDuration } from './ci-unit-timing-reporter.mjs'
+import UnitTimingReporter, {
+  buildUnitPhaseReport,
+  moduleDuration,
+  UNIT_PHASE_FILE_LIMIT
+} from './ci-unit-timing-reporter.mjs'
 
 const report = (index, timings) => ({
   metric: 'module-duration-v1',
@@ -38,6 +42,7 @@ describe('unit worker timing evidence', () => {
       const previous = process.env.ORCA_UNIT_TIMING_REPORT
       process.env.ORCA_UNIT_TIMING_REPORT = join(directory, 'unit-timings.json')
       vi.stubEnv('ORCA_TEST_NODE_VERSION', nodeVersion)
+      vi.stubEnv('ORCA_UNIT_PHASE_REPORT', undefined)
       try {
         const reporter = new UnitTimingReporter()
         reporter.onInit({ config: { root: process.cwd(), shard: { index: 1, count: 8 } } })
@@ -121,3 +126,217 @@ describe('unit worker timing evidence', () => {
     expect(() => importUnitTimingReports([first, report(2, { b: -1 })])).toThrow('timing')
   })
 })
+
+const phaseDiagnostic = (overrides = {}) => ({
+  environmentSetupDuration: 1.5,
+  prepareDuration: 2.5,
+  collectDuration: 30.25,
+  setupDuration: 4.5,
+  duration: 5.25,
+  workerId: 7,
+  concurrencyId: 2,
+  importDurations: {},
+  ...overrides
+})
+const phaseModule = (file, diagnostic = phaseDiagnostic(), project = 'bun') => ({
+  moduleId: resolve(file),
+  project: { name: project },
+  state: () => 'passed',
+  diagnostic: () => diagnostic
+})
+const phaseContext = (state) => ({
+  config: { root: process.cwd(), shard: { index: 1, count: 5 } },
+  version: '5.0.3',
+  state
+})
+
+function withReporterOutputs(run) {
+  const directory = mkdtempSync(join(tmpdir(), 'orca-unit-phases-'))
+  const timingPath = join(directory, 'unit-timings.json')
+  const phasePath = join(directory, 'unit-phase-diagnostics.json')
+  vi.stubEnv('ORCA_UNIT_TIMING_REPORT', timingPath)
+  vi.stubEnv('ORCA_UNIT_PHASE_REPORT', undefined)
+  vi.stubEnv('ORCA_UNIT_SELECTION_PLAN', join(directory, 'missing-selection.json'))
+  vi.stubEnv('ORCA_SHARD_SOURCE_SHA', 'phase-source')
+  vi.stubEnv('GITHUB_RUN_ID', 'phase-run')
+  vi.stubEnv('GITHUB_RUN_ATTEMPT', '3')
+  try {
+    run({ timingPath, phasePath })
+  } finally {
+    vi.unstubAllEnvs()
+    rmSync(directory, { recursive: true, force: true })
+  }
+}
+
+describe('optional unit phase diagnostics', () => {
+  it('leaves the timing report identical and writes nothing extra when disabled', () => {
+    withReporterOutputs(({ timingPath, phasePath }) => {
+      const modules = [
+        phaseModule(
+          'src/example.test.ts',
+          phaseDiagnostic({ importDurations: { dependency: { selfTime: 100, totalTime: 200 } } })
+        )
+      ]
+      const reporter = new UnitTimingReporter()
+      reporter.onInit(phaseContext({ workersSpawned: 3, startupTime: 500 }))
+      reporter.onTestRunEnd(modules, [], 'passed')
+      const timing = readFileSync(timingPath, 'utf8')
+      expect(existsSync(phasePath)).toBe(false)
+
+      vi.stubEnv('ORCA_UNIT_PHASE_REPORT', phasePath)
+      reporter.onTestRunEnd(modules, [], 'passed')
+      expect(readFileSync(timingPath, 'utf8')).toBe(timing)
+      expect(JSON.parse(timing).timings).toEqual({ 'src/example.test.ts': 44 })
+      expect(JSON.parse(readFileSync(phasePath, 'utf8'))).toMatchObject({
+        sourceSha: 'phase-source',
+        runId: 'phase-run',
+        runAttempt: '3',
+        metric: 'module-phase-diagnostics-v1',
+        vitestVersion: '5.0.3',
+        shard: { index: 1, count: 5 },
+        status: 'passed',
+        unhandledErrors: 0,
+        workerTotals: { scope: 'vitest-session', workersSpawned: 3, startupTimeMs: 500 }
+      })
+    })
+  })
+
+  it('records fractions and project totals without retaining sampled import records', () => {
+    const report = buildUnitPhaseReport(
+      phaseContext(),
+      [
+        phaseModule(
+          'src/first.test.ts',
+          phaseDiagnostic({
+            importDurations: {
+              first: { selfTime: 2, totalTime: 9, importer: 'src/private-account.ts' },
+              second: { selfTime: 3.5, totalTime: 5.5 },
+              invalid: { selfTime: Number.NaN, totalTime: 4 }
+            }
+          })
+        ),
+        phaseModule('src/second.test.ts'),
+        phaseModule('src/node.test.ts', phaseDiagnostic({ workerId: 9 }), 'node-runtime')
+      ],
+      [{ message: 'test error' }],
+      'failed'
+    )
+    expect(report.files[0]).toEqual({
+      file: 'src/first.test.ts',
+      project: 'bun',
+      workerId: 7,
+      concurrencyId: 2,
+      phases: {
+        environmentSetupDuration: 1.5,
+        prepareDuration: 2.5,
+        collectDuration: 30.25,
+        setupDuration: 4.5,
+        duration: 5.25
+      }
+    })
+    expect(report.projects).toMatchObject([
+      {
+        project: 'bun',
+        files: 2,
+        phases: { collectDuration: 60.5 },
+        phaseSamples: { collectDuration: 2 },
+        sampledImports: { entries: 2, selfTimeMs: 5.5, totalTimeMs: 14.5 }
+      },
+      { project: 'node-runtime', files: 1, sampledImports: { entries: 0 } }
+    ])
+    expect(report.phases.collectDuration).toBe(90.75)
+    expect(report).toMatchObject({
+      phaseTime: 'summed-worker-time',
+      status: 'failed',
+      unhandledErrors: 1
+    })
+    expect(JSON.stringify(report)).not.toContain('private-account')
+    expect(JSON.stringify(report)).not.toContain('importDurations')
+  })
+
+  it('reports successful worker startup counters for the current run separately', () => {
+    withReporterOutputs(({ phasePath }) => {
+      vi.stubEnv('ORCA_UNIT_PHASE_REPORT', phasePath)
+      const state = { workersSpawned: 2, startupTime: 10 }
+      const reporter = new UnitTimingReporter()
+      reporter.onInit(phaseContext(state))
+      reporter.onTestRunStart()
+      state.workersSpawned = 5
+      state.startupTime = 60.25
+      reporter.onTestRunEnd([phaseModule('src/example.test.ts')], [], 'passed')
+      expect(JSON.parse(readFileSync(phasePath, 'utf8')).workerTotals).toEqual({
+        scope: 'test-run',
+        workersSpawned: 3,
+        startupTimeMs: 50.25
+      })
+    })
+  })
+
+  it('caps file detail while keeping aggregates from omitted files', () => {
+    const module = phaseModule('src/repeated.test.ts')
+    const modules = Array.from({ length: UNIT_PHASE_FILE_LIMIT }, () => module)
+    modules.push(
+      phaseModule(
+        'src/omitted.test.ts',
+        phaseDiagnostic({
+          collectDuration: 11,
+          importDurations: { dependency: { selfTime: 2, totalTime: 3 } }
+        })
+      )
+    )
+    const report = buildUnitPhaseReport(phaseContext(), modules, [], 'passed')
+    expect(report.files).toHaveLength(UNIT_PHASE_FILE_LIMIT)
+    expect(report).toMatchObject({
+      fileLimit: UNIT_PHASE_FILE_LIMIT,
+      filesSeen: UNIT_PHASE_FILE_LIMIT + 1,
+      filesOmitted: 1,
+      phases: { collectDuration: UNIT_PHASE_FILE_LIMIT * 30.25 + 11 },
+      projects: [
+        {
+          files: UNIT_PHASE_FILE_LIMIT + 1,
+          sampledImports: { entries: 1, selfTimeMs: 2, totalTimeMs: 3 }
+        }
+      ]
+    })
+  })
+
+  it('marks unavailable counters, phases and worker IDs without inventing measurements', () => {
+    const report = buildUnitPhaseReport(
+      phaseContext({ workersSpawned: -1, startupTime: Number.NaN }),
+      [phaseModule('src/unavailable.test.ts', {})],
+      [],
+      'interrupted'
+    )
+    expect(report.workerTotals).toEqual({
+      scope: 'vitest-session',
+      workersSpawned: null,
+      startupTimeMs: null
+    })
+    expect(report.files[0]).toMatchObject({
+      workerId: null,
+      concurrencyId: null,
+      phases: { collectDuration: null }
+    })
+    expect(report.phaseSamples.collectDuration).toBe(0)
+    expect(report.status).toBe('interrupted')
+  })
+
+  it.each(['unwritable', 'same as timing'])(
+    'preserves timing evidence when the optional output is %s',
+    (target) => {
+      withReporterOutputs(({ timingPath }) => {
+        const reporter = new UnitTimingReporter()
+        reporter.onInit(phaseContext())
+        const modules = [phaseModule('src/example.test.ts')]
+        reporter.onTestRunEnd(modules, [], 'passed')
+        const timing = readFileSync(timingPath, 'utf8')
+        vi.stubEnv(
+          'ORCA_UNIT_PHASE_REPORT',
+          target === 'unwritable' ? `${timingPath}/unwritable.json` : timingPath
+        )
+        expect(() => reporter.onTestRunEnd(modules, [], 'passed')).not.toThrow()
+        expect(readFileSync(timingPath, 'utf8')).toBe(timing)
+      })
+    }
+  )
+})

config/scripts/dev-storage-reclaim.mjs

@@ -0,0 +1,41 @@
+import { resolve } from 'node:path'
+import { spawnProcess } from '@orca/process-host'
+
+export function startDevStorageReclaim({
+  repoRoot,
+  env = process.env,
+  spawn = spawnProcess,
+  report = console.error
+}) {
+  if (env.ORCA_SKIP_DEV_STORAGE_RECLAIM === '1') {
+    return
+  }
+  try {
+    // The worker owns its lock after dev exits and must not keep captured dev output open.
+    const child = spawn({
+      program: process.execPath,
+      args: [
+        resolve(import.meta.dirname, 'reclaim-worktree-space.mjs'),
+        '--repo',
+        repoRoot,
+        '--exclude-worktree',
+        repoRoot,
+        '--automatic',
+        '--apply'
+      ],
+      cwd: repoRoot,
+      env,
+      detached: true,
+      stdio: 'ignore'
+    })
+    child.on('error', (error) => report(`[orca-dev] Storage cleanup skipped: ${error.message}`))
+    child.on('exit', (code) => {
+      if (code !== 0) {
+        report('[orca-dev] Storage cleanup did not finish; retry with pnpm reclaim-space.')
+      }
+    })
+    child.unref()
+  } catch (error) {
+    report(`[orca-dev] Storage cleanup skipped: ${error.message}`)
+  }
+}

config/scripts/dev-storage-reclaim.test.mjs

@@ -0,0 +1,84 @@
+import { EventEmitter } from 'node:events'
+import { tmpdir } from 'node:os'
+import { resolve } from 'node:path'
+import { pathToFileURL } from 'node:url'
+import { runProcess } from '@orca/process-host'
+import { expect, it, vi } from 'vitest'
+import { startDevStorageReclaim } from './dev-storage-reclaim.mjs'
+
+function fixture() {
+  const child = new EventEmitter()
+  child.unref = vi.fn()
+  const spawn = vi.fn(() => child)
+  return { child, spawn, report: vi.fn(), repoRoot: '/dev-worktree', env: {} }
+}
+
+it('starts one non-interactive worker without waiting for its exit', () => {
+  const options = fixture()
+  startDevStorageReclaim(options)
+  expect(options.spawn).toHaveBeenCalledTimes(1)
+  expect(options.spawn.mock.calls[0][0]).toMatchObject({
+    args: expect.arrayContaining([
+      '--automatic',
+      '--apply',
+      '--exclude-worktree',
+      options.repoRoot
+    ]),
+    detached: true,
+    stdio: 'ignore'
+  })
+  expect(options.child.unref).toHaveBeenCalledTimes(1)
+  expect(options.report).not.toHaveBeenCalled()
+})
+
+it('allows tests and developers to skip the repository-wide sweep', () => {
+  const options = fixture()
+  startDevStorageReclaim({ ...options, env: { ORCA_SKIP_DEV_STORAGE_RECLAIM: '1' } })
+  expect(options.spawn).not.toHaveBeenCalled()
+})
+
+it('reports cleanup failure without throwing into dev startup', () => {
+  const options = fixture()
+  startDevStorageReclaim(options)
+  options.child.emit('exit', 1)
+  expect(options.report).toHaveBeenCalledWith(expect.stringContaining('pnpm reclaim-space'))
+})
+
+it.each([true, false])('handles both sync and async spawn errors (sync: %j)', (synchronous) => {
+  const options = fixture()
+  if (synchronous) {
+    options.spawn.mockImplementation(() => {
+      throw new Error('spawn failed')
+    })
+  }
+  expect(() => startDevStorageReclaim(options)).not.toThrow()
+  if (!synchronous) {
+    options.child.emit('error', new Error('spawn failed'))
+  }
+  expect(options.report).toHaveBeenCalledWith(expect.stringContaining('spawn failed'))
+})
+
+it('closes captured dev output before the detached worker exits', async () => {
+  const entry = pathToFileURL(resolve(import.meta.dirname, 'dev-storage-reclaim.mjs')).href
+  const result = await runProcess({
+    program: process.env.ORCA_TEST_NODE_EXECUTABLE ?? process.execPath,
+    args: [
+      '--input-type=module',
+      '-e',
+      `import { startDevStorageReclaim } from ${JSON.stringify(entry)};
+       import { spawnProcess } from '@orca/process-host';
+       startDevStorageReclaim({
+         repoRoot: ${JSON.stringify(tmpdir())},
+         env: { ...process.env, ORCA_SKIP_DEV_STORAGE_RECLAIM: '0' },
+         spawn: (spec) => spawnProcess({ ...spec, args: ['-e', 'setTimeout(() => {}, 5000)'] })
+       });
+       process.exit(0);`
+    ],
+    cwd: resolve(import.meta.dirname, '../..'),
+    env: { ...process.env, ORCA_BACKGROUND_LAUNCH: '1' },
+    timeoutMs: 2_000,
+    terminationBarrier: true
+  })
+  expect(result.timedOut).toBe(false)
+  expect(result.code, result.stderr).toBe(0)
+})

config/scripts/electron-builder-mac-channel-config.test.mjs

@@ -133,20 +133,4 @@ describe('electron-builder mac channel config', () => {
       }
     )
   })
-
-  // Why: the dev channels share every packaging decision except where they
-  // publish, so a future edit that collapses them must not also collapse the
-  // repos — a branch or daily build landing in orca-hourly would be offered to
-  // everyone riding main's hourlies.
-  it('keeps the dev channels on separate repos', () => {
-    withHourlyEnv((hourly) => {
-      withDailyEnv((daily) => {
-        withAdhocEnv((adhoc) => {
-          expect(new Set([hourly.publish.repo, daily.publish.repo, adhoc.publish.repo]).size).toBe(
-            3
-          )
-        })
-      })
-    })
-  })
 })

config/scripts/generated-artifact-line-ending-pin.test.mjs

@@ -16,10 +16,14 @@ it('keeps byte-compared artifacts and source LF in an autocrlf checkout', () =>
     ['src/renderer/src/example.tsx', lf],
     ['src/main/example.ts', lf],
     ['config/scripts/example.mjs', lf],
+    ['config/scripts/nested/deeper/example.mjs', lf],
     ...['cmd', 'bat', 'ps1', 'nsh'].map((extension) => [`config/example.${extension}`, crlf]),
     ['resources/skills-extra/example.json', crlf],
     ['vendor/resources/skills/example.json', crlf],
     ['scripts/example.mjs', crlf],
+    ['config/scripts-extra/example.mjs', crlf],
+    ['vendor/config/scripts/example.mjs', crlf],
+    ['config/scripts/example.mjsx', crlf],
     ['src/main/runtime/__fixtures__/example.txt', crlf],
     ['src/main/daemon/__fixtures__/pty-transcripts/example.txt', crlf]
   ])

config/scripts/happy-dom-offscreen-canvas.ts

@@ -49,7 +49,7 @@ export function installHappyDomOffscreenCanvasCompatibility(): boolean {
       return context
     }
 
-    const canvas = document.createElement('canvas') as PatchableCanvas
+    const canvas = document.createElement('canvas')
     return htmlCanvas.prototype.getContext!.call(canvas, contextType, contextAttributes)
   }
   prototype[OFFSCREEN_CANVAS_COMPATIBILITY_INSTALLED] = true

config/scripts/locale-ja-value-overrides.mjs

@@ -433,8 +433,6 @@ const JA_AUDIT_OVERRIDES = {
     'ステージ済みの変更からコミットメッセージを生成します。',
   lan: 'LAN',
   deploy: 'deploy',
-  'Connect this source to check for Hermes automations in the remote profile.':
-    'このソースに接続すると、リモートプロファイル内の Hermes オートメーションを確認できます。',
   'Notes sent to active agent.': 'メモをアクティブな Agent に送信しました。',
   'Review before attaching. Captured page context may include visible site content.':
     '添付する前に確認してください。取得したページのコンテキストには、表示中のサイトの内容が含まれる場合があります。',

config/scripts/locale-ko-key-overrides.test.mjs

@@ -1,14 +0,0 @@
-import fs from 'node:fs'
-
-import { describe, expect, it } from 'vitest'
-
-describe('locale-ko-key-overrides', () => {
-  it('keeps Korean key override data scoped to Korean values', () => {
-    const overrides = JSON.parse(
-      fs.readFileSync(new URL('./locale-ko-key-overrides.json', import.meta.url), 'utf8')
-    )
-    for (const value of Object.values(overrides)) {
-      expect(Object.keys(value)).toEqual(['ko'])
-    }
-  })
-})

config/scripts/locale-ko-value-overrides.mjs

@@ -293,6 +293,5 @@ export const KO_VALUE_OVERRIDES = {
   Server: '서버',
   Web: '웹',
   Code: '코드',
-  Folder: '폴더',
-  'Hermes automation created.': 'Hermes 자동화가 생성되었습니다.'
+  Folder: '폴더'
 }

config/scripts/localization-extraction-call-syntax.test.mjs

@@ -53,4 +53,47 @@ describe.each([
       rmSync(directory, { recursive: true, force: true })
     }
   })
+
+  it('preserves cross-file namespaces, source locations and fresh plugin transformations', async () => {
+    const directory = mkdtempSync(join(tmpdir(), 'orca-extraction-passes-'))
+    const file = join(directory, 'a-consumer.ts')
+    const config = {
+      locales: ['en'],
+      extract: {
+        input: [join(directory, '*.ts').replaceAll('\\', '/')],
+        functions: ['t', '*.t'],
+        disablePlurals: true
+      }
+    }
+    try {
+      writeFileSync(join(directory, 'z-constants.ts'), "export const NS = 'probe' as const\n")
+      const source = [
+        "import { NS } from './z-constants'",
+        '/* 🙂 */ const { t } = useTranslation(NS);',
+        "t('first', 'First')"
+      ].join('\n')
+      writeFileSync(file, source)
+      const first = await extractKeys(config)
+      expect([...first.allKeys.values()]).toMatchObject([
+        { key: 'first', ns: 'probe', locations: [{ file, line: 3, column: 2 }] }
+      ])
+
+      writeFileSync(file, source.replace("'first'", "'updated'"))
+      const updated = await extractKeys(config)
+      expect([...updated.allKeys.values()].map((entry) => entry.key)).toEqual(['updated'])
+
+      const transformed = await extractKeys({
+        ...config,
+        plugins: [
+          {
+            name: 'fixture-transform',
+            onLoad: (code) => code.replace("'updated'", "'transformed'")
+          }
+        ]
+      })
+      expect([...transformed.allKeys.values()].map((entry) => entry.key)).toEqual(['transformed'])
+    } finally {
+      rmSync(directory, { recursive: true, force: true })
+    }
+  })
 })

config/scripts/mobile-release-check-scope.test.mjs

@@ -102,7 +102,8 @@ it('gates Ruby independently and retains mobile static validation', () => {
   for (const file of [
     'config/scripts/mobile-release-check-scope.mjs',
     'config/scripts/mobile-release-check-scope.test.mjs',
-    'config/scripts/pr-code-change-scope.mjs'
+    'config/scripts/pr-code-change-scope.mjs',
+    'config/scripts/pr-package-test-paths.mjs'
   ]) {
     expect(workflow.on.pull_request.paths.some((pattern) => matchesGlob(file, pattern))).toBe(true)
   }
@@ -137,6 +138,7 @@ function fixture(detectorStep = detector) {
   for (const file of [
     'package.json',
     'config/scripts/pr-code-change-scope.mjs',
+    'config/scripts/pr-package-test-paths.mjs',
     'config/scripts/pr-layout-oracle-route.mjs',
     'config/scripts/mobile-release-check-scope.mjs',
     'config/scripts/mobile-test-change-scope.mjs'

config/scripts/mobile-test-change-scope.mjs

@@ -8,6 +8,7 @@ const NON_TEST_FILES = new Set([
   'mobile/README.md',
   '.github/workflows/mobile-ios-release.yml',
   'config/scripts/pr-code-change-scope.mjs',
+  'config/scripts/pr-package-test-paths.mjs',
   'config/scripts/pr-code-change-scope.test.mjs',
   'config/scripts/pr-layout-oracle-route.mjs',
   'config/scripts/mobile-release-check-scope.mjs',

config/scripts/mobile-test-change-scope.test.mjs

@@ -12,6 +12,7 @@ it.each([
   'mobile/fastlane/Fastfile',
   '.github/workflows/mobile-ios-release.yml',
   'config/scripts/pr-code-change-scope.mjs',
+  'config/scripts/pr-package-test-paths.mjs',
   'config/scripts/mobile-release-check-scope.test.mjs'
 ])('skips only known non-test inputs: %s', (file) => {
   expect(shouldRunMobileTests([file])).toBe(false)

config/scripts/mobile-web-app-browser-pane-text-inputs.test.mjs

@@ -1,108 +0,0 @@
-/**
- * The browser pane's two text inputs, read the way C4.2's census reads them.
- *
- * The pane is not in any page route today, so no route closure reaches it and the census that
- * enforces the seam is not run against it: it walks the source-control hub and the review route,
- * and the pane is in neither. C6 raises both inputs anyway, because the failure is not cosmetic —
- * an input under 16px makes an iOS browser zoom the page on focus, and `keyboard-occlusion.web.ts`
- * reads a visual viewport scale other than 1 as "no keyboard" for the rest of the typing session.
- *
- * This file is the census run by hand over the pane's own modules, so the raise is checked by the
- * rule that will judge it once a route lists the pane.
- */
-import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
-import { tmpdir } from 'node:os'
-import { join } from 'node:path'
-import { fileURLToPath } from 'node:url'
-import { describe, expect, it } from 'vitest'
-import {
-  modulesDeclaringTextInput,
-  textInputFontSizeOffenders,
-  unresolvedTextInputStyles
-} from './mobile-web-app-text-input-font-size-seam.mjs'
-
-const mobileDir = join(fileURLToPath(new URL('../..', import.meta.url)), 'mobile')
-
-/**
- * The pane's text-input modules, named as the bundler resolves them: a `.web.ts` where one exists,
- * which is what `mobileWebAppRouteClosure` reports for a real route.
- */
-const PANE_CLOSURE = {
-  local: [
-    'src/browser/MobileBrowserPaneView.tsx',
-    'src/browser/MobileBrowserAddressField.tsx',
-    'src/browser/mobile-browser-pane-styles.ts',
-    'src/browser/browser-address-field-styles.web.ts',
-    'src/platform/text-input-font-size.web.ts'
-  ]
-}
-
-const KEY_ROW_STYLES = 'src/browser/mobile-browser-pane-styles.ts'
-
-/** The modules in `PANE_CLOSURE` that render an input, which is what the rule below judges. */
-const PANE_INPUT_MODULES = [
-  'src/browser/MobileBrowserAddressField.tsx',
-  'src/browser/MobileBrowserPaneView.tsx'
-]
-
-function offendingFiles() {
-  return textInputFontSizeOffenders(mobileDir, PANE_CLOSURE).map((entry) =>
-    entry.slice(0, entry.lastIndexOf(':'))
-  )
-}
-
-describe('the browser pane text inputs under the C4.2 census', () => {
-  /**
-   * The precondition the rest of this file rests on.
-   *
-   * `PANE_CLOSURE` is written by hand, because the pane is in no page route and there is no
-   * closure to derive it from. So "no unresolved styles" says the walk read the files listed, not
-   * that those files are the pane's inputs — a third input module added under `src/browser/` would
-   * leave every assertion below green and unchecked. This scans for it instead.
-   */
-  it('lists every module under src/browser that renders an input', () => {
-    expect(modulesDeclaringTextInput(mobileDir, 'src/browser')).toEqual(PANE_INPUT_MODULES)
-    expect(PANE_CLOSURE.local).toEqual(expect.arrayContaining(PANE_INPUT_MODULES))
-  })
-
-  it('reads every input the pane declares, so the verdicts below are complete', () => {
-    // The completeness half: an empty offender list means nothing if the walk found no input.
-    expect(unresolvedTextInputStyles(mobileDir, PANE_CLOSURE)).toEqual([])
-    expect(PANE_CLOSURE.local).toContain('src/platform/text-input-font-size.web.ts')
-  })
-
-  // A scan that happens to find two things is not a scan that would find a third. Planted in a
-  // scratch tree rather than in src/browser, so no other census ever walks the plant.
-  it('would report a third input module, and ignores tests and non-JSX mentions', () => {
-    const scratch = mkdtempSync(join(tmpdir(), 'orca-browser-pane-inputs-'))
-    try {
-      mkdirSync(join(scratch, 'src/browser'), { recursive: true })
-      for (const [name, source] of Object.entries({
-        'Planted.tsx': 'export const P = () => <TextInput style={styles.input} />',
-        'Planted.test.tsx': 'it("x", () => <TextInput />)',
-        'mentions-only.ts': "import { TextInput } from 'react-native' // TextInput, named twice"
-      })) {
-        writeFileSync(join(scratch, 'src/browser', name), source)
-      }
-
-      expect(modulesDeclaringTextInput(scratch, 'src/browser')).toEqual(['src/browser/Planted.tsx'])
-    } finally {
-      rmSync(scratch, { recursive: true, force: true })
-    }
-  })
-
-  it('takes the key row input through the seam', () => {
-    expect(offendingFiles()).not.toContain(KEY_ROW_STYLES)
-  })
-
-  /**
-   * The address field is split, and the split is what the census now reads.
-   *
-   * Native keeps the 12px the toolbar has always rendered, because no phone has a page to zoom;
-   * the browser gets the seam. `resolveLocal` follows the `.web.ts` the builder would have
-   * resolved, so the size judged here is the size the page runs rather than the one it does not.
-   */
-  it('takes the address field through the seam, through its .web sibling', () => {
-    expect(offendingFiles()).toEqual([])
-  })
-})

config/scripts/mobile-web-app-module-closure.test.mjs

@@ -1,20 +0,0 @@
-import { mkdir, realpath, writeFile } from 'node:fs/promises'
-import { join } from 'node:path'
-import { expect, it } from 'vitest'
-import { mobileWebAppModuleClosure } from './build-mobile-web-app-bundle.mjs'
-import { withScratch } from './mobile-web-app-bundle-test-fixture.mjs'
-
-it.each(['app/_layout', 'app/_layout.tsx'])(
-  'resolves %s from the supplied directory without a TypeScript baseUrl',
-  async (entry) => {
-    await withScratch(async (scratch) => {
-      const root = await realpath(scratch)
-      await mkdir(join(root, 'app'))
-      await writeFile(join(root, 'app/_layout.tsx'), "export default 'native'")
-      await writeFile(join(root, 'app/_layout.web.tsx'), "export default 'web'")
-      const closure = await mobileWebAppModuleClosure([entry], { absWorkingDir: root })
-      expect(closure.modules).toEqual(['app/_layout.web.tsx'])
-      expect(closure.local).toEqual(['app/_layout.web.tsx'])
-    })
-  }
-)

config/scripts/mobile-web-app-safe-area-render.test.mjs

@@ -14,7 +14,7 @@ import {
 } from './mobile-web-app-render-harness.mjs'
 
 // The page lays out edge-to-edge under the shell's system bars and pads through its own
-// SafeAreaViews, by the insets `init` carries. Measured in a real browser because what is under
+// SafeAreaViews, and its root for the sides, by the insets `init` carries. Measured in a real browser because what is under
 // test is which providers and navigators expo-router actually mounts above a screen.
 const HOST_ROUTE = '/h/render-check-host'
 const HOST = {
@@ -106,7 +106,9 @@ function readEdges(page) {
     )
     return {
       titleTop: title === null ? null : range.getBoundingClientRect().top,
+      titleLeft: title === null ? null : range.getBoundingClientRect().left,
       fabBottom: fab?.getBoundingClientRect().bottom ?? null,
+      fabRight: fab?.getBoundingClientRect().right ?? null,
       envTop: (() => {
         const probe = document.createElement('div')
         probe.style.paddingTop = 'env(safe-area-inset-top)'
@@ -142,6 +144,19 @@ describeRender('the page pads for the system bars once, by the shell insets', ()
     expect(padded.titleTop - zero.titleTop).toBe(INSETS.top)
   }, 90_000)
 
+  // Landscape puts the iPhone sensor housing on a side. Unequal sides, so a swapped edge fails.
+  it('keeps the screens clear of the side insets, padding them once', async () => {
+    const upright = await openHostRoute(INSETS)
+    const base = await readEdges(upright.page)
+    await upright.page.close()
+    const sideways = await openHostRoute({ ...INSETS, left: 59, right: 44 })
+    const inset = await readEdges(sideways.page)
+    await sideways.page.close()
+    expect([...upright.errors, ...sideways.errors]).toEqual([])
+    expect(inset.titleLeft - base.titleLeft).toBe(59)
+    expect(base.fabRight - inset.fabRight).toBe(44)
+  }, 60_000)
+
   it('follows a re-sent init, which is how the keyboard and rotation reach the page', async () => {
     const opened = await openHostRoute(INSETS)
     const before = await readEdges(opened.page)

config/scripts/mobile-web-app-session-terminal-closure.test.mjs

@@ -1,6 +1,6 @@
 import { join } from 'node:path'
 import { fileURLToPath } from 'node:url'
-import { describe, expect, it } from 'vitest'
+import { beforeAll, describe, expect, it } from 'vitest'
 import { mobileWebAppRouteClosure } from './build-mobile-web-app-bundle.mjs'
 import { mobileWebAppDependenciesPresent } from './mobile-web-app-bundle-dependencies.mjs'
 import { mobileWebAppRouteChunkClosure } from './mobile-web-app-route-chunk-closure.mjs'
@@ -478,8 +478,13 @@ const describeClosure = bundles ? describe : describe.skip
 describeClosure(
   "the session route's page closure with the terminal on it",
   () => {
+    let closure
+    beforeAll(async () => {
+      closure = await mobileWebAppRouteClosure(SESSION_ROUTE)
+    }, 300_000)
+
     it('gains the document, xterm and the addons, and sheds the engine string', async () => {
-      const { local, modules } = await mobileWebAppRouteClosure(SESSION_ROUTE)
+      const { local, modules } = closure
       for (const gone of SHED) {
         expect(local, `${gone} is still in the closure`).not.toContain(gone)
       }
@@ -506,7 +511,7 @@ describeClosure(
     }, 300_000)
 
     it('enters through the web sibling and the route body, not the switch', async () => {
-      const { local } = await mobileWebAppRouteClosure(SESSION_ROUTE)
+      const { local } = closure
       for (const entry of ROUTE_ENTRY) {
         expect(local, `${entry} is not in the closure`).toContain(entry)
       }
@@ -518,7 +523,7 @@ describeClosure(
     }, 300_000)
 
     it('reaches the engine as one deferred module and never as part of the download', async () => {
-      const { modules } = await mobileWebAppRouteClosure(SESSION_ROUTE)
+      const { modules } = closure
       // The engine is here, as the one artifact the loader imports.
       expect(artifactModules(modules)).toHaveLength(1)
       // Package sources stay inside that artifact; unrelated module counts are not this boundary.
@@ -537,16 +542,17 @@ describeClosure(
     }, 600_000)
 
     it('leaves the 16px seam census exactly where C7.2 left it', async () => {
-      const closure = await mobileWebAppRouteClosure(SESSION_ROUTE)
       // Two preconditions, because zero offenders is what a walk that read nothing also reports:
       // the seam's own web module has to be in the closure, and no style may be unresolved.
       expect(closure.local).toContain('src/platform/text-input-font-size.web.ts')
+      expect(closure.local.length).toBeGreaterThan(900)
       expect(unresolvedTextInputStyles(mobileDir, closure)).toEqual([])
       expect(textInputFontSizeOffenders(mobileDir, closure)).toHaveLength(EXPECTED_OFFENDERS)
+      expect(closure.local).toContain('src/session/mobile-native-chat-input-styles.web.ts')
+      expect(closure.local).not.toContain('src/session/mobile-native-chat-input-styles.ts')
     }, 300_000)
 
     it('holds the editables the TextInput census cannot see to the same floor', async () => {
-      const closure = await mobileWebAppRouteClosure(SESSION_ROUTE)
       // The rich Markdown editor's surface is a `contenteditable` in a markup string, sized by a
       // rule in a stylesheet: `modulesDeclaringTextInput` matches JSX tags and never sees it, so it
       // shipped at 14 px and was measured at 14 px in both engines. The same floor, read by a rule

config/scripts/mobile-web-app-session-text-inputs.test.mjs

@@ -1,76 +0,0 @@
-/**
- * Every text input the session screen reaches, and the size it declares.
- *
- * iOS zooms the page on focus of any input under 16px and does not zoom back out, so the document
- * spends the rest of that typing session at a scale other than 1 — which `keyboard-occlusion.web.ts`
- * reads as "not a keyboard" on purpose, because geometry cannot separate a zoom from a keyboard.
- *
- * This closure is the one that cannot afford it. The terminal's own input is a hidden field the
- * keyboard seam's geometry is the whole basis of, and this screen reaches nine inputs that declare
- * a size — a custom-key capture field, a prompt modal, the chat's ask/composer/question fields, the
- * quick-command editor and its search, and the terminal command bar. One of them left off the seam
- * leaves every later focus on this screen measuring a zoomed document.
- *
- * The rule is the closure rather than the nine sites it happens to have today: a module entering it
- * later is held to it without anyone remembering to add it here.
- */
-import { fileURLToPath } from 'node:url'
-import { describe, expect, it } from 'vitest'
-import { mobileWebAppRouteClosure } from './build-mobile-web-app-bundle.mjs'
-import { mobileWebAppDependenciesPresent } from './mobile-web-app-bundle-dependencies.mjs'
-import {
-  TEXT_INPUT_FONT_SIZE_SEAM,
-  textInputFontSizeOffenders,
-  unresolvedTextInputStyles
-} from './mobile-web-app-text-input-font-size-seam.mjs'
-
-const mobileDir = fileURLToPath(new URL('../../mobile/', import.meta.url))
-const describeClosure = mobileWebAppDependenciesPresent() ? describe : describe.skip
-
-const SESSION = 'app/h/[hostId]/session/[worktreeId].tsx'
-
-/**
- * The style module this screen splits, as the page bundle resolves it.
- *
- * Named rather than left to the offender list because a split is the one fix that can be undone
- * without reopening the offence: delete the `.web.ts` and the native sibling's size is what the
- * page runs, which is a 15px chat composer and a zoomed document, and the offender list would say
- * so — but only the next time someone reads it. Listed here, the closure says which file the page
- * loads.
- */
-const SPLIT_WEB_STYLES = ['src/session/mobile-native-chat-input-styles.web.ts']
-
-describeClosure(
-  'the text inputs the session screen reaches',
-  () => {
-    it('takes every input size through the seam', async () => {
-      const closure = await mobileWebAppRouteClosure(SESSION)
-      expect(textInputFontSizeOffenders(mobileDir, closure)).toEqual([])
-    })
-
-    it('reads every input it found, so the list above is complete', async () => {
-      // The completeness half: an empty offender list is evidence only if every input was read.
-      const closure = await mobileWebAppRouteClosure(SESSION)
-      expect(unresolvedTextInputStyles(mobileDir, closure)).toEqual([])
-    })
-
-    it('carries the seam, so the rule is not vacuous', async () => {
-      // Without this an empty offender list would also be what a closure reaching no text input at
-      // all produces, and the census would pass against a page that has nothing to raise.
-      const closure = await mobileWebAppRouteClosure(SESSION)
-      expect(closure.local).toContain(TEXT_INPUT_FONT_SIZE_SEAM)
-      expect(closure.local.length).toBeGreaterThan(900)
-    })
-
-    it('loads the web half of the split style module, not the native one', async () => {
-      const closure = await mobileWebAppRouteClosure(SESSION)
-      expect(closure.local).toEqual(expect.arrayContaining(SPLIT_WEB_STYLES))
-      expect(
-        closure.local.filter((file) =>
-          SPLIT_WEB_STYLES.some((web) => file === web.replace('.web.ts', '.ts'))
-        )
-      ).toEqual([])
-    })
-  },
-  240_000
-)