Merge remote-tracking branch 'origin/main' into pr18513

.gitattributes

@@ -4,11 +4,24 @@
 /config/scripts/**/*.mjs text eol=lf
 /skill-guides/*.md text eol=lf
 /skill-stubs/*.md text eol=lf
+/skill-stubs/_shared/*.md text eol=lf
 /skills/*/SKILL.md text eol=lf
 /src/cli/bundled-skill-guides.ts text eol=lf
 # Bundled plugin trees are byte-hashed; CRLF checkout would break the pinned hash.
 /resources/plugins/** text eol=lf
-# pnpm hashes every patch byte-for-byte, so a CRLF checkout breaks the install.
+# Relay assets are copied verbatim into the bundle and hashed byte-for-byte into
+# .version, which names the immutable remote install dir. A CRLF checkout makes a
+# Windows-built client disagree with a mac/Linux-built one on the same release,
+# so one host ends up with two relay trees (#17886 review).
+/config/relay-assets/** text eol=lf
+# Pin the bytes so a patch reads and diffs identically on every host. It is NOT
+# what makes the hash right: pnpm hashes a patch LF-normalized, so a CRLF checkout
+# cannot change it. Believing otherwise put a hand-computed raw digest in the
+# lockfile twice and broke every install (#17886).
+# These files are stored LF, which is not always the encoding they were written
+# against -- @vscode/windows-process-tree ships CRLF sources -- so any code that
+# runs `git apply` on one must force `-c core.autocrlf=input` rather than trust
+# the host's setting. See config/scripts/windows-process-tree-gyp-rebuild.mjs.
 /config/patches/*.patch -text
 # The xterm bundle hunks also make a diff nobody can read; review the hand-written
 # source patch under xterm-src/ instead. The sibling patches stay diffable.

.github/actions/install-node-dependencies/action.yml

@@ -39,6 +39,9 @@ runs:
       with:
         install: false
 
+    # Why both lockfiles: setup-node keys the pnpm store on the root lockfile alone, so
+    # jobs that also install mobile restored a store with none of the React Native tree
+    # in it and re-downloaded the lot on every run.
     - name: Setup Node.js
       id: default-node
       if: inputs.node-version == ''
@@ -46,6 +49,9 @@ runs:
       with:
         node-version-file: package.json
         cache: pnpm
+        cache-dependency-path: |
+          pnpm-lock.yaml
+          mobile/pnpm-lock.yaml
 
     - name: Setup requested Node.js
       id: requested-node
@@ -54,6 +60,9 @@ runs:
       with:
         node-version: ${{ inputs.node-version }}
         cache: pnpm
+        cache-dependency-path: |
+          pnpm-lock.yaml
+          mobile/pnpm-lock.yaml
 
     - name: Validate native runtime
       shell: bash
@@ -68,14 +77,6 @@ runs:
             ;;
         esac
 
-    # pnpm's bundled gyp_main.py is not executable on fresh Linux runners.
-    - name: Use external node-gyp
-      if: runner.os == 'Linux' && inputs.native-runtime != 'none'
-      shell: bash
-      run: |
-        npm install -g node-gyp@11.5.0
-        echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV"
-
     - name: Prepare dependency install
       shell: bash
       run: |
@@ -166,6 +167,22 @@ runs:
           node_modules/.pnpm/@vscode+windows-process-tree@*/node_modules/@vscode/windows-process-tree/build
         key: native-modules-${{ runner.os }}-${{ steps.native-cache-scope.outputs.scope }}-${{ runner.arch }}-${{ inputs.native-runtime }}-node${{ steps.requested-node.outputs.node-version || steps.default-node.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch') }}
 
+    # pnpm's bundled gyp_main.py is not executable on fresh Linux runners.
+    - name: Use external node-gyp
+      if: runner.os == 'Linux' && inputs.native-runtime != 'none'
+      shell: bash
+      env:
+        NATIVE_RUNTIME: ${{ inputs.native-runtime }}
+        NATIVE_CACHE_HIT: ${{ steps.native-cache-restore.outputs.cache-hit || steps.native-cache-restore-only.outputs.cache-hit }}
+      run: |
+        # A cache hit can contain unusable addons; probe before skipping the rebuild toolchain.
+        if [ "$NATIVE_RUNTIME" = node ] && [ "$NATIVE_CACHE_HIT" = true ] &&
+          node config/scripts/ensure-native-runtime.mjs --check-only; then
+          exit 0
+        fi
+        npm install -g node-gyp@11.5.0
+        echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV"
+
     - name: Prepare native runtime
       if: inputs.native-runtime != 'none'
       shell: bash

.github/actions/setup-wsl-test-runtime/action.yml

@@ -0,0 +1,8 @@
+name: Set up WSL test runtime
+description: Install a checksum-pinned Ubuntu WSL1 guest with executable Node and Git for real terminal tests.
+runs:
+  using: composite
+  steps:
+    - name: Provision Ubuntu WSL1
+      shell: pwsh
+      run: '& "${{ github.action_path }}/setup.ps1"'

.github/actions/setup-wsl-test-runtime/setup.ps1

@@ -0,0 +1,32 @@
+$ErrorActionPreference = 'Stop'
+if (-not $IsWindows) { throw 'WSL test provisioning requires a Windows runner' }
+
+$rootfs = Join-Path $env:RUNNER_TEMP 'noble-rootfs.tar.gz'
+Invoke-WebRequest 'https://releases.ubuntu.com/24.04.4/ubuntu-24.04.4-wsl-amd64.wsl' -OutFile $rootfs
+if ((Get-FileHash $rootfs -Algorithm SHA256).Hash.ToLowerInvariant() -ne '9b2f7730dc68227dd04a9f3e5eab86ad85caf556b8606ad94f1f29ff5c4fd3f5') { throw 'Ubuntu rootfs checksum mismatch' }
+$distroDir = Join-Path $env:RUNNER_TEMP 'orca-wsl-ubuntu'
+wsl.exe --import Ubuntu $distroDir $rootfs --version 1
+if ($LASTEXITCODE -ne 0) { throw "WSL import failed: $LASTEXITCODE" }
+wsl.exe --distribution Ubuntu --user root --exec /usr/bin/true
+if ($LASTEXITCODE -ne 0) { throw "WSL guest did not start: $LASTEXITCODE" }
+wsl.exe --distribution Ubuntu --user root --exec /usr/bin/apt-get update
+if ($LASTEXITCODE -ne 0) { throw "WSL apt update failed: $LASTEXITCODE" }
+wsl.exe --distribution Ubuntu --user root --exec /usr/bin/apt-get install --yes git curl xz-utils
+if ($LASTEXITCODE -ne 0) { throw "WSL git install failed: $LASTEXITCODE" }
+$kernelMsi = Join-Path $env:RUNNER_TEMP 'wsl_update_x64.msi'
+Invoke-WebRequest 'https://wslstorestorage.blob.core.windows.net/wslblob/wsl_update_x64.msi' -OutFile $kernelMsi
+if ((Get-FileHash $kernelMsi -Algorithm SHA256).Hash.ToLowerInvariant() -ne '4d09c776c8d45f70a202281d18e19be1118f53159b0c217a5274a31ce18525fe') { throw 'WSL kernel installer checksum mismatch' }
+$installer = Start-Process msiexec.exe -ArgumentList @('/i', $kernelMsi, '/quiet', '/norestart') -Wait -PassThru
+if ($installer.ExitCode -ne 0) { throw "WSL kernel installation failed: $($installer.ExitCode)" }
+wsl.exe --status
+if ($LASTEXITCODE -ne 0) { throw "WSL status failed: $LASTEXITCODE" }
+wsl.exe --distribution Ubuntu --user root --exec /usr/bin/curl --fail --silent --show-error --location https://nodejs.org/dist/v22.14.0/node-v22.14.0-linux-x64.tar.xz --output /tmp/orca-node.tar.xz
+if ($LASTEXITCODE -ne 0) { throw 'Node download failed' }
+$nodeHash = wsl.exe --distribution Ubuntu --user root --exec /usr/bin/sha256sum /tmp/orca-node.tar.xz
+if ($LASTEXITCODE -ne 0 -or -not ($nodeHash -match '^69b09dba5c8dcb05c4e4273a4340db1005abeafe3927efda2bc5b249e80437ec')) { throw 'Node checksum mismatch' }
+wsl.exe --distribution Ubuntu --user root --exec /usr/bin/tar -xJf /tmp/orca-node.tar.xz -C /usr/local --strip-components=1
+if ($LASTEXITCODE -ne 0) { throw 'Node extraction failed' }
+wsl.exe --distribution Ubuntu --user root --exec /usr/local/bin/node --version
+if ($LASTEXITCODE -ne 0) { throw 'Node cannot execute in WSL' }
+wsl.exe --list --verbose
+if ($LASTEXITCODE -ne 0) { throw "WSL enumeration failed: $LASTEXITCODE" }

.github/scripts/e2e-with-window-manager.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+set -euo pipefail
+openbox --sm-disable > /tmp/orca-e2e-window-manager.log 2>&1 &
+wm_pid=$!
+cleanup() {
+  kill "$wm_pid" 2>/dev/null || true
+  wait "$wm_pid" 2>/dev/null || true
+}
+trap cleanup EXIT
+ready=false
+for attempt in {1..100}; do
+  if xprop -root _NET_SUPPORTING_WM_CHECK 2>/dev/null | rg -q 'window id # 0x[1-9a-fA-F]'; then
+    ready=true
+    break
+  fi
+  if ! kill -0 "$wm_pid" 2>/dev/null; then
+    cat /tmp/orca-e2e-window-manager.log
+    exit 1
+  fi
+  sleep 0.1
+done
+if [ "$ready" != true ]; then
+  echo 'Window manager did not acquire the Xvfb root window' >&2
+  exit 1
+fi
+"$@"

.github/workflows/adhoc-mac-build.yml

@@ -127,9 +127,12 @@ jobs:
           esac
           # Bare: a work-tree repo refuses to fetch over its own checked-out
           # branch. tree:0 keeps the fetch to the commit graph — no trees, no
-          # blobs — so this stays cheap next to the build it fronts.
+          # blobs — so this stays cheap next to the build it fronts. reftable
+          # because this repo has branches that differ only in casing, and the
+          # files backend cannot store both on a case-insensitive runner disk —
+          # it fails the entire fetch, not just the one ref.
           scratch="$RUNNER_TEMP/vet-requested-ref"
-          git init -q --bare "$scratch"
+          git init -q --bare --ref-format=reftable "$scratch"
           git -C "$scratch" fetch -q --filter=tree:0 "$REPO_URL" '+refs/heads/*:refs/heads/*' '+refs/tags/*:refs/tags/*'
           # Branch first to keep actions/checkout's old tie-break: bare
           # rev-parse would prefer the tag when a branch shares its name.
@@ -157,6 +160,9 @@ jobs:
 
       - name: Checkout the requested ref
         uses: actions/checkout@v6
+        env:
+          # Full-history checkout must also preserve case-twin branch and tag names.
+          GIT_DEFAULT_REF_FORMAT: reftable
         with:
           # Why an input at all rather than just github.ref: the whole point is to
           # build code that has not landed, and the workflow definition itself

.github/workflows/cloud-deploy-relay-production-director.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       image-digest:
-        description: "Immutable relay image digest (sha256: plus 64 lowercase hex characters)"
+        description: 'Immutable relay image digest (sha256: plus 64 lowercase hex characters)'
         required: true
         type: string
       regional-placement-mode:

.github/workflows/cloud-deploy-relay-production-same-cap-job.yml

@@ -91,7 +91,11 @@ jobs:
           test -n "${CAPACITY_SERVICE_ACCOUNT}"
           test -n "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}"
 
+      # Full history: the monitor evidence this job verifies is sealed at an ancestor commit,
+      # and the provenance check fails closed on a commit a shallow clone left out.
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - uses: pnpm/action-setup@v4
         with: { package_json_file: cloud/package.json }
@@ -177,11 +181,12 @@ jobs:
         env:
           ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
         run: |
-          RETRY_ARGS=()
-          if test "${WAVE_INDEX}" != 0; then RETRY_ARGS=(--retry-freshness); fi
+          # Freshness-only failures are publish lag, not health, on every wave
+          # including the first; the CLI still caps the retry at the wave's
+          # evidence-age budget, so this cannot mutate on aged evidence.
           pnpm incident:relay-preflight -- \
             --state-file "${OUTPUT_DIRECTORY}/relay-${MONITOR_RUN_ID}-dry-run.state.json" \
-            --wave-index "${WAVE_INDEX}" "${RETRY_ARGS[@]}"
+            --wave-index "${WAVE_INDEX}" --retry-freshness
 
       - name: Require durable rehome disabled and exact selector
         env:
@@ -271,10 +276,22 @@ jobs:
         env:
           ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
         run: |
-          CURRENT_RUNTIME="$(curl --fail-with-body --max-time 30 \
-            --request POST "${CELL_ORIGIN}/v1/admin/runtime-status" \
-            --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
-            --header 'Content-Type: application/json' --data '{"v":1}')"
+          # A single transient 5xx (LB warm-up behind a fresh instance) must not
+          # fail a canary; 4xx (auth, generation mismatch) still fails fast.
+          admin_post() {
+            local out="${RUNNER_TEMP}/$1.json"
+            if ! curl --fail-with-body --max-time 30 \
+              --retry 3 --retry-delay 2 --retry-connrefused --output "${out}" \
+              --request POST "$2" \
+              --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
+              --header 'Content-Type: application/json' --data "$3"; then
+              cat "${out}" >&2
+              return 1
+            fi
+            cat "${out}"
+          }
+          CURRENT_RUNTIME="$(admin_post current-runtime \
+            "${CELL_ORIGIN}/v1/admin/runtime-status" '{"v":1}')"
           # A rollback that failed between template apply and admission restore
           # leaves the cell already on the rollback image; resume from that
           # state instead of demanding the pre-rollback predecessor.
@@ -370,11 +387,9 @@ jobs:
             if .regionalRehomeProtocol == null then "regionalRehomeProtocol" else empty end
           ] | if length > 0 then "runtime predecessor normalized legacy fields=" + join(",") else empty end' \
             <<< "${CURRENT_RUNTIME}"
-          CURRENT_DIRECTOR_STATUS="$(curl --fail-with-body --max-time 30 \
-            --request POST "${DIRECTOR_ORIGIN}/v1/admin/cell-status" \
-            --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
-            --header 'Content-Type: application/json' \
-            --data "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"
+          CURRENT_DIRECTOR_STATUS="$(admin_post current-cell-status \
+            "${DIRECTOR_ORIGIN}/v1/admin/cell-status" \
+            "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"
           SOURCE_INCARNATION="$(jq -er '.status.runtime.cellIncarnation' \
             <<< "${CURRENT_DIRECTOR_STATUS}")"
           if test "${ROLLBACK_RESUME}" = true && ! jq -e \
@@ -418,13 +433,13 @@ jobs:
           # result's generation is authoritative either way.
           ISOLATE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
             --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
-            --cell-id "${TARGET_CELL_ID}" --mode isolate)"
+            --cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode isolate)"
           echo "${ISOLATE_RESULT}"
           ISOLATE_GENERATION="$(jq -er '.generation' <<< "${ISOLATE_RESULT}")"
           echo "SELECTOR_GENERATION_AFTER_ISOLATE=${ISOLATE_GENERATION}" >> "${GITHUB_ENV}"
           node dev/scripts/prepare-relay-production-capacity-canary.mjs \
             --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
-            --cell-id "${TARGET_CELL_ID}" --mode drain
+            --cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode drain
           node dev/scripts/verify-relay-capacity-transition.mjs \
             --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
             --cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
@@ -486,6 +501,7 @@ jobs:
                 --rollback-image "${DESIRED_IMAGE}" \
                 --rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
                 --rehome-audience https://relay.onorca.dev/v1/admin/host-drain \
+                --regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" \
               | jq -e '.changes == 2' >/dev/null
           fi
           gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
@@ -511,7 +527,8 @@ jobs:
               --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" --image "${DESIRED_IMAGE}" \
               --rollback-image "${IMAGE_REPOSITORY}@${CURRENT_IMAGE_DIGEST}" \
               --rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
-              --rehome-audience https://relay.onorca.dev/v1/admin/host-drain
+              --rehome-audience https://relay.onorca.dev/v1/admin/host-drain \
+              --regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}"
           terraform -chdir=infra/terraform apply -auto-approve \
             "${RUNNER_TEMP}/relay-same-cap.tfplan"
           gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
@@ -532,6 +549,20 @@ jobs:
         env:
           ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }}
         run: |
+          # A single transient 5xx (LB warm-up behind a fresh instance) must not
+          # fail a canary; 4xx (auth, generation mismatch) still fails fast.
+          admin_post() {
+            local out="${RUNNER_TEMP}/$1.json"
+            if ! curl --fail-with-body --max-time 30 \
+              --retry 3 --retry-delay 2 --retry-connrefused --output "${out}" \
+              --request POST "$2" \
+              --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
+              --header 'Content-Type: application/json' --data "$3"; then
+              cat "${out}" >&2
+              return 1
+            fi
+            cat "${out}"
+          }
           node dev/scripts/verify-relay-capacity-transition.mjs \
             --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
             --cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
@@ -539,19 +570,15 @@ jobs:
             --heartbeat fresh --admission migration-only --draining forbidden \
             --activity allowed --expected-image-digests "${DESIRED_IMAGE_DIGEST}" \
             --regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" --timeout-ms 900000
-          TARGET_RUNTIME="$(curl --fail-with-body --max-time 30 \
-            --request POST "${CELL_ORIGIN}/v1/admin/runtime-status" \
-            --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
-            --header 'Content-Type: application/json' --data '{"v":1}')"
+          TARGET_RUNTIME="$(admin_post target-runtime \
+            "${CELL_ORIGIN}/v1/admin/runtime-status" '{"v":1}')"
           jq -e --arg digest "${DESIRED_IMAGE_DIGEST}" \
             --argjson protocol "${DESIRED_REHOME_PROTOCOL}" \
             '.imageDigest == $digest and (.regionalRehomeProtocol // 0) == $protocol' \
             <<< "${TARGET_RUNTIME}" >/dev/null
-          TARGET_DIRECTOR_STATUS="$(curl --fail-with-body --max-time 30 \
-            --request POST "${DIRECTOR_ORIGIN}/v1/admin/cell-status" \
-            --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
-            --header 'Content-Type: application/json' \
-            --data "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"
+          TARGET_DIRECTOR_STATUS="$(admin_post target-cell-status \
+            "${DIRECTOR_ORIGIN}/v1/admin/cell-status" \
+            "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"
           TARGET_INCARNATION="$(jq -er '.status.runtime.cellIncarnation' \
             <<< "${TARGET_DIRECTOR_STATUS}")"
           if test "${ROLLBACK_RESUME}" = true; then
@@ -588,7 +615,7 @@ jobs:
           echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
           ACTIVATE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
             --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
-            --cell-id "${TARGET_CELL_ID}" --mode activate)"
+            --cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode activate)"
           echo "${ACTIVATE_RESULT}"
           SELECTOR_GENERATION_AFTER_ACTIVATE="$(jq -er '.generation' \
             <<< "${ACTIVATE_RESULT}")"
@@ -627,7 +654,7 @@ jobs:
           test "${MUTATION_STARTED:-false}" = true || exit 0
           ISOLATE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
             --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
-            --cell-id "${TARGET_CELL_ID}" --mode isolate)"
+            --cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode isolate)"
           echo "${ISOLATE_RESULT}"
           # The isolate result carries the authoritative post-isolate generation;
           # fixed offsets are wrong whenever an earlier isolate was a no-op.

.github/workflows/cloud-deploy-relay-production-same-cap.yml

@@ -87,13 +87,18 @@ jobs:
   gate:
     if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }}
     runs-on: blacksmith-2vcpu-ubuntu-2204
-    timeout-minutes: 10
+    # Headroom for the full-history checkout the canary provenance check needs.
+    timeout-minutes: 15
     environment: production
     outputs:
       cells: ${{ steps.wave.outputs.cells }}
       job-mode: ${{ steps.wave.outputs.job-mode }}
     steps:
+      # Full history: the canary authority a batch verifies is sealed at an ancestor commit, and
+      # the provenance check fails closed on a commit a shallow clone left out.
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - uses: actions/setup-node@v4
         with: { node-version: 24 }

.github/workflows/cloud-operate-relay-production-rehome-job.yml

@@ -14,6 +14,7 @@ on:
       not-before: { required: true, type: string }
       rate-per-minute: { required: true, type: string }
       preference-max-age-ms: { required: true, type: string }
+      host-cooldown-ms: { required: true, type: string }
       drain-grace-ms: { required: true, type: string }
       confirmation: { required: true, type: string }
       monitor-run-id: { required: true, type: string }
@@ -26,6 +27,9 @@ permissions:
 
 defaults:
   run:
+    # `shell: bash` adds pipefail; without it `node ... | tee` reports tee's exit code and a
+    # thrown inspect/apply passed green (Aug 28-29 and Sep 3 2026 runs).
+    shell: bash
     working-directory: cloud
 
 jobs:
@@ -51,6 +55,7 @@ jobs:
       NOT_BEFORE: ${{ inputs.not-before }}
       RATE_PER_MINUTE: ${{ inputs.rate-per-minute }}
       PREFERENCE_MAX_AGE_MS: ${{ inputs.preference-max-age-ms }}
+      HOST_COOLDOWN_MS: ${{ inputs.host-cooldown-ms }}
       DRAIN_GRACE_MS: ${{ inputs.drain-grace-ms }}
       CONFIRMATION: ${{ inputs.confirmation }}
       MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
@@ -92,7 +97,11 @@ jobs:
               ;;
           esac
 
+      # Full history: the monitor evidence this job verifies is sealed at an ancestor commit,
+      # and the provenance check fails closed on a commit a shallow clone left out.
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - uses: actions/setup-node@v4
         with:
@@ -121,6 +130,7 @@ jobs:
             --expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \
             --not-before "${NOT_BEFORE}" --rate-per-minute "${RATE_PER_MINUTE}" \
             --preference-max-age-ms "${PREFERENCE_MAX_AGE_MS}" \
+            --host-cooldown-ms "${HOST_COOLDOWN_MS}" \
             --drain-grace-ms "${DRAIN_GRACE_MS}" --confirmation "${CONFIRMATION}" \
             | tee "${RUNNER_TEMP}/relay-rehome-control.json"
 
@@ -292,6 +302,7 @@ jobs:
             --expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \
             --not-before "${NOT_BEFORE}" --rate-per-minute "${RATE_PER_MINUTE}" \
             --preference-max-age-ms "${PREFERENCE_MAX_AGE_MS}" \
+            --host-cooldown-ms "${HOST_COOLDOWN_MS}" \
             --drain-grace-ms "${DRAIN_GRACE_MS}" --confirmation "${CONFIRMATION}" \
             | tee "${RUNNER_TEMP}/relay-rehome-control.json"
 

.github/workflows/cloud-operate-relay-production-rehome.yml

@@ -52,6 +52,11 @@ on:
         required: true
         default: '86400000'
         type: string
+      host-cooldown-ms:
+        description: Minimum gap between two rehomes of the same host
+        required: true
+        default: '604800000'
+        type: string
       drain-grace-ms:
         description: Per-host source drain grace
         required: true
@@ -99,6 +104,7 @@ jobs:
       not-before: ${{ inputs.not-before }}
       rate-per-minute: ${{ inputs.rate-per-minute }}
       preference-max-age-ms: ${{ inputs.preference-max-age-ms }}
+      host-cooldown-ms: ${{ inputs.host-cooldown-ms }}
       drain-grace-ms: ${{ inputs.drain-grace-ms }}
       confirmation: ${{ inputs.confirmation }}
       monitor-run-id: ${{ inputs.monitor-run-id }}

.github/workflows/cloud-prove-relay-asia-staging.yml

@@ -55,9 +55,11 @@ jobs:
 
       - name: Validate the exact staging proof request
         shell: bash
+        env:
+          CONFIRMATION: ${{ inputs.confirmation }}
         run: |
           set -euo pipefail
-          test "${{ inputs.confirmation }}" = PROVE_ASIA_STAGING
+          test "${CONFIRMATION}" = PROVE_ASIA_STAGING
           [[ "${IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]
           [[ "${INITIAL_SELECTOR_GENERATION}" =~ ^[1-9][0-9]*$ ]]
           [[ "${PROMOTE_ATTEMPT_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]]

.github/workflows/cloud-push-deploy.yml

@@ -0,0 +1,566 @@
+name: Deploy Push Gateway Production
+
+on:
+  workflow_dispatch:
+    inputs:
+      source_sha:
+        description: Full reviewed commit SHA to build (feature may remain unmerged)
+        required: true
+        type: string
+      confirmation:
+        description: Enter DEPLOY_PUSH_GATEWAY to shift production traffic
+        required: true
+        type: string
+
+permissions:
+  contents: read
+  id-token: write
+
+# Serialize push traffic changes independently of Relay and the shared database.
+concurrency:
+  group: production-push-rollout
+  cancel-in-progress: false
+
+defaults:
+  run:
+    working-directory: cloud
+
+jobs:
+  deploy:
+    if: >-
+      ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' &&
+      github.ref == 'refs/heads/main' }}
+    runs-on: blacksmith-2vcpu-ubuntu-2204
+    environment: production
+    env:
+      GCP_PROJECT_ID: onorca-cloud
+      GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
+      SERVICE_NAME: orca-cloud-push
+      REPOSITORY_ID: orca-cloud
+      IMAGE_NAME: push
+      PUSH_ORIGIN: https://push.onorca.dev
+      PUSH_RUNTIME_SERVICE_ACCOUNT: orca-cloud-push@onorca-cloud.iam.gserviceaccount.com
+      # Scaling the serving revision must already hold, matching push_min_instances and
+      # push_max_instances. Terraform owns both, and the candidate inherits them from the
+      # service, so this deploy never passes a scaling flag: doing so would write a
+      # Terraform-owned field that `lifecycle.ignore_changes` does not cover, and a later
+      # `push_max_instances` raise would then be reverted by every deploy. These two values
+      # are the expected shape, asserted before the candidate is created and again on the
+      # candidate itself, so a deploy that would change the gateway's Cloud SQL draw fails.
+      PUSH_MIN_INSTANCES: 1
+      PUSH_MAX_INSTANCES: 2
+      CONFIRMATION: ${{ inputs.confirmation }}
+      SOURCE_SHA: ${{ inputs.source_sha }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Require the explicit deploy confirmation
+        shell: bash
+        run: |
+          set -euo pipefail
+          test "${CONFIRMATION}" = DEPLOY_PUSH_GATEWAY
+          [[ "${SOURCE_SHA}" =~ ^[a-f0-9]{40}$ ]]
+
+      # Keep the workflow and rollout lease on main; only the Docker build uses candidate code.
+      - name: Fetch the immutable gateway source
+        shell: bash
+        run: |
+          set -euo pipefail
+          git fetch --no-tags origin "${SOURCE_SHA}"
+          test "$(git rev-parse FETCH_HEAD)" = "${SOURCE_SHA}"
+          mkdir -p "${RUNNER_TEMP}/push-source"
+          git -C "${GITHUB_WORKSPACE}" archive "${SOURCE_SHA}" cloud \
+            | tar -x -C "${RUNNER_TEMP}/push-source"
+
+      - uses: google-github-actions/auth@v2
+        with:
+          workload_identity_provider: ${{ vars.PRODUCTION_GCP_PUSH_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ vars.PRODUCTION_GCP_PUSH_DEPLOY_SERVICE_ACCOUNT }}
+
+      - uses: google-github-actions/setup-gcloud@v2
+
+      - uses: docker/setup-buildx-action@v3
+
+      - name: Configure Docker auth
+        run: gcloud auth configure-docker "${GCP_REGION}-docker.pkg.dev" --quiet
+
+      # Building an image does not need the deployment lease.
+      - name: Build and publish the immutable gateway image
+        shell: bash
+        run: |
+          set -euo pipefail
+          image_tag="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}:sha-${SOURCE_SHA}"
+          docker buildx build --push --platform linux/amd64 --provenance=false --metadata-file "${RUNNER_TEMP}/push-image.json" \
+            -f "${RUNNER_TEMP}/push-source/cloud/apps/push/Dockerfile" \
+            -t "${image_tag}" "${RUNNER_TEMP}/push-source/cloud"
+          digest="$(jq -er '."containerimage.digest"' "${RUNNER_TEMP}/push-image.json")"
+          [[ "${digest}" =~ ^sha256:[a-f0-9]{64}$ ]]
+          echo "IMAGE=${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}@${digest}" \
+            >> "${GITHUB_ENV}"
+          echo "IMAGE_DIGEST=${digest}" >> "${GITHUB_ENV}"
+
+      # Refuse older images before they ever boot against production.
+      - name: Require image support for inert validation
+        shell: bash
+        run: |
+          set -euo pipefail
+          docker run --rm --network none --entrypoint node "${IMAGE}" --input-type=module -e '
+            import { loadPushConfig } from "./apps/push/dist/config.js";
+            const env = { ORCA_PUSH_PUBLIC_URL: "https://push.onorca.dev", ORCA_PUSH_MODE: "validation" };
+            if (loadPushConfig(env).mode !== "validation") throw new Error("validation_mode_unsupported");
+            let rejected = false;
+            try { loadPushConfig({ ...env, ORCA_PUSH_MODE: "invalid" }); } catch { rejected = true; }
+            if (!rejected) throw new Error("validation_mode_not_fail_closed");
+          '
+
+      # Held across the deploy, not just a separate schema step: the gateway opens its pool and
+      # applies its schema while the new revision starts, so the revision is the schema step.
+      - uses: ./.github/actions/cloud-sql-rollout-lease
+        with:
+          bucket: onorca-cloud-terraform-state
+          object: terraform/state/push-rollout/production.lock
+
+      # Why: the candidate inherits the serving revision's scaling. A serving revision that has
+      # drifted below the floor would hand the candidate a cold start on every notification, and
+      # one that has drifted above the ceiling would hand it a larger Cloud SQL draw than the
+      # rollout lease was taken for. Refuse to inherit either rather than latch it.
+      - name: Record the serving revision and require its Terraform-owned scaling
+        shell: bash
+        run: |
+          set -euo pipefail
+          serving="$(gcloud run services describe "${SERVICE_NAME}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
+            | jq -r '[.status.traffic[] | select((.percent // 0) > 0)]
+                     | if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
+          test -n "${serving}"
+          revisions="$(gcloud run revisions list --service "${SERVICE_NAME}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format='value(metadata.name)')"
+          if test "${revisions}" != "${serving}"; then
+            echo 'Retire leftover revisions under the rollout lease before deploying; three pools are the limit.' >&2
+            exit 1
+          fi
+          floor="$(gcloud run revisions describe "${serving}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
+            --format="value(metadata.annotations['autoscaling.knative.dev/minScale'])")"
+          if [[ "${floor:-0}" -lt "${PUSH_MIN_INSTANCES}" ]]; then
+            echo "serving revision ${serving} holds ${floor:-0} minimum instances," \
+              "below ${PUSH_MIN_INSTANCES}; deploying would inherit and latch it." >&2
+            echo "Restore the floor first: gcloud run services update ${SERVICE_NAME}" \
+              "--region ${GCP_REGION} --min-instances=${PUSH_MIN_INSTANCES}" >&2
+            exit 1
+          fi
+          ceiling="$(gcloud run revisions describe "${serving}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
+            --format="value(metadata.annotations['autoscaling.knative.dev/maxScale'])")"
+          test "${ceiling}" = "${PUSH_MAX_INSTANCES}"
+          echo "serving revision ${serving} holds ${floor} minimum and ${ceiling} maximum instances"
+          echo "ROLLBACK_REVISION=${serving}" >> "${GITHUB_ENV}"
+          gcloud run revisions describe "${serving}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
+            > "${RUNNER_TEMP}/push-rollback-revision.json"
+          image="$(jq -er '.status.imageDigest' "${RUNNER_TEMP}/push-rollback-revision.json")"
+          [[ "${image}" =~ @sha256:[a-f0-9]{64}$ ]]
+          echo "ROLLBACK_IMAGE=${image}" >> "${GITHUB_ENV}"
+          jq -e 'all(.spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE" or .value == "active")' \
+            "${RUNNER_TEMP}/push-rollback-revision.json" > /dev/null
+
+      # Validation has no schema writes, HTTP mutations, worker, or pruners; tags alone do not
+      # isolate background consumers from production.
+      - name: Deploy the candidate revision with no traffic
+        shell: bash
+        run: |
+          set -euo pipefail
+          tag="c${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          {
+            echo "VALIDATION_DEPLOY_ATTEMPTED=true"
+            echo "VALIDATION_REVISION=${SERVICE_NAME}-${tag}"
+            echo "VALIDATION_TAG=${tag}"
+            echo "CANDIDATE_TAG=${tag}"
+            echo "CANDIDATE_REVISION=${SERVICE_NAME}-${tag}"
+          } >> "${GITHUB_ENV}"
+          gcloud run deploy "${SERVICE_NAME}" \
+            --project "${GCP_PROJECT_ID}" \
+            --region "${GCP_REGION}" \
+            --image "${IMAGE}" \
+            --tag "${tag}" \
+            --revision-suffix "${tag}" \
+            --no-traffic \
+            --update-env-vars ORCA_PUSH_MODE=validation \
+            --quiet
+          candidate="$(gcloud run services describe "${SERVICE_NAME}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
+            | jq -er --arg tag "${tag}" \
+              '[.status.traffic[] | select(.tag == $tag)]
+               | if length == 1 then .[0] else error("tagged candidate is not unique") end')"
+          test "$(jq -r '.revisionName' <<< "${candidate}")" = "${SERVICE_NAME}-${tag}"
+          echo "CANDIDATE_URL=$(jq -r '.url' <<< "${candidate}")" >> "${GITHUB_ENV}"
+
+      # A tagged revision is directly addressable and sits outside the service-wide cap, so the
+      # candidate and the serving revision each draw up to the ceiling during the probe window.
+      # Successor creation later requires three revision pools; assert the inherited ceiling.
+      - name: Require the candidate to serve the exact image and inherited scaling
+        shell: bash
+        run: |
+          set -euo pipefail
+          served="$(gcloud run revisions describe "${CANDIDATE_REVISION}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
+            --format='value(spec.containers[0].image)')"
+          test "${served}" = "${IMAGE}"
+          test "${CANDIDATE_REVISION}" != "${ROLLBACK_REVISION}"
+          candidate_ceiling="$(gcloud run revisions describe "${CANDIDATE_REVISION}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
+            --format="value(metadata.annotations['autoscaling.knative.dev/maxScale'])")"
+          test "${candidate_ceiling}" = "${PUSH_MAX_INSTANCES}"
+
+      - name: Probe the candidate readiness endpoint
+        shell: bash
+        run: |
+          set -euo pipefail
+          [[ "${CANDIDATE_URL}" =~ ^https://[^/]+$ ]]
+          for attempt in $(seq 1 30); do
+            code="$(curl -sS -o "${RUNNER_TEMP}/push-ready.json" -w '%{http_code}' \
+              --max-time 10 "${CANDIDATE_URL}/ready" || true)"
+            if test "${code}" = 200; then
+              jq -e . < "${RUNNER_TEMP}/push-ready.json" > /dev/null
+              curl --fail --silent --show-error --max-time 10 "${CANDIDATE_URL}/health" \
+                | jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "validation"' > /dev/null
+              echo "candidate ${CANDIDATE_REVISION} is ready after ${attempt} attempt(s)"
+              exit 0
+            fi
+            echo "attempt ${attempt}: /ready returned ${code}"
+            sleep 5
+          done
+          echo "candidate ${CANDIDATE_REVISION} never reported ready" >&2
+          exit 1
+
+      # Why: a gateway that boots and answers /ready can still be unable to send. This proves the
+      # runtime account's FCM grant end to end without delivering anything: validate_only stops
+      # Google before any push, and the deliberately invalid token means a healthy credential
+      # answers INVALID_ARGUMENT. PERMISSION_DENIED is the failure this step exists to catch.
+      #
+      # Only the four verdicts below are conclusive. A 429, a 5xx, or a transport failure says
+      # nothing about the credential, so it is retried rather than treated as either answer; a
+      # denied credential still fails on the first attempt, without burning the retries.
+      - name: Prove the runtime identity can reach FCM
+        shell: bash
+        run: |
+          set -euo pipefail
+          token="$(gcloud auth print-access-token \
+            --impersonate-service-account "${PUSH_RUNTIME_SERVICE_ACCOUNT}")"
+          test -n "${token}"
+          echo "::add-mask::${token}"
+          body='{"validate_only":true,"message":{"token":"orca-push-deploy-probe-invalid-token","notification":{"title":"Orca","body":"deploy probe"}}}'
+          for attempt in $(seq 1 5); do
+            code="$(curl -sS -o "${RUNNER_TEMP}/push-fcm.json" -w '%{http_code}' --max-time 20 \
+              -X POST "https://fcm.googleapis.com/v1/projects/${GCP_PROJECT_ID}/messages:send" \
+              -H "Authorization: Bearer ${token}" \
+              -H 'Content-Type: application/json' \
+              --data "${body}" || true)"
+            status="$(jq -r '.error.status // empty' < "${RUNNER_TEMP}/push-fcm.json" || true)"
+            echo "attempt ${attempt}: FCM validate-only send returned HTTP ${code} status ${status:-OK}"
+            if test "${status}" = PERMISSION_DENIED || test "${status}" = INVALID_ARGUMENT ||
+              test "${code}" = 401 || test "${code}" = 403; then
+              break
+            fi
+            sleep 5
+          done
+          if test "${status}" = PERMISSION_DENIED || test "${code}" = 401 || test "${code}" = 403; then
+            echo "the push runtime identity cannot send through FCM" >&2
+            exit 1
+          fi
+          test "${status}" = INVALID_ARGUMENT
+
+      # Cloud Run requires a successor before the latest revision can be deleted.
+      - name: Retire inert validation and activate the verified image
+        shell: bash
+        run: |
+          set -euo pipefail
+          tag="a${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          {
+            echo "CANDIDATE_TAG=${tag}"
+            echo "CANDIDATE_REVISION=${SERVICE_NAME}-${tag}"
+            echo "ACTIVATION_ATTEMPTED=true"
+          } >> "${GITHUB_ENV}"
+          # This is the production-effect boundary: schema, pruners and workers start here.
+          gcloud run deploy "${SERVICE_NAME}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
+            --image "${IMAGE}" --tag "${tag}" --revision-suffix "${tag}" \
+            --remove-env-vars ORCA_PUSH_MODE --no-traffic --quiet
+          gcloud run services update-traffic "${SERVICE_NAME}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
+            --remove-tags "${VALIDATION_TAG}" --quiet
+          gcloud run revisions delete "${VALIDATION_REVISION}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet
+          echo "VALIDATION_RETIRED=true" >> "${GITHUB_ENV}"
+          revision="$(gcloud run revisions describe "${SERVICE_NAME}-${tag}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
+          jq -e --arg image "${IMAGE}" --arg account "${PUSH_RUNTIME_SERVICE_ACCOUNT}" \
+            --arg ceiling "${PUSH_MAX_INSTANCES}" --arg floor "${PUSH_MIN_INSTANCES}" \
+            --slurpfile prior "${RUNNER_TEMP}/push-rollback-revision.json" '
+             def shape: del(.containers[0].image) |
+               .containers[0].env = ((.containers[0].env // []) |
+                 map(select(.name != "ORCA_PUSH_MODE")) | sort_by(.name));
+             .spec.containers[0].image == $image and .spec.serviceAccountName == $account and
+             all(.spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE") and
+             (.spec | shape) == ($prior[0].spec | shape) and
+             .metadata.annotations["autoscaling.knative.dev/maxScale"] == $ceiling and
+             (.metadata.annotations["autoscaling.knative.dev/minScale"] | tonumber) >= ($floor | tonumber)' \
+            <<< "${revision}" > /dev/null
+          candidate="$(gcloud run services describe "${SERVICE_NAME}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
+            | jq -er --arg tag "${tag}" '[.status.traffic[] | select(.tag == $tag)]
+              | if length == 1 then .[0] else error("active candidate is not unique") end')"
+          test "$(jq -r '.revisionName' <<< "${candidate}")" = "${SERVICE_NAME}-${tag}"
+          url="$(jq -er '.url' <<< "${candidate}")"
+          [[ "${url}" =~ ^https://[^/]+$ ]]
+          curl --fail --silent --show-error --max-time 10 "${url}/ready" | jq -e '.ok == true'
+          curl --fail --silent --show-error --max-time 10 "${url}/health" \
+            | jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"'
+
+      - name: Shift all traffic to the verified candidate
+        shell: bash
+        run: |
+          set -euo pipefail
+          echo "TRAFFIC_SHIFT_ATTEMPTED=true" >> "${GITHUB_ENV}"
+          gcloud run services update-traffic "${SERVICE_NAME}" \
+            --project "${GCP_PROJECT_ID}" \
+            --region "${GCP_REGION}" \
+            --to-revisions "${CANDIDATE_REVISION}=100" \
+            --quiet
+          serving="$(gcloud run services describe "${SERVICE_NAME}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
+            | jq -r '[.status.traffic[] | select((.percent // 0) > 0)]
+                     | if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
+          test "${serving}" = "${CANDIDATE_REVISION}"
+          echo "TRAFFIC_SHIFTED=true" >> "${GITHUB_ENV}"
+
+      # Why: the summary is written before the origin check, not after it. Once traffic has
+      # moved, the rollback target is the single thing an operator needs, and a summary that only
+      # appeared on success would be missing in exactly the run that needs it.
+      - name: Publish the rollout summary
+        if: ${{ always() && env.CANDIDATE_REVISION != '' && env.ROLLBACK_REVISION != '' }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          {
+            echo '### Push gateway rollout'
+            echo
+            echo "Source: ${SOURCE_SHA}"
+            echo
+            echo "Revision: \`${CANDIDATE_REVISION}\`"
+            echo
+            echo "Image: \`${IMAGE_DIGEST}\`"
+            echo "Known-good image: \`${ROLLBACK_IMAGE}\`"
+            echo
+            echo "Recovery: deploy \`${ROLLBACK_IMAGE}\` as a new revision with" \
+              "\`--remove-env-vars ORCA_PUSH_MODE --no-traffic --tag <unique-recovery-tag> --revision-suffix <unique-recovery-suffix>\`."
+            echo 'Verify its exact digest, configuration, readiness and active mode, then promote and check the public origin.'
+            echo 'Only then remove obsolete tags and delete rejected/previous revisions; never delete the latest revision.'
+            echo 'The previous revision is retired after public checks; retain this immutable image for recovery under the rollout lease.'
+            echo "Activation attempted: ${ACTIVATION_ATTEMPTED:-false}; traffic rollback cannot undo schema or deliveries."
+          } >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Verify the public origin after the shift
+        shell: bash
+        run: |
+          set -euo pipefail
+          for attempt in $(seq 1 30); do
+            code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 \
+              "${PUSH_ORIGIN}/ready" || true)"
+            if test "${code}" = 200; then
+              curl --fail --silent --show-error --max-time 10 "${PUSH_ORIGIN}/health" \
+                | jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"' > /dev/null
+              echo "ROLLOUT_VERIFIED=true" >> "${GITHUB_ENV}"
+              echo "${PUSH_ORIGIN} is ready after ${attempt} attempt(s)"
+              exit 0
+            fi
+            echo "attempt ${attempt}: ${PUSH_ORIGIN}/ready returned ${code}"
+            sleep 5
+          done
+          echo "${PUSH_ORIGIN} never reported ready after the shift" >&2
+          exit 1
+
+      # Why: everything after the shift runs with production on the candidate. A failure there
+      # is not a failure to deploy, it is a live gateway that has to go back, so the traffic move
+      # is undone here rather than left to whoever reads the run.
+      - name: Roll traffic back to the previous revision
+        if: ${{ (failure() || cancelled()) && env.TRAFFIC_SHIFT_ATTEMPTED == 'true' && env.ROLLOUT_VERIFIED != 'true' }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          test -n "${ROLLBACK_REVISION:-}"
+          gcloud run services update-traffic "${SERVICE_NAME}" \
+            --project "${GCP_PROJECT_ID}" \
+            --region "${GCP_REGION}" \
+            --to-revisions "${ROLLBACK_REVISION}=100" \
+            --quiet
+          serving="$(gcloud run services describe "${SERVICE_NAME}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
+            | jq -r '[.status.traffic[] | select((.percent // 0) > 0)]
+                     | if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
+          test "${serving}" = "${ROLLBACK_REVISION}"
+          echo "TRAFFIC_ROLLED_BACK=true" >> "${GITHUB_ENV}"
+          {
+            echo
+            echo '### Push gateway rolled back'
+            echo
+            echo "Traffic returned to \`${ROLLBACK_REVISION}\`; the candidate" \
+              "\`${CANDIDATE_REVISION}\` no longer serves HTTP; deletion below must stop its workers."
+          } >> "${GITHUB_STEP_SUMMARY}"
+
+      # Deleting a revision does not restore the service template that Terraform reconciles.
+      - name: Restore the known-good service template
+        if: ${{ (failure() || cancelled()) && env.VALIDATION_DEPLOY_ATTEMPTED == 'true' && env.ROLLOUT_VERIFIED != 'true' }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          test "${TRAFFIC_SHIFT_ATTEMPTED:-false}" != true || test "${TRAFFIC_ROLLED_BACK:-false}" = true
+          test -n "${ROLLBACK_IMAGE}"
+          # A partial activation may leave validation plus active; free one slot before recovery.
+          latest="$(gcloud run services describe "${SERVICE_NAME}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
+            --format='value(status.latestCreatedRevisionName)')"
+          test -n "${latest}"
+          if test "${VALIDATION_RETIRED:-false}" != true && test "${latest}" != "${VALIDATION_REVISION}"; then
+            existing="$(gcloud run revisions list --service "${SERVICE_NAME}" \
+              --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
+              --filter "metadata.name=${VALIDATION_REVISION}" --format='value(metadata.name)')"
+            if test -n "${existing}"; then
+              test "${existing}" = "${VALIDATION_REVISION}"
+              gcloud run services update-traffic "${SERVICE_NAME}" \
+                --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
+                --remove-tags "${VALIDATION_TAG}" --quiet
+              gcloud run revisions delete "${VALIDATION_REVISION}" \
+                --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet
+            fi
+          fi
+          tag="r${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          echo "RECOVERY_TAG=${tag}" >> "${GITHUB_ENV}"
+          echo "TEMPLATE_RECOVERY_REVISION=${SERVICE_NAME}-${tag}" >> "${GITHUB_ENV}"
+          echo "Template recovery attempted: ${SERVICE_NAME}-${tag}, image ${ROLLBACK_IMAGE}." \
+            >> "${GITHUB_STEP_SUMMARY}"
+          # Known-good schema/workers can run here even though HTTP stays on the old revision.
+          gcloud run deploy "${SERVICE_NAME}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
+            --image "${ROLLBACK_IMAGE}" --revision-suffix "${tag}" --tag "${tag}" \
+            --remove-env-vars ORCA_PUSH_MODE --no-traffic --quiet
+          gcloud run services describe "${SERVICE_NAME}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
+            > "${RUNNER_TEMP}/push-recovered-service.json"
+          gcloud run revisions describe "${SERVICE_NAME}-${tag}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
+            > "${RUNNER_TEMP}/push-recovery-revision.json"
+          jq -e --arg image "${ROLLBACK_IMAGE}" --arg serving "${ROLLBACK_REVISION}" \
+            --arg floor "${PUSH_MIN_INSTANCES}" --arg ceiling "${PUSH_MAX_INSTANCES}" \
+            --slurpfile prior "${RUNNER_TEMP}/push-rollback-revision.json" \
+            --slurpfile recovered "${RUNNER_TEMP}/push-recovery-revision.json" '
+              def shape: del(.containers[0].image) |
+                .containers[0].env = ((.containers[0].env // []) |
+                  map(select(.name != "ORCA_PUSH_MODE")) | sort_by(.name));
+              .spec.template.spec.containers[0].image == $image and
+              all(.spec.template.spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE") and
+              $recovered[0].spec.containers[0].image == $image and
+              all($recovered[0].spec.containers[0].env[]?; .name != "ORCA_PUSH_MODE") and
+              ($recovered[0].spec | shape) == ($prior[0].spec | shape) and
+              .spec.template.metadata.annotations["autoscaling.knative.dev/maxScale"] == $ceiling and
+              (.spec.template.metadata.annotations["autoscaling.knative.dev/minScale"] | tonumber) >= ($floor | tonumber) and
+              ([.status.traffic[] | select((.percent // 0) > 0)] |
+                length == 1 and .[0].revisionName == $serving and .[0].percent == 100)
+            ' "${RUNNER_TEMP}/push-recovered-service.json" > /dev/null
+          echo "TEMPLATE_RESTORED=true" >> "${GITHUB_ENV}"
+          echo 'Known-good image and normal mode restored; previous revision still serves HTTP.' \
+            >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Promote and verify the known-good recovery revision
+        if: ${{ always() && env.TEMPLATE_RESTORED == 'true' }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          url="$(jq -er --arg tag "${RECOVERY_TAG}" \
+            '.status.traffic[] | select(.tag == $tag) | .url' "${RUNNER_TEMP}/push-recovered-service.json")"
+          [[ "${url}" =~ ^https://[^/]+$ ]]
+          curl --fail --silent --show-error --max-time 10 "${url}/ready" | jq -e '.ok == true'
+          curl --fail --silent --show-error --max-time 10 "${url}/health" \
+            | jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"'
+          gcloud run services update-traffic "${SERVICE_NAME}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
+            --to-revisions "${TEMPLATE_RECOVERY_REVISION}=100" --quiet
+          serving="$(gcloud run services describe "${SERVICE_NAME}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
+            | jq -er '[.status.traffic[] | select((.percent // 0) > 0)] |
+              if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
+          test "${serving}" = "${TEMPLATE_RECOVERY_REVISION}"
+          curl --fail --silent --show-error --max-time 10 "${PUSH_ORIGIN}/ready" | jq -e '.ok == true'
+          curl --fail --silent --show-error --max-time 10 "${PUSH_ORIGIN}/health" \
+            | jq -e '.ok == true and .deliveryProtocol == 2 and .mode == "active"'
+          echo "RECOVERY_VERIFIED=true" >> "${GITHUB_ENV}"
+          echo "Recovery revision ${TEMPLATE_RECOVERY_REVISION} now serves the known-good image." \
+            >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Delete the rejected candidate revision
+        if: ${{ always() && env.RECOVERY_VERIFIED == 'true' }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          if test -z "${CANDIDATE_REVISION:-}"; then
+            echo "CANDIDATE_DELETED=true" >> "${GITHUB_ENV}"
+            exit 0
+          fi
+          if test -n "${CANDIDATE_TAG:-}"; then
+            gcloud run services update-traffic "${SERVICE_NAME}" \
+              --project "${GCP_PROJECT_ID}" \
+              --region "${GCP_REGION}" \
+              --remove-tags "${CANDIDATE_TAG}" \
+              --quiet
+            echo "CANDIDATE_TAG=" >> "${GITHUB_ENV}"
+          fi
+          existing="$(gcloud run revisions list --service "${SERVICE_NAME}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
+            --filter "metadata.name=${CANDIDATE_REVISION}" --format='value(metadata.name)')"
+          if test -n "${existing}"; then
+            test "${existing}" = "${CANDIDATE_REVISION}"
+            gcloud run revisions delete "${CANDIDATE_REVISION}" \
+              --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet
+          fi
+          echo "CANDIDATE_DELETED=true" >> "${GITHUB_ENV}"
+          echo "candidate revision ${CANDIDATE_REVISION} is absent"
+
+      # Public checks commit the serving revision; cleanup failures must not roll it back.
+      - name: Retire previous consumers after public checks
+        if: ${{ always() && (env.ROLLOUT_VERIFIED == 'true' || (env.RECOVERY_VERIFIED == 'true' && env.CANDIDATE_DELETED == 'true')) }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          serving="${CANDIDATE_REVISION}"
+          if test "${RECOVERY_VERIFIED:-false}" = true; then
+            serving="${TEMPLATE_RECOVERY_REVISION}"
+          fi
+          gcloud run services update-traffic "${SERVICE_NAME}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --clear-tags --quiet
+          revisions="$(gcloud run revisions list --service "${SERVICE_NAME}" \
+            --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format='value(metadata.name)')"
+          while IFS= read -r revision; do
+            test -n "${revision}" || continue
+            test "${revision}" != "${serving}" || continue
+            case "${revision}" in
+              "${ROLLBACK_REVISION}"|"${VALIDATION_REVISION}"|"${CANDIDATE_REVISION}") ;;
+              *) echo "Unexpected revision ${revision}; manual retirement required." >&2; exit 1 ;;
+            esac
+            gcloud run revisions delete "${revision}" \
+              --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --quiet
+          done <<< "${revisions}"
+          echo "CANDIDATE_TAG=" >> "${GITHUB_ENV}"
+          echo 'Obsolete revision resources retired; verify actual SQL session drain operationally.' \
+            >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Drop the candidate traffic tag
+        if: always()
+        shell: bash
+        run: |
+          set -euo pipefail
+          test -n "${CANDIDATE_TAG:-}" || exit 0
+          gcloud run services update-traffic "${SERVICE_NAME}" \
+            --project "${GCP_PROJECT_ID}" \
+            --region "${GCP_REGION}" \
+            --remove-tags "${CANDIDATE_TAG}" \
+            --quiet

.github/workflows/cloud-verify.yml

@@ -25,9 +25,10 @@ defaults:
     working-directory: cloud
 
 jobs:
+  # Public-repository hosted runners preserve Blacksmith allowance for macOS.
   security:
     name: Secret scan
-    runs-on: blacksmith-2vcpu-ubuntu-2204
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
         with:
@@ -53,7 +54,7 @@ jobs:
   # Compiles the workspace. No Postgres service: nothing here reaches a
   # database, and the service container costs ~13s of startup.
   build:
-    runs-on: blacksmith-4vcpu-ubuntu-2204
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
 
@@ -73,7 +74,7 @@ jobs:
   # package it needs through the relay pretest hook, so it does not depend on
   # `pnpm build` having run.
   test:
-    runs-on: blacksmith-4vcpu-ubuntu-2204
+    runs-on: ubuntu-22.04
     services:
       postgres:
         image: postgres:16-alpine
@@ -107,7 +108,7 @@ jobs:
   # Fork pull requests reach this job, so it never configures a backend, never plans, and never
   # holds a credential. Only the relay root ships here; foundation and apps stay private.
   terraform:
-    runs-on: blacksmith-2vcpu-ubuntu-2204
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
 

.github/workflows/dev-channel-win-build.yml

@@ -149,9 +149,12 @@ jobs:
           fi
           # Reachability is the trust test: GitHub serves PR-only commits by SHA,
           # so resolving the object is not proof a branch or tag of this repo
-          # reaches it. Bare + tree:0 keeps this to the commit graph.
+          # reaches it. Bare + tree:0 keeps this to the commit graph; reftable
+          # because branches that differ only in casing cannot both be stored by
+          # the files backend on a case-insensitive runner disk, which fails the
+          # entire fetch rather than the one ref.
           scratch="$RUNNER_TEMP/vet-requested-ref"
-          git init -q --bare "$scratch"
+          git init -q --bare --ref-format=reftable "$scratch"
           git -C "$scratch" fetch -q --filter=tree:0 "$REPO_URL" '+refs/heads/*:refs/heads/*' '+refs/tags/*:refs/tags/*'
           if ! git -C "$scratch" rev-parse --verify --quiet "$REQUESTED_SHA^{commit}" >/dev/null; then
             echo "::error::Commit $REQUESTED_SHA is not in stablyai/orca."

.github/workflows/e2e.yml

@@ -27,6 +27,10 @@ on:
         description: Ref to check out (defaults to the workflow ref)
         required: false
         type: string
+      test_files:
+        description: JSON array of specs to run; empty runs the full suite
+        required: false
+        type: string
   schedule:
     # Why: GitHub cron uses UTC; these slots map to 10am and 3pm
     # America/Phoenix for the default-branch E2E run.
@@ -146,7 +150,7 @@ jobs:
       # Native cache misses need the compiler, Electron needs Xvfb, and paired
       # Quick Open needs ripgrep. Install them in one apt transaction per shard.
       - name: Install native build and headless UI tools
-        run: sudo apt-get update && sudo apt-get install -y build-essential fonts-noto-cjk python3 ripgrep xvfb zsh
+        run: sudo apt-get update && sudo apt-get install -y build-essential fonts-noto-cjk python3 ripgrep xvfb zsh openbox x11-utils
 
       - uses: ./.github/actions/install-node-dependencies
         with:
@@ -167,7 +171,7 @@ jobs:
       # ORCA_E2E_FORWARD_APP_LOGS keeps startup failures visible when Electron
       # launches but never creates a BrowserWindow.
       - name: Run E2E tests (${{ matrix.shard_name }})
-        run: xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 ORCA_E2E_WEB_CLIENT=1 ORCA_RELAY_PATH="$GITHUB_WORKSPACE/out/relay" pnpm run test:e2e --shard=${{ matrix.shard }}
+        run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 ORCA_E2E_WEB_CLIENT=1 ORCA_RELAY_PATH="$GITHUB_WORKSPACE/out/relay" pnpm run test:e2e --shard=${{ matrix.shard }}
 
       # Why: Playwright retains traces/screenshots only on failure. Uploading
       # them as an artifact makes post-mortem debugging on CI possible without
@@ -201,7 +205,7 @@ jobs:
         # unbounded inventory fallback; the paired fixture exercises that real boundary.
         # Why openssh-client: the Docker-SSH fixture shells out to ssh/ssh-keygen, and this
         # lane now receives those specs from pr.yml's SSH source mapping.
-        run: sudo apt-get update && sudo apt-get install -y build-essential fonts-noto-cjk openssh-client python3 ripgrep xvfb zsh
+        run: sudo apt-get update && sudo apt-get install -y build-essential fonts-noto-cjk openssh-client python3 ripgrep xvfb zsh openbox x11-utils
 
       - uses: ./.github/actions/install-node-dependencies
         with:
@@ -223,6 +227,12 @@ jobs:
           mapfile -t TEST_FILES < <(jq -r '.[] | select(
             . != "tests/e2e/ssh-startup-exec-readiness.spec.ts" and
             . != "tests/e2e/paired-startup-exec-readiness.spec.ts" and
+            . != "tests/e2e/ssh-docker-five-pane-input-under-flood.spec.ts" and
+            . != "tests/e2e/local-ssh-browser-routing.spec.ts" and
+            . != "tests/e2e/ssh-browser-network-execution-route.docker.unit.test.ts" and
+            . != "tests/e2e/ssh-localhost.spec.ts" and
+            . != "tests/e2e/ssh-client-hosted-browser-drop-reconnect.spec.ts" and
+            . != "tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts" and
             . != "tests/e2e/terminal-ibus-hangul-native.spec.ts"
           )' <<<"$TEST_FILES_JSON")
           if [ "${#TEST_FILES[@]}" -eq 0 ]; then
@@ -241,7 +251,7 @@ jobs:
           if grep -l '@headful' "${TEST_FILES[@]}" >/dev/null; then
             E2E_PROJECT_ARGS+=(--project=electron-headful)
           fi
-          xvfb-run --auto-servernum env "${E2E_ENV[@]}" \
+          xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env "${E2E_ENV[@]}" \
             pnpm run test:e2e "${TEST_FILES[@]}" --workers=1 "${E2E_PROJECT_ARGS[@]}"
 
       - name: Upload Playwright traces
@@ -258,12 +268,16 @@ jobs:
     needs: [build, prepare-native-cache]
     # effect of one route listing a startup-readiness spec — pruning that spec would have
     # silently retired the whole lane. The signal is now derived from the SSH routes directly.
-    # The two spec clauses stay for their honest purpose: changed-e2e hands these specs to this
+    # The explicit spec clauses stay for their honest purpose: changed-e2e hands these specs to this
     # lane, so editing one must still run it here.
     if: >-
       inputs.test_files == '' ||
       inputs.ssh_source_changed == 'true' ||
+      contains(inputs.test_files, 'tests/e2e/ssh-docker-five-pane-input-under-flood.spec.ts') ||
+      contains(inputs.test_files, 'tests/e2e/local-ssh-browser-routing.spec.ts') ||
+      contains(inputs.test_files, 'tests/e2e/ssh-client-hosted-browser-drop-reconnect.spec.ts') ||
       contains(inputs.test_files, 'tests/e2e/ssh-startup-exec-readiness.spec.ts') ||
+      contains(inputs.test_files, 'tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts') ||
       contains(inputs.test_files, 'tests/e2e/paired-startup-exec-readiness.spec.ts')
     runs-on: ubuntu-latest
     # Why 60: this lane now also runs the remaining Docker-SSH specs serially. They average
@@ -278,7 +292,7 @@ jobs:
           ref: ${{ inputs.ref || github.ref }}
 
       - name: Install native build and headless UI tools
-        run: sudo apt-get update && sudo apt-get install -y build-essential fonts-noto-cjk openssh-client python3 xvfb zsh
+        run: sudo apt-get update && sudo apt-get install -y build-essential fonts-noto-cjk openssh-client python3 ripgrep xvfb zsh openbox x11-utils
 
       - uses: ./.github/actions/install-node-dependencies
         with:
@@ -293,7 +307,7 @@ jobs:
       # Why: this is the release-path proof that the deployed Linux relay keeps
       # its PTY and explorer live across a real watcher SIGSEGV.
       - name: Run Docker SSH watcher isolation E2E
-        run: xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:ssh-docker-watcher-isolation
+        run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:ssh-docker-watcher-isolation
 
       # Why: Playwright empties test-results/ when it starts, so each step here used to
       # destroy the previous step's traces. Only the last lane's failure was ever
@@ -310,7 +324,7 @@ jobs:
       # readiness across live SSH, headed paired, and headless serve topologies.
       - name: Run Docker SSH terminal parking + startup readiness E2E
         if: always()
-        run: xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:ssh-docker-terminal-parking
+        run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:ssh-docker-terminal-parking
 
       - name: Keep terminal-parking traces
         if: always()
@@ -326,7 +340,7 @@ jobs:
       # legible as an SSH-named failure.
       - name: Run remaining Docker SSH E2E
         if: always()
-        run: xvfb-run --auto-servernum env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:ssh-docker
+        run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 pnpm run test:e2e:ssh-docker
 
       - name: Keep remaining-ssh-docker traces
         if: always()
@@ -344,3 +358,87 @@ jobs:
           path: e2e-traces/
           retention-days: 7
           if-no-files-found: ignore
+
+  ssh-browser-network-route:
+    name: ssh browser network route
+    if: inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/ssh-browser-network-execution-route.docker.unit.test.ts')
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+      - uses: ./.github/actions/install-node-dependencies
+        with:
+          native-runtime: node
+      - name: Install SSH client
+        run: sudo apt-get update && sudo apt-get install -y openssh-client
+      - name: Run Docker SSH browser network route journeys
+        env:
+          ORCA_BACKGROUND_LAUNCH: '1'
+          ORCA_RUN_DOCKER_SSH_BROWSER_E2E: '1'
+        run: node_modules/.bin/vitest run --config config/vitest.config.ts tests/e2e/ssh-browser-network-execution-route.docker.unit.test.ts
+
+  ssh-localhost:
+    name: localhost SSH terminal and hooks
+    needs: [build, prepare-native-cache]
+    if: inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/ssh-localhost.spec.ts')
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+      - name: Install SSH server and headless tools
+        run: sudo apt-get update && sudo apt-get install -y build-essential openssh-client openssh-server python3 ripgrep xvfb zsh openbox x11-utils
+      - uses: ./.github/actions/install-node-dependencies
+        with:
+          native-runtime: electron
+      - uses: actions/download-artifact@v8
+        with:
+          name: e2e-build-out
+          path: out/
+      - name: Start isolated localhost SSH server
+        shell: bash
+        run: |
+          # Bare shells install Pi extensions only for an existing agent home.
+          mkdir -p "$HOME/.pi/agent"
+          fixture="$RUNNER_TEMP/orca-localhost-sshd"
+          mkdir -p "$fixture"
+          ssh-keygen -q -t ed25519 -N '' -f "$fixture/host_key"
+          ssh-keygen -q -t ed25519 -N '' -f "$fixture/client_key"
+          cat > "$fixture/sshd_config" <<EOF
+          Port 22222
+          ListenAddress 127.0.0.1
+          HostKey $fixture/host_key
+          PidFile $fixture/sshd.pid
+          AuthorizedKeysFile $fixture/client_key.pub
+          StrictModes no
+          PasswordAuthentication no
+          KbdInteractiveAuthentication no
+          UsePAM yes
+          AllowUsers $(id -un)
+          Subsystem sftp internal-sftp
+          EOF
+          sudo mkdir -p /run/sshd
+          sudo /usr/sbin/sshd -f "$fixture/sshd_config" -E "$fixture/sshd.log"
+          ssh -i "$fixture/client_key" -p 22222 -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null 127.0.0.1 true || { sudo cat "$fixture/sshd.log"; exit 1; }
+          {
+            echo "ORCA_E2E_SSH_PORT=22222"
+            echo "ORCA_E2E_SSH_USER=$(id -un)"
+            echo "ORCA_E2E_SSH_IDENTITY_FILE=$fixture/client_key"
+          } >> "$GITHUB_ENV"
+      - name: Run localhost SSH terminal and hook journey
+        env:
+          SKIP_BUILD: '1'
+          ORCA_E2E_SSH_LOCALHOST: '1'
+          ORCA_FEATURE_REMOTE_AGENT_HOOKS: '1'
+          ORCA_E2E_FORWARD_APP_LOGS: '1'
+        run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/ssh-localhost.spec.ts --project=electron-headless --workers=1
+      - uses: actions/upload-artifact@v7
+        if: failure()
+        with:
+          name: localhost-ssh-traces
+          path: test-results/
+          retention-days: 7
+          if-no-files-found: ignore

.github/workflows/golden-e2e-experiment.yml

@@ -98,12 +98,17 @@ jobs:
           $env:SKIP_BUILD = '1'
           $env:ORCA_E2E_FORWARD_APP_LOGS = '1'
           pnpm run --if-present test:e2e:workspace-session-golden
+          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
           pnpm run --if-present test:e2e:windows-fresh-startup-golden
+          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
           pnpm run --if-present test:e2e:tab-bar-agent-launch-golden
+          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
           if (Test-Path tests/e2e/golden-fresh-profile-terminal.spec.ts) {
             pnpm run test:e2e -- tests/e2e/golden-fresh-profile-terminal.spec.ts tests/e2e/golden-shell-command.spec.ts
+            if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
           }
           pnpm run --if-present test:e2e:source-control-golden
+          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
 
       - name: Upload Playwright traces
         if: failure()

.github/workflows/hourly-mac-build.yml

@@ -26,7 +26,7 @@ name: Hourly macOS Dev Build
 #   HOURLY_RELEASE_APP_ID           the App's numeric id
 #   HOURLY_RELEASE_APP_PRIVATE_KEY  the App's .pem private key
 #
-# Installation tokens live one hour, which is why this mints twice. Install and
+# Installation tokens live one hour, so the build job mints twice. Install and
 # build need no token at all, and notarization can hold the publish step for tens
 # of minutes; minting again once the build is done starts the clock at the first
 # call that actually uses it rather than burning a third of it on `pnpm install`.
@@ -60,33 +60,15 @@ env:
   HOURLY_RETAIN_COUNT: 72
 
 jobs:
-  build-hourly-mac:
+  # Avoid occupying the limited Mac pool when main has not moved.
+  preflight:
     if: github.repository == 'stablyai/orca'
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
     outputs:
-      tag: ${{ steps.release.outputs.tag }}
-      version: ${{ steps.hourly.outputs.version }}
+      should_build: ${{ steps.freshness.outputs.should_build }}
       head_sha: ${{ steps.freshness.outputs.head_sha }}
-      published: ${{ steps.publish_live.outcome == 'success' && 'true' || 'false' }}
-    runs-on: blacksmith-6vcpu-macos-15
-    # Why 150: it must exceed the worst case the retry budgets below can produce
-    # (install 3x10 + publish 2x45 = 120, plus ~25 for checkout/build/verify), or
-    # the job is killed mid-retry and no cleanup step runs at all. A typical run
-    # is far shorter — this is the notary queue's tail, not its median.
-    timeout-minutes: 150
-    env:
-      NODE_OPTIONS: --max-old-space-size=4096
     steps:
-      - name: Checkout
-        uses: actions/checkout@v6
-        with:
-          ref: main
-          fetch-depth: 0
-          # Why: this job only reads stablyai/orca and never pushes; every write
-          # goes to the hourly repo through a minted App token passed by env.
-          # Not persisting the checkout credential shrinks the blast radius if a
-          # build step is compromised (zizmor: artipacked).
-          persist-credentials: false
-
       - name: Mint hourly repo token
         id: app_token
         uses: actions/create-github-app-token@v2
@@ -95,18 +77,19 @@ jobs:
           private-key: ${{ secrets.HOURLY_RELEASE_APP_PRIVATE_KEY }}
           owner: stablyai
           repositories: orca-hourly
+          permission-contents: read
 
-      # Why: main is often idle overnight. Rebuilding an unchanged commit burns a
-      # runner hour and adds a redundant tag to the retention window.
       - name: Check whether main moved since the last hourly
         id: freshness
         shell: bash
         env:
           GH_TOKEN: ${{ steps.app_token.outputs.token }}
+          MAIN_REPO_TOKEN: ${{ github.token }}
           FORCED: ${{ github.event_name == 'workflow_dispatch' && inputs.force }}
         run: |
           set -euo pipefail
-          head_sha="$(git rev-parse HEAD)"
+          head_sha="$(GH_TOKEN="$MAIN_REPO_TOKEN" gh api "repos/$GITHUB_REPOSITORY/commits/main" --jq .sha)"
+          [[ "$head_sha" =~ ^[0-9a-f]{40}$ ]] || { echo "::error::Could not resolve main"; exit 1; }
           echo "head_sha=$head_sha" >>"$GITHUB_OUTPUT"
           if [[ "$FORCED" == "true" ]]; then
             echo "should_build=true" >>"$GITHUB_OUTPUT"
@@ -133,21 +116,55 @@ jobs:
             echo "main moved to $head_sha (last hourly built $last_sha); building."
           fi
 
+  build-hourly-mac:
+    needs: preflight
+    if: needs.preflight.outputs.should_build == 'true'
+    outputs:
+      tag: ${{ steps.release.outputs.tag }}
+      version: ${{ steps.hourly.outputs.version }}
+      head_sha: ${{ needs.preflight.outputs.head_sha }}
+      published: ${{ steps.publish_live.outcome == 'success' && 'true' || 'false' }}
+    runs-on: blacksmith-6vcpu-macos-15
+    # Why 150: it must exceed the worst case the retry budgets below can produce
+    # (install 3x10 + publish 2x45 = 120, plus ~25 for checkout/build/verify), or
+    # the job is killed mid-retry and no cleanup step runs at all. A typical run
+    # is far shorter — this is the notary queue's tail, not its median.
+    timeout-minutes: 150
+    env:
+      NODE_OPTIONS: --max-old-space-size=4096
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ needs.preflight.outputs.head_sha }}
+          fetch-depth: 0
+          # Why: this job only reads stablyai/orca and never pushes; every write
+          # goes to the hourly repo through a minted App token passed by env.
+          # Not persisting the checkout credential shrinks the blast radius if a
+          # build step is compromised (zizmor: artipacked).
+          persist-credentials: false
+
+      - name: Mint hourly repo token
+        id: app_token
+        uses: actions/create-github-app-token@v2
+        with:
+          app-id: ${{ secrets.HOURLY_RELEASE_APP_ID }}
+          private-key: ${{ secrets.HOURLY_RELEASE_APP_PRIVATE_KEY }}
+          owner: stablyai
+          repositories: orca-hourly
+
       - name: Setup pnpm
-        if: steps.freshness.outputs.should_build == 'true'
         uses: pnpm/setup@v2
         with:
           install: false
 
       - name: Setup Node.js
-        if: steps.freshness.outputs.should_build == 'true'
         uses: actions/setup-node@v6
         with:
           node-version-file: package.json
           cache: pnpm
 
       - name: Cache electron-builder downloads
-        if: steps.freshness.outputs.should_build == 'true'
         uses: actions/cache@v5
         with:
           path: |
@@ -158,7 +175,6 @@ jobs:
             electron-builder-mac-
 
       - name: Install dependencies
-        if: steps.freshness.outputs.should_build == 'true'
         uses: nick-fields/retry@v4
         with:
           timeout_minutes: 10
@@ -169,7 +185,6 @@ jobs:
       # Why: signing is what makes an hourly installable over an existing Orca, so
       # a missing cert must fail here rather than after a 20-minute build.
       - name: Verify macOS signing environment
-        if: steps.freshness.outputs.should_build == 'true'
         run: node config/scripts/verify-macos-release-env.mjs
         env:
           CSC_LINK: ${{ secrets.MAC_CERTS }}
@@ -180,7 +195,6 @@ jobs:
 
       - name: Compute hourly version
         id: hourly
-        if: steps.freshness.outputs.should_build == 'true'
         shell: bash
         env:
           GH_TOKEN: ${{ steps.app_token.outputs.token }}
@@ -211,7 +225,7 @@ jobs:
             node config/scripts/hourly-build-version.mjs \
             >"$RUNNER_TEMP/hourly-identity.txt"
           grep -E '^(version|build_number)=' "$RUNNER_TEMP/hourly-identity.txt"
-          # Why check rather than trust: the checkout above pins `ref: main`, but a
+          # Why check rather than trust: the checkout above pins the resolved main commit, but a
           # workflow_dispatch runs this file from whatever branch was dispatched. A
           # branch that edits this step while main still has the old script yields
           # an empty name and an untitled release — silent, and only visible once
@@ -223,7 +237,6 @@ jobs:
           cat "$RUNNER_TEMP/hourly-identity.txt" >>"$GITHUB_OUTPUT"
 
       - name: Build app
-        if: steps.freshness.outputs.should_build == 'true'
         run: pnpm build:release
         env:
           NODE_OPTIONS: --max-old-space-size=4096
@@ -239,7 +252,6 @@ jobs:
       # part the full budget.
       - name: Re-mint hourly repo token for publish
         id: app_token_publish
-        if: steps.freshness.outputs.should_build == 'true'
         uses: actions/create-github-app-token@v2
         with:
           app-id: ${{ secrets.HOURLY_RELEASE_APP_ID }}
@@ -249,13 +261,12 @@ jobs:
 
       - name: Create hourly release
         id: release
-        if: steps.freshness.outputs.should_build == 'true'
         shell: bash
         env:
           GH_TOKEN: ${{ steps.app_token_publish.outputs.token }}
           TAG: v${{ steps.hourly.outputs.version }}
           NAME: ${{ steps.hourly.outputs.name }}
-          SHA: ${{ steps.freshness.outputs.head_sha }}
+          SHA: ${{ needs.preflight.outputs.head_sha }}
         run: |
           set -euo pipefail
           # Kept at 12 even though the title shows 7: the freshness check above
@@ -291,7 +302,6 @@ jobs:
           echo "tag=$TAG" >>"$GITHUB_OUTPUT"
 
       - name: Publish hourly macOS artifacts
-        if: steps.freshness.outputs.should_build == 'true'
         uses: nick-fields/retry@v4
         with:
           # Why 45 like the release pipeline: an attempt is pack + notarize +
@@ -322,7 +332,6 @@ jobs:
       # release missing that manifest is a tag the picker offers and the download
       # 404s on, so fail loudly instead of leaving a broken entry.
       - name: Verify update manifest published
-        if: steps.freshness.outputs.should_build == 'true'
         shell: bash
         env:
           GH_TOKEN: ${{ steps.app_token_publish.outputs.token }}
@@ -352,7 +361,6 @@ jobs:
       # means the picker can never offer a release whose assets are incomplete.
       - name: Publish the verified release
         id: publish_live
-        if: steps.freshness.outputs.should_build == 'true'
         shell: bash
         env:
           GH_TOKEN: ${{ steps.app_token_publish.outputs.token }}

.github/workflows/mobile-android-release.yml

@@ -104,11 +104,47 @@ jobs:
               --clobber \
               android/app/build/outputs/apk/release/*.apk
           else
+            # Why: release tags live on side branches, so GitHub's automatic
+            # previous-tag detection reaches back several releases; that body
+            # already exceeds the 125000-character API limit and grows each
+            # release. Pin the comparison base and cap the size.
+            notes_file="$RUNNER_TEMP/android-release-notes.md"
+            previous_tag="$(
+              gh release list --repo "$GITHUB_REPOSITORY" --limit 200 --json tagName --jq '.[].tagName' \
+                | grep '^mobile-android-v' | grep -Fxv "$tag" | sort -V | tail -1 || true
+            )"
+
+            if [ -n "$previous_tag" ]; then
+              # Why: gh writes the JSON error body to stdout on an HTTP error, so a
+              # non-empty file is not proof of success — gate on exit status.
+              if ! gh api "repos/$GITHUB_REPOSITORY/releases/generate-notes" -X POST \
+                -f tag_name="$tag" \
+                -f target_commitish="$GITHUB_SHA" \
+                -f previous_tag_name="$previous_tag" \
+                --jq .body > "$notes_file"; then
+                : > "$notes_file"
+              fi
+            fi
+            if [ ! -s "$notes_file" ]; then
+              printf 'Orca Mobile Android %s\n' "$tag" > "$notes_file"
+            fi
+            # Why: reuse the desktop release path's character-safe truncation so a
+            # multi-byte character cannot be split at the cap.
+            NOTES_FILE="$notes_file" \
+            NOTES_MODULE="$GITHUB_WORKSPACE/config/scripts/create-draft-release.mjs" \
+            node --input-type=module -e '
+              const { readFileSync, writeFileSync } = await import("node:fs")
+              const { pathToFileURL } = await import("node:url")
+              const { truncateReleaseBody } = await import(pathToFileURL(process.env.NOTES_MODULE).href)
+              const file = process.env.NOTES_FILE
+              writeFileSync(file, truncateReleaseBody(readFileSync(file, "utf8")))
+            '
+
             gh release create "$tag" \
               --repo "$GITHUB_REPOSITORY" \
               --title "Orca Mobile Android $tag" \
               --prerelease \
               --latest=false \
-              --generate-notes \
+              --notes-file "$notes_file" \
               android/app/build/outputs/apk/release/*.apk
           fi

.github/workflows/mobile.yml

@@ -15,8 +15,13 @@ on:
       # Why: this job holds the only checks that load the Fastfile, so edits to
       # it or to the release workflow it guards must re-run them.
       - '.github/workflows/mobile.yml'
+      - '.github/actions/install-node-dependencies/**'
       - '.github/workflows/mobile-ios-release.yml'
 
+concurrency:
+  group: mobile-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   verify:
     runs-on: ubuntu-latest
@@ -35,10 +40,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v6
 
-      - name: Setup Node.js
-        uses: actions/setup-node@v6
-        with:
-          node-version-file: package.json
+      - uses: ./.github/actions/install-node-dependencies
 
       # bundler-cache installs mobile/Gemfile.lock, so this job is also what
       # proves the pinned fastlane the release workflow depends on still
@@ -50,23 +52,6 @@ jobs:
           bundler-cache: true
           working-directory: mobile
 
-      - name: Setup pnpm
-        uses: pnpm/setup@v2
-        with:
-          install: false
-
-      # Why: the mobile typecheck imports shared types from ../src/shared, and
-      # some of those files import runtime deps (tweetnacl, ws) resolved from
-      # the repo-root node_modules. Without a root install, tsc fails with
-      # "Cannot find module 'tweetnacl'/'ws'". Mobile is a separate pnpm project
-      # (not in the root workspace), so this is a distinct install.
-      # --ignore-scripts skips the root postinstall (Electron native-module
-      # rebuild) which is irrelevant to a type-only check and would only add
-      # time and failure surface on this ubuntu mobile runner.
-      - name: Install root dependencies
-        working-directory: .
-        run: pnpm install --frozen-lockfile --ignore-scripts
-
       - name: Install dependencies
         run: pnpm install --frozen-lockfile
 

.github/workflows/packaged-browser-e2e.yml

@@ -0,0 +1,74 @@
+name: Packaged browser compatibility
+on:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: Commit SHA or ref to validate (defaults to the selected revision)
+        type: string
+        required: false
+  schedule:
+    - cron: '20 8 * * 1'
+  workflow_call:
+    inputs:
+      ref:
+        type: string
+        required: false
+permissions:
+  contents: read
+jobs:
+  compatibility:
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          ref: ${{ inputs.ref || github.sha }}
+          persist-credentials: false
+      - name: Install headless tools
+        run: sudo apt-get update && sudo apt-get install -y build-essential openssh-client python3 ripgrep xvfb zsh openbox x11-utils
+      - uses: ./.github/actions/install-node-dependencies
+        with:
+          native-runtime: electron
+      - name: Download pinned old release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh release download v1.4.188 --repo stablyai/orca --pattern orca-ide_1.4.188_amd64.deb --dir "$RUNNER_TEMP/old-orca"
+          python3 - <<'PYVERIFY'
+          import base64,hashlib,os,pathlib,subprocess
+          root=pathlib.Path(os.environ['RUNNER_TEMP'])/'old-orca'
+          package=root/'orca-ide_1.4.188_amd64.deb'
+          expected='uGONFUDfinYggxcT9ac72wnnlofLQaqasDDeP0HWOSqarBwTi1Ax3khmzKUY3vUnvuYOpSCEmsH4InzLZ2vg6g=='
+          assert base64.b64encode(hashlib.sha512(package.read_bytes()).digest()).decode()==expected
+          extracted=root/'extracted'
+          subprocess.run(['dpkg-deb','-x',str(package),str(extracted)],check=True)
+          executable=extracted/'opt'/'Orca'/'orca-ide'
+          assert executable.is_file() and os.access(executable,os.X_OK)
+          with open(os.environ['GITHUB_ENV'],'a') as env: env.write('ORCA_CROSS_VERSION_PACKAGED_EXECUTABLE='+str(executable)+'\n')
+          print('Verified old package:',executable)
+          PYVERIFY
+      - name: Build current Electron app
+        env:
+          VITE_EXPOSE_STORE: 'true'
+        run: |
+          pnpm run build:relay
+          pnpm exec electron-vite build --mode e2e
+          pnpm run build:web-from-renderer
+      - name: Run both mixed-version directions
+        env:
+          PLAYWRIGHT_JSON_OUTPUT_FILE: test-results/packaged-browser-results.json
+        run: >-
+          xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh
+          env SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1
+          pnpm exec playwright test --config tests/playwright.config.ts
+          tests/e2e/packaged-mixed-version-browser-placement.spec.ts
+          --project=electron-headless --workers=1 --retries=0 --repeat-each=3 --reporter=list,json
+      - name: Require all six compatibility executions
+        if: always()
+        run: node config/scripts/verify-packaged-browser-participation.mjs test-results/packaged-browser-results.json
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: packaged-mixed-version-audit
+          path: test-results/
+          retention-days: 3

.github/workflows/performance-contracts.yml

@@ -0,0 +1,63 @@
+name: Performance contracts
+
+on:
+  schedule:
+    - cron: '15 9 * * *'
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - '.github/workflows/performance-contracts.yml'
+      - 'config/vitest.performance.config.ts'
+      - 'config/oxlint-performance-audit.json'
+      - 'config/oxlint-plugins/*performance.mjs'
+      - 'config/oxlint-plugins/quadratic-buffer-concat.mjs'
+      - 'config/scripts/*-plugin.test.mjs'
+      # Keep in sync with the contract list in config/vitest.performance.config.ts;
+      # without these a rename lands green and only breaks the next nightly.
+      - 'src/main/sqlite/sync-database.test.ts'
+      - 'src/main/runtime/orchestration/db/row-column-lists.test.ts'
+      - 'src/relay/fs-path-metadata-symlink-concurrency.test.ts'
+      - 'src/renderer/src/components/editor/rich-markdown-list-tokenizers.test.ts'
+      - 'src/renderer/src/components/editor/rich-markdown-lowlight-cache.test.ts'
+      - 'src/renderer/src/components/terminal-pane/agent-completion-coordinator-queued-inspection-disposal.test.ts'
+      - 'src/renderer/src/lib/pane-manager/pane-terminal-output-scheduler-queue-retention.test.ts'
+
+permissions:
+  contents: read
+
+concurrency:
+  group: performance-contracts-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  contracts:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - uses: ./.github/actions/install-node-dependencies
+      - name: Run operation-count and retention contracts
+        run: pnpm test:perf:contracts --reporter=default --reporter=json --outputFile=performance-contracts.json
+      # Source-only scan: identical on every OS, so run it once.
+      - name: Audit production performance patterns
+        if: always() && matrix.os == 'ubuntu-latest'
+        shell: bash
+        run: pnpm --silent audit:perf > performance-audit.json
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: performance-contracts-${{ matrix.os }}
+          path: performance-contracts.json
+          if-no-files-found: error
+      - uses: actions/upload-artifact@v7
+        if: always() && matrix.os == 'ubuntu-latest'
+        with:
+          name: performance-audit
+          path: performance-audit.json
+          if-no-files-found: error

.github/workflows/pr.yml

@@ -28,6 +28,7 @@ jobs:
     outputs:
       should_run: ${{ steps.filter.outputs.should_run }}
       native_cache_changed: ${{ steps.filter.outputs.native_cache_changed }}
+      mobile_dependencies: ${{ steps.filter.outputs.mobile_dependencies }}
       static_analysis: ${{ steps.filter.outputs.static_analysis }}
       typecheck: ${{ steps.filter.outputs.typecheck }}
       git_compatibility: ${{ steps.filter.outputs.git_compatibility }}
@@ -40,6 +41,11 @@ jobs:
       managed_hook_node18: ${{ steps.filter.outputs.managed_hook_node18 }}
       package: ${{ steps.filter.outputs.package }}
       package_windows: ${{ steps.filter.outputs.package_windows }}
+      e2e_should_run: ${{ steps.e2e_filter.outputs.should_run }}
+      test_files: ${{ steps.e2e_filter.outputs.test_files }}
+      ssh_source_changed: ${{ steps.e2e_filter.outputs.ssh_source_changed }}
+      native_ime_source_changed: ${{ steps.e2e_filter.outputs.native_ime_source_changed }}
+      wsl_source_changed: ${{ steps.e2e_filter.outputs.wsl_source_changed }}
     steps:
       - name: Checkout
         uses: actions/checkout@v6
@@ -65,6 +71,41 @@ jobs:
           printf '%s\n' "$CHANGED"
           printf '%s\n' "$CHANGED" | node config/scripts/pr-code-change-scope.mjs | tee -a "$GITHUB_OUTPUT"
 
+      # Reuse the path-detector checkout instead of queuing another runner.
+      - name: Filter changed E2E specs
+        id: e2e_filter
+        if: github.event.pull_request.draft != true && steps.filter.outputs.should_run == 'true'
+        run: |
+          set -euo pipefail
+          BASE="${{ github.event.pull_request.base.sha }}"
+          HEAD="${{ github.event.pull_request.head.sha }}"
+          CHANGED="$(git diff --name-only --diff-filter=AMCR --merge-base "$BASE" "$HEAD")"
+          # Source routes are executable contracts so a test can prove exact
+          # authorities, exclusions, and sentinels without evaluating workflow shell.
+          TEST_FILES_JSON="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs)"
+          echo "test_files=$TEST_FILES_JSON" >> "$GITHUB_OUTPUT"
+          # Why a separate signal: the Docker-SSH lane must trigger on SSH source, not on a
+          # spec name surviving in a route's list. Same routes, so the two cannot drift.
+          SSH_SOURCE_CHANGED="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --ssh-source)"
+          echo "ssh_source_changed=$SSH_SOURCE_CHANGED" >> "$GITHUB_OUTPUT"
+          echo "SSH source changed: $SSH_SOURCE_CHANGED"
+          # Why its own signal: the real-IME lane is a whole ibus session, not a spec, so it must
+          # trigger on IME source rather than on a spec name in some route's list.
+          NATIVE_IME_SOURCE_CHANGED="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --native-ime-source)"
+          echo "native_ime_source_changed=$NATIVE_IME_SOURCE_CHANGED" >> "$GITHUB_OUTPUT"
+          WSL_CHANGED="$(git diff --name-only --no-renames --diff-filter=ACDMR --merge-base "$BASE" "$HEAD")"
+          WSL_SOURCE_CHANGED="$(printf '%s\n' "$WSL_CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --wsl-source)"
+          echo "wsl_source_changed=$WSL_SOURCE_CHANGED" >> "$GITHUB_OUTPUT"
+          echo "Native IME source changed: $NATIVE_IME_SOURCE_CHANGED"
+          SHOULD_RUN="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --reusable-workflow)"
+          if [ "$SHOULD_RUN" = true ]; then
+            echo "should_run=true" >> "$GITHUB_OUTPUT"
+            echo "Changed E2E specs: $TEST_FILES_JSON"
+          else
+            echo "should_run=false" >> "$GITHUB_OUTPUT"
+            echo "No specs requiring the reusable E2E workflow"
+          fi
+
   static_analysis:
     name: static analysis
     needs: [code_paths]
@@ -95,6 +136,25 @@ jobs:
       - name: Enforce type-aware code-quality baseline
         run: pnpm run audit:code-quality:type-aware
 
+      # Why: the changed-code gate lints mobile files too, and its type-aware pass
+      # resolves types from mobile/node_modules. Mobile is a separate pnpm project,
+      # so the root install above leaves it empty and every mobile type degrades to
+      # an `error` type — reported as phantom findings against the changed lines.
+      # Why no --ignore-scripts, unlike the root install: mobile's postinstall generates
+      # the gitignored terminal/mermaid webview engine modules that tracked source imports,
+      # and skipping it degrades those very types the step exists to resolve. The drift
+      # guard mirrors the root install so a stale mobile lockfile fails by name — mobile's
+      # lockfile carries patchedDependencies that a silent rewrite would drop.
+      - name: Install mobile dependencies
+        if: needs.code_paths.outputs.mobile_dependencies == 'true'
+        working-directory: mobile
+        run: |
+          pnpm install --frozen-lockfile
+          if [ "$(git -C "$GITHUB_WORKSPACE" rev-parse --is-inside-work-tree 2>/dev/null)" = true ]; then
+            git -C "$GITHUB_WORKSPACE" diff --exit-code -- \
+              mobile/package.json mobile/pnpm-lock.yaml mobile/pnpm-workspace.yaml
+          fi
+
       - name: Enforce changed-code quality
         run: pnpm run check:code-quality:changed -- "${{ github.event.pull_request.base.sha }}"
 
@@ -360,7 +420,7 @@ jobs:
       - uses: ./.github/actions/install-node-dependencies
 
       # Why: the check rebuilds every package in the manifest from a pinned upstream
-      # commit — @xterm/xterm and the two addons, each built twice (once unmodified to
+      # commit — @xterm/xterm and its three addons, each built twice (once unmodified to
       # prove the toolchain still reproduces the published bundles, once patched). Caching
       # the npm metadata and the shallow clone keeps the repeated cost to the builds
       # themselves; the key is the manifest, so a commit, package or toolchain bump
@@ -692,7 +752,11 @@ jobs:
       - name: Package unpacked app
         env:
           ORCA_REUSE_PREPARED_NATIVE_RUNTIME: '1'
-        run: pnpm exec electron-builder --config config/electron-builder.config.cjs --linux AppImage deb rpm --x64 --publish never
+        # PR artifacts are only inspected locally; gzip avoids release-size xz compression.
+        run: >-
+          pnpm exec electron-builder --config config/electron-builder.config.cjs
+          --linux AppImage deb rpm --x64 --publish never
+          --config.deb.compression=gz --config.rpm.compression=gzip
 
       - name: Verify root-package marker payloads
         run: |
@@ -772,10 +836,13 @@ jobs:
             node_modules/.pnpm/@vscode+windows-process-tree@*/node_modules/@vscode/windows-process-tree/build
           key: native-modules-${{ runner.os }}-${{ steps.deps.outputs.native-cache-scope }}-${{ runner.arch }}-node-node${{ steps.deps.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch') }}
 
+      # vitest runs here directly rather than through `pnpm test`, so the addon
+      # assertions only hold once install-node-dependencies has rebuilt natives.
       - name: Test Windows-specific boundaries
         run: >-
           pnpm exec vitest run --config config/vitest.config.ts
           config/scripts/rebuild-native-deps.test.mjs
+          config/scripts/rebuild-native-deps-windows-process-tree.test.mjs
           src/main/browser/browser-client-page-renderer-lifecycle.electron.test.ts
           src/main/browser/browser-route-tcp-egress.electron.test.ts
           src/main/browser/browser-route-webrtc-egress.electron.test.ts
@@ -784,9 +851,16 @@ jobs:
           src/main/providers/windows-conpty-wide-char-duplication.node-pty.test.ts
           src/main/providers/pty-repaint-wide-char-buffer.node-pty.test.ts
           src/shared/child-process/windows-command-line.win32.test.ts
+          src/shared/child-process/windows-cmd-shim-resolution.test.ts
+          src/shared/child-process/windows-cmd-shim-resolution.win32.test.ts
           src/main/agent-hooks/windows-hook-payload-delivery.test.ts
+          src/main/agent-hooks/windows-direct-cmd-hook-command.test.ts
           src/main/windows/windows-pty-job.win32.test.ts
+          src/main/windows/windows-msys-job.win32.test.ts
           src/main/windows/windows-host-job.win32.test.ts
+          src/main/windows/windows-process-tree-command-line-patch.test.ts
+          src/main/windows/windows-process-table-native-addon.win32.test.ts
+          src/main/windows-live-tree-kill.win32.test.ts
           src/main/wsl/wsl-runner.test.ts
           src/main/wsl/wsl-guest-environment.test.ts
           src/main/wsl/wsl-invocation-boundary.test.ts
@@ -794,13 +868,18 @@ jobs:
           src/main/wsl/wsl-w1-w3-contract.test.ts
           src/shared/source-scan/source-tree-scan.test.ts
           src/main/cli/wsl-cli-powershell-boundary.test.ts
+          src/main/computer/desktop-script-runtime-host.win32.test.ts
           src/main/cursor/hook-service.test.ts
           src/main/orca-profiles/profile-index-store.test.ts
+          src/main/startup/windows-install-dir-acl-repair.win32.test.ts
           src/main/runtime/repo-worktree-admin-fingerprint.test.ts
           src/main/runtime/worktree-scan-admin-fingerprint-gate.test.ts
           src/shared/secure-file-fsync-flags.test.ts
+          src/shared/secure-path-windows-acl.win32.test.ts
+          src/main/runtime/unreadable-secret-store-preservation.win32.test.ts
           src/main/ipc/pty-codex-account-attribution.test.ts
           src/main/ipc/pty-spawn-env-codex-resume-provenance.test.ts
+          src/relay/windows-port-scan.win32.test.ts
 
       # Why the :parallel variant: identical to build:release except the three
       # electron-vite targets overlap instead of running back to back. The Linux package
@@ -839,65 +918,10 @@ jobs:
       - name: Smoke packaged CLI
         run: node config/scripts/smoke-packaged-cli.mjs --app-dir=dist/win-unpacked
 
-  # Why: PR E2E is advisory and only validates changed specs; scheduled and
-  # release runs retain full-suite coverage.
-  e2e-paths:
-    name: detect changed e2e specs
-    needs: [code_paths]
-    runs-on: ubuntu-latest
-    if: github.event.pull_request.draft != true && needs.code_paths.outputs.should_run == 'true'
-    # Why: detector only needs to read the checkout; do not inherit repo defaults.
-    permissions:
-      contents: read
-    outputs:
-      should_run: ${{ steps.filter.outputs.should_run }}
-      test_files: ${{ steps.filter.outputs.test_files }}
-      ssh_source_changed: ${{ steps.filter.outputs.ssh_source_changed }}
-      native_ime_source_changed: ${{ steps.filter.outputs.native_ime_source_changed }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v6
-        with:
-          # Why blob:none: full history is needed for the merge-base diff, but historical
-          # file contents are not. Blobs are ~89% of this repo's pack, and Git fetches the
-          # few this job actually reads on demand.
-          fetch-depth: 0
-          filter: blob:none
-          persist-credentials: false
-
-      - name: Filter changed E2E specs
-        id: filter
-        run: |
-          set -euo pipefail
-          BASE="${{ github.event.pull_request.base.sha }}"
-          HEAD="${{ github.event.pull_request.head.sha }}"
-          CHANGED="$(git diff --name-only --diff-filter=AMCR --merge-base "$BASE" "$HEAD")"
-          # Source routes are executable contracts so a test can prove exact
-          # authorities, exclusions, and sentinels without evaluating workflow shell.
-          TEST_FILES_JSON="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs)"
-          echo "test_files=$TEST_FILES_JSON" >> "$GITHUB_OUTPUT"
-          # Why a separate signal: the Docker-SSH lane must trigger on SSH source, not on a
-          # spec name surviving in a route's list. Same routes, so the two cannot drift.
-          SSH_SOURCE_CHANGED="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --ssh-source)"
-          echo "ssh_source_changed=$SSH_SOURCE_CHANGED" >> "$GITHUB_OUTPUT"
-          echo "SSH source changed: $SSH_SOURCE_CHANGED"
-          # Why its own signal: the real-IME lane is a whole ibus session, not a spec, so it must
-          # trigger on IME source rather than on a spec name in some route's list.
-          NATIVE_IME_SOURCE_CHANGED="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --native-ime-source)"
-          echo "native_ime_source_changed=$NATIVE_IME_SOURCE_CHANGED" >> "$GITHUB_OUTPUT"
-          echo "Native IME source changed: $NATIVE_IME_SOURCE_CHANGED"
-          if [ "$TEST_FILES_JSON" != '[]' ]; then
-            echo "should_run=true" >> "$GITHUB_OUTPUT"
-            echo "Changed E2E specs: $TEST_FILES_JSON"
-          else
-            echo "should_run=false" >> "$GITHUB_OUTPUT"
-            echo "No changed E2E specs"
-          fi
-
   e2e:
     name: e2e
-    needs: e2e-paths
-    if: needs.e2e-paths.outputs.should_run == 'true'
+    needs: code_paths
+    if: needs.code_paths.outputs.e2e_should_run == 'true'
     # Why: reusable e2e.yml only checkouts, builds, and uploads artifacts.
     permissions:
       contents: read
@@ -906,8 +930,8 @@ jobs:
       # The synthetic pull-request merge ref can disappear while this reusable
       # workflow is queued. The head SHA is immutable and works for every PR.
       ref: ${{ github.event.pull_request.head.sha }}
-      test_files: ${{ needs.e2e-paths.outputs.test_files }}
-      ssh_source_changed: ${{ needs.e2e-paths.outputs.ssh_source_changed }}
+      test_files: ${{ needs.code_paths.outputs.test_files }}
+      ssh_source_changed: ${{ needs.code_paths.outputs.ssh_source_changed }}
 
   # Why this is not in verify's needs: it is the first PR-gate run of a harness whose reliability
   # is only known from nightly main runs (20/20 green, 2026-08-09..2026-08-29, p50 3m25s). It
@@ -917,13 +941,23 @@ jobs:
   # require `success || skipped` outside the strict loop — see the note on `e2e`.
   terminal_ime_native:
     name: real IME
-    needs: e2e-paths
-    if: needs.e2e-paths.outputs.native_ime_source_changed == 'true'
+    needs: code_paths
+    if: needs.code_paths.outputs.native_ime_source_changed == 'true'
     # Why: the reusable workflow only checks out, builds, and uploads artifacts.
     permissions:
       contents: read
     uses: ./.github/workflows/terminal-ime-e2e.yml
 
+  windows_wsl:
+    name: real WSL terminal
+    needs: code_paths
+    if: needs.code_paths.outputs.wsl_source_changed == 'true'
+    permissions:
+      contents: read
+    uses: ./.github/workflows/windows-wsl-e2e.yml
+    with:
+      ref: ${{ github.event.pull_request.head.sha }}
+
   verify:
     if: always()
     needs:

.github/workflows/release-cut.yml

@@ -809,13 +809,7 @@ jobs:
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           TAG: ${{ needs.cut.outputs.tag }}
-        run: |
-          if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
-            echo "Release $TAG already exists."
-            exit 0
-          fi
-
-          node config/scripts/create-draft-release.mjs "$TAG"
+        run: node config/scripts/create-draft-release.mjs "$TAG"
 
   terminal-rendering-golden:
     needs: cut
@@ -858,16 +852,17 @@ jobs:
         if: runner.os == 'Linux'
         run: sudo apt-get update && sudo apt-get install -y build-essential python3 xvfb
 
-      - name: Setup Node.js
-        uses: actions/setup-node@v6
-        with:
-          node-version-file: package.json
-
       - name: Setup pnpm
         uses: pnpm/setup@v2
         with:
           install: false
 
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version-file: package.json
+          cache: pnpm
+
       # Why: Linux terminal golden E2E uses the same native install path as
       # release CI, which needs pnpm to bypass its non-executable gyp_main.py.
       - name: Use external node-gyp to avoid pnpm's bundled copy (Linux only)
@@ -1074,16 +1069,17 @@ jobs:
         if: runner.os == 'Linux'
         run: sudo apt-get update && sudo apt-get install -y build-essential python3 xvfb
 
-      - name: Setup Node.js
-        uses: actions/setup-node@v6
-        with:
-          node-version-file: package.json
-
       - name: Setup pnpm
         uses: pnpm/setup@v2
         with:
           install: false
 
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version-file: package.json
+          cache: pnpm
+
       # Why: keep the non-blocking evidence lane on the same Linux native
       # install path as the blocking golden and release build jobs.
       - name: Use external node-gyp to avoid pnpm's bundled copy (Linux only)
@@ -1425,6 +1421,17 @@ jobs:
           command: ${{ matrix.release_command }}
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          # Why: the NSIS uninstaller only exists inside electron-builder's
+          # uninstaller pass, which deletes it right after embedding it. The sign
+          # hook in config/scripts/windows-uninstaller-signing.cjs copies it out
+          # here so it can ride the inner-binaries SignPath request below.
+          # Why runner.temp and never the workspace: `files` in
+          # config/electron-builder.config.cjs is all-negation, so app-builder
+          # prepends `**/*` and packs whatever is left in the checkout root. This
+          # step retries up to 3 times; attempt 1 writes the file after packing,
+          # but attempts 2 and 3 would then pack the unsigned uninstaller into
+          # app.asar - the exact defect this chain exists to remove.
+          ORCA_WIN_UNINSTALLER_EXPORT_PATH: ${{ runner.temp }}\uninstaller-signing\unsigned\orca-uninstaller.exe
 
       - name: Verify Windows node-pty ConPTY runtime
         if: matrix.platform == 'win' && github.run_attempt == 1
@@ -1451,7 +1458,10 @@ jobs:
       # Why: SignPath cannot deep-sign inside NSIS installers, so inner PE
       # files (Orca.exe, node-pty *.node, DLLs) are signed via a separate zip
       # request, then the installer is rebuilt from the signed tree before the
-      # existing installer signing request below. Every step in this chain is
+      # existing installer signing request below. The NSIS uninstaller rides
+      # this same request (it is the MDE update cluster: old-uninstaller.exe /
+      # Uninstall Orca.exe), captured through electron-builder's sign hook and
+      # swapped back in during the rebuild — no third approval wait. Every step is
       # fail-open (continue-on-error + outcome gating): any failure ships the
       # original installer with unsigned inner binaries, exactly like releases
       # did before this chain existed. Rehearsed end to end in run 28988432001
@@ -1498,6 +1508,36 @@ jobs:
           Write-Host "Skipped $($skipped.Count) already-signed files:"
           $skipped | ForEach-Object { Write-Host "  $_" }
 
+          # Why the uninstaller rides this request: it is the file MDE flagged in
+          # the whole update cluster (old-uninstaller.exe / Uninstall Orca.exe),
+          # and folding it in here costs no extra approval wait. Why it is kept
+          # out of inner-signing-list.txt: that list drives the copy-back into
+          # dist/win-unpacked, and the uninstaller does not live there — it is
+          # re-injected through the sign hook during the rebuild instead.
+          # Why this name and not "Uninstall Orca.exe": the restore loop below
+          # matches staged files by suffix (`-like "*$relative"`) and takes the
+          # first hit, so any staged path ending in "Orca.exe" is separated from
+          # the real Orca.exe only by Get-ChildItem's enumeration order. That
+          # order happens to favour the root file today, but it is not a
+          # documented guarantee; a name that cannot suffix-match is.
+          # Why the whole block is caught rather than just Test-Path'd: this
+          # step's outcome gates the upload of every inner binary, so a locked
+          # file or a full disk here would cost all of them their signatures -
+          # worse than shipping no uninstaller signature at all.
+          try {
+            $exportedUninstaller = Join-Path $env:RUNNER_TEMP 'uninstaller-signing\unsigned\orca-uninstaller.exe'
+            if (Test-Path -LiteralPath $exportedUninstaller) {
+              $uninstallerStagePath = Join-Path $stage.FullName 'uninstaller\orca-uninstaller.exe'
+              New-Item -ItemType Directory -Force -Path (Split-Path $uninstallerStagePath) -ErrorAction Stop | Out-Null
+              Copy-Item -LiteralPath $exportedUninstaller -Destination $uninstallerStagePath -Force -ErrorAction Stop
+              Write-Host 'Staged the NSIS uninstaller for signing: uninstaller\orca-uninstaller.exe'
+            } else {
+              Write-Host "::warning::No exported NSIS uninstaller at $exportedUninstaller; this release ships an unsigned uninstaller (fail-open)."
+            }
+          } catch {
+            Write-Host "::warning::Could not stage the NSIS uninstaller ($_); this release ships an unsigned uninstaller (fail-open)."
+          }
+
       - name: Upload unsigned inner binaries for SignPath
         id: upload-unsigned-inner
         if: matrix.platform == 'win' && github.run_attempt == 1 && steps.stage-inner.outcome == 'success'
@@ -1642,6 +1682,31 @@ jobs:
             throw "Signed inner artifact did not round-trip cleanly ($($failures.Count) failures)."
           }
 
+      # Why gated separately from the inner restore above: if SignPath's
+      # windows-inner-binaries-zip artifact configuration does not (yet) cover the
+      # uninstaller/ directory, the uninstaller comes back missing. That must cost
+      # only the uninstaller signature — the rebuild below still runs and still
+      # ships the signed inner binaries, exactly as it does today.
+      - name: Restore signed uninstaller for the installer rebuild
+        id: restore-signed-uninstaller
+        if: matrix.platform == 'win' && github.run_attempt == 1 && steps.restore-signed-inner.outcome == 'success'
+        continue-on-error: true
+        shell: pwsh
+        run: |
+          $signed = Get-ChildItem -Path signed-inner -Recurse -File -Filter 'orca-uninstaller.exe' |
+            Select-Object -First 1
+          if ($null -eq $signed) {
+            throw 'SignPath did not return uninstaller/orca-uninstaller.exe; check the windows-inner-binaries-zip artifact configuration covers it.'
+          }
+          $signature = Get-AuthenticodeSignature -FilePath $signed.FullName
+          if ($null -eq $signature.SignerCertificate) {
+            throw 'The returned NSIS uninstaller carries no signature.'
+          }
+          $signedDir = Join-Path $env:RUNNER_TEMP 'uninstaller-signing\signed'
+          New-Item -ItemType Directory -Force -Path $signedDir | Out-Null
+          Copy-Item -LiteralPath $signed.FullName -Destination (Join-Path $signedDir 'orca-uninstaller.exe') -Force
+          Write-Host ("{0,-14} uninstaller  <{1}>" -f $signature.Status, $signature.SignerCertificate.Subject)
+
       # Why this step exists: electron-builder's CopyElevateHelper re-copies a
       # pristine elevate.exe from its download cache over resources\elevate.exe
       # on EVERY nsis pack — including the --prepackaged rebuild below — which
@@ -1651,9 +1716,12 @@ jobs:
       # no-op. Known quirk: the cache persists across releases via actions/cache,
       # so later runs may see elevate.exe as already signed and skip staging it —
       # that is fine (the signature is timestamped) and the evidence gate checks
-      # elevate.exe in the shipped installer unconditionally. If this ever causes
-      # trouble, delete this step; the only effect is elevate.exe shipping
-      # unsigned again, which the evidence gate will flag.
+      # elevate.exe in the shipped installer unconditionally.
+      #
+      # The cache lookup lives in a script because the inline path this step used
+      # (`<cache>\nsis`) matches no app-builder-lib layout, and `SilentlyContinue`
+      # plus `exit 0` turned that miss into a green step — v1.4.193 and v1.4.194
+      # shipped an unsigned elevate.exe that way. A miss now fails the step.
       - name: Replace cached elevate.exe with the signed copy
         id: sign-elevate-cache
         if: matrix.platform == 'win' && github.run_attempt == 1 && steps.restore-signed-inner.outcome == 'success'
@@ -1665,20 +1733,26 @@ jobs:
             Write-Host '::warning::No elevate.exe in win-unpacked resources; nothing to protect from the rebuild clobber.'
             exit 0
           }
+          # Why this guard stays: windows-signing-rehearsal.yml shares the
+          # electron-builder-win-<lockfile hash> cache key with this workflow, so a
+          # test-certificate elevate.exe must never be staged into a release cache.
           $signature = Get-AuthenticodeSignature -FilePath $signed
           $subject = if ($null -eq $signature.SignerCertificate) { '<none>' } else { $signature.SignerCertificate.Subject }
           if ($signature.Status -ne 'Valid' -or $subject -notlike '*CN=SignPath Foundation*') {
             Write-Host "::warning::win-unpacked elevate.exe is not SignPath-signed ($($signature.Status), $subject); skipping cache swap."
             exit 0
           }
-          $cached = @(Get-ChildItem "$env:LOCALAPPDATA\electron-builder\Cache\nsis" -Recurse -Filter elevate.exe -ErrorAction SilentlyContinue)
-          if ($cached.Count -eq 0) {
-            Write-Host '::warning::No cached elevate.exe found (electron-builder cache layout changed?); the rebuild will pack the unsigned copy and the evidence gate will flag it.'
-            exit 0
-          }
-          foreach ($file in $cached) {
-            Copy-Item -Path $signed -Destination $file.FullName -Force
-            Write-Host "Replaced $($file.FullName) with the SignPath-signed copy."
+          node config/scripts/replace-cached-nsis-elevate.mjs $signed
+          if ($LASTEXITCODE -ne 0) {
+            $message = 'Cached elevate.exe swap found nothing to replace; the rebuilt installer ships an unsigned UAC elevation helper (issue #7785).'
+            if ($env:GITHUB_STEP_SUMMARY) {
+              try {
+                Add-Content -Path $env:GITHUB_STEP_SUMMARY -Value "**Windows elevate.exe cache swap:** FAILED — $message" -ErrorAction Stop
+              } catch {
+                Write-Host "::warning::Could not write the elevate.exe swap verdict to the job summary: $_"
+              }
+            }
+            throw $message
           }
 
       - name: Rebuild NSIS installer from signed unpacked app
@@ -1686,6 +1760,11 @@ jobs:
         if: matrix.platform == 'win' && github.run_attempt == 1 && steps.restore-signed-inner.outcome == 'success'
         continue-on-error: true
         shell: pwsh
+        env:
+          # Why unconditional: the sign hook keys off the file existing, which it
+          # only does when the restore step above succeeded. A missing file logs a
+          # warning and embeds the freshly built unsigned uninstaller instead.
+          ORCA_WIN_UNINSTALLER_SIGNED_PATH: ${{ runner.temp }}\uninstaller-signing\signed\orca-uninstaller.exe
         run: |
           # Why: keep the pre-rebuild artifacts so a failed rebuild can fall
           # back to shipping them unchanged (fail-open).
@@ -1716,6 +1795,7 @@ jobs:
         with:
           name: orca-windows-unsigned-${{ needs.cut.outputs.tag }}
           path: dist/orca-windows-setup.exe
+          compression-level: 0
           if-no-files-found: error
 
       # Why: SignPath Foundation production certificates require manual review,
@@ -1876,6 +1956,7 @@ jobs:
         env:
           ORCA_WINDOWS_INNER_SIGNATURE_REQUIRED: 'false'
           INNER_SIGNING_COMPLETED: ${{ steps.rebuild-nsis-signed.outcome == 'success' }}
+          UNINSTALLER_SIGNING_COMPLETED: ${{ steps.restore-signed-uninstaller.outcome == 'success' }}
         run: |
           $required = $env:ORCA_WINDOWS_INNER_SIGNATURE_REQUIRED -eq 'true'
 
@@ -1956,6 +2037,39 @@ jobs:
             if ($targets -notcontains 'resources\elevate.exe') {
               $targets += 'resources\elevate.exe'
             }
+            # Why the uninstaller is not in $targets: NSIS embeds it in its own
+            # compressed data section (`File /oname=${UNINSTALL_FILENAME}` in
+            # app-builder-lib templates/nsis/include/installer.nsh), not in the
+            # app 7z payload extracted above - the bundled 7za cannot see it.
+            # What the receipt proves and does not: the digest comparison is
+            # equal by construction (the hook digests the bytes it copied from
+            # this same file), so the real signal is that the receipt exists at
+            # all - the import leg ran, and these are the bytes it embedded. The
+            # signature check below is the part with teeth. The shipped-artifact
+            # check lives in windows-signing-rehearsal.yml, which installs the
+            # installer and inspects the uninstaller it drops on disk.
+            if ($env:UNINSTALLER_SIGNING_COMPLETED -eq 'true') {
+              $signedUninstaller = Join-Path $env:RUNNER_TEMP 'uninstaller-signing\signed\orca-uninstaller.exe'
+              $receipt = "$signedUninstaller.embedded-sha256"
+              if (-not (Test-Path -LiteralPath $receipt)) {
+                $failures.Add('the sign hook did not embed the signed uninstaller into the rebuilt installer')
+              } else {
+                $embedded = (Get-Content -LiteralPath $receipt -Raw).Trim()
+                $actual = (Get-FileHash -LiteralPath $signedUninstaller -Algorithm SHA256).Hash.ToLowerInvariant()
+                $signature = Get-AuthenticodeSignature -FilePath $signedUninstaller
+                $subject = if ($null -eq $signature.SignerCertificate) { '<none>' } else { $signature.SignerCertificate.Subject }
+                $line = "{0,-14} {1}  <{2}>" -f $signature.Status, 'Uninstall Orca.exe (embedded)', $subject
+                $report.Add($line)
+                Write-Host $line
+                if ($embedded -ne $actual) {
+                  $failures.Add("the rebuilt installer embedded different uninstaller bytes than the signed one ($embedded vs $actual)")
+                } elseif ($signature.Status -ne 'Valid' -or $subject -notlike '*CN=SignPath Foundation*') {
+                  $failures.Add("not signed by SignPath Foundation: Uninstall Orca.exe ($($signature.Status), $subject)")
+                }
+              }
+            } else {
+              Write-Host '::warning::The NSIS uninstaller was not signed on this run; it is excluded from the evidence gate (fail-open).'
+            }
             foreach ($relative in $targets) {
               $path = Join-Path $root $relative
               if (-not (Test-Path $path)) {
@@ -1988,7 +2102,9 @@ jobs:
               Add-GateEvidence "VERDICT: FAILED — $message"
               Add-GateSummary "FAILED — $message"
             } else {
-              $ok = "All $($targets.Count) inner binaries in the shipped installer are signed by SignPath Foundation."
+              # $report, not $targets: the embedded uninstaller is reported but
+              # is not one of the extracted payload targets.
+              $ok = "All $($report.Count) checked binaries are signed by SignPath Foundation."
               Add-GateEvidence "VERDICT: PASSED — $ok"
               Add-GateSummary "PASSED — $ok"
               Write-Host $ok

.github/workflows/release-ref-validation.yml

@@ -0,0 +1,38 @@
+name: Release ref validation
+
+on:
+  pull_request:
+    paths:
+      - '.github/workflows/adhoc-mac-build.yml'
+      - '.github/workflows/dev-channel-win-build.yml'
+      - '.github/workflows/release-ref-validation.yml'
+      - 'config/scripts/workflow-ref-reachability.test.mjs'
+      - 'config/scripts/workflow-ref-mirror-case-safety.test.mjs'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: release-ref-validation-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  validate:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-15, windows-2022]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - uses: ./.github/actions/install-node-dependencies
+      - name: Verify case-twin refs and release trust boundary
+        run: >-
+          pnpm exec vitest run --config config/vitest.config.ts
+          config/scripts/workflow-ref-reachability.test.mjs
+          config/scripts/workflow-ref-mirror-case-safety.test.mjs
+          config/scripts/dev-channel-windows-workflow-contract.test.mjs

.github/workflows/skill-update-roundtrip.yml

@@ -22,6 +22,10 @@ on:
       - main
     paths: *skill-roundtrip-paths
 
+concurrency:
+  group: skill-roundtrip-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   roundtrip:
     strategy:
@@ -41,7 +45,9 @@ jobs:
     steps:
       - uses: actions/checkout@v6
         with:
+          # Historical skill snapshots need tags, but only their blobs are read.
           fetch-depth: 0
+          filter: blob:none
           persist-credentials: false
       - uses: actions/setup-node@v6
         with:

.github/workflows/terminal-ime-e2e.yml

@@ -38,23 +38,9 @@ jobs:
           xfwm4
           xvfb
 
-      - name: Setup Node.js
-        uses: actions/setup-node@v6
+      - uses: ./.github/actions/install-node-dependencies
         with:
-          node-version-file: package.json
-
-      - name: Setup pnpm
-        uses: pnpm/setup@v2
-        with:
-          install: false
-
-      - name: Use external node-gyp to avoid pnpm bundled copy
-        run: |
-          npm install -g node-gyp@11.5.0
-          echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV"
-
-      - name: Install dependencies
-        run: pnpm install --frozen-lockfile
+          native-runtime: electron
 
       - name: Build Electron app for E2E
         run: pnpm exec electron-vite build --mode e2e
@@ -84,3 +70,40 @@ jobs:
           path: test-results/
           retention-days: 7
           if-no-files-found: ignore
+
+  linux-wayland:
+    name: Linux Wayland Hangul terminating digit
+    runs-on: ubuntu-22.04
+    timeout-minutes: 25
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - name: Install native build, nested compositor and IME tools
+        run: >-
+          sudo apt-get update && sudo apt-get install -y
+          build-essential python3 fonts-noto-cjk dbus-x11 dconf-gsettings-backend
+          ibus ibus-hangul gnome-shell gnome-settings-daemon libglib2.0-bin
+          xdotool xvfb x11-utils imagemagick
+      - uses: ./.github/actions/install-node-dependencies
+        with:
+          native-runtime: electron
+      - name: Build Electron app for E2E
+        env:
+          VITE_EXPOSE_STORE: 'true'
+        run: |
+          pnpm run build:relay
+          pnpm exec electron-vite build --mode e2e
+          pnpm run build:web-from-renderer
+      - name: Run native Wayland Hangul terminating digit
+        env:
+          SKIP_BUILD: '1'
+        run: node config/scripts/run-terminal-ibus-hangul-e2e.mjs --nested-wayland
+      - name: Upload Wayland terminal IME evidence
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: terminal-wayland-ime-evidence
+          path: test-results/
+          retention-days: 7
+          if-no-files-found: error

.github/workflows/terminal-perf.yml

@@ -67,16 +67,17 @@ jobs:
       - name: Install native build tools and xvfb
         run: sudo apt-get update && sudo apt-get install -y build-essential python3 xvfb zsh
 
-      - name: Setup Node.js
-        uses: actions/setup-node@v6
-        with:
-          node-version-file: package.json
-
       - name: Setup pnpm
         uses: pnpm/setup@v2
         with:
           install: false
 
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version-file: package.json
+          cache: pnpm
+
       # Why: this scheduled/manual workflow uses the same native install path as
       # PR and E2E CI, which needs pnpm to bypass its bundled gyp_main.py.
       - name: Use external node-gyp to avoid pnpm's bundled copy

.github/workflows/windows-signing-rehearsal.yml

@@ -3,9 +3,11 @@
 # Why: SignPath cannot deep-sign inside NSIS installers, so shipping signed
 # inner binaries (Orca.exe, node-pty *.node, DLLs — see issue #7785) requires
 # a two-request flow: sign the unpacked PE files first, then build the NSIS
-# installer from the signed tree, then sign the installer. This workflow
-# rehearses that entire flow from a branch, end to end, without publishing
-# anything — so the release pipeline on main is never at risk while we verify.
+# installer from the signed tree, then sign the installer. The NSIS uninstaller
+# rides that same first request — it is captured through electron-builder's sign
+# hook and swapped back in during the rebuild — so it adds no third approval.
+# This workflow rehearses that entire flow from a branch, end to end, without
+# publishing anything — so the release pipeline on main is never at risk.
 #
 # Runs only via manual dispatch. Use the test-signing policy for iteration
 # (auto-approved test certificate) and release-signing to rehearse the
@@ -81,15 +83,27 @@ jobs:
         env:
           NODE_OPTIONS: --max-old-space-size=4096
 
-      - name: Package unpacked Windows app
+      # Why a full --win build and not --dir: the NSIS uninstaller only exists
+      # inside the installer build, and it is the file the MDE update cluster
+      # flags. --dir would never produce it, so the rehearsal would not rehearse
+      # the uninstaller leg at all. This mirrors release-cut's first Windows pass.
+      - name: Package Windows app and export the NSIS uninstaller
         shell: pwsh
+        env:
+          # runner.temp, never the workspace: the all-negation `files` list in
+          # config/electron-builder.config.cjs packs whatever is left in the
+          # checkout root into app.asar.
+          ORCA_WIN_UNINSTALLER_EXPORT_PATH: ${{ runner.temp }}\uninstaller-signing\unsigned\orca-uninstaller.exe
         run: |
           node config/scripts/ensure-native-runtime.mjs --runtime=electron
           if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
-          pnpm exec electron-builder --config config/electron-builder.config.cjs --win --dir --publish never
+          pnpm exec electron-builder --config config/electron-builder.config.cjs --win --publish never
           if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
           if (-not (Test-Path 'dist/win-unpacked/Orca.exe')) {
-            throw 'electron-builder --dir did not produce dist/win-unpacked/Orca.exe'
+            throw 'electron-builder --win did not produce dist/win-unpacked/Orca.exe'
+          }
+          if (-not (Test-Path -LiteralPath $env:ORCA_WIN_UNINSTALLER_EXPORT_PATH)) {
+            throw "The sign hook did not export the NSIS uninstaller to $env:ORCA_WIN_UNINSTALLER_EXPORT_PATH"
           }
 
       # Why: only unsigned PE files go to SignPath. Files that already carry a
@@ -132,6 +146,17 @@ jobs:
           Write-Host "Skipped $($skipped.Count) already-signed files:"
           $skipped | ForEach-Object { Write-Host "  $_" }
 
+          # Why kept out of inner-signing-list.txt: that list drives the copy-back
+          # into dist/win-unpacked, and the uninstaller does not live there — it is
+          # re-injected through the electron-builder sign hook during the rebuild.
+          # No catch here, unlike the release job: the rehearsal exists to prove
+          # the flow, so a staging failure must fail it loudly.
+          $exportedUninstaller = Join-Path $env:RUNNER_TEMP 'uninstaller-signing\unsigned\orca-uninstaller.exe'
+          $uninstallerStagePath = Join-Path $stage.FullName 'uninstaller\orca-uninstaller.exe'
+          New-Item -ItemType Directory -Force -Path (Split-Path $uninstallerStagePath) | Out-Null
+          Copy-Item -LiteralPath $exportedUninstaller -Destination $uninstallerStagePath -Force
+          Write-Host 'Staged the NSIS uninstaller for signing: uninstaller\orca-uninstaller.exe'
+
       - name: Upload unsigned inner binaries for SignPath
         id: upload-unsigned-inner
         uses: actions/upload-artifact@v7
@@ -200,8 +225,27 @@ jobs:
             throw "Signed inner artifact did not round-trip cleanly ($($failures.Count) failures)."
           }
 
+      - name: Restore signed uninstaller for the installer rebuild
+        shell: pwsh
+        run: |
+          $signed = Get-ChildItem -Path signed-inner -Recurse -File -Filter 'orca-uninstaller.exe' |
+            Select-Object -First 1
+          if ($null -eq $signed) {
+            throw 'SignPath did not return uninstaller/orca-uninstaller.exe; check the inner-binaries artifact configuration covers it.'
+          }
+          $signature = Get-AuthenticodeSignature -FilePath $signed.FullName
+          if ($null -eq $signature.SignerCertificate) {
+            throw 'The returned NSIS uninstaller carries no signature.'
+          }
+          $signedDir = Join-Path $env:RUNNER_TEMP 'uninstaller-signing\signed'
+          New-Item -ItemType Directory -Force -Path $signedDir | Out-Null
+          Copy-Item -LiteralPath $signed.FullName -Destination (Join-Path $signedDir 'orca-uninstaller.exe') -Force
+          Write-Host ("{0,-14} uninstaller  <{1}>" -f $signature.Status, $signature.SignerCertificate.Subject)
+
       - name: Build NSIS installer from signed unpacked app
         shell: pwsh
+        env:
+          ORCA_WIN_UNINSTALLER_SIGNED_PATH: ${{ runner.temp }}\uninstaller-signing\signed\orca-uninstaller.exe
         run: |
           pnpm exec electron-builder --config config/electron-builder.config.cjs --win --publish never --prepackaged "$env:GITHUB_WORKSPACE\dist\win-unpacked"
           if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
@@ -215,6 +259,7 @@ jobs:
         with:
           name: orca-windows-installer-unsigned-${{ github.run_id }}
           path: dist/orca-windows-setup.exe
+          compression-level: 0
           if-no-files-found: error
 
       - name: Submit Windows installer signing request
@@ -288,20 +333,33 @@ jobs:
         run: |
           $report = New-Object System.Collections.Generic.List[string]
           $failures = New-Object System.Collections.Generic.List[string]
+          $advisories = New-Object System.Collections.Generic.List[string]
           $requireValid = $env:SIGNING_POLICY -eq 'release-signing'
 
-          function Test-Signature([string]$label, [string]$path) {
+          # -Advisory records a problem without failing the run. It exists for
+          # exactly one file (resources\elevate.exe, below) and must not be
+          # widened casually: the point of this workflow is to fail when signing
+          # is broken.
+          function Test-Signature([string]$label, [string]$path, [switch]$Advisory) {
             $signature = Get-AuthenticodeSignature -FilePath $path
             $subject = if ($null -eq $signature.SignerCertificate) { '<none>' } else { $signature.SignerCertificate.Subject }
             $line = "{0,-14} {1}  <{2}>" -f $signature.Status, $label, $subject
             $script:report.Add($line)
             Write-Host $line
+            $problem = $null
             if ($null -eq $signature.SignerCertificate -or $signature.Status -eq 'NotSigned') {
-              $script:failures.Add("unsigned: $label")
+              $problem = "unsigned: $label"
             } elseif ($script:requireValid -and $signature.Status -ne 'Valid') {
-              $script:failures.Add("not Valid under release-signing: $label ($($signature.Status))")
+              $problem = "not Valid under release-signing: $label ($($signature.Status))"
             } elseif ($script:requireValid -and $subject -notlike '*CN=SignPath Foundation*') {
-              $script:failures.Add("unexpected signer: $label ($subject)")
+              $problem = "unexpected signer: $label ($subject)"
+            }
+            if ($null -eq $problem) { return }
+            if ($Advisory) {
+              $script:advisories.Add($problem)
+              Write-Host "::warning::$problem - known pre-existing issue, not failing the rehearsal"
+            } else {
+              $script:failures.Add($problem)
             }
           }
 
@@ -323,21 +381,155 @@ jobs:
           & $7za x 'dist/orca-windows-setup.exe' '-oextracted-app' -y | Out-Null
 
           $root = Resolve-Path 'extracted-app'
+          # The receipt only proves the import leg ran; it cannot prove what NSIS
+          # embedded, because the uninstaller lives in a compressed NSIS data
+          # section rather than the app 7z payload above and the bundled 7za has
+          # no NSIS handler. So the rehearsal - unlike the release job, which
+          # must not mutate the runner it publishes from - goes all the way: it
+          # installs the installer silently and inspects the uninstaller the
+          # installer actually wrote to disk. That is the file MDE flags.
+          $signedUninstaller = Join-Path $env:RUNNER_TEMP 'uninstaller-signing\signed\orca-uninstaller.exe'
+          $receipt = "$signedUninstaller.embedded-sha256"
+          if (-not (Test-Path -LiteralPath $receipt)) {
+            $failures.Add('the sign hook did not embed the signed uninstaller into the rebuilt installer')
+          } else {
+            Test-Signature 'relayed: orca-uninstaller.exe' $signedUninstaller
+          }
+
+          # Why a full 7-Zip attempt first: it is non-invasive. The runner image
+          # ships the complete 7z.exe, which - unlike the reduced 7za - has an
+          # NSIS handler. If it cannot read the section either, fall back to a
+          # real silent install.
+          $installedUninstaller = $null
+          $installedVia = $null
+          $expectedDigest = if (Test-Path -LiteralPath $receipt) { (Get-Content -LiteralPath $receipt -Raw).Trim() } else { $null }
+          $full7z = 'C:\Program Files\7-Zip\7z.exe'
+          if (Test-Path -LiteralPath $full7z) {
+            New-Item -ItemType Directory -Path nsis-extract -Force | Out-Null
+            & $full7z x -tnsis 'dist/orca-windows-setup.exe' '-onsis-extract' -y 2>&1 | Out-Null
+            $installedUninstaller = Get-ChildItem -Path nsis-extract -Recurse -File -Filter 'Uninstall*.exe' -ErrorAction SilentlyContinue |
+              Select-Object -First 1
+            # Why the digest guard before trusting this route: 7-Zip's NSIS
+            # handler emits partial or garbled output on some NSIS builds, and a
+            # truncated extract would score NotSigned and fail the rehearsal as
+            # "the shipped uninstaller is unsigned" when nothing is wrong. Only
+            # trust it when it reproduces the bytes the relay embedded; otherwise
+            # fall through to the install route, which is ground truth. A name
+            # miss (the handler labelling the entry by its source name) falls
+            # through the same way.
+            if ($null -ne $installedUninstaller -and $null -ne $expectedDigest -and
+                (Get-FileHash -LiteralPath $installedUninstaller.FullName -Algorithm SHA256).Hash.ToLowerInvariant() -ne $expectedDigest) {
+              Write-Host "7-Zip's NSIS output did not match the relayed digest; falling back to a silent install."
+              $installedUninstaller = $null
+            }
+            if ($null -ne $installedUninstaller) {
+              $installedVia = "7-Zip's NSIS handler"
+              Write-Host "Read the embedded uninstaller with 7-Zip's NSIS handler: $($installedUninstaller.FullName)"
+            } else {
+              Write-Host "7-Zip's NSIS handler did not yield a usable uninstaller; falling back to a silent install."
+            }
+          }
+
+          if ($null -eq $installedUninstaller) {
+            # Nothing here is published, so mutating this runner is free.
+            # Why -PassThru and a bounded wait rather than -Wait: a bare -Wait on
+            # an installer that ever prompts hangs to the job's 360-minute cap.
+            $installerProcess = Start-Process -FilePath (Resolve-Path 'dist/orca-windows-setup.exe') -ArgumentList '/S' -PassThru
+            if (-not $installerProcess.WaitForExit(300000)) {
+              $installerProcess | Stop-Process -Force -ErrorAction SilentlyContinue
+              $failures.Add('the silent install did not exit within 5 minutes; it is likely prompting')
+            }
+            # Why a poll rather than one Stop-Process: the oneClick installer
+            # launches the app as it finishes, so Orca.exe can appear *after* the
+            # installer process exits. A single silenced Stop-Process would miss
+            # it and leave Orca plus orca-terminal-daemon.exe holding handles
+            # under %LOCALAPPDATA%\Programs for the rest of the job.
+            for ($attempt = 0; $attempt -lt 20; $attempt++) {
+              $running = @(Get-Process -Name 'Orca' -ErrorAction SilentlyContinue)
+              if ($running.Count -gt 0) {
+                $running | Stop-Process -Force -ErrorAction SilentlyContinue
+                break
+              }
+              Start-Sleep -Milliseconds 500
+            }
+            Get-Process -Name 'orca-terminal-daemon' -ErrorAction SilentlyContinue |
+              Stop-Process -Force -ErrorAction SilentlyContinue
+            $installedUninstaller = Get-ChildItem -Path "$env:LOCALAPPDATA\Programs" -Recurse -File -Filter 'Uninstall*.exe' -ErrorAction SilentlyContinue |
+              Where-Object { $_.FullName -like '*Orca*' } |
+              Select-Object -First 1
+            if ($null -ne $installedUninstaller) { $installedVia = 'a silent install' }
+          }
+
+          if ($null -eq $installedUninstaller) {
+            $failures.Add('could not obtain the uninstaller the installer ships; neither 7-Zip nor a silent install produced it')
+          } else {
+            # Why this digest comparison is the point of the whole rehearsal:
+            # unlike the release job's, it hashes a file NSIS itself wrote out
+            # rather than the file the hook copied, so it is the only check that
+            # proves the shipped installer embedded the SignPath-signed bytes. On
+            # the 7-Zip route the guard above already forced equality; on the
+            # install route this is the first time it is tested.
+            if ($null -ne $expectedDigest) {
+              $shippedDigest = (Get-FileHash -LiteralPath $installedUninstaller.FullName -Algorithm SHA256).Hash.ToLowerInvariant()
+              if ($shippedDigest -ne $expectedDigest) {
+                $failures.Add("the uninstaller the installer ships is not the relayed one (via $installedVia): $shippedDigest vs $expectedDigest")
+              }
+            }
+            Test-Signature "shipped: Uninstall Orca.exe (via $installedVia)" $installedUninstaller.FullName
+          }
+
           foreach ($relative in Get-Content 'inner-signing-list.txt') {
             $path = Join-Path $root $relative
             if (-not (Test-Path $path)) {
               $failures.Add("missing from installer payload: $relative")
               continue
             }
-            Test-Signature "installed: $relative" $path
+            # Why elevate.exe alone is advisory: app-builder-lib re-copies the
+            # pristine cached elevate.exe over resources\elevate.exe on EVERY nsis
+            # pack - AppPackageHelper.packArch calls elevateHelper.copy() before
+            # buildAppPackage (nsisUtil.js), and CopyElevateHelper.copy does
+            # `copyFile(elevatePath, outFile, false)` then `signIf(outFile)`, which
+            # signs nothing because this build configures no certificate. So the
+            # signed copy restored into win-unpacked is clobbered by the rebuild.
+            # This predates the uninstaller relay and is not caused by it: with no
+            # `sign` hook, signIf already returned false at "no signing info
+            # identified" (windowsSignToolManager.js), so no signtool call was
+            # displaced. release-cut.yml mitigates it separately by pre-seeding the
+            # electron-builder cache ("Replace cached elevate.exe with the signed
+            # copy"); this workflow has no such step, which is why the clobber is
+            # visible here and not there. Mirroring that step here would not help:
+            # it only swaps when the copy is already Valid and SignPath-signed, so
+            # it no-ops under the test certificate.
+            #
+            # DO NOT relax that Valid + SignPath-signed guard to make this
+            # rehearsal go green. This workflow and release-cut.yml share the
+            # cache key `electron-builder-win-<lockfile hash>`, and that guard is
+            # the only thing stopping a test certificate from being seeded into
+            # the cache a real release restores from. Shipping users a binary
+            # signed by "Test certificate for 'Orca agent ide [OSS]'" is worse
+            # than shipping it unsigned.
+            #
+            # Fixing elevate.exe belongs in its own PR - it is a UAC elevation
+            # helper, and it deserves more scrutiny than a footnote in an
+            # uninstaller change.
+            if ($relative -eq 'resources\elevate.exe') {
+              Test-Signature "installed: $relative" $path -Advisory
+            } else {
+              Test-Signature "installed: $relative" $path
+            }
           }
 
+          if ($advisories.Count -gt 0) {
+            $report.Add('')
+            $report.Add('ADVISORY (known pre-existing, did not fail this run):')
+            $advisories | ForEach-Object { $report.Add("  $_") }
+          }
           Set-Content -Path 'signing-evidence.txt' -Value ($report -join "`n")
           if ($failures.Count -gt 0) {
             $failures | ForEach-Object { Write-Host "::error::$_" }
             throw "Signing rehearsal failed with $($failures.Count) problems."
           }
-          Write-Host "All $((Get-Content 'inner-signing-list.txt').Count) inner binaries plus the installer are signed."
+          Write-Host "All checked binaries are signed, including the uninstaller the installer writes to disk ($($advisories.Count) advisory)."
 
       - name: Upload rehearsal evidence and installer
         if: always()

.github/workflows/windows-wsl-e2e.yml

@@ -0,0 +1,74 @@
+name: Windows WSL terminal E2E
+
+on:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: Commit to validate
+        type: string
+        required: false
+  workflow_call:
+    inputs:
+      ref:
+        type: string
+        required: false
+
+permissions:
+  contents: read
+
+concurrency:
+  group: windows-wsl-e2e-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  wsl-terminal:
+    runs-on: windows-2022
+    timeout-minutes: 30
+    env:
+      NODE_OPTIONS: --max-old-space-size=4096
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          ref: ${{ inputs.ref || github.sha }}
+          persist-credentials: false
+      - uses: ./.github/actions/setup-wsl-test-runtime
+      - uses: ./.github/actions/install-node-dependencies
+        with:
+          native-runtime: electron
+      - name: Build relay and Electron
+        run: |
+          pnpm run build:relay
+          if ($LASTEXITCODE -ne 0) { throw 'Relay build failed' }
+          pnpm exec electron-vite build --mode e2e
+          if ($LASTEXITCODE -ne 0) { throw 'Electron build failed' }
+      - name: Exercise real WSL launch and paste
+        env:
+          SKIP_BUILD: '1'
+          ORCA_E2E_FORWARD_APP_LOGS: '1'
+          PLAYWRIGHT_JSON_OUTPUT_FILE: test-results/wsl-results.json
+        run: >-
+          pnpm exec playwright test
+          tests/e2e/golden-tab-bar-agent-launch.spec.ts
+          tests/e2e/terminal-windows-shell-paste-ownership.spec.ts
+          --config tests/playwright.config.ts
+          --project=electron-headless
+          --grep "WSL"
+          --repeat-each=3
+          --workers=1
+          --reporter=list,json
+      - name: Require all nine WSL executions
+        if: always()
+        run: node config/scripts/verify-wsl-e2e-participation.mjs test-results/wsl-results.json
+      - name: Upload WSL participation report
+        uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: windows-wsl-participation-report
+          path: test-results/wsl-results.json
+          retention-days: 3
+      - uses: actions/upload-artifact@v7
+        if: failure()
+        with:
+          name: windows-wsl-terminal-traces
+          path: test-results/
+          retention-days: 7

.gitignore

@@ -110,6 +110,8 @@ docs/**
 !docs/reference/macos-press-and-hold.md
 !docs/reference/orcad-operations.md
 !docs/reference/relay-grace-time-reconfiguration.md
+!docs/reference/windows-cmd-shim-resolution.md
+!docs/reference/windows-daemon-host-relocation.md
 !docs/reference/windows-edr-posture.md
 !docs/reference/windows-process-enumeration.md
 !docs/reference/wsl-runner-verification.md
@@ -158,6 +160,7 @@ src/renderer/src/i18n/locales/.zh-catalog-cache.json
 src/renderer/src/i18n/locales/.ko-catalog-cache.json
 src/renderer/src/i18n/locales/.ja-catalog-cache.json
 src/renderer/src/i18n/locales/.es-catalog-cache.json
+src/renderer/src/i18n/locales/.fr-catalog-cache.json
 
 # Bench result JSONs are working artifacts
 tests/tools/benchmarks/results/terminal-pipeline-*.json

.oxlintrc.json

@@ -2,6 +2,10 @@
   "$schema": "./node_modules/oxlint/configuration_schema.json",
   "plugins": ["typescript", "react", "react-hooks", "react-perf", "unicorn"],
   "jsPlugins": [
+    {
+      "name": "sort-comparator-performance",
+      "specifier": "./config/oxlint-plugins/sort-comparator-performance.mjs"
+    },
     {
       "name": "mobile-pairing",
       "specifier": "./config/oxlint-plugins/mobile-pairing-qrcode-import.mjs"
@@ -23,6 +27,7 @@
     "correctness": "error"
   },
   "rules": {
+    "sort-comparator-performance/no-repeated-collator": "warn",
     "app-store-performance/require-selector": "error",
     "app-store-performance/no-identity-selector": "error",
     "app-store-performance/no-fresh-selector-result": "error",

AGENTS.md

@@ -4,6 +4,12 @@ All UI work — layout, color, typography, spacing, component selection, UX beha
 
 ## Electron UI Validation
 
+Always run tests and agent-launched apps in the background with `ORCA_BACKGROUND_LAUNCH=1`.
+Never steal monitor focus or reveal test windows: no `show()`, `showInactive()`, `bringToFront()`,
+`app.focus()`, or OS activation. Use CDP screenshots of hidden renderers. Keep native-focus and
+visible-window tests paused on the user's desktop; run them on an isolated display or CI.
+Rebuild modified launch-policy code before running an app; stale build wrappers are not safe.
+
 Use the `$electron` skill and Playwright CDP for rendered Orca UI checks. Do not use computer-use for Orca UI validation.
 
 # Style
@@ -47,8 +53,9 @@ Orca targets macOS, Linux, and Windows. Keep all platform-dependent behavior beh
 - **Shortcut labels in UI**: Display `⌘` / `⇧` on Mac and `Ctrl+` / `Shift+` on other platforms.
 - **File paths**: Use `path.join` or Electron/Node path utilities — never assume `/` or `\`.
 - **Windows setup scripts**: the setup/issue-command runner is a `.cmd` batch file unless the script starts with a `#!` line — never derive that from the user's terminal-shell preference, and never launch a `.cmd` runner with a bare `cmd.exe /c` from a Git Bash pane (MSYS rewrites the `/c`). See [`docs/reference/windows-setup-shell.md`](./docs/reference/windows-setup-shell.md).
-- **Windows child processes**: start them through `runProcess`/`spawnProcess` in `src/shared/child-process/` — never `child_process` directly. It pins `windowsHide`, refuses `shell: true`, and encodes `.cmd`/`.bat` arguments so neither `CommandLineToArgvW` nor `cmd.exe` mangles them. A ratchet test fails on any new direct import.
+- **Windows child processes**: start them through `runProcess`/`spawnProcess` in `src/shared/child-process/` — never `child_process` directly. It pins `windowsHide`, refuses `shell: true`, and encodes `.cmd`/`.bat` arguments so neither `CommandLineToArgvW` nor `cmd.exe` mangles them. A ratchet test fails on any new direct import. Recognised npm/pnpm `.cmd` shims are resolved to their real target so the spawn skips `cmd.exe` entirely; see [`docs/reference/windows-cmd-shim-resolution.md`](./docs/reference/windows-cmd-shim-resolution.md) before adding a shim shape or debugging one.
 - **Windows process enumeration**: read the table through `src/main/windows/windows-process-table.ts`, never by forking `powershell.exe`. See [`docs/reference/windows-process-enumeration.md`](./docs/reference/windows-process-enumeration.md).
+- **Windows daemon-host relocation**: the terminal daemon runs from a copy of the app runtime under `%LOCALAPPDATA%`, which is what survives an auto-update. Before touching that copy, its exe name, or the NSIS uninstall macro, read [`docs/reference/windows-daemon-host-relocation.md`](./docs/reference/windows-daemon-host-relocation.md).
 - **Windows EDR signal**: don't add `-ExecutionPolicy Bypass`, `-EncodedCommand`, `cmd.exe /c` with escaped free text, per-operation interpreter spawning, or runtime `Add-Type` compilation without reading [`docs/reference/windows-edr-posture.md`](./docs/reference/windows-edr-posture.md) first — behavioural EDR scores each of those, and being signed does not clear them.
 - **WSL commands**: build argv with `buildWslExecArgs` (always `--exec` — under `--`, `wsl.exe` expands `$name` in every argument and silently rewrites the script), and fence anything whose stdout you parse with `buildWslCapturedLoginShellCommand`, because the interactive login shell prints the distro banner to stdout. See [`docs/reference/wsl-command-execution.md`](./docs/reference/wsl-command-execution.md).
 - **Linux native modules**: keep the glibc floor at Ubuntu 20.04 / glibc 2.31. A module compiled from source on a newer runner can reference symbol versions absent on the floor and crash the app on startup. See [`docs/reference/linux-glibc-compatibility.md`](./docs/reference/linux-glibc-compatibility.md); packaging fails if a bundled native binary needs newer glibc.

README.md

@@ -36,7 +36,7 @@
 
 Monitor and steer your agents from your phone — get notified when an agent finishes and send follow-ups from anywhere.
 
-[iOS App Store](https://apps.apple.com/us/app/orca-ide/id6766130217) · [TestFlight](https://testflight.apple.com/join/YjeGMQBA) · [Android APK 0.0.47](https://github.com/stablyai/orca/releases/download/mobile-android-v0.0.47/app-release.apk) · [Docs →](https://www.onorca.dev/docs/mobile)
+[iOS App Store](https://apps.apple.com/us/app/orca-ide/id6766130217) · [TestFlight](https://testflight.apple.com/join/YjeGMQBA) · [Android APK 0.0.48](https://github.com/stablyai/orca/releases/download/mobile-android-v0.0.48/app-release.apk) · [Docs →](https://www.onorca.dev/docs/mobile)
 
 </td>
 <td width="50%">
@@ -230,7 +230,7 @@ yay -S stably-orca-bin
 Pair with your desktop app to monitor and steer your agents from your phone.
 
 - **iOS:** [Download on the App Store](https://apps.apple.com/us/app/orca-ide/id6766130217) or [join TestFlight](https://testflight.apple.com/join/YjeGMQBA)
-- **Android:** [Download APK 0.0.47](https://github.com/stablyai/orca/releases/download/mobile-android-v0.0.47/app-release.apk) · [Install guide](https://www.onorca.dev/docs/android-apk)
+- **Android:** [Download APK 0.0.48](https://github.com/stablyai/orca/releases/download/mobile-android-v0.0.48/app-release.apk) · [Install guide](https://www.onorca.dev/docs/android-apk)
 
 ---
 
@@ -238,9 +238,9 @@ Pair with your desktop app to monitor and steer your agents from your phone.
 
 - **Discord:** Join the community on **[Discord](https://discord.gg/fzjDKHxv8Q)**.
 - **Twitter / X:** Follow **[@orca_build](https://x.com/orca_build)** for updates and announcements.
-- **WeChat:** Scan to join the Orca community WeChat group 8.
+- **WeChat:** Scan to join the Orca community WeChat group 8. Group 8 may be full; if so, scan the Group 9 QR code instead.
 
-  <img src="docs/assets/wechat-qr-group8.jpg" alt="WeChat group 8 QR code for the Orca community" width="160" />
+  <img src="docs/assets/wechat-qr-group8.jpg" alt="WeChat group 8 QR code for the Orca community" width="160" />&nbsp;&nbsp;<img src="docs/assets/wechat-qr-group9.jpg" alt="WeChat group 9 QR code for the Orca community" width="160" />
 
 - **Feedback &amp; Ideas:** We ship fast. Missing something? [Request a new feature](https://github.com/stablyai/orca/issues).
 - **Privacy:** See the [privacy &amp; telemetry docs](https://www.onorca.dev/docs/telemetry) for what anonymous usage data Orca collects and how to opt out.

cloud/.gitleaks.toml

@@ -13,3 +13,10 @@ description = "Cloud SQL rollout lease holder keys in the action's unit tests"
 regexTarget = "secret"
 paths = ['''\.github/actions/cloud-sql-rollout-lease/[a-z-]+\.test\.mjs$''']
 regexes = ['''^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+/[0-9]+$''']
+
+# RFC 6455 §1.3 example handshake nonce ("the sample nonce" in base64), sent by the raw-socket
+# upgrade tests; the generic key rule reads any base64 header value as a secret.
+[[allowlists]]
+description = "RFC 6455 example Sec-WebSocket-Key in upgrade tests"
+regexTarget = "secret"
+regexes = ['''^dGhlIHNhbXBsZSBub25jZQ==$''']

cloud/apps/relay-fence-broker/package.json

@@ -14,8 +14,8 @@
     "typecheck": "tsc -p tsconfig.json --noEmit"
   },
   "dependencies": {
-    "@hono/node-server": "^1.19.14",
-    "hono": "^4.12.27",
+    "@hono/node-server": "^1.19.17",
+    "hono": "^4.13.7",
     "zod": "^3.25.76"
   },
   "devDependencies": {

cloud/apps/relay-ops/package.json

@@ -16,8 +16,8 @@
     "typecheck": "tsc -p tsconfig.json --noEmit"
   },
   "dependencies": {
-    "@hono/node-server": "^1.19.14",
-    "hono": "^4.12.27",
+    "@hono/node-server": "^1.19.17",
+    "hono": "^4.13.7",
     "zod": "^3.25.76"
   },
   "devDependencies": {

cloud/apps/relay-ops/src/incident-live-preflight-cli.test.ts

@@ -6,7 +6,10 @@ import {
   livePreflightGcloud,
   runIncidentLivePreflight
 } from './incident-live-preflight-cli.js'
-import type { IncidentSample } from './incident-monitor.js'
+import {
+  INCIDENT_MONITOR_THRESHOLDS,
+  type IncidentSample
+} from './incident-monitor.js'
 import type { AdmissionSelector } from './incident-selector.js'
 
 const directories: string[] = []
@@ -69,6 +72,7 @@ function sample(): IncidentSample {
     expectedSelector: selector,
     cells: [{
       cellId: 'production-gce-c1',
+      region: 'us-central1',
       runtimeKnown: true,
       powered: true,
       expectedAdmissionState: 'existing-only'
@@ -250,6 +254,30 @@ describe('relay incident live preflight', () => {
     )).rejects.toThrow('cloud-monitoring/threshold_max')
   })
 
+  // Why: a frozen wave has to name what froze it without re-reading the sample.
+  it('names the signal and its numbers in the failure message', async () => {
+    const slowCell = sample()
+    slowCell.sources['active-probe']!.signals[
+      'cell.production-gce-c1.latency_ms'
+    ]!.value = 2_568
+    await expect(runIncidentLivePreflight(
+      ['--state-file', stateFile()],
+      { now: () => now, collect: async () => slowCell }
+    )).rejects.toThrow(
+      'relay live preflight failed: active-probe/threshold_max cell.production-gce-c1.latency_ms observed=2568 threshold=2000'
+    )
+
+    // A failure with no signal keeps the source/code token and drops the rest.
+    const stale = sample()
+    stale.sources['active-probe']!.observedAt = new Date(now - 60_001).toISOString()
+    await expect(runIncidentLivePreflight(
+      ['--state-file', stateFile()],
+      { now: () => now, collect: async () => stale }
+    )).rejects.toThrow(
+      'relay live preflight failed: active-probe/source_stale observed=60001 threshold=60000'
+    )
+  })
+
   it('enforces the signed migration policy', async () => {
     const inactiveTarget = sample()
     inactiveTarget.sources['director-admin']!.signals[
@@ -313,7 +341,7 @@ describe('relay incident live preflight', () => {
   it('retries freshness-only failures when explicitly requested', async () => {
     const stale = sample()
     stale.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.observedAt =
-      new Date(now - 180_001).toISOString()
+      new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString()
     const missing = sample()
     delete missing.sources['relay-logs']
     const collect = vi.fn()
@@ -331,11 +359,44 @@ describe('relay incident live preflight', () => {
     expect(wait).toHaveBeenNthCalledWith(2, 15_000)
   })
 
+  it('retries a first-wave stale sample and passes on the fresh one', async () => {
+    const stale = sample()
+    stale.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.observedAt =
+      new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString()
+    const collect = vi.fn().mockResolvedValueOnce(stale).mockResolvedValueOnce(sample())
+    const wait = vi.fn(async () => undefined)
+    await expect(runIncidentLivePreflight(
+      ['--state-file', stateFile(), '--wave-index', '0', '--retry-freshness'],
+      { now: () => now, collect, wait }
+    )).resolves.toBeUndefined()
+    expect(collect).toHaveBeenCalledTimes(2)
+    expect(wait).toHaveBeenCalledOnce()
+  })
+
+  it('stops retrying when the next wait would exceed the evidence-age bound', async () => {
+    const completedAt = now - 290_000
+    const stale = sample()
+    stale.sources['cloud-monitoring']!.observedAt = new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString()
+    const collect = vi.fn(async () => stale)
+    const wait = vi.fn(async () => undefined)
+    await expect(runIncidentLivePreflight(
+      ['--state-file', stateFile('strict', {
+        startedAt: new Date(completedAt - 17 * 60_000).toISOString(),
+        windowStartedAt: new Date(completedAt - 16 * 60_000).toISOString(),
+        lastSampleAt: new Date(completedAt - 30_000).toISOString(),
+        completedAt: new Date(completedAt).toISOString()
+      }), '--retry-freshness'],
+      { now: () => now, collect, wait }
+    )).rejects.toThrow('cloud-monitoring/source_stale')
+    expect(collect).toHaveBeenCalledOnce()
+    expect(wait).not.toHaveBeenCalled()
+  })
+
   it('does not retry a threshold failure', async () => {
     const unhealthy = sample()
     unhealthy.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.value = 0.9
     unhealthy.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.observedAt =
-      new Date(now - 180_001).toISOString()
+      new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString()
     const collect = vi.fn(async () => unhealthy)
     const wait = vi.fn(async () => undefined)
     await expect(runIncidentLivePreflight(
@@ -348,7 +409,7 @@ describe('relay incident live preflight', () => {
 
   it('fails closed after the bounded freshness retry window', async () => {
     const stale = sample()
-    stale.sources['cloud-monitoring']!.observedAt = new Date(now - 180_001).toISOString()
+    stale.sources['cloud-monitoring']!.observedAt = new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString()
     const collect = vi.fn(async () => stale)
     const wait = vi.fn(async () => undefined)
     await expect(runIncidentLivePreflight(

cloud/apps/relay-ops/src/incident-live-preflight-cli.ts

@@ -7,7 +7,9 @@ import { suppliedIdentityToken } from './incident-monitor-cli.js'
 import { AdmissionSelectorSchema, type AdmissionSelector } from './incident-selector.js'
 import {
   evaluateIncidentSample,
+  FRESHNESS_FAILURE_CODES,
   preDrainDryRunPassed,
+  type IncidentFailure,
   type IncidentSample
 } from './incident-monitor.js'
 import { createIncidentSampleCollector } from './incident-monitor-sources.js'
@@ -18,12 +20,6 @@ const MONITOR_EVIDENCE_MAX_AGE_MS = 5 * 60_000
 // Matches the same-cap cell job timeout-minutes; bounds each predecessor wave.
 const WAVE_PREDECESSOR_TIMEOUT_MS = 75 * 60_000
 const WAVE_INDEX_PATTERN = /^[0-3]$/
-const FRESHNESS_FAILURE_CODES = new Set([
-  'signal_missing',
-  'signal_stale',
-  'source_missing',
-  'source_stale'
-])
 
 export function livePreflightGcloud(
   gcloud: ReturnType<typeof createGcloudClient>,
@@ -74,6 +70,17 @@ const PreflightStateSchema = z.object({
   }
 })
 
+// Keep the source/code prefix other tooling matches on, then name the signal and
+// its numbers so a frozen wave is attributable without re-reading the sample.
+function describeFailure(failure: IncidentFailure): string {
+  const detail = [
+    failure.signal,
+    failure.observed === undefined ? null : `observed=${failure.observed}`,
+    failure.threshold === undefined ? null : `threshold=${failure.threshold}`
+  ].filter((part): part is string => part !== null && part !== undefined)
+  return [`${failure.source}/${failure.code}`, ...detail].join(' ')
+}
+
 export async function runIncidentLivePreflight(
   argv: string[],
   dependencies: {
@@ -173,10 +180,14 @@ export async function runIncidentLivePreflight(
     const freshnessOnly = evaluation.failures.every((failure) =>
       FRESHNESS_FAILURE_CODES.has(failure.code)
     )
-    if (!freshnessOnly || attempt === attempts) {
+    // Waiting must never carry the mutation past the same evidence-age bound
+    // the entry check enforces, so the wave budget also caps the retry window.
+    const budgetExhausted =
+      now() + FRESHNESS_RETRY_INTERVAL_MS - completedAt > maxEvidenceAgeMs
+    if (!freshnessOnly || attempt === attempts || budgetExhausted) {
       throw new Error(
         `relay live preflight failed: ${evaluation.failures
-          .map((failure) => `${failure.source}/${failure.code}`)
+          .map(describeFailure)
         .join(',')}`
       )
     }

cloud/apps/relay-ops/src/incident-monitor-cli.test.ts

@@ -44,6 +44,7 @@ function sample(at: number): IncidentSample {
     expectedSelector: selector,
     cells: [{
       cellId,
+      region: 'us-central1',
       runtimeKnown: true,
       powered: true,
       expectedAdmissionState: 'general'

cloud/apps/relay-ops/src/incident-monitor-cli.ts

@@ -50,6 +50,8 @@ const StateSchema = z.object({
   continuityEvents: z.array(z.object({
     recordedAt: z.string(),
     windowSequence: z.number().int().nonnegative(),
+    // Pre-2026-09-05 state files predate tolerated freshness gaps.
+    tolerated: z.boolean().default(false),
     failures: z.array(z.object({
       code: z.string(),
       source: z.enum(['active-probe', 'cloud-monitoring', 'relay-logs', 'director-admin']),

cloud/apps/relay-ops/src/incident-monitor-sources.test.ts

@@ -93,7 +93,7 @@ describe('incident monitor sources', () => {
   })
 
   it('zero-fills an expired sparse lock-wait point', async () => {
-    let pointAt = now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs
+    let pointAt = now - INCIDENT_MONITOR_THRESHOLDS.cloudLockWaitCarryMs
     const fetchImpl: typeof fetch = async () => Response.json({
       timeSeries: [{
         points: [{
@@ -141,7 +141,7 @@ describe('incident monitor sources', () => {
 
   it('freshens a sparse zero without masking a recent nonzero lock wait', async () => {
     let value = 0
-    const pointAt = now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs
+    const pointAt = now - INCIDENT_MONITOR_THRESHOLDS.cloudLockWaitCarryMs
     const readAt = now + 11_879
     const fetchImpl: typeof fetch = async () => Response.json({
       timeSeries: [{
@@ -317,6 +317,49 @@ describe('incident monitor sources', () => {
     }, endAt)).toBeNull()
   })
 
+  // Why: the per-region cell latency bar is only correct if the tfvars region
+  // reaches the evaluator on every cell expectation.
+  it('carries the configured region onto every cell expectation', async () => {
+    const gcloud: GcloudClient = {
+      accessToken: async () => 'unused',
+      identityToken: async () => 'unused'
+    }
+    const selector = {
+      generation: 1,
+      membership: {
+        existingOnly: [],
+        migrationOnly: [],
+        general: productionCells
+      }
+    }
+    const fetchImpl: typeof fetch = async (_input, init) => {
+      const body = JSON.parse(String(init?.body)) as { cellId?: string; sourceCellId?: string }
+      if (!body.cellId && !body.sourceCellId) return Response.json({ selector })
+      if (body.cellId) {
+        return Response.json({
+          status: {
+            enabled: true,
+            connectionCapacity: { hardCap: 600 },
+            runtime: { lastHeartbeatAt: now - 1_000, heartbeatFresh: true }
+          }
+        })
+      }
+      return Response.json({
+        blocked: 0,
+        blockedExpiredUnregistered: 0,
+        registeredTargetInactive: 0
+      })
+    }
+    const result = await directorSignals('production', selector, gcloud, now, fetchImpl)
+    const regionById = new Map(result.cells.map((cell) => [cell.cellId, cell.region]))
+    expect(regionById.get('production-gce-c1')).toBe('us-central1')
+    expect(regionById.get('production-gce-c27')).toBe('asia-east2')
+    expect(result.cells).toHaveLength(productionCells.length)
+    for (const cell of RELAY_OPS_ENVIRONMENTS.production.cells) {
+      expect(regionById.get(cell.cellId)).toBe(cell.region)
+    }
+  })
+
   it('aggregates admin state without returning tokens or response identities', async () => {
     const identityToken = 'secret.header.signature'
     const sensitiveIdentity = 'user@example.test'

cloud/apps/relay-ops/src/incident-monitor-sources.ts

@@ -95,7 +95,7 @@ export const GOOGLE_METRICS: GoogleMetricDefinition[] = [
       'resource.type="cloudsql_database" AND metric.label."wait_event_type"="Lock"',
     aggregation: 'latest-max',
     emptyIsZero: true,
-    zeroAfterMs: INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs
+    zeroAfterMs: INCIDENT_MONITOR_THRESHOLDS.cloudLockWaitCarryMs
   },
   {
     signal: 'cloud_sql.deadlocks',
@@ -558,6 +558,7 @@ export async function directorSignals(
     selector,
     cells: statuses.map(({ cell }) => ({
       cellId: cell.cellId,
+      region: cell.region,
       runtimeKnown: true,
       powered: true,
       expectedAdmissionState: selectorCellState(expectedSelector, cell.cellId)

cloud/apps/relay-ops/src/incident-monitor.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest'
 import {
   evaluateIncidentSample,
   INCIDENT_CHECKPOINT_MINUTES,
+  INCIDENT_FRESHNESS_TOLERANCE_SAMPLES,
   INCIDENT_MONITOR_THRESHOLDS,
   INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS,
   initialIncidentMonitorState,
@@ -32,6 +33,7 @@ function healthySample(at = startedAt): IncidentSample {
     expectedSelector: selector,
     cells: [{
       cellId: 'production-gce-c1',
+      region: 'us-central1',
       runtimeKnown: true,
       powered: true,
       expectedAdmissionState: 'general'
@@ -111,14 +113,87 @@ describe('incident monitor evaluator', () => {
     })
   })
 
-  it('freezes when postgres retries exceed the recalibrated ceiling', () => {
-    const sample = healthySample()
-    sample.sources['relay-logs']!.signals['relay.postgres_retries'] =
-      signal(INCIDENT_MONITOR_THRESHOLDS.relayPostgresRetries + 1)
-    expect(evaluateIncidentSample(sample, startedAt)).toMatchObject({
+  // Why: the global relay_cells lock made retries a steady-state rate (24 h p99
+  // 1320/5min on 2026-09-04); the bar fences only unbounded growth beyond that.
+  it('tolerates the measured healthy retry rate and freezes above the bar', () => {
+    const healthy = healthySample()
+    healthy.sources['relay-logs']!.signals['relay.postgres_retries'] = signal(1504)
+    expect(evaluateIncidentSample(healthy, startedAt).status).toBe('green')
+
+    const incident = healthySample()
+    incident.sources['relay-logs']!.signals['relay.postgres_retries'] = signal(2001)
+    expect(evaluateIncidentSample(incident, startedAt)).toMatchObject({
       status: 'freeze',
       failures: [
-        expect.objectContaining({ signal: 'relay.postgres_retries', threshold: 300 })
+        expect.objectContaining({ signal: 'relay.postgres_retries', threshold: 2000 })
+      ]
+    })
+  })
+
+  // Why: since #18521 the request path fails fast on the cell-inventory lock, so
+  // exhaustion is a steady contention rate (post-#18521 p90 147/5min, max 220),
+  // not an anomaly. The bar bounds it below the 2026-08-23 incident peak of 467.
+  it('tolerates the measured healthy exhaustion rate and freezes above the bar', () => {
+    const healthy = healthySample()
+    healthy.sources['relay-logs']!.signals['relay.postgres_retry_exhausted'] = signal(220)
+    expect(evaluateIncidentSample(healthy, startedAt).status).toBe('green')
+
+    const atLimit = healthySample()
+    atLimit.sources['relay-logs']!.signals['relay.postgres_retry_exhausted'] = signal(300)
+    expect(evaluateIncidentSample(atLimit, startedAt).status).toBe('green')
+
+    const incident = healthySample()
+    incident.sources['relay-logs']!.signals['relay.postgres_retry_exhausted'] = signal(301)
+    expect(evaluateIncidentSample(incident, startedAt)).toMatchObject({
+      status: 'freeze',
+      failures: [
+        expect.objectContaining({ signal: 'relay.postgres_retry_exhausted', threshold: 300 })
+      ]
+    })
+  })
+
+  // Why: an asia-east2 cell's /ready reaches auth and Cloud SQL in us-central1, so
+  // from the US runner it measures p50 0.88 s / max 2.7 s and the flat 2 000 bar
+  // froze three healthy gates on 2026-09-05 (c27 at 2568/2668/2685 ms).
+  it('holds cell endpoint latency to a per-region bar', () => {
+    const asiaTail = healthySample()
+    asiaTail.cells[0]!.region = 'asia-east2'
+    asiaTail.sources['active-probe']!.signals['cell.production-gce-c1.latency_ms'] =
+      signal(2_685)
+    expect(evaluateIncidentSample(asiaTail, startedAt)).toMatchObject({
+      status: 'green',
+      failures: []
+    })
+
+    const asiaIncident = healthySample()
+    asiaIncident.cells[0]!.region = 'asia-east2'
+    asiaIncident.sources['active-probe']!.signals['cell.production-gce-c1.latency_ms'] =
+      signal(4_001)
+    expect(evaluateIncidentSample(asiaIncident, startedAt)).toMatchObject({
+      status: 'freeze',
+      failures: [
+        expect.objectContaining({
+          code: 'threshold_max',
+          source: 'active-probe',
+          signal: 'cell.production-gce-c1.latency_ms',
+          observed: 4_001,
+          threshold: 4_000
+        })
+      ]
+    })
+
+    const usIncident = healthySample()
+    usIncident.sources['active-probe']!.signals['cell.production-gce-c1.latency_ms'] =
+      signal(2_001)
+    expect(evaluateIncidentSample(usIncident, startedAt)).toMatchObject({
+      status: 'freeze',
+      failures: [
+        expect.objectContaining({
+          code: 'threshold_max',
+          signal: 'cell.production-gce-c1.latency_ms',
+          observed: 2_001,
+          threshold: 2_000
+        })
       ]
     })
   })
@@ -155,12 +230,49 @@ describe('incident monitor evaluator', () => {
       code: 'source_missing',
       source: 'relay-logs'
     })
-    const stale = healthySample(startedAt - 180_001)
+    const stale = healthySample(
+      startedAt - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1
+    )
     const failures = evaluateIncidentSample(stale, startedAt).failures
     expect(failures.some((failure) => failure.source === 'cloud-monitoring')).toBe(true)
     expect(failures.some((failure) => failure.source === 'active-probe')).toBe(true)
   })
 
+  // Why: production run 33944873727 at 2026-09-05T04:46:09Z read
+  // cloud_sql.lock_waits 189 286 ms old and restarted a 15-minute window on
+  // Google's publish lag. Cloud SQL documents 60 s sampling plus up to 165 s of
+  // invisibility, so that age is Google's clock, not our fleet.
+  it('reads a 189-second cloud signal as fresh and holds the other sources at 180 s', () => {
+    const lagged = healthySample()
+    lagged.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] =
+      signal(0, startedAt - 189_286)
+    expect(evaluateIncidentSample(lagged, startedAt)).toMatchObject({
+      status: 'green',
+      failures: []
+    })
+    const laggedDirector = healthySample()
+    laggedDirector.sources['director-admin']!.observedAt =
+      new Date(startedAt - 189_286).toISOString()
+    expect(evaluateIncidentSample(laggedDirector, startedAt).failures).toContainEqual(
+      expect.objectContaining({ code: 'source_stale', source: 'director-admin' })
+    )
+  })
+
+  it('still fails a cloud signal past the documented publish lag', () => {
+    const dark = healthySample()
+    dark.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] = signal(
+      0,
+      startedAt - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1
+    )
+    expect(evaluateIncidentSample(dark, startedAt).failures).toContainEqual(
+      expect.objectContaining({
+        code: 'signal_stale',
+        source: 'cloud-monitoring',
+        signal: 'cloud_sql.lock_waits'
+      })
+    )
+  })
+
   it('freezes on SQL, director, relay pool, heartbeat, and migration breaches', () => {
     const sample = healthySample()
     sample.sources['cloud-monitoring']!.signals['cloud_sql.cpu'] = signal(0.81)
@@ -336,12 +448,14 @@ describe('incident monitor evaluator', () => {
     ] = signal(0)
     sample.cells.push({
       cellId: 'production-gce-c2',
+      region: 'us-central1',
       runtimeKnown: true,
       powered: true,
       expectedAdmissionState: 'general'
     })
     sample.cells.push({
       cellId: 'production-gce-c3',
+      region: 'us-central1',
       runtimeKnown: true,
       powered: true,
       expectedAdmissionState: 'general'
@@ -565,7 +679,7 @@ describe('incident monitor lifecycle', () => {
     'restarts a %i-minute continuous window after stale telemetry',
     async (durationMinutes) => {
       let now = startedAt
-      let staleInjected = false
+      let staleSamples = INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1
       const checkpoints: Array<[number, number]> = []
       const state = initialIncidentMonitorState({
         incidentId: 'incident-1',
@@ -585,9 +699,11 @@ describe('incident monitor lifecycle', () => {
           now += ms
         },
         collect: async () => {
-          if (!staleInjected && now === startedAt + 5 * 60_000) {
-            staleInjected = true
-            return healthySample(now - 180_001)
+          if (staleSamples > 0 && now >= startedAt + 5 * 60_000) {
+            staleSamples--
+            return healthySample(
+              now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1
+            )
           }
           return healthySample(now)
         },
@@ -596,16 +712,20 @@ describe('incident monitor lifecycle', () => {
           checkpoints.push([summary.windowSequence, summary.checkpointMinute])
         }
       })
+      const restartMinute = 5 + INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1
       expect(result.windowSequence).toBe(1)
       expect(result.windowStartedAt).toBe(
-        new Date(startedAt + 6 * 60_000).toISOString()
+        new Date(startedAt + restartMinute * 60_000).toISOString()
       )
       expect(result.completedAt).toBe(
-        new Date(startedAt + (durationMinutes + 6) * 60_000).toISOString()
+        new Date(startedAt + (durationMinutes + restartMinute) * 60_000).toISOString()
       )
       expect(result.sampleCount).toBe(durationMinutes + 1)
-      expect(result.continuityEvents).toHaveLength(1)
-      expect(result.continuityEvents[0]!.failures).toEqual(
+      expect(result.continuityEvents.map((event) => event.tolerated)).toEqual([
+        ...Array<boolean>(INCIDENT_FRESHNESS_TOLERANCE_SAMPLES).fill(true),
+        false
+      ])
+      expect(result.continuityEvents.at(-1)!.failures).toEqual(
         expect.arrayContaining([
           expect.objectContaining({ code: 'source_stale' })
         ])
@@ -615,6 +735,188 @@ describe('incident monitor lifecycle', () => {
     }
   )
 
+  // Why: run 33944873727 on 2026-09-05 restarted at 04:46:09Z on a single
+  // 189-second cloud reading and then blew the 25-minute lineage cap, so a
+  // green fleet produced no verdict at all. One unread sample now continues the
+  // window; the sample is still checked against every threshold it can read.
+  it('carries a 15-minute window through a single stale cloud sample', async () => {
+    let now = startedAt
+    const state = initialIncidentMonitorState({
+      incidentId: 'incident-1',
+      environment: 'production',
+      expectedSelector: selector,
+      preDrainDryRun: true,
+      migrationPolicy: 'strict',
+      recoverySourceCellId: null,
+      capacityCellId: null,
+      startedAt: new Date(startedAt).toISOString(),
+      durationMinutes: 15,
+      intervalMs: 60_000
+    })
+    const result = await runIncidentMonitor(state, {
+      now: () => now,
+      wait: async (ms) => {
+        now += ms
+      },
+      collect: async () => {
+        const sample = healthySample(now)
+        if (now === startedAt + 10 * 60_000) {
+          sample.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] =
+            signal(0, now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1)
+        }
+        return sample
+      },
+      persist: async () => {},
+      checkpoint: async () => {}
+    })
+    expect(result.windowSequence).toBe(0)
+    expect(result.windowStartedAt).toBe(new Date(startedAt).toISOString())
+    expect(result.completedAt).toBe(new Date(startedAt + 15 * 60_000).toISOString())
+    expect(result.sampleCount).toBe(16)
+    expect(result.frozenAt).toBeNull()
+    expect(result.continuityEvents).toEqual([{
+      recordedAt: new Date(startedAt + 10 * 60_000).toISOString(),
+      windowSequence: 0,
+      tolerated: true,
+      failures: [expect.objectContaining({
+        code: 'signal_stale',
+        source: 'cloud-monitoring',
+        signal: 'cloud_sql.lock_waits'
+      })]
+    }])
+    expect(preDrainDryRunPassed(result)).toBe(true)
+  })
+
+  it('gives a signal a fresh budget only after it reads fresh again', async () => {
+    let now = startedAt
+    const staleMinutes = new Set([3, 5, 6, 9, 10])
+    const state = initialIncidentMonitorState({
+      incidentId: 'incident-1',
+      environment: 'production',
+      expectedSelector: selector,
+      preDrainDryRun: true,
+      migrationPolicy: 'strict',
+      recoverySourceCellId: null,
+      capacityCellId: null,
+      startedAt: new Date(startedAt).toISOString(),
+      durationMinutes: 15,
+      intervalMs: 60_000
+    })
+    const result = await runIncidentMonitor(state, {
+      now: () => now,
+      wait: async (ms) => {
+        now += ms
+      },
+      collect: async () => {
+        const sample = healthySample(now)
+        if (staleMinutes.has((now - startedAt) / 60_000)) {
+          sample.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] =
+            signal(0, now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1)
+        }
+        return sample
+      },
+      persist: async () => {},
+      checkpoint: async () => {}
+    })
+    expect(result.windowSequence).toBe(0)
+    expect(result.continuityEvents).toHaveLength(staleMinutes.size)
+    expect(result.continuityEvents.every((event) => event.tolerated)).toBe(true)
+    expect(preDrainDryRunPassed(result)).toBe(true)
+  })
+
+  it('does not hand a resumed monitor a fresh tolerance budget', async () => {
+    let now = startedAt + 3 * 60_000
+    const resumed = {
+      ...initialIncidentMonitorState({
+        incidentId: 'incident-1',
+        environment: 'production',
+        expectedSelector: selector,
+        preDrainDryRun: true,
+        migrationPolicy: 'strict',
+        recoverySourceCellId: null,
+        capacityCellId: null,
+        startedAt: new Date(startedAt).toISOString(),
+        durationMinutes: 15,
+        intervalMs: 60_000
+      }),
+      windowStartedAt: new Date(startedAt).toISOString(),
+      lastSampleAt: new Date(startedAt + 2 * 60_000).toISOString(),
+      sampleCount: 3,
+      totalSampleCount: 3,
+      continuityEvents: Array.from(
+        { length: INCIDENT_FRESHNESS_TOLERANCE_SAMPLES },
+        (_, index) => ({
+          recordedAt: new Date(startedAt + (index + 1) * 60_000).toISOString(),
+          windowSequence: 0,
+          tolerated: true,
+          failures: [{
+            code: 'signal_stale',
+            source: 'cloud-monitoring' as const,
+            signal: 'cloud_sql.lock_waits'
+          }]
+        })
+      )
+    }
+    const stop = new Error('stop after the resumed sample')
+    await expect(runIncidentMonitor(resumed, {
+      now: () => now,
+      wait: async () => {
+        throw stop
+      },
+      collect: async () => {
+        const sample = healthySample(now)
+        sample.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] =
+          signal(0, now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1)
+        return sample
+      },
+      persist: async (state) => {
+        expect(state.windowSequence).toBe(1)
+        expect(state.windowStartedAt).toBeNull()
+        expect(state.continuityEvents.at(-1)!.tolerated).toBe(false)
+      },
+      checkpoint: async () => {}
+    })).rejects.toThrow(stop)
+  })
+
+  it('freezes on a threshold breach that arrives with a tolerated stale signal', async () => {
+    let now = startedAt
+    const state = initialIncidentMonitorState({
+      incidentId: 'incident-1',
+      environment: 'production',
+      expectedSelector: selector,
+      preDrainDryRun: true,
+      migrationPolicy: 'strict',
+      recoverySourceCellId: null,
+      capacityCellId: null,
+      startedAt: new Date(startedAt).toISOString(),
+      durationMinutes: 15,
+      intervalMs: 60_000
+    })
+    const result = await runIncidentMonitor(state, {
+      now: () => now,
+      wait: async (ms) => {
+        now += ms
+      },
+      collect: async () => {
+        const sample = healthySample(now)
+        if (now === startedAt + 2 * 60_000) {
+          sample.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] =
+            signal(0, now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1)
+          sample.sources['cloud-monitoring']!.signals['cloud_sql.cpu'] = signal(0.81, now)
+        }
+        return sample
+      },
+      persist: async () => {},
+      checkpoint: async () => {}
+    })
+    expect(result.frozenAt).toBe(new Date(startedAt + 2 * 60_000).toISOString())
+    expect(result.failures).toContainEqual(expect.objectContaining({
+      code: 'threshold_max',
+      signal: 'cloud_sql.cpu'
+    }))
+    expect(preDrainDryRunPassed(result)).toBe(false)
+  })
+
   it('resets at the next fresh sample after a runner gap', async () => {
     let now = startedAt + 10 * 60_000
     const state = {
@@ -663,13 +965,21 @@ describe('incident monitor lifecycle', () => {
       durationMinutes: 15,
       intervalMs: 60_000
     })
+    let staleSamples = INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1
     const result = await runIncidentMonitor(state, {
       now: () => now,
       wait: async (ms) => {
         now += ms
       },
-      collect: async () =>
-        healthySample(now === startedAt + 10 * 60_000 ? now - 180_001 : now),
+      collect: async () => {
+        if (staleSamples > 0 && now >= startedAt + 10 * 60_000) {
+          staleSamples--
+          return healthySample(
+            now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1
+          )
+        }
+        return healthySample(now)
+      },
       persist: async () => {},
       checkpoint: async () => {}
     })
@@ -679,7 +989,7 @@ describe('incident monitor lifecycle', () => {
     )
     expect(result.frozenAt).not.toBeNull()
     expect(result.windowSequence).toBe(1)
-    expect(result.sampleCount).toBe(15)
+    expect(result.sampleCount).toBe(13)
     expect(result.failures).toContainEqual({
       code: 'continuity_deadline_exceeded',
       source: 'active-probe',

cloud/apps/relay-ops/src/incident-monitor.ts

@@ -1,3 +1,4 @@
+import type { RelayOpsRegion } from './environment-config.js'
 import {
   exactAdmissionSelector,
   type AdmissionSelector,
@@ -6,17 +7,46 @@ import {
 
 export const INCIDENT_MONITOR_THRESHOLDS = {
   activeProbeMaxAgeMs: 60_000,
-  cloudDataMaxAgeMs: 180_000,
+  // Why: Cloud Monitoring publishes on Google's clock, not ours. Per the metric
+  // list read 2026-09-05, Cloud Run instance_count / cpu / memory /
+  // max_request_concurrencies / request_count are "Sampled every 60 seconds.
+  // After sampling, data is not visible for up to 120 seconds" (60+120=180 s),
+  // and Cloud SQL cpu / memory / num_backends / backends_in_wait /
+  // deadlock_count say "up to 165 seconds" (60+165=225 s). Window-sum signals
+  // age differently: observedAt is the newest point in the 5-minute query
+  // window, so a label series that stops emitting reads as 300 s old while its
+  // summed value is still complete. 330 s clears the worst of the three (the
+  // 300 s query window) plus ~30 s of collect-to-evaluate latency. The old
+  // 180 s bar restarted healthy 15-minute windows at 181 s, 189 s and 255 s on
+  // 2026-09-04/05, once burning the whole 25-minute lineage with no verdict.
+  cloudDataMaxAgeMs: 330_000,
+  // Why: the director admin API answers live on our own request, so hold its
+  // freshness bar where it sat while it shared cloudDataMaxAgeMs.
+  directorAdminMaxAgeMs: 180_000,
+  // Why: how long a nonzero backends-in-wait point is carried before it reads as
+  // zero. Held at the pre-2026-09-05 cloud bar: carrying it for the full
+  // cloudDataMaxAgeMs would hand the evaluator a point older than its own
+  // freshness bar as soon as collection latency is added.
+  cloudLockWaitCarryMs: 180_000,
   relayLogMaxAgeMs: 180_000,
   heartbeatMaxAgeMs: 45_000,
   endpointLatencyMs: 2_000,
+  // Why: a cell's /ready fetches the auth JWKS and runs SELECT 1 against Cloud SQL,
+  // both in us-central1, so from the US runner asia-east2 cells measure p50 0.88 s /
+  // max 2.7 s against 0.08-0.5 s for us-central1. The flat 2 000 bar froze three
+  // healthy 15-minute gates on 2026-09-05 (c27 at 2568/2668/2685 ms); hard faults
+  // are still caught by the .health/.ready equal-1 checks and the 8 s fetch timeout.
+  cellEndpointLatencyMs: {
+    'us-central1': 2_000,
+    'asia-east2': 4_000
+  } as const satisfies Record<RelayOpsRegion, number>,
   cloudSqlCpuUtilization: 0.8,
   cloudSqlMemoryUtilization: 0.9,
   // Why: healthy latest-sum backends idle near 100 but spike to 216 in 1-minute
   // bursts (~10 min/day exceeded the old bar of 160 on 2026-08-26, freezing a
   // pre-drain gate on baseline noise). 250 clears measured healthy peaks while
-  // firing well before the verified 400-connection ceiling; pool-wait and
-  // exhausted-retry signals keep their strict thresholds.
+  // firing well before the verified 400-connection ceiling; the retry signals
+  // below discriminate incident-class contention.
   cloudSqlBackends: 250,
   // Bound the observed recovery load; deadlocks remain zero-tolerance.
   cloudSqlLockWaits: 20,
@@ -32,13 +62,35 @@ export const INCIDENT_MONITOR_THRESHOLDS = {
   relayPoolWaiting: 800,
   relayPoolWaitMs: 2_500,
   // Why: successful lock retries are the contention machinery working, not harm.
-  // Healthy 2026-08-26 baseline bursts to 234/5min (26% of windows crossed the old
-  // bar of 20, set unmeasured at the monitor's 2026-07-28 birth); the 2026-08-23
-  // incident ran ~2,200-3,000/5min. 300 clears healthy bursts with ~10x incident
-  // margin; relayPostgresRetryExhausted below stays at zero tolerance, so any
-  // transaction that terminally fails still freezes the gate.
-  relayPostgresRetries: 300,
-  relayPostgresRetryExhausted: 0,
+  // Recalibrated 2026-09-04 from 300, which was set 2026-08-26 when healthy bursts
+  // reached 234/5min. The global relay_cells FOR UPDATE lock has since become the
+  // fleet's steady state: measured fleet-wide (director + cells, summed per five
+  // minutes) 2026-09-03T05Z..2026-09-04T05Z p50 430 / p90 924 / p99 1320 / max
+  // 1504, with 55% of windows over 300 and only 22% of 15-minute gates clean, so
+  // the bar blocked the very cell roll that carries the 500 ms lock wait (#18521)
+  // and the beginProof crash guard to the cells. The 2026-08-23 lock incident on
+  // this same metric peaked at 1510 in one window and 646 in the next, so it is
+  // not separable from today's contention by retries alone; it is caught by
+  // relayPostgresRetryExhausted (467 at the peak vs a 300 bar), director
+  // concurrency, and the pool bars. 2000 passes every healthy 15-minute window
+  // measured in the last 24 h and still fences unbounded growth. Re-tighten once
+  // the fleet is on the 500 ms lock wait and the baseline is re-measured.
+  relayPostgresRetries: 2000,
+  // Why: 300 per five minutes, recalibrated 2026-09-04 from a bar of zero that no
+  // production window has cleared since #18521 shipped to the director. That
+  // change cut the request-path cell-inventory wait from the 1 s pool lock_timeout
+  // to 500 ms, so a contended waiter now fails fast (one /v1/assign 503 with
+  // Retry-After, which the client retries) instead of succeeding slowly, and the
+  // exhaustion count became a steady-state contention rate rather than an
+  // anomaly. Measured fleet-wide (director + cells) per five minutes over
+  // 2026-09-03T03Z..2026-09-04T02Z: every one of 236 windows was non-zero;
+  // quiet hours p50 2 / max 36; pre-#18521 daytime p50 10 / p90 25 / max 87;
+  // post-#18521 p50 42 / p90 147 / max 220. The 2026-08-23 lock incident peaked
+  // at 467. 300 clears every measured healthy window and still sits below the
+  // incident shape; retries above fence only unbounded growth.
+  // User-facing /v1/assign 503 share did not move with #18521 (13.9% old image
+  // vs 12.3% new, same evening), so exhaustion is not a proxy for user harm.
+  relayPostgresRetryExhausted: 300,
   // Why: public admission is a per-instance semaphore, so fleet assignment capacity is
   // concurrency x instances. A floor of 1 let the 2026-08-04 collapse from five instances
   // to two pass unnoticed, which is the exact failure this monitor exists to catch. Keep in
@@ -84,6 +136,7 @@ export type IncidentSource = {
 
 export type IncidentCellExpectation = {
   cellId: string
+  region: RelayOpsRegion
   runtimeKnown: boolean
   powered: boolean
   expectedAdmissionState: AdmissionState
@@ -153,6 +206,7 @@ export type IncidentMonitorState = {
   continuityEvents: {
     recordedAt: string
     windowSequence: number
+    tolerated: boolean
     failures: IncidentFailure[]
   }[]
   frozenAt: string | null
@@ -285,7 +339,7 @@ const SOURCE_MAX_AGE: Record<IncidentSourceName, number> = {
   'active-probe': INCIDENT_MONITOR_THRESHOLDS.activeProbeMaxAgeMs,
   'cloud-monitoring': INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs,
   'relay-logs': INCIDENT_MONITOR_THRESHOLDS.relayLogMaxAgeMs,
-  'director-admin': INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs
+  'director-admin': INCIDENT_MONITOR_THRESHOLDS.directorAdminMaxAgeMs
 }
 
 function ageMs(timestamp: string, nowMs: number): number {
@@ -362,7 +416,7 @@ function checkCell(
       'active-probe',
       probe,
       `cell.${cell.cellId}.latency_ms`,
-      INCIDENT_MONITOR_THRESHOLDS.endpointLatencyMs,
+      INCIDENT_MONITOR_THRESHOLDS.cellEndpointLatencyMs[cell.region],
       'max'
     ],
     [
@@ -586,14 +640,59 @@ function checkpointMinutes(durationMinutes: number): number[] {
   return INCIDENT_CHECKPOINT_MINUTES.filter((minute) => minute <= durationMinutes)
 }
 
-const CONTINUITY_FAILURE_CODES = new Set([
-  'collector_failed',
-  'monitor_gap',
+// Freshness-only failures: we could not read a signal this sample. Distinct from
+// collector_failed / monitor_gap, where the whole sample is absent.
+export const FRESHNESS_FAILURE_CODES = new Set([
+  'signal_missing',
   'signal_stale',
   'source_missing',
   'source_stale'
 ])
 
+const CONTINUITY_FAILURE_CODES = new Set([
+  'collector_failed',
+  'monitor_gap',
+  ...FRESHNESS_FAILURE_CODES
+])
+
+// Why: Cloud Monitoring overshoots its own publish bar, and one unread sample is
+// not evidence of an unhealthy fleet. Under the 25-minute lineage cap a restart
+// past minute 10 costs the entire verdict, so a healthy fleet produced none on
+// 2026-09-05. A signal may miss this many consecutive samples before the window
+// restarts; the sample is still evaluated against every threshold it can read,
+// and a threshold breach still freezes the run outright.
+export const INCIDENT_FRESHNESS_TOLERANCE_SAMPLES = 2
+
+function freshnessKey(failure: IncidentFailure): string {
+  return `${failure.source}/${failure.signal ?? '*'}`
+}
+
+// Rebuild the per-signal tolerated streak from the trailing continuity events so a
+// resumed monitor cannot hand a signal a fresh budget.
+function resumeFreshnessStreaks(
+  state: IncidentMonitorState
+): Map<string, number> {
+  const events = state.continuityEvents
+  const streaks = new Map<string, number>()
+  const last = events[events.length - 1]
+  if (!last?.tolerated) return streaks
+  for (const key of new Set(last.failures.map(freshnessKey))) {
+    let streak = 0
+    let laterAt: number | null = null
+    for (let index = events.length - 1; index >= 0; index--) {
+      const event = events[index]!
+      const recordedAt = Date.parse(event.recordedAt)
+      if (!event.tolerated) break
+      if (laterAt !== null && laterAt - recordedAt > state.intervalMs * 1.5) break
+      if (!event.failures.some((failure) => freshnessKey(failure) === key)) break
+      streak++
+      laterAt = recordedAt
+    }
+    streaks.set(key, streak)
+  }
+  return streaks
+}
+
 function resetContinuousWindow(
   state: IncidentMonitorState,
   recordedAt: string,
@@ -609,6 +708,7 @@ function resetContinuousWindow(
   state.continuityEvents.push({
     recordedAt,
     windowSequence: state.windowSequence,
+    tolerated: false,
     failures
   })
 }
@@ -659,6 +759,7 @@ export async function runIncidentMonitor(
     await dependencies.persist(state)
     return state
   }
+  const freshnessStreaks = resumeFreshnessStreaks(state)
   while (state.completedAt === null) {
     if (dependencies.now() > lineageDeadlineMs) {
       completeContinuityDeadline(state, dependencies.now(), lineageStartMs)
@@ -693,9 +794,34 @@ export async function runIncidentMonitor(
     const thresholdFailures = evaluation.failures.filter((failure) =>
       !CONTINUITY_FAILURE_CODES.has(failure.code)
     )
-    if (continuityFailures.length > 0) {
+    const toleratedKeys = new Set(
+      state.windowStartedAt !== null &&
+        continuityFailures.length > 0 &&
+        continuityFailures.every((failure) => FRESHNESS_FAILURE_CODES.has(failure.code))
+        ? continuityFailures.map(freshnessKey)
+        : []
+    )
+    for (const key of [...freshnessStreaks.keys()]) {
+      if (!toleratedKeys.has(key)) freshnessStreaks.delete(key)
+    }
+    let tolerated = toleratedKeys.size > 0
+    for (const key of toleratedKeys) {
+      const streak = (freshnessStreaks.get(key) ?? 0) + 1
+      freshnessStreaks.set(key, streak)
+      if (streak > INCIDENT_FRESHNESS_TOLERANCE_SAMPLES) tolerated = false
+    }
+    if (continuityFailures.length > 0 && !tolerated) {
+      freshnessStreaks.clear()
       resetContinuousWindow(state, evaluation.evaluatedAt, continuityFailures)
     } else {
+      if (tolerated) {
+        state.continuityEvents.push({
+          recordedAt: evaluation.evaluatedAt,
+          windowSequence: state.windowSequence,
+          tolerated: true,
+          failures: continuityFailures
+        })
+      }
       if (state.windowStartedAt === null) {
         state.windowStartedAt = evaluation.evaluatedAt
       }

cloud/apps/relay-ops/src/resource-inventory.test.ts

@@ -13,6 +13,41 @@ const runService = {
   latestReadyRevision: 'projects/project/revisions/revision-one'
 }
 
+const sleepingStagingGcloud: GcloudClient = { accessToken: async () => 'a'.repeat(40) }
+
+// Staging's Cloud SQL is stopped, so this inventory reads REST only and probes no endpoint.
+type MigOutcome = 'ok' | 'throw' | 'missing'
+const sleepingStagingFetch = (migOutcome: (migName: string) => MigOutcome): typeof fetch =>
+  async (input) => {
+    const url = new URL(String(input))
+    if (url.hostname === 'run.googleapis.com') return Response.json(runService)
+    if (url.hostname === 'sqladmin.googleapis.com') return Response.json({
+      state: 'STOPPED',
+      databaseVersion: 'POSTGRES_17',
+      settings: { activationPolicy: 'NEVER', availabilityType: 'ZONAL', tier: 'db-custom-1-3840' }
+    })
+    if (url.hostname === 'certificatemanager.googleapis.com') return Response.json({
+      managed: { domains: ['*.relay-staging.onorca.dev'], state: 'ACTIVE' }
+    })
+    if (url.pathname.includes('/instanceGroupManagers/')) {
+      const name = url.pathname.split('/').at(-1)!
+      const outcome = migOutcome(name)
+      if (outcome === 'throw') throw new TypeError('fetch failed')
+      if (outcome === 'missing') return new Response(null, { status: 404 })
+      return Response.json({
+        name,
+        targetSize: 0,
+        size: '0',
+        instanceGroup: `projects/project/zones/zone/instanceGroups/${name}`,
+        instanceTemplate: `projects/project/global/instanceTemplates/template-${name}`,
+        status: { isStable: true }
+      })
+    }
+    if (url.pathname.includes('/instanceTemplates/')) return Response.json({ properties: {} })
+    if (url.pathname.endsWith('/getHealth')) return Response.json([])
+    throw new Error(`Unexpected request to ${url.hostname}${url.pathname}`)
+  }
+
 describe('readResourceInventory', () => {
   it('does not delay a healthy endpoint sample', async () => {
     let calls = 0
@@ -23,8 +58,10 @@ describe('readResourceInventory', () => {
         calls += 1
         return new Response(null, { status: 200 })
       },
-      async () => {
-        waits += 1
+      {
+        wait: async () => {
+          waits += 1
+        }
       }
     )
 
@@ -45,8 +82,10 @@ describe('readResourceInventory', () => {
         calls.set(path, call)
         return new Response(null, { status: path === '/ready' && call === 1 ? 503 : 200 })
       },
-      async (ms) => {
-        waits.push(ms)
+      {
+        wait: async (ms) => {
+          waits.push(ms)
+        }
       }
     )
 
@@ -65,17 +104,130 @@ describe('readResourceInventory', () => {
         calls += 1
         return new Response(null, { status: 503 })
       },
-      async (ms) => {
-        waits.push(ms)
+      {
+        wait: async (ms) => {
+          waits.push(ms)
+        }
       }
     )
 
     expect(result.health).toBe(false)
     expect(result.ready).toBe(false)
     expect(calls).toBe(4)
+    // A refusing endpoint is a reading, so only the independent retry runs.
     expect(waits).toEqual([11_000])
   })
 
+  it('treats a thrown fetch as no reading and re-asks that path once', async () => {
+    const calls: string[] = []
+    const waits: number[] = []
+    const result = await probeEndpointHealth(
+      'https://c9.relay.onorca.dev',
+      async (input) => {
+        const path = new URL(String(input)).pathname
+        calls.push(path)
+        if (path === '/health' && calls.filter((call) => call === '/health').length === 1) {
+          throw new TypeError('fetch failed')
+        }
+        return new Response(null, { status: 200 })
+      },
+      {
+        wait: async (ms) => {
+          waits.push(ms)
+        }
+      }
+    )
+
+    expect(result.health).toBe(true)
+    expect(result.ready).toBe(true)
+    expect(calls.filter((call) => call === '/health')).toEqual(['/health', '/health'])
+    expect(waits).toEqual([1_000])
+  })
+
+  it('fails closed when both attempts of a path throw', async () => {
+    const calls: string[] = []
+    const waits: number[] = []
+    const result = await probeEndpointHealth(
+      'https://c9.relay.onorca.dev',
+      async (input) => {
+        const path = new URL(String(input)).pathname
+        calls.push(path)
+        if (path === '/health') throw new TypeError('fetch failed')
+        return new Response(null, { status: 200 })
+      },
+      {
+        wait: async (ms) => {
+          waits.push(ms)
+        }
+      }
+    )
+
+    expect(result.health).toBe(false)
+    expect(calls.filter((call) => call === '/health')).toHaveLength(4)
+    expect(waits).toEqual([1_000, 11_000, 1_000])
+  })
+
+  it('accepts an auth-shaped endpoint that serves no readiness path', async () => {
+    const calls: string[] = []
+    const waits: number[] = []
+    const result = await probeEndpointHealth(
+      'https://login.onorca.dev',
+      async (input) => {
+        const path = new URL(String(input)).pathname
+        calls.push(path)
+        return new Response(null, { status: path === '/ready' ? 404 : 200 })
+      },
+      {
+        requiresReady: false,
+        wait: async (ms) => {
+          waits.push(ms)
+        }
+      }
+    )
+
+    expect(result.health).toBe(true)
+    expect(result.ready).toBeNull()
+    expect(calls).toEqual(['/health'])
+    expect(waits).toEqual([])
+  })
+
+  it('still requires readiness for the director and cells', async () => {
+    const waits: number[] = []
+    const result = await probeEndpointHealth(
+      'https://relay.onorca.dev',
+      async (input) => new Response(null, {
+        status: new URL(String(input)).pathname === '/ready' ? 503 : 200
+      }),
+      {
+        wait: async (ms) => {
+          waits.push(ms)
+        }
+      }
+    )
+
+    expect(result.health).toBe(true)
+    expect(result.ready).toBe(false)
+    expect(waits).toEqual([11_000])
+  })
+
+  it('measures latency as the answering round trip, not the retry delay', async () => {
+    let healthCalls = 0
+    const result = await probeEndpointHealth(
+      'https://c9.relay.onorca.dev',
+      async (input) => {
+        if (new URL(String(input)).pathname !== '/health') return new Response(null, { status: 200 })
+        healthCalls += 1
+        if (healthCalls === 1) throw new TypeError('fetch failed')
+        return new Response(null, { status: 200 })
+      },
+      { wait: async (ms) => await new Promise((resolve) => setTimeout(resolve, Math.min(ms, 60))) }
+    )
+
+    expect(result.health).toBe(true)
+    expect(result.latencyMs).not.toBeNull()
+    expect(result.latencyMs!).toBeLessThan(60)
+  })
+
   it('uses aggregate REST inventory without probing sleeping staging endpoints', async () => {
     const gcloud: GcloudClient = { accessToken: async () => 'a'.repeat(40) }
     let publicProbeCalls = 0
@@ -132,6 +284,74 @@ describe('readResourceInventory', () => {
     expect(JSON.stringify(result)).not.toContain('SECRET_TEXT')
   })
 
+  it('re-asks a MIG read that failed once before calling a cell powered-unknown', async () => {
+    const parkedCell = RELAY_OPS_ENVIRONMENTS.staging.cells[0]!
+    const waits: number[] = []
+    let parkedMigCalls = 0
+    const result = await readResourceInventory(
+      RELAY_OPS_ENVIRONMENTS.staging,
+      sleepingStagingGcloud,
+      sleepingStagingFetch((migName) => {
+        if (!migName.endsWith(parkedCell.hostname)) return 'ok'
+        parkedMigCalls += 1
+        return parkedMigCalls === 1 ? 'throw' : 'ok'
+      }),
+      { wait: async (ms) => { waits.push(ms) } }
+    )
+
+    const parked = result.cells.find((cell) => cell.cellId === parkedCell.cellId)!
+    // The MIG was fine and parked at zero; one transient read must not erase that reading.
+    expect(parked.targetSize).toBe(0)
+    expect(parkedMigCalls).toBe(2)
+    expect(waits).toEqual([1_000])
+    expect(result.warnings).toEqual([])
+  })
+
+  it('reports a MIG unavailable only when the retry fails too', async () => {
+    const parkedCell = RELAY_OPS_ENVIRONMENTS.staging.cells[0]!
+    const waits: number[] = []
+    let parkedMigCalls = 0
+    const result = await readResourceInventory(
+      RELAY_OPS_ENVIRONMENTS.staging,
+      sleepingStagingGcloud,
+      sleepingStagingFetch((migName) => {
+        if (!migName.endsWith(parkedCell.hostname)) return 'ok'
+        parkedMigCalls += 1
+        return 'throw'
+      }),
+      { wait: async (ms) => { waits.push(ms) } }
+    )
+
+    const parked = result.cells.find((cell) => cell.cellId === parkedCell.cellId)!
+    expect(parked.targetSize).toBeNull()
+    expect(parked.backendHealth).toBe('unknown')
+    expect(parkedMigCalls).toBe(2)
+    expect(waits).toEqual([1_000])
+    expect(result.warnings).toEqual([
+      `${parkedCell.hostname.toUpperCase()} MIG inventory is unavailable.`
+    ])
+  })
+
+  it('does not re-ask a MIG read the API answered with 404', async () => {
+    const missingCell = RELAY_OPS_ENVIRONMENTS.staging.cells[0]!
+    const waits: number[] = []
+    let missingMigCalls = 0
+    const result = await readResourceInventory(
+      RELAY_OPS_ENVIRONMENTS.staging,
+      sleepingStagingGcloud,
+      sleepingStagingFetch((migName) => {
+        if (!migName.endsWith(missingCell.hostname)) return 'ok'
+        missingMigCalls += 1
+        return 'missing'
+      }),
+      { wait: async (ms) => { waits.push(ms) } }
+    )
+
+    expect(result.cells.find((cell) => cell.cellId === missingCell.cellId)!.targetSize).toBeNull()
+    expect(missingMigCalls).toBe(1)
+    expect(waits).toEqual([])
+  })
+
   it('represents missing credentials as unknown inventory, never sleeping', async () => {
     const gcloud: GcloudClient = {
       accessToken: async () => { throw new Error('sensitive context') }

cloud/apps/relay-ops/src/resource-inventory.ts

@@ -102,6 +102,9 @@ export type ResourceInventory = {
 
 const unavailableEndpoint = (): EndpointHealth => ({ health: null, ready: null, latencyMs: null })
 const independentEndpointRetryDelayMs = 11_000
+const transientProbeRetryDelayMs = 1_000
+const sleep = async (ms: number): Promise<void> =>
+  await new Promise((resolvePromise) => setTimeout(resolvePromise, ms))
 
 function finalSegment(value: string): string {
   return value.split('/').at(-1) ?? value
@@ -120,6 +123,12 @@ function parseService(value: unknown): ServiceInventory {
   }
 }
 
+class GoogleApiError extends Error {
+  constructor(readonly status: number) {
+    super(`Google API returned ${status}`)
+  }
+}
+
 async function googleRequest(
   fetchImpl: typeof fetch,
   token: string,
@@ -134,45 +143,97 @@ async function googleRequest(
     },
     signal: AbortSignal.timeout(30_000)
   })
-  if (!response.ok) throw new Error(`Google API returned ${response.status}`)
+  if (!response.ok) throw new GoogleApiError(response.status)
   return await response.json()
 }
 
-async function endpointProbe(origin: string, fetchImpl: typeof fetch): Promise<EndpointHealth> {
-  const startedAt = performance.now()
-  const check = async (path: '/health' | '/ready'): Promise<boolean> => {
+// A 404 is the API's answer about the resource; anything else is the absence of a reading, so re-ask.
+async function readOnceMore(
+  read: () => Promise<unknown>,
+  wait: (ms: number) => Promise<void>
+): Promise<unknown> {
+  try {
+    return await read()
+  } catch (error) {
+    if (error instanceof GoogleApiError && error.status === 404) throw error
+    await wait(transientProbeRetryDelayMs)
+    return await read()
+  }
+}
+
+// A reading the endpoint actually produced: ok is its answer, latencyMs is that answer's round trip.
+type PathReading = { ok: boolean; latencyMs: number | null }
+
+async function probePath(
+  origin: string,
+  path: '/health' | '/ready',
+  fetchImpl: typeof fetch,
+  wait: (ms: number) => Promise<void>
+): Promise<PathReading> {
+  // null means the request never produced an answer (DNS/TCP/TLS failure or the 8s abort).
+  const attempt = async (): Promise<PathReading | null> => {
+    const startedAt = performance.now()
     try {
       const response = await fetchImpl(`${origin}${path}`, {
         redirect: 'error',
         signal: AbortSignal.timeout(8_000)
       })
-      return response.ok
+      return { ok: response.ok, latencyMs: Math.round(performance.now() - startedAt) }
     } catch {
-      return false
+      return null
     }
   }
-  const [health, ready] = await Promise.all([check('/health'), check('/ready')])
-  return { health, ready, latencyMs: Math.round(performance.now() - startedAt) }
+  const first = await attempt()
+  if (first) return first
+  // A thrown fetch is the absence of a reading, not an unhealthy answer, so re-ask before concluding.
+  await wait(transientProbeRetryDelayMs)
+  return (await attempt()) ?? { ok: false, latencyMs: null }
+}
+
+async function endpointProbe(
+  origin: string,
+  fetchImpl: typeof fetch,
+  requiresReady: boolean,
+  wait: (ms: number) => Promise<void>
+): Promise<EndpointHealth> {
+  const [health, ready] = await Promise.all([
+    probePath(origin, '/health', fetchImpl, wait),
+    requiresReady ? probePath(origin, '/ready', fetchImpl, wait) : null
+  ])
+  // Latency is the slowest answering round trip in this probe; retry delays are not serving latency.
+  const latencies = [health.latencyMs, ready?.latencyMs ?? null].filter(
+    (value): value is number => value !== null
+  )
+  return {
+    health: health.ok,
+    ready: ready ? ready.ok : null,
+    latencyMs: latencies.length > 0 ? Math.max(...latencies) : null
+  }
+}
+
+export type EndpointProbeOptions = {
+  // Auth serves no /ready by design, so it is judged on /health and latency alone.
+  requiresReady?: boolean
+  wait?: (ms: number) => Promise<void>
 }
 
 export async function probeEndpointHealth(
   origin: string,
   fetchImpl: typeof fetch,
-  wait: (ms: number) => Promise<void> = async (ms) =>
-    await new Promise((resolvePromise) => setTimeout(resolvePromise, ms))
+  options: EndpointProbeOptions = {}
 ): Promise<EndpointHealth> {
-  const first = await endpointProbe(origin, fetchImpl)
-  if (
-    first.health &&
-    first.ready &&
-    first.latencyMs !== null &&
-    first.latencyMs <= INCIDENT_MONITOR_THRESHOLDS.endpointLatencyMs
-  ) {
-    return first
-  }
+  const requiresReady = options.requiresReady ?? true
+  const wait = options.wait ?? sleep
+  const accepted = (probe: EndpointHealth): boolean =>
+    probe.health === true &&
+    (!requiresReady || probe.ready === true) &&
+    probe.latencyMs !== null &&
+    probe.latencyMs <= INCIDENT_MONITOR_THRESHOLDS.endpointLatencyMs
+  const first = await endpointProbe(origin, fetchImpl, requiresReady, wait)
+  if (accepted(first)) return first
   // Outwait Relay's ten-second readiness cache before treating the retry as independent.
   await wait(independentEndpointRetryDelayMs)
-  return await endpointProbe(origin, fetchImpl)
+  return await endpointProbe(origin, fetchImpl, requiresReady, wait)
 }
 
 function imageDigest(template: z.infer<typeof TemplateSchema>): string | null {
@@ -285,11 +346,17 @@ function unavailableInventory(environment: RelayOpsEnvironment, warning: string)
   }
 }
 
+export type ResourceInventoryOptions = {
+  wait?: (ms: number) => Promise<void>
+}
+
 export async function readResourceInventory(
   environment: RelayOpsEnvironment,
   gcloud: GcloudClient,
-  fetchImpl: typeof fetch = fetch
+  fetchImpl: typeof fetch = fetch,
+  options: ResourceInventoryOptions = {}
 ): Promise<ResourceInventory> {
+  const wait = options.wait ?? sleep
   let token: string
   try {
     token = await gcloud.accessToken()
@@ -316,7 +383,10 @@ export async function readResourceInventory(
       token,
       `https://certificatemanager.googleapis.com/v1/projects/${environment.project}/locations/global/certificates/${environment.certificateName}`
     ),
-    ...environment.cells.map((cell) => googleRequest(fetchImpl, token, migUrl(cell)))
+    // One transient Compute read must never become a verdict on a cell's power state.
+    ...environment.cells.map((cell) =>
+      readOnceMore(async () => await googleRequest(fetchImpl, token, migUrl(cell)), wait)
+    )
   ])
   const warnings: string[] = []
   const directorValue = parsed(settled[0]!, RunServiceSchema, 'Director service inventory is unavailable.', warnings)
@@ -338,7 +408,8 @@ export async function readResourceInventory(
     ? [unavailableEndpoint(), unavailableEndpoint()]
     : await Promise.all([
         probeEndpointHealth(environment.directorOrigin, fetchImpl),
-        probeEndpointHealth(environment.authOrigin, fetchImpl)
+        // The auth service exposes no /ready, so requiring it would fail every first probe.
+        probeEndpointHealth(environment.authOrigin, fetchImpl, { requiresReady: false })
       ])
   const cells = await Promise.all(environment.cells.map((cell, index) =>
     readCell(environment, cell, migValues[index] ?? null, token, fetchImpl)

cloud/apps/relay/package.json

@@ -15,13 +15,13 @@
     "typecheck": "tsc -p tsconfig.json --noEmit"
   },
   "dependencies": {
-    "@hono/node-server": "^1.19.14",
+    "@hono/node-server": "^1.19.17",
     "@orca-cloud/relay-contract": "workspace:*",
-    "hono": "^4.12.27",
+    "hono": "^4.13.7",
     "jose": "^6.1.3",
     "pg": "^8.22.0",
     "tweetnacl": "^1.0.3",
-    "ws": "^8.18.3",
+    "ws": "^8.21.3",
     "zod": "^3.25.76"
   },
   "devDependencies": {

cloud/apps/relay/src/app.ts

@@ -606,8 +606,9 @@ export function createRelayApp(
       const source = await operations.assignments.cellDeploymentStatus(
         body.data.sourceCellId
       )
+      // Any cell that can be drained can be a rehome source, in either
+      // direction, so the probe is gated on the protocol and not on a region.
       if (
-        source.region !== RELAY_DEFAULT_REGION ||
         !source.runtime ||
         source.runtime.cellIncarnation !== body.data.sourceCellIncarnation ||
         !source.runtime.ready ||
@@ -1412,6 +1413,11 @@ const RegionalRehomeControlSchema = z.discriminatedUnion('action', [
       .int()
       .min(60_000)
       .max(30 * 24 * 60 * 60_000),
+    hostCooldownMs: z
+      .number()
+      .int()
+      .min(60_000)
+      .max(30 * 24 * 60 * 60_000),
     drainGraceMs: z.number().int().min(60_000).max(60 * 60_000),
     confirmation: z.enum([
       'ENABLE_REGIONAL_REHOMING',

cloud/apps/relay/src/assignment-connection-headroom-postgres.test.ts

@@ -44,6 +44,12 @@ describePostgres('PostgreSQL assignment connection headroom', () => {
         `DELETE FROM relay_assignments
          WHERE user_id LIKE 'connection-headroom-postgres-%'`
       )
+      // A snapshot left by an aborted run rejects the replayed watermark
+      // with stale_connection_snapshot.
+      await database.query(
+        `DELETE FROM relay_cell_connection_snapshots WHERE cell_id = ?`,
+        [cell.id]
+      )
       await database.query(
         `DELETE FROM relay_cell_connection_runtime WHERE cell_id = ?`,
         [cell.id]

cloud/apps/relay/src/assignment-control-supersession-postgres.test.ts

@@ -38,6 +38,10 @@ describePostgres('PostgreSQL control supersession', () => {
         [identity.userId]
       )
       await database.query(`DELETE FROM relay_assignments WHERE user_id = ?`, [identity.userId])
+      // A snapshot left by an aborted run rejects the replayed watermark with stale_connection_snapshot.
+      await database.query(`DELETE FROM relay_cell_connection_snapshots WHERE cell_id = ?`, [
+        cell.id
+      ])
       await database.query(`DELETE FROM relay_cell_connection_runtime WHERE cell_id = ?`, [cell.id])
       await database.query(`DELETE FROM relay_cell_connection_limits WHERE cell_id = ?`, [cell.id])
       await database.query(`DELETE FROM relay_cell_runtime WHERE cell_id = ?`, [cell.id])

cloud/apps/relay/src/assignment-inventory-snapshot.ts

@@ -1,3 +1,4 @@
+import { RELAY_DEFAULT_REGION } from '@orca-cloud/relay-contract'
 import type { RelayDatabase, SqlRow } from './database.js'
 
 export type CellInventorySnapshotRow = {
@@ -92,7 +93,7 @@ export async function readAssignmentInventorySnapshot(
   return {
     cells: cellRows.map((row) => ({
       cellId: asText(row, 'cell_id'),
-      region: optionalText(row, 'region') ?? 'us-central1',
+      region: optionalText(row, 'region') ?? RELAY_DEFAULT_REGION,
       admissionState: optionalText(row, 'admission_state') ?? 'unset',
       enabled: asInteger(row, 'enabled') === 1,
       capacityRequests: asInteger(row, 'capacity_requests'),

cloud/apps/relay/src/assignment-store.ts

@@ -30,8 +30,16 @@ import {
   ASSIGNMENT_CONNECTION_HEADROOM_QUERY
 } from './assignment-connection-headroom-query.js'
 import { AssignmentIdentityQueue } from './assignment-identity-queue.js'
+import {
+  REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS
+} from './database.js'
 import type { RelayCellConfig } from './config.js'
-import type { RelayDatabase, RelayTransactionOptions, SqlRow } from './database.js'
+import type {
+  RelayDatabase,
+  RelayLockOptions,
+  RelayTransactionOptions,
+  SqlRow
+} from './database.js'
 import type { RegionalRehomeSafetySnapshot } from './relay-observability.js'
 import {
   combineRegionalRehomeSafety,
@@ -116,7 +124,7 @@ export type RelayAssignmentMigration = AssignmentIdentity & {
 
 export type RegionalRehomeAttempt = AssignmentIdentity & {
   attemptId: string
-  preferredRegion: 'asia-east2'
+  preferredRegion: RelayRegion
   sourceCellId: string
   sourceCellUrl: string
   sourceCellIncarnation: string
@@ -146,6 +154,7 @@ export type RegionalRehomeControl = {
   notBefore: number
   ratePerMinute: number
   preferenceMaxAgeMs: number
+  hostCooldownMs: number
   drainGraceMs: number
 }
 
@@ -316,6 +325,25 @@ const ACTIVITY_REQUEST_UNITS: Record<AssignmentActivityKind, number> = {
 }
 
 const ASSIGNMENT_LOCK_RETRY_DEADLINE_MS = 15_000
+// Why: one global FOR UPDATE over a 23-row table serialises every director and
+// cell. At the 1s pool lock_timeout each blocked waiter also holds a pooled
+// client for a full second, so the queue converts contention into pool
+// exhaustion. The lock is held to COMMIT and the assignment path runs many
+// statements after taking it, and no hold-time telemetry existed before this
+// change, so 500ms is a first value to tune once cellInventoryHoldMsMax lands.
+export const CELL_INVENTORY_LOCK_TIMEOUT_MS = 500
+
+// The same inventory lock is taken by live requests and by background sweeps,
+// and the right failure mode differs per caller.
+export type CellInventoryLockMode =
+  // Bound the wait so a blocked request stops occupying a pooled client.
+  | 'request'
+  // Never queue: the caller handles database_lock_unavailable and moves on.
+  | 'nowait'
+  // A sweep can enter here, so keep the pool default. Failing sooner would turn
+  // ordinary contention into a 55P03 the retry wrapper reports as terminal, which
+  // spends the incident gate's bounded exhausted-retry budget (300 per 5 min).
+  | 'pool-default'
 // Why: stranded detection (issue #225) needs a grant old enough that a real
 // attach would have registered (the 90s activity lease covers dial +
 // activation), yet recent enough to prove an active retry loop rather than
@@ -536,29 +564,35 @@ export class RelayAssignmentStore {
   async assign(
     identity: AssignmentIdentity,
     preferredRegion?: RelayRegion,
-    placementRegion: RelayRegion = preferredRegion ?? RELAY_DEFAULT_REGION
+    placementRegion: RelayRegion = preferredRegion ?? RELAY_DEFAULT_REGION,
+    // evacuateDeadCells re-enters placement from a sweep; it must not take the
+    // bounded wait, whose 55P03 would surface as a terminal sweep failure.
+    lockMode: CellInventoryLockMode = 'request'
   ): Promise<RelayAssignment> {
-    const sticky = await this.assignStickyWithLockRetry(identity, preferredRegion)
+    const sticky = await this.assignStickyWithLockRetry(identity, lockMode, preferredRegion)
     if (sticky) return sticky
     // Only placement needs the global inventory critical section; queueing those
     // attempts locally avoids turning true placement bursts into NOWAIT storms.
     return await this.serializeAssignment(
-      async () => await this.assignWithLockRetry(identity, preferredRegion, placementRegion)
+      async () =>
+        await this.assignWithLockRetry(identity, lockMode, preferredRegion, placementRegion)
     )
   }
 
   private async assignStickyWithLockRetry(
     identity: AssignmentIdentity,
+    lockMode: CellInventoryLockMode,
     preferredRegion?: RelayRegion
   ): Promise<RelayAssignment | null> {
     return await this.withAssignmentLockRetry(
       async (inventoryFirst) =>
-        await this.assignStickyOnce(identity, inventoryFirst, preferredRegion)
+        await this.assignStickyOnce(identity, inventoryFirst, lockMode, preferredRegion)
     )
   }
 
   private async assignWithLockRetry(
     identity: AssignmentIdentity,
+    lockMode: CellInventoryLockMode,
     preferredRegion?: RelayRegion,
     placementRegion: RelayRegion = preferredRegion ?? RELAY_DEFAULT_REGION
   ): Promise<RelayAssignment> {
@@ -566,7 +600,13 @@ export class RelayAssignmentStore {
     let inventoryScope: AssignmentInventoryScope = 'none'
     while (true) {
       try {
-        return await this.assignOnce(identity, inventoryScope, preferredRegion, placementRegion)
+        return await this.assignOnce(
+          identity,
+          inventoryScope,
+          lockMode,
+          preferredRegion,
+          placementRegion
+        )
       } catch (error) {
         if (error instanceof AssignmentInventoryScopeChanged) {
           inventoryScope = 'all'
@@ -604,13 +644,22 @@ export class RelayAssignmentStore {
   private async assignStickyOnce(
     identity: AssignmentIdentity,
     inventoryFirst: boolean,
+    lockMode: CellInventoryLockMode,
     preferredRegion?: RelayRegion
   ): Promise<RelayAssignment | null> {
     const now = this.now()
     return await this.database.transaction(async (transaction) => {
-      const lockedCells = inventoryFirst
-        ? await this.lockCellInventory(transaction)
+      // Why: the retry exists to take a cell row before the assignment row, the
+      // order placement uses. It only ever needs the one cell this host is
+      // pinned to, so read the pin unlocked and lock that row alone; taking all
+      // 23 queued every sticky refresh in the fleet behind every other one.
+      const pinnedCellId = inventoryFirst
+        ? await this.pinnedCellId(transaction, identity)
         : undefined
+      const lockedCells =
+        pinnedCellId === undefined
+          ? undefined
+          : await this.lockCellRows(transaction, [pinnedCellId], lockMode)
       const existing = await this.assignmentRow(transaction, identity, inventoryFirst)
       if (!existing) return null
       const activityLeases = await this.lockAssignmentActivities(transaction, identity, true)
@@ -624,6 +673,11 @@ export class RelayAssignmentStore {
       }
 
       const currentCellId = text(existing, 'cell_id')
+      // The pin moved between the unlocked read and the assignment lock, so the
+      // row held is the wrong one. Same recovery as losing the lock: retry.
+      if (pinnedCellId !== undefined && pinnedCellId !== currentCellId) {
+        throw new Error('database_lock_unavailable')
+      }
       const hadControl = holdsControlLease(
         activityLeases,
         currentCellId,
@@ -664,14 +718,9 @@ export class RelayAssignmentStore {
       if (hadControl) {
         await this.touchAssignment(transaction, identity, leaseExpiresAt, now)
       } else {
-        const nextReservation = integer(currentRow, 'reserved_requests') + 1
-        if (nextReservation > integer(currentRow, 'capacity_requests')) {
-          throw new Error('relay_capacity_exhausted')
-        }
-        await transaction.query(
-          `UPDATE relay_cells SET reserved_requests = ?, updated_at = ? WHERE cell_id = ?`,
-          [nextReservation, now, currentCellId]
-        )
+        // Delta, not the value read from the snapshot: an absolute write here
+        // would clobber any concurrent movement of the same counter.
+        await this.adjustCellReservationAtomically(transaction, currentCellId, 1)
         await this.adjustActivityCount(transaction, identity, 'control', 1, leaseExpiresAt, now)
         await this.insertPendingControlLease(
           transaction,
@@ -751,6 +800,7 @@ export class RelayAssignmentStore {
   private async assignOnce(
     identity: AssignmentIdentity,
     inventoryScope: AssignmentInventoryScope,
+    lockMode: CellInventoryLockMode,
     preferredRegion?: RelayRegion,
     placementRegion: RelayRegion = preferredRegion ?? RELAY_DEFAULT_REGION
   ): Promise<RelayAssignment> {
@@ -760,9 +810,9 @@ export class RelayAssignmentStore {
     return await this.database.transaction(async (transaction) => {
       let lockedCells =
         inventoryScope === 'all'
-          ? await this.lockCellInventory(transaction)
+          ? await this.lockCellInventory(transaction, lockMode)
           : inventoryScope === 'general'
-            ? await this.lockGeneralCellInventory(transaction)
+            ? await this.lockGeneralCellInventory(transaction, lockMode)
             : undefined
       const existing = await this.assignmentRow(
         transaction,
@@ -779,7 +829,7 @@ export class RelayAssignmentStore {
       let connectionHeadroomReassignment = false
       let strandedReassignment = false
       if (existing && !mayNormallyReassign(activity(existing), now)) {
-        lockedCells ??= await this.lockCellInventory(transaction, true)
+        lockedCells ??= await this.lockCellInventory(transaction, 'nowait')
         const admission = await cellAdmissionStates(transaction)
         const currentRow = lockedCells.find(
           (row) => text(row, 'cell_id') === text(existing, 'cell_id')
@@ -859,8 +909,8 @@ export class RelayAssignmentStore {
       }
 
       lockedCells ??= existing
-        ? await this.lockCellInventory(transaction, true)
-        : await this.lockGeneralCellInventory(transaction, true)
+        ? await this.lockCellInventory(transaction, 'nowait')
+        : await this.lockGeneralCellInventory(transaction, 'nowait')
       const target = await this.leastLoadedCell(
         transaction,
         lockedCells,
@@ -2114,7 +2164,7 @@ export class RelayAssignmentStore {
          ORDER BY migration.user_id, migration.relay_host_id`,
         [input.cellId]
       )
-      const cells = await this.lockCellInventory(transaction)
+      const cells = await this.lockCellInventory(transaction, 'request')
       for (const migrationRow of migrations) {
         const identity = {
           userId: text(migrationRow, 'user_id'),
@@ -2608,10 +2658,12 @@ export class RelayAssignmentStore {
     let moved = 0
     for (const row of rows) {
       try {
-        const assignment = await this.assign({
-          userId: text(row, 'user_id'),
-          relayHostId: text(row, 'relay_host_id')
-        })
+        const assignment = await this.assign(
+          { userId: text(row, 'user_id'), relayHostId: text(row, 'relay_host_id') },
+          undefined,
+          undefined,
+          'pool-default'
+        )
         if (assignment.cellId !== text(row, 'cell_id')) moved++
       } catch (error) {
         if (!(error instanceof Error && error.message === 'relay_capacity_exhausted')) throw error
@@ -3162,8 +3214,7 @@ export class RelayAssignmentStore {
         )
         const requestDelta = ACTIVITY_REQUEST_UNITS[kind] * (after - before)
         if (requestDelta !== 0) {
-          await this.lockCellInventory(transaction)
-          await this.adjustCellReservation(transaction, text(row, 'cell_id'), requestDelta)
+          await this.adjustCellReservationAtomically(transaction, text(row, 'cell_id'), requestDelta)
         }
       })
     })
@@ -3223,9 +3274,12 @@ export class RelayAssignmentStore {
         }
         const units = ACTIVITY_REQUEST_UNITS[input.kind]
         if (existing) {
-          await this.lockCellInventory(transaction)
+          // Why: a client-chosen activity id can move between cells, so lock the
+          // one or two rows this path touches in cell_id order, the same order
+          // placement takes the inventory in, and no cycle can form.
+          await this.lockCellRows(transaction, [text(existing, 'cell_id'), input.cellId])
           await this.removeActivityLease(transaction, identity, existing, now)
-          await this.adjustCellReservation(transaction, input.cellId, units)
+          await this.adjustCellReservationAtomically(transaction, input.cellId, units)
         }
         await this.adjustActivityCount(transaction, identity, input.kind, 1, expiresAt, now)
         await transaction.query(
@@ -3540,8 +3594,7 @@ export class RelayAssignmentStore {
             )
             await this.touchAssignment(transaction, identity, expiresAt, now)
           } else {
-            await this.lockCellInventory(transaction)
-            await this.adjustCellReservation(transaction, input.cellId, 1)
+            await this.adjustCellReservationAtomically(transaction, input.cellId, 1)
             await this.adjustActivityCount(transaction, identity, 'control', 1, expiresAt, now)
             await transaction.query(
               `INSERT INTO relay_assignment_activity_leases
@@ -3614,7 +3667,7 @@ export class RelayAssignmentStore {
       }
       if (sourceCellId === targetCellId) throw new Error('target_matches_source')
       await this.lockAssignmentActivities(transaction, identity)
-      const cells = await this.lockCellInventory(transaction)
+      const cells = await this.lockCellInventory(transaction, 'request')
       const target = cells.find((row) => text(row, 'cell_id') === targetCellId)
       if (!target || integer(target, 'enabled') !== 1) throw new Error('target_cell_unavailable')
       if (!(await this.cellIsLive(transaction, targetCellId, now))) {
@@ -3822,7 +3875,7 @@ export class RelayAssignmentStore {
       let lockedCells: SqlRow[] | undefined
       if (inventoryFirst) {
         try {
-          lockedCells = await this.lockCellInventory(transaction)
+          lockedCells = await this.lockCellInventory(transaction, 'request')
         } catch (error) {
           if (isDatabaseLockTimeout(error)) {
             throw new Error('database_lock_unavailable')
@@ -3863,7 +3916,7 @@ export class RelayAssignmentStore {
       if (activityUnitsForCell(activityLeases, input.sourceCellId) > 0) {
         throw new Error('migration_source_still_active')
       }
-      const cells = lockedCells ?? (await this.lockCellInventory(transaction, true))
+      const cells = lockedCells ?? (await this.lockCellInventory(transaction, 'nowait'))
       const source = cells.find((cell) => text(cell, 'cell_id') === input.sourceCellId)
       const target = cells.find((cell) => text(cell, 'cell_id') === input.targetCellId)
       if (!source || integer(source, 'enabled') !== 0) {
@@ -3967,7 +4020,7 @@ export class RelayAssignmentStore {
     const now = this.now()
     return await this.database.transaction(async (transaction) => {
       const lockedCells = inventoryFirst
-        ? await this.lockCellInventory(transaction)
+        ? await this.lockCellInventory(transaction, 'request')
         : undefined
       const assignment = await this.assignmentRow(transaction, identity, inventoryFirst)
       const existing = (
@@ -4041,7 +4094,7 @@ export class RelayAssignmentStore {
       ) {
         throw new Error('migration_activity_topology_mismatch')
       }
-      const cells = lockedCells ?? (await this.lockCellInventory(transaction, true))
+      const cells = lockedCells ?? (await this.lockCellInventory(transaction, 'nowait'))
       const source = cells.find((cell) => text(cell, 'cell_id') === input.sourceCellId)
       const currentTarget = cells.find(
         (cell) => text(cell, 'cell_id') === input.currentTargetCellId
@@ -4458,7 +4511,7 @@ export class RelayAssignmentStore {
             throw new Error('migration_activity_topology_mismatch')
           }
         }
-        if (obsoleteLeases.length > 0) await this.lockCellInventory(transaction)
+        if (obsoleteLeases.length > 0) await this.lockCellInventory(transaction, 'request')
         for (const lease of obsoleteLeases) {
           await this.removeActivityLease(transaction, identity, lease, now)
         }
@@ -4634,7 +4687,7 @@ export class RelayAssignmentStore {
       ) {
         throw new Error('migration_activity_lease_shape_mismatch')
       }
-      await this.lockCellInventory(transaction)
+      await this.lockCellInventory(transaction, 'request')
       await this.adjustCellReservation(
         transaction,
         input.currentTargetCellId,
@@ -4723,7 +4776,7 @@ export class RelayAssignmentStore {
       if (this.requireLiveCells) {
         let cells: SqlRow[]
         try {
-          cells = await this.lockCellInventory(transaction, true)
+          cells = await this.lockCellInventory(transaction, 'nowait')
         } catch (error) {
           if (isDatabaseLockUnavailable(error)) {
             // Mixed-version workers may still hold a cell-first lock; defer
@@ -4779,7 +4832,7 @@ export class RelayAssignmentStore {
       )
       if (!targetIsActive) throw new Error('migration_target_not_active')
       const lease = activityLeaseById(activityLeases, migrationActivityId(assignmentEpoch))
-      if (lease && !cellsLocked) await this.lockCellInventory(transaction)
+      if (lease && !cellsLocked) await this.lockCellInventory(transaction, 'pool-default')
       if (lease) await this.removeActivityLease(transaction, identity, lease, now)
       await transaction.query(
         `UPDATE relay_assignment_migrations SET completed_at = ?, updated_at = ?
@@ -4801,7 +4854,7 @@ export class RelayAssignmentStore {
       const sourceCellId = text(assignment, 'cell_id')
       if (sourceCellId === targetCellId) throw new Error('target_matches_source')
       await this.lockAssignmentActivities(transaction, identity)
-      const cells = await this.lockCellInventory(transaction)
+      const cells = await this.lockCellInventory(transaction, 'request')
       const admission = await cellAdmissionStates(transaction)
       const targetRow = cells.find(
         (row) =>
@@ -4872,6 +4925,7 @@ export class RelayAssignmentStore {
     notBefore: number
     ratePerMinute: number
     preferenceMaxAgeMs: number
+    hostCooldownMs: number
     drainGraceMs: number
   }): Promise<RegionalRehomeControl> {
     if (!Number.isSafeInteger(input.expectedGeneration) || input.expectedGeneration < 0) {
@@ -4890,6 +4944,13 @@ export class RelayAssignmentStore {
     ) {
       throw new Error('invalid_regional_rehome_preference_age')
     }
+    if (
+      !Number.isSafeInteger(input.hostCooldownMs) ||
+      input.hostCooldownMs < 60_000 ||
+      input.hostCooldownMs > 30 * 24 * 60 * 60_000
+    ) {
+      throw new Error('invalid_regional_rehome_host_cooldown')
+    }
     if (
       !Number.isSafeInteger(input.drainGraceMs) ||
       input.drainGraceMs < 60_000 ||
@@ -4918,14 +4979,15 @@ export class RelayAssignmentStore {
       await transaction.query(
         `UPDATE relay_region_rehome_control
          SET generation = generation + 1, enabled = ?, not_before = ?,
-             rate_per_minute = ?, preference_max_age_ms = ?, drain_grace_ms = ?,
-             updated_at = ?
+             rate_per_minute = ?, preference_max_age_ms = ?, host_cooldown_ms = ?,
+             drain_grace_ms = ?, updated_at = ?
          WHERE control_id = 'global'`,
         [
           input.enabled ? 1 : 0,
           input.notBefore,
           input.ratePerMinute,
           input.preferenceMaxAgeMs,
+          input.hostCooldownMs,
           input.drainGraceMs,
           now
         ]
@@ -4957,10 +5019,17 @@ export class RelayAssignmentStore {
     await database.query(
       `INSERT INTO relay_region_rehome_control
        (control_id, generation, enabled, observation_started_at, not_before,
-        rate_per_minute, preference_max_age_ms, drain_grace_ms, updated_at)
-       VALUES ('global', 0, 0, ?, 0, 10, ?, ?, ?)
+        rate_per_minute, preference_max_age_ms, host_cooldown_ms, drain_grace_ms,
+        updated_at)
+       VALUES ('global', 0, 0, ?, 0, 10, ?, ?, ?, ?)
        ON CONFLICT (control_id) DO NOTHING`,
-      [now, 24 * 60 * 60_000, 60 * 60_000, now]
+      [
+        now,
+        24 * 60 * 60_000,
+        REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS,
+        60 * 60_000,
+        now
+      ]
     )
   }
 
@@ -4968,6 +5037,9 @@ export class RelayAssignmentStore {
     return await this.readRegionalRehomeFleetSafety(this.database, this.now())
   }
 
+  // The rehome fleet is every general cell that can be drained: those are the
+  // sources and, because a host must be movable back out again, the only legal
+  // targets. The region join stays so a cell with no region row is excluded.
   private async readRegionalRehomeFleetSafety(
     database: RelayDatabase,
     now: number
@@ -4988,10 +5060,7 @@ export class RelayAssignmentStore {
          ON safety.cell_id = runtime.cell_id
         AND safety.cell_incarnation = runtime.cell_incarnation
        WHERE cell.enabled = 1 AND admission.admission_state = 'general'
-         AND (
-           region.region = 'asia-east2' OR
-           (region.region = 'us-central1' AND capability.regional_rehome_protocol >= 1)
-         )`
+         AND capability.regional_rehome_protocol >= 1`
     )
     const valid = rows.filter(
       (row) =>
@@ -5051,7 +5120,13 @@ export class RelayAssignmentStore {
     }
     this.pendingRegionalRehomeDisableLog = null
     const candidateSkips: RegionalRehomeCandidateSkip[] = []
+    // A Postgres transaction is unusable after a NOWAIT abort, so a contended
+    // tick abandons the candidate it stopped on plus every one behind it.
+    let candidatesTotal = 0
+    let candidatesFinished = 0
     const claimResult = await this.database.transaction(async (transaction) => {
+      candidatesTotal = 0
+      candidatesFinished = 0
       candidateSkips.length = 0
       await this.initializeRegionalRehomeControl(transaction, now)
       const control = (
@@ -5064,6 +5139,10 @@ export class RelayAssignmentStore {
       }
       const intervalMs = Math.ceil(60_000 / integer(control, 'rate_per_minute'))
       const preferenceCutoff = now - integer(control, 'preference_max_age_ms')
+      // A host that was rehomed recently is left alone whichever way its
+      // preference now points: a flapping region probe must not walk one host
+      // back and forth across an ocean.
+      const cooldownCutoff = now - integer(control, 'host_cooldown_ms')
       await transaction.query(
         `INSERT INTO relay_region_rehome_worker_state
          (worker_id, next_dispatch_at, paused_until, consecutive_failures, updated_at)
@@ -5122,6 +5201,7 @@ export class RelayAssignmentStore {
         )
       )[0]
       if (retry) {
+        candidatesTotal = 1
         const fleetSafety = await this.lockedRegionalRehomeFleetSafety(transaction, now)
         if (
           !(await this.regionalRehomeSafetyAllowsClaim(
@@ -5190,6 +5270,7 @@ export class RelayAssignmentStore {
         )
       )[0]
       if (redrain) {
+        candidatesTotal = 1
         const fleetSafety = await this.lockedRegionalRehomeFleetSafety(transaction, now)
         if (
           !(await this.regionalRehomeSafetyAllowsClaim(
@@ -5227,9 +5308,8 @@ export class RelayAssignmentStore {
          JOIN relay_cell_capabilities capability
            ON capability.cell_id = runtime.cell_id
           AND capability.cell_incarnation = runtime.cell_incarnation
-         WHERE preference.preferred_region = 'asia-east2'
+         WHERE preference.preferred_region <> region.region
            AND preference.observed_at >= ?
-           AND region.region = 'us-central1'
            AND admission.admission_state = 'general'
            AND runtime.ready = 1 AND runtime.last_heartbeat_at > ?
            AND capability.regional_rehome_protocol >= 1
@@ -5249,10 +5329,40 @@ export class RelayAssignmentStore {
                AND migration.relay_host_id = assignment.relay_host_id
                AND migration.completed_at IS NULL AND migration.aborted_at IS NULL
            )
+           AND NOT EXISTS (
+             SELECT 1 FROM relay_region_rehome_attempts recent
+             WHERE recent.user_id = preference.user_id
+               AND recent.relay_host_id = preference.relay_host_id
+               AND recent.created_at > ?
+           )
+           AND EXISTS (
+             SELECT 1 FROM relay_cell_regions target_region
+             JOIN relay_cells target_cell ON target_cell.cell_id = target_region.cell_id
+             JOIN relay_cell_admission target_admission
+               ON target_admission.cell_id = target_region.cell_id
+             JOIN relay_cell_runtime target_runtime
+               ON target_runtime.cell_id = target_region.cell_id
+             JOIN relay_cell_capabilities target_capability
+               ON target_capability.cell_id = target_runtime.cell_id
+              AND target_capability.cell_incarnation = target_runtime.cell_incarnation
+             WHERE target_region.region = preference.preferred_region
+               AND target_cell.enabled = 1
+               AND target_admission.admission_state = 'general'
+               AND target_runtime.ready = 1
+               AND target_runtime.last_heartbeat_at > ?
+               AND target_capability.regional_rehome_protocol >= 1
+           )
          ORDER BY preference.observed_at, preference.user_id, preference.relay_host_id
          LIMIT 10`,
-        [preferenceCutoff, now - this.heartbeatTtlMs, now]
+        [
+          preferenceCutoff,
+          now - this.heartbeatTtlMs,
+          now,
+          cooldownCutoff,
+          now - this.heartbeatTtlMs
+        ]
       )
+      candidatesTotal = candidates.length
       for (const candidate of candidates) {
         const claimed = await this.startRegionalRehomeCandidate(transaction, {
           identity: {
@@ -5262,12 +5372,14 @@ export class RelayAssignmentStore {
           sourceCellId: text(candidate, 'source_cell_id'),
           assignmentEpoch: integer(candidate, 'assignment_epoch'),
           preferenceCutoff,
+          cooldownCutoff,
           drainGraceMs: integer(control, 'drain_grace_ms'),
           processSafety: effectiveProcessSafety,
           worker,
           now,
           skips: candidateSkips
         })
+        candidatesFinished++
         if (!claimed) continue
         await this.markRegionalRehomeDispatchClaimed(
           transaction,
@@ -5283,6 +5395,21 @@ export class RelayAssignmentStore {
         await this.markRegionalRehomeTickSkipped(transaction, now, intervalMs)
       }
       return null
+    }).catch((error: unknown): RegionalRehomeAttempt | null => {
+      // Only inventory contention is swallowed here; every other failure keeps
+      // its existing propagation and its dispatch-failure accounting.
+      if (!isDatabaseLockUnavailable(error)) throw error
+      // The dispatch tick runs every second; losing one to inventory contention
+      // costs a second of latency and never loses durable rehome state. The
+      // rolled-back transaction never disabled anything, so its pending disable
+      // log would describe a decision that did not happen.
+      candidateSkips.length = 0
+      this.pendingRegionalRehomeDisableLog = null
+      warnSweepCellInventoryBusy(
+        'claim-regional-rehome',
+        Math.max(1, candidatesTotal - candidatesFinished)
+      )
+      return null
     })
     const pendingDisableLog = this.pendingRegionalRehomeDisableLog
     this.pendingRegionalRehomeDisableLog = null
@@ -5300,6 +5427,7 @@ export class RelayAssignmentStore {
       sourceCellId: string
       assignmentEpoch: number
       preferenceCutoff: number
+      cooldownCutoff: number
       drainGraceMs: number
       processSafety: RegionalRehomeSafetySnapshot
       worker: SqlRow
@@ -5323,14 +5451,11 @@ export class RelayAssignmentStore {
         [input.identity.userId, input.identity.relayHostId]
       )
     )[0]
-    if (
-      !preference ||
-      text(preference, 'preferred_region') !== 'asia-east2' ||
-      integer(preference, 'observed_at') < input.preferenceCutoff
-    ) {
+    if (!preference || integer(preference, 'observed_at') < input.preferenceCutoff) {
       input.skips.push({ reason: 'candidate_stale' })
       return null
     }
+    const preferredRegion = relayRegion(preference, 'preferred_region')
     const activeMigration = await transaction.queryLocked(
       `SELECT assignment_epoch FROM relay_assignment_migrations
        WHERE user_id = ? AND relay_host_id = ?
@@ -5341,9 +5466,21 @@ export class RelayAssignmentStore {
       input.skips.push({ reason: 'candidate_stale' })
       return null
     }
+    // Re-read under the claim: an attempt committed between the scan and here
+    // would otherwise start a second move for the same host.
+    const recentAttempt = await transaction.query(
+      `SELECT 1 FROM relay_region_rehome_attempts
+       WHERE user_id = ? AND relay_host_id = ? AND created_at > ?
+       LIMIT 1`,
+      [input.identity.userId, input.identity.relayHostId, input.cooldownCutoff]
+    )
+    if (recentAttempt.length > 0) {
+      input.skips.push({ reason: 'host_cooldown' })
+      return null
+    }
     const activityLeases = await this.lockAssignmentActivities(transaction, input.identity)
     assertAssignmentActivityCounts(assignment, activityLeases, 0)
-    const cells = await this.lockCellInventory(transaction)
+    const cells = await this.lockCellInventory(transaction, 'nowait')
     const admission = await cellAdmissionStates(transaction)
     const regions = new Map(
       (await transaction.query(`SELECT cell_id, region FROM relay_cell_regions`)).map((row) => [
@@ -5395,11 +5532,17 @@ export class RelayAssignmentStore {
       )
       return null
     }
+    // The preference read under lock can now agree with the cell the host is
+    // already on: nothing to move, in either direction.
+    if (regions.get(input.sourceCellId) === preferredRegion) {
+      input.skips.push({ reason: 'candidate_stale' })
+      return null
+    }
     if (
       !source ||
       integer(source, 'enabled') !== 1 ||
       admission.get(input.sourceCellId) !== 'general' ||
-      regions.get(input.sourceCellId) !== RELAY_DEFAULT_REGION ||
+      regions.get(input.sourceCellId) === undefined ||
       !sourceRuntime ||
       integer(sourceRuntime, 'ready') !== 1 ||
       integer(sourceRuntime, 'last_heartbeat_at') <= input.now - this.heartbeatTtlMs ||
@@ -5428,17 +5571,25 @@ export class RelayAssignmentStore {
       return null
     }
     const connectionHeadroom = await this.connectionHeadroomByCell(transaction)
+    // A target must be drainable too, or the host lands somewhere it can never
+    // be rehomed out of again -- the trap this bidirectional move exists to undo.
     const eligibleTargets = cells.filter((row) => {
       const cellId = text(row, 'cell_id')
       const runtime = runtimes.find((candidate) => text(candidate, 'cell_id') === cellId)
+      const capability = capabilities.find(
+        (candidate) => text(candidate, 'cell_id') === cellId
+      )
       return (
         cellId !== input.sourceCellId &&
         integer(row, 'enabled') === 1 &&
         admission.get(cellId) === 'general' &&
-        regions.get(cellId) === 'asia-east2' &&
+        regions.get(cellId) === preferredRegion &&
         runtime !== undefined &&
         integer(runtime, 'ready') === 1 &&
-        integer(runtime, 'last_heartbeat_at') > input.now - this.heartbeatTtlMs
+        integer(runtime, 'last_heartbeat_at') > input.now - this.heartbeatTtlMs &&
+        capability !== undefined &&
+        text(capability, 'cell_incarnation') === text(runtime, 'cell_incarnation') &&
+        integer(capability, 'regional_rehome_protocol') >= 1
       )
     })
     const targetIsClean = (row: SqlRow): boolean => {
@@ -5594,12 +5745,13 @@ export class RelayAssignmentStore {
         drain_grace_ms, send_attempts, last_send_attempt_at,
         drain_receipt_at, drain_outcome, completed_at, aborted_at,
         created_at, updated_at)
-       VALUES (?, ?, ?, 'asia-east2', ?, ?, ?, ?, ?, ?, ?, 0, NULL,
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, NULL,
          NULL, NULL, NULL, NULL, ?, ?)`,
       [
         attemptId,
         input.identity.userId,
         input.identity.relayHostId,
+        preferredRegion,
         input.sourceCellId,
         text(sourceRuntime, 'cell_incarnation'),
         targetCellId,
@@ -5614,7 +5766,7 @@ export class RelayAssignmentStore {
     return {
       ...input.identity,
       attemptId,
-      preferredRegion: 'asia-east2',
+      preferredRegion,
       sourceCellId: input.sourceCellId,
       sourceCellUrl: text(source, 'cell_url'),
       sourceCellIncarnation: text(sourceRuntime, 'cell_incarnation'),
@@ -5630,7 +5782,7 @@ export class RelayAssignmentStore {
     transaction: RelayDatabase,
     now: number
   ): Promise<RegionalRehomeFleetSafety> {
-    const cells = await this.lockCellInventory(transaction)
+    const cells = await this.lockCellInventory(transaction, 'nowait')
     const admission = await cellAdmissionStates(transaction)
     const regions = new Map(
       (await transaction.query(`SELECT cell_id, region FROM relay_cell_regions`)).map((row) => [
@@ -5874,6 +6026,7 @@ export class RelayAssignmentStore {
       [...quarantined, limit]
     )
     let completed = 0
+    let inventoryBusy = 0
     for (const candidate of candidates) {
       // One poisoned row must not stall every later candidate: an invariant
       // throw here blocked fleet completions head-of-line in production.
@@ -5890,9 +6043,14 @@ export class RelayAssignmentStore {
         if (changed) completed++
         this.regionalRehomeCandidateQuarantine.delete(attemptId)
       } catch (error) {
+        if (isDatabaseLockUnavailable(error)) {
+          inventoryBusy++
+          continue
+        }
         this.recordRegionalRehomeCandidateFailure('complete', attemptId, now, error)
       }
     }
+    warnSweepCellInventoryBusy('complete-ready-regional-rehomes', inventoryBusy)
     return completed
   }
 
@@ -6112,7 +6270,7 @@ export class RelayAssignmentStore {
         leases,
         migration
       )
-      const cells = await this.lockCellInventory(transaction)
+      const cells = await this.lockCellInventory(transaction, 'nowait')
       const target = cells.find((cell) => text(cell, 'cell_id') === targetCellId)
       const admission = await cellAdmissionStates(transaction)
       if (
@@ -6261,6 +6419,7 @@ export class RelayAssignmentStore {
       [now - REGIONAL_REHOME_MAX_REFRESH_MS, ...quarantined, limit]
     )
     let aborted = 0
+    let inventoryBusy = 0
     for (const candidate of candidates) {
       const identity = {
         userId: text(candidate, 'user_id'),
@@ -6318,7 +6477,7 @@ export class RelayAssignmentStore {
             integer(lease, 'expires_at') > now
         )
         if (targetActive) return false
-        const cells = await this.lockCellInventory(transaction)
+        const cells = await this.lockCellInventory(transaction, 'nowait')
         const source = cells.find((cell) => text(cell, 'cell_id') === sourceCellId)
         const admission = await cellAdmissionStates(transaction)
         if (
@@ -6376,10 +6535,12 @@ export class RelayAssignmentStore {
         })
         this.regionalRehomeCandidateQuarantine.delete(attemptId)
       } catch (error) {
-        this.recordRegionalRehomeCandidateFailure('abort', attemptId, now, error)
+        if (isDatabaseLockUnavailable(error)) inventoryBusy++
+        else this.recordRegionalRehomeCandidateFailure('abort', attemptId, now, error)
       }
       if (changed) aborted++
     }
+    warnSweepCellInventoryBusy('abort-expired-regional-rehomes', inventoryBusy)
     return aborted
   }
 
@@ -6396,6 +6557,7 @@ export class RelayAssignmentStore {
       [now, now, abandonedBefore, abandonedBefore]
     )
     let aborted = 0
+    let inventoryBusy = 0
     for (const candidate of candidates) {
       const didAbort = await this.database.transaction(async (transaction) => {
         const identity = {
@@ -6480,7 +6642,7 @@ export class RelayAssignmentStore {
           ]
             .map((activityId) => activityLeaseById(activityLeases, activityId))
             .filter((lease): lease is SqlRow => lease !== undefined)
-          if (obsoleteLeases.length > 0) await this.lockCellInventory(transaction)
+          if (obsoleteLeases.length > 0) await this.lockCellInventory(transaction, 'nowait')
           for (const lease of obsoleteLeases) {
             await this.removeActivityLease(transaction, identity, lease, now)
           }
@@ -6498,7 +6660,7 @@ export class RelayAssignmentStore {
           )
           return true
         }
-        const cells = await this.lockCellInventory(transaction)
+        const cells = await this.lockCellInventory(transaction, 'nowait')
         const sourceCellId = text(row, 'source_cell_id')
         const admissionRows = await transaction.query(
           `SELECT cell_id, admission_state, updated_at FROM relay_cell_admission
@@ -6595,9 +6757,15 @@ export class RelayAssignmentStore {
           [now, now, identity.userId, identity.relayHostId, assignmentEpoch]
         )
         return true
+      }).catch((error: unknown): boolean => {
+        // Expiry is durable; another director settling this row is not a failure.
+        if (!isDatabaseLockUnavailable(error)) throw error
+        inventoryBusy++
+        return false
       })
       if (didAbort) aborted++
     }
+    warnSweepCellInventoryBusy('abort-expired-evacuations', inventoryBusy)
     return aborted
   }
 
@@ -6665,7 +6833,7 @@ export class RelayAssignmentStore {
           const activityLeases = await this.lockAssignmentActivities(transaction, identity, true)
           const lease = activityLeaseById(activityLeases, text(candidate, 'activity_id'))
           if (!lease || integer(lease, 'expires_at') > now) return false
-          await this.lockCellInventory(transaction, true)
+          await this.lockCellInventory(transaction, 'nowait')
           await this.removeActivityLease(transaction, identity, lease, now)
           return true
         })
@@ -6709,7 +6877,7 @@ export class RelayAssignmentStore {
           [now],
           { failIfUnavailable: true }
         )
-        if (expired.length > 0) await this.lockCellInventory(transaction, true)
+        if (expired.length > 0) await this.lockCellInventory(transaction, 'nowait')
         for (const row of expired) {
           await this.adjustCellReservation(transaction, text(row, 'cell_id'), -requestUnits(row))
           await transaction.query(
@@ -6782,13 +6950,24 @@ export class RelayAssignmentStore {
           targetCellId
         ]
       )
-      const cells = await this.lockCellInventory(transaction)
+      // Only the two cells this repairs need holding. The id set below is an
+      // existence check against a table that only reconcileCells writes, so it
+      // reads unlocked instead of dragging the other 21 rows into the section.
+      const cellIds = new Set(
+        (await transaction.query(`SELECT cell_id FROM relay_cells`)).map((row) =>
+          text(row, 'cell_id')
+        )
+      )
+      const cells = await this.lockCellRows(
+        transaction,
+        [sourceCellId, targetCellId],
+        'pool-default'
+      )
       const assignmentKeys = new Set(
         assignments.map((row) =>
           assignmentKey(text(row, 'user_id'), text(row, 'relay_host_id'))
         )
       )
-      const cellIds = new Set(cells.map((row) => text(row, 'cell_id')))
       const assignmentCounts = new Map<
         string,
         { counts: Record<AssignmentActivityKind, number>; leaseExpiresAt: number }
@@ -6842,9 +7021,7 @@ export class RelayAssignmentStore {
         )
       }
 
-      for (const row of cells.filter((cell) =>
-        [sourceCellId, targetCellId].includes(text(cell, 'cell_id'))
-      )) {
+      for (const row of cells) {
         const cellId = text(row, 'cell_id')
         const expected = cellUnits.get(cellId) ?? 0
         if (expected > integer(row, 'capacity_requests')) {
@@ -6861,38 +7038,78 @@ export class RelayAssignmentStore {
 
   private async lockCellInventory(
     database: RelayDatabase,
-    failIfUnavailable = false
+    mode: CellInventoryLockMode
   ): Promise<SqlRow[]> {
     // Every capacity-changing assignment takes the tiny cell inventory in one
     // order; dynamically locking only the selected target allowed cross-cell cycles.
-    return await database.queryLocked(
+    const rows = await database.queryLocked(
       `SELECT * FROM relay_cells ORDER BY cell_id ASC`,
       [],
-      { failIfUnavailable }
+      cellInventoryLockOptions(mode)
     )
+    return rows
+  }
+
+  // Per-connection paths touch one or two cells. Locking exactly those rows,
+  // in the same ascending order the inventory lock uses (ORDER BY fixes the
+  // row-lock order), keeps them off the fleet-wide lock without a cycle.
+  // The wait policy follows the caller for the same reason the inventory lock's
+  // does: a sweep must not fail terminally on ordinary contention. Hold time is
+  // deliberately not sampled here — the metric tracks the fleet-wide lock these
+  // rows replace, and mixing in short single-row holds would flatter it.
+  private async lockCellRows(
+    database: RelayDatabase,
+    cellIds: string[],
+    mode: CellInventoryLockMode = 'request'
+  ): Promise<SqlRow[]> {
+    const distinct = [...new Set(cellIds)]
+    const { measureHoldMs: _sampled, ...wait } = cellInventoryLockOptions(mode)
+    return await database.queryLocked(
+      `SELECT * FROM relay_cells WHERE cell_id IN (${distinct.map(() => '?').join(', ')})
+       ORDER BY cell_id ASC`,
+      distinct,
+      wait
+    )
+  }
+
+  // Unlocked on purpose: this only names the row to lock next, and the caller
+  // re-checks the pin once the assignment row is held.
+  private async pinnedCellId(
+    database: RelayDatabase,
+    identity: AssignmentIdentity
+  ): Promise<string | undefined> {
+    const row = (
+      await database.query(
+        `SELECT cell_id FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`,
+        [identity.userId, identity.relayHostId]
+      )
+    )[0]
+    return row ? text(row, 'cell_id') : undefined
   }
 
   private async lockGeneralCellInventory(
     database: RelayDatabase,
-    failIfUnavailable = false
+    mode: CellInventoryLockMode
   ): Promise<SqlRow[]> {
-    return await database.queryLocked(
+    const rows = await database.queryLocked(
       `SELECT * FROM relay_cells
        WHERE cell_id IN (
          SELECT cell_id FROM relay_cell_admission WHERE admission_state = 'general'
        )
        ORDER BY cell_id ASC`,
       [],
-      { failIfUnavailable }
+      cellInventoryLockOptions(mode)
     )
+    return rows
   }
 
   private async leastLoadedCell(
     database: RelayDatabase,
-    lockedCells: SqlRow[] | undefined,
+    // Required: the one caller has already locked the inventory it selects from,
+    // and an optional parameter left a second fleet-wide lock reachable here.
+    rows: SqlRow[],
     preferredRegion: RelayRegion
   ): Promise<CellRow | null> {
-    const rows = lockedCells ?? (await this.lockCellInventory(database))
     const regions = new Map(
       (await database.query(`SELECT cell_id, region FROM relay_cell_regions`)).map((row) => [
         text(row, 'cell_id'),
@@ -7507,7 +7724,10 @@ export class RelayAssignmentStore {
     ) {
       throw new Error('activity_lease_shape_mismatch')
     }
-    const cells = await this.lockCellInventory(database)
+    // Why: this recomputes one cell's reservation from its leases, so only that
+    // row needs to be held; the 23-row inventory lock here serialised every
+    // desktop control rebind in the fleet behind every other one.
+    const cellRow = (await this.lockCellRows(database, [cellId]))[0]
     await database.query(
       `DELETE FROM relay_assignment_activity_leases
        WHERE user_id = ? AND relay_host_id = ? AND activity_kind = 'control'
@@ -7528,7 +7748,6 @@ export class RelayAssignmentStore {
         [cellId]
       )
     )[0]!
-    const cellRow = cells.find((cell) => text(cell, 'cell_id') === cellId)
     const cellUnits = integer(cellUnitsRow, 'request_units')
     if (!cellRow) throw new Error('assigned_cell_missing')
     if (cellUnits > integer(cellRow, 'capacity_requests')) {
@@ -7886,6 +8105,23 @@ function isDatabaseLockUnavailable(error: unknown): boolean {
   return error instanceof Error && error.message === 'database_lock_unavailable'
 }
 
+export function cellInventoryLockOptions(mode: CellInventoryLockMode): RelayLockOptions {
+  if (mode === 'nowait') return { failIfUnavailable: true, measureHoldMs: true }
+  if (mode === 'pool-default') return { measureHoldMs: true }
+  return { lockTimeoutMs: CELL_INVENTORY_LOCK_TIMEOUT_MS, measureHoldMs: true }
+}
+
+// Background sweeps take the cell inventory NOWAIT so they never queue ahead of
+// assignment traffic. A skipped candidate is re-derived from durable state on
+// the next tick, so it is ordinary contention, not a sweep failure: one summary
+// line per tick, never an error and never a quarantine.
+function warnSweepCellInventoryBusy(sweep: string, skipped: number): void {
+  if (skipped === 0) return
+  console.warn(
+    JSON.stringify({ event: 'orca_relay_sweep_cell_inventory_busy', sweep, skipped })
+  )
+}
+
 function isDatabaseLockTimeout(error: unknown): boolean {
   return String((error as { code?: unknown }).code) === '55P03'
 }
@@ -7943,7 +8179,7 @@ function regionalRehomeAttempt(row: SqlRow): RegionalRehomeAttempt {
     attemptId: text(row, 'attempt_id'),
     userId: text(row, 'user_id'),
     relayHostId: text(row, 'relay_host_id'),
-    preferredRegion: 'asia-east2',
+    preferredRegion: relayRegion(row, 'preferred_region'),
     sourceCellId: text(row, 'source_cell_id'),
     sourceCellUrl: text(row, 'source_cell_url'),
     sourceCellIncarnation: text(row, 'source_cell_incarnation'),
@@ -7964,6 +8200,7 @@ function regionalRehomeControl(row: SqlRow): RegionalRehomeControl {
     notBefore: integer(row, 'not_before'),
     ratePerMinute: integer(row, 'rate_per_minute'),
     preferenceMaxAgeMs: integer(row, 'preference_max_age_ms'),
+    hostCooldownMs: integer(row, 'host_cooldown_ms'),
     drainGraceMs: integer(row, 'drain_grace_ms')
   }
 }
@@ -8003,10 +8240,9 @@ function regionalRehomeFleetSafetyFromInventory(input: {
     return (
       integer(row, 'enabled') === 1 &&
       input.admission.get(cellId) === 'general' &&
-      (input.regions.get(cellId) === 'asia-east2' ||
-        (input.regions.get(cellId) === RELAY_DEFAULT_REGION &&
-          capability !== undefined &&
-          integer(capability, 'regional_rehome_protocol') >= 1))
+      input.regions.get(cellId) !== undefined &&
+      capability !== undefined &&
+      integer(capability, 'regional_rehome_protocol') >= 1
     )
   })
   const valid = required.flatMap((row) => {
@@ -8073,6 +8309,7 @@ function regionalRehomeFleetSafetyFailure(
 type RegionalRehomeCandidateSkip = {
   reason:
     | 'candidate_stale'
+    | 'host_cooldown'
     | 'source_ineligible'
     | 'source_unclean'
     | 'source_control_inactive'

cloud/apps/relay/src/cell-heartbeat-client.ts

@@ -1,4 +1,5 @@
 import { randomUUID } from 'node:crypto'
+import { RELAY_DEFAULT_REGION } from '@orca-cloud/relay-contract'
 import type { RelayConfig } from './config.js'
 import { googleMetadataIdentityToken } from './google-metadata-identity-token.js'
 import type { RegionalRehomeSafetySnapshot } from './relay-observability.js'
@@ -57,7 +58,7 @@ export function startCellHeartbeat(
             v: 1,
             cellId: config.cellId,
             cellUrl: config.cellUrl,
-            region: config.region ?? 'us-central1',
+            region: config.region ?? RELAY_DEFAULT_REGION,
             cellIncarnation,
             startedAt,
             ready,

cloud/apps/relay/src/cell-inventory-hold-samples.test.ts

@@ -0,0 +1,70 @@
+import { describe, expect, it } from 'vitest'
+import {
+  CellInventoryHoldSamples,
+  emptyCellInventoryHoldCounts
+} from './cell-inventory-hold-samples.js'
+
+// Nearest rank, computed in integer arithmetic so it cannot inherit the float
+// error the implementation's `0.95 * n` could in principle carry.
+function nearestRankP95(sorted: number[]): number {
+  return sorted[Math.ceil((95 * sorted.length) / 100) - 1]!
+}
+
+function samplesOf(values: number[]): CellInventoryHoldSamples {
+  const samples = new CellInventoryHoldSamples()
+  for (const value of values) samples.record(value)
+  return samples
+}
+
+describe('cell inventory hold samples', () => {
+  it('reports nothing before the first hold', () => {
+    expect(new CellInventoryHoldSamples().readCounts()).toEqual(
+      emptyCellInventoryHoldCounts()
+    )
+  })
+
+  // Why: the 500ms bound will be tuned against this percentile, so an off-by-one
+  // here reads as a hold the fleet never had.
+  it('places p95 at the nearest rank for every window size', () => {
+    for (let size = 1; size <= 400; size++) {
+      const values = Array.from({ length: size }, (_, index) => index + 1)
+      const shuffled = [...values].reverse()
+
+      const counts = samplesOf(shuffled).readCounts()
+
+      expect(counts.cellInventoryHoldMsP95).toBe(nearestRankP95(values))
+      expect(counts.cellInventoryHoldMsMax).toBe(size)
+      expect(counts.cellInventoryHolds).toBe(size)
+    }
+  })
+
+  it('never reports a p95 above the max', () => {
+    for (let size = 1; size <= 200; size++) {
+      const counts = samplesOf(Array.from({ length: size }, (_, i) => i + 1)).readCounts()
+
+      expect(counts.cellInventoryHoldMsP95).toBeLessThanOrEqual(counts.cellInventoryHoldMsMax)
+    }
+  })
+
+  it('ignores a hold that is not a finite, non-negative duration', () => {
+    const samples = samplesOf([Number.NaN, Number.POSITIVE_INFINITY, -1])
+
+    expect(samples.readCounts()).toEqual(emptyCellInventoryHoldCounts())
+  })
+
+  // Why: the reservoir is bounded, so a heavy flush interval keeps the most
+  // recent holds rather than growing without limit or freezing on the oldest.
+  it('keeps the most recent holds once the reservoir is full', () => {
+    const counts = samplesOf(Array.from({ length: 2_100 }, (_, index) => index + 1)).readCounts()
+
+    expect(counts.cellInventoryHolds).toBe(2_048)
+    expect(counts.cellInventoryHoldMsMax).toBe(2_100)
+  })
+
+  it('resets the window on consume so each flush reports its own holds', () => {
+    const samples = samplesOf([5, 10])
+
+    expect(samples.consumeCounts().cellInventoryHolds).toBe(2)
+    expect(samples.consumeCounts()).toEqual(emptyCellInventoryHoldCounts())
+  })
+})

cloud/apps/relay/src/cell-inventory-hold-samples.ts

@@ -0,0 +1,46 @@
+// Why: the cell inventory lock is held to COMMIT, and the assignment path runs
+// many statements after taking it. Tuning the request-path wait bound needs the
+// hold distribution, and no runtime metric carried it before this change.
+export type CellInventoryHoldCounts = {
+  cellInventoryHoldMsMax: number
+  cellInventoryHoldMsP95: number
+  cellInventoryHolds: number
+}
+
+// Bounded so a flush interval with heavy assignment traffic cannot grow the array
+// without limit; the reservoir keeps the most recent holds.
+const MAX_SAMPLES = 2_048
+
+export function emptyCellInventoryHoldCounts(): CellInventoryHoldCounts {
+  return { cellInventoryHoldMsMax: 0, cellInventoryHoldMsP95: 0, cellInventoryHolds: 0 }
+}
+
+export class CellInventoryHoldSamples {
+  private samples: number[] = []
+
+  record(holdMs: number): void {
+    if (!Number.isFinite(holdMs) || holdMs < 0) return
+    if (this.samples.length === MAX_SAMPLES) this.samples.shift()
+    this.samples.push(holdMs)
+  }
+
+  consumeCounts(): CellInventoryHoldCounts {
+    const counts = this.readCounts()
+    this.samples = []
+    return counts
+  }
+
+  readCounts(): CellInventoryHoldCounts {
+    if (this.samples.length === 0) return emptyCellInventoryHoldCounts()
+    const sorted = [...this.samples].sort((left, right) => left - right)
+    return {
+      cellInventoryHoldMsMax: round(sorted[sorted.length - 1]!),
+      cellInventoryHoldMsP95: round(sorted[Math.ceil(0.95 * sorted.length) - 1] ?? 0),
+      cellInventoryHolds: sorted.length
+    }
+  }
+}
+
+function round(value: number): number {
+  return Number(value.toFixed(3))
+}

cloud/apps/relay/src/cell-inventory-lock-census.test.ts

@@ -0,0 +1,269 @@
+import { readFileSync } from 'node:fs'
+import { describe, expect, it } from 'vitest'
+import { cellInventoryLockOptions, type CellInventoryLockMode } from './assignment-store.js'
+
+// Which entry points can reach a call site. A site a sweep can enter must never
+// take the bounded wait: its 55P03 becomes a terminal transaction failure, and
+// the incident monitor freezes on a single one.
+type Reachability = 'request' | 'sweep' | 'both' | 'orphan'
+
+// 'caller' is not a CellInventoryLockMode: those sites take the mode threaded
+// from `assign`, which is 'request' for a client and 'pool-default' for the
+// evacuateDeadCells sweep.
+type CensusMode = CellInventoryLockMode | 'caller'
+
+type CensusEntry = { method: string; mode: CensusMode; reach: Reachability }
+
+// Every lockCellInventory / lockGeneralCellInventory call site in
+// assignment-store.ts, in source order. A new site fails this test until it is
+// classified here, which is the point.
+const CENSUS: CensusEntry[] = [
+  // assignStickyOnce is gone from this list: its retry now locks only the row
+  // the host is pinned to (lockCellRows), which is what a sticky refresh
+  // touches. Placement below is the one genuinely fleet-wide decision left.
+  { method: 'assignOnce', mode: 'caller', reach: 'both' },
+  { method: 'assignOnce', mode: 'caller', reach: 'both' },
+  { method: 'assignOnce', mode: 'nowait', reach: 'both' },
+  { method: 'assignOnce', mode: 'nowait', reach: 'both' },
+  { method: 'assignOnce', mode: 'nowait', reach: 'both' },
+  { method: 'refreshDrainMigrationLeasesOnce', mode: 'request', reach: 'request' },
+  // changeActivity, acquireActivity, activateControl and
+  // removeSupersededSameCellControls no longer take the inventory: they lock
+  // only the one or two cell rows they touch, in cell_id order (lockCellRows),
+  // so they cannot cycle with placement's ordered inventory lock, and the
+  // 23-row lock there had serialised every reconnect in the fleet behind every
+  // other one.
+  { method: 'startEvacuation', mode: 'request', reach: 'request' },
+  { method: 'completeEvacuationFromDeadSourceOnce', mode: 'request', reach: 'request' },
+  { method: 'completeEvacuationFromDeadSourceOnce', mode: 'nowait', reach: 'request' },
+  { method: 'supersedeRegisteredEvacuationOnce', mode: 'request', reach: 'request' },
+  { method: 'supersedeRegisteredEvacuationOnce', mode: 'nowait', reach: 'request' },
+  { method: 'prepareRegisteredCellSupersession', mode: 'request', reach: 'request' },
+  { method: 'prepareRegisteredCellSupersession', mode: 'request', reach: 'request' },
+  { method: 'completeEvacuation', mode: 'nowait', reach: 'both' },
+  { method: 'completeEvacuation', mode: 'pool-default', reach: 'both' },
+  { method: 'rebalanceDormant', mode: 'request', reach: 'request' },
+  { method: 'startRegionalRehomeCandidate', mode: 'nowait', reach: 'sweep' },
+  { method: 'lockedRegionalRehomeFleetSafety', mode: 'nowait', reach: 'sweep' },
+  { method: 'completeRegionalRehomeCandidate', mode: 'nowait', reach: 'sweep' },
+  { method: 'abortExpiredRegionalRehomes', mode: 'nowait', reach: 'sweep' },
+  { method: 'abortExpiredEvacuations', mode: 'nowait', reach: 'sweep' },
+  { method: 'abortExpiredEvacuations', mode: 'nowait', reach: 'sweep' },
+  { method: 'releaseExpiredActivityLeases', mode: 'nowait', reach: 'sweep' },
+  { method: 'releaseExpiredActivity', mode: 'nowait', reach: 'sweep' },
+  // reconcileReservationAccounting and leastLoadedCell are gone too: the first
+  // repairs exactly two cells' counters and now holds only those rows, and the
+  // second selects from the inventory its single caller has already locked.
+]
+
+// Every inline `FROM relay_cells ... FOR UPDATE` outside the named lock helpers,
+// in source order: whole-table locks in reconciliation and sticky placement,
+// and single-row locks for a cell the method is already scoped to (heartbeat,
+// fence, drain generation, configuration, or a reservation adjust that runs
+// under a lock its caller already holds). A new inline lock fails the census
+// below until it is listed here; per-connection paths that touch more than one
+// cell go through lockCellRows so the order is fixed.
+const NAMED_LOCK_HELPERS = ['lockCellInventory', 'lockGeneralCellInventory', 'lockCellRows']
+
+const INLINE_CELL_LOCK_SITES = [
+  'reconcileCellsWithOptions',
+  'assignStickyOnce',
+  'recordCellHeartbeat',
+  'attestCellFence',
+  'adoptLegacyCellFence',
+  'commitLegacyCellFenceAdoption',
+  'prepareCellFenceAttempt',
+  'attestCellFenceAttempt',
+  'attestCellFenceAttempt',
+  'configureCell',
+  'assertDrainCellGeneration',
+  'adjustCellReservation'
+]
+
+// The background sweeps, and nothing else. A method reachable from one of these
+// can be entered by a sweep tick, whatever else can also enter it. Both lists are
+// read from source, so a new sweep step or a new route widens the derivation here
+// instead of silently widening what a bounded wait can be entered from.
+const SWEEP_ENTRY_FILES = ['./assignment-cleanup-steps.ts', './regional-rehome-worker.ts']
+const REQUEST_ENTRY_FILES = [
+  './app.ts',
+  './relay-server.ts',
+  './host-session-registry.ts',
+  './cell-admission-startup.ts'
+]
+
+const DECLARATION = /^ {2}(?:private |public )?(?:static )?(?:async )?([A-Za-z_][\w]*)[(<]/
+
+function storeSource(): string[] {
+  return readFileSync(new URL('./assignment-store.ts', import.meta.url), 'utf8').split('\n')
+}
+
+function entryPoints(files: string[]): string[] {
+  return files.flatMap((file) =>
+    [
+      ...readFileSync(new URL(file, import.meta.url), 'utf8').matchAll(
+        /assignments\.([A-Za-z_][\w]*)\(/g
+      )
+    ].map((call) => call[1]!)
+  )
+}
+
+// Same-class call graph: store methods only ever reach each other through `this.`.
+function storeCallGraph(lines: string[]): Map<string, Set<string>> {
+  const bounds: { name: string; start: number }[] = []
+  lines.forEach((line, index) => {
+    const declaration = DECLARATION.exec(line)
+    if (declaration) bounds.push({ name: declaration[1]!, start: index })
+  })
+  const callees = new Map<string, Set<string>>()
+  bounds.forEach((method, index) => {
+    const end = bounds[index + 1]?.start ?? lines.length
+    const names = callees.get(method.name) ?? new Set<string>()
+    for (const call of lines.slice(method.start, end).join('\n').matchAll(
+      /this\.([A-Za-z_][\w]*)\s*\(/g
+    )) {
+      names.add(call[1]!)
+    }
+    callees.set(method.name, names)
+  })
+  return callees
+}
+
+function closure(callees: Map<string, Set<string>>, roots: string[]): Set<string> {
+  const reached = new Set<string>()
+  const pending = [...roots]
+  while (pending.length > 0) {
+    const name = pending.pop()!
+    if (reached.has(name)) continue
+    reached.add(name)
+    for (const callee of callees.get(name) ?? []) if (!reached.has(callee)) pending.push(callee)
+  }
+  return reached
+}
+
+// Why: a hand-written reachability column is a claim, not a check. Derive both
+// directions, so a new sweep edge into a bounded site fails here instead of in
+// production, and so 'sweep' and 'both' stop being asserted by hand.
+function derivedReachability(lines: string[]): (method: string) => Reachability {
+  const callees = storeCallGraph(lines)
+  const sweep = closure(callees, entryPoints(SWEEP_ENTRY_FILES))
+  const request = closure(callees, entryPoints(REQUEST_ENTRY_FILES))
+  return (method) =>
+    sweep.has(method)
+      ? request.has(method)
+        ? 'both'
+        : 'sweep'
+      : request.has(method)
+        ? 'request'
+        : 'orphan'
+}
+
+function readCallSites(): { method: string; mode: CensusMode }[] {
+  const sites: { method: string; mode: CensusMode }[] = []
+  let method = '<module>'
+  for (const line of storeSource()) {
+    const declaration = DECLARATION.exec(line)
+    if (declaration) method = declaration[1]!
+    if (/private async lock(General)?CellInventory\(/.test(line)) continue
+    const call = /lock(?:General)?CellInventory\(\s*\w+\s*,\s*(?:'([a-z-]+)'|(\w+))\s*\)/.exec(line)
+    if (!call) continue
+    sites.push({ method, mode: (call[1] ?? 'caller') as CensusMode })
+  }
+  return sites
+}
+
+describe('cell inventory lock call-site census', () => {
+  it('classifies every call site exactly as recorded', () => {
+    expect(readCallSites()).toEqual(
+      CENSUS.map(({ method, mode }) => ({ method, mode }))
+    )
+  })
+
+  // Why: the census only sees lockCellInventory calls, so a hand-written
+  // `relay_cells ... FOR UPDATE` would escape classification entirely.
+  it('routes every relay_cells row lock through a named lock helper', () => {
+    const lines = storeSource()
+    const rawSites: string[] = []
+    // Whole statements, not a fixed window: a wide column list or a raw
+    // FOR UPDATE inside query() must not slip past.
+    const source = lines.join('\n')
+    const bounds: { name: string; start: number }[] = []
+    lines.forEach((line, index) => {
+      const declaration = DECLARATION.exec(line)
+      if (declaration) bounds.push({ name: declaration[1]!, start: index })
+    })
+    const methodAt = (offset: number): string => {
+      const lineIndex = source.slice(0, offset).split('\n').length - 1
+      let name = '<module>'
+      for (const bound of bounds) if (bound.start <= lineIndex) name = bound.name
+      return name
+    }
+    const tick = String.fromCharCode(96)
+    const statementCall = new RegExp(
+      '\\.(queryLocked|query)\\(\\s*' + tick + '([^' + tick + ']*)' + tick,
+      'g'
+    )
+    for (const call of source.matchAll(statementCall)) {
+      const statement = call[2]!
+      if (!/\bFROM\s+relay_cells\b/.test(statement)) continue
+      const locks = call[1] === 'queryLocked' || /\bFOR\s+UPDATE\b/.test(statement)
+      if (!locks) continue
+      const method = methodAt(call.index)
+      if (NAMED_LOCK_HELPERS.includes(method)) continue
+      rawSites.push(method)
+    }
+    expect(rawSites).toEqual(INLINE_CELL_LOCK_SITES)
+  })
+
+  it('leaves no call site taking the inventory without naming a mode', () => {
+    const source = readFileSync(new URL('./assignment-store.ts', import.meta.url), 'utf8')
+    const unclassified = source
+      .split('\n')
+      .filter((line) => /lock(?:General)?CellInventory\(\s*\w+\s*\)/.test(line))
+      .filter((line) => !line.includes('private async'))
+
+    expect(unclassified).toEqual([])
+  })
+
+  it('derives the same reachability the census claims', () => {
+    const reachOf = derivedReachability(storeSource())
+
+    expect(readCallSites().map(({ method }) => reachOf(method))).toEqual(
+      CENSUS.map((entry) => entry.reach)
+    )
+  })
+
+  // Why: this is the whole point of the classification. A shorter wait on a
+  // sweep-reachable site turns contention into a terminal transaction failure
+  // that counts against the incident gate's relayPostgresRetryExhausted bar.
+  // Why: the hold distribution is what the 500ms bound will be tuned against, so
+  // a mode that stops asking for it goes unmeasured in exactly the lane that
+  // matters. Nothing else in the suite reads the pool-default branch.
+  it('measures the hold in every lock mode', () => {
+    const modes: CellInventoryLockMode[] = ['request', 'nowait', 'pool-default']
+
+    expect(modes.map((mode) => cellInventoryLockOptions(mode).measureHoldMs)).toEqual([
+      true,
+      true,
+      true
+    ])
+  })
+
+  it('never puts a sweep-reachable site on the bounded wait', () => {
+    const reachOf = derivedReachability(storeSource())
+    const bounded = readCallSites().filter(
+      (site) => site.mode === 'request' && ['sweep', 'both'].includes(reachOf(site.method))
+    )
+
+    expect(bounded).toEqual([])
+  })
+
+  it('routes every sweep-only site to NOWAIT so it can skip the tick', () => {
+    const reachOf = derivedReachability(storeSource())
+    const queueing = readCallSites().filter(
+      (site) => reachOf(site.method) === 'sweep' && site.mode !== 'nowait'
+    )
+
+    expect(queueing).toEqual([])
+  })
+})

cloud/apps/relay/src/cell-inventory-lock-contention.test.ts

@@ -0,0 +1,541 @@
+import { readFileSync } from 'node:fs'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+
+const fakes = vi.hoisted(() => ({
+  statements: [] as string[],
+  query: vi.fn(async (sql: string) => {
+    fakes.statements.push(sql)
+    return { rows: [], rowCount: 0 }
+  }),
+  release: vi.fn(),
+  end: vi.fn(async () => undefined)
+}))
+
+vi.mock('pg', () => ({
+  default: {
+    Pool: class {
+      totalCount = 1
+      idleCount = 1
+      waitingCount = 0
+      end = fakes.end
+      on = vi.fn()
+      connect = vi.fn(async () => ({ query: fakes.query, release: fakes.release }))
+    }
+  }
+}))
+
+const { CELL_INVENTORY_LOCK_TIMEOUT_MS, RelayAssignmentStore } = await import(
+  './assignment-store.js'
+)
+const { consumeRelayCellInventoryHold, openInMemoryRelayDatabase, openRelayDatabase, POSTGRES_LOCK_TIMEOUT_MS } =
+  await import('./database.js')
+const RESTORE = `SET LOCAL lock_timeout = '${POSTGRES_LOCK_TIMEOUT_MS}ms'`
+type RelayDatabase = import('./database.js').RelayDatabase
+type RelayLockOptions = import('./database.js').RelayLockOptions
+type RelayTransactionOptions = import('./database.js').RelayTransactionOptions
+type SqlRow = import('./database.js').SqlRow
+
+const CELL_INVENTORY_SQL = 'SELECT * FROM relay_cells ORDER BY cell_id ASC'
+
+// The assignment path locks the general-admission subset; both forms are the
+// same ordered scan of the same 23-row table and share its lock queue.
+function locksCellInventory(sql: string): boolean {
+  return sql.trim().startsWith('SELECT * FROM relay_cells') && sql.includes('ORDER BY cell_id ASC')
+}
+const CELLS = [
+  { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+  { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
+]
+const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+
+async function openFakePostgres(): Promise<RelayDatabase> {
+  const database = await openRelayDatabase({
+    databaseUrl: 'postgresql://relay:secret@127.0.0.1:5432/relay',
+    dataDir: './unused'
+  })
+  fakes.statements.length = 0
+  return database
+}
+
+afterEach(() => {
+  fakes.statements.length = 0
+  fakes.query.mockReset()
+  fakes.query.mockImplementation(async (sql: string) => {
+    fakes.statements.push(sql)
+    return { rows: [], rowCount: 0 }
+  })
+})
+
+describe('bounded cell-inventory lock wait', () => {
+  // Why: a bound at or above the pool default would fence nothing, and one far
+  // below the hold time would convert ordinary contention into terminal failures.
+  it('keeps the request bound strictly inside the pool default', () => {
+    expect(CELL_INVENTORY_LOCK_TIMEOUT_MS).toBe(500)
+    expect(CELL_INVENTORY_LOCK_TIMEOUT_MS).toBeLessThan(POSTGRES_LOCK_TIMEOUT_MS)
+  })
+
+  // Why: SET LOCAL lasts to COMMIT. Left in place it would govern every later
+  // locked statement in the transaction and misattribute their 55P03s.
+  it('restores the pool default before the next statement in the transaction', async () => {
+    const database = await openFakePostgres()
+
+    await database.transaction(async (transaction) => {
+      await transaction.queryLocked(CELL_INVENTORY_SQL, [], { lockTimeoutMs: 150 })
+      await transaction.queryLocked('SELECT * FROM relay_assignments', [])
+    })
+
+    expect(fakes.statements).toEqual([
+      'BEGIN',
+      "SET LOCAL lock_timeout = '150ms'",
+      `${CELL_INVENTORY_SQL} FOR UPDATE`,
+      RESTORE,
+      'SELECT * FROM relay_assignments FOR UPDATE',
+      'COMMIT'
+    ])
+    await database.close()
+  })
+
+  it('restores the pool default when the bounded lock itself times out', async () => {
+    const database = await openFakePostgres()
+    fakes.query.mockImplementation(async (sql: string) => {
+      fakes.statements.push(sql)
+      if (sql.includes('FOR UPDATE')) {
+        throw Object.assign(new Error('lock timeout'), { code: '55P03' })
+      }
+      return { rows: [], rowCount: 0 }
+    })
+
+    await expect(
+      database.transaction(async (transaction) => {
+        await transaction.queryLocked(CELL_INVENTORY_SQL, [], { lockTimeoutMs: 150 })
+      })
+    ).rejects.toMatchObject({ code: '55P03' })
+
+    // The retry wrapper makes three attempts; each one must leave the default back.
+    expect(fakes.statements.filter((sql) => sql.startsWith('SET LOCAL'))).toEqual(
+      Array.from({ length: 3 }, () => ["SET LOCAL lock_timeout = '150ms'", RESTORE]).flat()
+    )
+    await database.close()
+  })
+
+  it('rejects a lock bound that is not a positive whole number of milliseconds', async () => {
+    const database = await openFakePostgres()
+
+    for (const lockTimeoutMs of [0, -1, 1.5, Number.NaN]) {
+      await expect(
+        database.transaction(
+          async (transaction) =>
+            await transaction.queryLocked(CELL_INVENTORY_SQL, [], { lockTimeoutMs })
+        )
+      ).rejects.toThrow('invalid_lock_timeout')
+    }
+    await database.close()
+  })
+
+  it('skips the timeout for a NOWAIT lock, which never queues', async () => {
+    const database = await openFakePostgres()
+
+    await database.transaction(async (transaction) => {
+      await transaction.queryLocked(CELL_INVENTORY_SQL, [], {
+        failIfUnavailable: true,
+        lockTimeoutMs: 150
+      })
+    })
+
+    expect(fakes.statements.filter((sql) => sql.startsWith('SET LOCAL'))).toEqual([])
+    await database.close()
+  })
+
+  it('skips the timeout outside a transaction, where SET LOCAL cannot survive', async () => {
+    const database = await openFakePostgres()
+
+    await database.queryLocked(CELL_INVENTORY_SQL, [], { lockTimeoutMs: 150 })
+
+    expect(fakes.statements).toEqual([`${CELL_INVENTORY_SQL} FOR UPDATE`])
+    await database.close()
+  })
+
+  it('ignores the timeout on SQLite, which has no SET LOCAL', async () => {
+    const database = await openInMemoryRelayDatabase()
+
+    const rows = await database.transaction(
+      async (transaction) =>
+        await transaction.queryLocked(CELL_INVENTORY_SQL, [], { lockTimeoutMs: 150 })
+    )
+
+    expect(rows).toEqual([])
+    await database.close()
+  })
+
+  // Why: testing the helper alone would pass with the store still queueing for
+  // the pool's one-second default.
+  // Why: testing the helper alone would pass with the request path still queueing
+  // for the pool's full second.
+  it('never lets a request path take the unbounded wait', async () => {
+    const database = await openInMemoryRelayDatabase()
+    const probe = new InventoryLockProbe(database)
+    const store = new RelayAssignmentStore(probe, () => 1_000)
+    await store.reconcileCells(CELLS)
+    probe.inventoryLocks.length = 0
+
+    // Assignment takes the general-admission subset; evacuation takes them all.
+    await store.assign(identity)
+    const generalLocks = probe.inventoryLocks.length
+    await store.startEvacuation(identity, 'cell-b')
+
+    expect(generalLocks).toBeGreaterThan(0)
+    expect(probe.inventoryLocks.length).toBeGreaterThan(generalLocks)
+    for (const options of probe.inventoryLocks) {
+      const bounded = options?.lockTimeoutMs === CELL_INVENTORY_LOCK_TIMEOUT_MS
+      expect(bounded || options?.failIfUnavailable === true).toBe(true)
+    }
+    await database.close()
+  })
+
+  // Why: evacuateDeadCells re-enters placement from a sweep. A 55P03 there would
+  // be reported as a terminal sweep failure and freeze the incident gate.
+  it('keeps the pool default when a sweep re-enters placement', async () => {
+    const requestModes = await recordAssignInventoryModes(async (store) => {
+      await store.assign(identity)
+    })
+    const sweepModes = await recordAssignInventoryModes(async (store) => {
+      await store.assign(identity, undefined, undefined, 'pool-default')
+    })
+
+    // The inventory-first retry is the lane that carries the caller's mode.
+    expect(requestModes).toContain(CELL_INVENTORY_LOCK_TIMEOUT_MS)
+    expect(sweepModes).not.toContain(CELL_INVENTORY_LOCK_TIMEOUT_MS)
+    expect(sweepModes.filter((mode) => mode === 'nowait').length).toBe(
+      requestModes.filter((mode) => mode === 'nowait').length
+    )
+  })
+
+  it('sends the sweep that re-enters placement down the unbounded lane', async () => {
+    const database = await openInMemoryRelayDatabase()
+    const probe = new InventoryLockProbe(database)
+    let now = 1_000
+    const store = new RelayAssignmentStore(probe, () => now, {
+      requireLiveCells: true,
+      heartbeatTtlMs: 45_000
+    })
+    await store.reconcileCells(CELLS)
+    for (const cell of CELLS) {
+      await store.recordCellHeartbeat({
+        cellId: cell.id,
+        cellUrl: cell.url,
+        cellIncarnation: `1111111${cell.id.slice(-1)}-1111-4111-8111-111111111111`,
+        startedAt: 50,
+        ready: true,
+        observedRequests: 0
+      })
+    }
+    await store.assign(identity)
+    // Let every heartbeat lapse so the sweep sees the assigned cell as dead.
+    now += 45_001
+    probe.inventoryLocks.length = 0
+    probe.failActivityLockOnce = true
+
+    await store.evacuateDeadCells()
+
+    expect(probe.inventoryLocks).not.toEqual([])
+    for (const options of probe.inventoryLocks) {
+      expect(options?.lockTimeoutMs).toBeUndefined()
+    }
+    await database.close()
+  })
+
+  // Why: the SQLite hold test cannot reach PostgresDatabase.transaction, which is
+  // the only path production ever takes.
+  it('records the hold on the PostgreSQL transaction path', async () => {
+    const database = await openFakePostgres()
+
+    await database.transaction(async (transaction) => {
+      await transaction.queryLocked(CELL_INVENTORY_SQL, [], {
+        lockTimeoutMs: 150,
+        measureHoldMs: true
+      })
+    })
+
+    expect(consumeRelayCellInventoryHold(database).cellInventoryHolds).toBe(1)
+    await database.close()
+  })
+
+  it('records no hold for a PostgreSQL transaction that took no measured lock', async () => {
+    const database = await openFakePostgres()
+
+    await database.transaction(async (transaction) => {
+      await transaction.queryLocked(CELL_INVENTORY_SQL, [], { lockTimeoutMs: 150 })
+    })
+
+    expect(consumeRelayCellInventoryHold(database).cellInventoryHolds).toBe(0)
+    await database.close()
+  })
+
+  // Why: index.ts boots a server on import, so its wiring can only be read. An
+  // unspread hold metric is invisible: the flush simply omits the fields.
+  it('spreads the hold counts into the runtime metrics flush', () => {
+    const source = readFileSync(new URL('./index.ts', import.meta.url), 'utf8')
+    const flush = /observability\.start\(\(\) => \(\{([^}]*)\}\)\)/.exec(source)
+
+    expect(flush?.[1]).toContain('...consumeRelayCellInventoryHold(database)')
+  })
+
+  // Why: 500ms is a first value, not a measurement. Tuning it needs the hold
+  // distribution, which no runtime metric carried.
+  it('reports how long the inventory lock was held to COMMIT', async () => {
+    const database = await openInMemoryRelayDatabase()
+    const store = new RelayAssignmentStore(database, () => 1_000)
+    await store.reconcileCells(CELLS)
+    consumeRelayCellInventoryHold(database)
+
+    await store.assign(identity)
+
+    const counts = consumeRelayCellInventoryHold(database)
+    expect(counts.cellInventoryHolds).toBeGreaterThan(0)
+    expect(counts.cellInventoryHoldMsMax).toBeGreaterThanOrEqual(counts.cellInventoryHoldMsP95)
+    expect(counts.cellInventoryHoldMsMax).toBeGreaterThan(0)
+    // Consuming resets the window so the next flush reports its own holds.
+    expect(consumeRelayCellInventoryHold(database).cellInventoryHolds).toBe(0)
+    await database.close()
+  })
+})
+
+// Why: exhausted transactions count against the incident monitor's bounded bar.
+// A sweep that steps aside must not spend the retry budget or report a terminal failure.
+describe('sweep lock skips stay off the transaction retry counters', () => {
+  it('reports neither a retry nor an exhaustion when NOWAIT finds the lock held', async () => {
+    const database = await openFakePostgres()
+    fakes.query.mockImplementation(async (sql: string) => {
+      fakes.statements.push(sql)
+      if (sql.includes('FOR UPDATE NOWAIT')) {
+        throw Object.assign(new Error('could not obtain lock'), { code: '55P03' })
+      }
+      return { rows: [], rowCount: 0 }
+    })
+    const events: string[] = []
+    const warn = vi.spyOn(console, 'warn').mockImplementation((line: unknown) => {
+      try {
+        events.push(String((JSON.parse(line as string) as { event?: unknown }).event))
+      } catch {
+        // non-JSON lines are not transaction telemetry
+      }
+    })
+
+    try {
+      await expect(
+        database.transaction(async (transaction) => {
+          await transaction.queryLocked(CELL_INVENTORY_SQL, [], { failIfUnavailable: true })
+        })
+      ).rejects.toThrow('database_lock_unavailable')
+    } finally {
+      warn.mockRestore()
+    }
+
+    expect(events).not.toContain('orca_relay_postgres_transaction_retry')
+    expect(events).not.toContain('orca_relay_postgres_transaction_exhausted')
+    expect(fakes.statements.filter((sql) => sql === 'BEGIN')).toHaveLength(1)
+    await database.close()
+  })
+})
+
+describe('background sweeps skip a contended cell inventory', () => {
+  it('takes the inventory NOWAIT and skips the tick instead of queueing', async () => {
+    const database = await openInMemoryRelayDatabase()
+    const probe = new InventoryLockProbe(database)
+    let now = 1_000
+    const store = new RelayAssignmentStore(probe, () => now)
+    await store.reconcileCells(CELLS)
+    const assignment = await store.assign(identity)
+    await store.activateControl(identity, {
+      cellId: assignment.cellId,
+      assignmentEpoch: assignment.assignmentEpoch,
+      generation: 1
+    })
+    await store.startEvacuation(identity, 'cell-b')
+    now += 24 * 60 * 60_000
+    probe.inventoryLocks.length = 0
+    probe.failNoWait = true
+    const warnings = collectWarnings('orca_relay_sweep_cell_inventory_busy')
+
+    let aborted: number
+    try {
+      aborted = await store.abortExpiredEvacuations()
+    } finally {
+      warnings.restore()
+    }
+
+    expect(aborted).toBe(0)
+    expect(probe.inventoryLocks).not.toEqual([])
+    expect(probe.inventoryLocks.every((options) => options?.failIfUnavailable === true)).toBe(
+      true
+    )
+    expect(warnings.entries).toEqual([
+      { event: 'orca_relay_sweep_cell_inventory_busy', sweep: 'abort-expired-evacuations', skipped: 1 }
+    ])
+    await database.close()
+  })
+
+  // Why: a summary line on every quiet tick would bury the contended ones.
+  it('says nothing on a tick that skipped no candidate', async () => {
+    const database = await openInMemoryRelayDatabase()
+    let now = 1_000
+    const store = new RelayAssignmentStore(database, () => now)
+    await store.reconcileCells(CELLS)
+    const assignment = await store.assign(identity)
+    await store.activateControl(identity, {
+      cellId: assignment.cellId,
+      assignmentEpoch: assignment.assignmentEpoch,
+      generation: 1
+    })
+    await store.startEvacuation(identity, 'cell-b')
+    now += 24 * 60 * 60_000
+    const warnings = collectWarnings('orca_relay_sweep_cell_inventory_busy')
+
+    let aborted: number
+    try {
+      aborted = await store.abortExpiredEvacuations()
+    } finally {
+      warnings.restore()
+    }
+
+    expect(aborted).toBe(1)
+    expect(warnings.entries).toEqual([])
+    await database.close()
+  })
+
+  it('still aborts the expired evacuation once the inventory is free', async () => {
+    const database = await openInMemoryRelayDatabase()
+    const probe = new InventoryLockProbe(database)
+    let now = 1_000
+    const store = new RelayAssignmentStore(probe, () => now)
+    await store.reconcileCells(CELLS)
+    const assignment = await store.assign(identity)
+    await store.activateControl(identity, {
+      cellId: assignment.cellId,
+      assignmentEpoch: assignment.assignmentEpoch,
+      generation: 1
+    })
+    await store.startEvacuation(identity, 'cell-b')
+    now += 24 * 60 * 60_000
+
+    expect(await store.abortExpiredEvacuations()).toBe(1)
+    await database.close()
+  })
+})
+
+// Returns each inventory lock the run took, as its bound or 'nowait'.
+async function recordAssignInventoryModes(
+  drive: (store: InstanceType<typeof RelayAssignmentStore>) => Promise<void>
+): Promise<(number | 'nowait' | 'pool-default')[]> {
+  const database = await openInMemoryRelayDatabase()
+  const probe = new InventoryLockProbe(database)
+  const store = new RelayAssignmentStore(probe, () => 1_000)
+  await store.reconcileCells(CELLS)
+  probe.inventoryLocks.length = 0
+  probe.failActivityLockOnce = true
+  await drive(store)
+  await database.close()
+  return probe.inventoryLocks.map((options) =>
+    options?.failIfUnavailable ? 'nowait' : (options?.lockTimeoutMs ?? 'pool-default')
+  )
+}
+
+function collectWarnings(event: string) {
+  const entries: Record<string, unknown>[] = []
+  const original = console.warn
+  console.warn = (line: unknown, ...rest: unknown[]) => {
+    try {
+      const parsed = JSON.parse(line as string) as Record<string, unknown>
+      if (parsed.event === event) return void entries.push(parsed)
+    } catch {
+      // fall through to the real console for non-JSON lines
+    }
+    original(line, ...rest)
+  }
+  return { entries, restore: () => (console.warn = original) }
+}
+
+const ACTIVITY_LEASE_SQL = 'SELECT * FROM relay_assignment_activity_leases'
+
+class InventoryLockProbe implements RelayDatabase {
+  readonly inventoryLocks: (RelayLockOptions | undefined)[] = []
+  failNoWait = false
+  // Forces the next assign attempt down its inventory-first retry, the only lane
+  // that reaches the threaded lock mode.
+  failActivityLockOnce = false
+
+  constructor(private readonly delegate: RelayDatabase) {}
+
+  async query(sql: string, params?: unknown[]): Promise<SqlRow[]> {
+    return await this.delegate.query(sql, params)
+  }
+
+  async queryLocked(
+    sql: string,
+    params?: unknown[],
+    options?: RelayLockOptions
+  ): Promise<SqlRow[]> {
+    if (locksCellInventory(sql)) {
+      this.inventoryLocks.push(options)
+      if (this.failNoWait && options?.failIfUnavailable) {
+        throw new Error('database_lock_unavailable')
+      }
+    }
+    if (this.failActivityLockOnce && sql.trim().startsWith(ACTIVITY_LEASE_SQL) && options?.failIfUnavailable) {
+      this.failActivityLockOnce = false
+      throw new Error('database_lock_unavailable')
+    }
+    return await this.delegate.queryLocked(sql, params, options)
+  }
+
+  async transaction<T>(
+    operation: (transaction: RelayDatabase) => Promise<T>,
+    options?: RelayTransactionOptions
+  ): Promise<T> {
+    return await this.delegate.transaction(
+      async (transaction) => await operation(new InventoryLockProbeTransaction(transaction, this)),
+      options
+    )
+  }
+
+  async close(): Promise<void> {}
+}
+
+class InventoryLockProbeTransaction implements RelayDatabase {
+  constructor(
+    private readonly delegate: RelayDatabase,
+    private readonly probe: InventoryLockProbe
+  ) {}
+
+  async query(sql: string, params?: unknown[]): Promise<SqlRow[]> {
+    return await this.delegate.query(sql, params)
+  }
+
+  async queryLocked(
+    sql: string,
+    params?: unknown[],
+    options?: RelayLockOptions
+  ): Promise<SqlRow[]> {
+    if (locksCellInventory(sql)) {
+      this.probe.inventoryLocks.push(options)
+      if (this.probe.failNoWait && options?.failIfUnavailable) {
+        throw new Error('database_lock_unavailable')
+      }
+    }
+    if (
+      this.probe.failActivityLockOnce &&
+      sql.trim().startsWith(ACTIVITY_LEASE_SQL) &&
+      options?.failIfUnavailable
+    ) {
+      this.probe.failActivityLockOnce = false
+      throw new Error('database_lock_unavailable')
+    }
+    return await this.delegate.queryLocked(sql, params, options)
+  }
+
+  async transaction<T>(operation: (transaction: RelayDatabase) => Promise<T>): Promise<T> {
+    return await operation(this)
+  }
+
+  async close(): Promise<void> {}
+}

cloud/apps/relay/src/cell-inventory-per-cell-locking-postgres.test.ts

@@ -0,0 +1,206 @@
+import { afterAll, beforeAll, describe, expect, it } from 'vitest'
+import { RelayAssignmentStore } from './assignment-store.js'
+import { openRelayDatabase, type RelayDatabase } from './database.js'
+
+const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
+const describePostgres = databaseUrl ? describe : describe.skip
+
+// Sorted ascending, and the host is pinned to the LAST id on purpose: the
+// fleet-wide lock is one ordered scan, so it holds every earlier row while it
+// waits on the pinned one. Pinning to the first id would make the two locking
+// models indistinguishable.
+const cells = ['a', 'b', 'c'].map((suffix) => ({
+  id: `percell-postgres-${suffix}`,
+  url: `https://percell-postgres-${suffix}.example.com`,
+  capacityRequests: 1_000,
+  connectionHardCap: 600 as const,
+  connectionUnobservedBound: 50
+}))
+const [cellA, cellB, cellC] = cells as [(typeof cells)[0], (typeof cells)[0], (typeof cells)[0]]
+const identity = { userId: 'percell-postgres-user', relayHostId: 'percellhost00001' }
+
+function heartbeat(cell: (typeof cells)[number]) {
+  return {
+    cellId: cell.id,
+    cellUrl: cell.url,
+    cellIncarnation: '11111111-1111-4111-8111-111111111111',
+    startedAt: 50,
+    ready: true,
+    observedRequests: 0,
+    totalConnections: 0,
+    inFlightConnections: 0,
+    reservedConnectionUnits: 0,
+    enforcedConnectionUnits: 0,
+    connectionInclusionWatermark: 1,
+    connectionHardCap: 600 as const,
+    connectionUnobservedBound: 50
+  }
+}
+
+describePostgres('PostgreSQL per-cell inventory locking', () => {
+  const databases: RelayDatabase[] = []
+
+  beforeAll(async () => {
+    for (let index = 0; index < 3; index++) {
+      databases.push(await openRelayDatabase({ databaseUrl, dataDir: '' }))
+    }
+  })
+
+  async function removeTestRows(database: RelayDatabase): Promise<void> {
+    await database.query(
+      `DELETE FROM relay_control_connection_reservations WHERE user_id LIKE 'percell-postgres-%'`
+    )
+    for (const table of [
+      'relay_assignment_activity_leases',
+      'relay_post_drain_migration_pins',
+      'relay_assignment_migration_incarnations',
+      'relay_assignment_migrations',
+      'relay_assignment_region_preferences',
+      'relay_assignments'
+    ]) {
+      await database.query(`DELETE FROM ${table} WHERE user_id LIKE 'percell-postgres-%'`)
+    }
+    for (const cell of cells) {
+      for (const table of [
+        'relay_cell_connection_snapshots',
+        'relay_cell_connection_runtime',
+        'relay_cell_connection_limits',
+        'relay_cell_runtime',
+        'relay_cells'
+      ]) {
+        await database.query(`DELETE FROM ${table} WHERE cell_id = ?`, [cell.id])
+      }
+    }
+  }
+
+  afterAll(async () => {
+    if (databases[0]) await removeTestRows(databases[0])
+    for (const connection of databases) await connection.close()
+  })
+
+  async function pinHostToLastCell(store: RelayAssignmentStore): Promise<void> {
+    await store.reconcileCells(cells)
+    for (const cell of cells) await store.recordCellHeartbeat(heartbeat(cell))
+    await store.setCellEnabled(cellA.id, false)
+    await store.setCellEnabled(cellB.id, false)
+    const assignment = await store.assign(identity)
+    expect(assignment.cellId).toBe(cellC.id)
+    await store.setCellEnabled(cellA.id, true)
+    await store.setCellEnabled(cellB.id, true)
+  }
+
+  async function lockWaiterAppeared(database: RelayDatabase): Promise<boolean> {
+    const deadline = Date.now() + 4_000
+    while (Date.now() < deadline) {
+      const rows = await database.query(
+        `SELECT count(*) AS waiting FROM pg_stat_activity
+         WHERE datname = current_database() AND wait_event_type = 'Lock'`
+      )
+      if (Number(rows[0]!.waiting) > 0) return true
+      await new Promise((resolve) => setTimeout(resolve, 10))
+    }
+    return false
+  }
+
+  // Why: a sticky refresh whose first NOWAIT probe loses retries by taking a
+  // cell row before the assignment row. That retry used to take the whole
+  // inventory, so one busy cell stalled every other cell's reconnects.
+  it('waits only on the pinned cell row while refreshing a sticky assignment', async () => {
+    await removeTestRows(databases[0]!)
+    const store = new RelayAssignmentStore(databases[0]!, () => 100)
+    await pinHostToLastCell(store)
+    // A host whose control lease was already reaped still holds its pin; that
+    // is the shape that reaches the cell-row probe instead of touchAssignment.
+    await databases[0]!.query(
+      `DELETE FROM relay_assignment_activity_leases WHERE user_id = ?`,
+      [identity.userId]
+    )
+
+    let releaseRow!: () => void
+    const rowReleased = new Promise<void>((resolve) => {
+      releaseRow = resolve
+    })
+    let rowHeld!: () => void
+    const rowHeldPromise = new Promise<void>((resolve) => {
+      rowHeld = resolve
+    })
+    const holder = databases[1]!.transaction(async (transaction) => {
+      await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [cellC.id])
+      rowHeld()
+      await rowReleased
+    })
+    await rowHeldPromise
+
+    const refresh = store.assign(identity)
+    expect(await lockWaiterAppeared(databases[2]!)).toBe(true)
+    // The refresh is blocked on cell C. Every earlier row must still be free:
+    // the ordered fleet-wide scan would be holding both of them by now.
+    const heldWhileRefreshWaits: string[] = []
+    await databases[2]!.transaction(async (transaction) => {
+      for (const cell of [cellA, cellB]) {
+        try {
+          await transaction.queryLocked(
+            `SELECT * FROM relay_cells WHERE cell_id = ?`,
+            [cell.id],
+            { failIfUnavailable: true }
+          )
+        } catch {
+          heldWhileRefreshWaits.push(cell.id)
+        }
+      }
+    })
+    releaseRow()
+    await holder
+
+    expect(heldWhileRefreshWaits).toEqual([])
+    expect((await refresh).cellId).toBe(cellC.id)
+  }, 15_000)
+
+  // Why: the counter moves by a delta now instead of an absolute value read
+  // from a snapshot, so concurrent movement on the same cell must still sum.
+  it('keeps a cell reservation exact under concurrent same-cell activity', async () => {
+    await removeTestRows(databases[0]!)
+    const store = new RelayAssignmentStore(databases[0]!, () => 100)
+    await store.reconcileCells(cells)
+    for (const cell of cells) await store.recordCellHeartbeat(heartbeat(cell))
+    await store.setCellEnabled(cellA.id, false)
+    await store.setCellEnabled(cellB.id, false)
+
+    const hosts = Array.from({ length: 6 }, (_, index) => ({
+      userId: `percell-postgres-user-${index}`,
+      relayHostId: `percellhost0000${index}`
+    }))
+    const stores = databases.map((database) => new RelayAssignmentStore(database, () => 100))
+    await Promise.all(hosts.map((host, index) => stores[index % stores.length]!.assign(host)))
+
+    // One splice each (2 units) on the same cell, from three connections at once.
+    await Promise.all(
+      hosts.map((host, index) =>
+        stores[index % stores.length]!.acquireActivity(host, {
+          activityId: `splice:percell-${index}`,
+          kind: 'splice',
+          cellId: cellC.id
+        })
+      )
+    )
+    const afterAcquire = await databases[0]!.query(
+      `SELECT reserved_requests FROM relay_cells WHERE cell_id = ?`,
+      [cellC.id]
+    )
+    // 6 pending control grants + 6 splices at 2 units each.
+    expect(Number(afterAcquire[0]!.reserved_requests)).toBe(6 + 12)
+
+    await Promise.all(
+      hosts.map((host, index) =>
+        stores[index % stores.length]!.releaseActivity(host, `splice:percell-${index}`)
+      )
+    )
+    const afterRelease = await databases[0]!.query(
+      `SELECT reserved_requests FROM relay_cells WHERE cell_id = ?`,
+      [cellC.id]
+    )
+    expect(Number(afterRelease[0]!.reserved_requests)).toBe(6)
+    await store.setCellEnabled(cellA.id, true)
+    await store.setCellEnabled(cellB.id, true)
+  }, 15_000)
+})

cloud/apps/relay/src/control-rebind-inventory-lock-postgres.test.ts

@@ -0,0 +1,260 @@
+import { afterAll, beforeAll, describe, expect, it } from 'vitest'
+import { RelayAssignmentStore } from './assignment-store.js'
+import { openRelayDatabase, type RelayDatabase } from './database.js'
+
+const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
+const describePostgres = databaseUrl ? describe : describe.skip
+
+// Three cells: the inventory lock covers more than the rows a move touches, and
+// a high-to-low move exposes any lock taken out of cell_id order.
+const cells = [
+  {
+    id: 'rebind-inventory-postgres-a',
+    url: 'https://rebind-inventory-postgres-a.example.com',
+    capacityRequests: 1_000,
+    connectionHardCap: 600 as const,
+    connectionUnobservedBound: 50
+  },
+  {
+    id: 'rebind-inventory-postgres-b',
+    url: 'https://rebind-inventory-postgres-b.example.com',
+    capacityRequests: 1_000,
+    connectionHardCap: 600 as const,
+    connectionUnobservedBound: 50
+  },
+  {
+    id: 'rebind-inventory-postgres-c',
+    url: 'https://rebind-inventory-postgres-c.example.com',
+    capacityRequests: 1_000,
+    connectionHardCap: 600 as const,
+    connectionUnobservedBound: 50
+  }
+]
+const identity = { userId: 'rebind-inventory-postgres-user', relayHostId: 'rebindinvhost001' }
+
+function heartbeat(cell: (typeof cells)[number]) {
+  return {
+    cellId: cell.id,
+    cellUrl: cell.url,
+    cellIncarnation: '11111111-1111-4111-8111-111111111111',
+    startedAt: 50,
+    ready: true,
+    observedRequests: 0,
+    totalConnections: 0,
+    inFlightConnections: 0,
+    reservedConnectionUnits: 0,
+    enforcedConnectionUnits: 0,
+    connectionInclusionWatermark: 1,
+    connectionHardCap: 600 as const,
+    connectionUnobservedBound: 50
+  }
+}
+
+// Why: every desktop control rebind used to take the fleet-wide relay_cells
+// FOR UPDATE lock, so a rebind on one cell queued behind whatever held any
+// other cell's row, until COMMIT (55P03 at the request bound). A rebind only
+// touches its own cell row, so it must proceed while another cell's row is
+// held elsewhere.
+describePostgres('PostgreSQL control rebind under a held cell row', () => {
+  const databases: RelayDatabase[] = []
+
+  beforeAll(async () => {
+    databases.push(
+      await openRelayDatabase({ databaseUrl, dataDir: '' }),
+      await openRelayDatabase({ databaseUrl, dataDir: '' })
+    )
+  })
+
+  async function removeTestRows(database: RelayDatabase): Promise<void> {
+    await database.query(
+      `DELETE FROM relay_control_connection_reservations WHERE user_id = ?`,
+      [identity.userId]
+    )
+    for (const table of [
+      'relay_assignment_activity_leases',
+      'relay_post_drain_migration_pins',
+      'relay_assignment_migration_incarnations',
+      'relay_assignment_migrations',
+      'relay_assignments'
+    ]) {
+      await database.query(`DELETE FROM ${table} WHERE user_id = ?`, [identity.userId])
+    }
+    for (const cell of cells) {
+      for (const table of [
+        'relay_cell_connection_snapshots',
+        'relay_cell_connection_runtime',
+        'relay_cell_connection_limits',
+        'relay_cell_runtime',
+        'relay_cells'
+      ]) {
+        await database.query(`DELETE FROM ${table} WHERE cell_id = ?`, [cell.id])
+      }
+    }
+  }
+
+  afterAll(async () => {
+    if (databases[0]) await removeTestRows(databases[0])
+    for (const connection of databases) await connection.close()
+  })
+
+  it("rebinds and supersedes a control while another cell's row is held", async () => {
+    // A prior aborted run leaves connection snapshots that reject a replayed watermark.
+    await removeTestRows(databases[0]!)
+    const store = new RelayAssignmentStore(databases[0]!, () => 100)
+    await store.reconcileCells(cells)
+    for (const cell of cells) await store.recordCellHeartbeat(heartbeat(cell))
+    // Pin the host to cell A so placement is deterministic.
+    await store.setCellEnabled(cells[1]!.id, false)
+    await store.setCellEnabled(cells[2]!.id, false)
+    const assignment = await store.assign(identity)
+    expect(assignment.cellId).toBe(cells[0]!.id)
+    await store.setCellEnabled(cells[1]!.id, true)
+    await store.setCellEnabled(cells[2]!.id, true)
+    await store.activateControl(identity, {
+      cellId: cells[0]!.id,
+      assignmentEpoch: assignment.assignmentEpoch,
+      generation: 1,
+      connectionInclusionWatermark: 10
+    })
+
+    // Hold only cell B's row on a second connection, the way a rebind on B
+    // does, for longer than the request-path lock bound.
+    let releaseInventory!: () => void
+    const inventoryReleased = new Promise<void>((resolve) => {
+      releaseInventory = resolve
+    })
+    let inventoryHeld!: () => void
+    const inventoryHeldPromise = new Promise<void>((resolve) => {
+      inventoryHeld = resolve
+    })
+    const holder = databases[1]!.transaction(async (transaction) => {
+      await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [cells[1]!.id])
+      inventoryHeld()
+      await inventoryReleased
+    })
+    await inventoryHeldPromise
+
+    // A generation-2 rebind on cell A supersedes generation 1. It must not
+    // wait on cell B's row.
+    const startedAt = Date.now()
+    const blockedStatement = async (): Promise<string> => {
+      const rows = await databases[1]!.query(
+        `SELECT left(query, 160) AS q FROM pg_stat_activity
+         WHERE datname = current_database() AND wait_event_type = 'Lock'`
+      )
+      return rows.map((row) => String(row.q)).join(' | ')
+    }
+    const timeout = new Promise<never>((_, reject) =>
+      setTimeout(
+        () =>
+          void blockedStatement().then((statement) =>
+            reject(new Error(`rebind on cell A blocked behind cell B's row: ${statement}`))
+          ),
+        2_000
+      )
+    )
+    const rebound = await Promise.race([
+      store.activateControl(identity, {
+        cellId: cells[0]!.id,
+        assignmentEpoch: assignment.assignmentEpoch,
+        generation: 2,
+        connectionInclusionWatermark: 11
+      }),
+      timeout
+    ])
+    const elapsedMs = Date.now() - startedAt
+    releaseInventory()
+    await holder
+
+    expect(rebound).toBe(`control:${cells[0]!.id}:2`)
+    expect(elapsedMs).toBeLessThan(2_000)
+    const controls = await databases[0]!.query(
+      `SELECT activity_id FROM relay_assignment_activity_leases
+       WHERE user_id = ? AND activity_kind = 'control' ORDER BY activity_id`,
+      [identity.userId]
+    )
+    expect(controls).toEqual([{ activity_id: `control:${cells[0]!.id}:2` }])
+    const reserved = await databases[0]!.query(
+      `SELECT reserved_requests FROM relay_cells WHERE cell_id = ?`,
+      [cells[0]!.id]
+    )
+    expect(Number(reserved[0]!.reserved_requests)).toBe(1)
+  }, 15_000)
+
+  // Why: a phone's activity id is client-chosen and can follow the host across
+  // a migration, so acquireActivity may touch two cell rows. Moving from the
+  // higher cell to the lower one is where an unordered lock cycles with
+  // placement's ascending inventory lock (reproduced live before this fix).
+  it('moves an activity from a higher cell to a lower one in cell_id order', async () => {
+    await removeTestRows(databases[0]!)
+    const [cellA, cellB, cellC] = cells as [typeof cells[0], typeof cells[0], typeof cells[0]]
+    const store = new RelayAssignmentStore(databases[0]!, () => 100)
+    await store.reconcileCells(cells)
+    for (const cell of cells) await store.recordCellHeartbeat(heartbeat(cell))
+    await store.setCellEnabled(cellA.id, false)
+    await store.setCellEnabled(cellB.id, false)
+    const assignment = await store.assign(identity)
+    expect(assignment.cellId).toBe(cellC.id)
+    await store.setCellEnabled(cellA.id, true)
+    await store.setCellEnabled(cellB.id, true)
+    const activityId = 'splice:rebind-inventory-postgres'
+    await store.acquireActivity(identity, { activityId, kind: 'splice', cellId: cellC.id })
+    // The migration makes B authoritative; the lease still sits on C.
+    const migration = await store.startEvacuation(identity, cellB.id)
+    expect(migration.targetCellId).toBe(cellB.id)
+
+    // Hold B elsewhere. An ordered move locks B first and queues here holding
+    // nothing else. Locking C first (the old lease's row, as an unordered move
+    // does) or the whole inventory (which takes A) shows up as a held row.
+    let releaseRow!: () => void
+    const rowReleased = new Promise<void>((resolve) => {
+      releaseRow = resolve
+    })
+    let rowHeld!: () => void
+    const rowHeldPromise = new Promise<void>((resolve) => {
+      rowHeld = resolve
+    })
+    const heldWhileMoverWaits: string[] = []
+    const holder = databases[1]!.transaction(async (transaction) => {
+      await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [cellB.id])
+      rowHeld()
+      await rowReleased
+      for (const cell of [cellA, cellC]) {
+        try {
+          await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [cell.id], {
+            failIfUnavailable: true
+          })
+        } catch {
+          heldWhileMoverWaits.push(cell.id)
+        }
+      }
+    })
+    await rowHeldPromise
+    const move = store.acquireActivity(identity, { activityId, kind: 'splice', cellId: cellB.id })
+    let moved = false
+    void move.then(() => {
+      moved = true
+    })
+    await new Promise((resolve) => setTimeout(resolve, 250))
+    expect(moved).toBe(false)
+    releaseRow()
+    await holder
+    await move
+    expect(heldWhileMoverWaits).toEqual([])
+
+    const reservations = await databases[0]!.query(
+      `SELECT cell_id, reserved_requests FROM relay_cells
+       WHERE cell_id IN (?, ?, ?) ORDER BY cell_id ASC`,
+      [cellA.id, cellB.id, cellC.id]
+    )
+    const reserved = reservations.map((row) => [String(row.cell_id), Number(row.reserved_requests)])
+    expect(reserved).toEqual([
+      [cellA.id, 0],
+      // Migration grant plus the moved splice, as in the SQLite origin-scoped
+      // reservation case: the lock change did not alter accounting.
+      [cellB.id, 6],
+      // The sticky grant stays on the source until the migration completes.
+      [cellC.id, 1]
+    ])
+  }, 15_000)
+})

cloud/apps/relay/src/database-postgres-timeout.test.ts

@@ -2,7 +2,10 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 
 const fakes = vi.hoisted(() => ({
   configs: [] as Array<Record<string, unknown>>,
-  query: vi.fn(async () => ({ rows: [], rowCount: 0 })),
+  // Pool construction and pool shutdown interleaved, so "the schema pool is
+  // gone before the serving pool opens" is checkable rather than assumed.
+  lifecycle: [] as string[],
+  query: vi.fn(async (_sql: string) => ({ rows: [], rowCount: 0 })),
   release: vi.fn(),
   end: vi.fn(async () => undefined)
 }))
@@ -13,20 +16,41 @@ vi.mock('pg', () => ({
       totalCount = 1
       idleCount = 1
       waitingCount = 0
-      end = fakes.end
       on = vi.fn()
       connect = vi.fn(async () => ({ query: fakes.query, release: fakes.release }))
+      private readonly label: string
 
       constructor(config: Record<string, unknown>) {
         fakes.configs.push(config)
+        this.label = `max=${String(config.max)} statement_timeout=${String(config.statement_timeout)}`
+        fakes.lifecycle.push(`open ${this.label}`)
+      }
+
+      async end(): Promise<void> {
+        fakes.lifecycle.push(`end ${this.label}`)
+        await fakes.end()
       }
     }
   }
 }))
 
-import { openRelayDatabase } from './database.js'
+import {
+  openRelayDatabase,
+  POSTGRES_SCHEMA_MIGRATIONS,
+  relayPostgresStatementTimeoutMs
+} from './database.js'
 import { applyPostgresSchema } from './postgres-schema-startup.js'
 
+const SCHEMA_POOL = {
+  max: 1,
+  application_name: 'orca-relay/director/director/schema',
+  connectionTimeoutMillis: 2_000,
+  // Why: DDL must not inherit the request deadline.
+  statement_timeout: 0,
+  lock_timeout: 1_000,
+  idle_in_transaction_session_timeout: 5_000
+}
+
 afterEach(() => {
   vi.restoreAllMocks()
 })
@@ -34,9 +58,11 @@ afterEach(() => {
 describe('PostgreSQL relay deadlines', () => {
   beforeEach(() => {
     fakes.configs.length = 0
+    fakes.lifecycle.length = 0
     fakes.query.mockClear()
     fakes.release.mockClear()
     fakes.end.mockClear()
+    delete process.env.ORCA_RELAY_POSTGRES_STATEMENT_TIMEOUT_MS
   })
 
   it('bounds pool acquisition, statements, locks, and abandoned transactions', async () => {
@@ -48,6 +74,7 @@ describe('PostgreSQL relay deadlines', () => {
     })
 
     expect(fakes.configs).toEqual([
+      expect.objectContaining(SCHEMA_POOL),
       expect.objectContaining({
         max: 3,
         application_name: 'orca-relay/director/director',
@@ -59,6 +86,112 @@ describe('PostgreSQL relay deadlines', () => {
     ])
     await database.close()
   })
+
+  // Why: an untimed session left open would be a standing way for request work
+  // to escape the deadline this whole pool config exists to enforce.
+  it('closes the untimed schema pool before the serving pool opens', async () => {
+    const database = await openRelayDatabase({
+      databaseUrl: 'postgresql://relay:secret@127.0.0.1:5432/relay',
+      dataDir: './unused',
+      poolMax: 3,
+      applicationName: 'orca-relay/director/director'
+    })
+
+    expect(fakes.lifecycle).toEqual([
+      'open max=1 statement_timeout=0',
+      'end max=1 statement_timeout=0',
+      'open max=3 statement_timeout=5000'
+    ])
+    await database.close()
+  })
+
+  it('applies the schema on the untimed pool, never on the serving one', async () => {
+    fakes.query.mockClear()
+    const ddl: string[] = []
+    fakes.query.mockImplementation(async (sql: string) => {
+      // Every statement issued before the serving pool exists is schema work.
+      if (fakes.lifecycle.length === 1) ddl.push(sql)
+      return { rows: [], rowCount: 0 }
+    })
+    const database = await openRelayDatabase({
+      databaseUrl: 'postgresql://relay:secret@127.0.0.1:5432/relay',
+      dataDir: './unused'
+    })
+
+    expect(ddl.length).toBeGreaterThan(0)
+    // Statements can open with a leading `--` rationale comment.
+    const body = (statement: string): string =>
+      statement.replace(/^(?:\s*--[^\n]*\n)*\s*/, '')
+    expect(
+      ddl.every((statement) => /^(?:CREATE|ALTER TABLE)\b/i.test(body(statement)))
+    ).toBe(true)
+    // The backfill is DML, so it stays on the deadline-bearing serving pool.
+    expect(ddl.some((statement) => statement.includes('INSERT INTO'))).toBe(false)
+    await database.close()
+  })
+
+  it('takes the serving statement deadline from the environment', async () => {
+    process.env.ORCA_RELAY_POSTGRES_STATEMENT_TIMEOUT_MS = '2500'
+
+    const database = await openRelayDatabase({
+      databaseUrl: 'postgresql://relay:secret@127.0.0.1:5432/relay',
+      dataDir: './unused'
+    })
+
+    expect(fakes.configs).toEqual([
+      expect.objectContaining({ statement_timeout: 0 }),
+      expect.objectContaining({ statement_timeout: 2_500 })
+    ])
+    await database.close()
+  })
+
+  it.each(['0', '-1', '2.5', 'soon', ' '])(
+    'refuses %s as a statement deadline instead of running unbounded',
+    (value) => {
+      expect(() =>
+        relayPostgresStatementTimeoutMs({ ORCA_RELAY_POSTGRES_STATEMENT_TIMEOUT_MS: value })
+      ).toThrow('invalid_statement_timeout')
+    }
+  )
+
+  it.each([undefined, ''])('defaults to 5s when the environment says %s', (value) => {
+    expect(
+      relayPostgresStatementTimeoutMs(
+        value === undefined ? {} : { ORCA_RELAY_POSTGRES_STATEMENT_TIMEOUT_MS: value }
+      )
+    ).toBe(5_000)
+  })
+
+  // Why: a statement deadline that reaches the caller as a crash converts a
+  // transient stall into a failed assignment. It aborts the transaction exactly
+  // as a lock timeout does, so it belongs on the same bounded retry.
+  it('retries a statement timeout on a fresh client', async () => {
+    vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+    const database = await openRelayDatabase({
+      databaseUrl: 'postgresql://relay:secret@127.0.0.1:5432/relay',
+      dataDir: './unused'
+    })
+    let attempts = 0
+
+    const result = await database.transaction(async (transaction) => {
+      attempts += 1
+      if (attempts === 1) {
+        await transaction.query('SELECT 1')
+        throw Object.assign(new Error('canceling statement due to statement timeout'), {
+          code: '57014'
+        })
+      }
+      return 'committed'
+    })
+
+    expect(result).toBe('committed')
+    expect(attempts).toBe(2)
+    expect(console.warn).toHaveBeenCalledWith(
+      expect.stringContaining('"event":"orca_relay_postgres_transaction_retry"')
+    )
+    expect(console.warn).toHaveBeenCalledWith(expect.stringContaining('"code":"57014"'))
+    await database.close()
+  })
 })
 
 describe('PostgreSQL schema startup', () => {
@@ -118,6 +251,87 @@ describe('PostgreSQL schema startup', () => {
     expect(query).toHaveBeenCalledTimes(2)
   })
 
+  it.each([
+    ['42710', 'CREATE TABLE IF NOT EXISTS test'],
+    ['42P07', 'CREATE TABLE IF NOT EXISTS test'],
+    ['42P07', 'CREATE INDEX IF NOT EXISTS test_index ON test(id)'],
+    ['42P07', 'CREATE UNIQUE INDEX IF NOT EXISTS test_index ON test(id)']
+  ])('retries the committed-winner %s collision for %s', async (code, statement) => {
+    vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+    const collision = Object.assign(new Error('already exists'), { code })
+    const query = vi
+      .fn<(statement: string) => Promise<unknown>>()
+      .mockRejectedValueOnce(collision)
+      .mockResolvedValue(undefined)
+
+    await applyPostgresSchema([statement], query, { wait: async () => undefined })
+
+    expect(query).toHaveBeenCalledTimes(2)
+  })
+
+  it('treats an existing constraint as an applied ADD CONSTRAINT', async () => {
+    // Postgres has no `ADD CONSTRAINT IF NOT EXISTS`, and a retry would only
+    // repeat 42710, so a re-run and a concurrent startup both move on.
+    const error = Object.assign(new Error('already exists'), { code: '42710' })
+    const query = vi
+      .fn<(statement: string) => Promise<unknown>>()
+      .mockRejectedValueOnce(error)
+      .mockResolvedValue(undefined)
+    const pause = vi.fn(async () => undefined)
+
+    await applyPostgresSchema(
+      ['ALTER TABLE test ADD CONSTRAINT test_check CHECK (id > 0)', 'CREATE TABLE test2'],
+      query,
+      { wait: pause }
+    )
+
+    expect(pause).not.toHaveBeenCalled()
+    expect(query).toHaveBeenCalledTimes(2)
+    expect(query).toHaveBeenLastCalledWith('CREATE TABLE test2')
+  })
+
+  it('recognises every shipped ADD CONSTRAINT migration as re-runnable', async () => {
+    // Guards the statement text against the pattern that classifies it.
+    const shipped = POSTGRES_SCHEMA_MIGRATIONS.filter((statement) =>
+      statement.includes('ADD CONSTRAINT')
+    )
+    expect(shipped.length).toBeGreaterThan(0)
+    const error = Object.assign(new Error('already exists'), { code: '42710' })
+    const query = vi.fn<(statement: string) => Promise<unknown>>().mockRejectedValue(error)
+
+    await applyPostgresSchema(shipped, query, { wait: async () => undefined })
+
+    expect(query).toHaveBeenCalledTimes(shipped.length)
+  })
+
+  it('still fails an ADD CONSTRAINT that violates existing rows', async () => {
+    const error = Object.assign(new Error('check violation'), { code: '23514' })
+    const query = vi.fn<(statement: string) => Promise<unknown>>().mockRejectedValue(error)
+
+    await expect(
+      applyPostgresSchema(
+        ['ALTER TABLE test ADD CONSTRAINT test_check CHECK (id > 0)'],
+        query,
+        { wait: async () => undefined }
+      )
+    ).rejects.toBe(error)
+  })
+
+  it.each([
+    ['42710', 'CREATE INDEX IF NOT EXISTS test_index ON test(id)'],
+    ['42710', 'CREATE TABLE test'],
+    ['42P07', 'CREATE TABLE test'],
+    ['42P07', 'CREATE INDEX test_index ON test(id)']
+  ])('does not retry %s for %s', async (code, statement) => {
+    const error = Object.assign(new Error('already exists'), { code })
+    const query = vi.fn<(statement: string) => Promise<unknown>>().mockRejectedValue(error)
+    const pause = vi.fn(async () => undefined)
+
+    await expect(applyPostgresSchema([statement], query, { wait: pause })).rejects.toBe(error)
+
+    expect(pause).not.toHaveBeenCalled()
+  })
+
   it.each([
     ['pg_type_typname_nsp_index', 'CREATE TABLE test'],
     ['pg_class_relname_nsp_index', 'CREATE INDEX test_index ON test(id)']

cloud/apps/relay/src/database-statement-timeout-postgres.test.ts

@@ -0,0 +1,98 @@
+import { afterAll, beforeAll, describe, expect, it } from 'vitest'
+import { openRelayDatabase, type RelayDatabase } from './database.js'
+
+const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
+const describePostgres = databaseUrl ? describe : describe.skip
+const applicationName = 'orca-relay/statement-timeout-postgres'
+
+describePostgres('PostgreSQL statement deadline', () => {
+  const databases: RelayDatabase[] = []
+
+  beforeAll(async () => {
+    databases.push(await openRelayDatabase({ databaseUrl, dataDir: '' }))
+  })
+
+  afterAll(async () => {
+    for (const database of databases) await database.close()
+  })
+
+  it('serves requests under the configured deadline', async () => {
+    const database = await openRelayDatabase({ databaseUrl, dataDir: '', statementTimeoutMs: 300 })
+    databases.push(database)
+
+    expect(await database.query(`SELECT current_setting('statement_timeout') AS statement_timeout`)).toEqual([
+      { statement_timeout: '300ms' }
+    ])
+  })
+
+  // Why: a real 57014 aborts the transaction exactly as a lock timeout does. If
+  // it escapes the bounded retry it becomes a failed assignment instead of a
+  // slow one.
+  it('retries a real statement timeout on a fresh client', async () => {
+    const database = await openRelayDatabase({ databaseUrl, dataDir: '', statementTimeoutMs: 300 })
+    databases.push(database)
+    let attempts = 0
+
+    const result = await database.transaction(async (transaction) => {
+      attempts += 1
+      if (attempts === 1) await transaction.query(`SELECT pg_sleep(2)`)
+      return attempts
+    })
+
+    expect(result).toBe(2)
+  }, 15_000)
+
+  // Why: DDL runs on its own untimed connection. relay_invites carries a
+  // CREATE INDEX IF NOT EXISTS, which (unlike CREATE TABLE IF NOT EXISTS)
+  // really does queue behind an ACCESS EXCLUSIVE lock on the table.
+  it('applies the schema behind a held ACCESS EXCLUSIVE lock', async () => {
+    let releaseTable!: () => void
+    const tableReleased = new Promise<void>((resolve) => {
+      releaseTable = resolve
+    })
+    let tableHeld!: () => void
+    const tableHeldPromise = new Promise<void>((resolve) => {
+      tableHeld = resolve
+    })
+    const holder = databases[0]!.transaction(async (transaction) => {
+      await transaction.query(`LOCK TABLE relay_invites IN ACCESS EXCLUSIVE MODE`)
+      tableHeld()
+      await tableReleased
+    })
+    await tableHeldPromise
+
+    const opening = openRelayDatabase({
+      databaseUrl,
+      dataDir: '',
+      applicationName,
+      // Far too short for a blocked DDL; the serving pool wears it, the schema
+      // connection must not.
+      statementTimeoutMs: 200
+    })
+    const blockedOnSchemaConnection = async (): Promise<boolean> => {
+      const deadline = Date.now() + 4_000
+      while (Date.now() < deadline) {
+        const rows = await databases[0]!.query(
+          `SELECT count(*) AS waiting FROM pg_stat_activity
+           WHERE datname = current_database() AND wait_event_type = 'Lock'
+             AND application_name = ?`,
+          [`${applicationName}/schema`]
+        )
+        if (Number(rows[0]!.waiting) > 0) return true
+        await new Promise((resolve) => setTimeout(resolve, 10))
+      }
+      return false
+    }
+    const blocked = await blockedOnSchemaConnection()
+    releaseTable()
+    await holder
+
+    const database = await opening
+    databases.push(database)
+    expect(blocked).toBe(true)
+    // The serving pool still carries the short deadline it was opened with.
+    expect(await database.query(`SELECT current_setting('statement_timeout') AS statement_timeout`)).toEqual([
+      { statement_timeout: '200ms' }
+    ])
+  }, 15_000)
+})

cloud/apps/relay/src/database.test.ts

@@ -2,7 +2,12 @@ import { mkdtempSync, rmSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { afterEach, describe, expect, it } from 'vitest'
-import { openInMemoryRelayDatabase, openRelayDatabase } from './database.js'
+import {
+  openInMemoryRelayDatabase,
+  openRelayDatabase,
+  POSTGRES_SCHEMA_MIGRATIONS,
+  REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS
+} from './database.js'
 
 const temporaryDirectories: string[] = []
 
@@ -142,6 +147,41 @@ describe('relay database', () => {
     await second.close()
   })
 
+  it('renders every region check from the shared region list', async () => {
+    // Derived, not hand-written: a third region must not leave one column
+    // rejecting a value the rest of the relay already accepts.
+    const database = await openInMemoryRelayDatabase()
+    const checked = await database.query(
+      `SELECT name, sql FROM sqlite_master
+       WHERE type = 'table'
+         AND name IN ('relay_assignment_region_preferences', 'relay_cell_regions',
+                      'relay_region_rehome_attempts')
+       ORDER BY name`
+    )
+    const list = `IN ('us-central1', 'asia-east2')`
+    expect(checked.map((row) => row.name)).toEqual([
+      'relay_assignment_region_preferences',
+      'relay_cell_regions',
+      'relay_region_rehome_attempts'
+    ])
+    expect(checked.every((row) => String(row.sql).includes(list))).toBe(true)
+    expect(
+      POSTGRES_SCHEMA_MIGRATIONS.some((statement) => statement.includes(list))
+    ).toBe(true)
+    await database.close()
+  })
+
+  it('indexes rehome attempts by host recency for the per-host cooldown', async () => {
+    const database = await openInMemoryRelayDatabase()
+    const rows = await database.query(
+      `SELECT sql FROM sqlite_master
+       WHERE type = 'index' AND name = 'relay_region_rehome_attempts_host_recency'`
+    )
+    expect(rows[0]?.sql).toContain('(user_id, relay_host_id, created_at)')
+    expect(REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS).toBe(7 * 24 * 60 * 60_000)
+    await database.close()
+  })
+
   it('indexes region preference expiry by observation time', async () => {
     const database = await openInMemoryRelayDatabase()
     const rows = await database.query(

cloud/apps/relay/src/database.ts

@@ -1,16 +1,54 @@
 import { mkdirSync } from 'node:fs'
+import { performance } from 'node:perf_hooks'
 import { join } from 'node:path'
 import { DatabaseSync } from 'node:sqlite'
 import pg from 'pg'
+import { RELAY_REGIONS } from '@orca-cloud/relay-contract'
 import {
   emptyPostgresPoolPressureCounts,
   PostgresPoolPressure,
   type PostgresPoolPressureCounts
 } from './postgres-pool-pressure.js'
 import { applyPostgresSchema } from './postgres-schema-startup.js'
+import {
+  CellInventoryHoldSamples,
+  emptyCellInventoryHoldCounts,
+  type CellInventoryHoldCounts
+} from './cell-inventory-hold-samples.js'
+
+export const POSTGRES_LOCK_TIMEOUT_MS = 1_000
+
+function setLocalLockTimeout(milliseconds: number): string {
+  if (!Number.isInteger(milliseconds) || milliseconds < 1) {
+    throw new Error('invalid_lock_timeout')
+  }
+  return `SET LOCAL lock_timeout = '${milliseconds}ms'`
+}
+
+// Region CHECK lists come from the contract so a new region cannot leave a
+// column rejecting values the rest of the relay already accepts.
+const REGION_LIST = RELAY_REGIONS.map((region) => `'${region}'`).join(', ')
+
+// A host that was just moved is not a candidate again for this long, so a
+// desktop whose region probe flips cannot walk itself back and forth.
+export const REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS = 7 * 24 * 60 * 60_000
 
 export type SqlRow = Record<string, unknown>
-export type RelayLockOptions = { failIfUnavailable?: boolean }
+export type RelayLockOptions = {
+  failIfUnavailable?: boolean
+  // Only honoured inside a transaction: SET LOCAL is a no-op in autocommit.
+  lockTimeoutMs?: number
+  // Report how long this lock is held to COMMIT. The hold, not the wait, is what
+  // forms the queue, and nothing measured it before.
+  measureHoldMs?: boolean
+}
+
+// A transaction that can report how long it held a measured lock before COMMIT.
+type HoldMeasuringTransaction = { consumeHoldMs(): number | undefined }
+
+function measuredHoldMs(transaction: unknown): number | undefined {
+  return (transaction as HoldMeasuringTransaction).consumeHoldMs?.()
+}
 export type RelayTransactionOptions = { reportRetries?: boolean }
 
 export interface RelayDatabase {
@@ -152,7 +190,7 @@ CREATE TABLE IF NOT EXISTS relay_assignment_region_preferences (
   user_id TEXT NOT NULL,
   relay_host_id TEXT NOT NULL,
   preferred_region TEXT NOT NULL
-    CHECK (preferred_region IN ('us-central1', 'asia-east2')),
+    CHECK (preferred_region IN (${REGION_LIST})),
   observed_at BIGINT NOT NULL,
   PRIMARY KEY (user_id, relay_host_id)
 );
@@ -175,6 +213,8 @@ CREATE TABLE IF NOT EXISTS relay_region_rehome_control (
   not_before BIGINT NOT NULL,
   rate_per_minute BIGINT NOT NULL,
   preference_max_age_ms BIGINT NOT NULL,
+  host_cooldown_ms BIGINT NOT NULL
+    DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS},
   drain_grace_ms BIGINT NOT NULL,
   updated_at BIGINT NOT NULL
 );
@@ -183,7 +223,9 @@ CREATE TABLE IF NOT EXISTS relay_region_rehome_attempts (
   attempt_id TEXT PRIMARY KEY,
   user_id TEXT NOT NULL,
   relay_host_id TEXT NOT NULL,
-  preferred_region TEXT NOT NULL CHECK (preferred_region = 'asia-east2'),
+  preferred_region TEXT NOT NULL
+    CONSTRAINT relay_region_rehome_attempts_preferred_region_valid
+    CHECK (preferred_region IN (${REGION_LIST})),
   source_cell_id TEXT NOT NULL,
   source_cell_incarnation TEXT NOT NULL,
   target_cell_id TEXT NOT NULL,
@@ -205,6 +247,8 @@ CREATE TABLE IF NOT EXISTS relay_region_rehome_attempts (
 );
 CREATE INDEX IF NOT EXISTS relay_region_rehome_attempts_pending
   ON relay_region_rehome_attempts(drain_receipt_at, last_send_attempt_at, completed_at, aborted_at);
+CREATE INDEX IF NOT EXISTS relay_region_rehome_attempts_host_recency
+  ON relay_region_rehome_attempts(user_id, relay_host_id, created_at);
 
 CREATE TABLE IF NOT EXISTS relay_cells (
   cell_id TEXT PRIMARY KEY,
@@ -219,7 +263,7 @@ CREATE TABLE IF NOT EXISTS relay_cells (
 
 CREATE TABLE IF NOT EXISTS relay_cell_regions (
   cell_id TEXT PRIMARY KEY,
-  region TEXT NOT NULL CHECK (region IN ('us-central1', 'asia-east2'))
+  region TEXT NOT NULL CHECK (region IN (${REGION_LIST}))
 );
 
 CREATE TABLE IF NOT EXISTS relay_cell_admission (
@@ -551,6 +595,21 @@ CREATE TABLE IF NOT EXISTS relay_audit_events (
 CREATE INDEX IF NOT EXISTS relay_audit_events_at ON relay_audit_events(at);
 `
 
+// Rehoming is bidirectional, but tables created before that carry the
+// original single-region column check. The old constraint is the one Postgres
+// auto-named; the replacement is named, so both statements are no-ops on a
+// database the current schema created and neither can drop the other.
+export const POSTGRES_SCHEMA_MIGRATIONS = [
+  `ALTER TABLE relay_region_rehome_attempts
+     DROP CONSTRAINT IF EXISTS relay_region_rehome_attempts_preferred_region_check`,
+  `ALTER TABLE relay_region_rehome_attempts
+     ADD CONSTRAINT relay_region_rehome_attempts_preferred_region_valid
+     CHECK (preferred_region IN (${REGION_LIST}))`,
+  `ALTER TABLE relay_region_rehome_control
+     ADD COLUMN IF NOT EXISTS host_cooldown_ms BIGINT NOT NULL
+     DEFAULT ${REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS}`
+]
+
 function postgresSql(sql: string): string {
   let index = 0
   return sql.replace(/\?/g, () => `$${++index}`)
@@ -612,9 +671,23 @@ function postgresTransactionErrorPhase(error: unknown): string {
 
 class SqliteTransaction implements RelayDatabase {
   readonly dialect = 'sqlite' as const
+  private heldFromMs: number | undefined
 
   constructor(protected readonly database: DatabaseSync) {}
 
+  consumeHoldMs(): number | undefined {
+    if (this.heldFromMs === undefined) return undefined
+    const holdMs = performance.now() - this.heldFromMs
+    this.heldFromMs = undefined
+    return holdMs
+  }
+
+  protected noteHeld(options: RelayLockOptions): void {
+    if (options.measureHoldMs && this.heldFromMs === undefined) {
+      this.heldFromMs = performance.now()
+    }
+  }
+
   async query(sql: string, params: unknown[] = []): Promise<SqlRow[]> {
     const statement = this.database.prepare(sql)
     const bound = params.map((value) => (value === undefined ? null : value)) as never[]
@@ -626,9 +699,11 @@ class SqliteTransaction implements RelayDatabase {
   async queryLocked(
     sql: string,
     params: unknown[] = [],
-    _options: RelayLockOptions = {}
+    options: RelayLockOptions = {}
   ): Promise<SqlRow[]> {
-    return await this.query(sql, params)
+    const rows = await this.query(sql, params)
+    this.noteHeld(options)
+    return rows
   }
 
   async transaction<T>(
@@ -643,6 +718,11 @@ class SqliteTransaction implements RelayDatabase {
 
 class SqliteDatabase extends SqliteTransaction {
   private tail: Promise<void> = Promise.resolve()
+  private readonly holds = new CellInventoryHoldSamples()
+
+  consumeHoldCounts(): CellInventoryHoldCounts {
+    return this.holds.consumeCounts()
+  }
 
   override async query(sql: string, params: unknown[] = []): Promise<SqlRow[]> {
     await this.tail
@@ -655,9 +735,11 @@ class SqliteDatabase extends SqliteTransaction {
     this.tail = new Promise((resolve) => (release = resolve))
     await previous
     this.database.exec('BEGIN IMMEDIATE')
+    const transaction = new SqliteTransaction(this.database)
     try {
-      const result = await operation(new SqliteTransaction(this.database))
+      const result = await operation(transaction)
       this.database.exec('COMMIT')
+      this.holds.record(measuredHoldMs(transaction) ?? Number.NaN)
       return result
     } catch (error) {
       this.database.exec('ROLLBACK')
@@ -675,9 +757,17 @@ class SqliteDatabase extends SqliteTransaction {
 
 class PostgresTransaction implements RelayDatabase {
   readonly dialect = 'postgres' as const
+  private heldFromMs: number | undefined
 
   constructor(protected readonly client: pg.PoolClient) {}
 
+  consumeHoldMs(): number | undefined {
+    if (this.heldFromMs === undefined) return undefined
+    const holdMs = performance.now() - this.heldFromMs
+    this.heldFromMs = undefined
+    return holdMs
+  }
+
   async query(sql: string, params: unknown[] = []): Promise<SqlRow[]> {
     try {
       const result = await this.client.query(postgresSql(sql), params)
@@ -693,11 +783,21 @@ class PostgresTransaction implements RelayDatabase {
     params: unknown[] = [],
     options: RelayLockOptions = {}
   ): Promise<SqlRow[]> {
+    // SET LOCAL lasts to COMMIT, so a bound left in place would silently govern
+    // every later locked statement in the transaction and misattribute its 55P03s.
+    const bounded = options.lockTimeoutMs !== undefined && !options.failIfUnavailable
     try {
-      return await this.query(
+      // A blocked waiter holds its pooled client for the whole lock_timeout, so
+      // hot tiny-table locks bound their own wait well under the pool default.
+      if (bounded) await this.query(setLocalLockTimeout(options.lockTimeoutMs!))
+      const rows = await this.query(
         `${sql} FOR UPDATE${options.failIfUnavailable ? ' NOWAIT' : ''}`,
         params
       )
+      if (options.measureHoldMs && this.heldFromMs === undefined) {
+        this.heldFromMs = performance.now()
+      }
+      return rows
     } catch (error) {
       if (
         options.failIfUnavailable &&
@@ -706,6 +806,10 @@ class PostgresTransaction implements RelayDatabase {
         throw new Error('database_lock_unavailable')
       }
       throw error
+    } finally {
+      // Restore on the error path too: the transaction may still be retried or
+      // continue with unrelated locks after a caught lock failure.
+      if (bounded) await this.query(setLocalLockTimeout(POSTGRES_LOCK_TIMEOUT_MS)).catch(() => undefined)
     }
   }
 
@@ -722,13 +826,34 @@ class PostgresTransaction implements RelayDatabase {
 const POSTGRES_TRANSACTION_ATTEMPTS = 3
 const POSTGRES_RETRY_MAX_DELAY_MS = 25
 const POSTGRES_CONNECTION_TIMEOUT_MS = 2_000
-const POSTGRES_STATEMENT_TIMEOUT_MS = 5_000
-const POSTGRES_LOCK_TIMEOUT_MS = 1_000
+// Derivation: a control renewal must land inside its own 30s tick
+// (RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2), and a transaction gets
+// POSTGRES_TRANSACTION_ATTEMPTS tries, so the worst case a renewal can spend in
+// Postgres is attempts * timeout. 5s keeps that at 15s, half the tick, and still
+// leaves room for the connect timeout above.
+export const POSTGRES_STATEMENT_TIMEOUT_MS = 5_000
 const POSTGRES_IDLE_TRANSACTION_TIMEOUT_MS = 5_000
 
+export function relayPostgresStatementTimeoutMs(
+  env: NodeJS.ProcessEnv = process.env
+): number {
+  const configured = env.ORCA_RELAY_POSTGRES_STATEMENT_TIMEOUT_MS
+  if (configured === undefined || configured === '') return POSTGRES_STATEMENT_TIMEOUT_MS
+  const milliseconds = Number(configured)
+  // 0 is PostgreSQL's "no timeout"; refusing it keeps the deadline this exists
+  // to enforce from being disabled by a typo in an environment variable.
+  if (!Number.isInteger(milliseconds) || milliseconds < 1) {
+    throw new Error('invalid_statement_timeout')
+  }
+  return milliseconds
+}
+
 function retryablePostgresTransactionError(error: unknown): boolean {
   const code = String((error as { code?: unknown }).code)
-  return code === '40P01' || code === '40001' || code === '55P03'
+  // 57014 is the pool statement_timeout firing. It aborts the transaction the
+  // same way a lock timeout does, so it belongs on the bounded retry path
+  // rather than surfacing as a terminal failure to the caller.
+  return code === '40P01' || code === '40001' || code === '55P03' || code === '57014'
 }
 
 export function isRelayDatabaseTransientError(error: unknown): boolean {
@@ -749,6 +874,11 @@ async function waitForPostgresRetry(random: () => number = Math.random): Promise
 class PostgresDatabase implements RelayDatabase {
   readonly dialect = 'postgres' as const
   private readonly pressure: PostgresPoolPressure
+  private readonly holds = new CellInventoryHoldSamples()
+
+  consumeHoldCounts(): CellInventoryHoldCounts {
+    return this.holds.consumeCounts()
+  }
 
   constructor(private readonly pool: pg.Pool) {
     this.pressure = new PostgresPoolPressure(pool)
@@ -770,6 +900,8 @@ class PostgresDatabase implements RelayDatabase {
     options: RelayLockOptions = {}
   ): Promise<SqlRow[]> {
     try {
+      // No transaction here, so options.lockTimeoutMs cannot apply: SET LOCAL
+      // would be discarded at the autocommit boundary before the lock is taken.
       return await this.query(
         `${sql} FOR UPDATE${options.failIfUnavailable ? ' NOWAIT' : ''}`,
         params
@@ -791,10 +923,12 @@ class PostgresDatabase implements RelayDatabase {
   ): Promise<T> {
     for (let attempt = 1; attempt <= POSTGRES_TRANSACTION_ATTEMPTS; attempt++) {
       const client = await this.pressure.connect()
+      const transaction = new PostgresTransaction(client)
       try {
         await client.query('BEGIN')
-        const result = await operation(new PostgresTransaction(client))
+        const result = await operation(transaction)
         await client.query('COMMIT')
+        this.holds.record(measuredHoldMs(transaction) ?? Number.NaN)
         return result
       } catch (error) {
         await client.query('ROLLBACK').catch(() => undefined)
@@ -852,6 +986,13 @@ export function consumeRelayDatabasePoolPressure(
     : emptyPostgresPoolPressureCounts()
 }
 
+export function consumeRelayCellInventoryHold(
+  database: RelayDatabase
+): CellInventoryHoldCounts {
+  const holder = database as { consumeHoldCounts?: () => CellInventoryHoldCounts }
+  return holder.consumeHoldCounts?.() ?? emptyCellInventoryHoldCounts()
+}
+
 export function readRelayDatabasePoolPressure(
   database: RelayDatabase
 ): PostgresPoolPressureCounts {
@@ -874,11 +1015,39 @@ async function applySchema(database: RelayDatabase): Promise<void> {
   }
 }
 
-async function applySchemaWithPostgresRetries(database: RelayDatabase): Promise<void> {
-  await applyPostgresSchema(
-    SCHEMA.split(';').filter((statement) => statement.trim()),
-    async (statement) => await database.query(statement)
-  )
+// Why: DDL is not a request. A CREATE INDEX on a grown table legitimately runs
+// longer than the request statement_timeout, and inheriting that timeout would
+// make every startup fail at the same statement instead of finishing once. One
+// short-lived connection of its own, ended before the serving pool opens, keeps
+// the untimed session off the request path entirely.
+async function applySchemaOnUntimedPool(
+  databaseUrl: string,
+  applicationName: string | undefined
+): Promise<void> {
+  const pool = new pg.Pool({
+    connectionString: databaseUrl,
+    max: 1,
+    application_name: applicationName ? `${applicationName}/schema` : undefined,
+    connectionTimeoutMillis: POSTGRES_CONNECTION_TIMEOUT_MS,
+    statement_timeout: 0,
+    // Kept: a DDL blocked behind another director's ACCESS EXCLUSIVE lock must
+    // yield to the bounded schema retry instead of holding the connection.
+    lock_timeout: POSTGRES_LOCK_TIMEOUT_MS,
+    idle_in_transaction_session_timeout: POSTGRES_IDLE_TRANSACTION_TIMEOUT_MS
+  })
+  absorbPostgresIdleClientErrors(pool)
+  const database = new PostgresDatabase(pool)
+  try {
+    await applyPostgresSchema(
+      [
+        ...SCHEMA.split(';').filter((statement) => statement.trim()),
+        ...POSTGRES_SCHEMA_MIGRATIONS
+      ],
+      async (statement) => await database.query(statement)
+    )
+  } finally {
+    await database.close().catch(() => undefined)
+  }
 }
 
 async function backfillRelayCellRegions(database: RelayDatabase): Promise<void> {
@@ -894,15 +1063,17 @@ export async function openRelayDatabase(input: {
   dataDir: string
   poolMax?: number
   applicationName?: string
+  statementTimeoutMs?: number
 }): Promise<RelayDatabase> {
   let database: RelayDatabase
   if (input.databaseUrl) {
+    await applySchemaOnUntimedPool(input.databaseUrl, input.applicationName)
     const pool = new pg.Pool({
       connectionString: input.databaseUrl,
       max: input.poolMax ?? 10,
       application_name: input.applicationName,
       connectionTimeoutMillis: POSTGRES_CONNECTION_TIMEOUT_MS,
-      statement_timeout: POSTGRES_STATEMENT_TIMEOUT_MS,
+      statement_timeout: input.statementTimeoutMs ?? relayPostgresStatementTimeoutMs(),
       lock_timeout: POSTGRES_LOCK_TIMEOUT_MS,
       idle_in_transaction_session_timeout: POSTGRES_IDLE_TRANSACTION_TIMEOUT_MS
     })
@@ -915,8 +1086,7 @@ export async function openRelayDatabase(input: {
     database = new SqliteDatabase(sqlite)
   }
   try {
-    if (input.databaseUrl) await applySchemaWithPostgresRetries(database)
-    else await applySchema(database)
+    if (!input.databaseUrl) await applySchema(database)
     await backfillRelayCellRegions(database)
     return database
   } catch (error) {

cloud/apps/relay/src/host-close-reason-memory.test.ts

@@ -0,0 +1,82 @@
+import { ASSIGNMENT_LIMITS, RELAY_HOST_CLOSE_REASON } from '@orca-cloud/relay-contract'
+import { describe, expect, it } from 'vitest'
+import { HostCloseReasonMemory } from './host-close-reason-memory.js'
+
+function memoryAt(clock: { now: number }): HostCloseReasonMemory {
+  return new HostCloseReasonMemory(() => clock.now)
+}
+
+describe('HostCloseReasonMemory', () => {
+  it('remembers only reasons it knows', () => {
+    const clock = { now: 1_000 }
+    const memory = memoryAt(clock)
+
+    memory.record('a', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
+    memory.record('b', 'quitting')
+    memory.record('c', Buffer.alloc(0))
+    memory.record('d', undefined)
+
+    expect(memory.read('a')).toBe(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
+    expect(memory.read('b')).toBeNull()
+    expect(memory.read('c')).toBeNull()
+    expect(memory.read('d')).toBeNull()
+  })
+
+  it('accepts the reason as the Buffer a ws close delivers', () => {
+    const clock = { now: 1_000 }
+    const memory = memoryAt(clock)
+
+    memory.record('a', Buffer.from(RELAY_HOST_CLOSE_REASON.SIGNED_OUT))
+
+    expect(memory.read('a')).toBe(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
+  })
+
+  it('expires an entry once its host may have been rebalanced away', () => {
+    const clock = { now: 1_000 }
+    const memory = memoryAt(clock)
+    memory.record('a', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
+
+    clock.now += ASSIGNMENT_LIMITS.dormantTtlMs - 1
+    expect(memory.read('a')).toBe(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
+
+    clock.now += 1
+    expect(memory.read('a')).toBeNull()
+    expect(memory.size()).toBe(0)
+  })
+
+  it('forgets on demand', () => {
+    const clock = { now: 1_000 }
+    const memory = memoryAt(clock)
+    memory.record('a', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
+
+    memory.forget('a')
+
+    expect(memory.read('a')).toBeNull()
+  })
+
+  it('drops the oldest survivors rather than growing without bound', () => {
+    const clock = { now: 1_000 }
+    const memory = memoryAt(clock)
+    for (let index = 0; index < 50_050; index++) {
+      memory.record(`host-${index}`, RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
+    }
+
+    expect(memory.size()).toBe(50_000)
+    expect(memory.read('host-0')).toBeNull()
+    expect(memory.read('host-50049')).toBe(RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
+  })
+
+  it('re-recording refreshes recency so a live host is not evicted first', () => {
+    const clock = { now: 1_000 }
+    const memory = memoryAt(clock)
+    memory.record('a', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
+    memory.record('b', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
+    memory.record('a', RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
+
+    expect([...['a', 'b'].map((key) => memory.read(key))]).toEqual([
+      RELAY_HOST_CLOSE_REASON.SIGNED_OUT,
+      RELAY_HOST_CLOSE_REASON.SIGNED_OUT
+    ])
+    expect(memory.size()).toBe(2)
+  })
+})

cloud/apps/relay/src/host-close-reason-memory.ts

@@ -0,0 +1,72 @@
+import {
+  ASSIGNMENT_LIMITS,
+  relayHostCloseReasonFrom,
+  type RelayHostCloseReason
+} from '@orca-cloud/relay-contract'
+
+// Retention matches the dormant assignment TTL: past it the host may have been
+// rebalanced onto another cell, so this cell is no longer the one a phone asks.
+const RETENTION_MS = ASSIGNMENT_LIMITS.dormantTtlMs
+// A fleet-wide auth outage signs out every host at once; the cap bounds that
+// burst well above any single cell's host count without becoming a leak.
+const MAX_ENTRIES = 50_000
+
+// Why in-memory and not Postgres: a phone reaches the cell its host's assignment
+// row already names, which is the same cell that watched the control socket
+// close. Losing this on a cell restart degrades to the pre-existing generic
+// verdict, so the failure mode is the old behaviour rather than a wrong one.
+export class HostCloseReasonMemory {
+  private readonly entries = new Map<string, { reason: RelayHostCloseReason; expiresAt: number }>()
+
+  constructor(private readonly now: () => number = Date.now) {}
+
+  // Silently ignores anything that is not a known reason, which is every close
+  // from a host that predates the field and every abrupt 1006.
+  record(key: string, reason: unknown): void {
+    const parsed = relayHostCloseReasonFrom(reason)
+    if (!parsed) {
+      return
+    }
+    this.entries.delete(key)
+    this.entries.set(key, { reason: parsed, expiresAt: this.now() + RETENTION_MS })
+    this.evict()
+  }
+
+  forget(key: string): void {
+    this.entries.delete(key)
+  }
+
+  read(key: string): RelayHostCloseReason | null {
+    const entry = this.entries.get(key)
+    if (!entry) {
+      return null
+    }
+    if (entry.expiresAt <= this.now()) {
+      this.entries.delete(key)
+      return null
+    }
+    return entry.reason
+  }
+
+  size(): number {
+    return this.entries.size
+  }
+
+  private evict(): void {
+    const now = this.now()
+    for (const [key, entry] of this.entries) {
+      if (entry.expiresAt > now) {
+        break
+      }
+      this.entries.delete(key)
+    }
+    // Insertion order is recency order (record deletes before setting), so the
+    // head is always the oldest survivor.
+    for (const key of this.entries.keys()) {
+      if (this.entries.size <= MAX_ENTRIES) {
+        break
+      }
+      this.entries.delete(key)
+    }
+  }
+}

cloud/apps/relay/src/host-session-client-accept.test.ts

@@ -0,0 +1,590 @@
+import { EventEmitter } from 'node:events'
+import { RELAY_CLOSE_CODE, RELAY_PROTOCOL_LIMITS } from '@orca-cloud/relay-contract'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import type WebSocket from 'ws'
+import type { RelayAssignmentStore } from './assignment-store.js'
+import type { RelayConfig } from './config.js'
+import type { CredentialReservation, RelayCredentialStore } from './credential-store.js'
+import {
+  CONTROL_LEASE_JITTER_MS,
+  CONTROL_LEASE_MS,
+  HostSessionRegistry
+} from './host-session-registry.js'
+import type { RelayRuntimeObserver } from './relay-observability.js'
+import type { RelayTokenClaims } from './relay-token-verifier.js'
+import { ProcessQueuedByteBudget } from './splice-forwarder.js'
+
+// Incident 2026-09-04 ~01:05Z: the phone's dial bound ran out while the cell was
+// still inside acceptClient's serialized Postgres phase (cell-inventory lock
+// contention). The cell then finished the work for a socket nobody held, holding
+// an activity lease for the 10s attach deadline before its timer unwound it, and
+// logged `host_data_reservation_already_bound`.
+
+class FakeSocket extends EventEmitter {
+  readonly OPEN = 1
+  readonly CLOSING = 2
+  readonly CLOSED = 3
+  readyState = this.OPEN
+  readonly send = vi.fn()
+  readonly close = vi.fn((code?: number, reason?: string) => {
+    this.readyState = this.CLOSED
+    this.emit('close', code, Buffer.from(reason ?? ''))
+  })
+  readonly terminate = vi.fn(() => {
+    this.readyState = this.CLOSED
+    this.emit('close')
+  })
+}
+
+const config = {
+  port: 8080,
+  publicUrl: 'https://relay-c3.example.com',
+  cellUrl: 'https://relay-c3.example.com',
+  authIssuer: 'https://auth.example.com',
+  authAudience: 'orca-relay',
+  jwksUrl: 'https://auth.example.com/jwks',
+  assignmentSigningKey: new Uint8Array(32),
+  role: 'cell',
+  cellId: 'production-gce-c3',
+  cells: [{ id: 'production-gce-c3', url: 'https://relay-c3.example.com', capacityRequests: 4_000 }],
+  adminAudience: 'https://relay-c3.example.com/v1/admin/drain',
+  deployServiceAccount: 'deploy@example.com',
+  runtimeServiceAccount: 'runtime@example.com',
+  adminJwksUrl: 'https://auth.example.com/admin-jwks',
+  databasePoolMax: 10,
+  publicAssignmentsEnabled: true,
+  publicAssignmentConcurrency: 2,
+  publicAssignmentQueueMax: 128,
+  publicAssignmentWaitMs: 4_000,
+  publicResolveConcurrency: 1,
+  publicResolveWaitMs: 5_000,
+  publicAssignmentRetryAfterSeconds: 5,
+  dataDir: './test-data'
+} satisfies RelayConfig
+
+const identity = {
+  sub: 'user-1',
+  prof: 'profile-1',
+  relayHostId: 'abcdefghijklmnop',
+  purpose: 'host-control',
+  exp: 4_102_444_800
+} satisfies RelayTokenClaims
+
+function deferred<T>(): { promise: Promise<T>; resolve: (value: T) => void } {
+  let resolve!: (value: T) => void
+  const promise = new Promise<T>((next) => (resolve = next))
+  return { promise, resolve }
+}
+
+const reservation: CredentialReservation = {
+  userId: identity.sub,
+  relayHostId: identity.relayHostId,
+  credentialKind: 'resume',
+  relayDeviceId: 'device-1',
+  tokenHash: 'hash',
+  reservationId: 'reservation-1',
+  leaseExpiresAt: Date.now() + 60_000,
+  acceptedCredentialVersion: 2,
+  acceptedAs: 'current'
+}
+
+function harness(options: { random?: () => number; now?: () => number } = {}) {
+  const acquireActivity = vi.fn().mockResolvedValue(undefined)
+  const releaseActivity = vi.fn().mockResolvedValue(true)
+  const assignments = {
+    activateControl: vi.fn().mockResolvedValue('control:production-gce-c3:1'),
+    markMigrationTargetRegistered: vi.fn().mockResolvedValue(undefined),
+    resolve: vi.fn().mockResolvedValue({ cellId: config.cellId }),
+    acquireActivity,
+    renewControlActivity: vi.fn().mockResolvedValue(undefined),
+    releaseActivity
+  } as unknown as RelayAssignmentStore
+  const store = {
+    resolveResume: vi.fn().mockResolvedValue({ userId: identity.sub }),
+    reserveCredential: vi.fn().mockResolvedValue(reservation),
+    failReservation: vi.fn().mockResolvedValue(undefined),
+    recordConnectionBasis: vi.fn().mockResolvedValue(undefined),
+    deactivateBasis: vi.fn().mockResolvedValue(undefined)
+  }
+  const observer = {
+    recordAuth: vi.fn(),
+    recordForwardedBytes: vi.fn(),
+    recordHttp: vi.fn(),
+    recordReconnect: vi.fn(),
+    recordSql: vi.fn(),
+    recordClientAcceptAbandoned: vi.fn(),
+    recordClientAcceptCompleted: vi.fn(),
+    recordControlRtt: vi.fn()
+  } satisfies RelayRuntimeObserver
+  const registry = new HostSessionRegistry(
+    config,
+    vi.fn(),
+    store as unknown as RelayCredentialStore,
+    assignments,
+    new ProcessQueuedByteBudget(),
+    observer,
+    options.now,
+    options.random
+  )
+  const activate = (
+    registry as unknown as {
+      activate: (
+        socket: WebSocket,
+        identity: RelayTokenClaims,
+        existing: null,
+        generation: number,
+        rebind: boolean,
+        assignmentEpoch: number,
+        appVersion: string
+      ) => Promise<void>
+    }
+  ).activate.bind(registry)
+  return { registry, store, assignments, acquireActivity, releaseActivity, observer, activate }
+}
+
+async function activeHost(h: ReturnType<typeof harness>): Promise<FakeSocket> {
+  const control = new FakeSocket()
+  await h.activate(control as unknown as WebSocket, identity, null, 1, false, 1, '1.4.197')
+  return control
+}
+
+describe('client accept abandoned mid-DB-phase', () => {
+  beforeEach(() => vi.useFakeTimers())
+  afterEach(() => {
+    vi.clearAllTimers()
+    vi.useRealTimers()
+  })
+
+  it('stops after a slow activity acquire when the phone already hung up', async () => {
+    const h = harness()
+    const control = await activeHost(h)
+    const slowAcquire = deferred<void>()
+    h.acquireActivity.mockReturnValueOnce(slowAcquire.promise)
+    const capacity = { bind: vi.fn(), release: vi.fn() }
+    const client = new FakeSocket()
+    const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+    try {
+      const accepting = h.registry.acceptClient(
+        client as unknown as WebSocket,
+        identity.relayHostId,
+        'credential',
+        capacity
+      )
+      await vi.advanceTimersByTimeAsync(0)
+      expect(h.acquireActivity).toHaveBeenCalledOnce()
+      // The phone's 12s bound fires while the cell still waits on Postgres.
+      client.close(1000, 'client bound')
+      capacity.release()
+      slowAcquire.resolve()
+      await accepting
+
+      // No conn-open reached the desktop; nothing pending; the lease it just took is
+      // released instead of leaking to expiry cleanup; bind never throws.
+      expect(control.send).not.toHaveBeenCalledWith(expect.stringContaining('conn-open'))
+      expect(capacity.bind).not.toHaveBeenCalled()
+      const session = h.registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })
+      expect(session?.pendingConns.size).toBe(0)
+      expect(h.store.failReservation).toHaveBeenCalledWith(reservation)
+      expect(h.releaseActivity).toHaveBeenCalledWith(
+        { userId: identity.sub, relayHostId: identity.relayHostId },
+        expect.stringMatching(/^confirmation:/)
+      )
+      expect(h.observer.recordClientAcceptAbandoned).toHaveBeenCalledWith(
+        'activity',
+        expect.any(Number)
+      )
+      const line = warn.mock.calls.map((call) => String(call[0])).find((entry) =>
+        entry.includes('orca_relay_client_accept_abandoned')
+      )
+      expect(line).toBeDefined()
+      expect(JSON.parse(line!)).toMatchObject({ stage: 'activity' })
+      expect(line).not.toContain(identity.relayHostId)
+    } finally {
+      warn.mockRestore()
+      h.registry.drain(0)
+      vi.advanceTimersByTime(0)
+    }
+  })
+
+  it('stops after a slow credential reservation without acquiring an activity lease', async () => {
+    const h = harness()
+    await activeHost(h)
+    const slowReserve = deferred<CredentialReservation>()
+    h.store.reserveCredential.mockReturnValueOnce(slowReserve.promise)
+    const client = new FakeSocket()
+    const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+    try {
+      const accepting = h.registry.acceptClient(
+        client as unknown as WebSocket,
+        identity.relayHostId,
+        'credential'
+      )
+      await vi.advanceTimersByTimeAsync(0)
+      client.close(1000, 'client bound')
+      slowReserve.resolve(reservation)
+      await accepting
+
+      expect(h.acquireActivity).not.toHaveBeenCalled()
+      expect(h.store.failReservation).toHaveBeenCalledWith(reservation)
+      expect(h.observer.recordClientAcceptAbandoned).toHaveBeenCalledWith(
+        'credential',
+        expect.any(Number)
+      )
+    } finally {
+      warn.mockRestore()
+      h.registry.drain(0)
+      vi.advanceTimersByTime(0)
+    }
+  })
+
+  it('stops after a slow resume lookup before starting the invite and assignment lookups', async () => {
+    const h = harness()
+    await activeHost(h)
+    const store = h.store as typeof h.store & { resolveInviteForMove: ReturnType<typeof vi.fn> }
+    store.resolveInviteForMove = vi.fn().mockResolvedValue(null)
+    const slowResume = deferred<null>()
+    h.store.resolveResume.mockReturnValueOnce(slowResume.promise)
+    const resolveAssignment = (h.assignments as unknown as { resolve: ReturnType<typeof vi.fn> })
+      .resolve
+    resolveAssignment.mockClear()
+    const client = new FakeSocket()
+    const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+    try {
+      const accepting = h.registry.acceptClient(
+        client as unknown as WebSocket,
+        identity.relayHostId,
+        'credential'
+      )
+      await vi.advanceTimersByTimeAsync(0)
+      client.close(1000, 'client bound')
+      slowResume.resolve(null)
+      await accepting
+
+      expect(store.resolveInviteForMove).not.toHaveBeenCalled()
+      expect(resolveAssignment).not.toHaveBeenCalled()
+      expect(h.store.reserveCredential).not.toHaveBeenCalled()
+      expect(h.observer.recordClientAcceptAbandoned).toHaveBeenCalledWith(
+        'assignment',
+        expect.any(Number)
+      )
+    } finally {
+      warn.mockRestore()
+      h.registry.drain(0)
+      vi.advanceTimersByTime(0)
+    }
+  })
+
+  it('stops after a slow same-cell assignment resolve, before reserving a credential', async () => {
+    const h = harness()
+    await activeHost(h)
+    const resolveAssignment = (h.assignments as unknown as { resolve: ReturnType<typeof vi.fn> })
+      .resolve
+    const slowResolve = deferred<{ cellId: string }>()
+    resolveAssignment.mockReturnValueOnce(slowResolve.promise)
+    const client = new FakeSocket()
+    const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+    try {
+      const accepting = h.registry.acceptClient(
+        client as unknown as WebSocket,
+        identity.relayHostId,
+        'credential'
+      )
+      await vi.advanceTimersByTimeAsync(0)
+      client.close(1000, 'client bound')
+      // A correct, same-cell assignment: only the closed socket stops the accept.
+      slowResolve.resolve({ cellId: config.cellId })
+      await accepting
+
+      // Proves the accept reached the third guard, not the first.
+      expect(resolveAssignment).toHaveBeenCalled()
+      expect(h.store.reserveCredential).not.toHaveBeenCalled()
+      expect(h.observer.recordClientAcceptAbandoned).toHaveBeenCalledWith(
+        'assignment',
+        expect.any(Number)
+      )
+    } finally {
+      warn.mockRestore()
+      h.registry.drain(0)
+      vi.advanceTimersByTime(0)
+    }
+  })
+
+  it('still opens the connection when the phone is holding on', async () => {
+    const h = harness()
+    const control = await activeHost(h)
+    const capacity = { bind: vi.fn(), release: vi.fn() }
+    const client = new FakeSocket()
+    await h.registry.acceptClient(
+      client as unknown as WebSocket,
+      identity.relayHostId,
+      'credential',
+      capacity
+    )
+    expect(control.send).toHaveBeenCalledWith(expect.stringContaining('"type":"conn-open"'))
+    expect(capacity.bind).toHaveBeenCalledOnce()
+    expect(h.observer.recordClientAcceptAbandoned).not.toHaveBeenCalled()
+    expect(client.close).not.toHaveBeenCalled()
+    h.registry.drain(0)
+    vi.advanceTimersByTime(0)
+  })
+})
+
+describe('successful client accept timing', () => {
+  beforeEach(() => vi.useFakeTimers())
+  afterEach(() => {
+    vi.clearAllTimers()
+    vi.useRealTimers()
+  })
+
+  it('times every serialized stage plus the attach window once relay-hello lands', async () => {
+    let now = 1_700_000_000_000
+    const h = harness({ now: () => now })
+    const control = await activeHost(h)
+    h.store.resolveResume.mockImplementationOnce(async () => {
+      now += 5
+      return { userId: identity.sub }
+    })
+    h.store.reserveCredential.mockImplementationOnce(async () => {
+      now += 7
+      return reservation
+    })
+    h.acquireActivity.mockImplementationOnce(async () => {
+      now += 11
+    })
+    h.store.recordConnectionBasis.mockImplementationOnce(async () => {
+      now += 3
+    })
+    const client = new FakeSocket()
+    const hostData = new FakeSocket()
+    const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
+    try {
+      await h.registry.acceptClient(client as unknown as WebSocket, identity.relayHostId, 'cred')
+      const connOpen = JSON.parse(
+        String(control.send.mock.calls.find((call) => String(call[0]).includes('conn-open'))![0])
+      ) as { connId: string; connTicket: string }
+      // The desktop's data leg is the attach window this is meant to expose.
+      now += 23
+      const accepted = await h.registry.acceptHostData(
+        hostData as unknown as WebSocket,
+        connOpen.connId,
+        connOpen.connTicket,
+        1
+      )
+
+      expect(accepted).toBe(true)
+      expect(h.observer.recordClientAcceptCompleted).toHaveBeenCalledWith({
+        totalMs: 49,
+        stageMs: { assignment: 5, credential: 7, activity: 11, attach: 23, basis: 3 }
+      })
+      const line = log.mock.calls
+        .map((call) => String(call[0]))
+        .find((entry) => entry.includes('orca_relay_client_accept_completed'))
+      expect(line).toBeDefined()
+      const event = JSON.parse(line!) as {
+        role: string
+        cellId: string
+        region: string
+        credentialKind: string
+        stageMs: Record<string, number>
+        totalMs: number
+        relayHostIdDigest: string
+      }
+      expect(event.credentialKind).toBe('resume')
+      // Joins the line back to the emitting process, like the runtime metrics event.
+      expect(event).toMatchObject({ role: 'cell', cellId: config.cellId, region: 'us-central1' })
+      expect(Object.keys(event.stageMs).sort()).toEqual([
+        'activity',
+        'assignment',
+        'attach',
+        'basis',
+        'credential'
+      ])
+      for (const stage of Object.values(event.stageMs)) expect(stage).toBeGreaterThanOrEqual(0)
+      // The stages tile the accept end to end: every millisecond is attributed.
+      const summed = Object.values(event.stageMs).reduce((total, stage) => total + stage, 0)
+      expect(summed).toBe(event.totalMs)
+      expect(event.relayHostIdDigest).toMatch(/^[0-9a-f]{12}$/)
+      expect(line).not.toContain(identity.relayHostId)
+    } finally {
+      log.mockRestore()
+      h.registry.drain(0)
+      vi.advanceTimersByTime(0)
+    }
+  })
+})
+
+// Fires one heartbeat and returns the `t` of the ping it sent, which is the only
+// echo the registry will time.
+async function advanceToPing(control: FakeSocket, clock: { now: number }): Promise<number> {
+  clock.now += RELAY_PROTOCOL_LIMITS.controlPingIntervalMs
+  await vi.advanceTimersByTimeAsync(RELAY_PROTOCOL_LIMITS.controlPingIntervalMs)
+  const ping = control.send.mock.calls
+    .filter((call) => String(call[0]).includes('"type":"ping"'))
+    .at(-1)!
+  return (JSON.parse(String(ping[0])) as { t: number }).t
+}
+
+describe('control round-trip sampling', () => {
+  beforeEach(() => vi.useFakeTimers())
+  afterEach(() => {
+    vi.clearAllTimers()
+    vi.useRealTimers()
+  })
+
+  it('logs a host once at the fourth sample and not again within the hour', async () => {
+    const clock = { now: 1_700_000_000_000 }
+    const h = harness({ now: () => clock.now })
+    const control = await activeHost(h)
+    const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
+    const rttLines = (): string[] =>
+      log.mock.calls
+        .map((call) => String(call[0]))
+        .filter((entry) => entry.includes('orca_relay_host_control_rtt'))
+    // One heartbeat, then the desktop's echo of that ping's own `t` 40 ms later.
+    const roundTrip = async (): Promise<void> => {
+      const pingAt = await advanceToPing(control, clock)
+      clock.now += 40
+      control.emit('message', JSON.stringify({ type: 'pong', t: pingAt }), false)
+    }
+    try {
+      for (let round = 0; round < 3; round++) await roundTrip()
+      expect(h.observer.recordControlRtt).toHaveBeenCalledTimes(3)
+      expect(rttLines()).toHaveLength(0)
+
+      await roundTrip()
+      expect(h.observer.recordControlRtt).toHaveBeenLastCalledWith(40)
+      expect(rttLines()).toHaveLength(1)
+      expect(JSON.parse(rttLines()[0]!)).toMatchObject({
+        event: 'orca_relay_host_control_rtt',
+        role: 'cell',
+        cellId: config.cellId,
+        region: 'us-central1',
+        rttMsMedian: 40,
+        sampleCount: 4
+      })
+      expect(rttLines()[0]).not.toContain(identity.relayHostId)
+
+      // Later samples keep feeding the fleet metric, but stay silent for an hour.
+      for (let round = 0; round < 8; round++) await roundTrip()
+      expect(h.observer.recordControlRtt).toHaveBeenCalledTimes(12)
+      expect(rttLines()).toHaveLength(1)
+
+      const elapsedStart = clock.now
+      while (clock.now - elapsedStart < 60 * 60 * 1000) await roundTrip()
+      expect(rttLines()).toHaveLength(2)
+    } finally {
+      log.mockRestore()
+      h.registry.drain(0)
+      vi.advanceTimersByTime(0)
+    }
+  })
+
+  it('ignores a pong that answers no outstanding ping', async () => {
+    const clock = { now: 1_700_000_000_000 }
+    const h = harness({ now: () => clock.now })
+    const control = await activeHost(h)
+    try {
+      // Nothing has been pinged yet, so even a plausible echo is not a round trip.
+      control.emit('message', JSON.stringify({ type: 'pong' }), false)
+      control.emit('message', JSON.stringify({ type: 'pong', t: 'later' }), false)
+      control.emit('message', JSON.stringify({ type: 'pong', t: clock.now }), false)
+      control.emit('message', JSON.stringify({ type: 'pong', t: clock.now - 10 }), false)
+      expect(h.observer.recordControlRtt).not.toHaveBeenCalled()
+
+      const pingAt = await advanceToPing(control, clock)
+      // A guessed timestamp is not the outstanding ping's `t`, so it is dropped.
+      control.emit('message', JSON.stringify({ type: 'pong', t: pingAt - 1 }), false)
+      control.emit('message', JSON.stringify({ type: 'pong', t: pingAt + 1 }), false)
+      expect(h.observer.recordControlRtt).not.toHaveBeenCalled()
+
+      clock.now += 10
+      control.emit('message', JSON.stringify({ type: 'pong', t: pingAt }), false)
+      expect(h.observer.recordControlRtt).toHaveBeenCalledWith(10)
+    } finally {
+      h.registry.drain(0)
+      vi.advanceTimersByTime(0)
+    }
+  })
+
+  it('records one sample per ping however many pongs a host floods', async () => {
+    const clock = { now: 1_700_000_000_000 }
+    const h = harness({ now: () => clock.now })
+    const control = await activeHost(h)
+    const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
+    try {
+      const pingAt = await advanceToPing(control, clock)
+      clock.now += 12
+      for (let flood = 0; flood < 5_000; flood++) {
+        control.emit('message', JSON.stringify({ type: 'pong', t: pingAt }), false)
+        control.emit('message', JSON.stringify({ type: 'pong', t: clock.now }), false)
+      }
+      // One answered ping is one process-wide sample and one per-session sample, so
+      // neither the metric window nor the hourly log line can be flooded.
+      expect(h.observer.recordControlRtt).toHaveBeenCalledTimes(1)
+      expect(h.observer.recordControlRtt).toHaveBeenCalledWith(12)
+      expect(
+        log.mock.calls.filter((call) => String(call[0]).includes('orca_relay_host_control_rtt'))
+      ).toHaveLength(0)
+    } finally {
+      log.mockRestore()
+      h.registry.drain(0)
+      vi.advanceTimersByTime(0)
+    }
+  })
+})
+
+describe('control lease jitter', () => {
+  beforeEach(() => vi.useFakeTimers())
+  afterEach(() => {
+    vi.clearAllTimers()
+    vi.useRealTimers()
+  })
+
+  it('grants a lease uniformly around its mean so cohorts drift apart at the same mean rate', async () => {
+    const now = 1_700_000_000_000
+    const helloAck = (socket: FakeSocket) =>
+      JSON.parse(
+        String(socket.send.mock.calls.find((call) => String(call[0]).includes('host-hello-ack'))![0])
+      ) as { leaseExpiresAt: number }
+
+    const shortest = harness({ now: () => now, random: () => 0 })
+    const shortestAck = helloAck(await activeHost(shortest))
+    const centered = harness({ now: () => now, random: () => 0.5 })
+    const centeredAck = helloAck(await activeHost(centered))
+    const longestRoll = 0.999999
+    const longest = harness({ now: () => now, random: () => longestRoll })
+    const longestAck = helloAck(await activeHost(longest))
+
+    // Pinned, not bounded: a jitter clamped to one side still satisfies an upper
+    // bound, so only the exact top of the band proves it is symmetric.
+    const longestOffset = Math.floor((longestRoll * 2 - 1) * CONTROL_LEASE_JITTER_MS)
+    expect(shortestAck.leaseExpiresAt).toBe(now + CONTROL_LEASE_MS - CONTROL_LEASE_JITTER_MS)
+    expect(centeredAck.leaseExpiresAt).toBe(now + CONTROL_LEASE_MS)
+    expect(longestAck.leaseExpiresAt).toBe(now + CONTROL_LEASE_MS + longestOffset)
+    shortest.registry.drain(0)
+    centered.registry.drain(0)
+    longest.registry.drain(0)
+    vi.advanceTimersByTime(0)
+  })
+
+  it('rebinds re-roll the jitter instead of pinning the cohort phase', async () => {
+    const now = 1_700_000_000_000
+    let roll = 0
+    const h = harness({ now: () => now, random: () => roll })
+    const first = await activeHost(h)
+    const session = h.registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
+    const firstLease = session.leaseExpiresAt
+    roll = 0.75
+    const rebind = new FakeSocket()
+    await (
+      h.registry as unknown as {
+        activate: (...args: unknown[]) => Promise<void>
+      }
+    ).activate(rebind as unknown as WebSocket, identity, session, 1, true, 1, '1.4.197')
+    expect(session.leaseExpiresAt).toBe(now + CONTROL_LEASE_MS + CONTROL_LEASE_JITTER_MS / 2)
+    expect(session.leaseExpiresAt).not.toBe(firstLease)
+    expect(first.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.PEER_DROPPED, 'control rebound')
+    h.registry.drain(0)
+    vi.advanceTimersByTime(0)
+  })
+})

cloud/apps/relay/src/host-session-registry.test.ts

@@ -3,6 +3,7 @@ import {
   ASSIGNMENT_LIMITS,
   CONTROL_CONTINUITY_LIMITS,
   RELAY_CLOSE_CODE,
+  RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS,
   RELAY_PROTOCOL_LIMITS
 } from '@orca-cloud/relay-contract'
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
@@ -1005,3 +1006,115 @@ describe('control lease recovery after the session is gone', () => {
     }
   })
 })
+
+describe('host hello ack pending connections', () => {
+  const DETAILS = new Set([RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS])
+  const LEGACY_ENTRY = { connId: 'conn-1', connTicket: 'T'.repeat(43) }
+  const DETAILED_ENTRY = { ...LEGACY_ENTRY, kind: 'invite', relayDeviceId: 'device-1' }
+
+  beforeEach(() => vi.useFakeTimers())
+  afterEach(() => {
+    vi.clearAllTimers()
+    vi.useRealTimers()
+  })
+
+  function newRegistry(): ReturnType<typeof createRegistry> {
+    return createRegistry(
+      vi
+        .fn<RelayAssignmentStore['activateControl']>()
+        .mockResolvedValue('control:production-gce-c3:1')
+    )
+  }
+
+  function addPendingConnection(session: HostSession): void {
+    session.pendingConns.set('conn-1', {
+      ...LEGACY_ENTRY,
+      reservation: {
+        userId: identity.sub,
+        relayHostId: identity.relayHostId,
+        credentialKind: 'invite',
+        relayDeviceId: 'device-1'
+      },
+      client: new FakeSocket() as unknown as WebSocket,
+      attachTimer: setTimeout(() => {}, 60_000),
+      credentialActivityId: null
+    } as unknown as Parameters<typeof session.pendingConns.set>[1])
+  }
+
+  function sentAck(socket: FakeSocket): Record<string, unknown> {
+    const acks = socket.send.mock.calls
+      .map((call) => JSON.parse(String(call[0])) as Record<string, unknown>)
+      .filter((message) => message.type === 'host-hello-ack')
+    return acks.at(-1)!
+  }
+
+  function sessionOf(registry: HostSessionRegistry): HostSession {
+    return registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
+  }
+
+  async function ackFor(capabilities?: ReadonlySet<string>): Promise<Record<string, unknown>> {
+    const { registry, activate } = newRegistry()
+    const socket = new FakeSocket()
+    registry.acceptControl(
+      socket as unknown as WebSocket,
+      identity,
+      undefined,
+      capabilities ?? new Set()
+    )
+    await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
+    const session = sessionOf(registry)
+    addPendingConnection(session)
+    socket.send.mockClear()
+    ;(registry as unknown as { sendHelloAck(session: HostSession): void }).sendHelloAck(session)
+    return sentAck(socket)
+  }
+
+  async function ackAfterRebind(
+    first: ReadonlySet<string>,
+    successor: ReadonlySet<string>
+  ): Promise<{ opening: Record<string, unknown>; rebound: Record<string, unknown> }> {
+    const { registry, activate } = newRegistry()
+    const opening = new FakeSocket()
+    registry.acceptControl(opening as unknown as WebSocket, identity, undefined, first)
+    await activate(opening as unknown as WebSocket, identity, null, 1, false, 1)
+    const session = sessionOf(registry)
+    addPendingConnection(session)
+    opening.send.mockClear()
+    ;(registry as unknown as { sendHelloAck(session: HostSession): void }).sendHelloAck(session)
+
+    const rebound = new FakeSocket()
+    registry.acceptControl(rebound as unknown as WebSocket, identity, undefined, successor)
+    await activate(rebound as unknown as WebSocket, identity, session, 1, true, 1)
+    return { opening: sentAck(opening), rebound: sentAck(rebound) }
+  }
+
+  it('states the pending kind and device to a host that advertised it can read them', async () => {
+    const ack = await ackFor(DETAILS)
+
+    expect(ack.pendingConns).toEqual([DETAILED_ENTRY])
+  })
+
+  it('restates only the identifiers to a host that never advertised the capability', async () => {
+    // A shipped host parses these entries strictly, so an unannounced key fails
+    // the whole ack parse and kills a control that was working.
+    const ack = await ackFor()
+
+    expect(ack.pendingConns).toEqual([LEGACY_ENTRY])
+  })
+
+  it('downgrades the restated entry when the successor control drops the capability', async () => {
+    // The capability belongs to the socket, not the session: a rebind can land a
+    // control whose decoder is older than the one that opened the session.
+    const { opening, rebound } = await ackAfterRebind(DETAILS, new Set())
+
+    expect(opening.pendingConns).toEqual([DETAILED_ENTRY])
+    expect(rebound.pendingConns).toEqual([LEGACY_ENTRY])
+  })
+
+  it('upgrades the restated entry when the successor control adds the capability', async () => {
+    const { opening, rebound } = await ackAfterRebind(new Set(), DETAILS)
+
+    expect(opening.pendingConns).toEqual([LEGACY_ENTRY])
+    expect(rebound.pendingConns).toEqual([DETAILED_ENTRY])
+  })
+})

cloud/apps/relay/src/host-session-registry.ts

@@ -1,6 +1,7 @@
 import { createHash, createHmac, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto'
 import {
   ASSIGNMENT_LIMITS,
+  RELAY_DEFAULT_REGION,
   AuthRefreshSchema,
   buildHostChallengePlaintext,
   buildHostProofMacInput,
@@ -13,8 +14,11 @@ import {
   HostChallengeAckSchema,
   HostHelloSchema,
   InviteCreateSchema,
+  RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS,
   RELAY_PROTOCOL_LIMITS,
-  RELAY_CLOSE_CODE
+  RELAY_CLOSE_CODE,
+  type RelayHostCloseReason,
+  type RelayRegion
 } from '@orca-cloud/relay-contract'
 import nacl from 'tweetnacl'
 import type WebSocket from 'ws'
@@ -25,9 +29,15 @@ import {
   RelayCredentialStore,
   type CredentialReservation
 } from './credential-store.js'
+import { HostCloseReasonMemory } from './host-close-reason-memory.js'
 import { relayHostLogDigest } from './relay-host-log-digest.js'
 import type { RelayTokenClaims } from './relay-token-verifier.js'
-import type { RelayRuntimeObserver } from './relay-observability.js'
+import {
+  percentile,
+  type RelayClientAcceptStage,
+  type RelayClientAcceptTimedStage,
+  type RelayRuntimeObserver
+} from './relay-observability.js'
 import type { PendingHostDataReservation } from './relay-connection-ledger.js'
 import { closeRelayWebSocket } from './relay-websocket-close.js'
 import { ProcessQueuedByteBudget, wireSplice } from './splice-forwarder.js'
@@ -43,6 +53,20 @@ function printableCloseReason(reason: Buffer | string): string {
 type VerifyRelayToken = (token: string) => Promise<RelayTokenClaims | null>
 type HostState = 'proving' | 'active' | 'orphaned' | 'drain-only' | 'closed'
 
+// A host's distance to its cell moves on the scale of a rehome, not a heartbeat,
+// so a short window is enough to ride out one stalled ping.
+const CONTROL_RTT_WINDOW = 8
+const CONTROL_RTT_LOG_SAMPLE_THRESHOLD = 4
+const CONTROL_RTT_LOG_INTERVAL_MS = 60 * 60 * 1000
+// A pong claiming a multi-minute round trip is clock skew, not distance.
+const CONTROL_RTT_MAX_PLAUSIBLE_MS = 120_000
+
+// Wall clock can step backwards mid-accept; a negative latency would poison the
+// percentiles it feeds.
+function nonNegativeMs(elapsedMs: number): number {
+  return Math.max(0, elapsedMs)
+}
+
 const CONTROL_ACTIVITY_RENEWAL_INTERVAL_MS = RELAY_PROTOCOL_LIMITS.controlPingIntervalMs * 2
 // Preserve the existing 75s renewal runway after doubling the successful-call interval.
 const CONTROL_ACTIVITY_LEASE_MS =
@@ -66,6 +90,10 @@ export type HostSession = {
   orphanTimer: ReturnType<typeof setTimeout> | null
   heartbeatTimer: ReturnType<typeof setInterval> | null
   lastPongAt: number
+  // The `t` of the ping still waiting for its echo; null once one has answered it.
+  pendingPingAt: number | null
+  controlRttSamplesMs: number[]
+  controlRttLoggedAt: number | null
   activityRenewalDueAt: number
   activityRenewalAttempt: number
   activityRenewalCompletedAttempt: number
@@ -93,6 +121,15 @@ type PendingConnection = {
   attachTimer: ReturnType<typeof setTimeout>
   credentialActivityId: string | null
   capacityReservation?: PendingHostDataReservation
+  timing: ClientAcceptTiming
+}
+
+// Carries the phone-side accept clock across to the desktop's data leg, which
+// lands in a separate call and is the only place the accept is known to succeed.
+type ClientAcceptTiming = {
+  startedAt: number
+  connOpenAt: number
+  stageMs: Record<RelayClientAcceptStage, number>
 }
 
 function decodeCanonicalBase64(value: string, bytes: number): Uint8Array | null {
@@ -127,9 +164,24 @@ function send(socket: WebSocket, type: string, message: object): void {
 // stalled predecessor only accumulates doomed sockets.
 const ACTIVATION_QUEUE_WAIT_MS = 30_000
 
+// Why: this lease bounds how long a host lingers on a cell after a missed drain,
+// and rebinding it is the only passive rebalancing we have, so it has to stay
+// finite. 6h keeps both properties while cutting control-activation traffic on
+// the contended cell-inventory lock ~6x; the relay JWT (5 min, refreshed by the
+// desktop) and the 75s silence watchdog are enforced separately, so a longer
+// grant authorizes nothing extra. Symmetric jitter walks same-minute reconnect
+// cohorts apart across cycles without changing the mean rebind rate.
+export const CONTROL_LEASE_MS = 6 * 60 * 60 * 1000
+export const CONTROL_LEASE_JITTER_MS = 30 * 60 * 1000
+
 export class HostSessionRegistry {
   private readonly sessions = new Map<string, HostSession>()
   private readonly activationQueues = new Map<string, Promise<void>>()
+  // Why it outlives `sessions`: the orphan grace deletes the session within 30s,
+  // but a signed-out desktop never comes back, so the phone that asks minutes
+  // later would otherwise find nothing to explain its rejection with.
+  private readonly hostCloseReasons = new HostCloseReasonMemory(() => this.now())
+  private readonly hostCapabilities = new WeakMap<WebSocket, ReadonlySet<string>>()
   private draining = false
 
   constructor(
@@ -139,9 +191,16 @@ export class HostSessionRegistry {
     private readonly assignments: RelayAssignmentStore,
     private readonly queuedByteBudget: ProcessQueuedByteBudget,
     private readonly observer: RelayRuntimeObserver,
-    private readonly now: () => number = Date.now
+    private readonly now: () => number = Date.now,
+    private readonly random: () => number = Math.random
   ) {}
 
+  // Uniform over [CONTROL_LEASE_MS - jitter, CONTROL_LEASE_MS + jitter).
+  private controlLeaseExpiresAt(): number {
+    const offset = Math.floor((this.random() * 2 - 1) * CONTROL_LEASE_JITTER_MS)
+    return this.now() + CONTROL_LEASE_MS + offset
+  }
+
   async acceptClient(
     socket: WebSocket,
     hostId: string,
@@ -153,10 +212,42 @@ export class HostSessionRegistry {
       this.rejectClient(socket, RELAY_CLOSE_CODE.DRAINING)
       return
     }
+    // Why: the accept runs several serialized Postgres calls behind the contended
+    // cell-inventory lock, and phones bound their dial. Finishing the work for a
+    // phone that already hung up took an activity lease held for the 10s attach
+    // deadline, then failed at bind with host_data_reservation_already_bound.
+    const acceptStartedAt = this.now()
+    const abandonedByClient = (stage: RelayClientAcceptStage, cleanup?: () => void): boolean => {
+      if (socket.readyState === socket.OPEN) return false
+      capacityReservation?.release()
+      cleanup?.()
+      const elapsedMs = this.now() - acceptStartedAt
+      this.observer.recordClientAcceptAbandoned?.(stage, elapsedMs)
+      console.warn(
+        JSON.stringify({ event: 'orca_relay_client_accept_abandoned', stage, elapsedMs })
+      )
+      return true
+    }
+    const stageMs: Record<RelayClientAcceptStage, number> = {
+      assignment: 0,
+      credential: 0,
+      activity: 0
+    }
+    let stageCursor = acceptStartedAt
+    const markStage = (stage: RelayClientAcceptStage): void => {
+      const at = this.now()
+      stageMs[stage] = at - stageCursor
+      stageCursor = at
+    }
     if (this.config.role === 'cell') {
-      const outerIdentity =
-        (await this.store.resolveResume(hostId, credential)) ??
-        (await this.store.resolveInviteForMove(hostId, credential))
+      // Each lookup is its own pooled round trip; stop between them once the phone
+      // has left instead of running the rest of the chain for nobody.
+      let outerIdentity = await this.store.resolveResume(hostId, credential)
+      if (abandonedByClient('assignment')) return
+      if (!outerIdentity) {
+        outerIdentity = await this.store.resolveInviteForMove(hostId, credential)
+        if (abandonedByClient('assignment')) return
+      }
       const assignment = outerIdentity
         ? await this.assignments.resolve({ userId: outerIdentity.userId, relayHostId: hostId })
         : null
@@ -166,7 +257,9 @@ export class HostSessionRegistry {
         this.rejectClient(socket, RELAY_CLOSE_CODE.WRONG_CELL)
         return
       }
+      if (abandonedByClient('assignment')) return
     }
+    markStage('assignment')
     const reservation = await this.store.reserveCredential(hostId, credential)
     if (!reservation) {
       capacityReservation?.release()
@@ -175,7 +268,10 @@ export class HostSessionRegistry {
       return
     }
     this.observer.recordAuth(true)
-    const session = this.sessions.get(this.key(reservation.userId, hostId))
+    if (abandonedByClient('credential', () => this.failReservationBestEffort(reservation))) return
+    markStage('credential')
+    const sessionKey = this.key(reservation.userId, hostId)
+    const session = this.sessions.get(sessionKey)
     if (
       !session ||
       session.state !== 'active' ||
@@ -184,7 +280,13 @@ export class HostSessionRegistry {
     ) {
       capacityReservation?.release()
       await this.store.failReservation(reservation)
-      this.rejectClient(socket, RELAY_CLOSE_CODE.HOST_OFFLINE)
+      // The only rejection that can name a cause: the host is genuinely absent.
+      // The attach-deadline 4404 below fires while control is still connected.
+      this.rejectClient(
+        socket,
+        RELAY_CLOSE_CODE.HOST_OFFLINE,
+        this.hostCloseReasons.read(sessionKey)
+      )
       return
     }
     if (session.activeConnIds.size + session.pendingConns.size >= 8) {
@@ -214,6 +316,15 @@ export class HostSessionRegistry {
         return
       }
     }
+    if (
+      abandonedByClient('activity', () => {
+        this.failReservationBestEffort(reservation)
+        if (credentialActivityId) this.releaseActivityBestEffort(identity, credentialActivityId)
+      })
+    ) {
+      return
+    }
+    markStage('activity')
     const attachTimer = setTimeout(() => {
       session.pendingConns.delete(connId)
       capacityReservation?.release()
@@ -228,7 +339,10 @@ export class HostSessionRegistry {
       client: socket,
       attachTimer,
       credentialActivityId,
-      capacityReservation
+      capacityReservation,
+      // Attach starts where the activity stage ended, so the conn-open send is
+      // charged to it and no wall-clock gap goes unattributed.
+      timing: { startedAt: acceptStartedAt, connOpenAt: stageCursor, stageMs }
     }
     capacityReservation?.bind(connId)
     session.pendingConns.set(connId, pending)
@@ -275,6 +389,7 @@ export class HostSessionRegistry {
       return false
     }
     this.observer.recordAuth(true)
+    const attachedAt = this.now()
     clearTimeout(pending.attachTimer)
     session.pendingConns.delete(connId)
     session.activeConnIds.add(connId)
@@ -348,6 +463,7 @@ export class HostSessionRegistry {
       close()
       return false
     }
+    const helloAt = this.now()
     send(pending.client, 'relay-hello', {
       ok: true,
       credentialKind: pending.reservation.credentialKind,
@@ -363,14 +479,92 @@ export class HostSessionRegistry {
           }
         : {})
     })
+    this.recordClientAcceptCompleted(session, pending, attachedAt, helloAt)
     return true
   }
 
+  // The stages tile the whole accept, so their sum is the total minus only the
+  // clamping above: `basis` is the splice lease and connection-basis writes that
+  // land between the host data leg and relay-hello.
+  private recordClientAcceptCompleted(
+    session: HostSession,
+    pending: PendingConnection,
+    attachedAt: number,
+    helloAt: number
+  ): void {
+    const stageMs: Record<RelayClientAcceptTimedStage, number> = {
+      assignment: nonNegativeMs(pending.timing.stageMs.assignment),
+      credential: nonNegativeMs(pending.timing.stageMs.credential),
+      activity: nonNegativeMs(pending.timing.stageMs.activity),
+      attach: nonNegativeMs(attachedAt - pending.timing.connOpenAt),
+      basis: nonNegativeMs(helloAt - attachedAt)
+    }
+    const totalMs = nonNegativeMs(helloAt - pending.timing.startedAt)
+    this.observer.recordClientAcceptCompleted?.({ totalMs, stageMs })
+    console.log(
+      JSON.stringify({
+        event: 'orca_relay_client_accept_completed',
+        ...this.logIdentity(),
+        credentialKind: pending.reservation.credentialKind,
+        stageMs,
+        totalMs,
+        relayHostIdDigest: relayHostLogDigest(session.relayHostId)
+      })
+    )
+  }
+
+  // Matches the runtime metrics event so a log line and a metric point can be
+  // joined back to the process that emitted them.
+  private logIdentity(): { role: string; cellId: string; region: RelayRegion } {
+    return {
+      role: this.config.role,
+      cellId: this.config.cellId,
+      region: this.config.region ?? RELAY_DEFAULT_REGION
+    }
+  }
+
+  // Every desktop build already echoes the ping's `t`, so a pong is only timed when
+  // it answers the outstanding ping: at most one sample per ping this cell sent,
+  // however many a host floods. A pong that lost the race to the next ping is
+  // dropped here but still counts as proof of life for the silence watchdog.
+  private recordControlRtt(session: HostSession, echoedPingAt: unknown): void {
+    if (typeof echoedPingAt !== 'number' || echoedPingAt !== session.pendingPingAt) return
+    session.pendingPingAt = null
+    const now = this.now()
+    const rttMs = now - echoedPingAt
+    if (rttMs < 0 || rttMs > CONTROL_RTT_MAX_PLAUSIBLE_MS) return
+    this.observer.recordControlRtt?.(rttMs)
+    const samples = session.controlRttSamplesMs
+    samples.push(rttMs)
+    if (samples.length > CONTROL_RTT_WINDOW) samples.shift()
+    if (samples.length < CONTROL_RTT_LOG_SAMPLE_THRESHOLD) return
+    if (
+      session.controlRttLoggedAt !== null &&
+      now - session.controlRttLoggedAt < CONTROL_RTT_LOG_INTERVAL_MS
+    ) {
+      return
+    }
+    session.controlRttLoggedAt = now
+    console.log(
+      JSON.stringify({
+        event: 'orca_relay_host_control_rtt',
+        ...this.logIdentity(),
+        relayHostIdDigest: relayHostLogDigest(session.relayHostId),
+        rttMsMedian: percentile(samples, 0.5),
+        sampleCount: samples.length
+      })
+    )
+  }
+
   acceptControl(
     socket: WebSocket,
     identity: RelayTokenClaims,
-    connectionInclusionWatermark?: number
+    connectionInclusionWatermark?: number,
+    hostCapabilities?: ReadonlySet<string>
   ): void {
+    // Keyed by socket, not session: a rebind swaps the session's socket, and the
+    // successor's own advertisement is the only one that describes its decoder.
+    if (hostCapabilities?.size) this.hostCapabilities.set(socket, hostCapabilities)
     if (this.draining) {
       socket.close(RELAY_CLOSE_CODE.DRAINING, 'relay draining')
       return
@@ -727,8 +921,9 @@ export class HostSessionRegistry {
       existing.socket = socket
       existing.state = existing.regionalDrainAttemptId ? 'drain-only' : 'active'
       existing.appVersion = appVersion
-      existing.leaseExpiresAt = this.now() + 55 * 60 * 1000
+      existing.leaseExpiresAt = this.controlLeaseExpiresAt()
       existing.lastPongAt = this.now()
+      existing.pendingPingAt = null
       existing.activityRenewalDueAt =
         this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs
       this.wireActiveControl(existing)
@@ -778,10 +973,13 @@ export class HostSessionRegistry {
       appVersion,
       state: 'active',
       socket,
-      leaseExpiresAt: this.now() + 55 * 60 * 1000,
+      leaseExpiresAt: this.controlLeaseExpiresAt(),
       orphanTimer: null,
       heartbeatTimer: null,
       lastPongAt: this.now(),
+      pendingPingAt: null,
+      controlRttSamplesMs: [],
+      controlRttLoggedAt: null,
       activityRenewalDueAt: this.now() + RELAY_PROTOCOL_LIMITS.controlPingIntervalMs,
       activityRenewalAttempt: 0,
       activityRenewalCompletedAttempt: 0,
@@ -793,7 +991,10 @@ export class HostSessionRegistry {
       regionalDrainTimer: null,
       regionalDrainExpiresAt: null
     }
-    this.sessions.set(this.key(identity.sub, identity.relayHostId), session)
+    const sessionKey = this.key(identity.sub, identity.relayHostId)
+    // A host that proved itself again is not signed out, whatever it said last.
+    this.hostCloseReasons.forget(sessionKey)
+    this.sessions.set(sessionKey, session)
     this.wireActiveControl(session)
     this.sendHelloAck(session)
   }
@@ -813,6 +1014,11 @@ export class HostSessionRegistry {
     })
     socket.once('close', (code, reason) => {
       this.observer.recordControlClose?.(code)
+      // Guarded on identity: a predecessor retired by a rebind must not stamp a
+      // cause onto the live session that replaced it.
+      if (session.socket === socket) {
+        this.hostCloseReasons.record(this.key(session.identity.sub, session.relayHostId), reason)
+      }
       // One line per control close makes reconnect churners attributable by
       // host digest without exposing the raw relay host id.
       console.warn(
@@ -831,6 +1037,7 @@ export class HostSessionRegistry {
         const parsed = JSON.parse(raw.toString()) as Record<string, unknown>
         if (parsed.type === 'pong') {
           session.lastPongAt = this.now()
+          this.recordControlRtt(session, parsed.t)
           return
         }
         if (parsed.type === 'auth-refresh') {
@@ -978,11 +1185,18 @@ export class HostSessionRegistry {
       session.socket.close(RELAY_CLOSE_CODE.DRAINING, 'control lease expired')
       return
     }
+    session.pendingPingAt = now
     send(session.socket, 'ping', { t: now })
   }
 
   private sendHelloAck(session: HostSession): void {
     if (!session.socket) return
+    // Without these a host that missed the conn-open cannot dial the pending
+    // connection: it would have to guess the pairing kind and the device the
+    // relay authorized. Only sent to a host that said it can read them.
+    const details = this.hostCapabilities
+      .get(session.socket)
+      ?.has(RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS)
     send(session.socket, 'host-hello-ack', {
       v: 1,
       generation: session.generation,
@@ -991,7 +1205,13 @@ export class HostSessionRegistry {
       activeConnIds: [...session.activeConnIds],
       pendingConns: [...session.pendingConns.values()].map((pending) => ({
         connId: pending.connId,
-        connTicket: pending.connTicket
+        connTicket: pending.connTicket,
+        ...(details
+          ? {
+              kind: pending.reservation.credentialKind,
+              relayDeviceId: pending.reservation.relayDeviceId
+            }
+          : {})
       }))
     })
   }
@@ -1187,9 +1407,16 @@ export class HostSessionRegistry {
     if (session.socket) send(session.socket, 'control-error', { ...(reqId ? { reqId } : {}), code })
   }
 
-  private rejectClient(socket: WebSocket, code: number): void {
+  // hostCloseReason rides the WebSocket close reason, never relay-hello: every
+  // shipped phone parses relay-hello with a strict schema that rejects an
+  // unknown key, and none of them read the close reason at all.
+  private rejectClient(
+    socket: WebSocket,
+    code: number,
+    hostCloseReason?: RelayHostCloseReason | null
+  ): void {
     send(socket, 'relay-hello', { ok: false, code })
-    closeRelayWebSocket(socket, code, 'relay connection rejected')
+    closeRelayWebSocket(socket, code, hostCloseReason ?? 'relay connection rejected')
   }
 
   private releaseControlActivity(session: HostSession): void {

cloud/apps/relay/src/host-signed-out-rejection.test.ts

@@ -0,0 +1,206 @@
+import { EventEmitter } from 'node:events'
+import {
+  CONTROL_CONTINUITY_LIMITS,
+  RELAY_CLOSE_CODE,
+  RELAY_HOST_CLOSE_REASON
+} from '@orca-cloud/relay-contract'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import type WebSocket from 'ws'
+import type { RelayAssignmentStore } from './assignment-store.js'
+import type { RelayConfig } from './config.js'
+import type { RelayCredentialStore } from './credential-store.js'
+import { HostSessionRegistry } from './host-session-registry.js'
+import type { RelayRuntimeObserver } from './relay-observability.js'
+import type { RelayTokenClaims } from './relay-token-verifier.js'
+import { ProcessQueuedByteBudget } from './splice-forwarder.js'
+
+class FakeSocket extends EventEmitter {
+  readonly OPEN = 1
+  readonly CLOSED = 3
+  readyState = this.OPEN
+  readonly send = vi.fn()
+  readonly close = vi.fn((code?: number, reason?: string) => {
+    this.readyState = this.CLOSED
+    this.emit('close', code, Buffer.from(reason ?? ''))
+  })
+  readonly terminate = vi.fn(() => {
+    this.readyState = this.CLOSED
+    this.emit('close', 1006, Buffer.alloc(0))
+  })
+}
+
+const config = {
+  port: 8080,
+  publicUrl: 'https://relay-c3.example.com',
+  cellUrl: 'https://relay-c3.example.com',
+  authIssuer: 'https://auth.example.com',
+  authAudience: 'orca-relay',
+  jwksUrl: 'https://auth.example.com/jwks',
+  assignmentSigningKey: new Uint8Array(32),
+  role: 'cell',
+  cellId: 'production-gce-c3',
+  cells: []
+} as unknown as RelayConfig
+
+const identity = {
+  sub: 'user-1',
+  prof: 'profile-1',
+  org: 'org-1',
+  relayHostId: 'AbCdEf0123_-xyZ9'
+} as unknown as RelayTokenClaims
+
+const reservation = {
+  userId: identity.sub,
+  relayHostId: identity.relayHostId,
+  credentialKind: 'resume',
+  relayDeviceId: 'device-1',
+  leaseExpiresAt: Date.now() + 60_000
+}
+
+function createRegistry() {
+  const store = {
+    resolveResume: vi.fn().mockResolvedValue({ userId: identity.sub }),
+    reserveCredential: vi.fn().mockResolvedValue(reservation),
+    failReservation: vi.fn().mockResolvedValue(undefined)
+  }
+  const assignments = {
+    activateControl: vi.fn().mockResolvedValue('control:production-gce-c3:1'),
+    markMigrationTargetRegistered: vi.fn().mockResolvedValue(undefined),
+    resolve: vi.fn().mockResolvedValue({ cellId: config.cellId }),
+    acquireActivity: vi.fn().mockResolvedValue(undefined),
+    renewControlActivity: vi.fn().mockResolvedValue(undefined),
+    releaseActivity: vi.fn().mockResolvedValue(true)
+  } as unknown as RelayAssignmentStore
+  const observer = {
+    recordAuth: vi.fn(),
+    recordForwardedBytes: vi.fn(),
+    recordHttp: vi.fn(),
+    recordReconnect: vi.fn(),
+    recordSql: vi.fn(),
+    recordControlClose: vi.fn(),
+    recordSpliceClose: vi.fn()
+  } satisfies RelayRuntimeObserver
+  const registry = new HostSessionRegistry(
+    config,
+    vi.fn(),
+    store as unknown as RelayCredentialStore,
+    assignments,
+    new ProcessQueuedByteBudget(),
+    observer
+  )
+  const activate = (socket: WebSocket, generation: number): Promise<void> =>
+    (
+      registry as unknown as {
+        activate: (
+          socket: WebSocket,
+          identity: RelayTokenClaims,
+          existing: null,
+          generation: number,
+          rebind: boolean,
+          assignmentEpoch: number,
+          appVersion: string
+        ) => Promise<void>
+      }
+    ).activate(socket, identity, null, generation, false, 1, '1.4.173')
+  return { registry, activate }
+}
+
+async function dialPhone(registry: HostSessionRegistry): Promise<FakeSocket> {
+  const phone = new FakeSocket()
+  await registry.acceptClient(phone as unknown as WebSocket, identity.relayHostId, 'credential')
+  return phone
+}
+
+// The 4404 hello body is unchanged: every shipped phone parses it with a strict
+// schema, so the cause has to ride the close frame instead.
+const HOST_OFFLINE_HELLO = JSON.stringify({
+  type: 'relay-hello',
+  ok: false,
+  code: RELAY_CLOSE_CODE.HOST_OFFLINE
+})
+
+describe('host sign-out reason on phone rejection', () => {
+  beforeEach(() => vi.useFakeTimers())
+  afterEach(() => {
+    vi.clearAllTimers()
+    vi.useRealTimers()
+  })
+
+  it('names the sign-out to a phone that arrives after the host is gone', async () => {
+    const { registry, activate } = createRegistry()
+    const control = new FakeSocket()
+    await activate(control as unknown as WebSocket, 1)
+
+    control.close(1000, RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
+    vi.advanceTimersByTime(CONTROL_CONTINUITY_LIMITS.orphanGraceMs + 1)
+
+    const phone = await dialPhone(registry)
+    expect(phone.send).toHaveBeenCalledWith(HOST_OFFLINE_HELLO)
+    expect(phone.close).toHaveBeenCalledWith(
+      RELAY_CLOSE_CODE.HOST_OFFLINE,
+      RELAY_HOST_CLOSE_REASON.SIGNED_OUT
+    )
+  })
+
+  it('says nothing when the host died without naming a cause', async () => {
+    const { registry, activate } = createRegistry()
+    const control = new FakeSocket()
+    await activate(control as unknown as WebSocket, 1)
+
+    control.terminate()
+    vi.advanceTimersByTime(CONTROL_CONTINUITY_LIMITS.orphanGraceMs + 1)
+
+    const phone = await dialPhone(registry)
+    expect(phone.close).toHaveBeenCalledWith(
+      RELAY_CLOSE_CODE.HOST_OFFLINE,
+      'relay connection rejected'
+    )
+  })
+
+  it('ignores a close reason the host invented', async () => {
+    const { registry, activate } = createRegistry()
+    const control = new FakeSocket()
+    await activate(control as unknown as WebSocket, 1)
+
+    control.close(1000, 'signed-out-ish')
+    vi.advanceTimersByTime(CONTROL_CONTINUITY_LIMITS.orphanGraceMs + 1)
+
+    const phone = await dialPhone(registry)
+    expect(phone.close).toHaveBeenCalledWith(
+      RELAY_CLOSE_CODE.HOST_OFFLINE,
+      'relay connection rejected'
+    )
+  })
+
+  it('forgets the sign-out once the host proves itself again', async () => {
+    const { registry, activate } = createRegistry()
+    const control = new FakeSocket()
+    await activate(control as unknown as WebSocket, 1)
+    control.close(1000, RELAY_HOST_CLOSE_REASON.SIGNED_OUT)
+    vi.advanceTimersByTime(CONTROL_CONTINUITY_LIMITS.orphanGraceMs + 1)
+
+    const reconnected = new FakeSocket()
+    await activate(reconnected as unknown as WebSocket, 2)
+    // Drop it abruptly, as a network death would, so only the stale memory
+    // could still name a cause.
+    reconnected.terminate()
+    vi.advanceTimersByTime(CONTROL_CONTINUITY_LIMITS.orphanGraceMs + 1)
+
+    const phone = await dialPhone(registry)
+    expect(phone.close).toHaveBeenCalledWith(
+      RELAY_CLOSE_CODE.HOST_OFFLINE,
+      'relay connection rejected'
+    )
+  })
+
+  // A live host is present: the 4404 there is an attach deadline, not absence.
+  it('never names a cause while the host control is connected', async () => {
+    const { registry, activate } = createRegistry()
+    const control = new FakeSocket()
+    await activate(control as unknown as WebSocket, 1)
+
+    const phone = await dialPhone(registry)
+    expect(phone.close).not.toHaveBeenCalled()
+    expect(control.send).toHaveBeenCalledWith(expect.stringContaining('"type":"conn-open"'))
+  })
+})

cloud/apps/relay/src/index.ts

@@ -10,12 +10,14 @@ import {
   roleOwnsAssignmentMaintenance
 } from './cell-admission-startup.js'
 import {
+  consumeRelayCellInventoryHold,
   consumeRelayDatabasePoolPressure,
   openRelayDatabase,
   readRelayDatabasePoolPressure
 } from './database.js'
 import { runAssignmentCleanup } from './assignment-cleanup-steps.js'
 import { runRelayBackgroundOperation } from './relay-background-operation.js'
+import { jitteredSweepIntervalMs } from './relay-sweep-schedule.js'
 import { observedRelayRequests } from './relay-observability.js'
 import { startRegionalRehomeWorker } from './regional-rehome-worker.js'
 import { createRelayServer } from './relay-server.js'
@@ -54,7 +56,7 @@ const cleanupTimer = setInterval(
 const assignmentCleanupTimer = roleOwnsAssignmentMaintenance(config.role)
   ? setInterval(() => {
       void runAssignmentCleanup(assignments)
-    }, 30_000)
+    }, jitteredSweepIntervalMs(30_000))
   : null
 const inventorySnapshotTimer = roleOwnsAssignmentMaintenance(config.role)
   ? setInterval(() => {
@@ -78,7 +80,8 @@ inventorySnapshotTimer?.unref()
 migrationInventoryTimer?.unref()
 observability.start(() => ({
   ...runtimeCounts(),
-  ...consumeRelayDatabasePoolPressure(database)
+  ...consumeRelayDatabasePoolPressure(database),
+  ...consumeRelayCellInventoryHold(database)
 }))
 const regionalRehomeWorker = startRegionalRehomeWorker(config, assignments, {
   safetySnapshot: () => ({

cloud/apps/relay/src/postgres-schema-concurrency-postgres.test.ts

@@ -34,22 +34,28 @@ describePostgres('PostgreSQL schema concurrency', () => {
   })
 
   it('opens five directors when one new table is absent', async () => {
-    const initial = await openRelayDatabase({ databaseUrl: scopedUrl, dataDir: '' })
-    await initial.query(`DROP TABLE relay_cell_legacy_fence_adoptions`)
-    await initial.close()
+    // Which catalog step the race loser fails on depends on scheduling, so run several rounds and
+    // keep the loser's SQLSTATE in the failure instead of a bare boolean.
+    for (let round = 0; round < 10; round += 1) {
+      const initial = await openRelayDatabase({ databaseUrl: scopedUrl, dataDir: '' })
+      await initial.query(`DROP TABLE relay_cell_legacy_fence_adoptions`)
+      await initial.close()
 
-    const results = await Promise.allSettled(
-      Array.from({ length: 5 }, async (): Promise<RelayDatabase> =>
-        await openRelayDatabase({ databaseUrl: scopedUrl, dataDir: '' })
+      const results = await Promise.allSettled(
+        Array.from({ length: 5 }, async (): Promise<RelayDatabase> =>
+          await openRelayDatabase({ databaseUrl: scopedUrl, dataDir: '' })
+        )
+      )
+      const databases = results.flatMap((result) =>
+        result.status === 'fulfilled' ? [result.value] : []
       )
-    )
-    const databases = results.flatMap((result) =>
-      result.status === 'fulfilled' ? [result.value] : []
-    )
-    try {
-      expect(results.every((result) => result.status === 'fulfilled')).toBe(true)
-    } finally {
       await Promise.all(databases.map(async (database) => await database.close()))
+      const rejections = results.flatMap((result) =>
+        result.status === 'rejected'
+          ? [{ round, code: (result.reason as { code?: unknown }).code, message: String(result.reason) }]
+          : []
+      )
+      expect(rejections).toEqual([])
     }
-  })
+  }, 60_000)
 })

cloud/apps/relay/src/postgres-schema-startup.ts

@@ -22,15 +22,51 @@ function wait(delayMs: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, delayMs))
 }
 
+const CREATE_TABLE_IF_NOT_EXISTS = /^\s*CREATE\s+TABLE\s+IF\s+NOT\s+EXISTS\b/i
+const CREATE_INDEX_IF_NOT_EXISTS = /^\s*CREATE\s+(?:UNIQUE\s+)?INDEX\s+IF\s+NOT\s+EXISTS\b/i
+
+// `IF NOT EXISTS` only checks the name before the catalog inserts, so the loser of a concurrent
+// CREATE can fail on the catalog unique index (23505) or, when the winner has already committed by
+// the time the loser reaches TypeCreate/heap_create_with_catalog, on the name check those routines
+// repeat (42710 duplicate type, 42P07 duplicate relation). Each is a no-op on the next attempt.
+function concurrentCreateCollision(
+  value: { code?: unknown; constraint?: unknown },
+  statement: string
+): boolean {
+  if (CREATE_TABLE_IF_NOT_EXISTS.test(statement)) {
+    return (
+      (value.code === '23505' && value.constraint === 'pg_type_typname_nsp_index') ||
+      value.code === '42710' ||
+      value.code === '42P07'
+    )
+  }
+  if (CREATE_INDEX_IF_NOT_EXISTS.test(statement)) {
+    return (
+      (value.code === '23505' && value.constraint === 'pg_class_relname_nsp_index') ||
+      value.code === '42P07'
+    )
+  }
+  return false
+}
+
+const ALTER_TABLE_ADD_CONSTRAINT =
+  /^\s*ALTER\s+TABLE\s+\S+\s+ADD\s+CONSTRAINT\b/i
+
+// Postgres has no `ADD CONSTRAINT IF NOT EXISTS`, so a re-run and a concurrent
+// startup both land on 42710 once the constraint exists. Unlike a CREATE race
+// this is terminal, not transient: retrying only repeats it, so the statement
+// counts as applied.
+function constraintAlreadyApplied(error: unknown, statement: string): boolean {
+  return (
+    ALTER_TABLE_ADD_CONSTRAINT.test(statement) &&
+    (error as { code?: unknown }).code === '42710'
+  )
+}
+
 function retryableSchemaError(error: unknown, statement: string): boolean {
   const value = error as { code?: unknown; constraint?: unknown }
   return (
-    RETRYABLE_SCHEMA_CODES.has(String(value.code)) ||
-    (value.code === '23505' &&
-      ((value.constraint === 'pg_type_typname_nsp_index' &&
-        /^\s*CREATE\s+TABLE\s+IF\s+NOT\s+EXISTS\b/i.test(statement)) ||
-        (value.constraint === 'pg_class_relname_nsp_index' &&
-          /^\s*CREATE\s+(?:UNIQUE\s+)?INDEX\s+IF\s+NOT\s+EXISTS\b/i.test(statement))))
+    RETRYABLE_SCHEMA_CODES.has(String(value.code)) || concurrentCreateCollision(value, statement)
   )
 }
 
@@ -51,6 +87,7 @@ export async function applyPostgresSchema(
         await query(statement)
         break
       } catch (error) {
+        if (constraintAlreadyApplied(error, statement)) break
         const code = String((error as { code?: unknown }).code)
         const remainingMs = deadlineAt - now()
         const retryable = retryableSchemaError(error, statement)

cloud/apps/relay/src/postgres-transaction-recovery.test.ts

@@ -503,12 +503,19 @@ describePostgres('PostgreSQL transaction recovery', () => {
     const directorLockOrder: string[] = []
     const assignmentDatabase = new TransactionProbeDatabase(database, async (phase, sql) => {
       if (phase === 'before') {
-        if (sql.includes('FROM relay_assignments WHERE user_id = ?')) {
+        // Only locked statements reach this hook, so classifying the pin read
+        // is what proves it stays unlocked: if it ever grows a FOR UPDATE it
+        // shows up in the order below instead of silently joining the queue.
+        if (sql.includes('SELECT cell_id FROM relay_assignments')) {
+          directorLockOrder.push('pin-read')
+        } else if (sql.includes('FROM relay_assignments WHERE user_id = ?')) {
           directorLockOrder.push('assignment')
         } else if (sql.includes('FROM relay_assignment_activity_leases')) {
           directorLockOrder.push('activity')
         } else if (sql.includes('FROM relay_cells ORDER BY')) {
           directorLockOrder.push('cell-inventory')
+        } else if (sql.includes('FROM relay_cells WHERE cell_id IN')) {
+          directorLockOrder.push('cell-rows')
         } else if (sql.includes('FROM relay_cells WHERE cell_id = ?')) {
           directorLockOrder.push('cell')
         }
@@ -530,11 +537,14 @@ describePostgres('PostgreSQL transaction recovery', () => {
     })
     await expect(legacyTransaction).resolves.toBeUndefined()
     expect(assignmentDatabase.attempts).toBe(2)
+    // The retry still takes a cell row before the assignment row — the order
+    // that avoids the legacy cycle — but only the pinned row, never the
+    // inventory.
     expect(directorLockOrder).toEqual([
       'assignment',
       'activity',
       'cell',
-      'cell-inventory',
+      'cell-rows',
       'assignment',
       'activity'
     ])

cloud/apps/relay/src/regional-host-drain-app.test.ts

@@ -315,6 +315,7 @@ describe('regional rehome director controls', () => {
       notBefore: 100,
       ratePerMinute: 10,
       preferenceMaxAgeMs: 24 * 60 * 60_000,
+      hostCooldownMs: 7 * 24 * 60 * 60_000,
       drainGraceMs: 60_000,
       confirmation: 'ENABLE_REGIONAL_REHOMING'
     }
@@ -343,6 +344,14 @@ describe('regional rehome director controls', () => {
       'deploy-token',
       { ...apply, confirmation: 'DISABLE_REGIONAL_REHOMING' }
     )).status).toBe(400)
+    // The per-host cooldown is part of the durable shape an operator must state.
+    const { hostCooldownMs: _omitted, ...withoutCooldown } = apply
+    expect((await postPath(
+      app,
+      '/v1/admin/regional-rehome-control',
+      'deploy-token',
+      withoutCooldown
+    )).status).toBe(400)
   })
 
   it('probes dedicated trust twice and returns only aggregate proof', async () => {
@@ -411,6 +420,78 @@ describe('regional rehome director controls', () => {
     expect(JSON.stringify(responseBody)).not.toContain('rehome-token')
   })
 
+  it('probes a source cell in any region, not only the default one', async () => {
+    // Rehoming moves hosts in both directions, so an asia-east2 cell is a
+    // source too and its trust has to be provable the same way.
+    const cellDeploymentStatus = vi.fn().mockResolvedValue({
+      cellId: 'production-gce-c27',
+      cellUrl: 'https://c27.relay.example.test',
+      region: 'asia-east2',
+      runtime: {
+        cellIncarnation,
+        ready: true,
+        heartbeatFresh: true,
+        regionalRehomeProtocol: 1
+      }
+    })
+    const app = createRelayApp(config({ role: 'director', cellId: 'director' }), {
+      store: {} as never,
+      assignments: { cellDeploymentStatus } as never,
+      drain: vi.fn(),
+      regionalRehomeIdentityToken: vi.fn(async () => 'rehome-token'),
+      regionalRehomeFetch: (async () =>
+        Response.json({
+          v: 1,
+          outcome: 'host-not-connected',
+          sharedRuntimeIdentityRejected: true
+        })) as typeof fetch,
+      ready: vi.fn(async () => true)
+    })
+
+    const response = await postPath(
+      app,
+      '/v1/admin/regional-rehome-trust-probe',
+      'deploy-token',
+      { v: 1, sourceCellId: 'production-gce-c27', sourceCellIncarnation: cellIncarnation }
+    )
+
+    expect(response.status).toBe(200)
+    expect(await response.json()).toMatchObject({ proven: true })
+  })
+
+  it('still refuses a trust probe against a cell without the drain protocol', async () => {
+    const cellDeploymentStatus = vi.fn().mockResolvedValue({
+      cellId: 'production-gce-c27',
+      cellUrl: 'https://c27.relay.example.test',
+      region: 'asia-east2',
+      runtime: {
+        cellIncarnation,
+        ready: true,
+        heartbeatFresh: true,
+        regionalRehomeProtocol: 0
+      }
+    })
+    const sourceFetch = vi.fn<typeof fetch>()
+    const app = createRelayApp(config({ role: 'director', cellId: 'director' }), {
+      store: {} as never,
+      assignments: { cellDeploymentStatus } as never,
+      drain: vi.fn(),
+      regionalRehomeIdentityToken: vi.fn(async () => 'rehome-token'),
+      regionalRehomeFetch: sourceFetch,
+      ready: vi.fn(async () => true)
+    })
+
+    const response = await postPath(
+      app,
+      '/v1/admin/regional-rehome-trust-probe',
+      'deploy-token',
+      { v: 1, sourceCellId: 'production-gce-c27', sourceCellIncarnation: cellIncarnation }
+    )
+
+    expect(response.status).toBe(409)
+    expect(sourceFetch).not.toHaveBeenCalled()
+  })
+
   it('restricts trust probes to deploy authorization and strict input', async () => {
     const app = createRelayApp(config({ role: 'director', cellId: 'director' }), {
       store: {} as never,

cloud/apps/relay/src/regional-rehome-constraint-migration-postgres.test.ts

@@ -0,0 +1,195 @@
+import pg from 'pg'
+import { afterAll, beforeEach, describe, expect, it } from 'vitest'
+import {
+  openRelayDatabase,
+  REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS,
+  type RelayDatabase
+} from './database.js'
+
+const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
+const describePostgres = databaseUrl ? describe : describe.skip
+const schema = 'relay_rehome_constraint_migration_test'
+
+// The shape shipped before rehoming became bidirectional: a single-region
+// column check that Postgres auto-names.
+const LEGACY_ATTEMPTS_TABLE = `
+CREATE TABLE relay_region_rehome_attempts (
+  attempt_id TEXT PRIMARY KEY,
+  user_id TEXT NOT NULL,
+  relay_host_id TEXT NOT NULL,
+  preferred_region TEXT NOT NULL CHECK (preferred_region = 'asia-east2'),
+  source_cell_id TEXT NOT NULL,
+  source_cell_incarnation TEXT NOT NULL,
+  target_cell_id TEXT NOT NULL,
+  target_cell_incarnation TEXT NOT NULL,
+  previous_epoch BIGINT NOT NULL,
+  assignment_epoch BIGINT NOT NULL,
+  drain_grace_ms BIGINT NOT NULL,
+  send_attempts BIGINT NOT NULL,
+  last_send_attempt_at BIGINT,
+  drain_receipt_at BIGINT,
+  drain_outcome TEXT CHECK (
+    drain_outcome IN ('accepted', 'already-accepted', 'host-not-connected')
+  ),
+  completed_at BIGINT,
+  aborted_at BIGINT,
+  created_at BIGINT NOT NULL,
+  updated_at BIGINT NOT NULL,
+  UNIQUE (user_id, relay_host_id, assignment_epoch)
+)`
+
+// The control row as it shipped before the per-host cooldown existed.
+const LEGACY_CONTROL_TABLE = `
+CREATE TABLE relay_region_rehome_control (
+  control_id TEXT PRIMARY KEY,
+  generation BIGINT NOT NULL,
+  enabled BIGINT NOT NULL,
+  observation_started_at BIGINT NOT NULL,
+  not_before BIGINT NOT NULL,
+  rate_per_minute BIGINT NOT NULL,
+  preference_max_age_ms BIGINT NOT NULL,
+  drain_grace_ms BIGINT NOT NULL,
+  updated_at BIGINT NOT NULL
+)`
+
+const attemptValues = (attemptId: string, preferredRegion: string): unknown[] => [
+  attemptId,
+  'user-1',
+  'abcdefghijklmnop',
+  preferredRegion,
+  'cell-source',
+  '11111111-1111-4111-8111-111111111111',
+  'cell-target',
+  '22222222-2222-4222-8222-222222222222',
+  1,
+  Number(attemptId.at(-1)),
+  0,
+  0,
+  1_000_000,
+  1_000_000
+]
+
+const INSERT_ATTEMPT = `INSERT INTO relay_region_rehome_attempts
+  (attempt_id, user_id, relay_host_id, preferred_region, source_cell_id,
+   source_cell_incarnation, target_cell_id, target_cell_incarnation,
+   previous_epoch, assignment_epoch, drain_grace_ms, send_attempts,
+   created_at, updated_at)
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)`
+
+describePostgres('PostgreSQL regional rehome constraint migration', () => {
+  let scopedUrl = ''
+
+  async function withClient(
+    operation: (client: pg.Client) => Promise<void>
+  ): Promise<void> {
+    const client = new pg.Client({ connectionString: databaseUrl })
+    await client.connect()
+    try {
+      await operation(client)
+    } finally {
+      await client.end()
+    }
+  }
+
+  beforeEach(async () => {
+    await withClient(async (client) => {
+      await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
+      await client.query(`CREATE SCHEMA ${schema}`)
+      await client.query(`SET search_path = ${schema}`)
+      await client.query(LEGACY_ATTEMPTS_TABLE)
+      await client.query(LEGACY_CONTROL_TABLE)
+      await client.query(
+        `INSERT INTO relay_region_rehome_control
+         (control_id, generation, enabled, observation_started_at, not_before,
+          rate_per_minute, preference_max_age_ms, drain_grace_ms, updated_at)
+         VALUES ('global', 3, 0, 1, 0, 10, 86400000, 60000, 1)`
+      )
+      // Production data the replacement constraint has to validate.
+      await client.query(INSERT_ATTEMPT, attemptValues('attempt-1', 'asia-east2'))
+    })
+    const url = new URL(databaseUrl!)
+    url.searchParams.set('options', `-c search_path=${schema}`)
+    scopedUrl = url.toString()
+  })
+
+  afterAll(async () => {
+    await withClient(async (client) => {
+      await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
+    })
+  })
+
+  it('upgrades a legacy single-region constraint in place', async () => {
+    const database = await openRelayDatabase({ databaseUrl: scopedUrl, dataDir: '' })
+    try {
+      await withClient(async (client) => {
+        await client.query(`SET search_path = ${schema}`)
+        await client.query(INSERT_ATTEMPT, attemptValues('attempt-2', 'us-central1'))
+        await expect(
+          client.query(INSERT_ATTEMPT, attemptValues('attempt-3', 'europe-west1'))
+        ).rejects.toMatchObject({ code: '23514' })
+        const constraints = await client.query(
+          `SELECT conname FROM pg_constraint
+           WHERE conrelid = 'relay_region_rehome_attempts'::regclass
+             AND conname LIKE '%preferred_region%'
+           ORDER BY conname`
+        )
+        expect(constraints.rows).toEqual([
+          { conname: 'relay_region_rehome_attempts_preferred_region_valid' }
+        ])
+        // The existing control row keeps its tuning and gains the cooldown.
+        const control = await client.query(
+          `SELECT generation, preference_max_age_ms, host_cooldown_ms
+           FROM relay_region_rehome_control WHERE control_id = 'global'`
+        )
+        expect(control.rows).toEqual([
+          {
+            generation: '3',
+            preference_max_age_ms: '86400000',
+            host_cooldown_ms: String(REGIONAL_REHOME_DEFAULT_HOST_COOLDOWN_MS)
+          }
+        ])
+      })
+    } finally {
+      await database.close()
+    }
+  })
+
+  it('upgrades once across concurrent startups', async () => {
+    const results = await Promise.allSettled(
+      Array.from(
+        { length: 5 },
+        async (): Promise<RelayDatabase> =>
+          await openRelayDatabase({ databaseUrl: scopedUrl, dataDir: '' })
+      )
+    )
+    const databases = results.flatMap((result) =>
+      result.status === 'fulfilled' ? [result.value] : []
+    )
+    await Promise.all(databases.map(async (database) => await database.close()))
+
+    expect(
+      results.flatMap((result) =>
+        result.status === 'rejected'
+          ? [
+              {
+                code: (result.reason as { code?: unknown }).code,
+                message: String(result.reason)
+              }
+            ]
+          : []
+      )
+    ).toEqual([])
+    await withClient(async (client) => {
+      await client.query(`SET search_path = ${schema}`)
+      await client.query(INSERT_ATTEMPT, attemptValues('attempt-4', 'us-central1'))
+      const constraints = await client.query(
+        `SELECT conname FROM pg_constraint
+         WHERE conrelid = 'relay_region_rehome_attempts'::regclass
+           AND conname LIKE '%preferred_region%'`
+      )
+      expect(constraints.rows).toEqual([
+        { conname: 'relay_region_rehome_attempts_preferred_region_valid' }
+      ])
+    })
+  }, 60_000)
+})

cloud/apps/relay/src/regional-rehome-postgres.test.ts

@@ -81,6 +81,153 @@ describePostgres('PostgreSQL regional rehoming', () => {
     expect(await context.store.claimRegionalRehome()).not.toBeNull()
   })
 
+  it('moves a us-central1 host onto a cell in its preferred asia-east2 region', async () => {
+    const context = await fixture()
+
+    const attempt = await context.store.claimRegionalRehome()
+    expect(attempt).toMatchObject({
+      preferredRegion: 'asia-east2',
+      sourceCellId: context.source.id,
+      targetCellId: context.target.id
+    })
+    expect(await primary.query(
+      `SELECT preferred_region, source_cell_id, target_cell_id
+       FROM relay_region_rehome_attempts WHERE user_id = ?`,
+      [context.identity.userId]
+    )).toEqual([{
+      preferred_region: 'asia-east2',
+      source_cell_id: context.source.id,
+      target_cell_id: context.target.id
+    }])
+  })
+
+  it('moves an asia-east2 host back onto a cell in its preferred us-central1 region', async () => {
+    const context = await fixture({
+      sourceRegion: 'asia-east2',
+      targetRegion: 'us-central1'
+    })
+
+    const attempt = await context.store.claimRegionalRehome()
+    expect(attempt).toMatchObject({
+      preferredRegion: 'us-central1',
+      sourceCellId: context.source.id,
+      targetCellId: context.target.id
+    })
+    // The durable attempt row must accept the reverse direction too.
+    expect(await primary.query(
+      `SELECT preferred_region, source_cell_id, target_cell_id
+       FROM relay_region_rehome_attempts WHERE user_id = ?`,
+      [context.identity.userId]
+    )).toEqual([{
+      preferred_region: 'us-central1',
+      source_cell_id: context.source.id,
+      target_cell_id: context.target.id
+    }])
+    expect(await primary.query(
+      `SELECT cell_id FROM relay_assignments WHERE user_id = ?`,
+      [context.identity.userId]
+    )).toEqual([{ cell_id: context.target.id }])
+  })
+
+  it('leaves a host whose preference already matches its own region', async () => {
+    const context = await fixture({ preferredRegion: 'us-central1' })
+
+    await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
+    await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
+      generation: 1,
+      enabled: true
+    })
+    expect(await attemptAndMigrationCounts(context.identity)).toEqual({
+      attempts: 0,
+      migrations: 0
+    })
+  })
+
+  it('leaves a host whose preference is older than the configured max age', async () => {
+    const context = await fixture()
+    await primary.query(
+      `UPDATE relay_assignment_region_preferences SET observed_at = ?
+       WHERE user_id = ? AND relay_host_id = ?`,
+      [
+        context.now() - 24 * 60 * 60_000 - 1,
+        context.identity.userId,
+        context.identity.relayHostId
+      ]
+    )
+
+    await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
+    await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
+      generation: 1,
+      enabled: true
+    })
+    expect(await attemptAndMigrationCounts(context.identity)).toEqual({
+      attempts: 0,
+      migrations: 0
+    })
+  })
+
+  it('leaves a host inside its per-host rehome cooldown, in either direction', async () => {
+    const context = await fixture({ hostCooldownMs: 3 * 24 * 60 * 60_000 })
+    // A move this host already made, whichever way it went.
+    await primary.query(
+      `INSERT INTO relay_region_rehome_attempts
+       (attempt_id, user_id, relay_host_id, preferred_region, source_cell_id,
+        source_cell_incarnation, target_cell_id, target_cell_incarnation,
+        previous_epoch, assignment_epoch, drain_grace_ms, send_attempts,
+        completed_at, created_at, updated_at)
+       VALUES (?, ?, ?, 'us-central1', ?, ?, ?, ?, 0, 1, 0, 0, ?, ?, ?)`,
+      [
+        `pg-rehome-cooldown-${context.identity.relayHostId}`,
+        context.identity.userId,
+        context.identity.relayHostId,
+        context.target.id,
+        '22222222-2222-4222-8222-222222222222',
+        context.source.id,
+        '11111111-1111-4111-8111-111111111111',
+        context.now(),
+        context.now() - 3 * 24 * 60 * 60_000 + 1,
+        context.now()
+      ]
+    )
+
+    await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
+    await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
+      generation: 1,
+      enabled: true,
+      hostCooldownMs: 3 * 24 * 60 * 60_000
+    })
+    expect(await attemptAndMigrationCounts(context.identity)).toEqual({
+      attempts: 1,
+      migrations: 0
+    })
+
+    // One millisecond past the window the same host is a candidate again.
+    await primary.query(
+      `UPDATE relay_region_rehome_attempts SET created_at = ? WHERE user_id = ?`,
+      [context.now() - 3 * 24 * 60 * 60_000, context.identity.userId]
+    )
+    await expect(context.store.claimRegionalRehome()).resolves.toMatchObject({
+      sourceCellId: context.source.id,
+      targetCellId: context.target.id
+    })
+  })
+
+  it('leaves a host whose preferred region holds no drainable cell', async () => {
+    // A cell that cannot be drained cannot be a target: the host would land
+    // where no later rehome could move it out again.
+    const context = await fixture({ targetProtocol: 0 })
+
+    await expect(context.store.claimRegionalRehome()).resolves.toBeNull()
+    await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
+      generation: 1,
+      enabled: true
+    })
+    expect(await attemptAndMigrationCounts(context.identity)).toEqual({
+      attempts: 0,
+      migrations: 0
+    })
+  })
+
   it('skips an unclean cell without latching the control off', async () => {
     const context = await fixture()
     await primary.query(
@@ -281,7 +428,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
       context.store,
       context.target,
       '22222222-2222-4222-8222-222222222222',
-      0,
+      1,
       900_000,
       2
     )
@@ -322,7 +469,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
       context.store,
       context.target,
       '44444444-4444-4444-8444-444444444444',
-      0,
+      1,
       context.now()
     )
 
@@ -341,7 +488,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
       context.store,
       context.target,
       '22222222-2222-4222-8222-222222222222',
-      0,
+      1,
       900_000,
       2
     )
@@ -414,6 +561,26 @@ describePostgres('PostgreSQL regional rehoming', () => {
     })
   })
 
+  async function attemptAndMigrationCounts(identity: {
+    userId: string
+    relayHostId: string
+  }): Promise<{ attempts: number; migrations: number }> {
+    const attempts = await primary.query(
+      `SELECT COUNT(*) AS count FROM relay_region_rehome_attempts
+       WHERE user_id = ? AND relay_host_id = ?`,
+      [identity.userId, identity.relayHostId]
+    )
+    const migrations = await primary.query(
+      `SELECT COUNT(*) AS count FROM relay_assignment_migrations
+       WHERE user_id = ? AND relay_host_id = ?`,
+      [identity.userId, identity.relayHostId]
+    )
+    return {
+      attempts: Number(attempts[0]!.count),
+      migrations: Number(migrations[0]!.count)
+    }
+  }
+
   async function controlAccounting(identity: {
     userId: string
     relayHostId: string
@@ -436,12 +603,15 @@ describePostgres('PostgreSQL regional rehoming', () => {
     }
   }
 
-  async function fixture() {
+  async function fixture(options: FixtureOptions = {}) {
     sequence++
     let now = 1_000_000
     const suffix = String(sequence)
-    const source = cell(suffix, 'source', 'us-central1')
-    const target = cell(suffix, 'target', 'asia-east2')
+    const sourceRegion = options.sourceRegion ?? 'us-central1'
+    const targetRegion = options.targetRegion ?? 'asia-east2'
+    const preferredRegion = options.preferredRegion ?? targetRegion
+    const source = cell(suffix, 'source', sourceRegion)
+    const target = cell(suffix, 'target', targetRegion)
     const store = new RelayAssignmentStore(primary, () => now, storeOptions)
     const competingStore = new RelayAssignmentStore(secondary, () => now, storeOptions)
     await store.inspectRegionalRehomeControl()
@@ -452,6 +622,7 @@ describePostgres('PostgreSQL regional rehoming', () => {
       notBefore: now,
       ratePerMinute: 10,
       preferenceMaxAgeMs: 24 * 60 * 60_000,
+      hostCooldownMs: options.hostCooldownMs ?? 7 * 24 * 60 * 60_000,
       drainGraceMs: 60_000
     })
     await store.reconcileCells([source, target])
@@ -466,21 +637,22 @@ describePostgres('PostgreSQL regional rehoming', () => {
       store,
       target,
       '22222222-2222-4222-8222-222222222222',
-      0,
+      options.targetProtocol ?? 1,
       900_000
     )
     const identity = {
       userId: `pg-rehome-user-${suffix}`,
       relayHostId: `rehomehost${suffix.padStart(6, '0')}`
     }
-    const assignment = await store.assign(identity, undefined, 'us-central1')
+    const assignment = await store.assign(identity, undefined, sourceRegion)
     const sourceControl = await store.activateControl(identity, {
       cellId: source.id,
       assignmentEpoch: assignment.assignmentEpoch,
       generation: 1
     })
-    await store.assign(identity, 'asia-east2')
+    await store.assign(identity, preferredRegion)
     return {
+      preferredRegion,
       store,
       competingStore,
       identity,
@@ -500,7 +672,16 @@ const storeOptions = {
   heartbeatTtlMs: 45_000
 }
 
-function cell(suffix: string, role: string, region: 'us-central1' | 'asia-east2') {
+type Region = 'us-central1' | 'asia-east2'
+type FixtureOptions = {
+  sourceRegion?: Region
+  targetRegion?: Region
+  preferredRegion?: Region
+  targetProtocol?: number
+  hostCooldownMs?: number
+}
+
+function cell(suffix: string, role: string, region: Region) {
   return {
     id: `pg-rehome-cell-${suffix}-${role}`,
     url: `https://pg-rehome-${suffix}-${role}.example.test`,

cloud/apps/relay/src/regional-rehome-store.test.ts

@@ -5,7 +5,12 @@ import {
   REGIONAL_REHOME_QUARANTINE_MS,
   REGIONAL_REHOME_REDRAIN_SEND_LIMIT
 } from './assignment-store.js'
-import { openInMemoryRelayDatabase, type RelayDatabase, type SqlRow } from './database.js'
+import {
+  openInMemoryRelayDatabase,
+  type RelayDatabase,
+  type RelayLockOptions,
+  type SqlRow
+} from './database.js'
 import {
   REGIONAL_REHOME_SQL_FAILURES_LIMIT,
   REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT
@@ -76,6 +81,7 @@ describe('regional rehome assignment state', () => {
       notBefore: context.now(),
       ratePerMinute: 10,
       preferenceMaxAgeMs: 24 * 60 * 60_000,
+      hostCooldownMs: 7 * 24 * 60 * 60_000,
       drainGraceMs: 60_000
     })).rejects.toThrow('regional_rehome_generation_mismatch')
     await expect(context.store.applyRegionalRehomeControl({
@@ -84,6 +90,7 @@ describe('regional rehome assignment state', () => {
       notBefore: context.now(),
       ratePerMinute: 10,
       preferenceMaxAgeMs: 24 * 60 * 60_000,
+      hostCooldownMs: 7 * 24 * 60 * 60_000,
       drainGraceMs: 60_000
     })).resolves.toMatchObject({ generation: 3, enabled: true })
     await context.database.close()
@@ -202,7 +209,7 @@ describe('regional rehome assignment state', () => {
       databasePoolWaitersMax: 0,
       databasePoolWaitMsMax: 0
     })
-    await heartbeat(context.store, target, targetIncarnation, 0, 2, {
+    await heartbeat(context.store, target, targetIncarnation, 1, 2, {
       observedAt: context.now(),
       sqlFailures: 1,
       reconnects: 3,
@@ -221,6 +228,23 @@ describe('regional rehome assignment state', () => {
     await context.database.close()
   })
 
+  it('counts only drainable cells as the rehome fleet, in every region', async () => {
+    // The fleet whose health gates a rehome is exactly the cells that can be a
+    // source or a target, and both roles require the drain protocol.
+    const context = await setup({ targetProtocol: 0 })
+
+    expect(await context.store.regionalRehomeFleetSafety()).toMatchObject({
+      requiredCells: 1,
+      missingCells: 0
+    })
+    await heartbeat(context.store, target, targetIncarnation, 1, 2)
+    expect(await context.store.regionalRehomeFleetSafety()).toMatchObject({
+      requiredCells: 2,
+      missingCells: 0
+    })
+    await context.database.close()
+  })
+
   it('claims through the measured healthy baseline of pool micro-waits and churn', async () => {
     const context = await setup()
     const baseline = {
@@ -233,7 +257,7 @@ describe('regional rehome assignment state', () => {
       databasePoolWaitMsMax: 1
     }
     await heartbeat(context.store, source, sourceIncarnation, 1, 2, baseline)
-    await heartbeat(context.store, target, targetIncarnation, 0, 2, baseline)
+    await heartbeat(context.store, target, targetIncarnation, 1, 2, baseline)
     await activatePreferredSource(context, {
       userId: 'user-1',
       relayHostId: 'abcdefghijklmnop'
@@ -319,6 +343,204 @@ describe('regional rehome assignment state', () => {
     await context.database.close()
   })
 
+  it('moves a live host on an asia-east2 cell back to its preferred us-central1 cell', async () => {
+    const context = await setup()
+    const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
+    await activateReversePreferredSource(context, identity)
+
+    const attempt = await context.store.claimRegionalRehome()
+    expect(attempt).toMatchObject({
+      userId: identity.userId,
+      relayHostId: identity.relayHostId,
+      preferredRegion: 'us-central1',
+      sourceCellId: target.id,
+      sourceCellIncarnation: targetIncarnation,
+      targetCellId: source.id,
+      targetCellIncarnation: sourceIncarnation,
+      previousEpoch: 1,
+      assignmentEpoch: 2,
+      sendAttempts: 1
+    })
+    expect(
+      await context.database.query(
+        `SELECT preferred_region, source_cell_id, target_cell_id
+         FROM relay_region_rehome_attempts`
+      )
+    ).toEqual([{
+      preferred_region: 'us-central1',
+      source_cell_id: target.id,
+      target_cell_id: source.id
+    }])
+    expect(await context.store.resolve(identity)).toMatchObject({ cellId: source.id })
+    await context.database.close()
+  })
+
+  it('drops a candidate at scan time when no cell in the preferred region is usable', async () => {
+    const context = await setup()
+    await activatePreferredSource(context, {
+      userId: 'user-1',
+      relayHostId: 'abcdefghijklmnop'
+    })
+    // A disabled cell is not a target, and the scan must say so: leaving it to
+    // the claim would burn a slot of the candidate batch on a certain skip.
+    await context.database.query(`UPDATE relay_cells SET enabled = 0 WHERE cell_id = ?`, [
+      target.id
+    ])
+
+    const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped')
+    try {
+      expect(await context.store.claimRegionalRehome()).toBeNull()
+    } finally {
+      warnings.restore()
+    }
+    expect(warnings.entries).toEqual([])
+    expect(
+      await context.database.query(
+        `SELECT next_dispatch_at FROM relay_region_rehome_worker_state`
+      )
+    ).toEqual([{ next_dispatch_at: 0 }])
+    expect(await context.database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([])
+    await context.database.close()
+  })
+
+  it('names the skip when the last target is lost between scan and claim', async () => {
+    const database = await openInMemoryRelayDatabase()
+    const context = await setup({
+      database,
+      wrap: (delegate) =>
+        hookAfterCandidateScan(delegate, async (transaction) => {
+          await transaction.query(`UPDATE relay_cells SET enabled = 0 WHERE cell_id = ?`, [
+            target.id
+          ])
+        })
+    })
+    await activatePreferredSource(context, {
+      userId: 'user-1',
+      relayHostId: 'abcdefghijklmnop'
+    })
+
+    const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped')
+    try {
+      expect(await context.store.claimRegionalRehome()).toBeNull()
+    } finally {
+      warnings.restore()
+    }
+    expect(warnings.entries).toMatchObject([
+      { skips: [{ reason: 'no_eligible_target', candidates: 1 }] }
+    ])
+    expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
+      generation: 1,
+      enabled: true
+    })
+    expect(await database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([])
+    await database.close()
+  })
+
+  it('leaves a host alone until its cooldown expires, then moves it back', async () => {
+    const context = await setup({ hostCooldownMs: 3 * 24 * 60 * 60_000 })
+    const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
+    const targetControl = await completeRehomeToTarget(context, identity)
+    // Past the dispatch interval the earlier claim charged, so the next tick
+    // really does scan and the cooldown is the only thing holding this host.
+    context.advance(10_000)
+    // The desktop's region probe now says us-central1 again.
+    await context.store.assign(identity, 'us-central1')
+
+    const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped')
+    try {
+      expect(await context.store.claimRegionalRehome()).toBeNull()
+    } finally {
+      warnings.restore()
+    }
+    expect(warnings.entries).toEqual([])
+    expect(await context.store.resolve(identity)).toMatchObject({ cellId: target.id })
+
+    context.advance(3 * 24 * 60 * 60_000)
+    await freshHeartbeats(context)
+    await context.store.renewControlActivity(identity, {
+      activityId: targetControl,
+      cellId: target.id,
+      expiresAt: context.now() + 90_000
+    })
+    await context.store.assign(identity, 'us-central1')
+
+    const attempt = await context.store.claimRegionalRehome()
+    expect(attempt).toMatchObject({
+      preferredRegion: 'us-central1',
+      sourceCellId: target.id,
+      targetCellId: source.id
+    })
+    await context.database.close()
+  })
+
+  it('rejects a host whose attempt lands between the scan and the claim', async () => {
+    const database = await openInMemoryRelayDatabase()
+    const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
+    const context = await setup({
+      database,
+      wrap: (delegate) =>
+        hookAfterCandidateScan(delegate, async (transaction) => {
+          await transaction.query(
+            `INSERT INTO relay_region_rehome_attempts
+             (attempt_id, user_id, relay_host_id, preferred_region, source_cell_id,
+              source_cell_incarnation, target_cell_id, target_cell_incarnation,
+              previous_epoch, assignment_epoch, drain_grace_ms, send_attempts,
+              created_at, updated_at)
+             VALUES ('raced', ?, ?, 'asia-east2', ?, ?, ?, ?, 0, 1, 0, 0, ?, ?)`,
+            [
+              identity.userId,
+              identity.relayHostId,
+              source.id,
+              sourceIncarnation,
+              target.id,
+              targetIncarnation,
+              context.now(),
+              context.now()
+            ]
+          )
+        })
+    })
+    await activatePreferredSource(context, identity)
+
+    const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped')
+    try {
+      expect(await context.store.claimRegionalRehome()).toBeNull()
+    } finally {
+      warnings.restore()
+    }
+    expect(warnings.entries).toMatchObject([
+      { skips: [{ reason: 'host_cooldown', candidates: 1 }] }
+    ])
+    expect(await database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([])
+    await database.close()
+  })
+
+  it('does not scan a candidate whose preferred region has no drainable cell', async () => {
+    // A cell without the drain protocol cannot be a target: the host would land
+    // where no later rehome could move it out again. The candidate query drops
+    // it, so the tick stays idle instead of paying for an inventory scan.
+    const context = await setup({ targetProtocol: 0 })
+    await activatePreferredSource(context, {
+      userId: 'user-1',
+      relayHostId: 'abcdefghijklmnop'
+    })
+
+    const warnings = collectEventWarnings('orca_relay_regional_rehome_candidates_skipped')
+    try {
+      expect(await context.store.claimRegionalRehome()).toBeNull()
+    } finally {
+      warnings.restore()
+    }
+    expect(warnings.entries).toEqual([])
+    expect(
+      await context.database.query(
+        `SELECT next_dispatch_at FROM relay_region_rehome_worker_state`
+      )
+    ).toEqual([{ next_dispatch_at: 0 }])
+    expect(await context.database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([])
+    await context.database.close()
+  })
+
   it('skips an unclean cell without latching the control off', async () => {
     const context = await setup()
     await activatePreferredSource(context, {
@@ -452,7 +674,7 @@ describe('regional rehome assignment state', () => {
       databasePoolWaitersMax: 0,
       databasePoolWaitMsMax: 0
     })
-    await heartbeat(context.store, target, targetIncarnation, 0, 2, {
+    await heartbeat(context.store, target, targetIncarnation, 1, 2, {
       observedAt: context.now(),
       sqlFailures: 0,
       reconnects: 0,
@@ -472,6 +694,7 @@ describe('regional rehome assignment state', () => {
       notBefore: context.now(),
       ratePerMinute: 10,
       preferenceMaxAgeMs: 24 * 60 * 60_000,
+      hostCooldownMs: 7 * 24 * 60 * 60_000,
       drainGraceMs: 60_000
     })
     const retry = await context.store.claimRegionalRehome()
@@ -542,7 +765,7 @@ describe('regional rehome assignment state', () => {
     await activatePreferredSource(context, identity)
     await context.store.claimRegionalRehome()
     context.advance(6 * 60_000)
-    await heartbeat(context.store, target, targetIncarnation, 0, 2)
+    await heartbeat(context.store, target, targetIncarnation, 1, 2)
 
     expect(await context.store.refreshRegionalRehomeLeases()).toBe(0)
     expect(await context.store.abortExpiredEvacuations()).toBe(0)
@@ -556,6 +779,341 @@ describe('regional rehome assignment state', () => {
     await context.database.close()
   })
 
+  it('skips a rehome dispatch tick on a contended cell inventory', async () => {
+    const probe = new CellInventoryLockProbe()
+    const context = await setup({ wrap: (database) => probe.wrap(database) })
+    const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
+    await activatePreferredSource(context, identity)
+    probe.reset()
+    probe.failNoWait = true
+    const busy = collectEventWarnings('orca_relay_sweep_cell_inventory_busy')
+
+    let attempt: unknown
+    try {
+      attempt = await context.store.claimRegionalRehome()
+    } finally {
+      busy.restore()
+    }
+
+    expect(attempt).toBeNull()
+    expect(probe.locks).not.toEqual([])
+    expect(probe.locks.every((options) => options?.failIfUnavailable === true)).toBe(true)
+    expect(busy.entries).toEqual([
+      {
+        event: 'orca_relay_sweep_cell_inventory_busy',
+        sweep: 'claim-regional-rehome',
+        skipped: 1
+      }
+    ])
+
+    probe.failNoWait = false
+    expect(await context.store.claimRegionalRehome()).toMatchObject({
+      sourceCellId: source.id,
+      targetCellId: target.id
+    })
+    await context.database.close()
+  })
+
+  // Why: the redrain lane reaches the inventory through the fleet-safety read
+  // rather than through candidate selection, so it needs its own coverage.
+  // Why: one contended candidate must cost its own tick, not the whole page. The
+  // sweeps are explicitly per-candidate isolated for exactly this reason.
+  it('completes the candidates behind a contended one', async () => {
+    const probe = new CellInventoryLockProbe()
+    const context = await setup({ wrap: (database) => probe.wrap(database) })
+    const identities = [
+      { userId: 'user-1', relayHostId: 'abcdefghijklmnop' },
+      { userId: 'user-2', relayHostId: 'ponmlkjihgfedcba' }
+    ]
+    for (const identity of identities) {
+      // Dispatch is rate limited, so each claim needs its own interval.
+      context.advance(60_000)
+      await freshHeartbeats(context)
+      const sourceControl = await activatePreferredSource(context, identity)
+      const attempt = await context.store.claimRegionalRehome()
+      await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted')
+      await context.store.activateControl(identity, {
+        cellId: target.id,
+        assignmentEpoch: 2,
+        generation: 1
+      })
+      await context.store.markMigrationTargetRegistered(identity, {
+        cellId: target.id,
+        assignmentEpoch: 2
+      })
+      await context.store.releaseActivity(identity, sourceControl)
+    }
+    probe.reset()
+    probe.failNoWaitTimes = 1
+    const busy = collectEventWarnings('orca_relay_sweep_cell_inventory_busy')
+
+    let completed: number
+    try {
+      completed = await context.store.completeReadyRegionalRehomes()
+    } finally {
+      busy.restore()
+    }
+
+    expect(completed).toBe(1)
+    expect(busy.entries).toEqual([
+      {
+        event: 'orca_relay_sweep_cell_inventory_busy',
+        sweep: 'complete-ready-regional-rehomes',
+        skipped: 1
+      }
+    ])
+    await context.database.close()
+  })
+
+  // Why: with `continue` replaced by `break` a single contended candidate drops
+  // the rest of the page. Two in a row prove the sweep resumes, not just that it
+  // survived one, and that the summary counts both.
+  it('completes a candidate behind two contended ones', async () => {
+    const probe = new CellInventoryLockProbe()
+    const context = await setup({ wrap: (database) => probe.wrap(database) })
+    const identities = [
+      { userId: 'user-1', relayHostId: 'abcdefghijklmnop' },
+      { userId: 'user-2', relayHostId: 'ponmlkjihgfedcba' },
+      { userId: 'user-3', relayHostId: 'aaaabbbbccccdddd' }
+    ]
+    for (const identity of identities) {
+      // Dispatch is rate limited, so each claim needs its own interval.
+      context.advance(60_000)
+      await freshHeartbeats(context)
+      const sourceControl = await activatePreferredSource(context, identity)
+      const attempt = await context.store.claimRegionalRehome()
+      await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted')
+      await context.store.activateControl(identity, {
+        cellId: target.id,
+        assignmentEpoch: 2,
+        generation: 1
+      })
+      await context.store.markMigrationTargetRegistered(identity, {
+        cellId: target.id,
+        assignmentEpoch: 2
+      })
+      await context.store.releaseActivity(identity, sourceControl)
+    }
+    probe.reset()
+    probe.failNoWaitTimes = 2
+    const busy = collectEventWarnings('orca_relay_sweep_cell_inventory_busy')
+
+    let completed: number
+    try {
+      completed = await context.store.completeReadyRegionalRehomes()
+    } finally {
+      busy.restore()
+    }
+
+    expect(completed).toBe(1)
+    expect(busy.entries).toEqual([
+      {
+        event: 'orca_relay_sweep_cell_inventory_busy',
+        sweep: 'complete-ready-regional-rehomes',
+        skipped: 2
+      }
+    ])
+    await context.database.close()
+  })
+
+  // Why: only inventory contention is ordinary. Every other failure must keep its
+  // existing propagation and its dispatch-failure accounting.
+  it('propagates a claim failure that is not inventory contention', async () => {
+    const probe = new CellInventoryLockProbe()
+    const context = await setup({ wrap: (database) => probe.wrap(database) })
+    await activatePreferredSource(context, { userId: 'user-1', relayHostId: 'abcdefghijklmnop' })
+    probe.reset()
+    probe.failWith = new Error('relay_capacity_exhausted')
+    const busy = collectEventWarnings('orca_relay_sweep_cell_inventory_busy')
+
+    try {
+      await expect(context.store.claimRegionalRehome()).rejects.toThrow(
+        'relay_capacity_exhausted'
+      )
+    } finally {
+      busy.restore()
+    }
+
+    expect(busy.entries).toEqual([])
+    await context.database.close()
+  })
+
+  // Why: the transaction dies at the first contended candidate, so every
+  // candidate behind it is abandoned too. Reporting one would understate the tick.
+  it('reports every candidate the contended tick abandoned', async () => {
+    const probe = new CellInventoryLockProbe()
+    const context = await setup({ wrap: (database) => probe.wrap(database) })
+    await activatePreferredSource(context, { userId: 'user-1', relayHostId: 'abcdefghijklmnop' })
+    await activatePreferredSource(context, { userId: 'user-2', relayHostId: 'ponmlkjihgfedcba' })
+    await activatePreferredSource(context, { userId: 'user-3', relayHostId: 'aaaabbbbccccdddd' })
+    probe.reset()
+    probe.failNoWait = true
+    const busy = collectEventWarnings('orca_relay_sweep_cell_inventory_busy')
+
+    try {
+      expect(await context.store.claimRegionalRehome()).toBeNull()
+    } finally {
+      busy.restore()
+    }
+
+    expect(busy.entries).toEqual([
+      {
+        event: 'orca_relay_sweep_cell_inventory_busy',
+        sweep: 'claim-regional-rehome',
+        skipped: 3
+      }
+    ])
+    await context.database.close()
+  })
+
+  it('skips a redrain tick on a contended cell inventory', async () => {
+    const probe = new CellInventoryLockProbe()
+    const context = await setup({ wrap: (database) => probe.wrap(database) })
+    const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
+    await activatePreferredSource(context, identity)
+    const attempt = await context.store.claimRegionalRehome()
+    await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted')
+    await context.store.activateControl(identity, {
+      cellId: target.id,
+      assignmentEpoch: 2,
+      generation: 1
+    })
+    await context.store.markMigrationTargetRegistered(identity, {
+      cellId: target.id,
+      assignmentEpoch: 2
+    })
+    context.advance(60 * 60_000 + 1)
+    await freshHeartbeats(context)
+    probe.reset()
+    probe.failNoWait = true
+    const busy = collectEventWarnings('orca_relay_sweep_cell_inventory_busy')
+
+    let redrain: unknown
+    try {
+      redrain = await context.store.claimRegionalRehome()
+    } finally {
+      busy.restore()
+    }
+
+    expect(redrain).toBeNull()
+    expect(probe.locks).not.toEqual([])
+    expect(probe.locks.every((options) => options?.failIfUnavailable === true)).toBe(true)
+    expect(busy.entries).toEqual([
+      {
+        event: 'orca_relay_sweep_cell_inventory_busy',
+        sweep: 'claim-regional-rehome',
+        skipped: 1
+      }
+    ])
+
+    probe.failNoWait = false
+    expect(await context.store.claimRegionalRehome()).toMatchObject({
+      attemptId: attempt!.attemptId,
+      sendAttempts: 2
+    })
+    await context.database.close()
+  })
+
+  it('skips a completion tick on a contended cell inventory without quarantining it', async () => {
+    const probe = new CellInventoryLockProbe()
+    const context = await setup({ wrap: (database) => probe.wrap(database) })
+    const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
+    const sourceControl = await activatePreferredSource(context, identity)
+    const attempt = await context.store.claimRegionalRehome()
+    await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted')
+    await context.store.activateControl(identity, {
+      cellId: target.id,
+      assignmentEpoch: 2,
+      generation: 1
+    })
+    await context.store.markMigrationTargetRegistered(identity, {
+      cellId: target.id,
+      assignmentEpoch: 2
+    })
+    await context.store.releaseActivity(identity, sourceControl)
+    probe.reset()
+    probe.failNoWait = true
+    const busy = collectEventWarnings('orca_relay_sweep_cell_inventory_busy')
+    const failures = collectCandidateFailureWarnings()
+
+    let completed: number
+    try {
+      completed = await context.store.completeReadyRegionalRehomes()
+    } finally {
+      failures.restore()
+      busy.restore()
+    }
+
+    expect(completed).toBe(0)
+    expect(probe.locks).not.toEqual([])
+    expect(probe.locks.every((options) => options?.failIfUnavailable === true)).toBe(true)
+    expect(failures.entries).toEqual([])
+    expect(busy.entries).toEqual([
+      {
+        event: 'orca_relay_sweep_cell_inventory_busy',
+        sweep: 'complete-ready-regional-rehomes',
+        skipped: 1
+      }
+    ])
+
+    probe.failNoWait = false
+    expect(await context.store.completeReadyRegionalRehomes()).toBe(1)
+    await context.database.close()
+  })
+
+  // Why: a contended inventory is another director settling the same row, not a
+  // poisoned candidate. Quarantining on it would exclude a healthy attempt from
+  // the sweep's LIMIT pages for 15 minutes.
+  it('skips an abort tick on a contended cell inventory without quarantining it', async () => {
+    const probe = new CellInventoryLockProbe()
+    const context = await setup({ wrap: (database) => probe.wrap(database) })
+    const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
+    const sourceControl = await activatePreferredSource(context, identity)
+    const attempt = await context.store.claimRegionalRehome()
+    await context.store.recordRegionalRehomeDrainReceipt(attempt!.attemptId, 'accepted')
+    const targetControl = await context.store.activateControl(identity, {
+      cellId: target.id,
+      assignmentEpoch: 2,
+      generation: 1
+    })
+    await context.store.markMigrationTargetRegistered(identity, {
+      cellId: target.id,
+      assignmentEpoch: 2
+    })
+    await context.store.releaseActivity(identity, sourceControl)
+    await context.store.releaseActivity(identity, targetControl)
+    context.advance(24 * 60 * 60_000)
+    await heartbeat(context.store, source, sourceIncarnation, 1, 2)
+    probe.reset()
+    probe.failNoWait = true
+    const busy = collectEventWarnings('orca_relay_sweep_cell_inventory_busy')
+    const failures = collectCandidateFailureWarnings()
+
+    let aborted: number
+    try {
+      aborted = await context.store.abortExpiredRegionalRehomes()
+    } finally {
+      failures.restore()
+      busy.restore()
+    }
+
+    expect(aborted).toBe(0)
+    expect(probe.locks).not.toEqual([])
+    expect(probe.locks.every((options) => options?.failIfUnavailable === true)).toBe(true)
+    expect(failures.entries).toEqual([])
+    expect(busy.entries).toEqual([
+      {
+        event: 'orca_relay_sweep_cell_inventory_busy',
+        sweep: 'abort-expired-regional-rehomes',
+        skipped: 1
+      }
+    ])
+
+    probe.failNoWait = false
+    expect(await context.store.abortExpiredRegionalRehomes()).toBe(1)
+    await context.database.close()
+  })
+
   it('rolls back an inactive registered target only after the 24-hour bound', async () => {
     const context = await setup()
     const identity = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
@@ -1208,10 +1766,18 @@ function collectDisableWarnings() {
   }
 }
 
-async function setup(options: { sourceProtocol?: number } = {}) {
+async function setup(
+  options: {
+    sourceProtocol?: number
+    targetProtocol?: number
+    hostCooldownMs?: number
+    database?: RelayDatabase
+    wrap?: (database: RelayDatabase) => RelayDatabase
+  } = {}
+) {
   let clock = 1_000_000
-  const database = await openInMemoryRelayDatabase()
-  const store = new RelayAssignmentStore(database, () => clock, {
+  const database = options.database ?? (await openInMemoryRelayDatabase())
+  const store = new RelayAssignmentStore(options.wrap?.(database) ?? database, () => clock, {
     requireLiveCells: true,
     heartbeatTtlMs: 45_000
   })
@@ -1223,11 +1789,12 @@ async function setup(options: { sourceProtocol?: number } = {}) {
     notBefore: clock,
     ratePerMinute: 10,
     preferenceMaxAgeMs: 24 * 60 * 60_000,
+    hostCooldownMs: options.hostCooldownMs ?? 7 * 24 * 60 * 60_000,
     drainGraceMs: 60 * 60_000
   })
   await store.reconcileCells([source, target])
   await heartbeat(store, source, sourceIncarnation, options.sourceProtocol ?? 1)
-  await heartbeat(store, target, targetIncarnation, 0)
+  await heartbeat(store, target, targetIncarnation, options.targetProtocol ?? 1)
   return {
     database,
     store,
@@ -1329,7 +1896,7 @@ async function freshHeartbeats(context: Context): Promise<void> {
   }
   // The clock doubles as a strictly-increasing connection inclusion watermark.
   await heartbeat(context.store, source, sourceIncarnation, 1, context.now(), safety)
-  await heartbeat(context.store, target, targetIncarnation, 0, context.now(), safety)
+  await heartbeat(context.store, target, targetIncarnation, 1, context.now(), safety)
 }
 
 async function activatePreferredSource(
@@ -1346,6 +1913,68 @@ async function activatePreferredSource(
   return control
 }
 
+// Runs a hook inside the claim transaction, right after the candidate scan, so
+// a scan-versus-claim race is deterministic instead of timing-dependent.
+function hookAfterCandidateScan(
+  database: RelayDatabase,
+  hook: (transaction: RelayDatabase) => Promise<void>
+): RelayDatabase {
+  let fired = false
+  const decorate = (delegate: RelayDatabase): RelayDatabase => ({
+    query: async (sql, params) => {
+      const rows = await delegate.query(sql, params)
+      if (!fired && sql.includes('FROM relay_assignment_region_preferences preference')) {
+        fired = true
+        await hook(delegate)
+      }
+      return rows
+    },
+    queryLocked: async (sql, params, lockOptions) =>
+      await delegate.queryLocked(sql, params, lockOptions),
+    transaction: async (operation, transactionOptions) =>
+      await delegate.transaction(
+        async (transaction) => await operation(decorate(transaction)),
+        transactionOptions
+      ),
+    close: async () => undefined
+  })
+  return decorate(database)
+}
+
+async function completeRehomeToTarget(
+  context: Context,
+  identity: { userId: string; relayHostId: string }
+): Promise<string> {
+  const sourceControl = await activatePreferredSource(context, identity)
+  const attempt = await context.store.claimRegionalRehome()
+  const targetControl = await context.store.activateControl(identity, {
+    cellId: target.id,
+    assignmentEpoch: attempt!.assignmentEpoch,
+    generation: 1
+  })
+  await context.store.markMigrationTargetRegistered(identity, {
+    cellId: target.id,
+    assignmentEpoch: attempt!.assignmentEpoch
+  })
+  await context.store.releaseActivity(identity, sourceControl)
+  await context.store.completeReadyRegionalRehomes()
+  return targetControl
+}
+
+async function activateReversePreferredSource(
+  context: Context,
+  identity: { userId: string; relayHostId: string }
+): Promise<string> {
+  const assignment = await context.store.assign(identity, undefined, 'asia-east2')
+  const control = await context.store.activateControl(identity, {
+    cellId: target.id,
+    assignmentEpoch: assignment.assignmentEpoch,
+    generation: 1
+  })
+  await context.store.assign(identity, 'us-central1')
+  return control
+}
+
 async function activateSource(
   context: Context,
   identity: { userId: string; relayHostId: string }
@@ -1397,3 +2026,43 @@ async function heartbeat(
     }
   })
 }
+
+class CellInventoryLockProbe {
+  readonly locks: (RelayLockOptions | undefined)[] = []
+  failNoWait = false
+  // Contends the first N candidates only, so the sweep must carry on past them.
+  failNoWaitTimes = 0
+  failWith: Error | null = null
+
+  reset(): void {
+    this.locks.length = 0
+  }
+
+  wrap(database: RelayDatabase): RelayDatabase {
+    const probe = this
+    const decorate = (delegate: RelayDatabase): RelayDatabase => ({
+      query: async (sql, params) => await delegate.query(sql, params),
+      queryLocked: async (sql, params, options) => {
+        if (sql.trim() === 'SELECT * FROM relay_cells ORDER BY cell_id ASC') {
+          probe.locks.push(options)
+          if (probe.failWith) throw probe.failWith
+          if (options?.failIfUnavailable && probe.failNoWaitTimes > 0) {
+            probe.failNoWaitTimes--
+            throw new Error('database_lock_unavailable')
+          }
+          if (probe.failNoWait && options?.failIfUnavailable) {
+            throw new Error('database_lock_unavailable')
+          }
+        }
+        return await delegate.queryLocked(sql, params, options)
+      },
+      transaction: async (operation, options) =>
+        await delegate.transaction(
+          async (transaction) => await operation(decorate(transaction)),
+          options
+        ),
+      close: async () => undefined
+    })
+    return decorate(database)
+  }
+}

cloud/apps/relay/src/regional-rehome-target-selection.test.ts

@@ -36,6 +36,7 @@ async function setup() {
     notBefore: clock,
     ratePerMinute: 10,
     preferenceMaxAgeMs: 24 * 60 * 60_000,
+    hostCooldownMs: 7 * 24 * 60 * 60_000,
     drainGraceMs: 60 * 60_000
   })
   await store.reconcileCells([source, noHeadroom, unclean, highLoad, lowLoad])
@@ -100,22 +101,22 @@ describe('regional rehome target selection', () => {
       sqlFailures: 0
     })
     // Lowest load but the connection hard cap is exhausted.
-    await context.beat(noHeadroom, 2, 0, {
+    await context.beat(noHeadroom, 2, 1, {
       observedRequests: 0,
       enforcedConnections: 999,
       sqlFailures: 0
     })
-    await context.beat(unclean, 3, 0, {
+    await context.beat(unclean, 3, 1, {
       observedRequests: 0,
       enforcedConnections: 0,
       sqlFailures: UNCLEAN
     })
-    await context.beat(highLoad, 4, 0, {
+    await context.beat(highLoad, 4, 1, {
       observedRequests: 50,
       enforcedConnections: 0,
       sqlFailures: 0
     })
-    await context.beat(lowLoad, 5, 0, {
+    await context.beat(lowLoad, 5, 1, {
       observedRequests: 10,
       enforcedConnections: 0,
       sqlFailures: 0
@@ -134,22 +135,22 @@ describe('regional rehome target selection', () => {
       enforcedConnections: 0,
       sqlFailures: 0
     })
-    await context.beat(noHeadroom, 2, 0, {
+    await context.beat(noHeadroom, 2, 1, {
       observedRequests: 0,
       enforcedConnections: 999,
       sqlFailures: 0
     })
-    await context.beat(unclean, 3, 0, {
+    await context.beat(unclean, 3, 1, {
       observedRequests: 0,
       enforcedConnections: 0,
       sqlFailures: UNCLEAN
     })
-    await context.beat(highLoad, 4, 0, {
+    await context.beat(highLoad, 4, 1, {
       observedRequests: 50,
       enforcedConnections: 0,
       sqlFailures: 0
     })
-    await context.beat(lowLoad, 5, 0, {
+    await context.beat(lowLoad, 5, 1, {
       observedRequests: 10,
       enforcedConnections: 0,
       sqlFailures: UNCLEAN

cloud/apps/relay/src/regional-rehome-worker.ts

@@ -3,6 +3,7 @@ import type { RelayAssignmentStore } from './assignment-store.js'
 import type { RelayConfig } from './config.js'
 import { googleMetadataIdentityToken } from './google-metadata-identity-token.js'
 import type { RegionalRehomeSafetySnapshot } from './relay-observability.js'
+import { jitteredSweepIntervalMs } from './relay-sweep-schedule.js'
 
 type RegionalRehomeWorkerOptions = {
   fetch?: typeof fetch
@@ -10,6 +11,7 @@ type RegionalRehomeWorkerOptions = {
   now?: () => number
   intervalMs?: number
   requestTimeoutMs?: number
+  random?: () => number
   safetySnapshot?: () => RegionalRehomeSafetySnapshot
 }
 
@@ -109,7 +111,10 @@ export function startRegionalRehomeWorker(
       inFlight = false
     }
   }
-  const timer = setInterval(() => void run(), options.intervalMs ?? 1_000)
+  const timer = setInterval(
+    () => void run(),
+    options.intervalMs ?? jitteredSweepIntervalMs(1_000, options.random)
+  )
   timer.unref()
   void run()
   return {

cloud/apps/relay/src/relay-observability.test.ts

@@ -1,7 +1,9 @@
+import { RELAY_REGION_METRIC_SEGMENTS, RELAY_REGIONS } from '@orca-cloud/relay-contract'
 import { describe, expect, it, vi } from 'vitest'
 import type { RelayDatabase } from './database.js'
 import { observeRelayDatabase } from './observed-relay-database.js'
 import {
+  CONTROL_RTT_RESERVOIR_LIMIT,
   observedRelayRequests,
   RelayObservability,
   type RelayProcessCounts
@@ -22,6 +24,37 @@ const counts: RelayProcessCounts = {
   databasePoolWaitMsMax: 1_250
 }
 
+// Two schema keys legitimately spell a policed word: the abandoned-accept bucket
+// is keyed by stage name and one stage is `credential`. Rename those exact keys in
+// a clone instead of rewriting the JSON, so a stray raw field or value anywhere
+// else still trips the guard below.
+const SCHEMA_KEY_ALIASES: Record<string, string> = {
+  clientAcceptCredentialMsP95: 'clientAcceptStageTwoMsP95'
+}
+
+function scrubSchemaKeys(entries: Array<Record<string, unknown>>): string {
+  return JSON.stringify(
+    entries.map((entry) =>
+      Object.fromEntries(
+        Object.entries(entry).map(([key, value]) => [
+          SCHEMA_KEY_ALIASES[key] ?? key,
+          key === 'clientAcceptsAbandonedByStageDelta' ? renameStageKeys(value) : value
+        ])
+      )
+    )
+  )
+}
+
+function renameStageKeys(bucket: unknown): unknown {
+  if (bucket === null || typeof bucket !== 'object') return bucket
+  return Object.fromEntries(
+    Object.entries(bucket).map(([stage, count]) => [
+      stage === 'credential' ? 'stageTwo' : stage,
+      count
+    ])
+  )
+}
+
 describe('relay observability', () => {
   it('emits safe readiness dependency outcomes', () => {
     const entries: Array<Record<string, unknown>> = []
@@ -106,14 +139,31 @@ describe('relay observability', () => {
       requestedRegionsDelta: { 'asia-east2': 1, unhinted: 1 },
       selectedRegionsDelta: { 'us-central1': 1 },
       regionFallbacksDelta: { 'asia-east2': 1 },
-      unavailableRegionsDelta: { 'asia-east2': 1 }
+      unavailableRegionsDelta: { 'asia-east2': 1 },
+      // Flat per-region siblings the log-based metrics extract; `unhinted` stays map-only.
+      requestedRegionUsCentral1Delta: 0,
+      requestedRegionAsiaEast2Delta: 1,
+      selectedRegionUsCentral1Delta: 1,
+      selectedRegionAsiaEast2Delta: 0
     })
     expect(entries[1]).toMatchObject({
       requestedRegionsDelta: {},
       selectedRegionsDelta: {},
       regionFallbacksDelta: {},
-      unavailableRegionsDelta: {}
+      unavailableRegionsDelta: {},
+      // Zeros keep publishing so an idle window cannot drop a series out of the skew join.
+      requestedRegionUsCentral1Delta: 0,
+      requestedRegionAsiaEast2Delta: 0,
+      selectedRegionUsCentral1Delta: 0,
+      selectedRegionAsiaEast2Delta: 0
     })
+    // A region added to the contract has to reach the flat keys, or the skew alert's
+    // denominator silently misses it.
+    for (const segment of Object.values(RELAY_REGION_METRIC_SEGMENTS)) {
+      expect(entries[0]).toHaveProperty(`requestedRegion${segment}Delta`)
+      expect(entries[0]).toHaveProperty(`selectedRegion${segment}Delta`)
+    }
+    expect(Object.keys(RELAY_REGION_METRIC_SEGMENTS).sort()).toEqual([...RELAY_REGIONS].sort())
   })
 
   it('emits bounded aggregate runtime signals without identities or credentials', () => {
@@ -181,7 +231,7 @@ describe('relay observability', () => {
       controlActivityRecoveryFailuresDelta: 0,
       httpLatencyMsMax: 0
     })
-    expect(JSON.stringify(entries)).not.toMatch(/token|credential|userId|relayHostId/)
+    expect(scrubSchemaKeys(entries)).not.toMatch(/token|credential|userId|relayHostId/i)
   })
 
   it('aggregates control and splice closes as bounded per-reason deltas', () => {
@@ -195,19 +245,137 @@ describe('relay observability', () => {
     observability.recordControlClose(4402)
     observability.recordSpliceClose('host-oversize-frame')
     observability.recordSpliceClose('queue-limit')
+    observability.recordClientAcceptAbandoned('activity', 14_250.4)
+    observability.recordClientAcceptAbandoned('activity', 2_000)
+    observability.recordClientAcceptAbandoned('credential', 3_000)
     observability.flush(counts)
     observability.flush(counts)
 
     expect(entries[0]).toMatchObject({
       controlClosesByCodeDelta: { 1006: 2, 4402: 1 },
-      spliceClosesByTriggerDelta: { 'host-oversize-frame': 1, 'queue-limit': 1 }
+      spliceClosesByTriggerDelta: { 'host-oversize-frame': 1, 'queue-limit': 1 },
+      clientAcceptsAbandonedByStageDelta: { activity: 2, credential: 1 },
+      clientAcceptAbandonedMsMax: 14_250.4
     })
     expect(entries[1]).toMatchObject({
       controlClosesByCodeDelta: {},
-      spliceClosesByTriggerDelta: {}
+      spliceClosesByTriggerDelta: {},
+      clientAcceptsAbandonedByStageDelta: {},
+      clientAcceptAbandonedMsMax: 0
     })
   })
 
+  it('summarises completed client accepts and control round trips per window', () => {
+    const entries: Array<Record<string, unknown>> = []
+    const observability = new RelayObservability(
+      { role: 'cell', cellId: 'production-gce-c28', region: 'asia-east2' },
+      (entry) => entries.push(entry)
+    )
+    observability.recordClientAcceptCompleted({
+      totalMs: 812.4567,
+      stageMs: { assignment: 120, credential: 90, activity: 40, attach: 500, basis: 62 }
+    })
+    observability.recordClientAcceptCompleted({
+      totalMs: 6_400,
+      stageMs: { assignment: 4_100, credential: 95, activity: 60, attach: 2_000, basis: 145 }
+    })
+    observability.recordControlRtt(28)
+    observability.recordControlRtt(240)
+    observability.recordControlRtt(31)
+    observability.flush(counts)
+    observability.flush(counts)
+
+    expect(entries[0]).toMatchObject({
+      clientAcceptCompletedDelta: 2,
+      clientAcceptTotalMsP50: 812.457,
+      clientAcceptTotalMsP95: 6_400,
+      clientAcceptTotalMsMax: 6_400,
+      clientAcceptAssignmentMsP95: 4_100,
+      clientAcceptCredentialMsP95: 95,
+      clientAcceptActivityMsP95: 60,
+      clientAcceptAttachMsP95: 2_000,
+      clientAcceptBasisMsP95: 145,
+      controlRttSamplesDelta: 3,
+      controlRttMsP50: 31,
+      controlRttMsP95: 240,
+      controlRttMsMax: 240
+    })
+    // Only-add: the pre-existing fields still read the same after the extension.
+    expect(entries[0]).toMatchObject({
+      event: 'orca_relay_runtime_metrics',
+      metricVersion: 2,
+      clientAcceptsAbandonedByStageDelta: {},
+      clientAcceptAbandonedMsMax: 0
+    })
+    // An empty window publishes counts only: a zero percentile point is
+    // indistinguishable from a real zero once Cloud Logging aggregates it.
+    expect(entries[1]).toMatchObject({ clientAcceptCompletedDelta: 0, controlRttSamplesDelta: 0 })
+    for (const omitted of [
+      'clientAcceptTotalMsP50',
+      'clientAcceptTotalMsP95',
+      'clientAcceptTotalMsMax',
+      'clientAcceptAssignmentMsP95',
+      'clientAcceptCredentialMsP95',
+      'clientAcceptActivityMsP95',
+      'clientAcceptAttachMsP95',
+      'clientAcceptBasisMsP95',
+      'controlRttMsP50',
+      'controlRttMsP95',
+      'controlRttMsMax'
+    ]) {
+      expect(entries[1]).not.toHaveProperty(omitted)
+      expect(entries[0]).toHaveProperty(omitted)
+    }
+    expect(scrubSchemaKeys(entries)).not.toMatch(/token|credential|userId|relayHostId/i)
+  })
+
+  it('caps the control round-trip reservoir and reports what it dropped', () => {
+    const entries: Array<Record<string, unknown>> = []
+    const observability = new RelayObservability(
+      { role: 'cell', cellId: 'production-gce-c28', region: 'asia-east2' },
+      (entry) => entries.push(entry)
+    )
+    const flooded = CONTROL_RTT_RESERVOIR_LIMIT * 20
+    for (let sample = 0; sample < flooded; sample++) {
+      observability.recordControlRtt(10 + (sample % 40))
+    }
+    observability.flush(counts)
+
+    // Dropped is observed minus retained, so this pins the retained window at the cap.
+    expect(entries[0]).toMatchObject({
+      controlRttSamplesDelta: flooded,
+      controlRttSamplesDroppedDelta: flooded - CONTROL_RTT_RESERVOIR_LIMIT
+    })
+    // The kept samples are real observations, not a truncated or synthesised window.
+    expect(entries[0]!.controlRttMsP50 as number).toBeGreaterThanOrEqual(10)
+    expect(entries[0]!.controlRttMsMax as number).toBeLessThanOrEqual(49)
+
+    observability.flush(counts)
+    expect(entries[1]).toMatchObject({
+      controlRttSamplesDelta: 0,
+      controlRttSamplesDroppedDelta: 0
+    })
+    expect(entries[1]).not.toHaveProperty('controlRttMsP50')
+  })
+
+  it('samples the whole flooded window rather than its first samples', () => {
+    const entries: Array<Record<string, unknown>> = []
+    const observability = new RelayObservability(
+      { role: 'cell', cellId: 'production-gce-c28', region: 'asia-east2' },
+      (entry) => entries.push(entry)
+    )
+    const half = CONTROL_RTT_RESERVOIR_LIMIT * 10
+    for (let sample = 0; sample < half; sample++) observability.recordControlRtt(10)
+    for (let sample = 0; sample < half; sample++) observability.recordControlRtt(900)
+    observability.flush(counts)
+
+    // Keeping the first N instead would publish a window of nothing but 10s. Each
+    // reservoir slot ends up drawn from the late half with ~1/2 probability, so
+    // fewer than the 5% the p95 needs is out of reach of this suite.
+    expect(entries[0]!.controlRttMsP95).toBe(900)
+    expect(entries[0]!.controlRttMsMax).toBe(900)
+  })
+
   it('observes successful and failed database calls including transactions', async () => {
     const recordSql = vi.fn()
     const underlying: RelayDatabase = {

cloud/apps/relay/src/relay-observability.ts

@@ -1,6 +1,7 @@
 import { monitorEventLoopDelay, performance } from 'node:perf_hooks'
-import type { RelayRegion } from '@orca-cloud/relay-contract'
+import { RELAY_REGION_METRIC_SEGMENTS, type RelayRegion } from '@orca-cloud/relay-contract'
 import type { ControlRenewalOutcome } from './assignment-store.js'
+import type { CellInventoryHoldCounts } from './cell-inventory-hold-samples.js'
 import type { PostgresPoolPressureCounts } from './postgres-pool-pressure.js'
 import type { RelayReadinessObservation } from './relay-readiness.js'
 
@@ -20,7 +21,9 @@ export function observedRelayRequests(counts: RelayRuntimeCounts): number {
   return counts.preAuthConnections + counts.controls + counts.splices + counts.pendingSplices
 }
 
-export type RelayProcessCounts = RelayRuntimeCounts & PostgresPoolPressureCounts
+export type RelayProcessCounts = RelayRuntimeCounts &
+  PostgresPoolPressureCounts &
+  Partial<CellInventoryHoldCounts>
 
 export type RegionalRehomeRuntimeSafety = {
   observedAt: number
@@ -61,6 +64,30 @@ export interface RelayRuntimeObserver {
   }): void
   recordControlClose?(code: number): void
   recordSpliceClose?(trigger: string): void
+  recordClientAcceptAbandoned?(stage: RelayClientAcceptStage, elapsedMs: number): void
+  recordClientAcceptCompleted?(sample: RelayClientAcceptSample): void
+  recordControlRtt?(rttMs: number): void
+}
+
+// Which serialized accept step the phone had already hung up behind.
+export type RelayClientAcceptStage = 'assignment' | 'credential' | 'activity'
+
+// The attach window and the basis writes that follow it are only measurable once
+// the host data leg lands, so they join the serialized pre-attach steps on
+// completed accepts only.
+export type RelayClientAcceptTimedStage = RelayClientAcceptStage | 'attach' | 'basis'
+
+export const RELAY_CLIENT_ACCEPT_TIMED_STAGES = [
+  'assignment',
+  'credential',
+  'activity',
+  'attach',
+  'basis'
+] as const satisfies readonly RelayClientAcceptTimedStage[]
+
+export type RelayClientAcceptSample = {
+  totalMs: number
+  stageMs: Record<RelayClientAcceptTimedStage, number>
 }
 
 type RelayMetricDeltas = {
@@ -84,12 +111,22 @@ type RelayMetricDeltas = {
   unavailableRegions: Record<string, number>
   controlClosesByCode: Record<string, number>
   spliceClosesByTrigger: Record<string, number>
+  clientAcceptsAbandonedByStage: Record<string, number>
+  clientAcceptAbandonedMsMax: number
+  clientAcceptTotalsMs: number[]
+  clientAcceptStageSamplesMs: Record<RelayClientAcceptTimedStage, number[]>
+  controlRttSamplesMs: number[]
+  controlRttObserved: number
   controlRenewalLatenciesMs: number[]
   controlRenewalsByOutcome: Record<string, number>
   controlActivityRecoveries: number
   controlActivityRecoveryFailures: number
 }
 
+// A host chooses how often it answers a ping, so the process-wide window is a
+// reservoir: the heap cost of a flood is capped and the percentiles stay unbiased.
+export const CONTROL_RTT_RESERVOIR_LIMIT = 1024
+
 type MetricWriter = (entry: Record<string, unknown>) => void
 
 const emptyDeltas = (): RelayMetricDeltas => ({
@@ -113,18 +150,44 @@ const emptyDeltas = (): RelayMetricDeltas => ({
   unavailableRegions: {},
   controlClosesByCode: {},
   spliceClosesByTrigger: {},
+  clientAcceptsAbandonedByStage: {},
+  clientAcceptAbandonedMsMax: 0,
+  clientAcceptTotalsMs: [],
+  clientAcceptStageSamplesMs: {
+    assignment: [],
+    credential: [],
+    activity: [],
+    attach: [],
+    basis: []
+  },
+  controlRttSamplesMs: [],
+  controlRttObserved: 0,
   controlRenewalLatenciesMs: [],
   controlRenewalsByOutcome: {},
   controlActivityRecoveries: 0,
   controlActivityRecoveryFailures: 0
 })
 
-function percentile(values: number[], percentileRank: number): number {
+export function percentile(values: number[], percentileRank: number): number {
   if (values.length === 0) return 0
   const sorted = [...values].sort((left, right) => left - right)
   return sorted[Math.ceil(percentileRank * sorted.length) - 1] ?? 0
 }
 
+function roundMs(value: number): number {
+  return Number(value.toFixed(3))
+}
+
+// Spreading a window into Math.max blows the stack once a busy cell samples
+// enough of it, so the maximum is folded instead.
+function latencySummary(samples: number[]): { p50: number; p95: number; max: number } {
+  return {
+    p50: roundMs(percentile(samples, 0.5)),
+    p95: roundMs(percentile(samples, 0.95)),
+    max: roundMs(samples.reduce((highest, sample) => Math.max(highest, sample), 0))
+  }
+}
+
 export class RelayObservability implements RelayRuntimeObserver {
   private readonly eventLoop = monitorEventLoopDelay({ resolution: 20 })
   private deltas = emptyDeltas()
@@ -225,6 +288,33 @@ export class RelayObservability implements RelayRuntimeObserver {
       (this.deltas.spliceClosesByTrigger[trigger] ?? 0) + 1
   }
 
+  recordClientAcceptAbandoned(stage: RelayClientAcceptStage, elapsedMs: number): void {
+    increment(this.deltas.clientAcceptsAbandonedByStage, stage)
+    this.deltas.clientAcceptAbandonedMsMax = Math.max(
+      this.deltas.clientAcceptAbandonedMsMax,
+      elapsedMs
+    )
+  }
+
+  recordClientAcceptCompleted(sample: RelayClientAcceptSample): void {
+    this.deltas.clientAcceptTotalsMs.push(sample.totalMs)
+    for (const stage of RELAY_CLIENT_ACCEPT_TIMED_STAGES) {
+      this.deltas.clientAcceptStageSamplesMs[stage].push(sample.stageMs[stage])
+    }
+  }
+
+  recordControlRtt(rttMs: number): void {
+    const samples = this.deltas.controlRttSamplesMs
+    const observedBefore = this.deltas.controlRttObserved++
+    if (samples.length < CONTROL_RTT_RESERVOIR_LIMIT) {
+      samples.push(rttMs)
+      return
+    }
+    // Algorithm R: every round trip in the window keeps an equal chance of being kept.
+    const slot = Math.floor(Math.random() * (observedBefore + 1))
+    if (slot < CONTROL_RTT_RESERVOIR_LIMIT) samples[slot] = rttMs
+  }
+
   start(readCounts: () => RelayProcessCounts, intervalMs = 30_000): void {
     if (this.timer) return
     this.eventLoop.enable()
@@ -258,6 +348,11 @@ export class RelayObservability implements RelayRuntimeObserver {
       controlActivityRecoveryFailures: deltas.controlActivityRecoveryFailures
     }
     this.deltas = emptyDeltas()
+    const acceptTotals = latencySummary(deltas.clientAcceptTotalsMs)
+    const acceptStageP95 = (stage: RelayClientAcceptTimedStage): number =>
+      roundMs(percentile(deltas.clientAcceptStageSamplesMs[stage], 0.95))
+    const controlRtt = latencySummary(deltas.controlRttSamplesMs)
+    const controlRenewal = latencySummary(deltas.controlRenewalLatenciesMs)
     const memory = process.memoryUsage()
     const p99 = this.eventLoop.count === 0 ? 0 : this.eventLoop.percentile(99) / 1_000_000
     this.eventLoop.reset()
@@ -282,13 +377,43 @@ export class RelayObservability implements RelayRuntimeObserver {
       placementRejectionsByReasonDelta: deltas.placementRejectionsByReason,
       requestedRegionsDelta: deltas.requestedRegions,
       selectedRegionsDelta: deltas.selectedRegions,
+      ...regionCounterFields('requestedRegion', deltas.requestedRegions),
+      ...regionCounterFields('selectedRegion', deltas.selectedRegions),
       regionFallbacksDelta: deltas.regionFallbacks,
       unavailableRegionsDelta: deltas.unavailableRegions,
       controlClosesByCodeDelta: deltas.controlClosesByCode,
       spliceClosesByTriggerDelta: deltas.spliceClosesByTrigger,
+      clientAcceptsAbandonedByStageDelta: deltas.clientAcceptsAbandonedByStage,
+      clientAcceptAbandonedMsMax: roundMs(deltas.clientAcceptAbandonedMsMax),
+      clientAcceptCompletedDelta: deltas.clientAcceptTotalsMs.length,
+      // Accepts are sparse: publishing a zero percentile for every empty window
+      // would pin the p50 at 0 forever and collapse the p95 at low accept rates.
+      ...(deltas.clientAcceptTotalsMs.length === 0
+        ? {}
+        : {
+            clientAcceptTotalMsP50: acceptTotals.p50,
+            clientAcceptTotalMsP95: acceptTotals.p95,
+            clientAcceptTotalMsMax: acceptTotals.max,
+            clientAcceptAssignmentMsP95: acceptStageP95('assignment'),
+            clientAcceptCredentialMsP95: acceptStageP95('credential'),
+            clientAcceptActivityMsP95: acceptStageP95('activity'),
+            clientAcceptAttachMsP95: acceptStageP95('attach'),
+            clientAcceptBasisMsP95: acceptStageP95('basis')
+          }),
+      // Every round trip observed in the window, including the ones the reservoir
+      // above declined to keep; the percentiles summarise only what it kept.
+      controlRttSamplesDelta: deltas.controlRttObserved,
+      controlRttSamplesDroppedDelta: deltas.controlRttObserved - deltas.controlRttSamplesMs.length,
+      ...(deltas.controlRttSamplesMs.length === 0
+        ? {}
+        : {
+            controlRttMsP50: controlRtt.p50,
+            controlRttMsP95: controlRtt.p95,
+            controlRttMsMax: controlRtt.max
+          }),
       sqlQueriesDelta: deltas.sqlQueries,
       sqlFailuresDelta: deltas.sqlFailures,
-      sqlLatencyMsMax: Number(deltas.sqlLatencyMsMax.toFixed(3)),
+      sqlLatencyMsMax: roundMs(deltas.sqlLatencyMsMax),
       controlRenewalsByOutcomeDelta: deltas.controlRenewalsByOutcome,
       controlRenewalsDelta: deltas.controlRenewalLatenciesMs.length,
       controlRenewalSuccessesDelta: deltas.controlRenewalsByOutcome.renewed ?? 0,
@@ -296,16 +421,10 @@ export class RelayObservability implements RelayRuntimeObserver {
         deltas.controlRenewalsByOutcome.control_activity_not_found ?? 0,
       controlActivityRecoveriesDelta: deltas.controlActivityRecoveries,
       controlActivityRecoveryFailuresDelta: deltas.controlActivityRecoveryFailures,
-      controlRenewalLatencyMsP50: Number(
-        percentile(deltas.controlRenewalLatenciesMs, 0.5).toFixed(3)
-      ),
-      controlRenewalLatencyMsP95: Number(
-        percentile(deltas.controlRenewalLatenciesMs, 0.95).toFixed(3)
-      ),
-      controlRenewalLatencyMsMax: Number(
-        Math.max(0, ...deltas.controlRenewalLatenciesMs).toFixed(3)
-      ),
-      httpLatencyMsMax: Number(deltas.httpLatencyMsMax.toFixed(3)),
+      controlRenewalLatencyMsP50: controlRenewal.p50,
+      controlRenewalLatencyMsP95: controlRenewal.p95,
+      controlRenewalLatencyMsMax: controlRenewal.max,
+      httpLatencyMsMax: roundMs(deltas.httpLatencyMsMax),
       heapUsedBytes: memory.heapUsed,
       heapTotalBytes: memory.heapTotal,
       eventLoopDelayMsP99: Number(p99.toFixed(3))
@@ -313,6 +432,22 @@ export class RelayObservability implements RelayRuntimeObserver {
   }
 }
 
+// Flat siblings of the nested region maps, always emitted for every region including zeros.
+// A log-based metric cannot reach `requestedRegionsDelta."asia-east2"` without a quoted field
+// path, and an absent key would drop a series out of the inner join the region-skew alert does.
+// The maps stay authoritative and keep carrying anything outside the catalog, such as `unhinted`.
+function regionCounterFields(
+  prefix: 'requestedRegion' | 'selectedRegion',
+  counts: Record<string, number>
+): Record<string, number> {
+  return Object.fromEntries(
+    Object.entries(RELAY_REGION_METRIC_SEGMENTS).map(([region, segment]) => [
+      `${prefix}${segment}Delta`,
+      counts[region] ?? 0
+    ])
+  )
+}
+
 function increment(counts: Record<string, number>, key: string): void {
   counts[key] = (counts[key] ?? 0) + 1
 }

cloud/apps/relay/src/relay-server.ts

@@ -2,7 +2,9 @@ import { createAdaptorServer } from '@hono/node-server'
 import {
   hasAdmissionCapacity,
   HostDataAuthSchema,
+  parseRelayHostCapabilities,
   RELAY_ADMISSION_BUDGETS,
+  RELAY_HOST_CAPABILITIES_HEADER,
   RELAY_CLOSE_CODE,
   RELAY_DEFAULT_REGION,
   RELAY_PROTOCOL_LIMITS,
@@ -31,6 +33,16 @@ import { createRelayTokenVerifier, readBearer } from './relay-token-verifier.js'
 import { closeRelayWebSocket } from './relay-websocket-close.js'
 import { ProcessQueuedByteBudget } from './splice-forwarder.js'
 
+// A malformed percent-escape in the request target must be a client error, never a URIError
+// thrown out of the `upgrade` listener (which is uncaught and kills the process).
+function decodePathSegment(value: string): string | null {
+  try {
+    return decodeURIComponent(value)
+  } catch {
+    return null
+  }
+}
+
 function rejectUpgrade(socket: NodeJS.WritableStream, status: number, message: string): void {
   socket.write(`HTTP/1.1 ${status} ${message}\r\nConnection: close\r\nContent-Length: 0\r\n\r\n`)
   if ('destroy' in socket && typeof socket.destroy === 'function') socket.destroy()
@@ -78,6 +90,7 @@ export function createRelayServer(
   database: RelayDatabase,
   options: {
     now?: () => number
+    random?: () => number
     connectionLedgerLimits?: { hardCap: number; controlReserve: number }
     cellIncarnation?: string
   } = {}
@@ -113,7 +126,8 @@ export function createRelayServer(
     assignments,
     queuedBytes,
     observability,
-    options.now
+    options.now,
+    options.random
   )
   const app = createRelayApp(config, {
     store,
@@ -278,8 +292,8 @@ export function createRelayServer(
       return
     }
     if (url.pathname.startsWith('/v1/connect/')) {
-      const hostId = decodeURIComponent(url.pathname.slice('/v1/connect/'.length))
-      if (!/^[A-Za-z0-9_-]{16}$/.test(hostId)) {
+      const hostId = decodePathSegment(url.pathname.slice('/v1/connect/'.length))
+      if (hostId === null || !/^[A-Za-z0-9_-]{16}$/.test(hostId)) {
         rejectUpgrade(socket, 429, 'Too Many Requests')
         return
       }
@@ -373,7 +387,7 @@ export function createRelayServer(
         rejectUpgrade(socket, 404, 'Not Found')
         return
       }
-      const connId = decodeURIComponent(url.pathname.slice('/v1/host/data/'.length))
+      const connId = decodePathSegment(url.pathname.slice('/v1/host/data/'.length))
       if (!connId || connId.length > 128) {
         rejectUpgrade(socket, 429, 'Too Many Requests')
         return
@@ -474,7 +488,8 @@ export function createRelayServer(
           sessions.acceptControl(
             webSocket,
             identity,
-            controlUpgrade?.inclusionWatermark
+            controlUpgrade?.inclusionWatermark,
+            parseRelayHostCapabilities(request.headers[RELAY_HOST_CAPABILITIES_HEADER])
           )
         })
       } catch {

cloud/apps/relay/src/relay-sweep-schedule.test.ts

@@ -0,0 +1,55 @@
+import { readFileSync } from 'node:fs'
+import { describe, expect, it, vi } from 'vitest'
+import { startRegionalRehomeWorker } from './regional-rehome-worker.js'
+import { jitteredSweepIntervalMs, SWEEP_JITTER_FRACTION } from './relay-sweep-schedule.js'
+
+describe('sweep schedule jitter', () => {
+  it('spreads instances across a bounded window above the base period', () => {
+    expect(jitteredSweepIntervalMs(30_000, () => 0)).toBe(30_000)
+    expect(jitteredSweepIntervalMs(30_000, () => 0.5)).toBe(33_000)
+    // Math.random() never returns 1, so the open bound is the real ceiling.
+    expect(jitteredSweepIntervalMs(30_000, () => 0.999)).toBeLessThan(36_000)
+  })
+
+  // Why: a shorter period would raise the very lock traffic the offset spreads.
+  it('never schedules a sweep sooner than its base period', () => {
+    for (const random of [0, 0.25, 0.5, 0.75, 0.999]) {
+      expect(jitteredSweepIntervalMs(1_000, () => random)).toBeGreaterThanOrEqual(1_000)
+    }
+    expect(SWEEP_JITTER_FRACTION).toBeGreaterThan(0)
+  })
+
+  it('jitters the regional rehome dispatch tick, which every director runs each second', () => {
+    const timers: number[] = []
+    const setIntervalSpy = vi
+      .spyOn(globalThis, 'setInterval')
+      .mockImplementation(((_handler: unknown, delayMs?: number) => {
+        timers.push(delayMs ?? 0)
+        return { unref: () => undefined, [Symbol.dispose]: () => undefined } as never
+      }) as never)
+
+    try {
+      startRegionalRehomeWorker(
+        {
+          role: 'director',
+          rehomeAudience: 'https://rehome.example.test',
+          rehomeDirectorServiceAccount: 'rehome@example.test'
+        } as never,
+        { claimRegionalRehome: async () => null } as never,
+        { random: () => 0.5, safetySnapshot: () => ({}) as never }
+      )
+    } finally {
+      setIntervalSpy.mockRestore()
+    }
+
+    expect(timers).toEqual([1_100])
+  })
+
+  // Why: index.ts boots a server on import, so its wiring can only be read.
+  it('jitters the director assignment cleanup tick', () => {
+    const source = readFileSync(new URL('./index.ts', import.meta.url), 'utf8')
+    const cleanup = /runAssignmentCleanup\(assignments\)\s*\},\s*([^\n]*?)\)\n/.exec(source)
+
+    expect(cleanup?.[1]).toBe('jitteredSweepIntervalMs(30_000)')
+  })
+})

cloud/apps/relay/src/relay-sweep-schedule.ts

@@ -0,0 +1,13 @@
+// Why: every director instance boots from the same rollout, so its periodic
+// sweeps land on the same wall-clock second across instances and pile onto the
+// one global cell-inventory lock together. A per-process offset spreads the
+// arrivals; the sweeps are idempotent, so a slightly longer period is free.
+export const SWEEP_JITTER_FRACTION = 0.2
+
+export function jitteredSweepIntervalMs(
+  baseMs: number,
+  random: () => number = Math.random
+): number {
+  // Only ever longer: a shorter period would raise the very load being spread.
+  return baseMs + Math.floor(random() * baseMs * SWEEP_JITTER_FRACTION)
+}

cloud/apps/relay/src/relay-upgrade-malformed-uri.blackbox.test.ts

@@ -0,0 +1,122 @@
+import { connect, createServer as createNetServer } from 'node:net'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import type { RelayConfig } from './config.js'
+import type { RelayDatabase } from './database.js'
+import { createRelayServer } from './relay-server.js'
+
+async function unusedPort(): Promise<number> {
+  const server = createNetServer()
+  await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
+  const address = server.address()
+  if (!address || typeof address === 'string') throw new Error('missing test port')
+  await new Promise<void>((resolve) => server.close(() => resolve()))
+  return address.port
+}
+
+function rawUpgrade(port: number, target: string): Promise<{ status: string; closed: boolean }> {
+  return new Promise((resolve, reject) => {
+    const socket = connect(port, '127.0.0.1')
+    let data = ''
+    socket.once('connect', () => {
+      socket.write(
+        `GET ${target} HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: Upgrade\r\n` +
+          'Upgrade: websocket\r\nSec-WebSocket-Version: 13\r\n' +
+          // RFC 6455 §1.3 example nonce; allowlisted in cloud/.gitleaks.toml.
+          'Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n\r\n'
+      )
+    })
+    socket.on('data', (chunk) => {
+      data += chunk.toString()
+    })
+    socket.once('close', () => resolve({ status: data.split('\r\n')[0] ?? '', closed: true }))
+    socket.once('error', reject)
+    setTimeout(() => {
+      socket.destroy()
+      resolve({ status: data.split('\r\n')[0] ?? '', closed: false })
+    }, 1_500).unref()
+  })
+}
+
+describe('relay upgrade with a malformed request target', () => {
+  const cleanup: Array<() => Promise<void> | void> = []
+
+  afterEach(async () => {
+    for (const close of cleanup.splice(0).reverse()) await close()
+    vi.restoreAllMocks()
+  })
+
+  it('rejects an undecodable /v1/connect path without an uncaught exception', async () => {
+    const port = await unusedPort()
+    const relayUrl = `http://127.0.0.1:${port}`
+    const database: RelayDatabase = {
+      query: vi.fn(async () => []),
+      queryLocked: vi.fn(async () => []),
+      transaction: vi.fn(async (operation) => await operation(database)),
+      close: vi.fn(async () => undefined)
+    }
+    const config = {
+      port,
+      publicUrl: relayUrl,
+      cellUrl: relayUrl,
+      authIssuer: 'https://auth.example.com',
+      authAudience: 'orca-relay',
+      jwksUrl: 'https://auth.example.com/jwks',
+      assignmentSigningKey: new Uint8Array(32),
+      role: 'cell',
+      cellId: 'production-gce-c3',
+      cells: [{ id: 'production-gce-c3', url: relayUrl, capacityRequests: 4_000 }],
+      adminAudience: `${relayUrl}/admin`,
+      deployServiceAccount: 'deploy@example.com',
+      runtimeServiceAccount: 'runtime@example.com',
+      connectionHardCap: 600,
+      connectionUnobservedBound: 60,
+      adminJwksUrl: 'https://auth.example.com/admin-jwks',
+      databasePoolMax: 10,
+      publicAssignmentsEnabled: true,
+      publicAssignmentConcurrency: 2,
+      publicAssignmentQueueMax: 128,
+      publicAssignmentWaitMs: 4_000,
+      publicResolveConcurrency: 1,
+      publicResolveWaitMs: 5_000,
+      publicAssignmentRetryAfterSeconds: 5,
+      dataDir: './test-data'
+    } satisfies RelayConfig
+    const relay = createRelayServer(config, database, {
+      connectionLedgerLimits: { hardCap: 5, controlReserve: 1 }
+    })
+    relay.server.listen(port, '127.0.0.1')
+    await new Promise<void>((resolve) => relay.server.once('listening', resolve))
+    cleanup.push(() => new Promise<void>((resolve) => relay.server.close(() => resolve())))
+    vi.spyOn(console, 'log').mockImplementation(() => undefined)
+    vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+
+    // Vitest installs its own uncaughtException listener; capture ours first so the test reports
+    // the exception as a verdict instead of dying with it.
+    const uncaught: unknown[] = []
+    const onUncaught = (error: unknown): void => {
+      uncaught.push(error)
+    }
+    process.prependListener('uncaughtException', onUncaught)
+    cleanup.push(() => {
+      process.off('uncaughtException', onUncaught)
+    })
+
+    const results = []
+    for (const target of [
+      '/v1/connect/%',
+      '/v1/connect/%E0%A4%A',
+      '/v1/connect/%C0%AF',
+      '/v1/host/data/%'
+    ]) {
+      results.push(await rawUpgrade(port, target))
+    }
+    // A malformed percent-escape must be a client error, never a process-level throw.
+    expect(uncaught).toEqual([])
+    for (const result of results) {
+      expect(result.status).toMatch(/^HTTP\/1\.1 4\d\d/)
+    }
+    // The server must still serve a well-formed upgrade afterwards.
+    const after = await rawUpgrade(port, '/v1/connect/abcdefghijklmnop')
+    expect(after.status).toMatch(/^HTTP\/1\.1 101/)
+  })
+})

cloud/apps/relay/src/relay.blackbox.test.ts

@@ -9,7 +9,9 @@ import { fileURLToPath } from 'node:url'
 import { exportJWK, generateKeyPair, jwtVerify, SignJWT } from 'jose'
 import {
   buildHostProofMacInput,
-  HOST_CHALLENGE_PLAINTEXT_DOMAIN
+  HOST_CHALLENGE_PLAINTEXT_DOMAIN,
+  RELAY_HOST_CAPABILITIES_HEADER,
+  RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS
 } from '@orca-cloud/relay-contract'
 import nacl from 'tweetnacl'
 import { afterAll, beforeAll, describe, expect, it } from 'vitest'
@@ -282,11 +284,17 @@ async function openHostControl(input?: {
   previousGeneration?: number
   keyPair?: nacl.BoxKeyPair
   assignmentEpoch?: number
+  capabilities?: string
 }): Promise<{ socket: WebSocket; ack: Record<string, unknown>; keyPair: nacl.BoxKeyPair }> {
   const keyPair = input?.keyPair ?? nacl.box.keyPair()
   const hostId = createHash('sha256').update(keyPair.publicKey).digest('base64url').slice(0, 16)
   const socket = new WebSocket(`${relayUrl.replace('http:', 'ws:')}/v1/host/control`, {
-    headers: { authorization: `Bearer ${await relayToken('orca-relay', hostId)}` },
+    headers: {
+      authorization: `Bearer ${await relayToken('orca-relay', hostId)}`,
+      ...(input?.capabilities
+        ? { [RELAY_HOST_CAPABILITIES_HEADER]: input.capabilities }
+        : {})
+    },
     perMessageDeflate: false
   })
   await new Promise<void>((resolveOpen, reject) => {
@@ -653,6 +661,69 @@ describe('served relay URL', () => {
     expect(result.reason).not.toContain('http')
   })
 
+  it('restates a pending connection to the rebound control, detailed only when advertised', async () => {
+    // The one link the unit tests cannot reach: an upgrade that really carries
+    // x-orca-host-capabilities must reach acceptControl and change the ack. A
+    // typo in the header name here passes every other test in the suite.
+    const host = await openHostControl()
+    const hostId = createHash('sha256')
+      .update(host.keyPair.publicKey)
+      .digest('base64url')
+      .slice(0, 16)
+    const inviteResponse = nextMessage(host.socket)
+    host.socket.send(
+      JSON.stringify({
+        type: 'invite-create',
+        reqId: 'capability-invite',
+        relayDeviceId: 'capability-device'
+      })
+    )
+    const invite = await inviteResponse
+    const phone = new WebSocket(`${relayUrl.replace('http:', 'ws:')}/v1/connect/${hostId}`, {
+      headers: forwardedHeaders()
+    })
+    await new Promise<void>((resolveOpen, reject) => {
+      phone.once('open', resolveOpen)
+      phone.once('error', reject)
+    })
+    const connectionPromise = nextMessage(host.socket)
+    phone.send(
+      JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential: invite.inviteToken })
+    )
+    // Never attached: the connection stays pending, which is what the ack restates.
+    const connection = await connectionPromise
+    expect(connection.type).toBe('conn-open')
+
+    const capable = await openHostControl({
+      keyPair: host.keyPair,
+      controlResumeSecret: String(host.ack.controlResumeSecret),
+      previousGeneration: 1,
+      capabilities: RELAY_HOST_CAPABILITY_PENDING_CONN_DETAILS
+    })
+    expect(capable.ack.pendingConns).toEqual([
+      {
+        connId: connection.connId,
+        connTicket: connection.connTicket,
+        kind: 'invite',
+        relayDeviceId: 'capability-device'
+      }
+    ])
+
+    const legacy = await openHostControl({
+      keyPair: host.keyPair,
+      controlResumeSecret: String(capable.ack.controlResumeSecret),
+      previousGeneration: 1
+    })
+    // A shipped host parses these entries strictly, so an unannounced key would
+    // fail the whole ack and kill a control that was working.
+    expect(legacy.ack.pendingConns).toEqual([
+      { connId: connection.connId, connTicket: connection.connTicket }
+    ])
+
+    phone.close()
+    legacy.socket.close()
+  })
+
   it('keeps a pending attach usable after a bad ticket and rejects ticket replay', async () => {
     const host = await openHostControl()
     const hostId = createHash('sha256')

cloud/dev/fixtures/terraform-root-partition/families.json

@@ -133,10 +133,18 @@
     "google_logging_metric.relay_snapshot",
     "google_monitoring_alert_policy.relay_assignment_5xx",
     "google_monitoring_alert_policy.relay_assignment_edge_429",
+    "google_monitoring_alert_policy.relay_cell_control_rtt",
+    "google_monitoring_alert_policy.relay_cell_process_exit",
+    "google_monitoring_alert_policy.relay_cloud_nat_port_drops",
     "google_monitoring_alert_policy.relay_cloud_sql_backends",
+    "google_monitoring_alert_policy.relay_cloud_sql_checkpoint_loop",
+    "google_monitoring_alert_policy.relay_cloud_sql_disk",
     "google_monitoring_alert_policy.relay_custom",
+    "google_monitoring_alert_policy.relay_far_cell_accept_latency",
     "google_monitoring_alert_policy.relay_gce_connection_headroom",
     "google_monitoring_alert_policy.relay_postgres_retry_exhausted",
+    "google_monitoring_alert_policy.relay_region_hint_skew",
+    "google_monitoring_dashboard.relay_incident",
     "google_project_iam_custom_role.github_production_relay_capacity_mutation",
     "google_project_iam_custom_role.github_relay_asia_topology_mutation",
     "google_project_iam_custom_role.github_relay_asia_topology_read",

cloud/dev/scripts/cloud-sql-rollout-lock-census.mjs

@@ -298,6 +298,7 @@ export const LEASED_WORKFLOWS = named([
 ])
 
 export const NOT_A_CLOUD_SQL_CANDIDATE = named([
+  ['push-deploy.yml', 'Push uses dedicated SQL and its own production-push-rollout group and durable push-rollout lease.'],
   [
     'monitor-relay-production.yml',
     'Read-only. Its identity holds monitoring, logging, Cloud SQL and compute viewer roles only, and it runs `gcloud sql instances describe`, never a mutation. It consumes no connection budget, so the durable lease would only let monitoring block a rollout and a rollout block monitoring.'

cloud/dev/scripts/operate-relay-regional-rehome.mjs

@@ -1,4 +1,5 @@
 import { pathToFileURL } from 'node:url'
+import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs'
 import { inspectAdmissionSelector } from './relay-admission-selector.mjs'
 
 const DIRECTOR_ORIGIN = 'https://relay.onorca.dev'
@@ -44,6 +45,7 @@ export function parseRegionalRehomeArguments(argv, environment = process.env) {
     'not-before',
     'rate-per-minute',
     'preference-max-age-ms',
+    'host-cooldown-ms',
     'drain-grace-ms',
     'confirmation'
   ]
@@ -116,6 +118,11 @@ export function parseRegionalRehomeArguments(argv, environment = process.env) {
             '--preference-max-age-ms',
             { minimum: 60_000, maximum: 30 * 24 * 60 * 60_000 }
           ),
+          hostCooldownMs: integer(
+            values['host-cooldown-ms'],
+            '--host-cooldown-ms',
+            { minimum: 60_000, maximum: 30 * 24 * 60 * 60_000 }
+          ),
           drainGraceMs: integer(values['drain-grace-ms'], '--drain-grace-ms', {
             minimum: 60_000,
             maximum: 60 * 60_000
@@ -146,6 +153,11 @@ function assertControl(control, expected) {
     !Number.isSafeInteger(control.notBefore) ||
     !Number.isSafeInteger(control.ratePerMinute) ||
     !Number.isSafeInteger(control.preferenceMaxAgeMs) ||
+    // A director predating the per-host cooldown does not report it. Reading
+    // the control and both emergency brakes must keep working against that
+    // image; only enable requires the field.
+    (control.hostCooldownMs !== undefined &&
+      !Number.isSafeInteger(control.hostCooldownMs)) ||
     !Number.isSafeInteger(control.drainGraceMs)
   ) throw new Error('director returned an invalid regional rehome control')
   if (expected.enabled !== undefined && control.enabled !== expected.enabled) {
@@ -154,6 +166,12 @@ function assertControl(control, expected) {
   return control
 }
 
+// Echo the cooldown only when the director already reports it: a legacy
+// director rejects the unknown key outright and would refuse every brake.
+function cooldownField(before, value) {
+  return before.hostCooldownMs === undefined ? {} : { hostCooldownMs: value }
+}
+
 async function verifiedDisabledControl(post, generation) {
   return assertControl((await post('/v1/admin/regional-rehome-control', {
     v: 1,
@@ -170,6 +188,7 @@ async function applyDisabledControl(post, before) {
     notBefore: before.notBefore,
     ratePerMinute: before.ratePerMinute,
     preferenceMaxAgeMs: before.preferenceMaxAgeMs,
+    ...cooldownField(before, before.hostCooldownMs),
     drainGraceMs: before.drainGraceMs,
     confirmation: 'DISABLE_REGIONAL_REHOMING'
   })).control, { generation: before.generation + 1, enabled: false })
@@ -229,15 +248,20 @@ export async function recoverRegionalRehomeEnable(config, post) {
 export async function operateRegionalRehome(config, dependencies = {}) {
   const fetchImpl = dependencies.fetch ?? fetch
   const post = dependencies.post ?? (async (path, body) => await responseJson(
-    await fetchImpl(`${config.directorOrigin}${path}`, {
-      method: 'POST',
-      headers: {
-        authorization: `Bearer ${config.token}`,
-        'content-type': 'application/json'
+    // Generation-guarded writes make a retry a no-op or an explicit mismatch, never a double apply.
+    await fetchAdminOnceMore(
+      fetchImpl,
+      `${config.directorOrigin}${path}`,
+      {
+        method: 'POST',
+        headers: {
+          authorization: `Bearer ${config.token}`,
+          'content-type': 'application/json'
+        },
+        body: JSON.stringify(body)
       },
-      body: JSON.stringify(body),
-      signal: AbortSignal.timeout(30_000)
-    }),
+      { wait: dependencies.wait }
+    ),
     path
   ))
   if (config.mode === 'recover-enable') {
@@ -264,6 +288,11 @@ export async function operateRegionalRehome(config, dependencies = {}) {
     throw new Error('regional rehome is already paused')
   }
   const enabled = config.mode === 'enable'
+  if (enabled && before.hostCooldownMs === undefined) {
+    throw new Error(
+      'director does not report a per-host rehome cooldown; deploy a director that supports it before enabling'
+    )
+  }
   const applied = await post('/v1/admin/regional-rehome-control', {
     v: 1,
     action: 'apply',
@@ -272,6 +301,7 @@ export async function operateRegionalRehome(config, dependencies = {}) {
     notBefore: config.notBefore,
     ratePerMinute: config.ratePerMinute,
     preferenceMaxAgeMs: config.preferenceMaxAgeMs,
+    ...cooldownField(before, config.hostCooldownMs),
     drainGraceMs: config.drainGraceMs,
     confirmation: enabled
       ? 'ENABLE_REGIONAL_REHOMING'

cloud/dev/scripts/operate-relay-regional-rehome.test.mjs

@@ -26,6 +26,7 @@ function argumentsFor(mode, confirmation) {
       '--not-before', '2000000000000',
       '--rate-per-minute', '10',
       '--preference-max-age-ms', '86400000',
+      '--host-cooldown-ms', '604800000',
       '--drain-grace-ms', '60000',
       '--confirmation', confirmation
     ])
@@ -40,10 +41,29 @@ function control(generation, enabled) {
     notBefore: 2_000_000_000_000,
     ratePerMinute: 10,
     preferenceMaxAgeMs: 86_400_000,
+    hostCooldownMs: 604_800_000,
     drainGraceMs: 60_000
   }
 }
 
+// The control a director predating the per-host cooldown reports.
+function legacyControl(generation, enabled) {
+  const { hostCooldownMs: _absent, ...rest } = control(generation, enabled)
+  return rest
+}
+
+function legacyDirector(controls) {
+  const requests = []
+  const post = async (path, body) => {
+    requests.push({ path, body })
+    if (path === '/v1/admin/admission-selector/status') {
+      return { selector: { generation: 11, membership } }
+    }
+    return { v: 1, control: controls.shift() }
+  }
+  return { requests, post }
+}
+
 test('parses exact selector and typed control confirmation', () => {
   const parsed = parseRegionalRehomeArguments(
     argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING'),
@@ -52,6 +72,17 @@ test('parses exact selector and typed control confirmation', () => {
   assert.equal(parsed.expectedSelectorGeneration, 11)
   assert.equal(parsed.expectedControlGeneration, 4)
   assert.equal(parsed.ratePerMinute, 10)
+  assert.equal(parsed.hostCooldownMs, 604_800_000)
+  assert.throws(
+    () => parseRegionalRehomeArguments(
+      argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING').filter(
+        (value, index, all) =>
+          value !== '--host-cooldown-ms' && all[index - 1] !== '--host-cooldown-ms'
+      ),
+      { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
+    ),
+    /complete durable control shape/
+  )
   assert.throws(
     () => parseRegionalRehomeArguments(
       argumentsFor('pause', 'DISABLE_REGIONAL_REHOMING'),
@@ -79,6 +110,7 @@ test('binds enable to exact selector and durable control generations', async ()
     notBefore: 0,
     ratePerMinute: 10,
     preferenceMaxAgeMs: 86_400_000,
+    hostCooldownMs: 604_800_000,
     drainGraceMs: 60_000,
     ...control
   }))
@@ -96,6 +128,7 @@ test('binds enable to exact selector and durable control generations', async ()
     }
   })
   assert.equal(result.control.generation, 5)
+  assert.equal(result.control.hostCooldownMs, 604_800_000)
   assert.deepEqual(requests[2].body, {
     v: 1,
     action: 'apply',
@@ -104,11 +137,82 @@ test('binds enable to exact selector and durable control generations', async ()
     notBefore: 2_000_000_000_000,
     ratePerMinute: 10,
     preferenceMaxAgeMs: 86_400_000,
+    hostCooldownMs: 604_800_000,
     drainGraceMs: 60_000,
     confirmation: 'ENABLE_REGIONAL_REHOMING'
   })
 })
 
+test('inspects a director that predates the per-host cooldown', async () => {
+  const director = legacyDirector([legacyControl(4, true)])
+  const config = parseRegionalRehomeArguments(
+    argumentsFor('inspect'),
+    { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
+  )
+
+  const result = await operateRegionalRehome(config, { post: director.post })
+
+  assert.equal(result.control.generation, 4)
+  assert.equal(result.control.hostCooldownMs, undefined)
+})
+
+for (const [mode, confirmation, enabledBefore] of [
+  ['pause', 'PAUSE_REGIONAL_REHOMING', true],
+  ['disable', 'DISABLE_REGIONAL_REHOMING', false]
+]) {
+  test(`${mode} still brakes a director that predates the cooldown`, async () => {
+    const director = legacyDirector([
+      legacyControl(4, enabledBefore),
+      legacyControl(5, false),
+      legacyControl(5, false)
+    ])
+    const config = parseRegionalRehomeArguments(
+      argumentsFor(mode, confirmation),
+      { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
+    )
+
+    const result = await operateRegionalRehome(config, { post: director.post })
+
+    assert.equal(result.control.generation, 5)
+    // The unknown key would be refused by that director's strict schema.
+    assert.equal('hostCooldownMs' in director.requests[2].body, false)
+    assert.equal(director.requests[2].body.confirmation, 'DISABLE_REGIONAL_REHOMING')
+  })
+}
+
+test('failed-enable recovery brakes a director that predates the cooldown', async () => {
+  const requests = []
+  let current = legacyControl(7, true)
+  const result = await recoverRegionalRehomeEnable({
+    mode: 'recover-enable',
+    expectedControlGeneration: 4
+  }, async (_path, body) => {
+    requests.push(body)
+    if (body.action === 'inspect') return { control: current }
+    current = legacyControl(8, false)
+    return { control: current }
+  })
+
+  assert.equal(result.control.generation, 8)
+  assert.equal('hostCooldownMs' in requests[1], false)
+})
+
+test('refuses to enable a director that does not report the cooldown', async () => {
+  const director = legacyDirector([legacyControl(4, false)])
+  const config = parseRegionalRehomeArguments(
+    argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING'),
+    { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
+  )
+
+  await assert.rejects(
+    operateRegionalRehome(config, { post: director.post }),
+    /per-host rehome cooldown/
+  )
+  // Read-only: selector status and the control inspect, and nothing else.
+  assert.equal(director.requests.length, 2)
+  assert.equal(director.requests.every(({ body }) => body.action !== 'apply'), true)
+})
+
 test('fails closed on selector drift before reading or mutating control', async () => {
   let calls = 0
   const config = parseRegionalRehomeArguments(
@@ -150,6 +254,7 @@ test('failed-enable recovery CAS-disables an advanced enabled generation', async
     notBefore: 2_000_000_000_000,
     ratePerMinute: 10,
     preferenceMaxAgeMs: 86_400_000,
+    hostCooldownMs: 604_800_000,
     drainGraceMs: 60_000,
     confirmation: 'DISABLE_REGIONAL_REHOMING'
   })
@@ -263,3 +368,55 @@ test('main executes recovery mode and emits verified disabled control', async ()
     control: control(6, false)
   })
 })
+
+test('retries a transient 503 on the director control endpoint', async () => {
+  const config = parseRegionalRehomeArguments(
+    argumentsFor('inspect'),
+    { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
+  )
+  const paths = []
+  let selectorCalls = 0
+  const result = await operateRegionalRehome(config, {
+    wait: async () => {},
+    fetch: async (url) => {
+      const path = new URL(url).pathname
+      paths.push(path)
+      if (path === '/v1/admin/admission-selector/status') {
+        selectorCalls += 1
+        // The first read of each admin path 503s the way a warming instance does.
+        if (selectorCalls === 1) return new Response('warming up', { status: 503 })
+        return Response.json({ selector: { generation: 11, membership } })
+      }
+      if (paths.filter((value) => value === path).length === 1) {
+        return new Response('warming up', { status: 503 })
+      }
+      return Response.json({ v: 1, control: control(4, false) })
+    }
+  })
+  assert.equal(result.control.generation, 4)
+  assert.deepEqual(paths, [
+    '/v1/admin/admission-selector/status',
+    '/v1/admin/admission-selector/status',
+    '/v1/admin/regional-rehome-control',
+    '/v1/admin/regional-rehome-control'
+  ])
+})
+
+test('fails when both attempts at the director control endpoint return 503', async () => {
+  const config = parseRegionalRehomeArguments(
+    argumentsFor('inspect'),
+    { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
+  )
+  let calls = 0
+  await assert.rejects(
+    operateRegionalRehome(config, {
+      wait: async () => {},
+      fetch: async () => {
+        calls += 1
+        return new Response('warming up', { status: 503 })
+      }
+    }),
+    /returned 503/
+  )
+  assert.equal(calls, 2)
+})

cloud/dev/scripts/prepare-relay-production-capacity-canary.mjs

@@ -1,10 +1,12 @@
 import { pathToFileURL } from 'node:url'
+import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs'
 import {
   applyExactAdmissionSelector,
   inspectAdmissionSelector,
   membershipWithStates,
   selectorCellState
 } from './relay-admission-selector.mjs'
+import { SAME_CAP_CELLS } from './relay-production-same-cap-wave.mjs'
 
 const DIRECTOR_ORIGIN = 'https://relay.onorca.dev'
 export const PRODUCTION_CAPACITY_CELL_IDS = [
@@ -30,6 +32,9 @@ function cellOrigin(cellId) {
   return `https://${cellId.slice('production-gce-'.length)}.relay.onorca.dev`
 }
 
+// The same-cap roll covers the Asia cells the US-only capacity rollout never touches.
+const APPROVED_CELL_LISTS = { 'same-cap': SAME_CAP_CELLS }
+
 export function parseProductionCapacityCellArguments(argv) {
   const values = {}
   for (let index = 0; index < argv.length; index += 2) {
@@ -41,8 +46,15 @@ export function parseProductionCapacityCellArguments(argv) {
   if (!['isolate', 'drain', 'activate'].includes(values.mode)) {
     throw new Error('--mode must be isolate, drain, or activate')
   }
+  const approvedList = values['approved-cells']
+  if (approvedList !== undefined && !APPROVED_CELL_LISTS[approvedList]) {
+    throw new Error('--approved-cells is not a known allowlist')
+  }
+  const approvedCellIds = approvedList === undefined
+    ? PRODUCTION_CAPACITY_CELL_IDS
+    : APPROVED_CELL_LISTS[approvedList]
   const cellId = values['cell-id']
-  if (!PRODUCTION_CAPACITY_CELL_IDS.includes(cellId)) {
+  if (!approvedCellIds.includes(cellId)) {
     throw new Error('production capacity target is not approved')
   }
   const expectedCellOrigin = cellOrigin(cellId)
@@ -72,12 +84,16 @@ export async function prepareProductionCapacityCell(config, overrides = {}) {
   if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable')
   const postAt = async (origin, path, body) =>
     await responseJson(
-      await fetchImpl(`${origin}${path}`, {
-        method: 'POST',
-        headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
-        body: JSON.stringify(body),
-        signal: AbortSignal.timeout(30_000)
-      }),
+      await fetchAdminOnceMore(
+        fetchImpl,
+        `${origin}${path}`,
+        {
+          method: 'POST',
+          headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
+          body: JSON.stringify(body)
+        },
+        { wait: overrides.wait }
+      ),
       path
     )
   const post = async (path, body) => await postAt(config.directorOrigin, path, body)

cloud/dev/scripts/prepare-relay-production-capacity-canary.test.mjs

@@ -104,6 +104,47 @@ describe('production Relay capacity cell admission', () => {
       '--cell-id', 'production-gce-c7',
       '--mode', 'isolate'
     ]), /origin is not exact/)
+    assert.throws(() => parseProductionCapacityCellArguments([
+      '--director-origin', 'https://relay.onorca.dev',
+      '--cell-origin', 'https://c27.relay.onorca.dev',
+      '--cell-id', 'production-gce-c27',
+      '--mode', 'isolate'
+    ]), /not approved/)
+  })
+
+  it('admits the same-cap Asia cells only under the same-cap allowlist', () => {
+    for (const cellId of ['production-gce-c27', 'production-gce-c28', 'production-gce-c29']) {
+      const hostname = cellId.slice('production-gce-'.length)
+      assert.deepEqual(parseProductionCapacityCellArguments([
+        '--director-origin', 'https://relay.onorca.dev',
+        '--cell-origin', `https://${hostname}.relay.onorca.dev`,
+        '--cell-id', cellId,
+        '--approved-cells', 'same-cap',
+        '--mode', 'isolate'
+      ]), {
+        directorOrigin: 'https://relay.onorca.dev',
+        cellOrigin: `https://${hostname}.relay.onorca.dev`,
+        cellId,
+        mode: 'isolate'
+      })
+    }
+    for (const cellId of ['production-gce-c17', 'production-gce-c18', 'production-gce-c30']) {
+      const hostname = cellId.slice('production-gce-'.length)
+      assert.throws(() => parseProductionCapacityCellArguments([
+        '--director-origin', 'https://relay.onorca.dev',
+        '--cell-origin', `https://${hostname}.relay.onorca.dev`,
+        '--cell-id', cellId,
+        '--approved-cells', 'same-cap',
+        '--mode', 'isolate'
+      ]), /not approved/)
+    }
+    assert.throws(() => parseProductionCapacityCellArguments([
+      '--director-origin', 'https://relay.onorca.dev',
+      '--cell-origin', 'https://c27.relay.onorca.dev',
+      '--cell-id', 'production-gce-c27',
+      '--approved-cells', 'every-cell',
+      '--mode', 'isolate'
+    ]), /not a known allowlist/)
   })
 
   it('isolates only the selected cell without depending on its runtime', async () => {
@@ -170,4 +211,42 @@ describe('production Relay capacity cell admission', () => {
       /irreversible/
     )
   })
+
+  it('retries a transient 503 on the cell drain endpoint', async () => {
+    let calls = 0
+    const result = await prepareProductionCapacityCell(
+      { ...config, mode: 'drain' },
+      {
+        token: 'token',
+        wait: async () => {},
+        fetch: async (url) => {
+          assert.equal(new URL(url).pathname, '/v1/admin/drain')
+          calls += 1
+          if (calls === 1) return response({ error: 'warming up' }, 503)
+          return response({ v: 1, draining: true })
+        }
+      }
+    )
+    assert.equal(calls, 2)
+    assert.deepEqual(result, { changed: false, drained: true })
+  })
+
+  it('fails when both drain attempts return a transient 503', async () => {
+    let calls = 0
+    await assert.rejects(
+      prepareProductionCapacityCell(
+        { ...config, mode: 'drain' },
+        {
+          token: 'token',
+          wait: async () => {},
+          fetch: async () => {
+            calls += 1
+            return response({ error: 'warming up' }, 503)
+          }
+        }
+      ),
+      /returned 503/
+    )
+    assert.equal(calls, 2)
+  })
 })

cloud/dev/scripts/probe-relay-rehome-trust.mjs

@@ -1,6 +1,9 @@
 import { pathToFileURL } from 'node:url'
+import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs'
 
-const PRODUCTION_CELL = /^production-gce-c(?:7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26)$/
+// Every general cell that carries the rehome identity: the sixteen US cells and the
+// three asia-east2 cells that drain mis-homed hosts back the other way.
+const PRODUCTION_CELL = /^production-gce-c(?:7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26|27|28|29)$/
 const DIRECTOR_ORIGIN = 'https://relay.onorca.dev'
 
 export function parseRehomeTrustProbeArguments(argv, environment = process.env) {
@@ -35,7 +38,8 @@ export function parseRehomeTrustProbeArguments(argv, environment = process.env)
 
 export async function probeRehomeTrust(config, dependencies = {}) {
   const fetchImpl = dependencies.fetch ?? fetch
-  const response = await fetchImpl(
+  const response = await fetchAdminOnceMore(
+    fetchImpl,
     `${config.directorOrigin}/v1/admin/regional-rehome-trust-probe`,
     {
       method: 'POST',
@@ -47,9 +51,9 @@ export async function probeRehomeTrust(config, dependencies = {}) {
         v: 1,
         sourceCellId: config.cellId,
         sourceCellIncarnation: config.cellIncarnation
-      }),
-      signal: AbortSignal.timeout(30_000)
-    }
+      })
+    },
+    { wait: dependencies.wait }
   )
   const body = await response.json().catch(() => ({}))
   if (!response.ok) {

cloud/dev/scripts/probe-relay-rehome-trust.test.mjs

@@ -68,3 +68,66 @@ test('rejects partial or mismatched proof', async () => {
     /incomplete/
   )
 })
+
+const provenProbe = {
+  v: 1,
+  dedicatedIdentity: {
+    firstOutcome: 'host-not-connected',
+    secondOutcome: 'host-not-connected',
+    accepted: true,
+    idempotent: true
+  },
+  sharedRuntimeIdentityRejected: true,
+  proven: true
+}
+
+test('retries a transient 503 on the trust probe and proves on the second answer', async () => {
+  const config = parseRehomeTrustProbeArguments(argv, environment)
+  let calls = 0
+  const result = await probeRehomeTrust(config, {
+    wait: async () => {},
+    fetch: async () => {
+      calls += 1
+      if (calls === 1) return new Response('warming up', { status: 503 })
+      return Response.json(provenProbe)
+    }
+  })
+  assert.equal(calls, 2)
+  assert.equal(result.proven, true)
+})
+
+test('fails when both trust-probe attempts return a transient 503', async () => {
+  const config = parseRehomeTrustProbeArguments(argv, environment)
+  let calls = 0
+  await assert.rejects(
+    probeRehomeTrust(config, {
+      wait: async () => {},
+      fetch: async () => {
+        calls += 1
+        return new Response('warming up', { status: 503 })
+      }
+    }),
+    /returned 503/
+  )
+  assert.equal(calls, 2)
+})
+
+test('approves the asia-east2 rehome sources and still rejects unlisted cells', () => {
+  for (const cellId of ['production-gce-c27', 'production-gce-c28', 'production-gce-c29']) {
+    const parsed = parseRehomeTrustProbeArguments(
+      argv.map((value) => (value === 'production-gce-c7' ? cellId : value)),
+      environment
+    )
+    assert.equal(parsed.cellId, cellId)
+  }
+  for (const cellId of ['production-gce-c1', 'production-gce-c17', 'production-gce-c30']) {
+    assert.throws(
+      () =>
+        parseRehomeTrustProbeArguments(
+          argv.map((value) => (value === 'production-gce-c7' ? cellId : value)),
+          environment
+        ),
+      /--cell-id is not approved/
+    )
+  }
+})

cloud/dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs

@@ -0,0 +1,43 @@
+import assert from 'node:assert/strict'
+import { readFileSync } from 'node:fs'
+import { test } from 'node:test'
+import { fileURLToPath } from 'node:url'
+import { relayWorkflowUrl } from './relay-repository.mjs'
+
+const WORKFLOWS = [
+  'deploy-relay-production-same-cap-job.yml',
+  'operate-relay-production-rehome-job.yml'
+]
+
+function workflow(name) {
+  return readFileSync(fileURLToPath(relayWorkflowUrl(name)), 'utf8')
+}
+
+// A single transient 5xx from a warming instance behind the global load balancer
+// must not fail a canary, so no admin endpoint may be read by a bare curl.
+test('no admin endpoint is reached by a curl without a bounded retry', () => {
+  for (const name of WORKFLOWS) {
+    for (const invocation of workflow(name).split(/\bcurl\b/).slice(1)) {
+      const flags = invocation.split('\n          }')[0]
+      assert.match(flags, /--retry 3 --retry-delay 2 --retry-connrefused/, name)
+      assert.match(flags, /--max-time 30/, name)
+      // --retry-all-errors would also retry 401, 403, and 409, which are final.
+      assert.doesNotMatch(flags, /--retry-all-errors/, name)
+    }
+  }
+})
+
+test('every retried admin request captures only the final attempt body', () => {
+  const job = workflow('deploy-relay-production-same-cap-job.yml')
+  // --fail-with-body writes every failed attempt to stdout, so a retried
+  // request must land in a file curl truncates per attempt.
+  assert.match(job, /--output "\$\{out\}"/)
+  assert.equal(job.split('admin_post() {').length - 1, 2)
+  for (const call of [
+    /CURRENT_RUNTIME="\$\(admin_post current-runtime/,
+    /CURRENT_DIRECTOR_STATUS="\$\(admin_post current-cell-status/,
+    /TARGET_RUNTIME="\$\(admin_post target-runtime/,
+    /TARGET_DIRECTOR_STATUS="\$\(admin_post target-cell-status/
+  ]) assert.match(job, call)
+  assert.doesNotMatch(job, /\$\(curl /)
+})

cloud/dev/scripts/relay-admin-transient-retry.mjs

@@ -0,0 +1,29 @@
+// A single transient 5xx (load-balancer warm-up behind a fresh instance) must not fail a
+// deploy step. 4xx is never retried: auth and generation-mismatch answers are final.
+const TRANSIENT_STATUSES = [500, 502, 503, 504]
+const RETRY_DELAY_MS = 2_000
+const REQUEST_TIMEOUT_MS = 30_000
+
+export function isTransientAdminStatus(status) {
+  return TRANSIENT_STATUSES.includes(status)
+}
+
+// Each attempt gets its own timeout budget, so a reused signal cannot abort the retry.
+export async function fetchAdminOnceMore(fetchImpl, url, init, overrides = {}) {
+  const wait = overrides.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)))
+  const timeoutMs = overrides.timeoutMs ?? REQUEST_TIMEOUT_MS
+  const retryDelayMs = overrides.retryDelayMs ?? RETRY_DELAY_MS
+  const attempt = async () =>
+    await fetchImpl(url, { ...init, signal: AbortSignal.timeout(timeoutMs) })
+  let response
+  try {
+    response = await attempt()
+  } catch {
+    await wait(retryDelayMs)
+    return await attempt()
+  }
+  if (!isTransientAdminStatus(response.status)) return response
+  await response.arrayBuffer?.().catch(() => undefined)
+  await wait(retryDelayMs)
+  return await attempt()
+}

cloud/dev/scripts/relay-admin-transient-retry.test.mjs

@@ -0,0 +1,130 @@
+import assert from 'node:assert/strict'
+import { test } from 'node:test'
+import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs'
+
+const url = 'https://relay.onorca.dev/v1/admin/cell-status'
+const init = { method: 'POST', body: '{"v":1}' }
+
+function recordingWait(waits) {
+  return async (ms) => { waits.push(ms) }
+}
+
+test('a single transient 5xx is retried and the second answer is returned', async () => {
+  const waits = []
+  const statuses = [503, 200]
+  let calls = 0
+  const response = await fetchAdminOnceMore(
+    async () => {
+      calls += 1
+      const status = statuses.shift()
+      return new Response(JSON.stringify({ ok: status === 200 }), { status })
+    },
+    url,
+    init,
+    { wait: recordingWait(waits) }
+  )
+  assert.equal(calls, 2)
+  assert.equal(response.status, 200)
+  assert.deepEqual(waits, [2_000])
+  assert.deepEqual(await response.json(), { ok: true })
+})
+
+test('a connection failure is retried and the second answer is returned', async () => {
+  const waits = []
+  let calls = 0
+  const response = await fetchAdminOnceMore(
+    async () => {
+      calls += 1
+      if (calls === 1) throw new TypeError('fetch failed')
+      return Response.json({ ok: true })
+    },
+    url,
+    init,
+    { wait: recordingWait(waits) }
+  )
+  assert.equal(calls, 2)
+  assert.equal(response.status, 200)
+  assert.deepEqual(waits, [2_000])
+})
+
+test('two transient failures surface the second answer without a third attempt', async () => {
+  let calls = 0
+  const response = await fetchAdminOnceMore(
+    async () => {
+      calls += 1
+      return new Response('down', { status: 503 })
+    },
+    url,
+    init,
+    { wait: async () => {} }
+  )
+  assert.equal(calls, 2)
+  assert.equal(response.status, 503)
+})
+
+test('two connection failures rethrow the second error', async () => {
+  let calls = 0
+  await assert.rejects(
+    fetchAdminOnceMore(
+      async () => {
+        calls += 1
+        throw new TypeError(`fetch failed ${calls}`)
+      },
+      url,
+      init,
+      { wait: async () => {} }
+    ),
+    /fetch failed 2/
+  )
+  assert.equal(calls, 2)
+})
+
+test('4xx is final: auth and generation-mismatch answers are never retried', async () => {
+  for (const status of [400, 401, 403, 404, 409, 429]) {
+    let calls = 0
+    const response = await fetchAdminOnceMore(
+      async () => {
+        calls += 1
+        return new Response('no', { status })
+      },
+      url,
+      init,
+      { wait: async () => { throw new Error('must not wait') } }
+    )
+    assert.equal(calls, 1, `status ${status} must not be retried`)
+    assert.equal(response.status, status)
+  }
+})
+
+test('each attempt carries its own unexpired timeout signal', async () => {
+  const signals = []
+  await fetchAdminOnceMore(
+    async (_url, attemptInit) => {
+      signals.push(attemptInit.signal)
+      return new Response('down', { status: 502 })
+    },
+    url,
+    init,
+    { wait: async () => {}, timeoutMs: 30_000 }
+  )
+  assert.equal(signals.length, 2)
+  assert.notEqual(signals[0], signals[1])
+  assert.equal(signals[1].aborted, false)
+})
+
+test('the caller init is forwarded unchanged apart from the signal', async () => {
+  let seen
+  await fetchAdminOnceMore(
+    async (seenUrl, attemptInit) => {
+      seen = { seenUrl, attemptInit }
+      return Response.json({})
+    },
+    url,
+    { method: 'POST', headers: { authorization: 'Bearer t' }, body: '{"v":1}' },
+    { wait: async () => {} }
+  )
+  assert.equal(seen.seenUrl, url)
+  assert.equal(seen.attemptInit.method, 'POST')
+  assert.deepEqual(seen.attemptInit.headers, { authorization: 'Bearer t' })
+  assert.equal(seen.attemptInit.body, '{"v":1}')
+})

cloud/dev/scripts/relay-evidence-code-provenance.mjs

@@ -0,0 +1,94 @@
+import { spawnSync } from 'node:child_process'
+import { fileURLToPath } from 'node:url'
+import {
+  RELAY_REPOSITORY_ROOT,
+  relayTreePath,
+  relayWorkflowPath
+} from './relay-repository.mjs'
+
+const SHA = /^[a-f0-9]{40}$/
+
+// Every file that decides how relay evidence is produced, sealed, verified, and then spent against
+// production; identical content across two commits is what makes the older commit's verdict binding.
+export const TRUSTED_EVIDENCE_CODE_PATHS = [
+  // Produces and seals the 15-minute dry-run evidence.
+  relayWorkflowPath('monitor-relay-production.yml'),
+  relayWorkflowPath('monitor-relay-production-job.yml'),
+  // Download it, verify its authority, and mutate production on it.
+  relayWorkflowPath('deploy-relay-production-same-cap.yml'),
+  relayWorkflowPath('deploy-relay-production-same-cap-job.yml'),
+  relayWorkflowPath('operate-relay-production-rehome.yml'),
+  relayWorkflowPath('operate-relay-production-rehome-job.yml'),
+  // Sealing, verification, the wave/canary authority, and the path constants below.
+  relayTreePath('dev/scripts/relay-evidence-code-provenance.mjs'),
+  relayTreePath('dev/scripts/relay-monitor-evidence.mjs'),
+  relayTreePath('dev/scripts/relay-production-same-cap-wave.mjs'),
+  relayTreePath('dev/scripts/relay-repository.mjs'),
+  // Every other script those jobs run against live production.
+  relayTreePath('dev/scripts/infra.mjs'),
+  relayTreePath('dev/scripts/operate-relay-regional-rehome.mjs'),
+  relayTreePath('dev/scripts/prepare-relay-production-capacity-canary.mjs'),
+  relayTreePath('dev/scripts/probe-relay-rehome-trust.mjs'),
+  relayTreePath('dev/scripts/validate-relay-capacity-plan.mjs'),
+  relayTreePath('dev/scripts/verify-relay-capacity-transition.mjs'),
+  // The monitor itself and the live preflight recheck, plus anything that changes their behaviour.
+  relayTreePath('apps/relay-ops'),
+  relayTreePath('package.json'),
+  relayTreePath('pnpm-lock.yaml'),
+  relayTreePath('pnpm-workspace.yaml'),
+  // The Cloud SQL rollout lease every mutation job takes and releases.
+  '.github/actions/cloud-sql-rollout-lease'
+]
+
+function git(root, args) {
+  const result = spawnSync('git', ['-C', root, ...args], { encoding: 'utf8' })
+  if (result.error) throw new Error('relay evidence provenance cannot run git')
+  return result
+}
+
+/**
+ * Accepts evidence sealed at a different commit only when the current commit descends from it and
+ * every trusted path is byte-identical, so the verdict provably came from this exact code. Anything
+ * git cannot answer (no checkout, unknown commit, shallow clone) fails closed.
+ */
+export function requireSameEvidenceCode({
+  sealedSha,
+  currentSha,
+  label,
+  repositoryRoot = fileURLToPath(RELAY_REPOSITORY_ROOT)
+}) {
+  if (!SHA.test(sealedSha ?? '') || !SHA.test(currentSha ?? '')) {
+    throw new Error(`${label} commit is invalid`)
+  }
+  if (sealedSha === currentSha) return
+  if (git(repositoryRoot, ['rev-parse', '--git-dir']).status !== 0) {
+    throw new Error(`${label} commit cannot be compared without a git checkout`)
+  }
+  for (const sha of [sealedSha, currentSha]) {
+    if (git(repositoryRoot, ['rev-parse', '--verify', '--quiet', `${sha}^{commit}`]).status !== 0) {
+      throw new Error(
+        `${label} commit ${sha} is unknown to this checkout; check out with fetch-depth: 0`
+      )
+    }
+  }
+  const ancestry = git(repositoryRoot, ['merge-base', '--is-ancestor', sealedSha, currentSha])
+  if (ancestry.status === 1) {
+    throw new Error(`${label} commit ${sealedSha} is not an ancestor of ${currentSha}`)
+  }
+  if (ancestry.status !== 0) {
+    throw new Error(`${label} commit ancestry could not be determined`)
+  }
+  const diff = git(repositoryRoot, [
+    'diff',
+    '--name-only',
+    sealedSha,
+    currentSha,
+    '--',
+    ...TRUSTED_EVIDENCE_CODE_PATHS
+  ])
+  if (diff.status !== 0) throw new Error(`${label} commit comparison failed`)
+  const changed = diff.stdout.split('\n').filter(Boolean)
+  if (changed.length > 0) {
+    throw new Error(`${label} code changed after it was sealed: ${changed.join(',')}`)
+  }
+}

cloud/dev/scripts/relay-monitor-evidence.mjs

@@ -2,6 +2,7 @@ import { createHash } from 'node:crypto'
 import { chmod, readFile, readdir, stat, writeFile } from 'node:fs/promises'
 import { basename, join, resolve } from 'node:path'
 import { pathToFileURL } from 'node:url'
+import { requireSameEvidenceCode } from './relay-evidence-code-provenance.mjs'
 
 const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{1,127}$/
 const SHA = /^[a-f0-9]{40}$/
@@ -102,7 +103,7 @@ export async function createEvidenceManifest(argv) {
   return manifest
 }
 
-async function readAndVerifyManifest(directory, expected) {
+async function readAndVerifyManifest(directory, expected, sameCodeCommit) {
   const manifest = JSON.parse(
     await readFile(join(directory, 'evidence-manifest.json'), 'utf8')
   )
@@ -111,11 +112,23 @@ async function readAndVerifyManifest(directory, expected) {
     manifest.incidentId !== expected.incidentId ||
     manifest.runId !== expected.runId ||
     manifest.runAttempt !== expected.runAttempt ||
-    manifest.commitSha !== expected.commitSha ||
-    manifest.mode !== expected.mode
+    !SHA.test(manifest.commitSha ?? '') ||
+    manifest.mode !== expected.mode ||
+    (!sameCodeCommit && manifest.commitSha !== expected.commitSha)
   ) {
     throw new Error('relay monitor evidence provenance does not match')
   }
+  // Unrelated merges land on main every few minutes, so the deployer resolves a newer commit than
+  // the monitor it must trust; identical monitor and mutation code is the property the SHA stood in
+  // for. Restore and mutation keep the exact-SHA bind: both run at the commit that sealed them.
+  if (sameCodeCommit) {
+    requireSameEvidenceCode({
+      sealedSha: manifest.commitSha,
+      currentSha: expected.commitSha,
+      label: 'relay monitor evidence',
+      ...sameCodeCommit
+    })
+  }
   const names = Object.keys(manifest.files ?? {})
   if (!names.includes(`${expected.incidentId}.state.json`)) {
     throw new Error('relay monitor evidence has no durable state')
@@ -209,12 +222,12 @@ function validCompletedDryRunState(state, expected, nowMs, maxAgeMs) {
   )
 }
 
-export async function verifyDryRunAuthority(argv, now = Date.now) {
+export async function verifyDryRunAuthority(argv, now = Date.now, repositoryRoot) {
   const values = argumentsByName(argv)
   const directory = resolve(values.directory ?? '')
   const expected = provenance(values)
   if (expected.mode !== 'dry-run') throw new Error('relay mutation requires dry-run evidence')
-  const manifest = await readAndVerifyManifest(directory, expected)
+  const manifest = await readAndVerifyManifest(directory, expected, { repositoryRoot })
   const state = JSON.parse(
     await readFile(join(directory, `${expected.incidentId}.state.json`), 'utf8')
   )