diff --git a/.gitattributes b/.gitattributes
index ec7a93a19b9..2a99890023b 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,13 +1,7 @@
-/config/scripts/create-draft-release.mjs text eol=lf
-/config/scripts/orca-dev.mjs text eol=lf
-/config/scripts/latest-stable-release.mjs text eol=lf
-/config/scripts/publish-complete-draft-releases.mjs text eol=lf
-/config/scripts/release-rc-history.mjs text eol=lf
-/config/scripts/run-internal-dev-setup.mjs text eol=lf
-/config/scripts/verify-cli-bin.mjs text eol=lf
-/config/scripts/verify-release-required-assets.mjs text eol=lf
-/config/scripts/replace-cached-nsis-elevate.mjs text eol=lf
-/config/scripts/resolve-7za-path.mjs text eol=lf
+# A shebang plus CRLF makes vite's SSR transform emit a literal `#!` mid-module,
+# so any suite importing the script dies at load with a SyntaxError. Pin the whole
+# directory rather than the scripts that happen to have a test today.
+/config/scripts/**/*.mjs text eol=lf
/skill-guides/*.md text eol=lf
/skill-stubs/*.md text eol=lf
/skills/*/SKILL.md text eol=lf
@@ -24,3 +18,6 @@
# the reviewable change, and pin LF because they are compared byte-for-byte.
# Not -diff: the shell diff is the review surface when a wrapper does change.
/src/main/__fixtures__/shell-wrapper-snapshots/*.txt linguist-generated=true text eol=lf
+# Generated runtime English subset: compared byte-for-byte by
+# verify:localization-runtime-catalog, so a CRLF checkout would fail the gate.
+/src/renderer/src/i18n/en-runtime-required.json linguist-generated=true text eol=lf
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index b44287a9e18..dc1c1412470 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -3,3 +3,8 @@
/config/scripts/*localization*.mjs @brennanb2025
/config/scripts/*locale*.mjs @brennanb2025
/config/i18next.config.ts @brennanb2025
+
+# The relay's deploy and operate surface: workspace, workflows, and the rollout lease action.
+/cloud/ @Jinwoo-H
+/.github/workflows/cloud-*.yml @Jinwoo-H
+/.github/actions/cloud-sql-rollout-lease/ @Jinwoo-H
diff --git a/.github/actions/cloud-sql-rollout-lease/README.md b/.github/actions/cloud-sql-rollout-lease/README.md
new file mode 100644
index 00000000000..a06664b1f18
--- /dev/null
+++ b/.github/actions/cloud-sql-rollout-lease/README.md
@@ -0,0 +1,152 @@
+# Cloud SQL rollout lease
+
+A compare-and-swap lease on one Cloud Storage object, used to serialize Cloud SQL
+**connection-budget** rollouts across two repositories.
+
+`concurrency.group: production-cloud-sql-rollout` only serializes runs inside a single repository.
+Once the relay workflows live in `stablyai/orca` and the app workflows stay in
+`stablyai/orca-cloud`, there are two independent queues pointed at one shared Cloud SQL instance.
+`relay-cloud-sql-connection-budget.mjs` computes `rolloutOverlap` as a `Math.max` over the relay
+director, api, auth and relay-cell candidates, which is only sound when exactly one rollout is in
+flight. This lease is what keeps that assumption true. Keep the per-repo concurrency groups **and**
+the lease; they solve different halves of the problem.
+
+## What it protects
+
+No workflow runs a Cloud SQL schema migration. Every locked workflow either deploys a Cloud Run
+revision or applies a GCE instance template against the shared instance, so the lease must cover
+**all rollouts**, not just migrations.
+
+## Usage
+
+The lease step must run **after** `google-github-actions/setup-gcloud`, and after
+`actions/checkout` — `uses: ./.github/actions/...` resolves against the checked-out workspace.
+It belongs in the first job of the workflow that holds a GCP credential, which is not always the
+gate job: `deploy-relay-production-same-cap`'s gate runs no `gcloud`, so its first acquire happens
+in the first cell job.
+
+```yaml
+- uses: google-github-actions/auth@v2
+ with: { workload_identity_provider: ..., service_account: ... }
+- uses: google-github-actions/setup-gcloud@v2
+- uses: ./.github/actions/cloud-sql-rollout-lease
+ with:
+ bucket: onorca-cloud-terraform-state
+ object: terraform/state/cloud-sql-rollout/production.lock
+```
+
+Buckets and objects in use:
+
+| Environment | Bucket | Object |
+| ----------- | -------------------------------------- | --------------------------------------------------- |
+| production | `onorca-cloud-terraform-state` | `terraform/state/cloud-sql-rollout/production.lock` |
+| staging | `onorca-cloud-staging-terraform-state` | `terraform/state/cloud-sql-rollout/staging.lock` |
+
+Workflows that serve both environments (`deploy-relay-asia-topology`,
+`operate-relay-asia-admission`) select the pair with an `inputs.environment == 'production'`
+ternary on both `bucket` and `object`. `deploy-staging` keeps its own `deploy-artifacts-staging`
+concurrency group but takes the staging lease, because it rolls the staging API revision.
+
+The object sits beside `terraform/state/relay-fence-broker/.lock`. The IAM grant names both the
+relay and app service accounts, so it is a **foundation-root** resource: `roles/storage.objectAdmin`
+conditioned on the `terraform/state/cloud-sql-rollout/` prefix, **plus** an unconditioned
+`roles/storage.legacyBucketReader`. Without the second role the generation-matched write fails in a
+way that looks like a permissions flake.
+
+## One lease per run, not per job
+
+`deploy-relay-production-capacity` calls its reusable job six times and
+`deploy-relay-production-same-cap` four times. Each call is a separate job on a separate runner, so
+a naive per-job acquire/release would leave the object free between waves — for runs that have taken
+up to 85 minutes.
+
+The lease is therefore keyed to the **run**, not the job. `holder-key` defaults to
+`${{ github.repository }}/${{ github.run_id }}`, and a job that finds its own holder key on a live
+lease **re-enters** it: the record is refreshed, not rejected. Every job in the chain acquires; only
+the last one releases.
+
+```yaml
+jobs:
+ gate:
+ steps:
+ - uses: ./.github/actions/cloud-sql-rollout-lease
+ with: { bucket: ..., object: ..., release: 'false' } # intermediate
+
+ wave-1: # ... release: 'false' on every wave job
+
+ release_lease:
+ needs: [gate, wave-1, wave-2, wave-3, wave-4]
+ if: always()
+ steps:
+ - uses: google-github-actions/auth@v2
+ - uses: google-github-actions/setup-gcloud@v2
+ - uses: ./.github/actions/cloud-sql-rollout-lease
+ with: { bucket: ..., object: ..., release: 'true' } # final
+```
+
+`release: 'false'` still acquires and still runs its `post` step; `post` only skips the delete. A
+single-job workflow leaves `release` at its `true` default and needs no extra job. So does a
+workflow whose several jobs can never hold the lease at once: `prove-relay-staging-capacity`'s two
+lease-holding jobs are guarded by complementary `inputs.mode` conditions, and the contract test
+checks that exclusivity rather than assuming it.
+
+If the final job never runs (runner killed, run cancelled hard), the lease expires on its TTL.
+
+## Timing
+
+- **TTL 35 minutes**, matching `apps/relay-fence-broker/src/mutation-lease.ts`.
+- **Renewal every 5 minutes.** `main` spawns a detached background Node process that rewrites
+ `expires_at` on the same generation-matched path; `post` kills it by pid read back from
+ `$GITHUB_STATE`. The renewer stops on its own the moment the object stops being ours, and has a
+ six-hour backstop in case `post` never runs. Its log is written to
+ `$RUNNER_TEMP/cloud-sql-rollout-lease-renewer.log` and echoed by `post`.
+- Renewal is mandatory, not optional: capacity runs have taken 85 minutes, well past any sane TTL.
+
+## Failure behaviour
+
+| Situation | Behaviour |
+| -------------------------------- | ----------------------------------------------------------------------------------------------------------------- |
+| Object absent | Acquire with `ifGenerationMatch: 0`. |
+| Live lease, our own holder key | Re-enter. Refresh `expires_at`, keep `acquired_at`. Never fails. |
+| Live lease, another holder | **Fail the job immediately**, printing the holder's repository, workflow and run URL. Never queues, never steals. |
+| Expired lease | Take over with the observed generation and emit `::warning::` naming the stale holder. |
+| `412` on write | Someone raced us. Fail as a conflict. |
+| Bucket unreachable, `403`, `5xx` | **Fail closed.** |
+| Record present but unparseable | **Fail closed.** A record we cannot read is never treated as free; an operator must inspect and delete it. |
+| Release finds a foreign holder | Warn and leave it alone. Our lease had already expired. |
+| Release fails | Warn only. `post` never fails a job over a release; the TTL bounds the damage. |
+
+## Why `monitor-relay-production` must not use this
+
+`monitor-relay-production` is in the `production-cloud-sql-rollout` concurrency group but is
+**read-only**: its identity holds only monitoring, logging, Cloud SQL and compute _viewer_ roles,
+and it runs `gcloud sql instances describe`, never a mutation. It consumes no connection budget.
+Putting it on the durable lease would let a monitoring run block a real rollout, and a rollout block
+monitoring exactly when an operator most needs it. Keep its same-repo concurrency group; keep it off
+the lease. The lock census contract test records it in the not-a-candidate map with this reason.
+
+## Token acquisition
+
+`gcloud auth print-access-token`, not a hand-rolled exchange of the `external_account` credentials
+file. Every consuming workflow already runs `setup-gcloud`, gcloud already handles every ADC flavour
+including the service-account impersonation leg, and this action must stay zero-dependency because
+it is duplicated by hand into the public repo. The GCE metadata server that
+`apps/relay-fence-broker/src/google-metadata.ts` uses does **not** exist on GitHub or Blacksmith
+runners; only the compare-and-swap algorithm is shared with the fence broker.
+
+## Duplication
+
+This directory is copied verbatim into `stablyai/orca`. It has no `package.json`, no
+`node_modules`, and imports nothing outside itself — `action-contract.test.mjs` enforces all three.
+Cross-repo consumption via `uses: stablyai/orca/.github/actions/...@` was rejected: it would
+put public-repo code inside private app deploys that hold a production credential, and neither
+repository protects `main` today.
+
+## Tests
+
+```
+node --test .github/actions/cloud-sql-rollout-lease/
+```
+
+`storage-lease.test.mjs` drives the real compare-and-swap path against an in-memory Cloud Storage
+fake that enforces generations. No network.
diff --git a/.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs b/.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs
new file mode 100644
index 00000000000..631daef7d3e
--- /dev/null
+++ b/.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs
@@ -0,0 +1,52 @@
+import assert from 'node:assert/strict'
+import { readFileSync, readdirSync } from 'node:fs'
+import { test } from 'node:test'
+
+const here = new URL('./', import.meta.url)
+const action = readFileSync(new URL('action.yml', here), 'utf8')
+const modules = readdirSync(here).filter((name) => name.endsWith('.mjs'))
+const shipped = modules.filter((name) => !name.endsWith('.test.mjs'))
+
+test('is a node24 JavaScript action with an always-run post step', () => {
+ // A composite action has no `post:`, so the lease could never be released on cancel or failure.
+ assert.match(action, /^ {2}using: node24$/m)
+ assert.doesNotMatch(action, /using: composite/)
+ assert.match(action, /^ {2}main: main\.mjs$/m)
+ assert.match(action, /^ {2}post: post\.mjs$/m)
+ assert.match(action, /^ {2}post-if: always\(\)$/m)
+})
+
+test('declares the inputs the wave-chain callers depend on', () => {
+ for (const input of ['bucket:', 'object:', 'holder-key:', 'release:']) {
+ assert.match(action, new RegExp(`^ {2}${input}$`, 'm'), input)
+ }
+ assert.match(action, /default: \$\{\{ github\.repository \}\}\/\$\{\{ github\.run_id \}\}/)
+ assert.match(action, /default: 'true'/)
+})
+
+test('stays self-contained so it can be duplicated into the public repo', () => {
+ assert.deepEqual(
+ readdirSync(here).filter((name) => name === 'package.json' || name === 'node_modules'),
+ [],
+ 'the action must run with zero installed dependencies'
+ )
+ for (const name of modules) {
+ const source = readFileSync(new URL(name, here), 'utf8')
+ for (const match of source.matchAll(/^import\b[\s\S]*?from '([^']+)'/gm)) {
+ const specifier = match[1]
+ const local = specifier.startsWith('.')
+ assert.ok(
+ specifier.startsWith('node:') || (local && !specifier.includes('..')),
+ `${name} imports ${specifier}; only node: builtins and same-directory modules are allowed`
+ )
+ }
+ }
+})
+
+test('never reaches for the GCE metadata server', () => {
+ // Runners have no metadata.google.internal; the fence broker's token path must not be copied.
+ for (const name of shipped) {
+ const source = readFileSync(new URL(name, here), 'utf8')
+ assert.doesNotMatch(source, /metadata\.google\.internal/, name)
+ }
+})
diff --git a/.github/actions/cloud-sql-rollout-lease/action.yml b/.github/actions/cloud-sql-rollout-lease/action.yml
new file mode 100644
index 00000000000..afd4b8efee0
--- /dev/null
+++ b/.github/actions/cloud-sql-rollout-lease/action.yml
@@ -0,0 +1,41 @@
+name: Cloud SQL rollout lease
+description: >-
+ Serialize Cloud SQL connection-budget rollouts across repositories with a compare-and-swap lease
+ on a Cloud Storage object. Fails immediately when another run holds the lease; never queues,
+ never steals.
+
+inputs:
+ bucket:
+ description: Terraform state bucket that holds the lease object.
+ required: true
+ object:
+ description: Lease object name, e.g. terraform/state/cloud-sql-rollout/production.lock
+ required: true
+ holder-key:
+ description: >-
+ Identity that owns the lease. Every job in one run must pass the same value; a job that finds
+ its own holder key on a live lease re-enters it instead of failing.
+ required: false
+ default: ${{ github.repository }}/${{ github.run_id }}
+ release:
+ description: >-
+ Release the lease in the post step. Set to "false" on every job of a multi-job wave except the
+ final always() job, which sets "true".
+ required: false
+ default: 'true'
+
+outputs:
+ holder-key:
+ description: The holder key written to the lease object.
+ generation:
+ description: Cloud Storage generation of the lease object after acquisition.
+ expires-at:
+ description: ISO-8601 instant at which the lease expires without renewal.
+ reentrant:
+ description: '"true" when this job re-entered a lease its own run already held.'
+
+runs:
+ using: node24
+ main: main.mjs
+ post: post.mjs
+ post-if: always()
diff --git a/.github/actions/cloud-sql-rollout-lease/gcloud-access-token.mjs b/.github/actions/cloud-sql-rollout-lease/gcloud-access-token.mjs
new file mode 100644
index 00000000000..8ff2bc58646
--- /dev/null
+++ b/.github/actions/cloud-sql-rollout-lease/gcloud-access-token.mjs
@@ -0,0 +1,46 @@
+import { execFileSync } from 'node:child_process'
+
+// DESIGN CHOICE: shell out to `gcloud auth print-access-token` instead of exchanging the
+// external_account credentials file that google-github-actions/auth writes.
+//
+// Every workflow on the Cloud SQL rollout lease already runs google-github-actions/setup-gcloud
+// right after auth (verified across all 11 mutating members), so gcloud is on PATH and already
+// bound to the federated identity. Doing the exchange ourselves would mean reimplementing the STS
+// token swap plus the service-account impersonation leg, in an action that must stay
+// zero-dependency and is duplicated by hand into a second repo. gcloud already handles every ADC
+// flavour and refreshes on its own. The metadata server is not an option: it does not exist on
+// GitHub or Blacksmith runners.
+//
+// Consequence, documented in the README: the lease step MUST come after setup-gcloud.
+
+const TOKEN_REUSE_MS = 40 * 60 * 1_000 // GCP access tokens live ~60 min; re-mint well before that.
+
+export function createAccessTokenSource({ run = runGcloud, now = Date.now } = {}) {
+ let cached = null
+ return () => {
+ if (cached && cached.mintedAt + TOKEN_REUSE_MS > now()) {
+ return cached.token
+ }
+ const token = run()
+ if (!token) {
+ throw new Error('gcloud auth print-access-token returned an empty token')
+ }
+ cached = { token, mintedAt: now() }
+ return token
+ }
+}
+
+function runGcloud() {
+ const binary = process.platform === 'win32' ? 'gcloud.cmd' : 'gcloud'
+ try {
+ return execFileSync(binary, ['auth', 'print-access-token'], {
+ encoding: 'utf8',
+ stdio: ['ignore', 'pipe', 'pipe'],
+ timeout: 60_000
+ }).trim()
+ } catch (error) {
+ // Never surface stdout; it is the token on success and noise on failure.
+ const detail = String(error?.stderr ?? '').trim() || error?.message || 'unknown failure'
+ throw new Error(`could not mint a GCP access token via gcloud: ${detail}`)
+ }
+}
diff --git a/.github/actions/cloud-sql-rollout-lease/holder-identity.mjs b/.github/actions/cloud-sql-rollout-lease/holder-identity.mjs
new file mode 100644
index 00000000000..105d0b3bff0
--- /dev/null
+++ b/.github/actions/cloud-sql-rollout-lease/holder-identity.mjs
@@ -0,0 +1,20 @@
+// Who we claim to be on the lease object. Shared by main and the detached renewer so both agree
+// on the holder key without re-deriving it from a different set of environment variables.
+
+export function holderIdentity(explicitHolderKey) {
+ const repository = process.env.GITHUB_REPOSITORY ?? 'unknown'
+ const runId = process.env.GITHUB_RUN_ID ?? 'unknown'
+ const server = process.env.GITHUB_SERVER_URL ?? 'https://github.com'
+ const holderKey = explicitHolderKey || `${repository}/${runId}`
+ if (/[\r\n]/.test(holderKey)) {
+ throw new Error('holder-key must be single-line')
+ }
+ return {
+ holderKey,
+ repository,
+ workflow: process.env.GITHUB_WORKFLOW ?? 'unknown',
+ runId,
+ runUrl: `${server}/${repository}/actions/runs/${runId}`,
+ runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? 'unknown'
+ }
+}
diff --git a/.github/actions/cloud-sql-rollout-lease/main.mjs b/.github/actions/cloud-sql-rollout-lease/main.mjs
new file mode 100644
index 00000000000..0a429763a73
--- /dev/null
+++ b/.github/actions/cloud-sql-rollout-lease/main.mjs
@@ -0,0 +1,81 @@
+import { spawn } from 'node:child_process'
+import { fileURLToPath } from 'node:url'
+import { createAccessTokenSource } from './gcloud-access-token.mjs'
+import { holderIdentity } from './holder-identity.mjs'
+import { fail, input, notice, renewerLogPath, saveState, setOutput, warn } from './runner-state.mjs'
+import { CloudSqlRolloutLease, LeaseConflict, describeHolder } from './storage-lease.mjs'
+
+const bucket = input('bucket')
+const objectName = input('object')
+const release = input('release') !== 'false'
+
+if (!bucket || !objectName) {
+ fail('cloud-sql-rollout-lease requires both `bucket` and `object`')
+ process.exit(1)
+}
+
+const holder = holderIdentity(input('holder-key'))
+const lease = new CloudSqlRolloutLease({
+ bucket,
+ objectName,
+ accessToken: createAccessTokenSource()
+})
+
+let claim
+try {
+ claim = await lease.acquire(holder)
+} catch (error) {
+ if (error instanceof LeaseConflict) {
+ fail(
+ `${error.message}. Cloud SQL rollouts are serialized across repositories; this run will not queue or steal the lease. Wait for the holder to finish, then re-run.`
+ )
+ if (error.holder) {
+ console.log(`Lease holder repository: ${error.holder.repository}`)
+ console.log(`Lease holder workflow: ${error.holder.workflow}`)
+ console.log(`Lease holder run: ${error.holder.run_url}`)
+ }
+ } else {
+ // Bucket unreachable, permission denied, unreadable record: fail closed.
+ fail(`could not acquire ${lease.uri}: ${error.message}`)
+ }
+ process.exit(1)
+}
+
+// Persist before anything else can throw, so `post` always releases what we hold.
+saveState('acquired', 'true')
+saveState('bucket', bucket)
+saveState('object', objectName)
+saveState('holder_key', holder.holderKey)
+saveState('release', release ? 'true' : 'false')
+
+setOutput('holder-key', holder.holderKey)
+setOutput('generation', claim.generation)
+setOutput('expires-at', new Date(claim.record.expires_at).toISOString())
+setOutput('reentrant', claim.state === 'reentrant' ? 'true' : 'false')
+
+if (claim.state === 'reentrant') {
+ notice(
+ `Re-entered the Cloud SQL rollout lease on ${lease.uri} already held by this run; refreshed to ${new Date(claim.record.expires_at).toISOString()}.`
+ )
+} else if (claim.state === 'takeover') {
+ notice(`Took over ${lease.uri} from ${describeHolder(claim.previous)}.`)
+} else {
+ notice(
+ `Acquired ${lease.uri} until ${new Date(claim.record.expires_at).toISOString()} (holder ${holder.holderKey}).`
+ )
+}
+
+try {
+ const renewer = spawn(
+ process.execPath,
+ [fileURLToPath(new URL('./renew.mjs', import.meta.url)), bucket, objectName, holder.holderKey],
+ { detached: true, stdio: 'ignore', env: process.env }
+ )
+ renewer.unref()
+ saveState('renewer_pid', String(renewer.pid))
+ const log = renewerLogPath()
+ notice(`Lease renewer running as pid ${renewer.pid}${log ? `, logging to ${log}` : ''}.`)
+} catch (error) {
+ // A missing renewer is survivable for short jobs; the TTL still covers 35 minutes.
+ warn(`could not start the lease renewer: ${error.message}. The lease will expire on its TTL.`)
+}
diff --git a/.github/actions/cloud-sql-rollout-lease/post.mjs b/.github/actions/cloud-sql-rollout-lease/post.mjs
new file mode 100644
index 00000000000..5be970a34b3
--- /dev/null
+++ b/.github/actions/cloud-sql-rollout-lease/post.mjs
@@ -0,0 +1,81 @@
+import { readFileSync } from 'node:fs'
+import { createAccessTokenSource } from './gcloud-access-token.mjs'
+import { notice, renewerLogPath, savedState, warn } from './runner-state.mjs'
+import { CloudSqlRolloutLease, describeHolder } from './storage-lease.mjs'
+
+stopRenewer()
+printRenewerLog()
+
+if (savedState('acquired') !== 'true') {
+ notice('No Cloud SQL rollout lease was acquired by this step; nothing to release.')
+ process.exit(0)
+}
+
+const bucket = savedState('bucket')
+const objectName = savedState('object')
+const holderKey = savedState('holder_key')
+
+if (savedState('release') !== 'true') {
+ notice(
+ `Holding gs://${bucket}/${objectName} for the rest of run ${holderKey}; a later job with release=true must free it.`
+ )
+ process.exit(0)
+}
+
+const lease = new CloudSqlRolloutLease({
+ bucket,
+ objectName,
+ accessToken: createAccessTokenSource()
+})
+
+try {
+ const result = await lease.release(holderKey)
+ if (result.released) {
+ notice(`Released ${lease.uri} at generation ${result.generation}.`)
+ } else if (result.reason === 'absent') {
+ notice(`${lease.uri} was already gone; nothing to release.`)
+ } else if (result.reason === 'foreign') {
+ warn(
+ `${lease.uri} is now held by ${describeHolder(result.holder)}; leaving it alone. Our lease had already expired.`
+ )
+ } else {
+ warn(`${lease.uri} changed while releasing it; leaving it to expire on its TTL.`)
+ }
+} catch (error) {
+ // Never fail a job in post over a release; the TTL bounds the damage to 35 minutes.
+ warn(`could not release ${lease.uri}: ${error.message}. It will expire on its TTL.`)
+}
+
+function stopRenewer() {
+ const pid = Number(savedState('renewer_pid'))
+ if (!Number.isInteger(pid) || pid <= 0) {
+ return
+ }
+ try {
+ process.kill(pid, 'SIGTERM')
+ notice(`Stopped the lease renewer (pid ${pid}).`)
+ } catch (error) {
+ if (error?.code !== 'ESRCH') {
+ warn(`could not stop the lease renewer ${pid}: ${error.message}`)
+ }
+ }
+}
+
+function printRenewerLog() {
+ const path = renewerLogPath()
+ if (!path) {
+ return
+ }
+ let text = ''
+ try {
+ text = readFileSync(path, 'utf8')
+ } catch {
+ return
+ }
+ if (!text.trim()) {
+ return
+ }
+ console.log('::group::Cloud SQL rollout lease renewer log')
+ console.log(text.trimEnd())
+ console.log('::endgroup::')
+}
diff --git a/.github/actions/cloud-sql-rollout-lease/renew.mjs b/.github/actions/cloud-sql-rollout-lease/renew.mjs
new file mode 100644
index 00000000000..cc8e543ba4f
--- /dev/null
+++ b/.github/actions/cloud-sql-rollout-lease/renew.mjs
@@ -0,0 +1,73 @@
+import { appendFileSync } from 'node:fs'
+import { createAccessTokenSource } from './gcloud-access-token.mjs'
+import { renewerLogPath } from './runner-state.mjs'
+import { CloudSqlRolloutLease, RENEW_INTERVAL_MS } from './storage-lease.mjs'
+
+// Detached renewer. `main` spawns it, `post` kills it. It rewrites expires_at on the same
+// generation-matched path as acquisition, and stops the moment the object stops being ours.
+
+const MAX_LIFETIME_MS = 6 * 60 * 60 * 1_000 // Backstop if post never runs (runner killed).
+
+const [bucket, objectName, holderKey] = process.argv.slice(2)
+const logPath = renewerLogPath()
+const startedAt = Date.now()
+
+function log(message) {
+ if (!logPath) {
+ return
+ }
+ try {
+ appendFileSync(logPath, `${new Date().toISOString()} ${message}\n`)
+ } catch {
+ // A renewer that cannot log must still renew.
+ }
+}
+
+if (!bucket || !objectName || !holderKey) {
+ log('renewer started without bucket/object/holder-key; exiting')
+ process.exit(1)
+}
+
+const lease = new CloudSqlRolloutLease({
+ bucket,
+ objectName,
+ accessToken: createAccessTokenSource(),
+ warn: (message) => log(`warning ${message}`)
+})
+
+let stopping = false
+for (const signal of ['SIGTERM', 'SIGINT', 'SIGHUP']) {
+ process.on(signal, () => {
+ stopping = true
+ log(`received ${signal}; stopping`)
+ process.exit(0)
+ })
+}
+
+log(`renewer started for ${lease.uri} holder=${holderKey} interval=${RENEW_INTERVAL_MS}ms`)
+
+while (!stopping) {
+ await new Promise((resolve) => {
+ setTimeout(resolve, RENEW_INTERVAL_MS)
+ })
+ if (stopping) {
+ break
+ }
+ if (Date.now() - startedAt > MAX_LIFETIME_MS) {
+ log('renewer hit its maximum lifetime; stopping so the lease can expire')
+ break
+ }
+ try {
+ const result = await lease.renew(holderKey)
+ if (!result.renewed) {
+ log(`lease is no longer ours (${result.reason}); stopping`)
+ break
+ }
+ log(
+ `renewed until ${new Date(result.record.expires_at).toISOString()} at generation ${result.generation}`
+ )
+ } catch (error) {
+ // Transient GCS or token failures are retried on the next tick; the TTL covers 7 misses.
+ log(`renewal attempt failed: ${error.message}`)
+ }
+}
diff --git a/.github/actions/cloud-sql-rollout-lease/runner-state.mjs b/.github/actions/cloud-sql-rollout-lease/runner-state.mjs
new file mode 100644
index 00000000000..ec36a16e0f2
--- /dev/null
+++ b/.github/actions/cloud-sql-rollout-lease/runner-state.mjs
@@ -0,0 +1,55 @@
+import { appendFileSync } from 'node:fs'
+
+// Action-state and output plumbing via the runner's file protocol, so the action needs no
+// @actions/core dependency. Values are single-line by construction; anything else is rejected.
+
+/** The detached renewer's stdio is ignored, so it appends here instead and `post` echoes it. */
+export function renewerLogPath() {
+ const dir = process.env.RUNNER_TEMP
+ return dir ? `${dir}/cloud-sql-rollout-lease-renewer.log` : null
+}
+
+export function input(name) {
+ return (process.env[`INPUT_${name.replace(/ /g, '_').toUpperCase()}`] ?? '').trim()
+}
+
+export function savedState(name) {
+ return (process.env[`STATE_${name}`] ?? '').trim()
+}
+
+export function saveState(name, value) {
+ appendToEnvFile('GITHUB_STATE', name, value)
+}
+
+export function setOutput(name, value) {
+ appendToEnvFile('GITHUB_OUTPUT', name, value)
+}
+
+export function notice(message) {
+ console.log(`::notice::${oneLine(message)}`)
+}
+
+export function warn(message) {
+ console.log(`::warning::${oneLine(message)}`)
+}
+
+export function fail(message) {
+ console.log(`::error::${oneLine(message)}`)
+ process.exitCode = 1
+}
+
+function appendToEnvFile(variable, name, value) {
+ const text = String(value)
+ if (/[\r\n]/.test(text)) {
+ throw new Error(`${name} must be single-line`)
+ }
+ const path = process.env[variable]
+ if (!path) {
+ return
+ } // Running outside a runner (local smoke run); nothing to persist.
+ appendFileSync(path, `${name}=${text}\n`)
+}
+
+function oneLine(message) {
+ return String(message).replace(/\r?\n/g, ' ')
+}
diff --git a/.github/actions/cloud-sql-rollout-lease/storage-lease.mjs b/.github/actions/cloud-sql-rollout-lease/storage-lease.mjs
new file mode 100644
index 00000000000..44312377ee3
--- /dev/null
+++ b/.github/actions/cloud-sql-rollout-lease/storage-lease.mjs
@@ -0,0 +1,240 @@
+// Compare-and-swap lease over a single Cloud Storage object.
+//
+// Ported from apps/relay-fence-broker/src/mutation-lease.ts rather than imported: this action is
+// duplicated verbatim into stablyai/orca, so it must carry no repo-local imports. Only the
+// algorithm is shared (read metadata -> write with ifGenerationMatch -> 412 is a conflict ->
+// generation-matched delete -> an expired record is free). The broker's token path is NOT shared;
+// it reads the GCE metadata server, which does not exist on Actions runners.
+
+export const LEASE_TTL_MS = 35 * 60 * 1_000
+export const RENEW_INTERVAL_MS = 5 * 60 * 1_000
+
+const GENERATION = /^[1-9][0-9]{0,30}$/
+
+export class LeaseConflict extends Error {
+ constructor(message, holder) {
+ super(message)
+ this.name = 'LeaseConflict'
+ this.holder = holder ?? null
+ }
+}
+
+/** A live record we cannot parse is never treated as free; wedging beats double-rollout. */
+export class LeaseUnreadable extends Error {
+ constructor(message) {
+ super(message)
+ this.name = 'LeaseUnreadable'
+ }
+}
+
+function parseRecord(raw) {
+ if (!raw || typeof raw !== 'object') {
+ return null
+ }
+ if (typeof raw.holder_key !== 'string' || raw.holder_key.length === 0) {
+ return null
+ }
+ if (!Number.isSafeInteger(raw.acquired_at) || !Number.isSafeInteger(raw.expires_at)) {
+ return null
+ }
+ return {
+ repository: typeof raw.repository === 'string' ? raw.repository : 'unknown',
+ workflow: typeof raw.workflow === 'string' ? raw.workflow : 'unknown',
+ run_id: typeof raw.run_id === 'string' ? raw.run_id : 'unknown',
+ run_url: typeof raw.run_url === 'string' ? raw.run_url : 'unknown',
+ run_attempt: typeof raw.run_attempt === 'string' ? raw.run_attempt : 'unknown',
+ acquired_at: raw.acquired_at,
+ expires_at: raw.expires_at,
+ holder_key: raw.holder_key
+ }
+}
+
+function describe(record) {
+ return `${record.repository} / ${record.workflow} (run ${record.run_id}, attempt ${record.run_attempt}) ${record.run_url}`
+}
+
+export class CloudSqlRolloutLease {
+ #bucket
+ #objectName
+ #accessToken
+ #fetcher
+ #now
+ #warn
+
+ constructor({
+ bucket,
+ objectName,
+ accessToken,
+ fetcher = fetch,
+ now = Date.now,
+ warn = (message) => console.log(`::warning::${message}`)
+ }) {
+ this.#bucket = bucket
+ this.#objectName = objectName
+ this.#accessToken = accessToken
+ this.#fetcher = fetcher
+ this.#now = now
+ this.#warn = warn
+ }
+
+ get uri() {
+ return `gs://${this.#bucket}/${this.#objectName}`
+ }
+
+ async acquire(holder) {
+ const existing = await this.read()
+ const now = this.#now()
+ if (!existing) {
+ return this.#claim(holder, '0', now, now, 'created')
+ }
+ if (existing.record.holder_key === holder.holderKey) {
+ // Same run, another job in the wave chain. Refresh, never fail.
+ return this.#claim(
+ holder,
+ existing.generation,
+ existing.record.acquired_at,
+ now,
+ 'reentrant',
+ existing.record
+ )
+ }
+ if (existing.record.expires_at > now) {
+ throw new LeaseConflict(
+ `${this.uri} is held by ${describe(existing.record)} until ${new Date(existing.record.expires_at).toISOString()}`,
+ existing.record
+ )
+ }
+ this.#warn(
+ `Taking over an expired Cloud SQL rollout lease on ${this.uri}. Stale holder: ${describe(existing.record)}, expired ${new Date(existing.record.expires_at).toISOString()}.`
+ )
+ return this.#claim(holder, existing.generation, now, now, 'takeover', existing.record)
+ }
+
+ async renew(holderKey) {
+ const existing = await this.read()
+ if (!existing) {
+ return { renewed: false, reason: 'absent' }
+ }
+ if (existing.record.holder_key !== holderKey) {
+ return { renewed: false, reason: 'foreign' }
+ }
+ const now = this.#now()
+ const record = { ...existing.record, expires_at: now + LEASE_TTL_MS }
+ const written = await this.#write(record, existing.generation)
+ return { renewed: true, generation: written.generation, record }
+ }
+
+ async release(holderKey) {
+ const existing = await this.read()
+ if (!existing) {
+ return { released: false, reason: 'absent' }
+ }
+ if (existing.record.holder_key !== holderKey) {
+ return { released: false, reason: 'foreign', holder: existing.record }
+ }
+ const response = await this.#fetcher(
+ `${this.#metadataUrl()}?ifGenerationMatch=${encodeURIComponent(existing.generation)}`,
+ { method: 'DELETE', headers: { Authorization: `Bearer ${await this.#token()}` } }
+ )
+ if (response.status === 412) {
+ return { released: false, reason: 'conflict' }
+ }
+ if (!response.ok && response.status !== 404) {
+ throw new Error(`lease release failed: ${response.status}`)
+ }
+ return { released: true, generation: existing.generation }
+ }
+
+ async read() {
+ const token = await this.#token()
+ const metadataResponse = await this.#fetcher(this.#metadataUrl(), {
+ headers: { Authorization: `Bearer ${token}` }
+ })
+ if (metadataResponse.status === 404) {
+ return null
+ }
+ if (!metadataResponse.ok) {
+ throw new Error(`lease inspection failed: ${metadataResponse.status}`)
+ }
+ const metadata = await metadataResponse.json()
+ if (!GENERATION.test(metadata?.generation ?? '')) {
+ throw new LeaseUnreadable(`${this.uri} has no valid generation`)
+ }
+ const bodyResponse = await this.#fetcher(`${this.#metadataUrl()}?alt=media`, {
+ headers: { Authorization: `Bearer ${token}` }
+ })
+ if (bodyResponse.status === 404) {
+ return null
+ }
+ if (!bodyResponse.ok) {
+ throw new Error(`lease body read failed: ${bodyResponse.status}`)
+ }
+ let raw = null
+ try {
+ raw = await bodyResponse.json()
+ } catch {
+ raw = null
+ }
+ const record = parseRecord(raw)
+ if (!record) {
+ throw new LeaseUnreadable(
+ `${this.uri} holds an unreadable lease record; an operator must inspect and delete it before rollouts can resume`
+ )
+ }
+ return { generation: metadata.generation, record }
+ }
+
+ async #claim(holder, generation, acquiredAt, now, state, previous) {
+ const record = {
+ repository: holder.repository,
+ workflow: holder.workflow,
+ run_id: holder.runId,
+ run_url: holder.runUrl,
+ run_attempt: holder.runAttempt,
+ acquired_at: acquiredAt,
+ expires_at: now + LEASE_TTL_MS,
+ holder_key: holder.holderKey
+ }
+ const written = await this.#write(record, generation)
+ return { state, generation: written.generation, record, previous: previous ?? null }
+ }
+
+ async #write(record, generation) {
+ const response = await this.#fetcher(
+ `${this.#uploadUrl()}&ifGenerationMatch=${encodeURIComponent(generation)}`,
+ {
+ method: 'POST',
+ headers: {
+ Authorization: `Bearer ${await this.#token()}`,
+ 'Content-Type': 'application/json'
+ },
+ body: JSON.stringify(record)
+ }
+ )
+ if (response.status === 412) {
+ throw new LeaseConflict(`${this.uri} changed concurrently while we were claiming it`)
+ }
+ if (!response.ok) {
+ throw new Error(`lease write failed: ${response.status}`)
+ }
+ const metadata = await response.json()
+ if (!GENERATION.test(metadata?.generation ?? '')) {
+ throw new LeaseUnreadable(`${this.uri} write returned no valid generation`)
+ }
+ return { generation: metadata.generation }
+ }
+
+ async #token() {
+ return typeof this.#accessToken === 'function' ? await this.#accessToken() : this.#accessToken
+ }
+
+ #metadataUrl() {
+ return `https://storage.googleapis.com/storage/v1/b/${encodeURIComponent(this.#bucket)}/o/${encodeURIComponent(this.#objectName)}`
+ }
+
+ #uploadUrl() {
+ return `https://storage.googleapis.com/upload/storage/v1/b/${encodeURIComponent(this.#bucket)}/o?uploadType=media&name=${encodeURIComponent(this.#objectName)}`
+ }
+}
+
+export const describeHolder = describe
diff --git a/.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs b/.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs
new file mode 100644
index 00000000000..f13e01517be
--- /dev/null
+++ b/.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs
@@ -0,0 +1,324 @@
+import assert from 'node:assert/strict'
+import { test } from 'node:test'
+import { createAccessTokenSource } from './gcloud-access-token.mjs'
+import {
+ CloudSqlRolloutLease,
+ LEASE_TTL_MS,
+ LeaseConflict,
+ LeaseUnreadable
+} from './storage-lease.mjs'
+
+const BUCKET = 'onorca-cloud-terraform-state'
+const OBJECT = 'terraform/state/cloud-sql-rollout/production.lock'
+const NOW = 1_756_000_000_000
+
+/**
+ * Enough of the Cloud Storage JSON API to exercise real compare-and-swap semantics: generations
+ * increment, ifGenerationMatch is enforced, and a mismatch is a 412. `faults` injects failures.
+ */
+function fakeStorage({ object = null, faults = [] } = {}) {
+ const state = { object, requests: [] }
+ const fetcher = async (rawUrl, init = {}) => {
+ const url = new URL(rawUrl)
+ const method = init.method ?? 'GET'
+ const record = { method, url, path: url.pathname, search: url.searchParams }
+ state.requests.push(record)
+ const fault = faults.find((candidate) => candidate.when(record))
+ if (fault) {
+ return json(fault.status, fault.body ?? {})
+ }
+
+ if (url.pathname.startsWith('/upload/')) {
+ const want = url.searchParams.get('ifGenerationMatch')
+ const have = state.object ? state.object.generation : '0'
+ if (want !== have) {
+ return json(412, {})
+ }
+ const generation = String(Number(have === '0' ? '1000' : have) + 1)
+ state.object = { generation, body: JSON.parse(init.body) }
+ return json(200, { generation })
+ }
+ if (method === 'DELETE') {
+ if (!state.object) {
+ return json(404, {})
+ }
+ if (url.searchParams.get('ifGenerationMatch') !== state.object.generation) {
+ return json(412, {})
+ }
+ state.object = null
+ return json(204, {})
+ }
+ if (!state.object) {
+ return json(404, {})
+ }
+ if (url.searchParams.get('alt') === 'media') {
+ return json(200, state.object.body)
+ }
+ return json(200, { generation: state.object.generation })
+ }
+ return { state, fetcher }
+}
+
+function json(status, body) {
+ return {
+ status,
+ ok: status >= 200 && status < 300,
+ json: async () => body
+ }
+}
+
+function storedRecord({
+ holderKey,
+ expiresAt,
+ repository = 'stablyai/orca',
+ workflow = 'Deploy Relay Production'
+}) {
+ return {
+ repository,
+ workflow,
+ run_id: '9001',
+ run_url: 'https://github.com/stablyai/orca/actions/runs/9001',
+ run_attempt: '1',
+ acquired_at: NOW - 60_000,
+ expires_at: expiresAt,
+ holder_key: holderKey
+ }
+}
+
+function leaseFor(storage, { warn = () => {} } = {}) {
+ return new CloudSqlRolloutLease({
+ bucket: BUCKET,
+ objectName: OBJECT,
+ accessToken: 'test-token',
+ fetcher: storage.fetcher,
+ now: () => NOW,
+ warn
+ })
+}
+
+const HOLDER = {
+ holderKey: 'stablyai/orca-cloud/42',
+ repository: 'stablyai/orca-cloud',
+ workflow: 'Deploy Relay Production Same-Cap',
+ runId: '42',
+ runUrl: 'https://github.com/stablyai/orca-cloud/actions/runs/42',
+ runAttempt: '1'
+}
+
+test('acquires a lease on an empty object with ifGenerationMatch=0', async () => {
+ const storage = fakeStorage()
+ const claim = await leaseFor(storage).acquire(HOLDER)
+
+ assert.equal(claim.state, 'created')
+ const upload = storage.state.requests.find((request) => request.path.startsWith('/upload/'))
+ assert.equal(upload.search.get('ifGenerationMatch'), '0')
+ assert.equal(upload.search.get('name'), OBJECT)
+ assert.deepEqual(storage.state.object.body, {
+ repository: 'stablyai/orca-cloud',
+ workflow: 'Deploy Relay Production Same-Cap',
+ run_id: '42',
+ run_url: 'https://github.com/stablyai/orca-cloud/actions/runs/42',
+ run_attempt: '1',
+ acquired_at: NOW,
+ expires_at: NOW + LEASE_TTL_MS,
+ holder_key: 'stablyai/orca-cloud/42'
+ })
+})
+
+test('refuses a live lease held by another run and never writes', async () => {
+ const storage = fakeStorage({
+ object: {
+ generation: '1500',
+ body: storedRecord({ holderKey: 'stablyai/orca/9001', expiresAt: NOW + 60_000 })
+ }
+ })
+
+ const error = await leaseFor(storage)
+ .acquire(HOLDER)
+ .catch((thrown) => thrown)
+
+ assert.ok(error instanceof LeaseConflict)
+ assert.equal(error.holder.repository, 'stablyai/orca')
+ assert.equal(error.holder.run_url, 'https://github.com/stablyai/orca/actions/runs/9001')
+ assert.equal(
+ storage.state.requests.filter((request) => request.method !== 'GET').length,
+ 0,
+ 'a foreign live lease must not be written'
+ )
+ assert.equal(storage.state.object.generation, '1500')
+})
+
+test('re-enters a live lease this run already holds and extends it', async () => {
+ const storage = fakeStorage({
+ object: {
+ generation: '1500',
+ body: storedRecord({ holderKey: HOLDER.holderKey, expiresAt: NOW + 60_000 })
+ }
+ })
+ const warnings = []
+ const claim = await leaseFor(storage, { warn: (message) => warnings.push(message) }).acquire(
+ HOLDER
+ )
+
+ assert.equal(claim.state, 'reentrant')
+ assert.deepEqual(warnings, [], 're-entering our own lease is not a takeover')
+ assert.equal(claim.record.acquired_at, NOW - 60_000, 'original acquisition time is preserved')
+ assert.equal(claim.record.expires_at, NOW + LEASE_TTL_MS)
+ const upload = storage.state.requests.find((request) => request.path.startsWith('/upload/'))
+ assert.equal(upload.search.get('ifGenerationMatch'), '1500')
+ assert.equal(storage.state.object.generation, '1501')
+})
+
+test('takes over an expired lease and warns naming the stale holder', async () => {
+ const storage = fakeStorage({
+ object: {
+ generation: '1500',
+ body: storedRecord({
+ holderKey: 'stablyai/orca/9001',
+ expiresAt: NOW - 1,
+ repository: 'stablyai/orca',
+ workflow: 'Deploy Relay Production Capacity'
+ })
+ }
+ })
+ const warnings = []
+ const claim = await leaseFor(storage, { warn: (message) => warnings.push(message) }).acquire(
+ HOLDER
+ )
+
+ assert.equal(claim.state, 'takeover')
+ assert.equal(warnings.length, 1)
+ assert.match(warnings[0], /stablyai\/orca/)
+ assert.match(warnings[0], /Deploy Relay Production Capacity/)
+ assert.match(warnings[0], /actions\/runs\/9001/)
+ const upload = storage.state.requests.find((request) => request.path.startsWith('/upload/'))
+ assert.equal(upload.search.get('ifGenerationMatch'), '1500')
+ assert.equal(storage.state.object.body.holder_key, HOLDER.holderKey)
+})
+
+test('releases with a generation match and leaves the object gone', async () => {
+ const storage = fakeStorage()
+ const lease = leaseFor(storage)
+ const claim = await lease.acquire(HOLDER)
+
+ const released = await lease.release(HOLDER.holderKey)
+
+ assert.deepEqual(released, { released: true, generation: claim.generation })
+ const remove = storage.state.requests.find((request) => request.method === 'DELETE')
+ assert.equal(remove.search.get('ifGenerationMatch'), claim.generation)
+ assert.equal(storage.state.object, null)
+})
+
+test('refuses to release a lease another run now holds', async () => {
+ const storage = fakeStorage({
+ object: {
+ generation: '1500',
+ body: storedRecord({ holderKey: 'stablyai/orca/9001', expiresAt: NOW + 60_000 })
+ }
+ })
+
+ const released = await leaseFor(storage).release(HOLDER.holderKey)
+
+ assert.equal(released.released, false)
+ assert.equal(released.reason, 'foreign')
+ assert.equal(storage.state.object.generation, '1500')
+})
+
+test('fails closed when the bucket answers 5xx', async () => {
+ const storage = fakeStorage({
+ faults: [{ when: (request) => request.method === 'GET', status: 503 }]
+ })
+
+ const error = await leaseFor(storage)
+ .acquire(HOLDER)
+ .catch((thrown) => thrown)
+
+ assert.match(error.message, /lease inspection failed: 503/)
+ assert.equal(storage.state.requests.filter((request) => request.method !== 'GET').length, 0)
+})
+
+test('fails closed when permission is denied', async () => {
+ const storage = fakeStorage({
+ faults: [{ when: (request) => request.method === 'GET', status: 403 }]
+ })
+
+ const error = await leaseFor(storage)
+ .acquire(HOLDER)
+ .catch((thrown) => thrown)
+
+ assert.match(error.message, /lease inspection failed: 403/)
+})
+
+test('treats an unreadable record as held, not free', async () => {
+ const storage = fakeStorage({
+ object: { generation: '1500', body: { holder_key: 'stablyai/orca/9001' } }
+ })
+
+ const error = await leaseFor(storage)
+ .acquire(HOLDER)
+ .catch((thrown) => thrown)
+
+ assert.ok(error instanceof LeaseUnreadable)
+ assert.equal(storage.state.object.generation, '1500')
+})
+
+test('reports a 412 during acquisition as a conflict', async () => {
+ const storage = fakeStorage({
+ faults: [{ when: (request) => request.path.startsWith('/upload/'), status: 412 }]
+ })
+
+ const error = await leaseFor(storage)
+ .acquire(HOLDER)
+ .catch((thrown) => thrown)
+
+ assert.ok(error instanceof LeaseConflict)
+ assert.match(error.message, /changed concurrently/)
+})
+
+test('renewal rewrites only expires_at on the observed generation', async () => {
+ const storage = fakeStorage()
+ const lease = leaseFor(storage)
+ await lease.acquire(HOLDER)
+ storage.state.object.body.expires_at = NOW - 1
+
+ const renewed = await lease.renew(HOLDER.holderKey)
+
+ assert.equal(renewed.renewed, true)
+ assert.equal(storage.state.object.body.expires_at, NOW + LEASE_TTL_MS)
+ assert.equal(storage.state.object.body.acquired_at, NOW)
+ assert.equal(storage.state.object.body.holder_key, HOLDER.holderKey)
+})
+
+test('renewal stops once the object belongs to someone else', async () => {
+ const storage = fakeStorage({
+ object: {
+ generation: '1500',
+ body: storedRecord({ holderKey: 'stablyai/orca/9001', expiresAt: NOW + 60_000 })
+ }
+ })
+
+ assert.deepEqual(await leaseFor(storage).renew(HOLDER.holderKey), {
+ renewed: false,
+ reason: 'foreign'
+ })
+})
+
+test('the access token source re-mints only after the reuse window', () => {
+ let clock = 0
+ let mints = 0
+ const source = createAccessTokenSource({
+ run: () => `token-${++mints}`,
+ now: () => clock
+ })
+
+ assert.equal(source(), 'token-1')
+ clock = 39 * 60 * 1_000
+ assert.equal(source(), 'token-1')
+ clock = 41 * 60 * 1_000
+ assert.equal(source(), 'token-2')
+})
+
+test('the access token source rejects an empty gcloud response', () => {
+ const source = createAccessTokenSource({ run: () => '' })
+ assert.throws(() => source(), /empty token/)
+})
diff --git a/.github/actions/install-node-dependencies/action.yml b/.github/actions/install-node-dependencies/action.yml
index 46edfc54111..7695d2bec9b 100644
--- a/.github/actions/install-node-dependencies/action.yml
+++ b/.github/actions/install-node-dependencies/action.yml
@@ -39,6 +39,9 @@ runs:
with:
install: false
+ # Why both lockfiles: setup-node keys the pnpm store on the root lockfile alone, so
+ # jobs that also install mobile restored a store with none of the React Native tree
+ # in it and re-downloaded the lot on every run.
- name: Setup Node.js
id: default-node
if: inputs.node-version == ''
@@ -46,6 +49,9 @@ runs:
with:
node-version-file: package.json
cache: pnpm
+ cache-dependency-path: |
+ pnpm-lock.yaml
+ mobile/pnpm-lock.yaml
- name: Setup requested Node.js
id: requested-node
@@ -54,6 +60,9 @@ runs:
with:
node-version: ${{ inputs.node-version }}
cache: pnpm
+ cache-dependency-path: |
+ pnpm-lock.yaml
+ mobile/pnpm-lock.yaml
- name: Validate native runtime
shell: bash
@@ -68,14 +77,6 @@ runs:
;;
esac
- # pnpm's bundled gyp_main.py is not executable on fresh Linux runners.
- - name: Use external node-gyp
- if: runner.os == 'Linux' && inputs.native-runtime != 'none'
- shell: bash
- run: |
- npm install -g node-gyp@11.5.0
- echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV"
-
- name: Prepare dependency install
shell: bash
run: |
@@ -166,6 +167,22 @@ runs:
node_modules/.pnpm/@vscode+windows-process-tree@*/node_modules/@vscode/windows-process-tree/build
key: native-modules-${{ runner.os }}-${{ steps.native-cache-scope.outputs.scope }}-${{ runner.arch }}-${{ inputs.native-runtime }}-node${{ steps.requested-node.outputs.node-version || steps.default-node.outputs.node-version }}-${{ hashFiles('pnpm-lock.yaml', '.github/actions/install-node-dependencies/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/rebuild-native-deps.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch') }}
+ # pnpm's bundled gyp_main.py is not executable on fresh Linux runners.
+ - name: Use external node-gyp
+ if: runner.os == 'Linux' && inputs.native-runtime != 'none'
+ shell: bash
+ env:
+ NATIVE_RUNTIME: ${{ inputs.native-runtime }}
+ NATIVE_CACHE_HIT: ${{ steps.native-cache-restore.outputs.cache-hit || steps.native-cache-restore-only.outputs.cache-hit }}
+ run: |
+ # A cache hit can contain unusable addons; probe before skipping the rebuild toolchain.
+ if [ "$NATIVE_RUNTIME" = node ] && [ "$NATIVE_CACHE_HIT" = true ] &&
+ node config/scripts/ensure-native-runtime.mjs --check-only; then
+ exit 0
+ fi
+ npm install -g node-gyp@11.5.0
+ echo "npm_config_node_gyp=$(npm root -g)/node-gyp/bin/node-gyp.js" >> "$GITHUB_ENV"
+
- name: Prepare native runtime
if: inputs.native-runtime != 'none'
shell: bash
diff --git a/.github/scripts/check-root-directory-entries.mjs b/.github/scripts/check-root-directory-entries.mjs
index 5e7dcf3f0ff..5b63326691d 100644
--- a/.github/scripts/check-root-directory-entries.mjs
+++ b/.github/scripts/check-root-directory-entries.mjs
@@ -13,6 +13,10 @@ function readRootEntries(sha) {
return stdout.split('\0').filter(Boolean)
}
+// Why: the Cloud workspace import is the one reviewed root addition; it stays
+// listed until it lands on main, after which the base tree carries it.
+const REVIEWED_ROOT_ENTRIES = new Set(['cloud'])
+
function checkRootDirectoryEntries(argv) {
if (argv.length !== 2) {
console.error(`Usage: ${process.argv[1]} `)
@@ -21,7 +25,9 @@ function checkRootDirectoryEntries(argv) {
const [baseSha, headSha] = argv
const baseEntries = new Set(readRootEntries(baseSha))
- const blockedEntries = readRootEntries(headSha).filter((entry) => !baseEntries.has(entry))
+ const blockedEntries = readRootEntries(headSha).filter(
+ (entry) => !baseEntries.has(entry) && !REVIEWED_ROOT_ENTRIES.has(entry)
+ )
if (blockedEntries.length === 0) {
console.log('Root directory guard passed: no new root-level files or folders.')
diff --git a/.github/workflows/cloud-bootstrap-relay-staging-capacity.yml b/.github/workflows/cloud-bootstrap-relay-staging-capacity.yml
new file mode 100644
index 00000000000..29e68510b21
--- /dev/null
+++ b/.github/workflows/cloud-bootstrap-relay-staging-capacity.yml
@@ -0,0 +1,676 @@
+name: Bootstrap Relay Staging Capacity
+
+on:
+ workflow_dispatch:
+ inputs:
+ confirmation:
+ description: Enter BOOTSTRAP_STAGING_CAPACITY
+ required: true
+ type: string
+
+permissions:
+ contents: read
+ id-token: write
+
+concurrency:
+ group: relay-staging-mutation
+ cancel-in-progress: false
+
+defaults:
+ run:
+ working-directory: cloud
+
+jobs:
+ bootstrap:
+ if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ environment: staging
+ env:
+ GCP_PROJECT_ID: onorca-cloud-staging
+ GCP_REGION: us-central1
+ DIRECTOR_ORIGIN: https://relay-staging.onorca.dev
+ CAPACITY_SERVICE_ACCOUNT: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
+ LEGACY_C3_IMAGE_DIGEST: sha256:2d0f6e6db2b0eb9d6aba188698de8330f8c30b4e76badfcf0fac3f3eb9508a87
+ steps:
+ - uses: actions/checkout@v4
+
+ - id: deploy-auth
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - id: capacity-auth
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
+ token_format: access_token
+
+ - uses: google-github-actions/setup-gcloud@v2
+
+ - uses: ./.github/actions/cloud-sql-rollout-lease
+ with:
+ bucket: onorca-cloud-staging-terraform-state
+ object: terraform/state/cloud-sql-rollout/staging.lock
+
+ - uses: hashicorp/setup-terraform@v3
+ with:
+ terraform_wrapper: false
+
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 24
+
+ - name: Require explicit bootstrap confirmation
+ env:
+ CONFIRMATION: ${{ inputs.confirmation }}
+ run: test "${CONFIRMATION}" = "BOOTSTRAP_STAGING_CAPACITY"
+
+ - name: Read and verify reviewed 600/60 topology
+ shell: bash
+ run: |
+ node dev/scripts/infra.mjs init --env staging
+ terraform -chdir=infra/terraform output -json relay_gce_cell_deployments \
+ > "${RUNNER_TEMP}/relay-gce-state.json"
+ DESIRED_CELLS_JSON="$(terraform -chdir=infra/terraform console \
+ -var-file=environments/staging.tfvars \
+ <<< 'local.relay_director_cells_json' | jq -r '.')"
+ DESIRED_IMAGES_JSON="$(terraform -chdir=infra/terraform console \
+ -var-file=environments/staging.tfvars \
+ <<< 'jsonencode({ for cell_id, cell in var.relay_gce_cells : cell_id => cell.image })' \
+ | jq -r '.')"
+ DESIRED_ZONES_JSON="$(terraform -chdir=infra/terraform console \
+ -var-file=environments/staging.tfvars \
+ <<< 'jsonencode({ for cell_id, cell in var.relay_gce_cells : cell_id => cell.zone })' \
+ | jq -r '.')"
+ DESIRED_TARGET_SIZES_JSON="$(terraform -chdir=infra/terraform console \
+ -var-file=environments/staging.tfvars \
+ <<< 'jsonencode(local.relay_gce_cell_target_sizes)' | jq -r '.')"
+ jq -e \
+ --argjson images "${DESIRED_IMAGES_JSON}" \
+ --argjson target_sizes "${DESIRED_TARGET_SIZES_JSON}" \
+ '([.[] | select(.id == "staging-gce-c2")] | length == 1) and
+ ([.[] | select(.id == "staging-gce-c3")] | length == 1) and
+ (any(.[]; .id == "staging-gce-c2" and .connectionHardCap == 600 and .connectionUnobservedBound == 60)) and
+ (any(.[]; .id == "staging-gce-c3" and .connectionHardCap == 600 and .connectionUnobservedBound == 60)) and
+ ($images["staging-gce-c2"] == $images["staging-gce-c3"]) and
+ ($target_sizes["staging-gce-c2"] == 1) and
+ ($target_sizes["staging-gce-c3"] == 1)' \
+ <<< "${DESIRED_CELLS_JSON}" >/dev/null
+ jq \
+ --argjson desired "${DESIRED_CELLS_JSON}" \
+ --argjson images "${DESIRED_IMAGES_JSON}" \
+ --argjson zones "${DESIRED_ZONES_JSON}" \
+ '($desired | map({key: .id, value: .}) | from_entries) as $cells |
+ to_entries | map(. as $entry | {
+ key: $entry.key,
+ value: ($entry.value + {
+ origin: $cells[$entry.key].url,
+ image: $images[$entry.key],
+ zone: $zones[$entry.key],
+ connection_hard_cap: $cells[$entry.key].connectionHardCap,
+ connection_unobserved_bound: $cells[$entry.key].connectionUnobservedBound
+ })
+ }) | from_entries' \
+ "${RUNNER_TEMP}/relay-gce-state.json" \
+ > "${RUNNER_TEMP}/relay-gce-topology.json"
+ ACTIVE_REVISION="$(gcloud run services describe orca-cloud-relay-staging \
+ --project "${GCP_PROJECT_ID}" \
+ --region us-central1 \
+ --format=json \
+ | jq -r '
+ [.status.traffic[] | select((.percent // 0) > 0)] |
+ if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
+ test -n "${ACTIVE_REVISION}"
+ DESIRED_IMAGE="$(jq -r '.["staging-gce-c2"]' <<< "${DESIRED_IMAGES_JSON}")"
+ gcloud run revisions describe "${ACTIVE_REVISION}" \
+ --project "${GCP_PROJECT_ID}" \
+ --region us-central1 \
+ --format=json \
+ | jq -e \
+ --arg image "${DESIRED_IMAGE}" \
+ --arg capacity_service_account "${CAPACITY_SERVICE_ACCOUNT}" \
+ '(.spec.containers[0].image == $image) and
+ any(.spec.containers[0].env[]?;
+ .name == "ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT" and
+ .value == $capacity_service_account)' >/dev/null
+ CURRENT_CELLS_JSON="$(gcloud run revisions describe "${ACTIVE_REVISION}" \
+ --project "${GCP_PROJECT_ID}" \
+ --region "${GCP_REGION}" \
+ --format=json \
+ | jq -cer '
+ [.spec.containers[0].env[]? |
+ select(.name == "ORCA_RELAY_CELLS_JSON") | .value] |
+ if length == 1 then .[0] | fromjson else error("missing director topology") end')"
+ jq -e --argjson desired "${DESIRED_CELLS_JSON}" '
+ def without_bootstrap_capacity:
+ map(if .id == "staging-gce-c2" or .id == "staging-gce-c3"
+ then del(.connectionHardCap, .connectionUnobservedBound)
+ else . end);
+ without_bootstrap_capacity == ($desired | without_bootstrap_capacity)
+ ' <<< "${CURRENT_CELLS_JSON}" >/dev/null
+
+ - name: Bootstrap C2 then C3
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
+ shell: bash
+ run: |
+ set -euo pipefail
+
+ topology="${RUNNER_TEMP}/relay-gce-topology.json"
+
+ fixed_one_instance_name() {
+ local cell_id="$1"
+ local mig_name zone
+ mig_name="$(jq -r --arg cell "${cell_id}" '.[$cell].mig_name' "${topology}")"
+ zone="$(jq -r --arg cell "${cell_id}" '.[$cell].zone' "${topology}")"
+ gcloud compute instance-groups managed list-instances \
+ "${mig_name}" \
+ --project "${GCP_PROJECT_ID}" \
+ --zone "${zone}" \
+ --format=json \
+ | jq -er '
+ if length == 1 and .[0].instanceStatus == "RUNNING" and
+ .[0].currentAction == "NONE"
+ then .[0].instance | split("/") | last
+ else error("legacy cell does not have one stable running instance") end'
+ }
+
+ fixed_one_instance_id() {
+ local cell_id="$1"
+ local instance_name="$2"
+ local zone
+ zone="$(jq -r --arg cell "${cell_id}" '.[$cell].zone' "${topology}")"
+ gcloud compute instances describe "${instance_name}" \
+ --project "${GCP_PROJECT_ID}" \
+ --zone "${zone}" \
+ --format='value(id)'
+ }
+
+ write_legacy_metrics() {
+ local cell_id="$1"
+ local instance_id="$2"
+ local after="$3"
+ local output="$4"
+ gcloud logging read \
+ "resource.type=\"gce_instance\" AND
+ resource.labels.instance_id=\"${instance_id}\" AND
+ jsonPayload.event=\"orca_relay_runtime_metrics\" AND
+ jsonPayload.cellId=\"${cell_id}\" AND
+ timestamp>=\"${after}\"" \
+ --project "${GCP_PROJECT_ID}" \
+ --limit 10 \
+ --order desc \
+ --format json \
+ | jq '[.[] | {
+ timestamp,
+ cellId: .jsonPayload.cellId,
+ metricVersion: .jsonPayload.metricVersion,
+ totalConnections: .jsonPayload.totalConnections,
+ preAuthConnections: .jsonPayload.preAuthConnections,
+ controls: .jsonPayload.controls,
+ splices: .jsonPayload.splices,
+ pendingSplices: .jsonPayload.pendingSplices,
+ queuedBytes: .jsonPayload.queuedBytes
+ }]' > "${output}"
+ }
+
+ verify_legacy_cell() {
+ local cell_id="$1"
+ local admission="$2"
+ local after="$3"
+ local expected_instance_id="$4"
+ local runtime_started_after="${5:-}"
+ local previous_incarnation_digest="${6:-}"
+ local hard_cap="${7:-}"
+ local unobserved_bound="${8:-}"
+ local capacity_state="${9:-}"
+ local current_instance current_instance_id metrics origin result
+ local runtime_start_args=() incarnation_args=() capacity_args=()
+ current_instance="$(fixed_one_instance_name "${cell_id}")"
+ current_instance_id="$(fixed_one_instance_id "${cell_id}" "${current_instance}")"
+ test "${current_instance_id}" = "${expected_instance_id}"
+ metrics="${RUNNER_TEMP}/${cell_id}-legacy-runtime-metrics.json"
+ origin="$(jq -r --arg cell "${cell_id}" '.[$cell].origin' "${topology}")"
+ if test -n "${runtime_started_after}"; then
+ runtime_start_args=(--runtime-started-after "${runtime_started_after}")
+ fi
+ if test -n "${previous_incarnation_digest}"; then
+ incarnation_args=(--previous-incarnation-digest "${previous_incarnation_digest}")
+ fi
+ if test -n "${hard_cap}"; then
+ capacity_args=(--hard-cap "${hard_cap}" --unobserved-bound "${unobserved_bound}")
+ fi
+ if test -n "${capacity_state}"; then
+ capacity_args+=(--capacity-state "${capacity_state}")
+ fi
+ for _attempt in $(seq 1 18); do
+ write_legacy_metrics \
+ "${cell_id}" "${current_instance_id}" "${after}" "${metrics}"
+ if result="$(node dev/scripts/verify-relay-legacy-bootstrap.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${origin}" \
+ --cell-id "${cell_id}" \
+ --admission "${admission}" \
+ --expected-image-digest "${LEGACY_C3_IMAGE_DIGEST}" \
+ --metrics-after "${after}" \
+ --metrics-file "${metrics}" \
+ "${runtime_start_args[@]}" \
+ "${incarnation_args[@]}" \
+ "${capacity_args[@]}")"; then
+ echo "${result}"
+ return 0
+ fi
+ sleep 10
+ done
+ return 1
+ }
+
+ post_admin() {
+ local origin="$1"
+ local path="$2"
+ local body="$3"
+ curl --fail --silent --show-error \
+ --request POST \
+ --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
+ --header 'Content-Type: application/json' \
+ --data "${body}" \
+ "${origin}${path}"
+ }
+
+ runtime_kind() {
+ local cell_id="$1"
+ local origin desired_digest digest
+ origin="$(jq -r --arg cell "${cell_id}" '.[$cell].origin' "${topology}")"
+ desired_digest="$(jq -r --arg cell "${cell_id}" \
+ '.[$cell].image | split("@") | last' "${topology}")"
+ digest="$(post_admin "${origin}" /v1/admin/runtime-status '{"v":1}' \
+ | jq -er '.imageDigest')"
+ if test "${digest}" = "${LEGACY_C3_IMAGE_DIGEST}"; then
+ echo legacy
+ elif test "${digest}" = "${desired_digest}"; then
+ echo modern
+ else
+ echo 'bootstrap cell image is neither legacy nor reviewed' >&2
+ return 1
+ fi
+ }
+
+ cell_admission() {
+ local cell_id="$1"
+ post_admin "${DIRECTOR_ORIGIN}" /v1/admin/cell-status \
+ "$(jq -cn --arg cell "${cell_id}" '{v:1, cellId:$cell}')" \
+ | jq -er '.status.admissionState |
+ if . == "general" or . == "migration-only" then .
+ else error("bootstrap admission is not recoverable") end'
+ }
+
+ verify_modern_cell() {
+ local cell_id="$1"
+ local admission="$2"
+ local origin
+ origin="$(jq -r --arg cell "${cell_id}" '.[$cell].origin' "${topology}")"
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${origin}" \
+ --cell-id "${cell_id}" \
+ --hard-cap 600 \
+ --unobserved-bound 60 \
+ --heartbeat fresh \
+ --admission "${admission}" \
+ --draining forbidden \
+ --activity allowed
+ }
+
+ ensure_modern_general() {
+ local cell_id="$1"
+ local admission="$2"
+ verify_modern_cell "${cell_id}" "${admission}"
+ node dev/scripts/prepare-relay-capacity-canary.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-id "${cell_id}" \
+ --mode restore \
+ --general-cell-ids "${cell_id}"
+ verify_modern_cell "${cell_id}" general
+ }
+
+ prepare_legacy_c3_fallback() {
+ legacy_c3_metrics_boundary="$(node -e \
+ 'process.stdout.write(new Date(Date.now() - 120_000).toISOString())')"
+ legacy_c3_instance="$(fixed_one_instance_name staging-gce-c3)"
+ legacy_c3_instance_id="$(fixed_one_instance_id \
+ staging-gce-c3 "${legacy_c3_instance}")"
+ verify_legacy_cell \
+ staging-gce-c3 general \
+ "${legacy_c3_metrics_boundary}" "${legacy_c3_instance_id}"
+ node dev/scripts/probe-relay-legacy-admission.mjs \
+ --cell-origin https://c3.relay-staging.onorca.dev
+ legacy_c3_restart_started_after=
+ legacy_c3_old_incarnation=
+ }
+
+ normalize_legacy_c3() {
+ legacy_pre_boundary="$(node -e \
+ 'process.stdout.write(new Date(Date.now() - 120_000).toISOString())')"
+ legacy_c2_instance="$(fixed_one_instance_name staging-gce-c2)"
+ legacy_c2_instance_id="$(fixed_one_instance_id \
+ staging-gce-c2 "${legacy_c2_instance}")"
+ verify_legacy_cell \
+ staging-gce-c2 general "${legacy_pre_boundary}" "${legacy_c2_instance_id}"
+ node dev/scripts/probe-relay-legacy-admission.mjs \
+ --cell-origin https://c2.relay-staging.onorca.dev
+
+ legacy_c3_isolated=false
+ restore_legacy_c3_fallback() {
+ if test "${legacy_c3_isolated}" = true; then
+ verify_legacy_cell \
+ staging-gce-c2 general "${legacy_pre_boundary}" "${legacy_c2_instance_id}"
+ node dev/scripts/prepare-relay-capacity-canary.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-id staging-gce-c3 \
+ --mode restore-fallback \
+ --general-cell-ids staging-gce-c2
+ fi
+ }
+
+ trap restore_legacy_c3_fallback EXIT
+ legacy_c3_isolated=true
+ node dev/scripts/prepare-relay-capacity-canary.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin https://c3.relay-staging.onorca.dev \
+ --cell-id staging-gce-c3 \
+ --mode isolate
+ legacy_c3_drain_boundary="$(node -e \
+ 'process.stdout.write(new Date().toISOString())')"
+ legacy_c3_instance="$(fixed_one_instance_name staging-gce-c3)"
+ legacy_c3_instance_id="$(fixed_one_instance_id \
+ staging-gce-c3 "${legacy_c3_instance}")"
+ legacy_c3_drained="$(verify_legacy_cell \
+ staging-gce-c3 migration-only \
+ "${legacy_c3_drain_boundary}" "${legacy_c3_instance_id}")"
+ echo "${legacy_c3_drained}"
+ legacy_c3_old_incarnation="$(jq -er '.incarnationDigest' \
+ <<< "${legacy_c3_drained}")"
+ legacy_c3_restart_started_after="$(node -e \
+ 'process.stdout.write(new Date().toISOString())')"
+ gcloud compute instance-groups managed recreate-instances \
+ orca-cloud-staging-relay-gce-c3 \
+ --instances "${legacy_c3_instance}" \
+ --project "${GCP_PROJECT_ID}" \
+ --zone us-central1-a \
+ --quiet
+ gcloud compute instance-groups managed wait-until \
+ orca-cloud-staging-relay-gce-c3 \
+ --stable \
+ --project "${GCP_PROJECT_ID}" \
+ --zone us-central1-a \
+ --timeout 900
+ legacy_c3_instance="$(fixed_one_instance_name staging-gce-c3)"
+ legacy_c3_instance_id="$(fixed_one_instance_id \
+ staging-gce-c3 "${legacy_c3_instance}")"
+ legacy_c3_metrics_boundary="$(node -e \
+ 'process.stdout.write(new Date().toISOString())')"
+ verify_legacy_cell \
+ staging-gce-c3 migration-only \
+ "${legacy_c3_metrics_boundary}" "${legacy_c3_instance_id}" \
+ "${legacy_c3_restart_started_after}" "${legacy_c3_old_incarnation}"
+ node dev/scripts/prepare-relay-capacity-canary.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-id staging-gce-c3 \
+ --mode restore \
+ --general-cell-ids staging-gce-c2,staging-gce-c3
+ verify_legacy_cell \
+ staging-gce-c3 general \
+ "${legacy_c3_metrics_boundary}" "${legacy_c3_instance_id}" \
+ "${legacy_c3_restart_started_after}" "${legacy_c3_old_incarnation}"
+ legacy_c3_isolated=false
+ trap - EXIT
+ }
+
+ roll_cell() (
+ local cell_id="$1"
+ local fallback_cell_id="$2"
+ local target_kind="$3"
+ local fallback_kind="$4"
+ local active_revision cell_origin current_cells_json desired_bound desired_cap
+ local desired_cells_json director_result image mig_name plan
+ local fallback_origin plan_changes plan_result restored target_drain_boundary
+ local target_instance target_instance_id zone
+ cell_origin="$(jq -r --arg cell "${cell_id}" '.[$cell].origin' "${topology}")"
+ fallback_origin="$(jq -r --arg cell "${fallback_cell_id}" \
+ '.[$cell].origin' "${topology}")"
+ zone="$(jq -r --arg cell "${cell_id}" '.[$cell].zone' "${topology}")"
+ mig_name="$(jq -r --arg cell "${cell_id}" '.[$cell].mig_name' "${topology}")"
+ image="$(jq -r --arg cell "${cell_id}" '.[$cell].image' "${topology}")"
+ desired_cap="$(jq -r --arg cell "${cell_id}" \
+ '.[$cell].connection_hard_cap' "${topology}")"
+ desired_bound="$(jq -r --arg cell "${cell_id}" \
+ '.[$cell].connection_unobserved_bound' "${topology}")"
+ plan="${RUNNER_TEMP}/${cell_id}-capacity-bootstrap.tfplan"
+ restored=false
+
+ restore_fallback() {
+ if test "${restored}" = false; then
+ verify_fallback
+ node dev/scripts/prepare-relay-capacity-canary.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-id "${cell_id}" \
+ --mode restore-fallback \
+ --general-cell-ids "${fallback_cell_id}"
+ fi
+ }
+
+ verify_fallback() {
+ if test "${fallback_kind}" = legacy; then
+ verify_legacy_cell \
+ "${fallback_cell_id}" general \
+ "${legacy_c3_metrics_boundary}" "${legacy_c3_instance_id}" \
+ "${legacy_c3_restart_started_after}" "${legacy_c3_old_incarnation}"
+ return
+ fi
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${fallback_origin}" \
+ --cell-id "${fallback_cell_id}" \
+ --hard-cap 600 \
+ --unobserved-bound 60 \
+ --heartbeat fresh \
+ --admission general \
+ --draining forbidden \
+ --activity allowed
+ }
+
+ verify_fallback
+ trap restore_fallback EXIT
+ node dev/scripts/prepare-relay-capacity-canary.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${cell_origin}" \
+ --cell-id "${cell_id}" \
+ --mode isolate
+ target_drain_boundary="$(node -e \
+ 'process.stdout.write(new Date().toISOString())')"
+ if test "${target_kind}" = legacy; then
+ target_instance="$(fixed_one_instance_name "${cell_id}")"
+ target_instance_id="$(fixed_one_instance_id "${cell_id}" "${target_instance}")"
+ verify_legacy_cell \
+ "${cell_id}" migration-only \
+ "${target_drain_boundary}" "${target_instance_id}" \
+ '' '' "${desired_cap}" "${desired_bound}" absent-or-stale
+ else
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${cell_origin}" \
+ --cell-id "${cell_id}" \
+ --heartbeat either \
+ --admission migration-only \
+ --draining required \
+ --activity quiescent
+ fi
+
+ active_revision="$(gcloud run services describe orca-cloud-relay-staging \
+ --project "${GCP_PROJECT_ID}" \
+ --region "${GCP_REGION}" \
+ --format=json \
+ | jq -r '
+ [.status.traffic[] | select((.percent // 0) > 0)] |
+ if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
+ test -n "${active_revision}"
+ current_cells_json="$(gcloud run revisions describe "${active_revision}" \
+ --project "${GCP_PROJECT_ID}" \
+ --region "${GCP_REGION}" \
+ --format=json \
+ | jq -cer '
+ [.spec.containers[0].env[]? |
+ select(.name == "ORCA_RELAY_CELLS_JSON") | .value] |
+ if length == 1 then .[0] | fromjson else error("missing director topology") end')"
+ desired_cells_json="$(jq -ce \
+ --arg cell "${cell_id}" \
+ --argjson cap "${desired_cap}" \
+ --argjson bound "${desired_bound}" \
+ 'map(if .id == $cell then . + {
+ connectionHardCap: $cap,
+ connectionUnobservedBound: $bound
+ } else . end)' <<< "${current_cells_json}")"
+ director_result="$(node dev/scripts/deploy-relay-blue-green.mjs \
+ --project "${GCP_PROJECT_ID}" \
+ --region "${GCP_REGION}" \
+ --service orca-cloud-relay-staging \
+ --image "${image}" \
+ --role director \
+ --capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
+ --capacity-cell-id "${cell_id}" \
+ --director-cells-json "${desired_cells_json}" \
+ --min-instances 0 \
+ --prune-revisions true \
+ --release-id "bootstrap-${cell_id}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}")"
+ echo "${director_result}"
+ if test "${target_kind}" = legacy; then
+ verify_legacy_cell \
+ "${cell_id}" migration-only \
+ "${target_drain_boundary}" "${target_instance_id}" \
+ '' '' "${desired_cap}" "${desired_bound}"
+ else
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${cell_origin}" \
+ --cell-id "${cell_id}" \
+ --hard-cap "${desired_cap}" \
+ --unobserved-bound "${desired_bound}" \
+ --heartbeat either \
+ --admission migration-only \
+ --draining required \
+ --activity quiescent
+ fi
+
+ terraform -chdir=infra/terraform plan \
+ -var-file=environments/staging.tfvars \
+ "-target=google_compute_instance_template.relay_gce_cell[\"${cell_id}\"]" \
+ "-target=google_compute_instance_group_manager.relay_gce_cell[\"${cell_id}\"]" \
+ -out="${plan}"
+ plan_result="$(terraform -chdir=infra/terraform show -json "${plan}" \
+ | node dev/scripts/validate-relay-capacity-plan.mjs \
+ --mode bootstrap-cell \
+ --cell-id "${cell_id}" \
+ --hard-cap 600 \
+ --unobserved-bound 60 \
+ --image "${image}" \
+ --capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}")"
+ echo "${plan_result}"
+ plan_changes="$(jq -r '.changes' <<< "${plan_result}")"
+ [[ "${plan_changes}" =~ ^(0|2)$ ]]
+ if test "${plan_changes}" = 2; then
+ terraform -chdir=infra/terraform apply -auto-approve "${plan}"
+ else
+ target_instance="$(fixed_one_instance_name "${cell_id}")"
+ gcloud compute instance-groups managed recreate-instances "${mig_name}" \
+ --instances "${target_instance}" \
+ --project "${GCP_PROJECT_ID}" \
+ --zone "${zone}" \
+ --quiet
+ fi
+ gcloud compute instance-groups managed wait-until "${mig_name}" \
+ --stable \
+ --project "${GCP_PROJECT_ID}" \
+ --zone "${zone}" \
+ --timeout 900
+
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${cell_origin}" \
+ --cell-id "${cell_id}" \
+ --hard-cap 600 \
+ --unobserved-bound 60 \
+ --heartbeat fresh \
+ --admission migration-only \
+ --draining forbidden \
+ --activity allowed
+ node dev/scripts/prepare-relay-capacity-canary.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-id "${cell_id}" \
+ --mode restore \
+ --general-cell-ids staging-gce-c2,staging-gce-c3
+ restored=true
+ trap - EXIT
+ )
+
+ c2_kind="$(runtime_kind staging-gce-c2)"
+ c3_kind="$(runtime_kind staging-gce-c3)"
+ c2_admission="$(cell_admission staging-gce-c2)"
+ c3_admission="$(cell_admission staging-gce-c3)"
+ bootstrap_phase="$(node dev/scripts/classify-relay-staging-bootstrap.mjs \
+ --c2-kind "${c2_kind}" \
+ --c2-admission "${c2_admission}" \
+ --c3-kind "${c3_kind}" \
+ --c3-admission "${c3_admission}")"
+ jq -cn --arg phase "${bootstrap_phase}" \
+ '{event:"relay_staging_bootstrap_phase", phase:$phase}'
+
+ case "${bootstrap_phase}" in
+ normalize-and-roll-both)
+ normalize_legacy_c3
+ roll_cell staging-gce-c2 staging-gce-c3 legacy legacy
+ roll_cell staging-gce-c3 staging-gce-c2 legacy modern
+ ;;
+ resume-c2-then-c3)
+ prepare_legacy_c3_fallback
+ roll_cell staging-gce-c2 staging-gce-c3 legacy legacy
+ roll_cell staging-gce-c3 staging-gce-c2 legacy modern
+ ;;
+ roll-c2)
+ ensure_modern_general staging-gce-c3 "${c3_admission}"
+ roll_cell staging-gce-c2 staging-gce-c3 legacy modern
+ ;;
+ roll-c3)
+ ensure_modern_general staging-gce-c2 "${c2_admission}"
+ roll_cell staging-gce-c3 staging-gce-c2 legacy modern
+ ;;
+ complete)
+ ensure_modern_general staging-gce-c2 "${c2_admission}"
+ ensure_modern_general staging-gce-c3 "${c3_admission}"
+ ;;
+ *)
+ echo 'unsupported staging bootstrap phase' >&2
+ exit 1
+ ;;
+ esac
+
+ - name: Verify both bootstrapped cells
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
+ run: |
+ for number in 2 3; do
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "https://c${number}.relay-staging.onorca.dev" \
+ --cell-id "staging-gce-c${number}" \
+ --hard-cap 600 \
+ --unobserved-bound 60 \
+ --heartbeat fresh \
+ --admission general \
+ --draining forbidden \
+ --activity allowed
+ done
diff --git a/.github/workflows/cloud-deploy-relay-asia-topology.yml b/.github/workflows/cloud-deploy-relay-asia-topology.yml
new file mode 100644
index 00000000000..62e5ebb1426
--- /dev/null
+++ b/.github/workflows/cloud-deploy-relay-asia-topology.yml
@@ -0,0 +1,243 @@
+name: Deploy Relay Asia Topology
+
+on:
+ workflow_dispatch:
+ inputs:
+ environment:
+ description: Target Relay environment
+ required: true
+ type: choice
+ options: [staging, production]
+ mode:
+ description: Validate a saved plan or apply that exact plan
+ required: true
+ default: plan
+ type: choice
+ options: [plan, apply]
+ cell-ids:
+ description: Exact reviewed comma-separated Asia cell set
+ required: true
+ type: string
+ image:
+ description: Full environment Relay image pinned by sha256 digest
+ required: true
+ type: string
+ confirmation:
+ description: Enter APPLY_RELAY_ASIA_TOPOLOGY for apply mode
+ required: false
+ type: string
+
+permissions:
+ contents: read
+ id-token: write
+
+concurrency:
+ group: ${{ inputs.environment == 'production' && 'production-cloud-sql-rollout' || 'relay-staging-mutation' }}
+ cancel-in-progress: false
+
+defaults:
+ run:
+ working-directory: cloud
+
+jobs:
+ topology:
+ if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }}
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ timeout-minutes: 30
+ environment: ${{ inputs.environment }}
+ env:
+ DEPLOY_MODE: ${{ inputs.mode }}
+ TARGET_ENVIRONMENT: ${{ inputs.environment }}
+ TARGET_CELL_IDS: ${{ inputs.cell-ids }}
+ TARGET_IMAGE: ${{ inputs.image }}
+ TARGET_REGION: asia-east2
+ GCP_PROJECT_ID: ${{ inputs.environment == 'production' && 'onorca-cloud' || 'onorca-cloud-staging' }}
+ CLOUD_SQL_INSTANCE: ${{ inputs.environment == 'production' && 'orca-cloud-auth-db' || 'orca-cloud-staging-auth-db' }}
+ VERIFIED_DEFAULT_MAX_CONNECTIONS_TIER: db-custom-4-15360
+ VERIFIED_DEFAULT_MAX_CONNECTIONS_DATABASE_VERSION: POSTGRES_17
+ TF_BACKEND: ${{ inputs.environment == 'production' && 'backend/production.hcl' || 'backend/staging.hcl' }}
+ TF_VARS: ${{ inputs.environment == 'production' && 'environments/production.tfvars' || 'environments/staging.tfvars' }}
+ TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_RELAY_ASIA_TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER || vars.STAGING_GCP_RELAY_ASIA_TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER }}
+ TOPOLOGY_SERVICE_ACCOUNT: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_RELAY_ASIA_TOPOLOGY_SERVICE_ACCOUNT || vars.STAGING_GCP_RELAY_ASIA_TOPOLOGY_SERVICE_ACCOUNT }}
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Validate the reviewed request before authentication
+ shell: bash
+ env:
+ CONFIRMATION: ${{ inputs.confirmation }}
+ run: |
+ set -euo pipefail
+ test -n "${TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER}"
+ test -n "${TOPOLOGY_SERVICE_ACCOUNT}"
+ case "${TARGET_ENVIRONMENT}:${TARGET_CELL_IDS}" in
+ staging:staging-gce-c4) ;;
+ production:production-gce-c27,production-gce-c28,production-gce-c29) ;;
+ *) echo "cell-ids do not match the reviewed environment topology" >&2; exit 1 ;;
+ esac
+ [[ "${TARGET_IMAGE}" =~ ^us-central1-docker\.pkg\.dev/${GCP_PROJECT_ID}/orca-cloud/relay@sha256:[0-9a-f]{64}$ ]]
+ if test "${DEPLOY_MODE}" = apply; then
+ test "${CONFIRMATION}" = APPLY_RELAY_ASIA_TOPOLOGY
+ else
+ test "${DEPLOY_MODE}" = plan
+ test -z "${CONFIRMATION}"
+ fi
+
+ - uses: hashicorp/setup-terraform@v3
+ with:
+ terraform_version: 1.15.8
+ terraform_wrapper: false
+
+ - uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ env.TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ env.TOPOLOGY_SERVICE_ACCOUNT }}
+
+ - uses: google-github-actions/setup-gcloud@v2
+
+ - uses: ./.github/actions/cloud-sql-rollout-lease
+ with:
+ bucket: ${{ inputs.environment == 'production' && 'onorca-cloud-terraform-state' || 'onorca-cloud-staging-terraform-state' }}
+ object: ${{ inputs.environment == 'production' && 'terraform/state/cloud-sql-rollout/production.lock' || 'terraform/state/cloud-sql-rollout/staging.lock' }}
+
+ - name: Require the checked Cloud SQL connection budget
+ shell: bash
+ run: |
+ set -euo pipefail
+ budget="$(node dev/scripts/relay-cloud-sql-connection-budget.mjs)"
+ checked_max="$(jq -er '.maxConnections' <<< "${budget}")"
+ jq -e '.withinBudget == true' <<< "${budget}" >/dev/null
+ if test "${TARGET_ENVIRONMENT}" = production; then
+ instance="$(gcloud sql instances describe "${CLOUD_SQL_INSTANCE}" \
+ --project "${GCP_PROJECT_ID}" --format=json)"
+ live_flag="$(jq -er '[.settings.databaseFlags[]? |
+ select(.name == "max_connections") | .value] |
+ if length <= 1 then (.[0] // "") else error("duplicate max_connections flags") end' \
+ <<< "${instance}")"
+ if test -n "${live_flag}"; then
+ live_max="${live_flag}"
+ live_source=explicit-flag
+ else
+ # The verified production database uses Cloud SQL's 400-connection
+ # default for this exact shape; fail closed if its shape changes.
+ test "$(jq -er '.settings.tier' <<< "${instance}")" = \
+ "${VERIFIED_DEFAULT_MAX_CONNECTIONS_TIER}"
+ test "$(jq -er '.databaseVersion' <<< "${instance}")" = \
+ "${VERIFIED_DEFAULT_MAX_CONNECTIONS_DATABASE_VERSION}"
+ live_max=400
+ live_source=verified-shape-default
+ fi
+ test "${live_max}" = "${checked_max}"
+ else
+ live_max="not-read-for-staging"
+ live_source=not-read-for-staging
+ fi
+ {
+ echo "### Relay Cloud SQL connection budget"
+ echo "- Checked maximum: ${checked_max}"
+ echo "- Configured maximum: $(jq -er '.configuredMaximum' <<< "${budget}")"
+ echo "- Rollout operating maximum: $(jq -er '.operatingMaximum' <<< "${budget}")"
+ echo "- Explicit reserve: $(jq -er '.explicitReserve' <<< "${budget}")"
+ echo "- Production live max_connections: ${live_max}"
+ echo "- Production live maximum source: ${live_source}"
+ } >> "${GITHUB_STEP_SUMMARY}"
+
+ - name: Initialize the exact environment state
+ run: terraform -chdir=infra/terraform init -reconfigure -input=false -backend-config="${TF_BACKEND}"
+
+ - id: targets
+ name: Build the exact additive target set
+ shell: bash
+ run: |
+ set -euo pipefail
+ file="${RUNNER_TEMP}/relay-asia-targets"
+ : > "${file}"
+ printf '%s\n' \
+ '-target=google_compute_subnetwork.relay_gce_additional["asia-east2"]' \
+ '-target=google_compute_router.relay_gce_additional["asia-east2"]' \
+ '-target=google_compute_router_nat.relay_gce_additional["asia-east2"]' \
+ '-target=google_compute_url_map.relay_gce[0]' >> "${file}"
+ IFS=, read -ra cells <<< "${TARGET_CELL_IDS}"
+ for cell_id in "${cells[@]}"; do
+ printf '%s\n' \
+ "-target=google_compute_instance_template.relay_gce_cell[\"${cell_id}\"]" \
+ "-target=google_compute_instance_group_manager.relay_gce_cell[\"${cell_id}\"]" \
+ "-target=google_compute_backend_service.relay_gce_cell[\"${cell_id}\"]" >> "${file}"
+ done
+ echo "file=${file}" >> "${GITHUB_OUTPUT}"
+
+ - name: Create and validate the saved topology plan
+ id: plan
+ shell: bash
+ run: |
+ set -euo pipefail
+ plan="${RUNNER_TEMP}/relay-asia-topology.tfplan"
+ plan_json="${RUNNER_TEMP}/relay-asia-topology.json"
+ mapfile -t targets < "${{ steps.targets.outputs.file }}"
+ terraform -chdir=infra/terraform plan -input=false -lock-timeout=30s \
+ -var-file="${TF_VARS}" \
+ "${targets[@]}" -out="${plan}"
+ terraform -chdir=infra/terraform show -json "${plan}" > "${plan_json}"
+ committed="${RUNNER_TEMP}/relay-committed-asia-topology.json"
+ jq -e '{
+ relay_gce_cells: .variables.relay_gce_cells.value,
+ relay_gce_additional_region_subnetwork_cidrs:
+ .variables.relay_gce_additional_region_subnetwork_cidrs.value
+ }' "${plan_json}" > "${committed}"
+ node dev/scripts/prepare-relay-asia-topology-input.mjs \
+ --existing-json "${committed}" \
+ --environment "${TARGET_ENVIRONMENT}" \
+ --cell-ids "${TARGET_CELL_IDS}" \
+ --image "${TARGET_IMAGE}"
+ result="$(node dev/scripts/validate-relay-asia-topology-plan.mjs \
+ --plan-json "${plan_json}" \
+ --environment "${TARGET_ENVIRONMENT}" \
+ --cell-ids "${TARGET_CELL_IDS}" \
+ --region "${TARGET_REGION}" \
+ --image "${TARGET_IMAGE}")"
+ changes="$(jq -er '.changes' <<< "${result}")"
+ digest="$(sha256sum "${plan}" | awk '{print $1}')"
+ echo "plan=${plan}" >> "${GITHUB_OUTPUT}"
+ echo "changes=${changes}" >> "${GITHUB_OUTPUT}"
+ {
+ echo "### Relay Asia topology saved plan"
+ echo "- Environment: ${TARGET_ENVIRONMENT}"
+ echo "- Cells: ${TARGET_CELL_IDS}"
+ echo "- Region: ${TARGET_REGION}"
+ echo "- Mutating resources: ${changes}"
+ echo "- Saved-plan SHA-256: ${digest}"
+ } >> "${GITHUB_STEP_SUMMARY}"
+
+ - name: Apply only the validated saved plan
+ if: ${{ inputs.mode == 'apply' }}
+ run: terraform -chdir=infra/terraform apply -input=false -auto-approve "${{ steps.plan.outputs.plan }}"
+
+ - name: Prove the exact topology targets converged
+ if: ${{ inputs.mode == 'apply' }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ mapfile -t targets < "${{ steps.targets.outputs.file }}"
+ plan="${RUNNER_TEMP}/relay-asia-topology-readback.tfplan"
+ plan_json="${RUNNER_TEMP}/relay-asia-topology-readback.json"
+ terraform -chdir=infra/terraform plan -input=false -lock-timeout=30s \
+ -var-file="${TF_VARS}" \
+ "${targets[@]}" -out="${plan}"
+ terraform -chdir=infra/terraform show -json "${plan}" > "${plan_json}"
+ result="$(node dev/scripts/validate-relay-asia-topology-plan.mjs \
+ --plan-json "${plan_json}" \
+ --environment "${TARGET_ENVIRONMENT}" \
+ --cell-ids "${TARGET_CELL_IDS}" \
+ --region "${TARGET_REGION}" \
+ --image "${TARGET_IMAGE}")"
+ test "$(jq -er '.changes' <<< "${result}")" = 0
+
+ - name: Record the required selector-safe next step
+ if: ${{ inputs.mode == 'apply' }}
+ run: |
+ {
+ echo "### Required next step"
+ echo "The VMs are not eligible for ordinary placement yet."
+ echo "Register the exact new cells atomically as migration-only before any director configuration lists them."
+ echo "Rollback is migration-only admission; do not destroy the Asia network on rollout day."
+ } >> "${GITHUB_STEP_SUMMARY}"
diff --git a/.github/workflows/cloud-deploy-relay-fence-broker.yml b/.github/workflows/cloud-deploy-relay-fence-broker.yml
new file mode 100644
index 00000000000..624bac69a88
--- /dev/null
+++ b/.github/workflows/cloud-deploy-relay-fence-broker.yml
@@ -0,0 +1,90 @@
+name: Deploy Relay Fence Broker
+
+on:
+ workflow_dispatch:
+ inputs:
+ image-digest:
+ description: Immutable broker image digest built from this main commit
+ required: true
+ type: string
+
+permissions:
+ contents: read
+ id-token: write
+
+concurrency:
+ group: production-cloud-sql-rollout
+ cancel-in-progress: false
+
+defaults:
+ run:
+ working-directory: cloud
+
+jobs:
+ deploy:
+ if: >-
+ ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' &&
+ github.ref == 'refs/heads/main' }}
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ environment: production
+ env:
+ GCP_PROJECT_ID: onorca-cloud
+ GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
+ SERVICE_NAME: orca-cloud-relay-fence
+ IMAGE_REPOSITORY: us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay-fence-broker
+ IMAGE_DIGEST: ${{ inputs.image-digest }}
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+
+ - uses: google-github-actions/setup-gcloud@v2
+
+ - uses: ./.github/actions/cloud-sql-rollout-lease
+ with:
+ bucket: onorca-cloud-terraform-state
+ object: terraform/state/cloud-sql-rollout/production.lock
+
+ - name: Resolve exact-commit broker image
+ run: |
+ [[ "${IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]
+ IMAGE="${IMAGE_REPOSITORY}@${IMAGE_DIGEST}"
+ SERVED_DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" \
+ --format='value(image_summary.digest)')"
+ test "${SERVED_DIGEST}" = "${IMAGE_DIGEST}"
+ TAGS="$(gcloud artifacts docker tags list "${IMAGE_REPOSITORY}" \
+ --filter="version:${IMAGE_DIGEST}" \
+ --format=json)"
+ jq -e --arg tag "/tags/sha-${GITHUB_SHA}" \
+ 'any(.[]; .tag | endswith($tag))' <<< "${TAGS}"
+ echo "IMAGE=${IMAGE}" >> "${GITHUB_ENV}"
+
+ - name: Deploy broker image only
+ run: |
+ gcloud run services update "${SERVICE_NAME}" \
+ --project "${GCP_PROJECT_ID}" \
+ --region "${GCP_REGION}" \
+ --image "${IMAGE}" \
+ --quiet
+
+ - name: Verify ready singleton revision
+ run: |
+ SERVICE="$(gcloud run services describe "${SERVICE_NAME}" \
+ --project "${GCP_PROJECT_ID}" \
+ --region "${GCP_REGION}" \
+ --format=json)"
+ jq -e '.status.conditions[] | select(.type == "Ready" and .status == "True")' \
+ <<< "${SERVICE}"
+ REVISION="$(jq -r \
+ '[.status.traffic[] | select((.percent // 0) == 100)] |
+ if length == 1 then .[0].revisionName // empty else empty end' \
+ <<< "${SERVICE}")"
+ test -n "${REVISION}"
+ SERVED_IMAGE="$(gcloud run revisions describe "${REVISION}" \
+ --project "${GCP_PROJECT_ID}" \
+ --region "${GCP_REGION}" \
+ --format='value(spec.containers[0].image)')"
+ test "${SERVED_IMAGE}" = "${IMAGE}"
diff --git a/.github/workflows/cloud-deploy-relay-production-capacity-job.yml b/.github/workflows/cloud-deploy-relay-production-capacity-job.yml
new file mode 100644
index 00000000000..4cf39571371
--- /dev/null
+++ b/.github/workflows/cloud-deploy-relay-production-capacity-job.yml
@@ -0,0 +1,815 @@
+name: Deploy Relay Production Capacity Job
+
+on:
+ workflow_call:
+ inputs:
+ mode:
+ required: true
+ type: string
+ target-cell-id:
+ required: true
+ type: string
+ confirmation:
+ required: true
+ type: string
+ monitor-run-id:
+ required: true
+ type: string
+ monitor-run-attempt:
+ required: true
+ type: string
+ evidence-mode:
+ required: true
+ type: string
+ wave-cell-ids:
+ required: true
+ type: string
+ wave-index:
+ required: true
+ type: string
+ source-wave-run-id:
+ required: true
+ type: string
+
+permissions:
+ actions: read
+ contents: read
+ id-token: write
+
+defaults:
+ run:
+ working-directory: cloud
+
+jobs:
+ capacity:
+ if: ${{ github.ref == 'refs/heads/main' }}
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ timeout-minutes: 75
+ environment: production
+ env:
+ GCP_PROJECT_ID: onorca-cloud
+ GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
+ DIRECTOR_SERVICE_NAME: orca-cloud-relay
+ DIRECTOR_ORIGIN: https://relay.onorca.dev
+ TARGET_CELL_ID: ${{ inputs.target-cell-id }}
+ CAPACITY_CELL_IDS: production-gce-c7,production-gce-c8,production-gce-c9,production-gce-c10,production-gce-c13,production-gce-c14,production-gce-c15,production-gce-c16,production-gce-c19,production-gce-c20,production-gce-c21,production-gce-c22,production-gce-c23,production-gce-c24,production-gce-c25,production-gce-c26
+ PREDECESSOR_IMAGE_DIGEST: sha256:0e83408b0dc08531f1e8182019dc151afc38d63ddde4ad5cc01e40247ef3681d
+ COMPATIBLE_DIRECTOR_IMAGE_DIGEST: sha256:01b7fc3e6dce66180034f268a2dc92c05458706c5b3a0dc4450dcdd6161f6e73
+ COMPATIBLE_CELL_IMAGE_DIGEST: sha256:c77ec7aef565009fdb645b0989806859bfa40a7aa14e4a57ab55ac92fee6c34f
+ CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
+ DEPLOY_MODE: ${{ inputs.mode }}
+ EVIDENCE_MODE: ${{ inputs.evidence-mode }}
+ MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
+ MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }}
+ WAVE_CELL_IDS: ${{ inputs.wave-cell-ids }}
+ WAVE_INDEX: ${{ inputs.wave-index }}
+ SOURCE_WAVE_RUN_ID: ${{ inputs.source-wave-run-id }}
+ steps:
+ - name: Require exact reusable-workflow invocation
+ working-directory: .
+ run: |
+ [[ "${DEPLOY_MODE}" =~ ^(verify|apply|rollback)$ ]]
+ if test "${EVIDENCE_MODE}" = continuation; then
+ test "${DEPLOY_MODE}" = apply
+ [[ "${WAVE_INDEX}" =~ ^[0-3]$ ]]
+ test "${WAVE_CELL_IDS}" != none
+ test "${SOURCE_WAVE_RUN_ID}" = none
+ elif test "${EVIDENCE_MODE}" = resume; then
+ test "${DEPLOY_MODE}" = apply
+ test "${WAVE_INDEX}" = resume
+ test "${WAVE_CELL_IDS}" != none
+ [[ "${SOURCE_WAVE_RUN_ID}" =~ ^[0-9]+$ ]]
+ else
+ test "${EVIDENCE_MODE}" = single
+ test "${WAVE_CELL_IDS}" = none
+ test "${WAVE_INDEX}" = 0
+ test "${SOURCE_WAVE_RUN_ID}" = none
+ fi
+
+ - name: Require production workflow configuration
+ working-directory: .
+ env:
+ DEPLOY_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+ DEPLOY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+ CAPACITY_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
+ run: |
+ test -n "${GCP_REGION}"
+ test -n "${DEPLOY_WORKLOAD_IDENTITY_PROVIDER}"
+ test -n "${DEPLOY_SERVICE_ACCOUNT}"
+ test -n "${CAPACITY_WORKLOAD_IDENTITY_PROVIDER}"
+ test -n "${CAPACITY_SERVICE_ACCOUNT}"
+
+ - uses: actions/checkout@v4
+
+ - id: resume-provenance
+ if: ${{ inputs.evidence-mode == 'resume' }}
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: |
+ test "${MONITOR_RUN_ATTEMPT}" = 1
+ case "${MONITOR_RUN_ID}:${SOURCE_WAVE_RUN_ID}:${WAVE_CELL_IDS}:${TARGET_CELL_ID}" in
+ 31554591366:31555510376:production-gce-c16,production-gce-c15,production-gce-c14,production-gce-c13:production-gce-c13)
+ EXPECTED_SHA=a917e8e1fc1a2654e8cb81ba39b57733ec56be9c
+ EXPECTED_SOURCE_ATTEMPT=1
+ ;;
+ 31562760783:31563664692:production-gce-c10,production-gce-c9,production-gce-c8,production-gce-c7:production-gce-c10)
+ EXPECTED_SHA=6082e9ca89a918ca51f0c87db003f5e8805b64b7
+ EXPECTED_SOURCE_ATTEMPT=1
+ ;;
+ 31571019947:31572080665:production-gce-c9,production-gce-c8,production-gce-c7:production-gce-c8)
+ EXPECTED_SHA=e59958130c9d9b7a6cd805df2678d08997842c7c
+ EXPECTED_SOURCE_ATTEMPT=2
+ ;;
+ *) exit 1 ;;
+ esac
+ MONITOR_SHA="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/runs/${MONITOR_RUN_ID}" \
+ --jq 'select(.name == "Monitor Relay Production" and
+ .path == ".github/workflows/cloud-monitor-relay-production.yml" and
+ .head_branch == "main" and .head_repository.full_name == env.GITHUB_REPOSITORY and
+ .event == "workflow_dispatch" and .conclusion == "success" and .run_attempt == 1) |
+ .head_sha')"
+ SOURCE_SHA="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_WAVE_RUN_ID}" \
+ --jq 'select(.name == "Deploy Relay Production Capacity" and
+ .path == ".github/workflows/cloud-deploy-relay-production-capacity.yml" and
+ .head_branch == "main" and .head_repository.full_name == env.GITHUB_REPOSITORY and
+ .event == "workflow_dispatch" and .conclusion == "failure") |
+ .head_sha')"
+ SOURCE_ATTEMPT="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_WAVE_RUN_ID}" \
+ --jq '.run_attempt')"
+ [[ "${MONITOR_SHA}" =~ ^[0-9a-f]{40}$ ]]
+ test "${MONITOR_SHA}" = "${EXPECTED_SHA}"
+ test "${SOURCE_SHA}" = "${MONITOR_SHA}"
+ test "${SOURCE_ATTEMPT}" = "${EXPECTED_SOURCE_ATTEMPT}"
+ echo "commit-sha=${MONITOR_SHA}" >> "${GITHUB_OUTPUT}"
+
+ - name: Require fresh dry-run evidence reference
+ if: ${{ inputs.mode == 'apply' }}
+ run: |
+ [[ "${MONITOR_RUN_ID}" =~ ^[0-9]+$ ]]
+ [[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
+
+ - name: Download private dry-run evidence
+ if: ${{ inputs.mode == 'apply' }}
+ uses: actions/download-artifact@v4
+ with:
+ name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
+ path: ${{ runner.temp }}/relay-monitor-evidence
+ github-token: ${{ github.token }}
+ run-id: ${{ inputs.monitor-run-id }}
+
+ - uses: pnpm/action-setup@v4
+ with:
+ package_json_file: cloud/package.json
+
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 24
+
+ - run: pnpm install --frozen-lockfile
+
+ - uses: hashicorp/setup-terraform@v3
+ with:
+ terraform_wrapper: false
+
+ - name: Verify dry-run artifact before cloud authentication
+ if: ${{ inputs.mode == 'apply' }}
+ run: |
+ EVIDENCE_COMMIT_SHA="${GITHUB_SHA}"
+ if test "${EVIDENCE_MODE:-single}" = resume; then
+ EVIDENCE_COMMIT_SHA="${{ steps.resume-provenance.outputs.commit-sha }}"
+ fi
+ node dev/scripts/relay-monitor-evidence.mjs verify-restore \
+ --directory "${RUNNER_TEMP}/relay-monitor-evidence" \
+ --incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
+ --run-id "${MONITOR_RUN_ID}" \
+ --run-attempt "${MONITOR_RUN_ATTEMPT}" \
+ --commit-sha "${EVIDENCE_COMMIT_SHA}" \
+ --mode dry-run
+
+ - name: Reject previously consumed dry-run evidence
+ if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'single' }}
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: |
+ MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
+ COUNT="$(gh api \
+ "/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \
+ --jq '.total_count')"
+ test "${COUNT}" = "0"
+
+ - name: Download this workflow's wave authority
+ if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'continuation' }}
+ uses: actions/download-artifact@v4
+ with:
+ name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
+ path: ${{ runner.temp }}/relay-wave-authority
+ github-token: ${{ github.token }}
+ run-id: ${{ github.run_id }}
+
+ - name: Download the failed wave authority for resume
+ if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'resume' }}
+ uses: actions/download-artifact@v4
+ with:
+ name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
+ path: ${{ runner.temp }}/relay-wave-authority
+ github-token: ${{ github.token }}
+ run-id: ${{ inputs.source-wave-run-id }}
+
+ - name: Require wave evidence consumed by this workflow
+ if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'continuation' }}
+ run: |
+ MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
+ test "$(< "${RUNNER_TEMP}/relay-wave-authority/${MARKER_NAME}")" = "${GITHUB_RUN_ID}"
+
+ - name: Require wave evidence consumed by the failed source workflow
+ if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'resume' }}
+ run: |
+ MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
+ test "$(< "${RUNNER_TEMP}/relay-wave-authority/${MARKER_NAME}")" = "${SOURCE_WAVE_RUN_ID}"
+
+ - id: deploy-auth
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - uses: google-github-actions/setup-gcloud@v2
+
+ - uses: ./.github/actions/cloud-sql-rollout-lease
+ with:
+ bucket: onorca-cloud-terraform-state
+ object: terraform/state/cloud-sql-rollout/production.lock
+ release: 'false'
+
+ - name: Require exact mutation confirmation
+ if: ${{ inputs.mode != 'verify' }}
+ env:
+ CONFIRMATION: ${{ inputs.confirmation }}
+ run: |
+ if test "${EVIDENCE_MODE:-single}" = resume; then
+ test "${CONFIRMATION}" = "RESUME_SELECTED_CELL_TO_1000 ${TARGET_CELL_ID}"
+ elif test "${DEPLOY_MODE}" = apply; then
+ test "${CONFIRMATION}" = "RAISE_SELECTED_CELL_TO_1000"
+ else
+ test "${CONFIRMATION}" = "ROLL_BACK_SELECTED_CELL_TO_600 ${TARGET_CELL_ID}"
+ fi
+
+ - name: Initialize the exact production backend
+ run: node dev/scripts/infra.mjs init --env production
+
+ - name: Build the exact selected-cell configuration
+ shell: bash
+ run: |
+ if test "${DEPLOY_MODE}" = rollback; then
+ TARGET_HARD_CAP=600
+ else
+ TARGET_HARD_CAP=1000
+ fi
+ TARGET_UNOBSERVED_BOUND=60
+ TARGET_HOSTNAME="${TARGET_CELL_ID#production-gce-}"
+ [[ "${TARGET_HOSTNAME}" =~ ^c(7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26)$ ]]
+ CELL_ORIGIN="https://${TARGET_HOSTNAME}.relay.onorca.dev"
+ CELLS_JSON="$(terraform -chdir=infra/terraform console \
+ -var-file=environments/production.tfvars \
+ <<< 'jsonencode(var.relay_gce_cells)' | jq -er '.')"
+ OVERRIDE_CELLS_JSON="$(jq -ce \
+ --arg cell "${TARGET_CELL_ID}" \
+ --argjson cap "${TARGET_HARD_CAP}" \
+ --argjson bound "${TARGET_UNOBSERVED_BOUND}" \
+ '.[$cell].connection_hard_cap = $cap |
+ .[$cell].connection_unobserved_bound = $bound' \
+ <<< "${CELLS_JSON}")"
+ jq -n --argjson cells "${OVERRIDE_CELLS_JSON}" \
+ '{relay_gce_cells:$cells}' > "${RUNNER_TEMP}/relay-capacity.tfvars.json"
+ BASE_CELLS_JSON="$(terraform -chdir=infra/terraform console \
+ -var-file=environments/production.tfvars \
+ <<< 'local.relay_director_cells_json' | jq -er '.')"
+ IMAGE_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].image"
+ ZONE_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].zone"
+ DESIRED_IMAGE="$(terraform -chdir=infra/terraform console \
+ -var-file=environments/production.tfvars \
+ -var-file="${RUNNER_TEMP}/relay-capacity.tfvars.json" \
+ <<< "${IMAGE_EXPRESSION}" | jq -r '.')"
+ TARGET_ZONE="$(terraform -chdir=infra/terraform console \
+ -var-file=environments/production.tfvars \
+ -var-file="${RUNNER_TEMP}/relay-capacity.tfvars.json" \
+ <<< "${ZONE_EXPRESSION}" | jq -r '.')"
+ MIG_NAME="$(terraform -chdir=infra/terraform output -json relay_gce_cell_deployments \
+ | jq -r --arg cell "${TARGET_CELL_ID}" '.[$cell].mig_name')"
+ jq -e --arg cell "${TARGET_CELL_ID}" \
+ 'any(.[]; .id == $cell and .connectionHardCap == 1000 and
+ .connectionUnobservedBound == 60)' \
+ <<< "${BASE_CELLS_JSON}" >/dev/null
+ [[ "${DESIRED_IMAGE}" =~ @sha256:[0-9a-f]{64}$ ]]
+ DESIRED_IMAGE_DIGEST="${DESIRED_IMAGE##*@}"
+ [[ "${TARGET_ZONE}" =~ ^[a-z0-9-]+$ ]]
+ test "${MIG_NAME}" = "orca-cloud-relay-gce-${TARGET_HOSTNAME}"
+ {
+ echo "CELL_ORIGIN=${CELL_ORIGIN}"
+ echo "TARGET_HOSTNAME=${TARGET_HOSTNAME}"
+ echo "TARGET_HARD_CAP=${TARGET_HARD_CAP}"
+ echo "TARGET_UNOBSERVED_BOUND=${TARGET_UNOBSERVED_BOUND}"
+ echo "DESIRED_IMAGE=${DESIRED_IMAGE}"
+ echo "DESIRED_IMAGE_DIGEST=${DESIRED_IMAGE_DIGEST}"
+ echo "TARGET_ZONE=${TARGET_ZONE}"
+ echo "MIG_NAME=${MIG_NAME}"
+ echo "BASE_CELLS_JSON=${BASE_CELLS_JSON}"
+ } >> "${GITHUB_ENV}"
+
+ - name: Require the exact compatible production image and topology
+ shell: bash
+ run: |
+ SERVICE_JSON="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" \
+ --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
+ ACTIVE_REVISION="$(jq -r \
+ '[.status.traffic[] | select((.percent // 0) > 0)] |
+ if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end' \
+ <<< "${SERVICE_JSON}")"
+ test -n "${ACTIVE_REVISION}"
+ ACTIVE_REVISION_JSON="$(gcloud run revisions describe "${ACTIVE_REVISION}" \
+ --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
+ ACTIVE_IMAGE="$(jq -er '.spec.containers[0].image' <<< "${ACTIVE_REVISION_JSON}")"
+ ACTIVE_IMAGE_DIGEST="${ACTIVE_IMAGE##*@}"
+ if test "${ACTIVE_IMAGE}" != "${DESIRED_IMAGE}"; then
+ test "${ACTIVE_IMAGE_DIGEST}" = "${COMPATIBLE_DIRECTOR_IMAGE_DIGEST}"
+ test "${DESIRED_IMAGE_DIGEST}" = "${COMPATIBLE_CELL_IMAGE_DIGEST}"
+ fi
+ CURRENT_CELLS_JSON="$(jq -cer '[.spec.containers[0].env[]? |
+ select(.name == "ORCA_RELAY_CELLS_JSON") | .value] |
+ if length == 1 then .[0] | fromjson else error("missing director topology") end' \
+ <<< "${ACTIVE_REVISION_JSON}")"
+ CURRENT_CAPACITY_SERVICE_ACCOUNT_JSON="$(
+ node dev/scripts/read-relay-production-capacity-identity.mjs \
+ <<< "${ACTIVE_REVISION_JSON}"
+ )"
+ CLASSIFICATION="$(jq -nc \
+ --argjson baseCells "${BASE_CELLS_JSON}" \
+ --argjson currentCells "${CURRENT_CELLS_JSON}" \
+ --arg capacityCellIds "${CAPACITY_CELL_IDS}" \
+ --arg targetCellId "${TARGET_CELL_ID}" \
+ --argjson targetHardCap "${TARGET_HARD_CAP}" \
+ --argjson currentCapacityServiceAccount \
+ "${CURRENT_CAPACITY_SERVICE_ACCOUNT_JSON}" \
+ '{baseCells:$baseCells, currentCells:$currentCells,
+ capacityCellIds:($capacityCellIds | split(",")),
+ targetCellId:$targetCellId, targetHardCap:$targetHardCap,
+ currentCapacityServiceAccount:$currentCapacityServiceAccount}' \
+ | node dev/scripts/classify-relay-production-capacity-director.mjs \
+ --capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}")"
+ TOPOLOGY_PHASE="$(jq -er '.topologyPhase' <<< "${CLASSIFICATION}")"
+ DESIRED_CELLS_JSON="$(jq -cer '.desiredCells' <<< "${CLASSIFICATION}")"
+ DIRECTOR_READY="$(jq -er \
+ 'if (.directorReady | type) == "boolean" then
+ (.directorReady | tostring)
+ else error("invalid directorReady classification") end' \
+ <<< "${CLASSIFICATION}")"
+ {
+ echo "ACTIVE_IMAGE=${ACTIVE_IMAGE}"
+ echo "TOPOLOGY_PHASE=${TOPOLOGY_PHASE}"
+ echo "DIRECTOR_READY=${DIRECTOR_READY}"
+ echo "DESIRED_CELLS_JSON=${DESIRED_CELLS_JSON}"
+ } >> "${GITHUB_ENV}"
+
+ - name: Require the exact wave predecessor topology
+ if: ${{ inputs.mode == 'apply' && (inputs.evidence-mode == 'continuation' || inputs.evidence-mode == 'resume') }}
+ run: test "${TOPOLOGY_PHASE}" = predecessor
+
+ - name: Verify fresh dry-run evidence against the live selector
+ if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'single' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
+ run: |
+ node dev/scripts/relay-monitor-evidence.mjs verify-mutation \
+ --directory "${RUNNER_TEMP}/relay-monitor-evidence" \
+ --incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
+ --run-id "${MONITOR_RUN_ID}" \
+ --run-attempt "${MONITOR_RUN_ATTEMPT}" \
+ --commit-sha "${GITHUB_SHA}" \
+ --mode dry-run \
+ --mutation-mode capacity-transition \
+ --source-cell-id "${TARGET_CELL_ID}" \
+ --director-origin "${DIRECTOR_ORIGIN}"
+
+ - name: Recheck every live safety signal
+ if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'single' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
+ run: |
+ pnpm incident:relay-preflight -- \
+ --state-file "${RUNNER_TEMP}/relay-monitor-evidence/relay-${MONITOR_RUN_ID}-dry-run.state.json"
+
+ - name: Recheck exact wave state and every live safety signal
+ if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'continuation' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
+ run: |
+ node dev/scripts/relay-production-capacity-wave.mjs build-preflight \
+ --state-file "${RUNNER_TEMP}/relay-monitor-evidence/relay-${MONITOR_RUN_ID}-dry-run.state.json" \
+ --wave-cell-ids "${WAVE_CELL_IDS}" \
+ --wave-index "${WAVE_INDEX}" \
+ --target-cell-id "${TARGET_CELL_ID}" \
+ --output-file "${RUNNER_TEMP}/relay-capacity-wave-preflight.json"
+ RETRY_ARGS=()
+ if test "${WAVE_INDEX}" != 0; then RETRY_ARGS=(--retry-freshness); fi
+ pnpm incident:relay-preflight -- \
+ --state-file "${RUNNER_TEMP}/relay-capacity-wave-preflight.json" \
+ "${RETRY_ARGS[@]}"
+
+ - name: Recheck exact isolated resume state and every live safety signal
+ if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'resume' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
+ run: |
+ node dev/scripts/relay-production-capacity-wave.mjs build-resume-preflight \
+ --state-file "${RUNNER_TEMP}/relay-monitor-evidence/relay-${MONITOR_RUN_ID}-dry-run.state.json" \
+ --wave-cell-ids "${WAVE_CELL_IDS}" \
+ --target-cell-id "${TARGET_CELL_ID}" \
+ --output-file "${RUNNER_TEMP}/relay-capacity-wave-preflight.json"
+ pnpm incident:relay-preflight -- \
+ --state-file "${RUNNER_TEMP}/relay-capacity-wave-preflight.json" \
+ --retry-freshness
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --hard-cap 600 \
+ --unobserved-bound 60 \
+ --heartbeat fresh \
+ --admission migration-only \
+ --draining required \
+ --activity allowed \
+ --runtime required \
+ --expected-image-digests "${PREDECESSOR_IMAGE_DIGEST}"
+
+ - name: Consume the single-use dry-run evidence
+ if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'single' }}
+ run: |
+ MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
+ mkdir -p "${RUNNER_TEMP}/relay-monitor-consumption"
+ printf '%s\n' "${GITHUB_RUN_ID}" \
+ > "${RUNNER_TEMP}/relay-monitor-consumption/${MARKER_NAME}"
+
+ - name: Publish the consumed-evidence marker
+ if: ${{ inputs.mode == 'apply' && inputs.evidence-mode == 'single' }}
+ uses: actions/upload-artifact@v4
+ with:
+ name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
+ path: ${{ runner.temp }}/relay-monitor-consumption/relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
+ retention-days: 90
+ if-no-files-found: error
+
+ - name: Verify current selected-cell capacity
+ if: ${{ inputs.mode == 'verify' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
+ run: |
+ CURRENT_CAP="${TARGET_HARD_CAP}"
+ CURRENT_IMAGE_DIGEST="${DESIRED_IMAGE_DIGEST}"
+ if test "${TOPOLOGY_PHASE}" = predecessor; then
+ CURRENT_CAP=600
+ CURRENT_IMAGE_DIGEST="${DESIRED_IMAGE_DIGEST},${PREDECESSOR_IMAGE_DIGEST}"
+ fi
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --hard-cap "${CURRENT_CAP}" \
+ --unobserved-bound "${TARGET_UNOBSERVED_BOUND}" \
+ --heartbeat fresh \
+ --admission general \
+ --draining forbidden \
+ --activity allowed \
+ --expected-image-digests "${CURRENT_IMAGE_DIGEST}"
+
+ - name: Arm fail-closed mutation cleanup
+ if: ${{ inputs.mode != 'verify' }}
+ run: echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
+
+ - name: Reversibly isolate only the selected cell
+ if: ${{ inputs.mode != 'verify' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
+ run: |
+ test "${MUTATION_STARTED:-false}" = true || exit 0
+ node dev/scripts/prepare-relay-production-capacity-canary.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --mode isolate
+
+ - name: Drain the selected cell or prove an offline rollback
+ if: ${{ inputs.mode != 'verify' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
+ run: |
+ if node dev/scripts/prepare-relay-production-capacity-canary.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --mode drain; then
+ echo "OFFLINE_ROLLBACK=false" >> "${GITHUB_ENV}"
+ elif test "${DEPLOY_MODE}" = rollback; then
+ echo "OFFLINE_ROLLBACK=true" >> "${GITHUB_ENV}"
+ else
+ exit 1
+ fi
+
+ - id: restart-auth-one
+ if: ${{ inputs.mode != 'verify' }}
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - id: restart-gate-one
+ name: Require restart-safe selected-cell activity
+ if: ${{ inputs.mode != 'verify' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.restart-auth-one.outputs.id_token }}
+ run: |
+ if test "${OFFLINE_ROLLBACK:-false}" = true; then
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --heartbeat stale \
+ --admission migration-only \
+ --draining either \
+ --activity restart-safe \
+ --runtime unavailable
+ echo "settled=true" >> "${GITHUB_OUTPUT}"
+ exit 0
+ fi
+ CURRENT_CAP=600
+ if test "${TOPOLOGY_PHASE}" = desired; then
+ CURRENT_CAP="${TARGET_HARD_CAP}"
+ elif test "${TARGET_HARD_CAP}" = 600; then
+ CURRENT_CAP=1000
+ fi
+ GATE_LOG="${RUNNER_TEMP}/relay-capacity-restart-gate-one.log"
+ set +e
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --hard-cap "${CURRENT_CAP}" \
+ --unobserved-bound 60 \
+ --heartbeat either \
+ --admission migration-only \
+ --draining required \
+ --activity restart-safe \
+ --runtime required \
+ --timeout-ms 450000 \
+ --expected-image-digests \
+ "${DESIRED_IMAGE_DIGEST},${PREDECESSOR_IMAGE_DIGEST}" \
+ 2> "${GATE_LOG}"
+ GATE_EXIT=$?
+ set -e
+ cat "${GATE_LOG}" >&2
+ if test "${GATE_EXIT}" = 0; then
+ echo "settled=true" >> "${GITHUB_OUTPUT}"
+ exit 0
+ fi
+ if test "$(wc -l < "${GATE_LOG}" | tr -d ' ')" = 1 &&
+ grep -Eq '^capacity transition verification timed out: \{.*\}$' "${GATE_LOG}"; then
+ echo "settled=false" >> "${GITHUB_OUTPUT}"
+ exit 0
+ fi
+ exit "${GATE_EXIT}"
+
+ - id: restart-auth-two
+ if: ${{ steps.restart-gate-one.outputs.settled == 'false' }}
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - name: Require extended restart-safe selected-cell activity
+ if: ${{ steps.restart-gate-one.outputs.settled == 'false' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.restart-auth-two.outputs.id_token }}
+ run: |
+ CURRENT_CAP=600
+ if test "${TOPOLOGY_PHASE}" = desired; then
+ CURRENT_CAP="${TARGET_HARD_CAP}"
+ elif test "${TARGET_HARD_CAP}" = 600; then
+ CURRENT_CAP=1000
+ fi
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --hard-cap "${CURRENT_CAP}" \
+ --unobserved-bound 60 \
+ --heartbeat either \
+ --admission migration-only \
+ --draining required \
+ --activity restart-safe \
+ --runtime required \
+ --timeout-ms 450000 \
+ --expected-image-digests \
+ "${DESIRED_IMAGE_DIGEST},${PREDECESSOR_IMAGE_DIGEST}"
+
+ - name: Deploy only the reviewed director topology
+ if: ${{ inputs.mode != 'verify' }}
+ run: |
+ if test "${DIRECTOR_READY}" = true; then exit 0; fi
+ RELEASE_ID="capacity-${TARGET_HOSTNAME}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}"
+ node dev/scripts/deploy-relay-blue-green.mjs \
+ --project "${GCP_PROJECT_ID}" \
+ --region "${GCP_REGION}" \
+ --service "${DIRECTOR_SERVICE_NAME}" \
+ --image "${ACTIVE_IMAGE}" \
+ --role director \
+ --max-instances 5 \
+ --capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
+ --capacity-cell-id "${TARGET_CELL_ID}" \
+ --director-cells-json "${DESIRED_CELLS_JSON}" \
+ --min-instances 5 \
+ --prune-revisions false \
+ --release-id "${RELEASE_ID}"
+
+ - id: director-transition-auth
+ if: ${{ inputs.mode != 'verify' }}
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - name: Require fail-closed director transition
+ if: ${{ inputs.mode != 'verify' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.director-transition-auth.outputs.id_token }}
+ run: |
+ if test "${OFFLINE_ROLLBACK:-false}" = true; then
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --heartbeat stale \
+ --admission migration-only \
+ --draining either \
+ --activity restart-safe \
+ --runtime unavailable
+ exit 0
+ fi
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --hard-cap "${TARGET_HARD_CAP}" \
+ --unobserved-bound "${TARGET_UNOBSERVED_BOUND}" \
+ --heartbeat either \
+ --admission migration-only \
+ --draining required \
+ --activity restart-safe \
+ --runtime required \
+ --expected-image-digests \
+ "${DESIRED_IMAGE_DIGEST},${PREDECESSOR_IMAGE_DIGEST}"
+
+ - id: capacity-auth
+ if: ${{ inputs.mode != 'verify' }}
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - name: Plan and apply only the empty selected cell
+ if: ${{ inputs.mode != 'verify' }}
+ shell: bash
+ run: |
+ terraform -chdir=infra/terraform plan \
+ -var-file=environments/production.tfvars \
+ -var-file="${RUNNER_TEMP}/relay-capacity.tfvars.json" \
+ "-target=google_compute_instance_template.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
+ "-target=google_compute_instance_group_manager.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
+ -out="${RUNNER_TEMP}/relay-capacity-cell.tfplan"
+ PLAN_RESULT="$(terraform -chdir=infra/terraform show -json \
+ "${RUNNER_TEMP}/relay-capacity-cell.tfplan" \
+ | node dev/scripts/validate-relay-capacity-plan.mjs \
+ --mode bootstrap-cell \
+ --cell-id "${TARGET_CELL_ID}" \
+ --hard-cap "${TARGET_HARD_CAP}" \
+ --unobserved-bound "${TARGET_UNOBSERVED_BOUND}" \
+ --image "${DESIRED_IMAGE}" \
+ --capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}")"
+ echo "${PLAN_RESULT}"
+ PLAN_CHANGES="$(jq -r '.changes' <<< "${PLAN_RESULT}")"
+ [[ "${PLAN_CHANGES}" =~ ^(0|1|2)$ ]]
+ if test "${PLAN_CHANGES}" != 0; then
+ terraform -chdir=infra/terraform apply \
+ -auto-approve "${RUNNER_TEMP}/relay-capacity-cell.tfplan"
+ else
+ INSTANCE="$(gcloud compute instance-groups managed list-instances \
+ "${MIG_NAME}" --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" \
+ --format=json | jq -er 'if length == 1 and
+ .[0].instanceStatus == "RUNNING" and .[0].currentAction == "NONE"
+ then .[0].instance | split("/") | last
+ else error("selected cell is not one stable running instance") end')"
+ gcloud compute instance-groups managed recreate-instances \
+ "${MIG_NAME}" --instances "${INSTANCE}" \
+ --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --quiet
+ fi
+ gcloud compute instance-groups managed wait-until \
+ "${MIG_NAME}" --stable --project "${GCP_PROJECT_ID}" \
+ --zone "${TARGET_ZONE}" --timeout 900
+
+ - id: capacity-transition-auth
+ if: ${{ inputs.mode != 'verify' }}
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - name: Verify fresh exact selected-cell heartbeat before admission
+ if: ${{ inputs.mode != 'verify' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.capacity-transition-auth.outputs.id_token }}
+ run: |
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --hard-cap "${TARGET_HARD_CAP}" \
+ --unobserved-bound "${TARGET_UNOBSERVED_BOUND}" \
+ --heartbeat fresh \
+ --admission migration-only \
+ --draining forbidden \
+ --activity allowed \
+ --expected-image-digests "${DESIRED_IMAGE_DIGEST}"
+
+ - name: Restore only the selected cell to general admission
+ if: ${{ inputs.mode != 'verify' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.capacity-transition-auth.outputs.id_token }}
+ run: |
+ node dev/scripts/prepare-relay-production-capacity-canary.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --mode activate
+
+ - name: Verify the live general selected cell
+ if: ${{ inputs.mode != 'verify' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.capacity-transition-auth.outputs.id_token }}
+ run: |
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --hard-cap "${TARGET_HARD_CAP}" \
+ --unobserved-bound "${TARGET_UNOBSERVED_BOUND}" \
+ --heartbeat fresh \
+ --admission general \
+ --draining forbidden \
+ --activity allowed \
+ --expected-image-digests "${DESIRED_IMAGE_DIGEST}"
+
+ - id: cleanup-auth
+ if: ${{ failure() && inputs.mode != 'verify' }}
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - name: Keep the selected cell isolated after a failed mutation
+ if: ${{ failure() && inputs.mode != 'verify' }}
+ continue-on-error: true
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.cleanup-auth.outputs.id_token }}
+ run: |
+ test "${MUTATION_STARTED:-false}" = true || exit 0
+ CLEANUP_STATUS=0
+ node dev/scripts/prepare-relay-production-capacity-canary.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --mode isolate || CLEANUP_STATUS=$?
+ node dev/scripts/prepare-relay-production-capacity-canary.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --mode drain || CLEANUP_STATUS=$?
+ exit "${CLEANUP_STATUS}"
diff --git a/.github/workflows/cloud-deploy-relay-production-capacity.yml b/.github/workflows/cloud-deploy-relay-production-capacity.yml
new file mode 100644
index 00000000000..5f6a1897d73
--- /dev/null
+++ b/.github/workflows/cloud-deploy-relay-production-capacity.yml
@@ -0,0 +1,351 @@
+name: Deploy Relay Production Capacity
+
+on:
+ workflow_dispatch:
+ inputs:
+ mode:
+ description: Verify, change one cell, or raise a sequential wave
+ required: true
+ default: verify
+ type: choice
+ options:
+ - verify
+ - apply
+ - rollback
+ - wave-apply
+ - wave-resume
+ target-cell-id:
+ description: Exact serving cell for verify, apply, or rollback
+ required: true
+ default: production-gce-c26
+ type: choice
+ options:
+ - production-gce-c7
+ - production-gce-c8
+ - production-gce-c9
+ - production-gce-c10
+ - production-gce-c13
+ - production-gce-c14
+ - production-gce-c15
+ - production-gce-c16
+ - production-gce-c19
+ - production-gce-c20
+ - production-gce-c21
+ - production-gce-c22
+ - production-gce-c23
+ - production-gce-c24
+ - production-gce-c25
+ - production-gce-c26
+ wave-cell-ids:
+ description: Ordered comma-separated wave of two to four serving cells
+ required: false
+ default: none
+ type: string
+ confirmation:
+ description: Enter the exact single-cell or wave confirmation
+ required: false
+ type: string
+ monitor-run-id:
+ description: Successful fresh dry-run monitor workflow run ID for apply
+ required: false
+ type: string
+ monitor-run-attempt:
+ description: Exact dry-run monitor workflow attempt for apply
+ required: false
+ type: string
+ source-wave-run-id:
+ description: Failed wave run that isolated the resume target
+ required: false
+ type: string
+
+permissions:
+ actions: read
+ contents: read
+ id-token: write
+
+concurrency:
+ group: production-cloud-sql-rollout
+ cancel-in-progress: false
+
+defaults:
+ run:
+ working-directory: cloud
+
+jobs:
+ single_cell:
+ if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (inputs.mode != 'wave-apply' && inputs.mode != 'wave-resume') }}
+ uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml
+ with:
+ mode: ${{ inputs.mode }}
+ target-cell-id: ${{ inputs.target-cell-id }}
+ confirmation: ${{ inputs.confirmation }}
+ monitor-run-id: ${{ inputs.monitor-run-id }}
+ monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
+ evidence-mode: single
+ wave-cell-ids: none
+ wave-index: '0'
+ source-wave-run-id: none
+ secrets: inherit
+
+ resume_cell:
+ if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (inputs.mode == 'wave-resume' && github.ref == 'refs/heads/main') }}
+ uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml
+ with:
+ mode: apply
+ target-cell-id: ${{ inputs.target-cell-id }}
+ confirmation: ${{ inputs.confirmation }}
+ monitor-run-id: ${{ inputs.monitor-run-id }}
+ monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
+ evidence-mode: resume
+ wave-cell-ids: ${{ inputs.wave-cell-ids }}
+ wave-index: resume
+ source-wave-run-id: ${{ inputs.source-wave-run-id }}
+ secrets: inherit
+
+ wave_gate:
+ if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (inputs.mode == 'wave-apply' && github.ref == 'refs/heads/main') }}
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ timeout-minutes: 30
+ environment: production
+ outputs:
+ cells: ${{ steps.wave.outputs.cells }}
+ env:
+ DIRECTOR_ORIGIN: https://relay.onorca.dev
+ MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
+ MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }}
+ OUTPUT_DIRECTORY: ${{ github.workspace }}/relay-monitor-evidence
+ PREDECESSOR_IMAGE_DIGEST: sha256:0e83408b0dc08531f1e8182019dc151afc38d63ddde4ad5cc01e40247ef3681d
+ COMPATIBLE_CELL_IMAGE_DIGEST: sha256:c77ec7aef565009fdb645b0989806859bfa40a7aa14e4a57ab55ac92fee6c34f
+ WAVE_CELL_IDS: ${{ inputs.wave-cell-ids }}
+ steps:
+ - name: Require production workflow configuration
+ working-directory: .
+ env:
+ DEPLOY_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+ DEPLOY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+ run: |
+ test -n "${DEPLOY_WORKLOAD_IDENTITY_PROVIDER}"
+ test -n "${DEPLOY_SERVICE_ACCOUNT}"
+
+ - uses: actions/checkout@v4
+
+ - uses: pnpm/action-setup@v4
+ with:
+ package_json_file: cloud/package.json
+
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 24
+
+ - run: pnpm install --frozen-lockfile
+
+ - id: wave
+ name: Validate the exact wave request
+ env:
+ CONFIRMATION: ${{ inputs.confirmation }}
+ run: |
+ CELLS="$(node dev/scripts/relay-production-capacity-wave.mjs validate \
+ --wave-cell-ids "${WAVE_CELL_IDS}" \
+ --confirmation "${CONFIRMATION}")"
+ echo "cells=${CELLS}" >> "${GITHUB_OUTPUT}"
+
+ - name: Require fresh dry-run evidence reference
+ run: |
+ [[ "${MONITOR_RUN_ID}" =~ ^[0-9]+$ ]]
+ [[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
+
+ - name: Download private dry-run evidence
+ uses: actions/download-artifact@v4
+ with:
+ name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
+ path: ${{ github.workspace }}/relay-monitor-evidence
+ github-token: ${{ github.token }}
+ run-id: ${{ inputs.monitor-run-id }}
+
+ - name: Verify dry-run artifact before cloud authentication
+ run: |
+ node dev/scripts/relay-monitor-evidence.mjs verify-restore \
+ --directory "${OUTPUT_DIRECTORY}" \
+ --incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
+ --run-id "${MONITOR_RUN_ID}" \
+ --run-attempt "${MONITOR_RUN_ATTEMPT}" \
+ --commit-sha "${GITHUB_SHA}" \
+ --mode dry-run
+
+ - name: Reject previously consumed dry-run evidence
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: |
+ MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
+ COUNT="$(gh api \
+ "/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \
+ --jq '.total_count')"
+ test "${COUNT}" = "0"
+
+ - id: deploy-auth
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - uses: google-github-actions/setup-gcloud@v2
+
+ - uses: ./.github/actions/cloud-sql-rollout-lease
+ with:
+ bucket: onorca-cloud-terraform-state
+ object: terraform/state/cloud-sql-rollout/production.lock
+ release: 'false'
+
+ - name: Verify wave evidence against the live selector
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
+ run: |
+ FIRST_CELL="$(jq -er '.[0]' <<< '${{ steps.wave.outputs.cells }}')"
+ node dev/scripts/relay-monitor-evidence.mjs verify-mutation \
+ --directory "${OUTPUT_DIRECTORY}" \
+ --incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
+ --run-id "${MONITOR_RUN_ID}" \
+ --run-attempt "${MONITOR_RUN_ATTEMPT}" \
+ --commit-sha "${GITHUB_SHA}" \
+ --mode dry-run \
+ --mutation-mode capacity-transition \
+ --source-cell-id "${FIRST_CELL}" \
+ --director-origin "${DIRECTOR_ORIGIN}"
+
+ - name: Recheck every live safety signal
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
+ run: |
+ pnpm incident:relay-preflight -- \
+ --state-file "${OUTPUT_DIRECTORY}/relay-${MONITOR_RUN_ID}-dry-run.state.json"
+
+ - name: Require exact 600/60 predecessor wave cells
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
+ run: |
+ while read -r CELL_ID; do
+ HOSTNAME="${CELL_ID#production-gce-}"
+ [[ "${HOSTNAME}" =~ ^c(7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26)$ ]]
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "https://${HOSTNAME}.relay.onorca.dev" \
+ --cell-id "${CELL_ID}" \
+ --hard-cap 600 \
+ --unobserved-bound 60 \
+ --heartbeat fresh \
+ --admission general \
+ --draining forbidden \
+ --activity allowed \
+ --expected-image-digests \
+ "${PREDECESSOR_IMAGE_DIGEST},${COMPATIBLE_CELL_IMAGE_DIGEST}"
+ done < <(jq -r '.[]' <<< '${{ steps.wave.outputs.cells }}')
+
+ - name: Consume the single-use dry-run evidence
+ run: |
+ MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
+ mkdir -p "${RUNNER_TEMP}/relay-monitor-consumption"
+ printf '%s\n' "${GITHUB_RUN_ID}" \
+ > "${RUNNER_TEMP}/relay-monitor-consumption/${MARKER_NAME}"
+
+ - name: Publish the consumed-evidence marker
+ uses: actions/upload-artifact@v4
+ with:
+ name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
+ path: ${{ runner.temp }}/relay-monitor-consumption/relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
+ retention-days: 90
+ if-no-files-found: error
+
+ wave_cell_1:
+ needs: wave_gate
+ uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml
+ with:
+ mode: apply
+ target-cell-id: ${{ fromJSON(needs.wave_gate.outputs.cells)[0] }}
+ confirmation: RAISE_SELECTED_CELL_TO_1000
+ monitor-run-id: ${{ inputs.monitor-run-id }}
+ monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
+ evidence-mode: continuation
+ wave-cell-ids: ${{ inputs.wave-cell-ids }}
+ wave-index: '0'
+ source-wave-run-id: none
+ secrets: inherit
+
+ wave_cell_2:
+ needs: [wave_gate, wave_cell_1]
+ uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml
+ with:
+ mode: apply
+ target-cell-id: ${{ fromJSON(needs.wave_gate.outputs.cells)[1] }}
+ confirmation: RAISE_SELECTED_CELL_TO_1000
+ monitor-run-id: ${{ inputs.monitor-run-id }}
+ monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
+ evidence-mode: continuation
+ wave-cell-ids: ${{ inputs.wave-cell-ids }}
+ wave-index: '1'
+ source-wave-run-id: none
+ secrets: inherit
+
+ wave_cell_3:
+ if: ${{ needs.wave_cell_2.result == 'success' && fromJSON(needs.wave_gate.outputs.cells)[2] != null }}
+ needs: [wave_gate, wave_cell_2]
+ uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml
+ with:
+ mode: apply
+ target-cell-id: ${{ fromJSON(needs.wave_gate.outputs.cells)[2] }}
+ confirmation: RAISE_SELECTED_CELL_TO_1000
+ monitor-run-id: ${{ inputs.monitor-run-id }}
+ monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
+ evidence-mode: continuation
+ wave-cell-ids: ${{ inputs.wave-cell-ids }}
+ wave-index: '2'
+ source-wave-run-id: none
+ secrets: inherit
+
+ wave_cell_4:
+ if: ${{ needs.wave_cell_3.result == 'success' && fromJSON(needs.wave_gate.outputs.cells)[3] != null }}
+ needs: [wave_gate, wave_cell_3]
+ uses: ./.github/workflows/cloud-deploy-relay-production-capacity-job.yml
+ with:
+ mode: apply
+ target-cell-id: ${{ fromJSON(needs.wave_gate.outputs.cells)[3] }}
+ confirmation: RAISE_SELECTED_CELL_TO_1000
+ monitor-run-id: ${{ inputs.monitor-run-id }}
+ monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
+ evidence-mode: continuation
+ wave-cell-ids: ${{ inputs.wave-cell-ids }}
+ wave-index: '3'
+ source-wave-run-id: none
+ secrets: inherit
+
+ # Every wave job re-enters the run's lease with release: 'false'; only this job frees it.
+ release_lease:
+ if: always()
+ needs:
+ - single_cell
+ - resume_cell
+ - wave_gate
+ - wave_cell_1
+ - wave_cell_2
+ - wave_cell_3
+ - wave_cell_4
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ timeout-minutes: 10
+ environment: production
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+
+ - uses: google-github-actions/setup-gcloud@v2
+
+ - uses: ./.github/actions/cloud-sql-rollout-lease
+ with:
+ bucket: onorca-cloud-terraform-state
+ object: terraform/state/cloud-sql-rollout/production.lock
+ release: 'true'
diff --git a/.github/workflows/cloud-deploy-relay-production-director.yml b/.github/workflows/cloud-deploy-relay-production-director.yml
new file mode 100644
index 00000000000..97abe2d227b
--- /dev/null
+++ b/.github/workflows/cloud-deploy-relay-production-director.yml
@@ -0,0 +1,267 @@
+name: Deploy Relay Production Director
+
+on:
+ workflow_dispatch:
+ inputs:
+ image-digest:
+ description: "Immutable relay image digest (sha256: plus 64 lowercase hex characters)"
+ required: true
+ type: string
+ regional-placement-mode:
+ description: Preserve the live switch, explicitly enable Asia preference, or force US-first
+ required: true
+ default: preserve
+ type: choice
+ options: [preserve, enable, disable]
+ prune-incompatible-revisions:
+ description: Retain only the newly verified serving and rollback revisions
+ required: true
+ default: false
+ type: boolean
+ confirmation:
+ description: Enter the exact confirmation required by a destructive option
+ required: false
+ type: string
+ expected-rehome-generation:
+ description: Exact durable regional-rehome generation; it must remain disabled
+ required: true
+ type: string
+ bootstrap-runtime-identity:
+ description: One-time move from the stamped-cell identity to the director identity
+ required: true
+ default: false
+ type: boolean
+ predecessor-image-digest:
+ description: Exact immutable serving predecessor digest for the one-time identity bootstrap
+ required: true
+ type: string
+
+permissions:
+ contents: read
+ id-token: write
+
+# Director updates and candidate operations both mutate production relay control state.
+concurrency:
+ group: production-cloud-sql-rollout
+ cancel-in-progress: false
+
+defaults:
+ run:
+ working-directory: cloud
+
+jobs:
+ deploy:
+ if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ environment: production
+ env:
+ GCP_PROJECT_ID: onorca-cloud
+ GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
+ DIRECTOR_SERVICE_NAME: orca-cloud-relay
+ IMAGE_REPOSITORY: us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay
+ REGIONAL_PLACEMENT_SECRET: orca-cloud-relay-regional-placement-enabled
+ IMAGE_DIGEST: ${{ inputs.image-digest }}
+ REGIONAL_PLACEMENT_MODE: ${{ inputs.regional-placement-mode }}
+ PRUNE_INCOMPATIBLE_REVISIONS: ${{ inputs.prune-incompatible-revisions }}
+ # Floor the served revision must keep, matching relay_min_instances in
+ # environments/production.tfvars. This gate only fails a bad deploy; Terraform
+ # still owns the value, and the candidate inherits it from the serving revision.
+ DIRECTOR_MIN_INSTANCES: 5
+ DIRECTOR_MAX_INSTANCES: 5
+ DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
+ PREDECESSOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_RUNTIME_SERVICE_ACCOUNT }}
+ REHOME_AUDIENCE: https://relay.onorca.dev/v1/admin/host-drain
+ EXPECTED_REHOME_GENERATION: ${{ inputs.expected-rehome-generation }}
+ BOOTSTRAP_RUNTIME_IDENTITY: ${{ inputs.bootstrap-runtime-identity }}
+ PREDECESSOR_IMAGE_DIGEST: ${{ inputs.predecessor-image-digest }}
+ steps:
+ - uses: actions/checkout@v4
+
+ - id: google-auth
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - uses: google-github-actions/setup-gcloud@v2
+
+ - uses: ./.github/actions/cloud-sql-rollout-lease
+ with:
+ bucket: onorca-cloud-terraform-state
+ object: terraform/state/cloud-sql-rollout/production.lock
+
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 24
+
+ - name: Resolve immutable production image
+ shell: bash
+ env:
+ CONFIRMATION: ${{ inputs.confirmation }}
+ run: |
+ if [[ ! "${IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]; then
+ echo "image-digest must be an immutable lowercase sha256 digest" >&2
+ exit 1
+ fi
+ IMAGE="${IMAGE_REPOSITORY}@${IMAGE_DIGEST}"
+ SERVED_DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" --format='value(image_summary.digest)')"
+ test "${SERVED_DIGEST}" = "${IMAGE_DIGEST}"
+ [[ "${PRUNE_INCOMPATIBLE_REVISIONS}" =~ ^(true|false)$ ]]
+ [[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
+ [[ "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" =~ ^[a-z][a-z0-9-]+@${GCP_PROJECT_ID}[.]iam[.]gserviceaccount[.]com$ ]]
+ [[ "${PREDECESSOR_RUNTIME_SERVICE_ACCOUNT}" =~ ^[a-z][a-z0-9-]+@${GCP_PROJECT_ID}[.]iam[.]gserviceaccount[.]com$ ]]
+ [[ "${BOOTSTRAP_RUNTIME_IDENTITY}" =~ ^(true|false)$ ]]
+ if test "${BOOTSTRAP_RUNTIME_IDENTITY}" = true; then
+ [[ "${PREDECESSOR_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
+ test "${PRUNE_INCOMPATIBLE_REVISIONS}" = false
+ test "${REGIONAL_PLACEMENT_MODE}" = preserve
+ test "${CONFIRMATION}" = BOOTSTRAP_RELAY_DIRECTOR_REHOME_IDENTITY
+ elif test "${PRUNE_INCOMPATIBLE_REVISIONS}" = true; then
+ test "${REGIONAL_PLACEMENT_MODE}" = preserve
+ test "${CONFIRMATION}" = PRUNE_INCOMPATIBLE_RELAY_DIRECTOR_REVISIONS
+ elif test "${REGIONAL_PLACEMENT_MODE}" = disable; then
+ test "${CONFIRMATION}" = FORCE_RELAY_US_FIRST
+ else
+ test -z "${CONFIRMATION}"
+ fi
+ echo "IMAGE=${IMAGE}" >> "${GITHUB_ENV}"
+
+ # Why: the deploy INHERITS the serving revision's floor, so when that revision has
+ # already lost it the candidate inherits zero, the in-script gate compares zero against
+ # zero and passes, and the post-deploy check below only notices after traffic moved.
+ # The documented rollback target is created at minimum instances zero, so promoting it
+ # arms exactly that. Refuse to inherit a degraded floor rather than latch it.
+ - name: Require a healthy serving floor before deploying
+ shell: bash
+ run: |
+ SERVING="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" \
+ --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
+ | jq -r '[.status.traffic[] | select((.percent // 0) > 0)]
+ | if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
+ test -n "${SERVING}"
+ FLOOR="$(gcloud run revisions describe "${SERVING}" \
+ --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
+ --format="value(metadata.annotations['autoscaling.knative.dev/minScale'])")"
+ if [[ "${FLOOR:-0}" -lt "${DIRECTOR_MIN_INSTANCES}" ]]; then
+ echo "serving revision ${SERVING} holds ${FLOOR:-0} minimum instances," \
+ "below ${DIRECTOR_MIN_INSTANCES}; deploying would inherit and latch it." >&2
+ echo "Restore the floor first: gcloud run services update ${DIRECTOR_SERVICE_NAME}" \
+ "--min-instances=${DIRECTOR_MIN_INSTANCES}" >&2
+ exit 1
+ fi
+ CEILING="$(gcloud run revisions describe "${SERVING}" \
+ --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
+ --format="value(metadata.annotations['autoscaling.knative.dev/maxScale'])")"
+ test "${CEILING}" = "${DIRECTOR_MAX_INSTANCES}"
+ echo "serving revision ${SERVING} holds ${FLOOR} minimum instances"
+ echo "SERVING_REVISION=${SERVING}" >> "${GITHUB_ENV}"
+
+ # Why: no --min-instances here. The candidate inherits the Terraform-owned
+ # scaling, and this step ends with 100% traffic on it. Pinning 1 rebuilt the
+ # per-instance admission shortage that took placement failures to ~70%.
+ - name: Deploy director blue/green
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ run: |
+ served_version="$(gcloud run revisions describe "${SERVING_REVISION}" \
+ --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
+ | jq -r '[.spec.containers[0].env[]? |
+ select(.name == "ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED") |
+ (.valueSource.secretKeyRef // .valueFrom.secretKeyRef // {}) |
+ (.version // .key // empty)] |
+ if length == 1 then .[0] else empty end')"
+ if [[ "${served_version}" =~ ^[1-9][0-9]*$ ]]; then
+ current_version="${served_version}"
+ else
+ test "${REGIONAL_PLACEMENT_MODE}" = preserve
+ current_version="$(gcloud secrets versions describe latest \
+ --project "${GCP_PROJECT_ID}" --secret "${REGIONAL_PLACEMENT_SECRET}" \
+ --format='value(name)' | awk -F/ '{print $NF}')"
+ [[ "${current_version}" =~ ^[1-9][0-9]*$ ]]
+ fi
+ current="$(gcloud secrets versions access "${current_version}" \
+ --project "${GCP_PROJECT_ID}" --secret "${REGIONAL_PLACEMENT_SECRET}")"
+ [[ "${current}" =~ ^(true|false)$ ]]
+ case "${REGIONAL_PLACEMENT_MODE}" in
+ preserve) desired="${current}" ;;
+ enable) desired=true ;;
+ disable) desired=false ;;
+ *) echo "regional-placement-mode is invalid" >&2; exit 1 ;;
+ esac
+ if test "${current}" != "${desired}"; then
+ target_version="$(printf '%s' "${desired}" | gcloud secrets versions add \
+ "${REGIONAL_PLACEMENT_SECRET}" --project "${GCP_PROJECT_ID}" --data-file=- \
+ --format='value(name)' --quiet | awk -F/ '{print $NF}')"
+ else
+ target_version="${current_version}"
+ fi
+ [[ "${target_version}" =~ ^[1-9][0-9]*$ ]]
+ RELEASE_ID="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}"
+ node dev/scripts/deploy-relay-blue-green.mjs \
+ --project "${GCP_PROJECT_ID}" \
+ --region "${GCP_REGION}" \
+ --service "${DIRECTOR_SERVICE_NAME}" \
+ --image "${IMAGE}" \
+ --role director \
+ --runtime-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
+ --predecessor-runtime-service-account "${PREDECESSOR_RUNTIME_SERVICE_ACCOUNT}" \
+ --bootstrap-runtime-identity "${BOOTSTRAP_RUNTIME_IDENTITY}" \
+ --predecessor-image-digest "${PREDECESSOR_IMAGE_DIGEST}" \
+ --rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
+ --rehome-audience "${REHOME_AUDIENCE}" \
+ --rehome-control-origin https://relay.onorca.dev \
+ --admin-audience https://relay.onorca.dev/v1/admin/drain \
+ --expected-rehome-generation "${EXPECTED_REHOME_GENERATION}" \
+ --max-instances "${DIRECTOR_MAX_INSTANCES}" \
+ --prune-revisions "${PRUNE_INCOMPATIBLE_REVISIONS}" \
+ --release-id "${RELEASE_ID}" \
+ --regional-placement-secret-version "${target_version}"
+ echo "REGIONAL_PLACEMENT_ENABLED=${desired}" >> "${GITHUB_ENV}"
+ echo "REGIONAL_PLACEMENT_VERSION=${target_version}" >> "${GITHUB_ENV}"
+
+ - name: Verify served revision and native health
+ shell: bash
+ run: |
+ SERVICE_JSON="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" \
+ --project "${GCP_PROJECT_ID}" \
+ --region "${GCP_REGION}" \
+ --format=json)"
+ REVISION="$(jq -r '[.status.traffic[] | select((.percent // 0) > 0)] | if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end' <<< "${SERVICE_JSON}")"
+ test -n "${REVISION}"
+ SERVED_IMAGE="$(gcloud run revisions describe "${REVISION}" \
+ --project "${GCP_PROJECT_ID}" \
+ --region "${GCP_REGION}" \
+ --format='value(spec.containers[0].image)')"
+ test "${SERVED_IMAGE}" = "${IMAGE}"
+ SERVED_REGIONAL_PLACEMENT_SECRET="$(gcloud run revisions describe "${REVISION}" \
+ --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
+ | jq -cer '[.spec.containers[0].env[] |
+ select(.name == "ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED") |
+ (.valueSource.secretKeyRef // .valueFrom.secretKeyRef // {}) |
+ {secret: (.secret // .name), version: (.version // .key)}] |
+ if length == 1 then .[0] else error("regional placement secret missing") end')"
+ test "$(jq -r '.secret' <<< "${SERVED_REGIONAL_PLACEMENT_SECRET}")" = \
+ "${REGIONAL_PLACEMENT_SECRET}"
+ test "$(jq -r '.version' <<< "${SERVED_REGIONAL_PLACEMENT_SECRET}")" = \
+ "${REGIONAL_PLACEMENT_VERSION}"
+ test "$(gcloud secrets versions access "${REGIONAL_PLACEMENT_VERSION}" --project "${GCP_PROJECT_ID}" \
+ --secret "${REGIONAL_PLACEMENT_SECRET}")" = "${REGIONAL_PLACEMENT_ENABLED}"
+ # Why: a served revision with no warm-instance floor still passes health and digest
+ # checks while quietly shrinking per-instance admission capacity.
+ SERVED_MIN_INSTANCES="$(gcloud run revisions describe "${REVISION}" \
+ --project "${GCP_PROJECT_ID}" \
+ --region "${GCP_REGION}" \
+ --format="value(metadata.annotations['autoscaling.knative.dev/minScale'])")"
+ if [[ "${SERVED_MIN_INSTANCES:-0}" -lt "${DIRECTOR_MIN_INSTANCES}" ]]; then
+ echo "served revision ${REVISION} holds ${SERVED_MIN_INSTANCES:-0} minimum instances, expected at least ${DIRECTOR_MIN_INSTANCES}" >&2
+ exit 1
+ fi
+ SERVED_MAX_INSTANCES="$(gcloud run revisions describe "${REVISION}" \
+ --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
+ --format="value(metadata.annotations['autoscaling.knative.dev/maxScale'])")"
+ test "${SERVED_MAX_INSTANCES}" = "${DIRECTOR_MAX_INSTANCES}"
+ SERVICE_URL="$(jq -r '.status.url' <<< "${SERVICE_JSON}")"
+ node dev/scripts/smoke-relay.mjs "${SERVICE_URL}"
diff --git a/.github/workflows/cloud-deploy-relay-production-multi-target.yml b/.github/workflows/cloud-deploy-relay-production-multi-target.yml
new file mode 100644
index 00000000000..8753cf81895
--- /dev/null
+++ b/.github/workflows/cloud-deploy-relay-production-multi-target.yml
@@ -0,0 +1,497 @@
+name: Deploy Relay Production Multi-Target
+
+on:
+ workflow_dispatch:
+ inputs:
+ source-cell-id:
+ description: Existing Terraform source cell ID
+ required: true
+ type: string
+ target-cell-ids:
+ description: Comma-separated distinct Terraform target cell IDs
+ required: true
+ type: string
+ general-cell-ids:
+ description: Comma-separated proven cells that remain eligible for ordinary placement
+ required: false
+ type: string
+ unobserved-connection-bound:
+ description: Exact worst-case unobserved connection bound proven by the passing load gate
+ required: false
+ type: string
+ failed-target-cell-id:
+ description: Registered failed target to fence and supersede
+ required: false
+ type: string
+ replacement-target-cell-id:
+ description: Healthy replacement for registered failed target
+ required: false
+ type: string
+ mode:
+ description: Preflight/audit are read-only; other modes mutate production
+ required: true
+ default: preflight
+ type: choice
+ options:
+ - audit
+ - preflight
+ - cutover-admission
+ - add-migration-cells
+ - promote-general-cell
+ - retire-migration-cell
+ - execute
+ - recover-forward
+ - fence-source
+ - supersede-target
+ confirmation:
+ description: Enter CUTOVER_SELECTOR, ADD_MIGRATION_CELLS, PROMOTE_GENERAL_CELL, RETIRE_MIGRATION_CELL, EVACUATE_MULTI, RECOVER_FORWARD, or FENCE_SOURCE
+ required: false
+ type: string
+ selector-attempt-id:
+ description: Exact durable selector attempt ID for admission mutations
+ required: false
+ type: string
+ monitor-run-id:
+ description: Successful fresh dry-run monitor workflow run ID
+ required: false
+ type: string
+ monitor-run-attempt:
+ description: Exact dry-run monitor workflow attempt
+ required: false
+ type: string
+ broker-operation-id:
+ description: Stable durable broker operation ID for target supersession
+ required: false
+ type: string
+ completed-fence-attempt-id:
+ description: Exact older completed fence attempt to recover without replay
+ required: false
+ type: string
+ completed-fence-commit:
+ description: Exact older fence commit bound to the completed attempt
+ required: false
+ type: string
+ completed-fence-operation:
+ description: Exact DONE Compute resize operation to adopt
+ required: false
+ type: string
+ completed-fence-state-serial:
+ description: Exact Terraform serial before the completed fence
+ required: false
+ type: string
+ completed-fence-plan-generation:
+ description: Exact saved-plan object generation
+ required: false
+ type: string
+ completed-fence-state-generation:
+ description: Exact current Terraform state object generation
+ required: false
+ type: string
+ completed-fence-state-sha256:
+ description: Exact current Terraform state object SHA-256
+ required: false
+ type: string
+ expected-lease-generation:
+ description: Exact live lease generation authorized for conditional takeover
+ required: false
+ type: string
+ expected-lease-operation-id:
+ description: Exact live lease operation ID authorized for takeover
+ required: false
+ type: string
+ expected-lease-request-digest:
+ description: Exact live lease request digest authorized for takeover
+ required: false
+ type: string
+
+permissions:
+ actions: read
+ contents: read
+ id-token: write
+
+concurrency:
+ group: production-cloud-sql-rollout
+ cancel-in-progress: false
+
+defaults:
+ run:
+ working-directory: cloud
+
+jobs:
+ deploy:
+ if: >-
+ ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' &&
+ github.ref == 'refs/heads/main' }}
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ environment: production
+ env:
+ GCP_PROJECT_ID: onorca-cloud
+ DIRECTOR_ORIGIN: https://relay.onorca.dev
+ ADMIN_AUDIENCE: https://relay.onorca.dev/v1/admin/drain
+ SOURCE_CELL_ID: ${{ inputs.source-cell-id }}
+ TARGET_CELL_IDS: ${{ inputs.target-cell-ids }}
+ GENERAL_CELL_IDS: ${{ inputs.general-cell-ids }}
+ UNOBSERVED_CONNECTION_BOUND: ${{ inputs.unobserved-connection-bound }}
+ FAILED_TARGET_CELL_ID: ${{ inputs.failed-target-cell-id }}
+ REPLACEMENT_TARGET_CELL_ID: ${{ inputs.replacement-target-cell-id }}
+ DEPLOY_MODE: ${{ inputs.mode }}
+ MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
+ MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }}
+ SELECTOR_ATTEMPT_ID: ${{ inputs.selector-attempt-id }}
+ BROKER_OPERATION_ID: ${{ inputs.broker-operation-id }}
+ COMPLETED_FENCE_ATTEMPT_ID: ${{ inputs.completed-fence-attempt-id }}
+ COMPLETED_FENCE_COMMIT: ${{ inputs.completed-fence-commit }}
+ COMPLETED_FENCE_OPERATION: ${{ inputs.completed-fence-operation }}
+ COMPLETED_FENCE_STATE_SERIAL: ${{ inputs.completed-fence-state-serial }}
+ COMPLETED_FENCE_PLAN_GENERATION: ${{ inputs.completed-fence-plan-generation }}
+ COMPLETED_FENCE_STATE_GENERATION: ${{ inputs.completed-fence-state-generation }}
+ COMPLETED_FENCE_STATE_SHA256: ${{ inputs.completed-fence-state-sha256 }}
+ EXPECTED_LEASE_GENERATION: ${{ inputs.expected-lease-generation }}
+ EXPECTED_LEASE_OPERATION_ID: ${{ inputs.expected-lease-operation-id }}
+ EXPECTED_LEASE_REQUEST_DIGEST: ${{ inputs.expected-lease-request-digest }}
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Require private fence-broker environment
+ if: >-
+ ${{ inputs.mode == 'fence-source' ||
+ inputs.mode == 'supersede-target' }}
+ env:
+ FENCE_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_WORKLOAD_IDENTITY_PROVIDER }}
+ FENCE_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_SERVICE_ACCOUNT }}
+ FENCE_BROKER_URI: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_BROKER_URI }}
+ run: |
+ test -n "${FENCE_WORKLOAD_IDENTITY_PROVIDER}"
+ test -n "${FENCE_SERVICE_ACCOUNT}"
+ test -n "${FENCE_BROKER_URI}"
+
+ - name: Reject direct-runner Terraform fence aborts
+ if: ${{ inputs.mode == 'abort-fence-source' }}
+ run: |
+ echo "Terraform fence aborts require a reviewed private-broker recovery path." >&2
+ exit 1
+
+ - name: Require fresh dry-run evidence reference
+ if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
+ run: |
+ [[ "${MONITOR_RUN_ID}" =~ ^[0-9]+$ ]]
+ [[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
+
+ - name: Download private dry-run evidence
+ if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
+ uses: actions/download-artifact@v4
+ with:
+ name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
+ path: ${{ runner.temp }}/relay-monitor-evidence
+ github-token: ${{ github.token }}
+ run-id: ${{ inputs.monitor-run-id }}
+
+ - uses: pnpm/action-setup@v4
+ with:
+ package_json_file: cloud/package.json
+
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 24
+
+ - run: pnpm install --frozen-lockfile
+
+ - uses: hashicorp/setup-terraform@v3
+ with:
+ terraform_wrapper: false
+
+ - name: Verify dry-run artifact before cloud authentication
+ if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
+ run: |
+ node dev/scripts/relay-monitor-evidence.mjs verify-restore \
+ --directory "${RUNNER_TEMP}/relay-monitor-evidence" \
+ --incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
+ --run-id "${MONITOR_RUN_ID}" \
+ --run-attempt "${MONITOR_RUN_ATTEMPT}" \
+ --commit-sha "${GITHUB_SHA}" \
+ --mode dry-run
+
+ - name: Reject previously consumed dry-run evidence
+ if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: |
+ MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
+ COUNT="$(gh api \
+ "/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \
+ --jq '.total_count')"
+ test "${COUNT}" = "0"
+
+ - id: google-auth
+ if: ${{ inputs.mode != 'supersede-target' }}
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - uses: google-github-actions/setup-gcloud@v2
+
+ - uses: ./.github/actions/cloud-sql-rollout-lease
+ with:
+ bucket: onorca-cloud-terraform-state
+ object: terraform/state/cloud-sql-rollout/production.lock
+
+ - name: Require explicit mutation confirmation
+ if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
+ env:
+ CONFIRMATION: ${{ inputs.confirmation }}
+ run: |
+ if [[ "${DEPLOY_MODE}" = "cutover-admission" ]]; then
+ test "${CONFIRMATION}" = "CUTOVER_SELECTOR"
+ elif [[ "${DEPLOY_MODE}" = "add-migration-cells" ]]; then
+ test "${CONFIRMATION}" = "ADD_MIGRATION_CELLS"
+ elif [[ "${DEPLOY_MODE}" = "promote-general-cell" ]]; then
+ test "${CONFIRMATION}" = "PROMOTE_GENERAL_CELL"
+ elif [[ "${DEPLOY_MODE}" = "retire-migration-cell" ]]; then
+ test "${CONFIRMATION}" = "RETIRE_MIGRATION_CELL"
+ elif [[ "${DEPLOY_MODE}" = "execute" ]]; then
+ test "${CONFIRMATION}" = "EVACUATE_MULTI"
+ elif [[ "${DEPLOY_MODE}" = "recover-forward" ]]; then
+ test "${CONFIRMATION}" = "RECOVER_FORWARD"
+ elif [[ "${DEPLOY_MODE}" = "fence-source" ]]; then
+ test "${CONFIRMATION}" = "FENCE_SOURCE"
+ elif [[ "${DEPLOY_MODE}" = "supersede-target" ]]; then
+ test "${CONFIRMATION}" = "SUPERSEDE_TARGET"
+ elif [[ "${DEPLOY_MODE}" = "abort-fence-source" ]]; then
+ test "${CONFIRMATION}" = "ABORT_FENCE"
+ else
+ test "${CONFIRMATION}" = "FENCE_SOURCE"
+ fi
+
+ - name: Require exact source-fence broker contract
+ if: ${{ inputs.mode == 'fence-source' }}
+ run: |
+ test "${SOURCE_CELL_ID}" = "production-gce-c3"
+ test "${TARGET_CELL_IDS}" = "production-gce-c7,production-gce-c8,production-gce-c10,production-gce-c13,production-gce-c17,production-gce-c18"
+ [[ "${BROKER_OPERATION_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]]
+ if [[ -n "${EXPECTED_LEASE_GENERATION}" ]]; then
+ [[ "${EXPECTED_LEASE_GENERATION}" =~ ^[1-9][0-9]*$ ]]
+ test "${EXPECTED_LEASE_OPERATION_ID}" = "${BROKER_OPERATION_ID}"
+ [[ "${EXPECTED_LEASE_REQUEST_DIGEST}" =~ ^[0-9a-f]{64}$ ]]
+ else
+ test -z "${EXPECTED_LEASE_OPERATION_ID}"
+ test -z "${EXPECTED_LEASE_REQUEST_DIGEST}"
+ fi
+
+ - name: Require exact broker cell contract
+ if: ${{ inputs.mode == 'supersede-target' }}
+ run: |
+ test "${SOURCE_CELL_ID}" = "production-gce-c3"
+ test "${FAILED_TARGET_CELL_ID}" = "production-gce-c12"
+ test "${REPLACEMENT_TARGET_CELL_ID}" = "production-gce-c13"
+ test "${TARGET_CELL_IDS}" = "production-gce-c12,production-gce-c13"
+ if [[ -n "${COMPLETED_FENCE_ATTEMPT_ID}" ]]; then
+ [[ "${COMPLETED_FENCE_ATTEMPT_ID}" =~ ^[0-9a-f-]{36}$ ]]
+ [[ "${COMPLETED_FENCE_COMMIT}" =~ ^[0-9a-f]{40}$ ]]
+ [[ "${COMPLETED_FENCE_OPERATION}" =~ ^[A-Za-z0-9._-]{1,256}$ ]]
+ [[ "${COMPLETED_FENCE_STATE_SERIAL}" =~ ^[0-9]+$ ]]
+ [[ "${COMPLETED_FENCE_PLAN_GENERATION}" =~ ^[1-9][0-9]*$ ]]
+ [[ "${COMPLETED_FENCE_STATE_GENERATION}" =~ ^[1-9][0-9]*$ ]]
+ [[ "${COMPLETED_FENCE_STATE_SHA256}" =~ ^[0-9a-f]{64}$ ]]
+ test -n "${EXPECTED_LEASE_GENERATION}"
+ fi
+ if [[ -n "${EXPECTED_LEASE_GENERATION}" ]]; then
+ [[ "${EXPECTED_LEASE_GENERATION}" =~ ^[1-9][0-9]*$ ]]
+ test "${EXPECTED_LEASE_OPERATION_ID}" = "${BROKER_OPERATION_ID}"
+ [[ "${EXPECTED_LEASE_REQUEST_DIGEST}" =~ ^[0-9a-f]{64}$ ]]
+ else
+ test -z "${EXPECTED_LEASE_OPERATION_ID}"
+ test -z "${EXPECTED_LEASE_REQUEST_DIGEST}"
+ fi
+
+ - name: Read reviewed Terraform topology
+ if: ${{ inputs.mode != 'supersede-target' }}
+ run: |
+ node dev/scripts/infra.mjs init --env production
+ terraform -chdir=infra/terraform output -json relay_gce_cell_deployments > "${RUNNER_TEMP}/relay-gce-topology.json"
+ RUNTIME_SERVICE_ACCOUNT="$(terraform -chdir=infra/terraform output -raw relay_runtime_service_account)"
+ DIRECTOR_MIN_INSTANCES="$(terraform -chdir=infra/terraform console \
+ -var-file=environments/production.tfvars <<< 'var.relay_min_instances')"
+ [[ "${DIRECTOR_MIN_INSTANCES}" =~ ^[1-9][0-9]*$ ]]
+ echo "RUNTIME_SERVICE_ACCOUNT=${RUNTIME_SERVICE_ACCOUNT}" >> "${GITHUB_ENV}"
+ echo "DIRECTOR_MIN_INSTANCES=${DIRECTOR_MIN_INSTANCES}" >> "${GITHUB_ENV}"
+
+ - name: Verify fresh dry-run evidence against live selector
+ if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ run: |
+ SCOPED_RECOVERY_ARGS=()
+ if [[ ("${DEPLOY_MODE}" = "execute" ||
+ "${DEPLOY_MODE}" = "recover-forward") &&
+ "${SOURCE_CELL_ID}" = "production-gce-c12" ]]; then
+ SCOPED_RECOVERY_ARGS=(
+ --scoped-recovery-source-cell-id
+ production-gce-c3
+ )
+ fi
+ node dev/scripts/relay-monitor-evidence.mjs verify-mutation \
+ --directory "${RUNNER_TEMP}/relay-monitor-evidence" \
+ --incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
+ --run-id "${MONITOR_RUN_ID}" \
+ --run-attempt "${MONITOR_RUN_ATTEMPT}" \
+ --commit-sha "${GITHUB_SHA}" \
+ --mode dry-run \
+ --mutation-mode "${DEPLOY_MODE}" \
+ --source-cell-id "${SOURCE_CELL_ID}" \
+ "${SCOPED_RECOVERY_ARGS[@]}" \
+ --director-origin "${DIRECTOR_ORIGIN}"
+
+ - name: Recheck all live safety signals
+ if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ run: |
+ pnpm incident:relay-preflight -- \
+ --state-file "${RUNNER_TEMP}/relay-monitor-evidence/relay-${MONITOR_RUN_ID}-dry-run.state.json"
+
+ - name: Create single-use dry-run marker
+ if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
+ run: |
+ MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
+ mkdir -p "${RUNNER_TEMP}/relay-monitor-consumption"
+ printf '%s\n' "${GITHUB_RUN_ID}" \
+ > "${RUNNER_TEMP}/relay-monitor-consumption/${MARKER_NAME}"
+
+ - name: Consume dry-run evidence
+ if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' && inputs.mode != 'cutover-admission' && inputs.mode != 'add-migration-cells' && inputs.mode != 'promote-general-cell' && inputs.mode != 'retire-migration-cell' && inputs.mode != 'supersede-target' }}
+ uses: actions/upload-artifact@v4
+ with:
+ name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
+ path: ${{ runner.temp }}/relay-monitor-consumption/relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
+ retention-days: 90
+ if-no-files-found: error
+
+ - id: google-fence-broker-auth
+ if: >-
+ ${{ inputs.mode == 'fence-source' ||
+ inputs.mode == 'supersede-target' }}
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_BROKER_URI }}
+ id_token_include_email: true
+
+ - name: Invoke private target-supersession broker
+ if: ${{ inputs.mode == 'supersede-target' }}
+ env:
+ BROKER_ID_TOKEN: ${{ steps.google-fence-broker-auth.outputs.id_token }}
+ BROKER_URI: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_BROKER_URI }}
+ run: |
+ [[ "${BROKER_OPERATION_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]]
+ if [[ -n "${COMPLETED_FENCE_ATTEMPT_ID}" ]]; then
+ REQUEST="$(jq -cn \
+ --arg operationId "${BROKER_OPERATION_ID}" \
+ --arg fenceCommit "${GITHUB_SHA}" \
+ --arg attemptId "${COMPLETED_FENCE_ATTEMPT_ID}" \
+ --arg completedCommit "${COMPLETED_FENCE_COMMIT}" \
+ --arg gceOperation "${COMPLETED_FENCE_OPERATION}" \
+ --arg stateSerial "${COMPLETED_FENCE_STATE_SERIAL}" \
+ --arg planGeneration "${COMPLETED_FENCE_PLAN_GENERATION}" \
+ --arg stateGeneration "${COMPLETED_FENCE_STATE_GENERATION}" \
+ --arg stateSha256 "${COMPLETED_FENCE_STATE_SHA256}" \
+ --arg leaseGeneration "${EXPECTED_LEASE_GENERATION}" \
+ --arg leaseOperationId "${EXPECTED_LEASE_OPERATION_ID}" \
+ --arg leaseRequestDigest "${EXPECTED_LEASE_REQUEST_DIGEST}" \
+ '{v:1,operationId:$operationId,fenceCommit:$fenceCommit,
+ completedFenceRecovery:{attemptId:$attemptId,fenceCommit:$completedCommit,
+ gceOperation:$gceOperation,terraformStateSerial:($stateSerial|tonumber),
+ planObjectGeneration:$planGeneration,
+ terraformStateObjectGeneration:$stateGeneration,
+ terraformStateObjectSha256:$stateSha256},
+ expectedLease:{generation:$leaseGeneration,operationId:$leaseOperationId,
+ requestDigest:$leaseRequestDigest},confirmation:"SUPERSEDE_TARGET"}')"
+ elif [[ -n "${EXPECTED_LEASE_GENERATION}" ]]; then
+ REQUEST="$(jq -cn \
+ --arg operationId "${BROKER_OPERATION_ID}" \
+ --arg fenceCommit "${GITHUB_SHA}" \
+ --arg leaseGeneration "${EXPECTED_LEASE_GENERATION}" \
+ --arg leaseOperationId "${EXPECTED_LEASE_OPERATION_ID}" \
+ --arg leaseRequestDigest "${EXPECTED_LEASE_REQUEST_DIGEST}" \
+ '{v:1,operationId:$operationId,fenceCommit:$fenceCommit,
+ expectedLease:{generation:$leaseGeneration,operationId:$leaseOperationId,
+ requestDigest:$leaseRequestDigest},confirmation:"SUPERSEDE_TARGET"}')"
+ else
+ REQUEST="$(jq -cn \
+ --arg operationId "${BROKER_OPERATION_ID}" \
+ --arg fenceCommit "${GITHUB_SHA}" \
+ '{v:1,operationId:$operationId,fenceCommit:$fenceCommit,confirmation:"SUPERSEDE_TARGET"}')"
+ fi
+ curl --fail-with-body --max-time 1790 \
+ --request POST "${BROKER_URI}/v1/supersede-target" \
+ --header "Authorization: Bearer ${BROKER_ID_TOKEN}" \
+ --header 'Content-Type: application/json' \
+ --data "${REQUEST}"
+
+ - name: Invoke private source-fence broker
+ if: ${{ inputs.mode == 'fence-source' }}
+ env:
+ BROKER_ID_TOKEN: ${{ steps.google-fence-broker-auth.outputs.id_token }}
+ BROKER_URI: ${{ vars.PRODUCTION_GCP_RELAY_FENCE_BROKER_URI }}
+ run: |
+ if [[ -n "${EXPECTED_LEASE_GENERATION}" ]]; then
+ REQUEST="$(jq -cn \
+ --arg operationId "${BROKER_OPERATION_ID}" \
+ --arg fenceCommit "${GITHUB_SHA}" \
+ --arg targetCellIds "${TARGET_CELL_IDS}" \
+ --arg leaseGeneration "${EXPECTED_LEASE_GENERATION}" \
+ --arg leaseOperationId "${EXPECTED_LEASE_OPERATION_ID}" \
+ --arg leaseRequestDigest "${EXPECTED_LEASE_REQUEST_DIGEST}" \
+ '{v:1,operationId:$operationId,fenceCommit:$fenceCommit,
+ targetCellIds:($targetCellIds|split(",")),
+ expectedLease:{generation:$leaseGeneration,operationId:$leaseOperationId,
+ requestDigest:$leaseRequestDigest},confirmation:"FENCE_SOURCE"}')"
+ else
+ REQUEST="$(jq -cn \
+ --arg operationId "${BROKER_OPERATION_ID}" \
+ --arg fenceCommit "${GITHUB_SHA}" \
+ --arg targetCellIds "${TARGET_CELL_IDS}" \
+ '{v:1,operationId:$operationId,fenceCommit:$fenceCommit,
+ targetCellIds:($targetCellIds|split(",")),confirmation:"FENCE_SOURCE"}')"
+ fi
+ curl --fail-with-body --max-time 1790 \
+ --request POST "${BROKER_URI}/v1/fence-source" \
+ --header "Authorization: Bearer ${BROKER_ID_TOKEN}" \
+ --header 'Content-Type: application/json' \
+ --data "${REQUEST}"
+
+ - name: Preflight or run multi-target evacuation
+ if: >-
+ ${{ inputs.mode != 'fence-source' &&
+ inputs.mode != 'supersede-target' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ run: |
+ node dev/scripts/deploy-relay-gce-multi-target.mjs \
+ --project "${GCP_PROJECT_ID}" \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --admin-audience "${ADMIN_AUDIENCE}" \
+ --topology-file "${RUNNER_TEMP}/relay-gce-topology.json" \
+ --source-cell-id "${SOURCE_CELL_ID}" \
+ --target-cell-ids "${TARGET_CELL_IDS}" \
+ --general-cell-ids "${GENERAL_CELL_IDS}" \
+ --unobserved-connection-bound "${UNOBSERVED_CONNECTION_BOUND}" \
+ --director-region "${{ vars.PRODUCTION_GCP_REGION }}" \
+ --director-service "orca-cloud-relay" \
+ --director-min-instances "${DIRECTOR_MIN_INSTANCES}" \
+ --selector-attempt-id "${SELECTOR_ATTEMPT_ID}" \
+ --failed-target-cell-id "${FAILED_TARGET_CELL_ID}" \
+ --replacement-target-cell-id "${REPLACEMENT_TARGET_CELL_ID}" \
+ --runtime-service-account "${RUNTIME_SERVICE_ACCOUNT}" \
+ --environment production \
+ --fence-commit "${GITHUB_SHA}" \
+ --terraform-dir infra/terraform \
+ --terraform-var-file environments/production.tfvars \
+ --mode "${DEPLOY_MODE}" \
+ --connection-ceiling 1000 \
+ --minimum-lease-remaining-ms 600000
diff --git a/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml b/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
new file mode 100644
index 00000000000..8ef61507088
--- /dev/null
+++ b/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
@@ -0,0 +1,668 @@
+name: Deploy Relay Production Same-Cap Job
+
+on:
+ workflow_call:
+ inputs:
+ mode: { required: true, type: string }
+ target-cell-id: { required: true, type: string }
+ target-image-digest: { required: true, type: string }
+ rollback-image-digest: { required: true, type: string }
+ target-rehome-protocol: { required: true, type: string }
+ rollback-rehome-protocol: { required: true, type: string }
+ expected-selector-generation: { required: true, type: string }
+ expected-existing-only-cells: { required: true, type: string }
+ expected-migration-only-cells: { required: true, type: string }
+ expected-general-cells: { required: true, type: string }
+ expected-rehome-generation: { required: true, type: string }
+ monitor-run-id: { required: true, type: string }
+ monitor-run-attempt: { required: true, type: string }
+ wave-index: { required: true, type: string }
+
+permissions:
+ actions: read
+ contents: read
+ id-token: write
+
+defaults:
+ run:
+ working-directory: cloud
+
+jobs:
+ rollout:
+ if: ${{ github.ref == 'refs/heads/main' }}
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ timeout-minutes: 75
+ environment: production
+ env:
+ GCP_PROJECT_ID: onorca-cloud
+ GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
+ DIRECTOR_ORIGIN: https://relay.onorca.dev
+ IMAGE_REPOSITORY: us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay
+ TARGET_CELL_ID: ${{ inputs.target-cell-id }}
+ DEPLOY_MODE: ${{ inputs.mode }}
+ TARGET_IMAGE_DIGEST: ${{ inputs.target-image-digest }}
+ ROLLBACK_IMAGE_DIGEST: ${{ inputs.rollback-image-digest }}
+ TARGET_REHOME_PROTOCOL: ${{ inputs.target-rehome-protocol }}
+ ROLLBACK_REHOME_PROTOCOL: ${{ inputs.rollback-rehome-protocol }}
+ EXPECTED_SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }}
+ EXPECTED_EXISTING_ONLY_CELLS: ${{ inputs.expected-existing-only-cells }}
+ EXPECTED_MIGRATION_ONLY_CELLS: ${{ inputs.expected-migration-only-cells }}
+ EXPECTED_GENERAL_CELLS: ${{ inputs.expected-general-cells }}
+ EXPECTED_REHOME_GENERATION: ${{ inputs.expected-rehome-generation }}
+ WAVE_INDEX: ${{ inputs.wave-index }}
+ MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
+ MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }}
+ OUTPUT_DIRECTORY: ${{ github.workspace }}/relay-monitor-evidence
+ steps:
+ - name: Require exact reusable-workflow configuration
+ working-directory: .
+ env:
+ DEPLOY_WIF: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+ DEPLOY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+ CAPACITY_WIF: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
+ CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
+ DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
+ run: |
+ [[ "${DEPLOY_MODE}" =~ ^(verify|apply|rollback)$ ]]
+ [[ "${TARGET_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
+ [[ "${ROLLBACK_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
+ test "${TARGET_IMAGE_DIGEST}" != "${ROLLBACK_IMAGE_DIGEST}"
+ [[ "${TARGET_REHOME_PROTOCOL}" =~ ^[01]$ ]]
+ [[ "${ROLLBACK_REHOME_PROTOCOL}" =~ ^[01]$ ]]
+ [[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
+ [[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
+ [[ "${WAVE_INDEX}" =~ ^[0-3]$ ]]
+ if test "${DEPLOY_MODE}" = verify; then
+ EFFECTIVE_SELECTOR_GENERATION="${EXPECTED_SELECTOR_GENERATION}"
+ else
+ EFFECTIVE_SELECTOR_GENERATION="$((EXPECTED_SELECTOR_GENERATION + (2 * WAVE_INDEX)))"
+ fi
+ echo "EFFECTIVE_SELECTOR_GENERATION=${EFFECTIVE_SELECTOR_GENERATION}" >> "${GITHUB_ENV}"
+ if test "${DEPLOY_MODE}" != verify && test "${GITHUB_RUN_ATTEMPT}" != 1; then
+ echo "mutations are single-dispatch: re-runs replay aged evidence," >&2
+ echo "so recover each remaining cell with its own fresh monitor" >&2
+ echo "dry-run and canary-apply dispatch instead" >&2
+ exit 1
+ fi
+ test -n "${GCP_REGION}"
+ test -n "${DEPLOY_WIF}"
+ test -n "${DEPLOY_SERVICE_ACCOUNT}"
+ test -n "${CAPACITY_WIF}"
+ test -n "${CAPACITY_SERVICE_ACCOUNT}"
+ test -n "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}"
+
+ # Full history: the monitor evidence this job verifies is sealed at an ancestor commit,
+ # and the provenance check fails closed on a commit a shallow clone left out.
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - uses: pnpm/action-setup@v4
+ with: { package_json_file: cloud/package.json }
+
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 24
+
+ - run: pnpm install --frozen-lockfile
+
+ - uses: hashicorp/setup-terraform@v3
+ with: { terraform_wrapper: false }
+
+ - name: Require fresh aggregate monitor evidence reference
+ if: ${{ inputs.mode != 'verify' }}
+ run: |
+ [[ "${MONITOR_RUN_ID}" =~ ^[1-9][0-9]*$ ]]
+ [[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
+
+ - name: Download private aggregate monitor evidence
+ if: ${{ inputs.mode != 'verify' }}
+ uses: actions/download-artifact@v4
+ with:
+ name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
+ path: ${{ github.workspace }}/relay-monitor-evidence
+ github-token: ${{ github.token }}
+ run-id: ${{ inputs.monitor-run-id }}
+
+ - name: Verify monitor evidence provenance
+ if: ${{ inputs.mode != 'verify' }}
+ run: |
+ node dev/scripts/relay-monitor-evidence.mjs verify-authority \
+ --directory "${OUTPUT_DIRECTORY}" \
+ --incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
+ --run-id "${MONITOR_RUN_ID}" \
+ --run-attempt "${MONITOR_RUN_ATTEMPT}" \
+ --commit-sha "${GITHUB_SHA}" \
+ --mode dry-run \
+ --required-migration-policy strict \
+ --wave-index "${WAVE_INDEX}"
+
+ - name: Download this wave's single-use safety authority
+ if: ${{ inputs.mode != 'verify' }}
+ uses: actions/download-artifact@v4
+ with:
+ name: relay-same-cap-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
+ path: ${{ runner.temp }}/relay-same-cap-monitor-authority
+ github-token: ${{ github.token }}
+ run-id: ${{ github.run_id }}
+
+ - name: Require safety evidence consumed by this workflow
+ if: ${{ inputs.mode != 'verify' }}
+ run: |
+ # Mutations are single-dispatch: a fresh dispatch cannot resume a
+ # partial batch (the canary authority binds the batch-entry selector
+ # generation), so each remaining cell is recovered by its own fresh
+ # monitor dry-run and canary-apply dispatch, never by re-running
+ # aged evidence.
+ test "${GITHUB_RUN_ATTEMPT}" = 1
+ MARKER_NAME="relay-same-cap-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
+ test "$(< "${RUNNER_TEMP}/relay-same-cap-monitor-authority/${MARKER_NAME}")" = \
+ "${GITHUB_RUN_ID}"
+
+ - id: deploy-auth
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - uses: google-github-actions/setup-gcloud@v2
+
+ - uses: ./.github/actions/cloud-sql-rollout-lease
+ with:
+ bucket: onorca-cloud-terraform-state
+ object: terraform/state/cloud-sql-rollout/production.lock
+ release: 'false'
+
+ - name: Recheck aggregate SQL, pool, reconnect, migration, and selector safety
+ if: ${{ inputs.mode != 'verify' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
+ run: |
+ # Freshness-only failures are publish lag, not health, on every wave
+ # including the first; the CLI still caps the retry at the wave's
+ # evidence-age budget, so this cannot mutate on aged evidence.
+ pnpm incident:relay-preflight -- \
+ --state-file "${OUTPUT_DIRECTORY}/relay-${MONITOR_RUN_ID}-dry-run.state.json" \
+ --wave-index "${WAVE_INDEX}" --retry-freshness
+
+ - name: Require durable rehome disabled and exact selector
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
+ run: |
+ node dev/scripts/operate-relay-regional-rehome.mjs \
+ --mode inspect \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --expected-selector-generation "${EFFECTIVE_SELECTOR_GENERATION}" \
+ --expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
+ --expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \
+ --expected-general-cells "${EXPECTED_GENERAL_CELLS}" \
+ --expected-control-generation "${EXPECTED_REHOME_GENERATION}" \
+ | jq -e '.control.enabled == false' >/dev/null
+
+ - name: Initialize the exact production backend
+ run: node dev/scripts/infra.mjs init --env production
+
+ - name: Resolve immutable same-cap cell configuration
+ shell: bash
+ run: |
+ TARGET_HOSTNAME="${TARGET_CELL_ID#production-gce-}"
+ case "${TARGET_HOSTNAME}" in
+ c7|c8|c9|c10|c13|c14|c15|c16|c19|c20|c21|c22|c23|c24|c25|c26)
+ EXPECTED_HARD_CAP=1000
+ EXPECTED_REGION=us-central1
+ ;;
+ c27|c28|c29)
+ EXPECTED_HARD_CAP=3000
+ EXPECTED_REGION=asia-east2
+ ;;
+ *) exit 1 ;;
+ esac
+ EXPECTED_UNOBSERVED_BOUND=60
+ CELL_ORIGIN="https://${TARGET_HOSTNAME}.relay.onorca.dev"
+ CELLS_JSON="$(terraform -chdir=infra/terraform console \
+ -var-file=environments/production.tfvars \
+ <<< 'jsonencode(var.relay_gce_cells)' | jq -er '.')"
+ SOURCE_CELLS="$(terraform -chdir=infra/terraform console \
+ -var-file=environments/production.tfvars \
+ <<< 'jsonencode(var.relay_region_rehome_source_cell_ids)' | jq -er '.')"
+ if test "${EXPECTED_REGION}" = us-central1; then
+ jq -e --arg cell "${TARGET_CELL_ID}" 'index($cell) != null' \
+ <<< "${SOURCE_CELLS}" >/dev/null
+ fi
+ CURRENT_SHAPE="$(jq -cer --arg cell "${TARGET_CELL_ID}" '.[$cell]' <<< "${CELLS_JSON}")"
+ test "$(jq -r '.connection_hard_cap' <<< "${CURRENT_SHAPE}")" = "${EXPECTED_HARD_CAP}"
+ test "$(jq -r '.connection_unobserved_bound' <<< "${CURRENT_SHAPE}")" = \
+ "${EXPECTED_UNOBSERVED_BOUND}"
+ TARGET_ZONE="$(jq -r '.zone' <<< "${CURRENT_SHAPE}")"
+ MIG_NAME="orca-cloud-relay-gce-${TARGET_HOSTNAME}"
+ if test "${DEPLOY_MODE}" = rollback; then
+ DESIRED_IMAGE_DIGEST="${ROLLBACK_IMAGE_DIGEST}"
+ CURRENT_IMAGE_DIGEST="${TARGET_IMAGE_DIGEST}"
+ DESIRED_REHOME_PROTOCOL="${ROLLBACK_REHOME_PROTOCOL}"
+ CURRENT_REHOME_PROTOCOL="${TARGET_REHOME_PROTOCOL}"
+ else
+ DESIRED_IMAGE_DIGEST="${TARGET_IMAGE_DIGEST}"
+ CURRENT_IMAGE_DIGEST="${ROLLBACK_IMAGE_DIGEST}"
+ DESIRED_REHOME_PROTOCOL="${TARGET_REHOME_PROTOCOL}"
+ CURRENT_REHOME_PROTOCOL="${ROLLBACK_REHOME_PROTOCOL}"
+ fi
+ DESIRED_IMAGE="${IMAGE_REPOSITORY}@${DESIRED_IMAGE_DIGEST}"
+ OVERRIDE_CELLS_JSON="$(jq -ce --arg cell "${TARGET_CELL_ID}" \
+ --arg image "${DESIRED_IMAGE}" '.[$cell].image = $image' <<< "${CELLS_JSON}")"
+ jq -n --argjson cells "${OVERRIDE_CELLS_JSON}" \
+ '{relay_gce_cells:$cells}' > "${RUNNER_TEMP}/relay-same-cap.tfvars.json"
+ SERVED_DIGEST="$(gcloud artifacts docker images describe "${DESIRED_IMAGE}" \
+ --project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')"
+ test "${SERVED_DIGEST}" = "${DESIRED_IMAGE_DIGEST}"
+ {
+ echo "TARGET_HOSTNAME=${TARGET_HOSTNAME}"
+ echo "CELL_ORIGIN=${CELL_ORIGIN}"
+ echo "TARGET_ZONE=${TARGET_ZONE}"
+ echo "MIG_NAME=${MIG_NAME}"
+ echo "EXPECTED_HARD_CAP=${EXPECTED_HARD_CAP}"
+ echo "EXPECTED_UNOBSERVED_BOUND=${EXPECTED_UNOBSERVED_BOUND}"
+ echo "EXPECTED_REGION=${EXPECTED_REGION}"
+ echo "DESIRED_IMAGE=${DESIRED_IMAGE}"
+ echo "DESIRED_IMAGE_DIGEST=${DESIRED_IMAGE_DIGEST}"
+ echo "CURRENT_IMAGE_DIGEST=${CURRENT_IMAGE_DIGEST}"
+ echo "DESIRED_REHOME_PROTOCOL=${DESIRED_REHOME_PROTOCOL}"
+ echo "CURRENT_REHOME_PROTOCOL=${CURRENT_REHOME_PROTOCOL}"
+ } >> "${GITHUB_ENV}"
+
+ - name: Verify exact current generation, digest, cap, and rollback point
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
+ run: |
+ # A single transient 5xx (LB warm-up behind a fresh instance) must not
+ # fail a canary; 4xx (auth, generation mismatch) still fails fast.
+ admin_post() {
+ local out="${RUNNER_TEMP}/$1.json"
+ if ! curl --fail-with-body --max-time 30 \
+ --retry 3 --retry-delay 2 --retry-connrefused --output "${out}" \
+ --request POST "$2" \
+ --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
+ --header 'Content-Type: application/json' --data "$3"; then
+ cat "${out}" >&2
+ return 1
+ fi
+ cat "${out}"
+ }
+ CURRENT_RUNTIME="$(admin_post current-runtime \
+ "${CELL_ORIGIN}/v1/admin/runtime-status" '{"v":1}')"
+ # A rollback that failed between template apply and admission restore
+ # leaves the cell already on the rollback image; resume from that
+ # state instead of demanding the pre-rollback predecessor.
+ LIVE_IMAGE_DIGEST="$(jq -r '.imageDigest' <<< "${CURRENT_RUNTIME}")"
+ if test "${DEPLOY_MODE}" = rollback \
+ && test "${LIVE_IMAGE_DIGEST}" = "${DESIRED_IMAGE_DIGEST}"; then
+ ROLLBACK_RESUME=true
+ PREDECESSOR_IMAGE_DIGEST="${DESIRED_IMAGE_DIGEST}"
+ PREDECESSOR_REHOME_PROTOCOL="${DESIRED_REHOME_PROTOCOL}"
+ else
+ ROLLBACK_RESUME=false
+ PREDECESSOR_IMAGE_DIGEST="${CURRENT_IMAGE_DIGEST}"
+ PREDECESSOR_REHOME_PROTOCOL="${CURRENT_REHOME_PROTOCOL}"
+ fi
+ RESTORED_MIGRATION_CELLS="$(jq -rn \
+ --arg value "${EXPECTED_MIGRATION_ONLY_CELLS/none/}" \
+ --arg target "${TARGET_CELL_ID}" \
+ '$value | split(",") | map(select(length > 0 and . != $target)) | unique | join(",")')"
+ RESTORED_GENERAL_CELLS="$(jq -rn \
+ --arg value "${EXPECTED_GENERAL_CELLS/none/}" \
+ --arg target "${TARGET_CELL_ID}" \
+ '$value | split(",") | map(select(length > 0)) + [$target] | unique | join(",")')"
+ test -n "${RESTORED_MIGRATION_CELLS}" || RESTORED_MIGRATION_CELLS=none
+ test -n "${RESTORED_GENERAL_CELLS}" || RESTORED_GENERAL_CELLS=none
+ ISOLATED_MIGRATION_CELLS="$(jq -rn \
+ --arg value "${EXPECTED_MIGRATION_ONLY_CELLS/none/}" \
+ --arg target "${TARGET_CELL_ID}" \
+ '$value | split(",") | map(select(length > 0)) + [$target] | unique | join(",")')"
+ ISOLATED_GENERAL_CELLS="$(jq -rn \
+ --arg value "${EXPECTED_GENERAL_CELLS/none/}" \
+ --arg target "${TARGET_CELL_ID}" \
+ '$value | split(",") | map(select(length > 0 and . != $target)) | unique | join(",")')"
+ test -n "${ISOLATED_MIGRATION_CELLS}" || ISOLATED_MIGRATION_CELLS=none
+ test -n "${ISOLATED_GENERAL_CELLS}" || ISOLATED_GENERAL_CELLS=none
+ {
+ echo "ROLLBACK_RESUME=${ROLLBACK_RESUME}"
+ # The failsafe consumes these; deriving them here keeps them
+ # defined for a failure in any later step.
+ echo "ISOLATED_MIGRATION_CELLS=${ISOLATED_MIGRATION_CELLS}"
+ echo "ISOLATED_GENERAL_CELLS=${ISOLATED_GENERAL_CELLS}"
+ # No restart happens on resume, so isolate below is skipped and
+ # cannot advance the selector generation.
+ echo "SELECTOR_GENERATION_AFTER_ISOLATE=${EFFECTIVE_SELECTOR_GENERATION}"
+ # A failed-canary rollback enters with the target migration-only,
+ # so the restore inspect cannot reuse the entry membership inputs.
+ echo "RESTORED_MIGRATION_CELLS=${RESTORED_MIGRATION_CELLS}"
+ echo "RESTORED_GENERAL_CELLS=${RESTORED_GENERAL_CELLS}"
+ } >> "${GITHUB_ENV}"
+ if ! jq -e --arg cell "${TARGET_CELL_ID}" --arg origin "${CELL_ORIGIN}" \
+ --arg digest "${PREDECESSOR_IMAGE_DIGEST}" \
+ --arg region "${EXPECTED_REGION}" \
+ --argjson hardCap "${EXPECTED_HARD_CAP}" \
+ --argjson unobservedBound "${EXPECTED_UNOBSERVED_BOUND}" \
+ --argjson protocol "${PREDECESSOR_REHOME_PROTOCOL}" \
+ --argjson drainingOk "$(test "${DEPLOY_MODE}" = rollback \
+ && test "${ROLLBACK_RESUME}" != true && echo true || echo false)" \
+ '.role == "cell" and .cellId == $cell and .cellUrl == $origin and
+ (.region == $region or
+ ($region == "us-central1" and $protocol == 0 and .region == null)) and
+ .imageDigest == $digest and
+ .connectionCapacity.hardCap == $hardCap and
+ .connectionCapacity.unobservedBound == $unobservedBound and
+ (.draining == false or $drainingOk) and
+ (.regionalRehomeProtocol // 0) == $protocol' <<< "${CURRENT_RUNTIME}" >/dev/null
+ then
+ jq -r --arg cell "${TARGET_CELL_ID}" --arg origin "${CELL_ORIGIN}" \
+ --arg digest "${PREDECESSOR_IMAGE_DIGEST}" \
+ --arg region "${EXPECTED_REGION}" \
+ --argjson hardCap "${EXPECTED_HARD_CAP}" \
+ --argjson unobservedBound "${EXPECTED_UNOBSERVED_BOUND}" \
+ --argjson protocol "${PREDECESSOR_REHOME_PROTOCOL}" \
+ --argjson drainingOk "$(test "${DEPLOY_MODE}" = rollback \
+ && test "${ROLLBACK_RESUME}" != true && echo true || echo false)" \
+ '[
+ if .role != "cell" then "role" else empty end,
+ if .cellId != $cell then "cellId" else empty end,
+ if .cellUrl != $origin then "cellUrl" else empty end,
+ if (.region != $region and
+ ($region != "us-central1" or $protocol != 0 or .region != null))
+ then "region" else empty end,
+ if .imageDigest != $digest then "imageDigest" else empty end,
+ if .connectionCapacity.hardCap != $hardCap then "hardCap" else empty end,
+ if .connectionCapacity.unobservedBound != $unobservedBound then "unobservedBound" else empty end,
+ if (.draining != false and ($drainingOk | not)) then "draining" else empty end,
+ if (.regionalRehomeProtocol // 0) != $protocol then "regionalRehomeProtocol" else empty end
+ ] | "runtime predecessor mismatch fields=" + join(",")' \
+ <<< "${CURRENT_RUNTIME}" >&2
+ exit 1
+ fi
+ # The exact legacy digest binds omitted pre-region fields to US and protocol 0.
+ jq -r '[
+ if .region == null then "region" else empty end,
+ if .regionalRehomeProtocol == null then "regionalRehomeProtocol" else empty end
+ ] | if length > 0 then "runtime predecessor normalized legacy fields=" + join(",") else empty end' \
+ <<< "${CURRENT_RUNTIME}"
+ CURRENT_DIRECTOR_STATUS="$(admin_post current-cell-status \
+ "${DIRECTOR_ORIGIN}/v1/admin/cell-status" \
+ "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"
+ SOURCE_INCARNATION="$(jq -er '.status.runtime.cellIncarnation' \
+ <<< "${CURRENT_DIRECTOR_STATUS}")"
+ if test "${ROLLBACK_RESUME}" = true && ! jq -e \
+ '.status.admissionState == "migration-only"' \
+ <<< "${CURRENT_DIRECTOR_STATUS}" >/dev/null; then
+ echo 'resume requires the isolated migration-only cell a failed rollback leaves' >&2
+ exit 1
+ fi
+ [[ "${SOURCE_INCARNATION}" =~ ^[0-9a-f-]{36}$ ]]
+ echo "SOURCE_INCARNATION=${SOURCE_INCARNATION}" >> "${GITHUB_ENV}"
+ # Rollback is the documented recovery from a failed canary, which
+ # leaves the cell migration-only (and possibly still marked
+ # draining); apply and verify still require a pristine general cell.
+ if test "${DEPLOY_MODE}" = rollback; then
+ PRECHECK_ADMISSION=general-or-migration-only
+ PRECHECK_DRAINING=either
+ else
+ PRECHECK_ADMISSION=general
+ PRECHECK_DRAINING=forbidden
+ fi
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
+ --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
+ --heartbeat fresh --admission "${PRECHECK_ADMISSION}" \
+ --draining "${PRECHECK_DRAINING}" --activity allowed \
+ --expected-image-digests "${PREDECESSOR_IMAGE_DIGEST}"
+
+ - name: Finish read-only verification
+ if: ${{ inputs.mode == 'verify' }}
+ run: echo 'Exact same-cap rollback point verified.'
+
+ - name: Reversibly isolate and drain only the selected cell
+ if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME != 'true' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }}
+ run: |
+ echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
+ # A cell isolated by a failed canary is already migration-only, so
+ # isolate is a no-op there that does not advance the selector; the
+ # result's generation is authoritative either way.
+ ISOLATE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode isolate)"
+ echo "${ISOLATE_RESULT}"
+ ISOLATE_GENERATION="$(jq -er '.generation' <<< "${ISOLATE_RESULT}")"
+ echo "SELECTOR_GENERATION_AFTER_ISOLATE=${ISOLATE_GENERATION}" >> "${GITHUB_ENV}"
+ node dev/scripts/prepare-relay-production-capacity-canary.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode drain
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
+ --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
+ --heartbeat either --admission migration-only --draining required \
+ --activity restart-safe --expected-image-digests "${CURRENT_IMAGE_DIGEST}" \
+ --timeout-ms 900000
+
+ - id: capacity-auth
+ if: ${{ inputs.mode != 'verify' }}
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
+
+ - name: Require converged Terraform state and a stable MIG on resume
+ if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME == 'true' }}
+ shell: bash
+ env:
+ DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
+ run: |
+ # Zero resource changes prove the prior run's apply completed and no
+ # restart will follow, keeping the incarnation check honest. Root
+ # outputs may lag a targeted apply, so judge resource_changes only.
+ terraform -chdir=infra/terraform plan \
+ -var-file=environments/production.tfvars \
+ -var-file="${RUNNER_TEMP}/relay-same-cap.tfvars.json" \
+ "-target=google_compute_instance_template.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
+ "-target=google_compute_instance_group_manager.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
+ -out="${RUNNER_TEMP}/relay-same-cap-resume.tfplan"
+ if ! terraform -chdir=infra/terraform show -json \
+ "${RUNNER_TEMP}/relay-same-cap-resume.tfplan" \
+ | jq -e '[.resource_changes[]?
+ | select(.change.actions | any(. != "no-op" and . != "read"))]
+ | length == 0' >/dev/null
+ then
+ # An apply that failed before its template apply also resumes here
+ # (the cell still serves the rollback image), and repo drift since
+ # the cell's last roll (for example newly added rehome trust
+ # config) then legitimately replaces the template. Nothing is
+ # applied on resume either way, so accept exactly the drift the
+ # reviewed validator would let a real apply ship for the image the
+ # cell already serves: the template leaves and re-enters the
+ # rollback image, as exactly the template-and-MIG change pair.
+ terraform -chdir=infra/terraform show -json \
+ "${RUNNER_TEMP}/relay-same-cap-resume.tfplan" \
+ | jq -r '"resume found unconverged resources: " +
+ ([.resource_changes[]?
+ | select(.change.actions | any(. != "no-op" and . != "read"))
+ | .address] | join(","))'
+ echo 'requiring reviewed rollback-image drift'
+ terraform -chdir=infra/terraform show -json \
+ "${RUNNER_TEMP}/relay-same-cap-resume.tfplan" \
+ | node dev/scripts/validate-relay-capacity-plan.mjs \
+ --mode same-cap-cell --cell-id "${TARGET_CELL_ID}" \
+ --hard-cap "${EXPECTED_HARD_CAP}" \
+ --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
+ --image "${DESIRED_IMAGE}" \
+ --rollback-image "${DESIRED_IMAGE}" \
+ --rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
+ --rehome-audience https://relay.onorca.dev/v1/admin/host-drain \
+ --regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" \
+ | jq -e '.changes == 2' >/dev/null
+ fi
+ gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
+ --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
+
+ - name: Apply only the selected same-cap template and MIG
+ if: ${{ inputs.mode != 'verify' && env.ROLLBACK_RESUME != 'true' }}
+ shell: bash
+ env:
+ CAPACITY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
+ DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
+ run: |
+ terraform -chdir=infra/terraform plan \
+ -var-file=environments/production.tfvars \
+ -var-file="${RUNNER_TEMP}/relay-same-cap.tfvars.json" \
+ "-target=google_compute_instance_template.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
+ "-target=google_compute_instance_group_manager.relay_gce_cell[\"${TARGET_CELL_ID}\"]" \
+ -out="${RUNNER_TEMP}/relay-same-cap.tfplan"
+ terraform -chdir=infra/terraform show -json "${RUNNER_TEMP}/relay-same-cap.tfplan" \
+ | node dev/scripts/validate-relay-capacity-plan.mjs \
+ --mode same-cap-cell --cell-id "${TARGET_CELL_ID}" \
+ --hard-cap "${EXPECTED_HARD_CAP}" \
+ --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" --image "${DESIRED_IMAGE}" \
+ --rollback-image "${IMAGE_REPOSITORY}@${CURRENT_IMAGE_DIGEST}" \
+ --rehome-director-service-account "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}" \
+ --rehome-audience https://relay.onorca.dev/v1/admin/host-drain \
+ --regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}"
+ terraform -chdir=infra/terraform apply -auto-approve \
+ "${RUNNER_TEMP}/relay-same-cap.tfplan"
+ gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
+ --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
+
+ - id: post-auth
+ if: ${{ inputs.mode != 'verify' }}
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - name: Verify new incarnation, exact image, protocol, and durable safety
+ if: ${{ inputs.mode != 'verify' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }}
+ run: |
+ # A single transient 5xx (LB warm-up behind a fresh instance) must not
+ # fail a canary; 4xx (auth, generation mismatch) still fails fast.
+ admin_post() {
+ local out="${RUNNER_TEMP}/$1.json"
+ if ! curl --fail-with-body --max-time 30 \
+ --retry 3 --retry-delay 2 --retry-connrefused --output "${out}" \
+ --request POST "$2" \
+ --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
+ --header 'Content-Type: application/json' --data "$3"; then
+ cat "${out}" >&2
+ return 1
+ fi
+ cat "${out}"
+ }
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
+ --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
+ --heartbeat fresh --admission migration-only --draining forbidden \
+ --activity allowed --expected-image-digests "${DESIRED_IMAGE_DIGEST}" \
+ --regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" --timeout-ms 900000
+ TARGET_RUNTIME="$(admin_post target-runtime \
+ "${CELL_ORIGIN}/v1/admin/runtime-status" '{"v":1}')"
+ jq -e --arg digest "${DESIRED_IMAGE_DIGEST}" \
+ --argjson protocol "${DESIRED_REHOME_PROTOCOL}" \
+ '.imageDigest == $digest and (.regionalRehomeProtocol // 0) == $protocol' \
+ <<< "${TARGET_RUNTIME}" >/dev/null
+ TARGET_DIRECTOR_STATUS="$(admin_post target-cell-status \
+ "${DIRECTOR_ORIGIN}/v1/admin/cell-status" \
+ "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"
+ TARGET_INCARNATION="$(jq -er '.status.runtime.cellIncarnation' \
+ <<< "${TARGET_DIRECTOR_STATUS}")"
+ if test "${ROLLBACK_RESUME}" = true; then
+ echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
+ # No restart happened; the incarnation legitimately stays put.
+ test "${TARGET_INCARNATION}" = "${SOURCE_INCARNATION}"
+ else
+ test "${TARGET_INCARNATION}" != "${SOURCE_INCARNATION}"
+ fi
+ echo "TARGET_INCARNATION=${TARGET_INCARNATION}" >> "${GITHUB_ENV}"
+ node dev/scripts/operate-relay-regional-rehome.mjs \
+ --mode inspect --director-origin "${DIRECTOR_ORIGIN}" \
+ --expected-selector-generation "${SELECTOR_GENERATION_AFTER_ISOLATE}" \
+ --expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
+ --expected-migration-only-cells "${ISOLATED_MIGRATION_CELLS}" \
+ --expected-general-cells "${ISOLATED_GENERAL_CELLS}" \
+ --expected-control-generation "${EXPECTED_REHOME_GENERATION}" \
+ | jq -e '.control.enabled == false' >/dev/null
+
+ - name: Prove exact per-host trust and idempotent no-neighbor behavior
+ if: ${{ inputs.mode != 'verify' && ((inputs.mode == 'rollback' && inputs.rollback-rehome-protocol == '1') || (inputs.mode != 'rollback' && inputs.target-rehome-protocol == '1')) }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }}
+ run: |
+ node dev/scripts/probe-relay-rehome-trust.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" --cell-id "${TARGET_CELL_ID}" \
+ --cell-incarnation "${TARGET_INCARNATION}"
+
+ - name: Restore only the verified selected cell to general admission
+ if: ${{ inputs.mode != 'verify' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }}
+ run: |
+ echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
+ ACTIVATE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode activate)"
+ echo "${ACTIVATE_RESULT}"
+ SELECTOR_GENERATION_AFTER_ACTIVATE="$(jq -er '.generation' \
+ <<< "${ACTIVATE_RESULT}")"
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \
+ --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
+ --heartbeat fresh --admission general --draining forbidden --activity allowed \
+ --expected-image-digests "${DESIRED_IMAGE_DIGEST}" \
+ --regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}"
+ node dev/scripts/operate-relay-regional-rehome.mjs \
+ --mode inspect --director-origin "${DIRECTOR_ORIGIN}" \
+ --expected-selector-generation "${SELECTOR_GENERATION_AFTER_ACTIVATE}" \
+ --expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
+ --expected-migration-only-cells "${RESTORED_MIGRATION_CELLS}" \
+ --expected-general-cells "${RESTORED_GENERAL_CELLS}" \
+ --expected-control-generation "${EXPECTED_REHOME_GENERATION}" \
+ | jq -e '.control.enabled == false' >/dev/null
+
+ - id: cleanup-auth
+ if: ${{ failure() && inputs.mode != 'verify' }}
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - name: Keep a failed cell isolated and rehome disabled
+ if: ${{ failure() && inputs.mode != 'verify' }}
+ continue-on-error: true
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.cleanup-auth.outputs.id_token }}
+ run: |
+ test "${MUTATION_STARTED:-false}" = true || exit 0
+ ISOLATE_RESULT="$(node dev/scripts/prepare-relay-production-capacity-canary.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" --approved-cells same-cap --mode isolate)"
+ echo "${ISOLATE_RESULT}"
+ # The isolate result carries the authoritative post-isolate generation;
+ # fixed offsets are wrong whenever an earlier isolate was a no-op.
+ FAILSAFE_GENERATION="$(jq -er '.generation' <<< "${ISOLATE_RESULT}")"
+ node dev/scripts/operate-relay-regional-rehome.mjs \
+ --mode inspect --director-origin "${DIRECTOR_ORIGIN}" \
+ --expected-selector-generation "${FAILSAFE_GENERATION}" \
+ --expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
+ --expected-migration-only-cells "${ISOLATED_MIGRATION_CELLS}" \
+ --expected-general-cells "${ISOLATED_GENERAL_CELLS}" \
+ --expected-control-generation "${EXPECTED_REHOME_GENERATION}"
diff --git a/.github/workflows/cloud-deploy-relay-production-same-cap.yml b/.github/workflows/cloud-deploy-relay-production-same-cap.yml
new file mode 100644
index 00000000000..fba5df0dcb9
--- /dev/null
+++ b/.github/workflows/cloud-deploy-relay-production-same-cap.yml
@@ -0,0 +1,315 @@
+name: Deploy Relay Production Same-Cap
+
+on:
+ workflow_dispatch:
+ inputs:
+ mode:
+ description: Verify, roll one canary, roll a bounded batch, or roll back
+ required: true
+ default: verify
+ type: choice
+ options: [verify, canary-apply, batch-apply, rollback]
+ cell-ids:
+ description: Ordered comma-separated serving cells; one canary or two to four batch cells
+ required: true
+ type: string
+ target-image-digest:
+ description: Exact immutable compatibility image digest
+ required: true
+ type: string
+ rollback-image-digest:
+ description: Exact immutable currently serving rollback digest
+ required: true
+ type: string
+ target-rehome-protocol:
+ description: Exact target regional-rehome protocol
+ required: true
+ default: '1'
+ type: choice
+ options: ['0', '1']
+ rollback-rehome-protocol:
+ description: Exact rollback regional-rehome protocol
+ required: true
+ default: '0'
+ type: choice
+ options: ['0', '1']
+ expected-selector-generation:
+ description: Exact selector generation before the first cell
+ required: true
+ type: string
+ expected-existing-only-cells:
+ description: Exact existing-only membership, or none
+ required: true
+ type: string
+ expected-migration-only-cells:
+ description: Exact migration-only membership, or none
+ required: true
+ type: string
+ expected-general-cells:
+ description: Exact general membership, or none
+ required: true
+ type: string
+ expected-rehome-generation:
+ description: Exact durable regional-rehome control generation; it must be disabled
+ required: true
+ type: string
+ monitor-run-id:
+ description: Fresh successful aggregate dry-run monitor workflow run
+ required: false
+ type: string
+ monitor-run-attempt:
+ description: Exact monitor attempt
+ required: false
+ type: string
+ canary-run-id:
+ description: Successful same-commit canary run required for batch-apply
+ required: false
+ type: string
+ confirmation:
+ description: Exact digest-and-cell-bound mutation confirmation
+ required: false
+ type: string
+
+permissions:
+ actions: read
+ contents: read
+ id-token: write
+
+concurrency:
+ group: production-cloud-sql-rollout
+ cancel-in-progress: false
+
+defaults:
+ run:
+ working-directory: cloud
+
+jobs:
+ gate:
+ if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }}
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ # Headroom for the full-history checkout the canary provenance check needs.
+ timeout-minutes: 15
+ environment: production
+ outputs:
+ cells: ${{ steps.wave.outputs.cells }}
+ job-mode: ${{ steps.wave.outputs.job-mode }}
+ steps:
+ # Full history: the canary authority a batch verifies is sealed at an ancestor commit, and
+ # the provenance check fails closed on a commit a shallow clone left out.
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - uses: actions/setup-node@v4
+ with: { node-version: 24 }
+
+ - id: wave
+ env:
+ MODE: ${{ inputs.mode }}
+ CELL_IDS: ${{ inputs.cell-ids }}
+ TARGET_DIGEST: ${{ inputs.target-image-digest }}
+ ROLLBACK_DIGEST: ${{ inputs.rollback-image-digest }}
+ CONFIRMATION: ${{ inputs.confirmation }}
+ CANARY_RUN_ID: ${{ inputs.canary-run-id }}
+ run: |
+ CELLS="$(node dev/scripts/relay-production-same-cap-wave.mjs validate \
+ --mode "${MODE}" --cell-ids "${CELL_IDS}" \
+ --target-digest "${TARGET_DIGEST}" --rollback-digest "${ROLLBACK_DIGEST}" \
+ --confirmation "${CONFIRMATION}" --canary-run-id "${CANARY_RUN_ID}")"
+ echo "cells=${CELLS}" >> "${GITHUB_OUTPUT}"
+ if [[ "${MODE}" =~ ^(canary-apply|batch-apply)$ ]]; then
+ echo 'job-mode=apply' >> "${GITHUB_OUTPUT}"
+ else
+ echo "job-mode=${MODE}" >> "${GITHUB_OUTPUT}"
+ fi
+
+ - name: Download exact prior canary authority
+ if: ${{ inputs.mode == 'batch-apply' }}
+ uses: actions/download-artifact@v4
+ with:
+ name: relay-same-cap-canary-${{ inputs.canary-run-id }}
+ path: ${{ runner.temp }}/relay-same-cap-canary
+ github-token: ${{ github.token }}
+ run-id: ${{ inputs.canary-run-id }}
+
+ - name: Verify canary authority against this batch
+ if: ${{ inputs.mode == 'batch-apply' }}
+ env:
+ CANARY_RUN_ID: ${{ inputs.canary-run-id }}
+ run: |
+ node dev/scripts/relay-production-same-cap-wave.mjs verify-canary \
+ --file "${RUNNER_TEMP}/relay-same-cap-canary/authority.json" \
+ --commit-sha "${GITHUB_SHA}" --run-id "${CANARY_RUN_ID}" \
+ --target-digest "${{ inputs.target-image-digest }}" \
+ --rollback-digest "${{ inputs.rollback-image-digest }}" \
+ --selector-generation "${{ inputs.expected-selector-generation }}" \
+ --rehome-generation "${{ inputs.expected-rehome-generation }}"
+
+ - name: Reject previously consumed aggregate safety evidence
+ if: ${{ inputs.mode != 'verify' }}
+ env:
+ GH_TOKEN: ${{ github.token }}
+ MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
+ MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }}
+ run: |
+ [[ "${MONITOR_RUN_ID}" =~ ^[1-9][0-9]*$ ]]
+ [[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
+ MARKER_NAME="relay-same-cap-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
+ COUNT="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \
+ --jq '.total_count')"
+ test "${COUNT}" = 0
+ mkdir -p "${RUNNER_TEMP}/relay-same-cap-monitor-authority"
+ printf '%s\n' "${GITHUB_RUN_ID}" \
+ > "${RUNNER_TEMP}/relay-same-cap-monitor-authority/${MARKER_NAME}"
+
+ - name: Consume aggregate safety evidence for this exact wave
+ if: ${{ inputs.mode != 'verify' }}
+ uses: actions/upload-artifact@v4
+ with:
+ name: relay-same-cap-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
+ path: ${{ runner.temp }}/relay-same-cap-monitor-authority/relay-same-cap-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
+ retention-days: 90
+ if-no-files-found: error
+
+ cell_1:
+ needs: gate
+ uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
+ with:
+ mode: ${{ needs.gate.outputs.job-mode }}
+ target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[0] }}
+ target-image-digest: ${{ inputs.target-image-digest }}
+ rollback-image-digest: ${{ inputs.rollback-image-digest }}
+ target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
+ rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
+ expected-selector-generation: ${{ inputs.expected-selector-generation }}
+ expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
+ expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
+ expected-general-cells: ${{ inputs.expected-general-cells }}
+ expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
+ monitor-run-id: ${{ inputs.monitor-run-id }}
+ monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
+ wave-index: '0'
+ secrets: inherit
+
+ cell_2:
+ if: ${{ needs.cell_1.result == 'success' && fromJSON(needs.gate.outputs.cells)[1] != null }}
+ needs: [gate, cell_1]
+ uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
+ with:
+ mode: ${{ needs.gate.outputs.job-mode }}
+ target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[1] }}
+ target-image-digest: ${{ inputs.target-image-digest }}
+ rollback-image-digest: ${{ inputs.rollback-image-digest }}
+ target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
+ rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
+ expected-selector-generation: ${{ inputs.expected-selector-generation }}
+ expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
+ expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
+ expected-general-cells: ${{ inputs.expected-general-cells }}
+ expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
+ monitor-run-id: ${{ inputs.monitor-run-id }}
+ monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
+ wave-index: '1'
+ secrets: inherit
+
+ cell_3:
+ if: ${{ needs.cell_2.result == 'success' && fromJSON(needs.gate.outputs.cells)[2] != null }}
+ needs: [gate, cell_2]
+ uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
+ with:
+ mode: ${{ needs.gate.outputs.job-mode }}
+ target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[2] }}
+ target-image-digest: ${{ inputs.target-image-digest }}
+ rollback-image-digest: ${{ inputs.rollback-image-digest }}
+ target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
+ rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
+ expected-selector-generation: ${{ inputs.expected-selector-generation }}
+ expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
+ expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
+ expected-general-cells: ${{ inputs.expected-general-cells }}
+ expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
+ monitor-run-id: ${{ inputs.monitor-run-id }}
+ monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
+ wave-index: '2'
+ secrets: inherit
+
+ cell_4:
+ if: ${{ needs.cell_3.result == 'success' && fromJSON(needs.gate.outputs.cells)[3] != null }}
+ needs: [gate, cell_3]
+ uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
+ with:
+ mode: ${{ needs.gate.outputs.job-mode }}
+ target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[3] }}
+ target-image-digest: ${{ inputs.target-image-digest }}
+ rollback-image-digest: ${{ inputs.rollback-image-digest }}
+ target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
+ rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
+ expected-selector-generation: ${{ inputs.expected-selector-generation }}
+ expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
+ expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
+ expected-general-cells: ${{ inputs.expected-general-cells }}
+ expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
+ monitor-run-id: ${{ inputs.monitor-run-id }}
+ monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
+ wave-index: '3'
+ secrets: inherit
+
+ seal_canary:
+ if: ${{ inputs.mode == 'canary-apply' }}
+ needs: [gate, cell_1]
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ timeout-minutes: 5
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: actions/setup-node@v4
+ with: { node-version: 24 }
+
+ - name: Seal exact successful canary authority
+ run: |
+ mkdir -p "${RUNNER_TEMP}/relay-same-cap-canary"
+ node dev/scripts/relay-production-same-cap-wave.mjs create-canary \
+ --cell-id "${{ inputs.cell-ids }}" \
+ --target-digest "${{ inputs.target-image-digest }}" \
+ --rollback-digest "${{ inputs.rollback-image-digest }}" \
+ --confirmation "${{ inputs.confirmation }}" \
+ --commit-sha "${GITHUB_SHA}" --run-id "${GITHUB_RUN_ID}" \
+ --selector-generation "${{ inputs.expected-selector-generation }}" \
+ --rehome-generation "${{ inputs.expected-rehome-generation }}" \
+ > "${RUNNER_TEMP}/relay-same-cap-canary/authority.json"
+
+ - uses: actions/upload-artifact@v4
+ with:
+ name: relay-same-cap-canary-${{ github.run_id }}
+ path: ${{ runner.temp }}/relay-same-cap-canary/authority.json
+ retention-days: 30
+ if-no-files-found: error
+
+ # Every cell job re-enters the run's lease with release: 'false'; only this job frees it.
+ release_lease:
+ if: always()
+ needs:
+ - gate
+ - cell_1
+ - cell_2
+ - cell_3
+ - cell_4
+ - seal_canary
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ timeout-minutes: 10
+ environment: production
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+
+ - uses: google-github-actions/setup-gcloud@v2
+
+ - uses: ./.github/actions/cloud-sql-rollout-lease
+ with:
+ bucket: onorca-cloud-terraform-state
+ object: terraform/state/cloud-sql-rollout/production.lock
+ release: 'true'
diff --git a/.github/workflows/cloud-deploy-relay-production.yml b/.github/workflows/cloud-deploy-relay-production.yml
new file mode 100644
index 00000000000..4c8691b9510
--- /dev/null
+++ b/.github/workflows/cloud-deploy-relay-production.yml
@@ -0,0 +1,217 @@
+name: Deploy Relay Production Candidate
+
+on:
+ workflow_dispatch:
+ inputs:
+ source-cell-id:
+ description: Existing Terraform cell ID to evacuate
+ required: true
+ type: string
+ target-cell-id:
+ description: Distinct Terraform candidate cell ID
+ required: true
+ type: string
+ mode:
+ description: Audit/preflight are read-only; recover/continue resume committed work; disable/enable/reset/execute mutate admission
+ required: true
+ default: preflight
+ type: choice
+ options:
+ - audit
+ - preflight
+ - recover-forward
+ - continue-evacuation
+ - disable-cell
+ - enable-empty-cell
+ - reset-empty-candidate
+ - execute
+ confirmation:
+ description: Enter RECOVER_FORWARD, CONTINUE_EVACUATION, DISABLE_CELL, ENABLE_CELL, RESET_CANDIDATE, or EVACUATE for the matching mutation
+ required: false
+ type: string
+ monitor-run-id:
+ description: Successful fresh dry-run monitor workflow run ID
+ required: false
+ type: string
+ monitor-run-attempt:
+ description: Exact dry-run monitor workflow attempt
+ required: false
+ type: string
+
+permissions:
+ actions: read
+ contents: read
+ id-token: write
+
+concurrency:
+ group: production-cloud-sql-rollout
+ cancel-in-progress: false
+
+defaults:
+ run:
+ working-directory: cloud
+
+jobs:
+ candidate:
+ if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }}
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ environment: production
+ env:
+ GCP_PROJECT_ID: onorca-cloud
+ DIRECTOR_ORIGIN: https://relay.onorca.dev
+ ADMIN_AUDIENCE: https://relay.onorca.dev/v1/admin/drain
+ SOURCE_CELL_ID: ${{ inputs.source-cell-id }}
+ TARGET_CELL_ID: ${{ inputs.target-cell-id }}
+ DEPLOY_MODE: ${{ inputs.mode }}
+ MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
+ MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }}
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Require fresh dry-run evidence reference
+ if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
+ run: |
+ [[ "${MONITOR_RUN_ID}" =~ ^[0-9]+$ ]]
+ [[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
+
+ - name: Download private dry-run evidence
+ if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
+ uses: actions/download-artifact@v4
+ with:
+ name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
+ path: ${{ runner.temp }}/relay-monitor-evidence
+ github-token: ${{ github.token }}
+ run-id: ${{ inputs.monitor-run-id }}
+
+ - uses: pnpm/action-setup@v4
+ with:
+ package_json_file: cloud/package.json
+
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 24
+
+ - run: pnpm install --frozen-lockfile
+
+ - uses: hashicorp/setup-terraform@v3
+ with:
+ terraform_wrapper: false
+
+ - name: Verify dry-run artifact before cloud authentication
+ if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
+ run: |
+ node dev/scripts/relay-monitor-evidence.mjs verify-restore \
+ --directory "${RUNNER_TEMP}/relay-monitor-evidence" \
+ --incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
+ --run-id "${MONITOR_RUN_ID}" \
+ --run-attempt "${MONITOR_RUN_ATTEMPT}" \
+ --commit-sha "${GITHUB_SHA}" \
+ --mode dry-run
+
+ - name: Reject previously consumed dry-run evidence
+ if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: |
+ MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
+ COUNT="$(gh api \
+ "/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \
+ --jq '.total_count')"
+ test "${COUNT}" = "0"
+
+ - id: google-auth
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - uses: google-github-actions/setup-gcloud@v2
+
+ - uses: ./.github/actions/cloud-sql-rollout-lease
+ with:
+ bucket: onorca-cloud-terraform-state
+ object: terraform/state/cloud-sql-rollout/production.lock
+
+ - name: Require explicit mutation confirmation
+ if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
+ env:
+ CONFIRMATION: ${{ inputs.confirmation }}
+ run: |
+ if [[ "${DEPLOY_MODE}" = "execute" ]]; then
+ test "${CONFIRMATION}" = "EVACUATE"
+ elif [[ "${DEPLOY_MODE}" = "recover-forward" ]]; then
+ test "${CONFIRMATION}" = "RECOVER_FORWARD"
+ elif [[ "${DEPLOY_MODE}" = "continue-evacuation" ]]; then
+ test "${CONFIRMATION}" = "CONTINUE_EVACUATION"
+ elif [[ "${DEPLOY_MODE}" = "disable-cell" ]]; then
+ test "${CONFIRMATION}" = "DISABLE_CELL"
+ elif [[ "${DEPLOY_MODE}" = "enable-empty-cell" ]]; then
+ test "${CONFIRMATION}" = "ENABLE_CELL"
+ else
+ test "${CONFIRMATION}" = "RESET_CANDIDATE"
+ fi
+
+ - name: Read reviewed Terraform topology
+ run: |
+ node dev/scripts/infra.mjs init --env production
+ terraform -chdir=infra/terraform output -json relay_gce_cell_deployments > "${RUNNER_TEMP}/relay-gce-topology.json"
+ RUNTIME_SERVICE_ACCOUNT="$(terraform -chdir=infra/terraform output -raw relay_runtime_service_account)"
+ echo "RUNTIME_SERVICE_ACCOUNT=${RUNTIME_SERVICE_ACCOUNT}" >> "${GITHUB_ENV}"
+
+ - name: Verify fresh dry-run evidence against live selector
+ if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ run: |
+ node dev/scripts/relay-monitor-evidence.mjs verify-mutation \
+ --directory "${RUNNER_TEMP}/relay-monitor-evidence" \
+ --incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
+ --run-id "${MONITOR_RUN_ID}" \
+ --run-attempt "${MONITOR_RUN_ATTEMPT}" \
+ --commit-sha "${GITHUB_SHA}" \
+ --mode dry-run \
+ --mutation-mode "${DEPLOY_MODE}" \
+ --source-cell-id "${SOURCE_CELL_ID}" \
+ --director-origin "${DIRECTOR_ORIGIN}"
+
+ - name: Recheck all live safety signals
+ if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ run: |
+ pnpm incident:relay-preflight -- \
+ --state-file "${RUNNER_TEMP}/relay-monitor-evidence/relay-${MONITOR_RUN_ID}-dry-run.state.json"
+
+ - name: Create single-use dry-run marker
+ if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
+ run: |
+ MARKER_NAME="relay-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
+ mkdir -p "${RUNNER_TEMP}/relay-monitor-consumption"
+ printf '%s\n' "${GITHUB_RUN_ID}" \
+ > "${RUNNER_TEMP}/relay-monitor-consumption/${MARKER_NAME}"
+
+ - name: Consume dry-run evidence
+ if: ${{ inputs.mode != 'audit' && inputs.mode != 'preflight' }}
+ uses: actions/upload-artifact@v4
+ with:
+ name: relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
+ path: ${{ runner.temp }}/relay-monitor-consumption/relay-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
+ retention-days: 90
+ if-no-files-found: error
+
+ - name: Preflight or evacuate exact GCE candidate
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ run: |
+ node dev/scripts/deploy-relay-gce-candidate.mjs \
+ --project "${GCP_PROJECT_ID}" \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --admin-audience "${ADMIN_AUDIENCE}" \
+ --topology-file "${RUNNER_TEMP}/relay-gce-topology.json" \
+ --source-cell-id "${SOURCE_CELL_ID}" \
+ --target-cell-id "${TARGET_CELL_ID}" \
+ --runtime-service-account "${RUNTIME_SERVICE_ACCOUNT}" \
+ --mode "${DEPLOY_MODE}"
diff --git a/.github/workflows/cloud-deploy-relay-staging-gce-candidate.yml b/.github/workflows/cloud-deploy-relay-staging-gce-candidate.yml
new file mode 100644
index 00000000000..e646980a787
--- /dev/null
+++ b/.github/workflows/cloud-deploy-relay-staging-gce-candidate.yml
@@ -0,0 +1,109 @@
+name: Deploy Relay Staging GCE Candidate
+
+on:
+ workflow_dispatch:
+ inputs:
+ source-cell-id:
+ description: Existing Terraform cell ID to evacuate
+ required: true
+ type: string
+ target-cell-id:
+ description: Distinct Terraform candidate cell ID
+ required: true
+ type: string
+ mode:
+ description: Preflight is read-only; reset repairs an empty candidate; execute evacuates
+ required: true
+ default: preflight
+ type: choice
+ options:
+ - preflight
+ - reset-empty-candidate
+ - execute
+ confirmation:
+ description: Enter RESET_CANDIDATE for reset or EVACUATE for execute
+ required: false
+ type: string
+
+permissions:
+ contents: read
+ id-token: write
+
+concurrency:
+ group: relay-staging-mutation
+ cancel-in-progress: false
+
+defaults:
+ run:
+ working-directory: cloud
+
+jobs:
+ candidate:
+ if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ environment: staging
+ env:
+ GCP_PROJECT_ID: onorca-cloud-staging
+ DIRECTOR_ORIGIN: https://relay-staging.onorca.dev
+ ADMIN_AUDIENCE: https://relay-staging.onorca.dev/v1/admin/drain
+ SOURCE_CELL_ID: ${{ inputs.source-cell-id }}
+ TARGET_CELL_ID: ${{ inputs.target-cell-id }}
+ DEPLOY_MODE: ${{ inputs.mode }}
+ steps:
+ - uses: actions/checkout@v4
+
+ - id: google-auth
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - uses: google-github-actions/setup-gcloud@v2
+
+ - uses: ./.github/actions/cloud-sql-rollout-lease
+ with:
+ bucket: onorca-cloud-staging-terraform-state
+ object: terraform/state/cloud-sql-rollout/staging.lock
+
+ - uses: hashicorp/setup-terraform@v3
+ with:
+ terraform_wrapper: false
+
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 24
+
+ - name: Require explicit mutation confirmation
+ if: ${{ inputs.mode != 'preflight' }}
+ env:
+ CONFIRMATION: ${{ inputs.confirmation }}
+ run: |
+ if [[ "${DEPLOY_MODE}" = "execute" ]]; then
+ test "${CONFIRMATION}" = "EVACUATE"
+ else
+ test "${CONFIRMATION}" = "RESET_CANDIDATE"
+ fi
+
+ - name: Read reviewed Terraform topology
+ run: |
+ node dev/scripts/infra.mjs init --env staging
+ terraform -chdir=infra/terraform output -json relay_gce_cell_deployments > "${RUNNER_TEMP}/relay-gce-topology.json"
+ RUNTIME_SERVICE_ACCOUNT="$(terraform -chdir=infra/terraform output -raw relay_runtime_service_account)"
+ echo "RUNTIME_SERVICE_ACCOUNT=${RUNTIME_SERVICE_ACCOUNT}" >> "${GITHUB_ENV}"
+
+ - name: Preflight or evacuate exact GCE candidate
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ run: |
+ node dev/scripts/deploy-relay-gce-candidate.mjs \
+ --project "${GCP_PROJECT_ID}" \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --admin-audience "${ADMIN_AUDIENCE}" \
+ --topology-file "${RUNNER_TEMP}/relay-gce-topology.json" \
+ --source-cell-id "${SOURCE_CELL_ID}" \
+ --target-cell-id "${TARGET_CELL_ID}" \
+ --runtime-service-account "${RUNTIME_SERVICE_ACCOUNT}" \
+ --mode "${DEPLOY_MODE}"
diff --git a/.github/workflows/cloud-deploy-relay-staging.yml b/.github/workflows/cloud-deploy-relay-staging.yml
new file mode 100644
index 00000000000..7ddfb9d7c73
--- /dev/null
+++ b/.github/workflows/cloud-deploy-relay-staging.yml
@@ -0,0 +1,112 @@
+name: Deploy Relay Staging
+
+on:
+ workflow_dispatch:
+ inputs:
+ expected-image-digest:
+ description: Exact checked-in production Relay sha256 digest to deploy
+ required: true
+ type: string
+
+permissions:
+ contents: read
+ id-token: write
+
+concurrency:
+ # Staging deploy, candidate, auth, and power operations must never overlap.
+ group: relay-staging-mutation
+ cancel-in-progress: false
+
+defaults:
+ run:
+ working-directory: cloud
+
+jobs:
+ deploy:
+ if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ environment: staging
+ env:
+ GCP_PROJECT_ID: onorca-cloud-staging
+ GCP_REGION: ${{ vars.STAGING_GCP_REGION }}
+ DIRECTOR_SERVICE_NAME: orca-cloud-relay-staging
+ REPOSITORY_ID: orca-cloud
+ IMAGE_NAME: relay
+ EXPECTED_IMAGE_DIGEST: ${{ inputs.expected-image-digest }}
+ CAPACITY_SERVICE_ACCOUNT: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
+ ASIA_PROOF_SERVICE_ACCOUNT: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT }}
+ REGIONAL_PLACEMENT_SECRET: orca-cloud-relay-regional-placement-enabled
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Require the expected immutable image
+ run: '[[ "${EXPECTED_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]'
+
+ - uses: hashicorp/setup-terraform@v3
+ with:
+ terraform_version: 1.15.8
+ terraform_wrapper: false
+
+ - id: google-auth
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - name: Bind the request to the checked-in staging C4 image
+ shell: bash
+ run: |
+ set -euo pipefail
+ terraform -chdir=infra/terraform init -reconfigure \
+ -backend-config=backend/staging.hcl -input=false
+ IMAGE="$(terraform -chdir=infra/terraform console \
+ -var-file=environments/staging.tfvars \
+ <<< 'var.relay_gce_cells["staging-gce-c4"].image' | jq -er '.')"
+ test "${IMAGE}" = \
+ "${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}@${EXPECTED_IMAGE_DIGEST}"
+ echo "IMAGE=${IMAGE}" >> "${GITHUB_ENV}"
+
+ - uses: google-github-actions/setup-gcloud@v2
+
+ - uses: ./.github/actions/cloud-sql-rollout-lease
+ with:
+ bucket: onorca-cloud-staging-terraform-state
+ object: terraform/state/cloud-sql-rollout/staging.lock
+
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 24
+
+ - name: Require the mirrored immutable image
+ run: |
+ DIGEST="$(gcloud artifacts docker images describe "${IMAGE}" \
+ --project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')"
+ test "${DIGEST}" = "${EXPECTED_IMAGE_DIGEST}"
+
+ - name: Deploy director blue/green
+ run: |
+ regional_version="$(gcloud secrets versions describe latest \
+ --project "${GCP_PROJECT_ID}" --secret "${REGIONAL_PLACEMENT_SECRET}" \
+ --format='value(name)' | awk -F/ '{print $NF}')"
+ [[ "${regional_version}" =~ ^[1-9][0-9]*$ ]]
+ RELEASE_ID="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}"
+ node dev/scripts/deploy-relay-blue-green.mjs \
+ --project "${GCP_PROJECT_ID}" \
+ --region "${GCP_REGION}" \
+ --service "${DIRECTOR_SERVICE_NAME}" \
+ --image "${IMAGE}" \
+ --role director \
+ --max-instances 2 \
+ --capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
+ --asia-proof-service-account "${ASIA_PROOF_SERVICE_ACCOUNT}" \
+ --regional-placement-secret-version "${regional_version}" \
+ --min-instances 0 \
+ --release-id "${RELEASE_ID}"
+
+ - name: Smoke director health
+ run: |
+ URL="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format='value(status.url)')"
+ node dev/scripts/smoke-relay.mjs "${URL}"
diff --git a/.github/workflows/cloud-monitor-relay-clock-skew.yml b/.github/workflows/cloud-monitor-relay-clock-skew.yml
new file mode 100644
index 00000000000..9d4de846519
--- /dev/null
+++ b/.github/workflows/cloud-monitor-relay-clock-skew.yml
@@ -0,0 +1,77 @@
+name: Monitor Relay Cell Clock Skew
+
+# Why: the 2026-08-01 sustained HOST_OFFLINE incident traced to one cell's
+# clock running ~100ms ahead, which deterministically failed every host
+# challenge under a zero-tolerance freshness check. /health and /ready cannot
+# see clock skew; this monitor alarms before drift reaches the (now 2s)
+# client tolerance.
+
+on:
+ workflow_dispatch:
+ schedule:
+ - cron: '17 * * * *'
+
+permissions:
+ contents: read
+
+defaults:
+ run:
+ working-directory: cloud
+
+jobs:
+ skew:
+ if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
+ runs-on: ubuntu-latest
+ steps:
+ - name: Measure Date-header skew for every relay cell
+ working-directory: .
+ run: |
+ set -u
+ # Date headers carry whole seconds; compare floored seconds on both
+ # sides so perfect sync reads 0/±1 and never flaps. An absolute
+ # floor-skew of >= 2 means real drift of at least ~1s — approaching
+ # the client's 2s challenge tolerance. Millisecond-precision deltas
+ # come from the desktop's named-check logging when activations fail.
+ ALARM_S=2
+ failures=0
+ reachable=0
+ unserved=0
+ # Keep this upper bound at or above the highest provisioned cell in
+ # environments/production.tfvars; unlisted cells are silently unmonitored.
+ for n in $(seq 1 22); do
+ cell="c${n}"
+ url="https://${cell}.relay.onorca.dev/health"
+ # Why: *.relay.onorca.dev is a wildcard, so the load balancer answers with its own
+ # accurate Date for a fenced, dead, or never-provisioned cell. Timing that reads as
+ # perfect sync. Only an HTTP 200 is the relay process itself answering, so only a
+ # 200 carries a clock worth judging.
+ response="$(curl -sS -D - -o /dev/null --max-time 8 "${url}" 2>/dev/null || true)"
+ status="$(printf '%s' "${response}" | awk 'NR==1 {print $2}')"
+ header="$(printf '%s' "${response}" | tr -d '\r' | grep -i '^date:' || true)"
+ if [ "${status:-000}" != "200" ] || [ -z "${header}" ]; then
+ # Expected for the fenced cells; a dead unfenced cell is caught by the heartbeat
+ # and readiness alerts, not here. Named either way so it is never invisible.
+ echo "${cell}: not serving (status ${status:-none}); no relay clock to judge"
+ unserved=$((unserved + 1))
+ continue
+ fi
+ reachable=$((reachable + 1))
+ server_s="$(date -d "${header#*: }" +%s)"
+ local_s="$(date +%s)"
+ skew=$((server_s - local_s))
+ abs=${skew#-}
+ if [ "${abs}" -ge "${ALARM_S}" ]; then
+ echo "::error::${cell}: clock skew ${skew}s reaches ±${ALARM_S}s alarm"
+ failures=$((failures + 1))
+ else
+ echo "${cell}: skew ${skew}s"
+ fi
+ done
+ echo "cells serving: ${reachable}, not serving: ${unserved}, alarms: ${failures}"
+ if [ "${reachable}" -eq 0 ]; then
+ # Now meaningful: previously the load balancer answered for every name, so this
+ # could never fire and a total fleet outage reported "all healthy".
+ echo "::error::no relay cell is serving; monitor blind"
+ exit 1
+ fi
+ exit "$((failures > 0 ? 1 : 0))"
diff --git a/.github/workflows/cloud-monitor-relay-production-job.yml b/.github/workflows/cloud-monitor-relay-production-job.yml
new file mode 100644
index 00000000000..4278b1420bc
--- /dev/null
+++ b/.github/workflows/cloud-monitor-relay-production-job.yml
@@ -0,0 +1,213 @@
+name: Monitor Relay Production Job
+
+on:
+ workflow_call:
+ inputs:
+ mode:
+ required: true
+ type: string
+ expected-selector-generation:
+ required: true
+ type: string
+ expected-existing-only-cells:
+ required: true
+ type: string
+ expected-migration-only-cells:
+ required: true
+ type: string
+ expected-general-cells:
+ required: true
+ type: string
+ migration-policy:
+ required: true
+ type: string
+ recovery-source-cell-id:
+ required: true
+ type: string
+ capacity-cell-id:
+ required: true
+ type: string
+
+permissions:
+ actions: read
+ contents: read
+ id-token: write
+
+defaults:
+ run:
+ working-directory: cloud
+
+jobs:
+ monitor:
+ if: ${{ github.ref == 'refs/heads/main' }}
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ timeout-minutes: 100
+ environment: production
+ env:
+ EXPECTED_SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }}
+ EXPECTED_EXISTING_ONLY_CELLS: ${{ inputs.expected-existing-only-cells }}
+ EXPECTED_MIGRATION_ONLY_CELLS: ${{ inputs.expected-migration-only-cells }}
+ EXPECTED_GENERAL_CELLS: ${{ inputs.expected-general-cells }}
+ MIGRATION_POLICY: ${{ inputs.migration-policy }}
+ RECOVERY_SOURCE_CELL_ID: ${{ inputs.recovery-source-cell-id }}
+ CAPACITY_CELL_ID: ${{ inputs.capacity-cell-id }}
+ INCIDENT_ID: relay-${{ github.run_id }}-${{ inputs.mode }}
+ MONITOR_MODE: ${{ inputs.mode }}
+ OUTPUT_DIRECTORY: ${{ github.workspace }}/relay-incident
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: pnpm/action-setup@v4
+ with:
+ package_json_file: cloud/package.json
+
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 24
+
+ - run: pnpm install --frozen-lockfile
+
+ - id: prior-attempt
+ if: ${{ github.run_attempt > 1 }}
+ run: echo "value=$((GITHUB_RUN_ATTEMPT - 1))" >> "${GITHUB_OUTPUT}"
+
+ - name: Restore prior private monitor state
+ if: ${{ github.run_attempt > 1 }}
+ uses: actions/download-artifact@v4
+ with:
+ name: relay-monitor-${{ inputs.mode }}-${{ github.run_id }}-${{ steps.prior-attempt.outputs.value }}
+ path: ${{ github.workspace }}/relay-incident
+ github-token: ${{ github.token }}
+ run-id: ${{ github.run_id }}
+
+ - name: Verify restored state provenance
+ if: ${{ github.run_attempt > 1 }}
+ run: |
+ node dev/scripts/relay-monitor-evidence.mjs verify-restore \
+ --directory "${OUTPUT_DIRECTORY}" \
+ --incident-id "${INCIDENT_ID}" \
+ --run-id "${GITHUB_RUN_ID}" \
+ --run-attempt "${{ steps.prior-attempt.outputs.value }}" \
+ --commit-sha "${GITHUB_SHA}" \
+ --mode "${MONITOR_MODE}"
+ echo "RESTART_FLAG=--restart" >> "${GITHUB_ENV}"
+
+ - id: google-auth
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_MONITOR_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.PRODUCTION_GCP_RELAY_MONITOR_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - uses: google-github-actions/setup-gcloud@v2
+
+ - name: Verify exact-audience admin identity
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ run: |
+ node -e "if (!/^[^.]+[.][^.]+[.][^.]+$/.test(process.env.ORCA_RELAY_ADMIN_ID_TOKEN ?? '')) process.exit(1)"
+
+ - name: Run read-only relay dry-run
+ if: ${{ inputs.mode == 'dry-run' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ run: |
+ pnpm --filter @orca-cloud/relay-ops incident:monitor \
+ --environment production \
+ --incident-id "${INCIDENT_ID}" \
+ --expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
+ --expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
+ --expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \
+ --expected-general-cells "${EXPECTED_GENERAL_CELLS}" \
+ --migration-policy "${MIGRATION_POLICY}" \
+ --recovery-source-cell-id "${RECOVERY_SOURCE_CELL_ID}" \
+ --capacity-cell-id "${CAPACITY_CELL_ID}" \
+ --interval-seconds 60 \
+ --output-directory "${OUTPUT_DIRECTORY}" \
+ --duration-minutes 15 \
+ --pre-drain-dry-run \
+ ${RESTART_FLAG:-}
+
+ - name: Run first relay monitor segment
+ if: ${{ inputs.mode == 'monitor' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ run: |
+ pnpm --filter @orca-cloud/relay-ops incident:monitor \
+ --environment production \
+ --incident-id "${INCIDENT_ID}" \
+ --expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
+ --expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
+ --expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \
+ --expected-general-cells "${EXPECTED_GENERAL_CELLS}" \
+ --migration-policy "${MIGRATION_POLICY}" \
+ --recovery-source-cell-id "${RECOVERY_SOURCE_CELL_ID}" \
+ --capacity-cell-id "${CAPACITY_CELL_ID}" \
+ --interval-seconds 60 \
+ --output-directory "${OUTPUT_DIRECTORY}" \
+ --duration-minutes 90 \
+ --max-samples-this-run 45 \
+ ${RESTART_FLAG:-}
+
+ - id: google-auth-refresh
+ if: ${{ inputs.mode == 'monitor' }}
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_MONITOR_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.PRODUCTION_GCP_RELAY_MONITOR_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - name: Run remaining relay monitor window
+ if: ${{ inputs.mode == 'monitor' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth-refresh.outputs.id_token }}
+ run: |
+ node -e "if (!/^[^.]+[.][^.]+[.][^.]+$/.test(process.env.ORCA_RELAY_ADMIN_ID_TOKEN ?? '')) process.exit(1)"
+ pnpm --filter @orca-cloud/relay-ops incident:monitor \
+ --environment production \
+ --incident-id "${INCIDENT_ID}" \
+ --expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
+ --expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
+ --expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \
+ --expected-general-cells "${EXPECTED_GENERAL_CELLS}" \
+ --migration-policy "${MIGRATION_POLICY}" \
+ --recovery-source-cell-id "${RECOVERY_SOURCE_CELL_ID}" \
+ --capacity-cell-id "${CAPACITY_CELL_ID}" \
+ --interval-seconds 60 \
+ --output-directory "${OUTPUT_DIRECTORY}" \
+ --duration-minutes 90 \
+ --restart
+
+ - name: Seal private evidence provenance
+ if: ${{ always() }}
+ run: |
+ node dev/scripts/relay-monitor-evidence.mjs create \
+ --directory "${OUTPUT_DIRECTORY}" \
+ --incident-id "${INCIDENT_ID}" \
+ --run-id "${GITHUB_RUN_ID}" \
+ --run-attempt "${GITHUB_RUN_ATTEMPT}" \
+ --commit-sha "${GITHUB_SHA}" \
+ --mode "${MONITOR_MODE}"
+
+ - name: Publish aggregate job summary
+ if: ${{ always() }}
+ run: |
+ if [[ -f "${OUTPUT_DIRECTORY}/${INCIDENT_ID}.summary.md" ]]; then
+ cat "${OUTPUT_DIRECTORY}/${INCIDENT_ID}.summary.md" >> "${GITHUB_STEP_SUMMARY}"
+ else
+ echo "Relay monitor failed before its first aggregate checkpoint." \
+ >> "${GITHUB_STEP_SUMMARY}"
+ fi
+
+ - name: Upload private aggregate evidence
+ if: ${{ always() }}
+ uses: actions/upload-artifact@v4
+ with:
+ name: relay-monitor-${{ inputs.mode }}-${{ github.run_id }}-${{ github.run_attempt }}
+ path: ${{ github.workspace }}/relay-incident
+ if-no-files-found: error
+ retention-days: 14
diff --git a/.github/workflows/cloud-monitor-relay-production.yml b/.github/workflows/cloud-monitor-relay-production.yml
new file mode 100644
index 00000000000..402e2aa9f69
--- /dev/null
+++ b/.github/workflows/cloud-monitor-relay-production.yml
@@ -0,0 +1,75 @@
+name: Monitor Relay Production
+
+on:
+ workflow_dispatch:
+ inputs:
+ mode:
+ description: Run the required 15-minute pre-drain gate or a 90-minute incident watch
+ required: true
+ default: dry-run
+ type: choice
+ options:
+ - dry-run
+ - monitor
+ expected-selector-generation:
+ description: Exact durable admission-selector generation
+ required: true
+ type: string
+ expected-existing-only-cells:
+ description: Exact comma-separated existing-only cells, or none
+ required: true
+ type: string
+ expected-migration-only-cells:
+ description: Exact comma-separated migration-only cells, or none
+ required: true
+ type: string
+ expected-general-cells:
+ description: Exact comma-separated general cells, or none
+ required: true
+ type: string
+ migration-policy:
+ description: Migration checks matched to the intended mutation
+ required: true
+ default: strict
+ type: choice
+ options:
+ - strict
+ - recover-forward
+ - capacity-transition
+ recovery-source-cell-id:
+ description: Existing-only recovery source cell, or none for strict monitoring
+ required: true
+ default: none
+ type: string
+ capacity-cell-id:
+ description: General cell for a capacity-transition monitor, or none
+ required: true
+ default: none
+ type: string
+
+permissions:
+ actions: read
+ contents: read
+ id-token: write
+
+concurrency:
+ group: production-cloud-sql-rollout
+ cancel-in-progress: false
+
+defaults:
+ run:
+ working-directory: cloud
+
+jobs:
+ monitor:
+ if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
+ uses: ./.github/workflows/cloud-monitor-relay-production-job.yml
+ with:
+ mode: ${{ inputs.mode }}
+ expected-selector-generation: ${{ inputs.expected-selector-generation }}
+ expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
+ expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
+ expected-general-cells: ${{ inputs.expected-general-cells }}
+ migration-policy: ${{ inputs.migration-policy }}
+ recovery-source-cell-id: ${{ inputs.recovery-source-cell-id }}
+ capacity-cell-id: ${{ inputs.capacity-cell-id }}
diff --git a/.github/workflows/cloud-operate-relay-asia-admission.yml b/.github/workflows/cloud-operate-relay-asia-admission.yml
new file mode 100644
index 00000000000..7abae2f8646
--- /dev/null
+++ b/.github/workflows/cloud-operate-relay-asia-admission.yml
@@ -0,0 +1,514 @@
+name: Operate Relay Asia Admission
+
+on:
+ workflow_dispatch:
+ inputs:
+ environment:
+ description: Target Relay environment
+ required: true
+ type: choice
+ options: [staging, production]
+ mode:
+ description: Inspect, initialize, verify, atomically register, promote, or roll back admission
+ required: true
+ default: verify
+ type: choice
+ options: [inspect, initialize, verify, register, configure, promote, rollback]
+ cell-ids:
+ description: Exact reviewed comma-separated Asia cell wave
+ required: true
+ type: string
+ selector-generation:
+ description: Exact live selector generation; leave empty only for inspect
+ required: false
+ type: string
+ selector-membership-sha256:
+ description: Exact fingerprint printed by inspect; required only for initialize
+ required: false
+ type: string
+ selector-attempt-id:
+ description: Durable unique attempt ID; empty for inspect, verify, and configure
+ required: false
+ type: string
+ image-digest:
+ description: Expected compatible Relay sha256 digest
+ required: true
+ type: string
+ director-image-digest:
+ description: Director sha256 digest; required only for configure
+ required: false
+ type: string
+ evidence-run-id:
+ description: Successful staging or C27 evidence workflow run ID; required for production promotion
+ required: false
+ type: string
+ evidence-run-attempt:
+ description: Exact evidence workflow run attempt; required for production promotion
+ required: false
+ type: string
+ confirmation:
+ description: Exact typed confirmation for a mutation
+ required: false
+ type: string
+
+permissions:
+ actions: read
+ contents: read
+ id-token: write
+
+concurrency:
+ group: ${{ inputs.environment == 'production' && 'production-cloud-sql-rollout' || 'relay-staging-mutation' }}
+ cancel-in-progress: false
+
+defaults:
+ run:
+ working-directory: cloud
+
+jobs:
+ admission:
+ if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }}
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ timeout-minutes: 30
+ environment: ${{ inputs.environment }}
+ env:
+ DIRECTOR_ORIGIN: ${{ inputs.environment == 'production' && 'https://relay.onorca.dev' || 'https://relay-staging.onorca.dev' }}
+ AUTH_ORIGIN: ${{ inputs.environment == 'production' && 'https://login.onorca.dev' || 'https://auth-staging.onorca.dev' }}
+ DIRECTOR_SERVICE: ${{ inputs.environment == 'production' && 'orca-cloud-relay' || 'orca-cloud-relay-staging' }}
+ REGIONAL_PLACEMENT_SECRET: orca-cloud-relay-regional-placement-enabled
+ GCP_PROJECT_ID: ${{ inputs.environment == 'production' && 'onorca-cloud' || 'onorca-cloud-staging' }}
+ GCP_REGION: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_REGION || vars.STAGING_GCP_REGION }}
+ DIRECTOR_MAX_INSTANCES: ${{ inputs.environment == 'production' && '5' || '2' }}
+ TF_BACKEND: ${{ inputs.environment == 'production' && 'backend/production.hcl' || 'backend/staging.hcl' }}
+ TARGET_ENVIRONMENT: ${{ inputs.environment }}
+ OPERATION_MODE: ${{ inputs.mode }}
+ TARGET_CELL_IDS: ${{ inputs.cell-ids }}
+ EXPECTED_SELECTOR_GENERATION: ${{ inputs.selector-generation }}
+ EXPECTED_SELECTOR_MEMBERSHIP_SHA256: ${{ inputs.selector-membership-sha256 }}
+ SELECTOR_ATTEMPT_ID: ${{ inputs.selector-attempt-id }}
+ IMAGE_DIGEST: ${{ inputs.image-digest }}
+ DIRECTOR_IMAGE_DIGEST: ${{ inputs.director-image-digest }}
+ EVIDENCE_RUN_ID: ${{ inputs.evidence-run-id }}
+ EVIDENCE_RUN_ATTEMPT: ${{ inputs.evidence-run-attempt }}
+ OPERATION_CONFIRMATION: ${{ inputs.confirmation }}
+ DEPLOY_WORKLOAD_IDENTITY_PROVIDER: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER || vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+ DEPLOY_SERVICE_ACCOUNT: ${{ inputs.environment == 'production' && vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT || vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Validate exact operation inputs before authentication
+ id: inputs
+ shell: bash
+ run: |
+ set -euo pipefail
+ test -n "${DEPLOY_WORKLOAD_IDENTITY_PROVIDER}"
+ test -n "${DEPLOY_SERVICE_ACCOUNT}"
+ [[ "${IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]
+ evidence_kind=none
+ if test "${OPERATION_MODE}" = inspect; then
+ test -z "${EXPECTED_SELECTOR_GENERATION}"
+ test -z "${SELECTOR_ATTEMPT_ID}"
+ test -z "${OPERATION_CONFIRMATION}"
+ test -z "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}"
+ test -z "${DIRECTOR_IMAGE_DIGEST}"
+ else
+ [[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
+ fi
+ if test "${OPERATION_MODE}" = initialize; then
+ test "${EXPECTED_SELECTOR_GENERATION}" = 0
+ [[ "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}" =~ ^[a-f0-9]{64}$ ]]
+ test -z "${DIRECTOR_IMAGE_DIGEST}"
+ [[ "${SELECTOR_ATTEMPT_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]]
+ test "${OPERATION_CONFIRMATION}" = INITIALIZE_ADMISSION_SELECTOR
+ elif test "${OPERATION_MODE}" = verify; then
+ test -z "${SELECTOR_ATTEMPT_ID}"
+ test -z "${OPERATION_CONFIRMATION}"
+ test -z "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}"
+ test -z "${DIRECTOR_IMAGE_DIGEST}"
+ elif test "${OPERATION_MODE}" = inspect; then
+ :
+ elif test "${OPERATION_MODE}" = configure; then
+ test -z "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}"
+ test -z "${SELECTOR_ATTEMPT_ID}"
+ [[ "${DIRECTOR_IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]
+ test "${OPERATION_CONFIRMATION}" = CONFIGURE_ASIA_DIRECTOR
+ else
+ test -z "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}"
+ test -z "${DIRECTOR_IMAGE_DIGEST}"
+ [[ "${SELECTOR_ATTEMPT_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]]
+ case "${OPERATION_MODE}:${OPERATION_CONFIRMATION}" in
+ register:REGISTER_ASIA_MIGRATION_ONLY) ;;
+ promote:PROMOTE_ASIA_GENERAL) ;;
+ rollback:ROLLBACK_ASIA_MIGRATION_ONLY) ;;
+ *) echo "typed confirmation does not match the requested mutation" >&2; exit 1 ;;
+ esac
+ fi
+ if test "${TARGET_ENVIRONMENT}:${OPERATION_MODE}" = production:promote; then
+ [[ "${EVIDENCE_RUN_ID}" =~ ^[1-9][0-9]*$ ]]
+ [[ "${EVIDENCE_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
+ case "${TARGET_CELL_IDS}" in
+ production-gce-c27)
+ test "${#SELECTOR_ATTEMPT_ID}" -le 119
+ evidence_kind=staging
+ artifact_name="relay-asia-staging-${EVIDENCE_RUN_ID}-${EVIDENCE_RUN_ATTEMPT}"
+ ;;
+ production-gce-c28,production-gce-c29)
+ evidence_kind=c27
+ artifact_name="relay-asia-c27-canary-${EVIDENCE_RUN_ID}-${EVIDENCE_RUN_ATTEMPT}"
+ ;;
+ *) echo "production promotion wave is not reviewed" >&2; exit 1 ;;
+ esac
+ else
+ test -z "${EVIDENCE_RUN_ID}"
+ test -z "${EVIDENCE_RUN_ATTEMPT}"
+ artifact_name=none
+ fi
+ {
+ echo "evidence_kind=${evidence_kind}"
+ echo "artifact_name=${artifact_name}"
+ } >> "${GITHUB_OUTPUT}"
+
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 24
+
+ - uses: pnpm/action-setup@v4
+ with:
+ package_json_file: cloud/package.json
+ if: ${{ inputs.environment == 'production' && inputs.mode == 'promote' && inputs.cell-ids == 'production-gce-c27' }}
+
+ - name: Install exact C27 canary dependencies
+ if: ${{ inputs.environment == 'production' && inputs.mode == 'promote' && inputs.cell-ids == 'production-gce-c27' }}
+ run: pnpm install --frozen-lockfile
+
+ - name: Build the C27 canary Relay contract
+ if: ${{ inputs.environment == 'production' && inputs.mode == 'promote' && inputs.cell-ids == 'production-gce-c27' }}
+ run: pnpm --filter @orca-cloud/relay-contract build
+
+ - name: Download immutable rollout evidence
+ if: ${{ steps.inputs.outputs.evidence_kind != 'none' }}
+ uses: actions/download-artifact@v4
+ with:
+ name: ${{ steps.inputs.outputs.artifact_name }}
+ path: ${{ runner.temp }}/relay-asia-input-evidence
+ github-token: ${{ github.token }}
+ run-id: ${{ inputs.evidence-run-id }}
+
+ - name: Verify evidence provenance and rollout binding before authentication
+ if: ${{ steps.inputs.outputs.evidence_kind != 'none' }}
+ env:
+ GH_TOKEN: ${{ github.token }}
+ EVIDENCE_KIND: ${{ steps.inputs.outputs.evidence_kind }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ run_json="${RUNNER_TEMP}/relay-asia-evidence-run.json"
+ verified="${RUNNER_TEMP}/relay-asia-evidence-verified"
+ gh api "/repos/${GITHUB_REPOSITORY}/actions/runs/${EVIDENCE_RUN_ID}/attempts/${EVIDENCE_RUN_ATTEMPT}" > "${run_json}"
+ evidence_commit_sha="$(
+ jq -er '.head_sha | select(type == "string" and test("^[a-f0-9]{40}$"))' "${run_json}"
+ )"
+ command=(node dev/scripts/relay-asia-rollout-evidence.mjs "verify-${EVIDENCE_KIND}"
+ --evidence "${RUNNER_TEMP}/relay-asia-input-evidence/evidence.json"
+ --run-json "${run_json}"
+ --commit-sha "${evidence_commit_sha}"
+ --image-digest "${IMAGE_DIGEST}"
+ --now "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+ --output "${verified}")
+ if test "${EVIDENCE_KIND}" = c27; then
+ command+=(--selector-generation "${EXPECTED_SELECTOR_GENERATION}")
+ fi
+ "${command[@]}"
+ test "$(< "${verified}")" = verified
+
+ - id: auth
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ env.DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ env.DEPLOY_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: ${{ env.DIRECTOR_ORIGIN }}/v1/admin/drain
+ id_token_include_email: true
+
+ - name: Require the exact director image before promotion
+ if: ${{ inputs.mode == 'promote' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ runtime="$(curl --fail-with-body --max-time 30 --request POST \
+ "${DIRECTOR_ORIGIN}/v1/admin/runtime-status" \
+ --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
+ --header 'Content-Type: application/json' --data '{"v":1}')"
+ test "$(jq -r '.role' <<< "${runtime}")" = director
+ test "$(jq -r '.imageDigest' <<< "${runtime}")" = "${IMAGE_DIGEST}"
+
+ - uses: google-github-actions/setup-gcloud@v2
+
+ - uses: ./.github/actions/cloud-sql-rollout-lease
+ with:
+ bucket: ${{ inputs.environment == 'production' && 'onorca-cloud-terraform-state' || 'onorca-cloud-staging-terraform-state' }}
+ object: ${{ inputs.environment == 'production' && 'terraform/state/cloud-sql-rollout/production.lock' || 'terraform/state/cloud-sql-rollout/staging.lock' }}
+ if: ${{ inputs.mode == 'configure' || (inputs.environment == 'production' && inputs.mode == 'promote' && inputs.cell-ids == 'production-gce-c27') }}
+
+ - uses: hashicorp/setup-terraform@v3
+ if: ${{ inputs.mode == 'configure' }}
+ with:
+ terraform_version: 1.15.8
+ terraform_wrapper: false
+
+ - name: Run the exact generation-bound admission operation
+ id: admission-operation
+ if: ${{ inputs.mode != 'configure' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
+ run: |
+ result="$(node dev/scripts/operate-relay-asia-admission.mjs \
+ --environment "${TARGET_ENVIRONMENT}" \
+ --mode "${OPERATION_MODE}" \
+ --cell-ids "${TARGET_CELL_IDS}" \
+ --expected-generation "${EXPECTED_SELECTOR_GENERATION}" \
+ --expected-membership-sha256 "${EXPECTED_SELECTOR_MEMBERSHIP_SHA256}" \
+ --attempt-id "${SELECTOR_ATTEMPT_ID}" \
+ --image-digest "${IMAGE_DIGEST}")"
+ generation="$(jq -er '.generation' <<< "${result}")"
+ states="$(jq -cS '.states // {}' <<< "${result}")"
+ membership="$(jq -cS '.membership // empty' <<< "${result}")"
+ membership_sha256="$(jq -r '.membershipSha256 // empty' <<< "${result}")"
+ echo "generation=${generation}" >> "${GITHUB_OUTPUT}"
+ result_dir="${RUNNER_TEMP}/relay-asia-admission-result"
+ mkdir -p "${result_dir}"
+ node dev/scripts/sanitize-relay-asia-admission-result.mjs \
+ <<< "${result}" > "${result_dir}/result.json"
+ {
+ echo "### Relay Asia admission"
+ echo "- Mode: ${OPERATION_MODE}"
+ echo "- Cells: ${TARGET_CELL_IDS}"
+ echo "- Result generation: ${generation}"
+ echo "- States: \`${states}\`"
+ if test -n "${membership}"; then echo "- Membership: \`${membership}\`"; fi
+ if test -n "${membership_sha256}"; then
+ echo "- Membership SHA-256: \`${membership_sha256}\`"
+ fi
+ } >> "${GITHUB_STEP_SUMMARY}"
+
+ - name: Verify C27 state and start the timed canary
+ id: c27-start
+ if: ${{ inputs.environment == 'production' && inputs.mode == 'promote' && inputs.cell-ids == 'production-gce-c27' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ result="$(node dev/scripts/operate-relay-asia-admission.mjs \
+ --environment production \
+ --mode verify \
+ --cell-ids production-gce-c27,production-gce-c28,production-gce-c29 \
+ --expected-generation "${{ steps.admission-operation.outputs.generation }}" \
+ --image-digest "${IMAGE_DIGEST}")"
+ test "$(jq -r '.states["production-gce-c27"]' <<< "${result}")" = general
+ test "$(jq -r '.states["production-gce-c28"]' <<< "${result}")" = migration-only
+ test "$(jq -r '.states["production-gce-c29"]' <<< "${result}")" = migration-only
+ echo "started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "${GITHUB_OUTPUT}"
+
+ - name: Run a real five-minute C27 control and splice canary
+ id: c27-load
+ if: ${{ inputs.environment == 'production' && inputs.mode == 'promote' && inputs.cell-ids == 'production-gce-c27' }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ log="${RUNNER_TEMP}/relay-asia-c27-load.jsonl"
+ report="${RUNNER_TEMP}/relay-asia-c27-load.json"
+ node dev/scripts/load-relay-controls.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --auth-origin "${AUTH_ORIGIN}" \
+ --preferred-region asia-east2 \
+ --relay-asia-load-principals 1 \
+ --controls 1 \
+ --splices 1 \
+ --capacity-hard-cap 3000 \
+ --ramp-seconds 0 \
+ --duration-seconds 300 \
+ --splice-hold-seconds 60 \
+ --required-lease-horizons 2 > "${log}"
+ jq -cer 'select(.event == "relay_load_complete")' "${log}" | tail -n 1 > "${report}"
+ echo "ended_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "${GITHUB_OUTPUT}"
+
+ - name: Collect regional, Relay SQL, and Cloud SQL canary evidence
+ if: ${{ inputs.environment == 'production' && inputs.mode == 'promote' && inputs.cell-ids == 'production-gce-c27' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
+ CANARY_STARTED_AT: ${{ steps.c27-start.outputs.started_at }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ result="$(node dev/scripts/operate-relay-asia-admission.mjs \
+ --environment production \
+ --mode verify \
+ --cell-ids production-gce-c27,production-gce-c28,production-gce-c29 \
+ --expected-generation "${{ steps.admission-operation.outputs.generation }}" \
+ --image-digest "${IMAGE_DIGEST}")"
+ test "$(jq -r '.states["production-gce-c27"]' <<< "${result}")" = general
+ ended_at="${{ steps.c27-load.outputs.ended_at }}"
+ sleep 60
+ output="${RUNNER_TEMP}/relay-asia-output-evidence"
+ logs="${RUNNER_TEMP}/relay-asia-c27-runtime-metrics.json"
+ mkdir -p "${output}"
+ gcloud logging read \
+ "timestamp>=\"${CANARY_STARTED_AT}\" AND timestamp<=\"${ended_at}\" AND jsonPayload.event=\"orca_relay_runtime_metrics\"" \
+ --project "${GCP_PROJECT_ID}" \
+ --limit 20000 \
+ --format json > "${logs}"
+ node dev/scripts/relay-asia-rollout-evidence.mjs create-c27 \
+ --repository "${GITHUB_REPOSITORY}" \
+ --run-id "${GITHUB_RUN_ID}" \
+ --run-attempt "${GITHUB_RUN_ATTEMPT}" \
+ --commit-sha "${GITHUB_SHA}" \
+ --image-digest "${IMAGE_DIGEST}" \
+ --selector-generation "${{ steps.admission-operation.outputs.generation }}" \
+ --started-at "${CANARY_STARTED_AT}" \
+ --ended-at "${ended_at}" \
+ --load-report "${RUNNER_TEMP}/relay-asia-c27-load.json" \
+ --logs-json "${logs}" \
+ --output "${output}/evidence.json"
+ jq -r '.metrics | to_entries[] | "- \(.key): \(.value)"' \
+ "${output}/evidence.json" >> "${GITHUB_STEP_SUMMARY}"
+
+ - name: Upload immutable C27 canary evidence
+ id: c27-evidence-upload
+ if: ${{ inputs.environment == 'production' && inputs.mode == 'promote' && inputs.cell-ids == 'production-gce-c27' }}
+ uses: actions/upload-artifact@v4
+ with:
+ name: relay-asia-c27-canary-${{ github.run_id }}-${{ github.run_attempt }}
+ path: ${{ runner.temp }}/relay-asia-output-evidence/evidence.json
+ if-no-files-found: error
+ retention-days: 7
+
+ - name: Upload sanitized admission result
+ if: ${{ inputs.mode != 'configure' && steps.admission-operation.outcome == 'success' }}
+ uses: actions/upload-artifact@v4
+ with:
+ name: relay-asia-admission-result-${{ github.run_id }}-${{ github.run_attempt }}
+ path: ${{ runner.temp }}/relay-asia-admission-result/result.json
+ if-no-files-found: error
+ retention-days: 7
+
+ - name: Return an unproven C27 canary to migration-only
+ if: ${{ always() && inputs.environment == 'production' && inputs.mode == 'promote' && inputs.cell-ids == 'production-gce-c27' && steps.admission-operation.outcome != 'skipped' && steps.c27-evidence-upload.outcome != 'success' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ promoted="$(node dev/scripts/operate-relay-asia-admission.mjs \
+ --environment production \
+ --mode recover-promotion \
+ --cell-ids production-gce-c27 \
+ --expected-generation "${EXPECTED_SELECTOR_GENERATION}" \
+ --attempt-id "${SELECTOR_ATTEMPT_ID}" \
+ --image-digest "${IMAGE_DIGEST}")"
+ if test "$(jq -r '.promoted' <<< "${promoted}")" = false; then exit 0; fi
+ promoted_generation="$(jq -er '.generation' <<< "${promoted}")"
+ result="$(node dev/scripts/operate-relay-asia-admission.mjs \
+ --environment production \
+ --mode rollback \
+ --cell-ids production-gce-c27 \
+ --expected-generation "${promoted_generation}" \
+ --attempt-id "${SELECTOR_ATTEMPT_ID}-rollback" \
+ --image-digest "${IMAGE_DIGEST}")"
+ test "$(jq -r '.states["production-gce-c27"]' <<< "${result}")" = migration-only
+
+ - name: Require registered migration-only cells before director configuration
+ if: ${{ inputs.mode == 'configure' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
+ run: |
+ result="$(node dev/scripts/operate-relay-asia-admission.mjs \
+ --environment "${TARGET_ENVIRONMENT}" \
+ --mode registered \
+ --cell-ids "${TARGET_CELL_IDS}" \
+ --expected-generation "${EXPECTED_SELECTOR_GENERATION}" \
+ --image-digest "${IMAGE_DIGEST}")"
+ test "$(jq -r '[.states[] == "migration-only"] | all' <<< "${result}")" = true
+
+ - name: Build the additive director cell configuration
+ if: ${{ inputs.mode == 'configure' }}
+ id: director-config
+ shell: bash
+ run: |
+ set -euo pipefail
+ terraform -chdir=infra/terraform init -reconfigure -input=false -backend-config="${TF_BACKEND}"
+ topology="${RUNNER_TEMP}/relay-asia-state-topology.json"
+ current="${RUNNER_TEMP}/relay-current-director-cells.json"
+ desired="${RUNNER_TEMP}/relay-asia-director-cells.json"
+ terraform -chdir=infra/terraform output -json relay_gce_cell_deployments > "${topology}"
+ service="$(gcloud run services describe "${DIRECTOR_SERVICE}" \
+ --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
+ revision="$(jq -er '[.status.traffic[] | select((.percent // 0) > 0)] |
+ if length == 1 and .[0].percent == 100 then .[0].revisionName else error("split traffic") end' \
+ <<< "${service}")"
+ gcloud run revisions describe "${revision}" \
+ --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
+ | jq -er '.spec.containers[0].env[] | select(.name == "ORCA_RELAY_CELLS_JSON") | .value | fromjson' \
+ > "${current}"
+ node dev/scripts/prepare-relay-asia-director-cells.mjs \
+ --current-json "${current}" \
+ --topology-json "${topology}" \
+ --output "${desired}" \
+ --cell-ids "${TARGET_CELL_IDS}" \
+ --image-digest "${IMAGE_DIGEST}"
+ echo "file=${desired}" >> "${GITHUB_OUTPUT}"
+
+ - name: Deploy the registered additive director topology
+ if: ${{ inputs.mode == 'configure' }}
+ env:
+ DIRECTOR_CELLS_FILE: ${{ steps.director-config.outputs.file }}
+ run: |
+ current="$(gcloud secrets versions access latest \
+ --project "${GCP_PROJECT_ID}" --secret "${REGIONAL_PLACEMENT_SECRET}")"
+ [[ "${current}" =~ ^(true|false)$ ]]
+ image="us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay@${DIRECTOR_IMAGE_DIGEST}"
+ release_id="asia-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}"
+ node dev/scripts/deploy-relay-blue-green.mjs \
+ --project "${GCP_PROJECT_ID}" \
+ --region "${GCP_REGION}" \
+ --service "${DIRECTOR_SERVICE}" \
+ --image "${image}" \
+ --role director \
+ --max-instances "${DIRECTOR_MAX_INSTANCES}" \
+ --release-id "${release_id}" \
+ --director-cells-json "$(< "${DIRECTOR_CELLS_FILE}")" \
+ --prune-revisions false
+
+ - name: Verify selector and heartbeats after director configuration
+ if: ${{ inputs.mode == 'configure' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
+ run: |
+ result="$(node dev/scripts/operate-relay-asia-admission.mjs \
+ --environment "${TARGET_ENVIRONMENT}" \
+ --mode verify \
+ --cell-ids "${TARGET_CELL_IDS}" \
+ --expected-generation "${EXPECTED_SELECTOR_GENERATION}" \
+ --image-digest "${IMAGE_DIGEST}")"
+ test "$(jq -r '[.states[] == "migration-only"] | all' <<< "${result}")" = true
+ revision="$(gcloud run services describe "${DIRECTOR_SERVICE}" \
+ --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
+ | jq -er '[.status.traffic[] | select((.percent // 0) > 0)] |
+ if length == 1 and .[0].percent == 100 then .[0].revisionName else error("split traffic") end')"
+ revision_json="$(gcloud run revisions describe "${revision}" \
+ --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
+ jq -e --arg image "us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay@${DIRECTOR_IMAGE_DIGEST}" \
+ '.spec.containers[0].image == $image' <<< "${revision_json}" > /dev/null
+ jq -er --arg secret "${REGIONAL_PLACEMENT_SECRET}" \
+ '[.spec.containers[0].env[] |
+ select(.name == "ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED") |
+ (.valueSource.secretKeyRef // .valueFrom.secretKeyRef // {}) |
+ {secret: (.secret // .name), version: (.version // .key)} |
+ select(.secret == $secret and (.version | test("^[1-9][0-9]*$")))] |
+ if length == 1 then .[0].version else error("regional switch version missing") end' \
+ <<< "${revision_json}" > "${RUNNER_TEMP}/relay-regional-placement-version"
+ regional_version="$(< "${RUNNER_TEMP}/relay-regional-placement-version")"
+ [[ "$(gcloud secrets versions access "${regional_version}" --project "${GCP_PROJECT_ID}" \
+ --secret "${REGIONAL_PLACEMENT_SECRET}")" =~ ^(true|false)$ ]]
+ echo "Director configuration now lists the registered migration-only Asia cells." >> "${GITHUB_STEP_SUMMARY}"
diff --git a/.github/workflows/cloud-operate-relay-production-rehome-job.yml b/.github/workflows/cloud-operate-relay-production-rehome-job.yml
new file mode 100644
index 00000000000..fdb1aca45e0
--- /dev/null
+++ b/.github/workflows/cloud-operate-relay-production-rehome-job.yml
@@ -0,0 +1,335 @@
+name: Operate Relay Production Rehome Job
+
+on:
+ workflow_call:
+ inputs:
+ mode: { required: true, type: string }
+ director-image-digest: { required: true, type: string }
+ rollback-image-digest: { required: true, type: string }
+ expected-selector-generation: { required: true, type: string }
+ expected-existing-only-cells: { required: true, type: string }
+ expected-migration-only-cells: { required: true, type: string }
+ expected-general-cells: { required: true, type: string }
+ expected-control-generation: { required: true, type: string }
+ not-before: { required: true, type: string }
+ rate-per-minute: { required: true, type: string }
+ preference-max-age-ms: { required: true, type: string }
+ drain-grace-ms: { required: true, type: string }
+ confirmation: { required: true, type: string }
+ monitor-run-id: { required: true, type: string }
+ monitor-run-attempt: { required: true, type: string }
+
+permissions:
+ actions: read
+ contents: read
+ id-token: write
+
+defaults:
+ run:
+ # `shell: bash` adds pipefail; without it `node ... | tee` reports tee's exit code and a
+ # thrown inspect/apply passed green (Aug 28-29 and Sep 3 2026 runs).
+ shell: bash
+ working-directory: cloud
+
+jobs:
+ control:
+ if: ${{ github.ref == 'refs/heads/main' }}
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ timeout-minutes: 30
+ environment: production
+ env:
+ GCP_PROJECT_ID: onorca-cloud
+ GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
+ DIRECTOR_SERVICE: orca-cloud-relay
+ DIRECTOR_ORIGIN: https://relay.onorca.dev
+ REHOME_AUDIENCE: https://relay.onorca.dev/v1/admin/host-drain
+ MODE: ${{ inputs.mode }}
+ DIRECTOR_IMAGE_DIGEST: ${{ inputs.director-image-digest }}
+ ROLLBACK_IMAGE_DIGEST: ${{ inputs.rollback-image-digest }}
+ EXPECTED_SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }}
+ EXPECTED_EXISTING_ONLY_CELLS: ${{ inputs.expected-existing-only-cells }}
+ EXPECTED_MIGRATION_ONLY_CELLS: ${{ inputs.expected-migration-only-cells }}
+ EXPECTED_GENERAL_CELLS: ${{ inputs.expected-general-cells }}
+ EXPECTED_CONTROL_GENERATION: ${{ inputs.expected-control-generation }}
+ NOT_BEFORE: ${{ inputs.not-before }}
+ RATE_PER_MINUTE: ${{ inputs.rate-per-minute }}
+ PREFERENCE_MAX_AGE_MS: ${{ inputs.preference-max-age-ms }}
+ DRAIN_GRACE_MS: ${{ inputs.drain-grace-ms }}
+ CONFIRMATION: ${{ inputs.confirmation }}
+ MONITOR_RUN_ID: ${{ inputs.monitor-run-id }}
+ MONITOR_RUN_ATTEMPT: ${{ inputs.monitor-run-attempt }}
+ OUTPUT_DIRECTORY: ${{ github.workspace }}/relay-monitor-evidence
+ steps:
+ - name: Require exact reusable-workflow configuration
+ working-directory: .
+ env:
+ DEPLOY_WIF: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+ DEPLOY_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+ DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
+ run: |
+ [[ "${MODE}" =~ ^(inspect|enable|pause|disable)$ ]]
+ [[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
+ [[ "${EXPECTED_CONTROL_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
+ test -n "${DEPLOY_WIF}"
+ test -n "${DEPLOY_SERVICE_ACCOUNT}"
+ if [[ "${MODE}" =~ ^(inspect|enable)$ ]]; then
+ [[ "${DIRECTOR_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
+ [[ "${ROLLBACK_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
+ test -n "${GCP_REGION}"
+ test -n "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}"
+ fi
+ case "${MODE}" in
+ inspect)
+ test -z "${CONFIRMATION}"
+ ;;
+ enable)
+ test "${CONFIRMATION}" = ENABLE_REGIONAL_REHOMING
+ test "${RATE_PER_MINUTE}" = 10
+ [[ "${NOT_BEFORE}" =~ ^[1-9][0-9]*$ ]]
+ ;;
+ pause)
+ test "${CONFIRMATION}" = PAUSE_REGIONAL_REHOMING
+ ;;
+ disable)
+ test "${CONFIRMATION}" = DISABLE_REGIONAL_REHOMING
+ ;;
+ esac
+
+ # Full history: the monitor evidence this job verifies is sealed at an ancestor commit,
+ # and the provenance check fails closed on a commit a shallow clone left out.
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 24
+
+ - id: google-auth
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - name: Apply emergency durable pause or disable before diagnostics
+ if: ${{ inputs.mode == 'pause' || inputs.mode == 'disable' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ run: |
+ node dev/scripts/operate-relay-regional-rehome.mjs \
+ --mode "${MODE}" --director-origin "${DIRECTOR_ORIGIN}" \
+ --expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
+ --expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
+ --expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \
+ --expected-general-cells "${EXPECTED_GENERAL_CELLS}" \
+ --expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \
+ --not-before "${NOT_BEFORE}" --rate-per-minute "${RATE_PER_MINUTE}" \
+ --preference-max-age-ms "${PREFERENCE_MAX_AGE_MS}" \
+ --drain-grace-ms "${DRAIN_GRACE_MS}" --confirmation "${CONFIRMATION}" \
+ | tee "${RUNNER_TEMP}/relay-rehome-control.json"
+
+ - uses: pnpm/action-setup@v4
+ if: ${{ inputs.mode == 'enable' }}
+ with: { package_json_file: cloud/package.json }
+
+ - run: pnpm install --frozen-lockfile
+ if: ${{ inputs.mode == 'enable' }}
+
+ - name: Download fresh aggregate safety evidence
+ if: ${{ inputs.mode == 'enable' }}
+ uses: actions/download-artifact@v4
+ with:
+ name: relay-monitor-dry-run-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
+ path: ${{ github.workspace }}/relay-monitor-evidence
+ github-token: ${{ github.token }}
+ run-id: ${{ inputs.monitor-run-id }}
+
+ - name: Verify enable evidence provenance
+ if: ${{ inputs.mode == 'enable' }}
+ run: |
+ [[ "${MONITOR_RUN_ID}" =~ ^[1-9][0-9]*$ ]]
+ [[ "${MONITOR_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
+ node dev/scripts/relay-monitor-evidence.mjs verify-authority \
+ --directory "${OUTPUT_DIRECTORY}" \
+ --incident-id "relay-${MONITOR_RUN_ID}-dry-run" \
+ --run-id "${MONITOR_RUN_ID}" --run-attempt "${MONITOR_RUN_ATTEMPT}" \
+ --commit-sha "${GITHUB_SHA}" --mode dry-run \
+ --required-migration-policy strict
+
+ - name: Reject previously consumed enable safety evidence
+ if: ${{ inputs.mode == 'enable' }}
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: |
+ MARKER_NAME="relay-rehome-enable-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
+ COUNT="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/artifacts?name=${MARKER_NAME}&per_page=1" \
+ --jq '.total_count')"
+ test "${COUNT}" = 0
+
+ - uses: google-github-actions/setup-gcloud@v2
+
+ - uses: ./.github/actions/cloud-sql-rollout-lease
+ with:
+ bucket: onorca-cloud-terraform-state
+ object: terraform/state/cloud-sql-rollout/production.lock
+
+ - name: Verify exact serving and rollback director identities
+ if: ${{ inputs.mode == 'inspect' || inputs.mode == 'enable' }}
+ env:
+ DIRECTOR_RUNTIME_SERVICE_ACCOUNT: ${{ vars.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT }}
+ run: |
+ SERVICE_JSON="$(gcloud run services describe "${DIRECTOR_SERVICE}" \
+ --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
+ SERVING_REVISION="$(jq -er \
+ '[.status.traffic[] | select((.percent // 0) > 0)] |
+ if length == 1 and .[0].percent == 100 then .[0].revisionName
+ else error("director does not have one serving revision") end' \
+ <<< "${SERVICE_JSON}")"
+ ROLLBACK_REVISION="$(jq -er \
+ '[.status.traffic[] | select(.tag == "selector-rollback")] |
+ if length == 1 then .[0].revisionName else error("rollback tag missing") end' \
+ <<< "${SERVICE_JSON}")"
+ verify_revision() {
+ local revision="$1" expected_digest="$2"
+ local json
+ json="$(gcloud run revisions describe "${revision}" \
+ --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
+ test "$(jq -r '.spec.serviceAccountName' <<< "${json}")" = \
+ "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}"
+ test "$(jq -r '.spec.containers[0].image | split("@") | last' <<< "${json}")" = \
+ "${expected_digest}"
+ test "$(jq -r '[.spec.containers[0].env[] | select(.name ==
+ "ORCA_RELAY_REHOME_DIRECTOR_SERVICE_ACCOUNT") | .value] | if length == 1
+ then .[0] else empty end' <<< "${json}")" = "${DIRECTOR_RUNTIME_SERVICE_ACCOUNT}"
+ test "$(jq -r '[.spec.containers[0].env[] | select(.name ==
+ "ORCA_RELAY_REHOME_AUDIENCE") | .value] | if length == 1 then .[0]
+ else empty end' <<< "${json}")" = "${REHOME_AUDIENCE}"
+ }
+ verify_revision "${SERVING_REVISION}" "${DIRECTOR_IMAGE_DIGEST}"
+ verify_revision "${ROLLBACK_REVISION}" "${ROLLBACK_IMAGE_DIGEST}"
+
+ - name: Seal 24-hour aggregate region observation evidence
+ if: ${{ inputs.mode == 'enable' }}
+ run: |
+ mkdir -p "${RUNNER_TEMP}/relay-region-observation"
+ # 6 director instances x 120 samples/hour x 25h = 18000; a clipped
+ # read empties the oldest hourly buckets and fails the seal.
+ gcloud logging read \
+ 'resource.type="cloud_run_revision" AND resource.labels.service_name="orca-cloud-relay" AND jsonPayload.event="orca_relay_runtime_metrics" AND jsonPayload.role="director"' \
+ --project "${GCP_PROJECT_ID}" --freshness=25h --limit=30000 --format=json \
+ | node dev/scripts/relay-region-observation-evidence.mjs create \
+ --commit-sha "${GITHUB_SHA}" \
+ --director-image-digest "${DIRECTOR_IMAGE_DIGEST}" \
+ --selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
+ --control-generation "${EXPECTED_CONTROL_GENERATION}" \
+ > "${RUNNER_TEMP}/relay-region-observation/evidence.json"
+
+ - name: Upload sealed 24-hour aggregate region evidence
+ if: ${{ inputs.mode == 'enable' }}
+ uses: actions/upload-artifact@v4
+ with:
+ name: relay-region-observation-${{ github.run_id }}-${{ github.run_attempt }}
+ path: ${{ runner.temp }}/relay-region-observation/evidence.json
+ retention-days: 90
+ if-no-files-found: error
+
+ - name: Verify sealed 24-hour enable authority
+ if: ${{ inputs.mode == 'enable' }}
+ run: |
+ node dev/scripts/relay-region-observation-evidence.mjs verify \
+ --file "${RUNNER_TEMP}/relay-region-observation/evidence.json" \
+ --commit-sha "${GITHUB_SHA}" \
+ --director-image-digest "${DIRECTOR_IMAGE_DIGEST}" \
+ --selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
+ --control-generation "${EXPECTED_CONTROL_GENERATION}"
+
+ - name: Recheck every aggregate safety signal before enable
+ if: ${{ inputs.mode == 'enable' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ run: |
+ pnpm incident:relay-preflight -- \
+ --state-file "${OUTPUT_DIRECTORY}/relay-${MONITOR_RUN_ID}-dry-run.state.json"
+
+ - name: Seal single-use enable safety authority
+ if: ${{ inputs.mode == 'enable' }}
+ run: |
+ MARKER_NAME="relay-rehome-enable-monitor-consumed-${MONITOR_RUN_ID}-${MONITOR_RUN_ATTEMPT}"
+ mkdir -p "${RUNNER_TEMP}/relay-rehome-enable-authority"
+ printf '%s\n' "${GITHUB_RUN_ID}" \
+ > "${RUNNER_TEMP}/relay-rehome-enable-authority/${MARKER_NAME}"
+
+ - name: Consume enable safety evidence before durable mutation
+ if: ${{ inputs.mode == 'enable' }}
+ uses: actions/upload-artifact@v4
+ with:
+ name: relay-rehome-enable-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
+ path: ${{ runner.temp }}/relay-rehome-enable-authority/relay-rehome-enable-monitor-consumed-${{ inputs.monitor-run-id }}-${{ inputs.monitor-run-attempt }}
+ retention-days: 90
+ if-no-files-found: error
+
+ - name: Inspect regional rehome control
+ if: ${{ inputs.mode == 'inspect' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ run: |
+ node dev/scripts/operate-relay-regional-rehome.mjs \
+ --mode inspect --director-origin "${DIRECTOR_ORIGIN}" \
+ --expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
+ --expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
+ --expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \
+ --expected-general-cells "${EXPECTED_GENERAL_CELLS}" \
+ --expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \
+ | tee "${RUNNER_TEMP}/relay-rehome-control.json"
+
+ - name: Apply exact durable regional rehome enable
+ if: ${{ inputs.mode == 'enable' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ run: |
+ node dev/scripts/operate-relay-regional-rehome.mjs \
+ --mode "${MODE}" --director-origin "${DIRECTOR_ORIGIN}" \
+ --expected-selector-generation "${EXPECTED_SELECTOR_GENERATION}" \
+ --expected-existing-only-cells "${EXPECTED_EXISTING_ONLY_CELLS}" \
+ --expected-migration-only-cells "${EXPECTED_MIGRATION_ONLY_CELLS}" \
+ --expected-general-cells "${EXPECTED_GENERAL_CELLS}" \
+ --expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \
+ --not-before "${NOT_BEFORE}" --rate-per-minute "${RATE_PER_MINUTE}" \
+ --preference-max-age-ms "${PREFERENCE_MAX_AGE_MS}" \
+ --drain-grace-ms "${DRAIN_GRACE_MS}" --confirmation "${CONFIRMATION}" \
+ | tee "${RUNNER_TEMP}/relay-rehome-control.json"
+
+ - name: Read fresh aggregate completion and abort evidence
+ run: |
+ gcloud logging read \
+ 'resource.type="cloud_run_revision" AND resource.labels.service_name="orca-cloud-relay" AND textPayload:"[orca-relay] regional rehome inventory"' \
+ --project "${GCP_PROJECT_ID}" --freshness=15m --limit=20 --format=json \
+ | node dev/scripts/relay-rehome-aggregate-evidence.mjs --max-age-ms 900000 \
+ | tee "${RUNNER_TEMP}/relay-rehome-inventory.json"
+
+ - name: Publish aggregate control evidence
+ run: |
+ {
+ echo '### Regional rehome control'
+ jq -r '"- mode: `\(.mode)`\n- generation: `\(.control.generation)`\n- enabled: `\(.control.enabled)`"' \
+ "${RUNNER_TEMP}/relay-rehome-control.json"
+ jq -r '"- active: `\(.active)`\n- awaiting receipt: `\(.awaitingReceipt)`\n- target registered: `\(.targetRegistered)`\n- completed (24h): `\(.completedLast24Hours)`\n- aborted (24h): `\(.abortedLast24Hours)`"' \
+ "${RUNNER_TEMP}/relay-rehome-inventory.json"
+ } >> "${GITHUB_STEP_SUMMARY}"
+
+ - name: Fail closed after an unsuccessful enable run
+ if: ${{ failure() && inputs.mode == 'enable' && steps.google-auth.outcome == 'success' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ run: |
+ node dev/scripts/operate-relay-regional-rehome.mjs \
+ --mode recover-enable --director-origin "${DIRECTOR_ORIGIN}" \
+ --expected-control-generation "${EXPECTED_CONTROL_GENERATION}" \
+ --confirmation RECOVER_FAILED_REGIONAL_REHOME_ENABLE \
+ | tee "${RUNNER_TEMP}/relay-rehome-enable-recovery.json"
+ jq -e \
+ '.mode == "recover-enable" and .control.enabled == false' \
+ "${RUNNER_TEMP}/relay-rehome-enable-recovery.json" >/dev/null
diff --git a/.github/workflows/cloud-operate-relay-production-rehome.yml b/.github/workflows/cloud-operate-relay-production-rehome.yml
new file mode 100644
index 00000000000..40bf5ebbd4f
--- /dev/null
+++ b/.github/workflows/cloud-operate-relay-production-rehome.yml
@@ -0,0 +1,106 @@
+name: Operate Relay Production Rehome
+
+on:
+ workflow_dispatch:
+ inputs:
+ mode:
+ description: Inspect or apply the durable regional-rehome switch
+ required: true
+ default: inspect
+ type: choice
+ options: [inspect, enable, pause, disable]
+ director-image-digest:
+ description: Exact immutable serving director digest
+ required: true
+ type: string
+ rollback-image-digest:
+ description: Exact immutable selector-rollback director digest
+ required: true
+ type: string
+ expected-selector-generation:
+ description: Exact admission selector generation
+ required: true
+ type: string
+ expected-existing-only-cells:
+ description: Exact existing-only membership, or none
+ required: true
+ type: string
+ expected-migration-only-cells:
+ description: Exact migration-only membership, or none
+ required: true
+ type: string
+ expected-general-cells:
+ description: Exact general membership, or none
+ required: true
+ type: string
+ expected-control-generation:
+ description: Exact durable rehome generation
+ required: true
+ type: string
+ not-before:
+ description: Exact epoch milliseconds; ignored only by inspect
+ required: true
+ default: '0'
+ type: string
+ rate-per-minute:
+ description: Exact global host rate; initial enable is fixed at 10
+ required: true
+ default: '10'
+ type: string
+ preference-max-age-ms:
+ description: Maximum fresh preference age
+ required: true
+ default: '86400000'
+ type: string
+ drain-grace-ms:
+ description: Per-host source drain grace
+ required: true
+ default: '3600000'
+ type: string
+ monitor-run-id:
+ description: Fresh successful aggregate dry-run required only by enable
+ required: false
+ type: string
+ monitor-run-attempt:
+ description: Exact monitor attempt required only by enable
+ required: false
+ type: string
+ confirmation:
+ description: ENABLE_REGIONAL_REHOMING, PAUSE_REGIONAL_REHOMING, or DISABLE_REGIONAL_REHOMING
+ required: false
+ type: string
+
+permissions:
+ actions: read
+ contents: read
+ id-token: write
+
+concurrency:
+ group: production-cloud-sql-rollout
+ cancel-in-progress: false
+
+defaults:
+ run:
+ working-directory: cloud
+
+jobs:
+ operate:
+ if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
+ uses: ./.github/workflows/cloud-operate-relay-production-rehome-job.yml
+ with:
+ mode: ${{ inputs.mode }}
+ director-image-digest: ${{ inputs.director-image-digest }}
+ rollback-image-digest: ${{ inputs.rollback-image-digest }}
+ expected-selector-generation: ${{ inputs.expected-selector-generation }}
+ expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
+ expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
+ expected-general-cells: ${{ inputs.expected-general-cells }}
+ expected-control-generation: ${{ inputs.expected-control-generation }}
+ not-before: ${{ inputs.not-before }}
+ rate-per-minute: ${{ inputs.rate-per-minute }}
+ preference-max-age-ms: ${{ inputs.preference-max-age-ms }}
+ drain-grace-ms: ${{ inputs.drain-grace-ms }}
+ confirmation: ${{ inputs.confirmation }}
+ monitor-run-id: ${{ inputs.monitor-run-id }}
+ monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
+ secrets: inherit
diff --git a/.github/workflows/cloud-power-relay-staging.yml b/.github/workflows/cloud-power-relay-staging.yml
new file mode 100644
index 00000000000..0e8311e4ec2
--- /dev/null
+++ b/.github/workflows/cloud-power-relay-staging.yml
@@ -0,0 +1,101 @@
+name: Power Relay Staging
+
+on:
+ schedule:
+ # A zero-activity guard makes this a no-op when an internal test is still running.
+ - cron: '0 9 * * *'
+ workflow_dispatch:
+ inputs:
+ mode:
+ description: Inspect, wake, or sleep the staging Relay data plane
+ required: true
+ default: status
+ type: choice
+ options:
+ - status
+ - wake
+ - sleep
+ wake-cells:
+ description: Wake configured admission cells, or include disabled candidate cells
+ required: true
+ default: configured
+ type: choice
+ options:
+ - configured
+ - all
+ confirmation:
+ description: Enter WAKE_STAGING or SLEEP_STAGING for a manual mutation
+ required: false
+ type: string
+
+permissions:
+ contents: read
+ id-token: write
+
+concurrency:
+ group: relay-staging-mutation
+ cancel-in-progress: false
+
+defaults:
+ run:
+ working-directory: cloud
+
+jobs:
+ power:
+ if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ environment: staging
+ env:
+ POWER_MODE: ${{ github.event_name == 'schedule' && 'sleep' || inputs.mode }}
+ WAKE_CELLS: ${{ inputs.wake-cells || 'configured' }}
+ steps:
+ - uses: actions/checkout@v4
+
+ - id: google-auth
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - uses: google-github-actions/setup-gcloud@v2
+
+ - uses: ./.github/actions/cloud-sql-rollout-lease
+ with:
+ bucket: onorca-cloud-staging-terraform-state
+ object: terraform/state/cloud-sql-rollout/staging.lock
+
+ - uses: hashicorp/setup-terraform@v3
+ with:
+ terraform_wrapper: false
+
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 24
+
+ - name: Require explicit manual mutation confirmation
+ if: ${{ github.event_name == 'workflow_dispatch' && inputs.mode != 'status' }}
+ env:
+ CONFIRMATION: ${{ inputs.confirmation }}
+ run: |
+ if [[ "${POWER_MODE}" = "wake" ]]; then
+ test "${CONFIRMATION}" = "WAKE_STAGING"
+ else
+ test "${CONFIRMATION}" = "SLEEP_STAGING"
+ fi
+
+ - name: Read reviewed staging topology
+ run: |
+ node dev/scripts/infra.mjs init --env staging
+ terraform -chdir=infra/terraform output -json relay_gce_cell_deployments > "${RUNNER_TEMP}/relay-gce-topology.json"
+
+ - name: Inspect or change staging power state
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ run: |
+ node dev/scripts/power-staging-relay.mjs \
+ --mode "${POWER_MODE}" \
+ --wake-cells "${WAKE_CELLS}" \
+ --topology-file "${RUNNER_TEMP}/relay-gce-topology.json"
diff --git a/.github/workflows/cloud-prove-relay-asia-staging.yml b/.github/workflows/cloud-prove-relay-asia-staging.yml
new file mode 100644
index 00000000000..56677a98600
--- /dev/null
+++ b/.github/workflows/cloud-prove-relay-asia-staging.yml
@@ -0,0 +1,333 @@
+name: Prove Relay Asia Staging
+
+on:
+ workflow_dispatch:
+ inputs:
+ image-digest:
+ description: Exact immutable Relay digest deployed on staging C4
+ required: true
+ type: string
+ selector-generation:
+ description: Exact selector generation with C4 migration-only
+ required: true
+ type: string
+ promote-attempt-id:
+ description: Durable unique C4 promotion attempt ID
+ required: true
+ type: string
+ rollback-attempt-id:
+ description: Durable unique C4 rollback attempt ID
+ required: true
+ type: string
+ confirmation:
+ description: Enter PROVE_ASIA_STAGING
+ required: true
+ type: string
+
+permissions:
+ contents: read
+ id-token: write
+
+concurrency:
+ group: relay-staging-mutation
+ cancel-in-progress: false
+
+defaults:
+ run:
+ working-directory: cloud
+
+jobs:
+ prove:
+ if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.ref == 'refs/heads/main') }}
+ runs-on: [self-hosted, linux, x64, relay-asia-east2-load]
+ timeout-minutes: 75
+ environment: staging
+ env:
+ GCP_PROJECT_ID: onorca-cloud-staging
+ DIRECTOR_ORIGIN: https://relay-staging.onorca.dev
+ AUTH_ORIGIN: https://auth-staging.onorca.dev
+ IMAGE_DIGEST: ${{ inputs.image-digest }}
+ INITIAL_SELECTOR_GENERATION: ${{ inputs.selector-generation }}
+ PROMOTE_ATTEMPT_ID: ${{ inputs.promote-attempt-id }}
+ ROLLBACK_ATTEMPT_ID: ${{ inputs.rollback-attempt-id }}
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Validate the exact staging proof request
+ shell: bash
+ env:
+ CONFIRMATION: ${{ inputs.confirmation }}
+ run: |
+ set -euo pipefail
+ test "${CONFIRMATION}" = PROVE_ASIA_STAGING
+ [[ "${IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]
+ [[ "${INITIAL_SELECTOR_GENERATION}" =~ ^[1-9][0-9]*$ ]]
+ [[ "${PROMOTE_ATTEMPT_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]]
+ [[ "${ROLLBACK_ATTEMPT_ID}" =~ ^[A-Za-z0-9_-]{8,128}$ ]]
+ test "${PROMOTE_ATTEMPT_ID}" != "${ROLLBACK_ATTEMPT_ID}"
+
+ - id: auth
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - uses: google-github-actions/setup-gcloud@v2
+
+ - uses: ./.github/actions/cloud-sql-rollout-lease
+ with:
+ bucket: onorca-cloud-staging-terraform-state
+ object: terraform/state/cloud-sql-rollout/staging.lock
+
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 24
+
+ - uses: pnpm/action-setup@v4
+ with:
+ package_json_file: cloud/package.json
+
+ - name: Require the exact staging director image before promotion
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ runtime="$(curl --fail-with-body --max-time 30 --request POST \
+ "${DIRECTOR_ORIGIN}/v1/admin/runtime-status" \
+ --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
+ --header 'Content-Type: application/json' --data '{"v":1}')"
+ test "$(jq -r '.role' <<< "${runtime}")" = director
+ test "$(jq -r '.imageDigest' <<< "${runtime}")" = "${IMAGE_DIGEST}"
+
+ - name: Install exact load-harness dependencies
+ run: pnpm install --frozen-lockfile
+
+ - name: Build the Relay load-harness contract
+ run: pnpm --filter @orca-cloud/relay-contract build
+
+ - name: Promote only staging C4
+ id: promote
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ result="$(node dev/scripts/operate-relay-asia-admission.mjs \
+ --environment staging \
+ --mode promote \
+ --cell-ids staging-gce-c4 \
+ --expected-generation "${INITIAL_SELECTOR_GENERATION}" \
+ --attempt-id "${PROMOTE_ATTEMPT_ID}" \
+ --image-digest "${IMAGE_DIGEST}")"
+ generation="$(jq -er '.generation' <<< "${result}")"
+ test "$(jq -r '.states["staging-gce-c4"]' <<< "${result}")" = general
+ echo "generation=${generation}" >> "${GITHUB_OUTPUT}"
+ echo "started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "${GITHUB_OUTPUT}"
+
+ - name: Run sharded two-horizon and mixed splice proofs
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ proof_dir="${RUNNER_TEMP}/relay-asia-staging-proof"
+ mkdir -p "${proof_dir}"
+ fd_limit="$(ulimit -n)"
+ if test "${fd_limit}" != unlimited; then
+ [[ "${fd_limit}" =~ ^[0-9]+$ ]]
+ test "${fd_limit}" -ge 4096
+ fi
+ run_phase() {
+ phase="$1"
+ controls="$2"
+ splices="$3"
+ pids=()
+ stop_shards() {
+ for pid in "${pids[@]}"; do kill "${pid}" 2>/dev/null || true; done
+ for pid in "${pids[@]}"; do wait "${pid}" 2>/dev/null || true; done
+ }
+ trap stop_shards EXIT
+ for shard in 0 1 2 3; do
+ slow=0
+ wedged=0
+ boundary_args=()
+ request_unit_args=()
+ if test "${phase}" = launch && test "${shard}" = 0; then
+ slow=4
+ wedged=1
+ fi
+ if test "${phase}" = launch && test "${shard}" = 0; then
+ boundary_args=(
+ --region-behavior-probes 1
+ --capacity-cell-id staging-gce-c4
+ --capacity-cell-origin https://c4.relay-staging.onorca.dev
+ --capacity-unobserved-bound 60
+ --rebind-probes 2
+ --skip-rebind-overflow-check
+ )
+ fi
+ node dev/scripts/load-relay-controls.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --auth-origin "${AUTH_ORIGIN}" \
+ --preferred-region asia-east2 \
+ --relay-asia-load-principals 32 \
+ --controls "${controls}" \
+ --splices "${splices}" \
+ --slow-reader-splices "${slow}" \
+ --wedged-reader-splices "${wedged}" \
+ --capacity-hard-cap 3000 \
+ "${boundary_args[@]}" \
+ "${request_unit_args[@]}" \
+ --phase-barrier-dir "${proof_dir}/${phase}-barrier" \
+ --aggregate-controls "$((controls * 4))" \
+ --aggregate-splices "$((splices * 4))" \
+ --aggregate-reader-splices "$([[ "${phase}" = launch ]] && echo 5 || echo 0)" \
+ --aggregate-reader-bytes "$([[ "${phase}" = launch ]] && echo 12582912 || echo 0)" \
+ --required-lease-horizons 2 \
+ --splice-ramp-seconds 120 \
+ --max-generator-rss-growth-mib 512 \
+ --ramp-seconds 180 \
+ --duration-seconds 210 \
+ --shard-count 4 \
+ --shard-index "${shard}" \
+ > "${proof_dir}/${phase}-${shard}.jsonl" &
+ pids+=("$!")
+ done
+ failed=0
+ for pid in "${pids[@]}"; do
+ if ! wait "${pid}"; then failed=1; break; fi
+ done
+ if test "${failed}" = 1; then
+ stop_shards
+ for shard in 0 1 2 3; do
+ jq -cer 'select(.event == "relay_load_progress" or .event == "relay_load_complete") |
+ {event, shardIndex, active, peakActive, connected, connectionFailures,
+ rampConnectionFailures, connectionFailuresByReason, unexpectedCloses,
+ protocolErrors, refreshErrors, socketErrors, elapsedSeconds}' \
+ "${proof_dir}/${phase}-${shard}.jsonl" | tail -n 1 || true
+ done
+ trap - EXIT
+ return 1
+ fi
+ trap - EXIT
+ for shard in 0 1 2 3; do
+ jq -cer 'select(.event == "relay_load_complete")' \
+ "${proof_dir}/${phase}-${shard}.jsonl" | tail -n 1
+ done | jq -s . > "${proof_dir}/${phase}.json"
+ }
+ run_phase launch 5 5
+
+ - name: Collect and validate aggregate staging proof evidence
+ env:
+ PROOF_STARTED_AT: ${{ steps.promote.outputs.started_at }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ proof_dir="${RUNNER_TEMP}/relay-asia-staging-proof"
+ ended_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+ sleep 60
+ gcloud logging read \
+ "timestamp>=\"${PROOF_STARTED_AT}\" AND timestamp<=\"${ended_at}\" AND jsonPayload.event=\"orca_relay_runtime_metrics\"" \
+ --project "${GCP_PROJECT_ID}" --limit 20000 --format json \
+ > "${proof_dir}/runtime-metrics.json"
+ node dev/scripts/relay-asia-rollout-evidence.mjs create-staging \
+ --repository "${GITHUB_REPOSITORY}" \
+ --run-id "${GITHUB_RUN_ID}" \
+ --run-attempt "${GITHUB_RUN_ATTEMPT}" \
+ --commit-sha "${GITHUB_SHA}" \
+ --image-digest "${IMAGE_DIGEST}" \
+ --selector-generation "${{ steps.promote.outputs.generation }}" \
+ --started-at "${PROOF_STARTED_AT}" \
+ --ended-at "${ended_at}" \
+ --launch-report "${proof_dir}/launch.json" \
+ --logs-json "${proof_dir}/runtime-metrics.json" \
+ --output "${proof_dir}/evidence.json"
+
+ - name: Return staging C4 to migration-only
+ if: ${{ always() && steps.promote.outcome != 'skipped' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ promoted="$(node dev/scripts/operate-relay-asia-admission.mjs \
+ --environment staging \
+ --mode recover-promotion \
+ --cell-ids staging-gce-c4 \
+ --expected-generation "${INITIAL_SELECTOR_GENERATION}" \
+ --attempt-id "${PROMOTE_ATTEMPT_ID}" \
+ --image-digest "${IMAGE_DIGEST}")"
+ if test "$(jq -r '.promoted' <<< "${promoted}")" = false; then exit 0; fi
+ promoted_generation="$(jq -er '.generation' <<< "${promoted}")"
+ result="$(node dev/scripts/operate-relay-asia-admission.mjs \
+ --environment staging \
+ --mode rollback \
+ --cell-ids staging-gce-c4 \
+ --expected-generation "${promoted_generation}" \
+ --attempt-id "${ROLLBACK_ATTEMPT_ID}" \
+ --image-digest "${IMAGE_DIGEST}")"
+ test "$(jq -r '.states["staging-gce-c4"]' <<< "${result}")" = migration-only
+
+ - name: Upload immutable staging readiness evidence
+ if: ${{ success() }}
+ uses: actions/upload-artifact@v4
+ with:
+ name: relay-asia-staging-${{ github.run_id }}-${{ github.run_attempt }}
+ path: ${{ runner.temp }}/relay-asia-staging-proof/evidence.json
+ if-no-files-found: error
+ retention-days: 7
+
+ recover:
+ if: ${{ always() && github.ref == 'refs/heads/main' }}
+ needs: prove
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ timeout-minutes: 15
+ environment: staging
+ env:
+ IMAGE_DIGEST: ${{ inputs.image-digest }}
+ INITIAL_SELECTOR_GENERATION: ${{ inputs.selector-generation }}
+ PROMOTE_ATTEMPT_ID: ${{ inputs.promote-attempt-id }}
+ ROLLBACK_ATTEMPT_ID: ${{ inputs.rollback-attempt-id }}
+ steps:
+ - uses: actions/checkout@v4
+
+ - id: auth
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 24
+
+ - name: Recover staging C4 with a fresh identity
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.auth.outputs.id_token }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ promoted="$(node dev/scripts/operate-relay-asia-admission.mjs \
+ --environment staging \
+ --mode recover-promotion \
+ --cell-ids staging-gce-c4 \
+ --expected-generation "${INITIAL_SELECTOR_GENERATION}" \
+ --attempt-id "${PROMOTE_ATTEMPT_ID}" \
+ --image-digest "${IMAGE_DIGEST}")"
+ if test "$(jq -r '.promoted' <<< "${promoted}")" = false; then exit 0; fi
+ promoted_generation="$(jq -er '.generation' <<< "${promoted}")"
+ result="$(node dev/scripts/operate-relay-asia-admission.mjs \
+ --environment staging \
+ --mode rollback \
+ --cell-ids staging-gce-c4 \
+ --expected-generation "${promoted_generation}" \
+ --attempt-id "${ROLLBACK_ATTEMPT_ID}" \
+ --image-digest "${IMAGE_DIGEST}")"
+ test "$(jq -r '.states["staging-gce-c4"]' <<< "${result}")" = migration-only
diff --git a/.github/workflows/cloud-prove-relay-staging-capacity.yml b/.github/workflows/cloud-prove-relay-staging-capacity.yml
new file mode 100644
index 00000000000..52a7538d40b
--- /dev/null
+++ b/.github/workflows/cloud-prove-relay-staging-capacity.yml
@@ -0,0 +1,917 @@
+name: Prove Relay Staging Capacity
+
+on:
+ workflow_dispatch:
+ inputs:
+ mode:
+ description: Verify or change C3 capacity, restore admission, or refresh empty Asia C4
+ required: true
+ default: verify
+ type: choice
+ options:
+ - verify
+ - apply
+ - restore-admission
+ - refresh-asia-c4-image
+ expected-hard-cap:
+ description: Exact cap declared for staging-gce-c3 in the reviewed staging tfvars
+ required: true
+ default: '600'
+ type: choice
+ options:
+ - '1000'
+ - '600'
+ expected-unobserved-bound:
+ description: Exact bound declared for staging-gce-c3 in the reviewed staging tfvars
+ required: true
+ default: '60'
+ type: choice
+ options:
+ - '60'
+ - '0'
+ confirmation:
+ description: Exact confirmation required for a mutation
+ required: false
+ type: string
+ expected-selector-generation:
+ description: Exact staging selector generation for a C4 image refresh
+ required: false
+ type: string
+ predecessor-image-digest:
+ description: Exact current C4 sha256 digest
+ required: false
+ type: string
+ target-image-digest:
+ description: Exact desired C4 sha256 digest
+ required: false
+ type: string
+
+permissions:
+ contents: read
+ id-token: write
+
+concurrency:
+ group: relay-staging-mutation
+ cancel-in-progress: false
+
+defaults:
+ run:
+ working-directory: cloud
+
+jobs:
+ capacity:
+ if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && inputs.mode != 'refresh-asia-c4-image' }}
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ environment: staging
+ env:
+ GCP_PROJECT_ID: onorca-cloud-staging
+ GCP_REGION: ${{ vars.STAGING_GCP_REGION }}
+ DIRECTOR_SERVICE_NAME: orca-cloud-relay-staging
+ DIRECTOR_ORIGIN: https://relay-staging.onorca.dev
+ CELL_ORIGIN: https://c3.relay-staging.onorca.dev
+ TARGET_CELL_ID: staging-gce-c3
+ FALLBACK_CELL_ORIGIN: https://c2.relay-staging.onorca.dev
+ FALLBACK_CELL_ID: staging-gce-c2
+ CAPACITY_SERVICE_ACCOUNT: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
+ EXPECTED_HARD_CAP: ${{ inputs.expected-hard-cap }}
+ EXPECTED_UNOBSERVED_BOUND: ${{ inputs.expected-unobserved-bound }}
+ steps:
+ - uses: actions/checkout@v4
+
+ - id: google-auth
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - uses: google-github-actions/setup-gcloud@v2
+
+ - uses: ./.github/actions/cloud-sql-rollout-lease
+ with:
+ bucket: onorca-cloud-staging-terraform-state
+ object: terraform/state/cloud-sql-rollout/staging.lock
+
+ - uses: hashicorp/setup-terraform@v3
+ if: ${{ inputs.mode != 'restore-admission' }}
+ with:
+ terraform_wrapper: false
+
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 24
+
+ - name: Initialize the exact staging backend
+ if: ${{ inputs.mode != 'restore-admission' }}
+ run: node dev/scripts/infra.mjs init --env staging
+
+ - name: Require reviewed desired capacity and image
+ if: ${{ inputs.mode != 'restore-admission' }}
+ shell: bash
+ run: |
+ CAP_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].connection_hard_cap"
+ BOUND_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].connection_unobserved_bound"
+ IMAGE_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].image"
+ ZONE_EXPRESSION="var.relay_gce_cells[\"${TARGET_CELL_ID}\"].zone"
+ DESIRED_CAP="$(terraform -chdir=infra/terraform console \
+ -var-file=environments/staging.tfvars <<< "${CAP_EXPRESSION}")"
+ DESIRED_BOUND="$(terraform -chdir=infra/terraform console \
+ -var-file=environments/staging.tfvars <<< "${BOUND_EXPRESSION}")"
+ DESIRED_IMAGE="$(terraform -chdir=infra/terraform console \
+ -var-file=environments/staging.tfvars <<< "${IMAGE_EXPRESSION}" | jq -r '.')"
+ TARGET_ZONE="$(terraform -chdir=infra/terraform console \
+ -var-file=environments/staging.tfvars <<< "${ZONE_EXPRESSION}" | jq -r '.')"
+ MIG_NAME="$(terraform -chdir=infra/terraform output -json relay_gce_cell_deployments \
+ | jq -r --arg cell "${TARGET_CELL_ID}" '.[$cell].mig_name')"
+ DESIRED_CELLS_JSON="$(terraform -chdir=infra/terraform console \
+ -var-file=environments/staging.tfvars \
+ <<< 'local.relay_director_cells_json' | jq -r '.')"
+ CELL_ORIGIN="$(jq -r --arg cell "${TARGET_CELL_ID}" \
+ '.[] | select(.id == $cell) | .url' <<< "${DESIRED_CELLS_JSON}")"
+ test "${DESIRED_CAP}" = "${EXPECTED_HARD_CAP}"
+ test "${DESIRED_BOUND}" = "${EXPECTED_UNOBSERVED_BOUND}"
+ [[ "${DESIRED_IMAGE}" =~ @sha256:[0-9a-f]{64}$ ]]
+ [[ "${TARGET_ZONE}" =~ ^[a-z0-9-]+$ ]]
+ [[ "${MIG_NAME}" =~ ^[a-z0-9-]+$ ]]
+ test "${CELL_ORIGIN}" = "https://c3.relay-staging.onorca.dev"
+ jq -e \
+ --arg cell "${TARGET_CELL_ID}" \
+ --arg fallback "${FALLBACK_CELL_ID}" \
+ --argjson cap "${EXPECTED_HARD_CAP}" \
+ --argjson bound "${EXPECTED_UNOBSERVED_BOUND}" \
+ '(any(.[]; .id == $cell and .connectionHardCap == $cap and .connectionUnobservedBound == $bound)) and
+ (any(.[]; .id == $fallback and .connectionHardCap == 600 and .connectionUnobservedBound == 60))' \
+ <<< "${DESIRED_CELLS_JSON}" >/dev/null
+ echo "DESIRED_IMAGE=${DESIRED_IMAGE}" >> "${GITHUB_ENV}"
+ echo "TARGET_ZONE=${TARGET_ZONE}" >> "${GITHUB_ENV}"
+ echo "MIG_NAME=${MIG_NAME}" >> "${GITHUB_ENV}"
+ echo "CELL_ORIGIN=${CELL_ORIGIN}" >> "${GITHUB_ENV}"
+ echo "DESIRED_CELLS_JSON=${DESIRED_CELLS_JSON}" >> "${GITHUB_ENV}"
+
+ - name: Verify exact compatible director image
+ if: ${{ inputs.mode != 'restore-admission' }}
+ shell: bash
+ run: |
+ SERVICE_JSON="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" \
+ --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json)"
+ ACTIVE_REVISION="$(jq -r \
+ '[.status.traffic[] | select((.percent // 0) > 0)]
+ | if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end' \
+ <<< "${SERVICE_JSON}")"
+ test -n "${ACTIVE_REVISION}"
+ ACTIVE_IMAGE="$(gcloud run revisions describe "${ACTIVE_REVISION}" \
+ --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" \
+ --format='value(spec.containers[0].image)')"
+ test "${ACTIVE_IMAGE}" = "${DESIRED_IMAGE}"
+ echo "ACTIVE_IMAGE=${ACTIVE_IMAGE}" >> "${GITHUB_ENV}"
+
+ - name: Require explicit transition confirmation
+ if: ${{ inputs.mode == 'apply' }}
+ env:
+ CONFIRMATION: ${{ inputs.confirmation }}
+ run: test "${CONFIRMATION}" = "FENCE_AND_TRANSITION_STAGING_C3"
+
+ - name: Require explicit admission restore confirmation
+ if: ${{ inputs.mode == 'restore-admission' }}
+ env:
+ CONFIRMATION: ${{ inputs.confirmation }}
+ run: test "${CONFIRMATION}" = "RESTORE_STAGING_C2_C3_GENERAL"
+
+ - name: Require capacity identity and exact predecessor state
+ if: ${{ inputs.mode == 'apply' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ case "${EXPECTED_HARD_CAP}/${EXPECTED_UNOBSERVED_BOUND}" in
+ 1000/0)
+ PREDECESSOR_C3_CAP=600
+ PREDECESSOR_C3_BOUND=60
+ ;;
+ 1000/60)
+ PREDECESSOR_C3_CAP=1000
+ PREDECESSOR_C3_BOUND=0
+ ;;
+ 600/60)
+ PREDECESSOR_C3_CAP=1000
+ PREDECESSOR_C3_BOUND=60
+ ;;
+ *)
+ echo "Unsupported staging capacity transition" >&2
+ exit 1
+ ;;
+ esac
+ ACTIVE_REVISION="$(gcloud run services describe "${DIRECTOR_SERVICE_NAME}" \
+ --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
+ | jq -r '
+ [.status.traffic[] | select((.percent // 0) > 0)] |
+ if length == 1 and .[0].percent == 100 then .[0].revisionName else empty end')"
+ test -n "${ACTIVE_REVISION}"
+ CURRENT_CELLS_JSON="$(gcloud run revisions describe "${ACTIVE_REVISION}" \
+ --project "${GCP_PROJECT_ID}" --region "${GCP_REGION}" --format=json \
+ | jq -cer '
+ [.spec.containers[0].env[]? |
+ select(.name == "ORCA_RELAY_CELLS_JSON") | .value] |
+ if length == 1 then .[0] | fromjson else error("missing director topology") end')"
+ PREDECESSOR_CELLS_JSON="$(jq -ce \
+ --arg cell "${TARGET_CELL_ID}" \
+ --argjson cap "${PREDECESSOR_C3_CAP}" \
+ --argjson bound "${PREDECESSOR_C3_BOUND}" \
+ 'map(if .id == $cell then . + {
+ connectionHardCap: $cap,
+ connectionUnobservedBound: $bound
+ } else . end)' <<< "${DESIRED_CELLS_JSON}")"
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${FALLBACK_CELL_ORIGIN}" \
+ --cell-id "${FALLBACK_CELL_ID}" \
+ --hard-cap 600 \
+ --unobserved-bound 60 \
+ --heartbeat fresh \
+ --admission either \
+ --draining forbidden \
+ --activity allowed
+ if jq -ne \
+ --argjson current "${CURRENT_CELLS_JSON}" \
+ --argjson expected "${DESIRED_CELLS_JSON}" \
+ '$current == $expected'; then
+ if node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --hard-cap "${EXPECTED_HARD_CAP}" \
+ --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
+ --heartbeat fresh \
+ --admission general \
+ --draining forbidden \
+ --activity allowed; then
+ TRANSITION_PHASE=cell-active
+ elif node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --hard-cap "${EXPECTED_HARD_CAP}" \
+ --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
+ --heartbeat fresh \
+ --admission migration-only \
+ --draining forbidden \
+ --activity allowed; then
+ TRANSITION_PHASE=cell-ready
+ else
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --heartbeat either \
+ --admission migration-only \
+ --draining required \
+ --activity restart-safe
+ TRANSITION_PHASE=director-ready
+ fi
+ else
+ jq -ne \
+ --argjson current "${CURRENT_CELLS_JSON}" \
+ --argjson expected "${PREDECESSOR_CELLS_JSON}" \
+ '$current == $expected'
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --hard-cap "${PREDECESSOR_C3_CAP}" \
+ --unobserved-bound "${PREDECESSOR_C3_BOUND}" \
+ --heartbeat fresh \
+ --admission either \
+ --draining either \
+ --activity allowed
+ TRANSITION_PHASE=predecessor
+ fi
+ echo "TRANSITION_PHASE=${TRANSITION_PHASE}" >> "${GITHUB_ENV}"
+
+ - name: Restore C2 as the safe placement fallback
+ if: ${{ inputs.mode == 'restore-admission' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ run: |
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${FALLBACK_CELL_ORIGIN}" \
+ --cell-id "${FALLBACK_CELL_ID}" \
+ --heartbeat fresh \
+ --admission either \
+ --draining forbidden \
+ --activity allowed
+ node dev/scripts/prepare-relay-capacity-canary.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --mode restore-fallback \
+ --general-cell-ids "${FALLBACK_CELL_ID}"
+
+ - name: Require a healthy non-draining C3 before restoring it
+ if: ${{ inputs.mode == 'restore-admission' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ run: |
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --hard-cap "${EXPECTED_HARD_CAP}" \
+ --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
+ --heartbeat fresh \
+ --admission either \
+ --draining forbidden \
+ --activity allowed
+
+ - name: Require a healthy general C2 before isolating C3
+ if: ${{ inputs.mode == 'apply' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ run: |
+ if test "${TRANSITION_PHASE}" = cell-active; then
+ node dev/scripts/prepare-relay-capacity-canary.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --mode restore-fallback \
+ --general-cell-ids "${FALLBACK_CELL_ID}"
+ fi
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${FALLBACK_CELL_ORIGIN}" \
+ --cell-id "${FALLBACK_CELL_ID}" \
+ --hard-cap 600 \
+ --unobserved-bound 60 \
+ --heartbeat fresh \
+ --admission general \
+ --draining forbidden \
+ --activity allowed
+
+ - name: Reversibly isolate and drain C3
+ if: ${{ inputs.mode == 'apply' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ run: |
+ if test "${TRANSITION_PHASE}" = cell-ready; then
+ exit 0
+ fi
+ node dev/scripts/prepare-relay-capacity-canary.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --mode isolate
+
+ - name: Verify restart-safe migration-only target
+ if: ${{ inputs.mode == 'apply' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ run: |
+ if test "${TRANSITION_PHASE}" = cell-ready; then
+ exit 0
+ fi
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --heartbeat either \
+ --admission migration-only \
+ --draining required \
+ --activity restart-safe
+
+ - name: Verify current capacity
+ if: ${{ inputs.mode == 'verify' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ run: |
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --hard-cap "${EXPECTED_HARD_CAP}" \
+ --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
+ --heartbeat fresh \
+ --admission general \
+ --draining forbidden \
+ --activity allowed
+
+ - name: Deploy reviewed director topology and remove pre-protocol revisions
+ if: ${{ inputs.mode == 'apply' }}
+ run: |
+ if test "${TRANSITION_PHASE}" != predecessor; then
+ echo "DIRECTOR_CONFIG_CHANGED=false" >> "${GITHUB_ENV}"
+ exit 0
+ fi
+ RELEASE_ID="capacity-compat-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA:0:8}"
+ DEPLOY_RESULT="$(node dev/scripts/deploy-relay-blue-green.mjs \
+ --project "${GCP_PROJECT_ID}" \
+ --region "${GCP_REGION}" \
+ --service "${DIRECTOR_SERVICE_NAME}" \
+ --image "${ACTIVE_IMAGE}" \
+ --role director \
+ --capacity-service-account "${CAPACITY_SERVICE_ACCOUNT}" \
+ --capacity-cell-id "${TARGET_CELL_ID}" \
+ --director-cells-json "${DESIRED_CELLS_JSON}" \
+ --min-instances 0 \
+ --prune-revisions true \
+ --release-id "${RELEASE_ID}")"
+ echo "${DEPLOY_RESULT}"
+ DIRECTOR_CONFIG_CHANGED="$(jq -r '.topologyChanged' <<< "${DEPLOY_RESULT}")"
+ [[ "${DIRECTOR_CONFIG_CHANGED}" =~ ^(true|false)$ ]]
+ echo "DIRECTOR_CONFIG_CHANGED=${DIRECTOR_CONFIG_CHANGED}" >> "${GITHUB_ENV}"
+
+ - name: Require fail-closed director transition
+ if: ${{ inputs.mode == 'apply' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ run: |
+ if test "${TRANSITION_PHASE}" = cell-ready; then
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --hard-cap "${EXPECTED_HARD_CAP}" \
+ --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
+ --heartbeat fresh \
+ --admission migration-only \
+ --draining forbidden \
+ --activity allowed
+ exit 0
+ fi
+ HEARTBEAT_EXPECTATION=either
+ if test "${DIRECTOR_CONFIG_CHANGED}" = true; then
+ HEARTBEAT_EXPECTATION=stale
+ fi
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --hard-cap "${EXPECTED_HARD_CAP}" \
+ --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
+ --heartbeat "${HEARTBEAT_EXPECTATION}" \
+ --admission migration-only \
+ --draining required \
+ --activity restart-safe
+
+ - name: Plan and apply only the exact empty cell
+ if: ${{ inputs.mode == 'apply' }}
+ shell: bash
+ run: |
+ recreate_fixed_one_instance() {
+ local instance
+ instance="$(gcloud compute instance-groups managed list-instances \
+ "${MIG_NAME}" \
+ --project "${GCP_PROJECT_ID}" \
+ --zone "${TARGET_ZONE}" \
+ --format=json \
+ | jq -er '
+ if length == 1 and .[0].instanceStatus == "RUNNING" and
+ .[0].currentAction == "NONE"
+ then .[0].instance | split("/") | last
+ else error("capacity cell does not have one stable running instance") end')"
+ gcloud compute instance-groups managed recreate-instances \
+ "${MIG_NAME}" \
+ --instances "${instance}" \
+ --project "${GCP_PROJECT_ID}" \
+ --zone "${TARGET_ZONE}" \
+ --quiet
+ }
+
+ terraform -chdir=infra/terraform plan \
+ -var-file=environments/staging.tfvars \
+ '-target=google_compute_instance_template.relay_gce_cell["staging-gce-c3"]' \
+ '-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c3"]' \
+ -out="${RUNNER_TEMP}/relay-capacity-cell.tfplan"
+ PLAN_RESULT="$(terraform -chdir=infra/terraform show -json \
+ "${RUNNER_TEMP}/relay-capacity-cell.tfplan" \
+ | node dev/scripts/validate-relay-capacity-plan.mjs \
+ --mode cell \
+ --cell-id "${TARGET_CELL_ID}" \
+ --hard-cap "${EXPECTED_HARD_CAP}" \
+ --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
+ --image "${DESIRED_IMAGE}")"
+ echo "${PLAN_RESULT}"
+ CELL_PLAN_CHANGES="$(jq -r '.changes' <<< "${PLAN_RESULT}")"
+ [[ "${CELL_PLAN_CHANGES}" =~ ^(0|2)$ ]]
+ if test "${TRANSITION_PHASE}" = cell-ready; then
+ test "${CELL_PLAN_CHANGES}" = 0
+ elif test "${TRANSITION_PHASE}" = cell-active; then
+ test "${CELL_PLAN_CHANGES}" = 0
+ recreate_fixed_one_instance
+ elif test "${CELL_PLAN_CHANGES}" = 0; then
+ recreate_fixed_one_instance
+ else
+ terraform -chdir=infra/terraform apply \
+ -auto-approve "${RUNNER_TEMP}/relay-capacity-cell.tfplan"
+ fi
+ gcloud compute instance-groups managed wait-until \
+ "${MIG_NAME}" \
+ --stable \
+ --project "${GCP_PROJECT_ID}" \
+ --zone "${TARGET_ZONE}" \
+ --timeout 900
+
+ - name: Verify exact live cap and fresh matching heartbeat
+ if: ${{ inputs.mode == 'apply' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ run: |
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --hard-cap "${EXPECTED_HARD_CAP}" \
+ --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
+ --heartbeat fresh \
+ --admission migration-only \
+ --draining forbidden \
+ --activity allowed
+
+ - name: Make C3 the only staging placement cell
+ if: ${{ inputs.mode == 'apply' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ run: |
+ node dev/scripts/prepare-relay-capacity-canary.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --mode activate
+
+ - name: Verify the sole general canary after transition
+ if: ${{ inputs.mode == 'apply' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ run: |
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --hard-cap "${EXPECTED_HARD_CAP}" \
+ --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
+ --heartbeat fresh \
+ --admission general \
+ --draining forbidden \
+ --activity allowed
+
+ - name: Restore the reviewed C2 and C3 placement set
+ if: ${{ inputs.mode == 'restore-admission' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ run: |
+ node dev/scripts/prepare-relay-capacity-canary.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --mode restore \
+ --general-cell-ids staging-gce-c2,staging-gce-c3
+
+ - name: Verify restored C3 admission and capacity
+ if: ${{ inputs.mode == 'restore-admission' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ run: |
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --hard-cap "${EXPECTED_HARD_CAP}" \
+ --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
+ --heartbeat fresh \
+ --admission general \
+ --draining forbidden \
+ --activity allowed
+
+ - name: Preserve C2 as the safe fallback after a failed transition
+ if: ${{ failure() && inputs.mode == 'apply' }}
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+ run: |
+ node dev/scripts/prepare-relay-capacity-canary.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" \
+ --mode restore-fallback \
+ --general-cell-ids "${FALLBACK_CELL_ID}"
+
+ refresh-asia-c4-image:
+ if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && github.ref == 'refs/heads/main' && inputs.mode == 'refresh-asia-c4-image' }}
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ timeout-minutes: 90
+ environment: staging
+ env:
+ GCP_PROJECT_ID: onorca-cloud-staging
+ GCP_REGION: ${{ vars.STAGING_GCP_REGION }}
+ DIRECTOR_ORIGIN: https://relay-staging.onorca.dev
+ CELL_ORIGIN: https://c4.relay-staging.onorca.dev
+ TARGET_CELL_ID: staging-gce-c4
+ EXPECTED_SELECTOR_GENERATION: ${{ inputs.expected-selector-generation }}
+ PREDECESSOR_IMAGE_DIGEST: ${{ inputs.predecessor-image-digest }}
+ TARGET_IMAGE_DIGEST: ${{ inputs.target-image-digest }}
+ APPROVED_PREDECESSOR_IMAGE_DIGEST: sha256:ce16d13ce6b633c6fbb1a2afdd6cdb8369645a329d42a8355efa7ad1e60a44f7
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Require the exact bounded C4 refresh
+ env:
+ CONFIRMATION: ${{ inputs.confirmation }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ test "${CONFIRMATION}" = REFRESH_STAGING_ASIA_C4_IMAGE
+ [[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
+ [[ "${PREDECESSOR_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
+ [[ "${TARGET_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
+ test "${PREDECESSOR_IMAGE_DIGEST}" != "${TARGET_IMAGE_DIGEST}"
+ test "${PREDECESSOR_IMAGE_DIGEST}" = "${APPROVED_PREDECESSOR_IMAGE_DIGEST}"
+
+ - id: google-auth
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - uses: google-github-actions/setup-gcloud@v2
+
+ - uses: ./.github/actions/cloud-sql-rollout-lease
+ with:
+ bucket: onorca-cloud-staging-terraform-state
+ object: terraform/state/cloud-sql-rollout/staging.lock
+
+ - uses: hashicorp/setup-terraform@v3
+ with:
+ terraform_version: 1.15.8
+ terraform_wrapper: false
+
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 24
+
+ - name: Initialize the exact staging backend
+ run: node dev/scripts/infra.mjs init --env staging
+
+ - name: Resolve the reviewed C4 image and shape
+ shell: bash
+ run: |
+ set -euo pipefail
+ cells="$(terraform -chdir=infra/terraform console \
+ -var-file=environments/staging.tfvars \
+ <<< 'jsonencode(var.relay_gce_cells)' | jq -er '.')"
+ shape="$(jq -cer --arg cell "${TARGET_CELL_ID}" '.[$cell]' <<< "${cells}")"
+ test "$(jq -r '.hostname' <<< "${shape}")" = c4
+ test "$(jq -r '.region' <<< "${shape}")" = asia-east2
+ test "$(jq -r '.zone' <<< "${shape}")" = asia-east2-a
+ test "$(jq -r '.machine_type' <<< "${shape}")" = e2-standard-4
+ test "$(jq -r '.capacity_requests' <<< "${shape}")" = 6000
+ test "$(jq -r '.database_pool_max' <<< "${shape}")" = 10
+ test "$(jq -r '.connection_hard_cap' <<< "${shape}")" = 3000
+ test "$(jq -r '.connection_unobserved_bound' <<< "${shape}")" = 60
+ test "$(jq -r '.initially_enabled' <<< "${shape}")" = false
+ desired_image="$(jq -r '.image' <<< "${shape}")"
+ test "${desired_image}" = \
+ "us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay@${TARGET_IMAGE_DIGEST}"
+ served_digest="$(gcloud artifacts docker images describe "${desired_image}" \
+ --project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')"
+ test "${served_digest}" = "${TARGET_IMAGE_DIGEST}"
+ mig_name="$(terraform -chdir=infra/terraform output -json relay_gce_cell_deployments \
+ | jq -r --arg cell "${TARGET_CELL_ID}" '.[$cell].mig_name')"
+ test "${mig_name}" = orca-cloud-staging-relay-gce-c4
+ {
+ echo "DESIRED_IMAGE=${desired_image}"
+ echo "ROLLBACK_IMAGE=us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay@${PREDECESSOR_IMAGE_DIGEST}"
+ echo "TARGET_ZONE=asia-east2-a"
+ echo "MIG_NAME=${mig_name}"
+ } >> "${GITHUB_ENV}"
+
+ - name: Save, validate, and classify the exact C4 plan
+ id: plan
+ shell: bash
+ run: |
+ set -euo pipefail
+ plan="${RUNNER_TEMP}/relay-c4-image-refresh.tfplan"
+ terraform -chdir=infra/terraform plan \
+ -var-file=environments/staging.tfvars \
+ '-target=google_compute_instance_template.relay_gce_cell["staging-gce-c4"]' \
+ '-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]' \
+ -out="${plan}"
+ result="$(terraform -chdir=infra/terraform show -json "${plan}" \
+ | node dev/scripts/validate-relay-capacity-plan.mjs \
+ --mode same-cap-image --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 \
+ --unobserved-bound 60 --image "${DESIRED_IMAGE}" \
+ --rollback-image "${ROLLBACK_IMAGE}")"
+ case "$(jq -r '[.changes,.changeKind] | join(":")' <<< "${result}")" in
+ 2:replacement) refresh_phase=predecessor ;;
+ 0:none|*:obsolete-template-delete) refresh_phase=applied ;;
+ *:manager-convergence|*:replacement-with-obsolete-template) refresh_phase=converging ;;
+ *) exit 1 ;;
+ esac
+ {
+ echo "PLAN_CHANGES=$(jq -r '.changes' <<< "${result}")"
+ echo "REFRESH_PHASE=${refresh_phase}"
+ } >> "${GITHUB_ENV}"
+
+ - id: state-auth
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - name: Verify the exact selector and current C4 state
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.state-auth.outputs.id_token }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ inspect="$(node dev/scripts/operate-relay-asia-admission.mjs \
+ --environment staging --mode inspect --cell-ids "${TARGET_CELL_ID}" \
+ --expected-generation '' --expected-membership-sha256 '' --attempt-id '' \
+ --image-digest "${TARGET_IMAGE_DIGEST}")"
+ test "$(jq -r '.generation' <<< "${inspect}")" = "${EXPECTED_SELECTOR_GENERATION}"
+ test "$(jq -r --arg cell "${TARGET_CELL_ID}" '.states[$cell]' <<< "${inspect}")" = \
+ migration-only
+ expected_digests="${TARGET_IMAGE_DIGEST}"
+ if test "${REFRESH_PHASE}" = predecessor; then
+ expected_digests="${PREDECESSOR_IMAGE_DIGEST}"
+ elif test "${REFRESH_PHASE}" = converging; then
+ expected_digests="${PREDECESSOR_IMAGE_DIGEST},${TARGET_IMAGE_DIGEST}"
+ fi
+ if runtime="$(curl --silent --show-error --fail-with-body --max-time 30 --request POST \
+ "${CELL_ORIGIN}/v1/admin/runtime-status" \
+ --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
+ --header 'Content-Type: application/json' --data '{"v":1}')"; then
+ current_digest="$(jq -er '.imageDigest' <<< "${runtime}")"
+ case ",${expected_digests}," in
+ *,"${current_digest}",*) ;;
+ *) exit 1 ;;
+ esac
+ source_incarnation=''
+ draining=forbidden
+ if test "${REFRESH_PHASE}" != applied; then
+ status="$(curl --fail-with-body --max-time 30 --request POST \
+ "${DIRECTOR_ORIGIN}/v1/admin/cell-status" \
+ --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
+ --header 'Content-Type: application/json' \
+ --data "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"
+ source_incarnation="$(jq -er '.status.runtime.cellIncarnation' <<< "${status}")"
+ [[ "${source_incarnation}" =~ ^[0-9a-f-]{36}$ ]]
+ if test "$(jq -r '.draining' <<< "${runtime}")" = true; then
+ draining=required
+ fi
+ fi
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 --unobserved-bound 60 \
+ --heartbeat fresh --admission migration-only --draining "${draining}" \
+ --activity quiescent --expected-image-digests "${expected_digests}"
+ runtime_available=true
+ else
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" --runtime unavailable --heartbeat stale \
+ --admission migration-only --draining either --activity restart-safe \
+ --timeout-ms 300000
+ source_incarnation=''
+ runtime_available=false
+ fi
+ {
+ echo "MIG_STABLE_AT_MS=0"
+ echo "MUTATION_STARTED=false"
+ echo "REPLACEMENT_STARTED_AT_MS=0"
+ echo "RUNTIME_AVAILABLE=${runtime_available}"
+ echo "SOURCE_INCARNATION=${source_incarnation}"
+ } >> "${GITHUB_ENV}"
+
+ - id: fence-auth
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - name: Fence C4 and prove it stayed empty before replacement
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.fence-auth.outputs.id_token }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ if test "${REFRESH_PHASE}" = applied; then exit 0; fi
+ echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
+ if test "${RUNTIME_AVAILABLE}" = false; then exit 0; fi
+ node dev/scripts/prepare-relay-capacity-canary.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" --mode isolate
+ fence_digests="${PREDECESSOR_IMAGE_DIGEST}"
+ if test "${REFRESH_PHASE}" = converging; then
+ fence_digests="${PREDECESSOR_IMAGE_DIGEST},${TARGET_IMAGE_DIGEST}"
+ fi
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 --unobserved-bound 60 \
+ --heartbeat fresh --admission migration-only --draining required \
+ --activity quiescent --expected-image-digests "${fence_digests}"
+
+ - name: Apply the exact saved C4 plan
+ shell: bash
+ run: |
+ set -euo pipefail
+ if test "${PLAN_CHANGES}" = 0 && test "${RUNTIME_AVAILABLE}" = true; then exit 0; fi
+ if test "${REFRESH_PHASE}" = applied && test "${RUNTIME_AVAILABLE}" = false; then
+ echo "MUTATION_STARTED=true" >> "${GITHUB_ENV}"
+ fi
+ if test "${REFRESH_PHASE}" != applied; then
+ replacement_started_at_ms="$(date -u +%s%3N)"
+ echo "REPLACEMENT_STARTED_AT_MS=${replacement_started_at_ms}" >> "${GITHUB_ENV}"
+ fi
+ if test "${PLAN_CHANGES}" != 0; then
+ terraform -chdir=infra/terraform apply -auto-approve \
+ "${RUNNER_TEMP}/relay-c4-image-refresh.tfplan"
+ fi
+ gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
+ --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
+ if test "${RUNTIME_AVAILABLE}" = false && test "${REFRESH_PHASE}" = applied; then
+ instance="$(gcloud compute instance-groups managed list-instances "${MIG_NAME}" \
+ --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --format=json \
+ | jq -er 'if length == 1 and .[0].instanceStatus == "RUNNING" and
+ .[0].currentAction == "NONE" then .[0].instance | split("/") | last
+ else error("C4 is not one stable running instance") end')"
+ gcloud compute instance-groups managed recreate-instances "${MIG_NAME}" \
+ --instances "${instance}" --project "${GCP_PROJECT_ID}" \
+ --zone "${TARGET_ZONE}" --quiet
+ gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
+ --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
+ fi
+ echo "MIG_STABLE_AT_MS=$(date -u +%s%3N)" >> "${GITHUB_ENV}"
+
+ - id: post-auth
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - name: Verify new C4 incarnation, image, and unchanged isolation
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 --unobserved-bound 60 \
+ --heartbeat fresh --admission migration-only --draining forbidden \
+ --activity quiescent --expected-image-digests "${TARGET_IMAGE_DIGEST}" \
+ --timeout-ms 240000
+ result="$(node dev/scripts/operate-relay-asia-admission.mjs \
+ --environment staging --mode verify --cell-ids "${TARGET_CELL_ID}" \
+ --expected-generation "${EXPECTED_SELECTOR_GENERATION}" \
+ --expected-membership-sha256 '' --attempt-id '' \
+ --image-digest "${TARGET_IMAGE_DIGEST}")"
+ test "$(jq -r '.generation' <<< "${result}")" = "${EXPECTED_SELECTOR_GENERATION}"
+ test "$(jq -r --arg cell "${TARGET_CELL_ID}" '.states[$cell]' <<< "${result}")" = \
+ migration-only
+ for _ in $(seq 1 36); do
+ status="$(curl --fail-with-body --max-time 30 --request POST \
+ "${DIRECTOR_ORIGIN}/v1/admin/cell-status" \
+ --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
+ --header 'Content-Type: application/json' \
+ --data "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"
+ if test "$(jq -r '.status.runtime.ready' <<< "${status}")" = true && \
+ test "$(jq -r '.status.runtime.lastHeartbeatAt' <<< "${status}")" \
+ -ge "${MIG_STABLE_AT_MS}"; then break; fi
+ sleep 5
+ done
+ target_incarnation="$(jq -er '.status.runtime.cellIncarnation' <<< "${status}")"
+ test "$(jq -r '.status.runtime.ready' <<< "${status}")" = true
+ test "$(jq -r '.status.runtime.lastHeartbeatAt' <<< "${status}")" \
+ -ge "${MIG_STABLE_AT_MS}"
+ if test "${REFRESH_PHASE}" != applied; then
+ if test -n "${SOURCE_INCARNATION}"; then
+ test "${target_incarnation}" != "${SOURCE_INCARNATION}"
+ fi
+ test "$(jq -r '.status.runtime.startedAt' <<< "${status}")" \
+ -ge "${REPLACEMENT_STARTED_AT_MS}"
+ fi
+
+ - name: Require an empty targeted Terraform readback
+ shell: bash
+ run: |
+ set -euo pipefail
+ plan="${RUNNER_TEMP}/relay-c4-image-readback.tfplan"
+ terraform -chdir=infra/terraform plan \
+ -var-file=environments/staging.tfvars \
+ '-target=google_compute_instance_template.relay_gce_cell["staging-gce-c4"]' \
+ '-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]' \
+ -out="${plan}"
+ result="$(terraform -chdir=infra/terraform show -json "${plan}" \
+ | node dev/scripts/validate-relay-capacity-plan.mjs \
+ --mode same-cap-image --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 \
+ --unobserved-bound 60 --image "${DESIRED_IMAGE}" \
+ --rollback-image "${ROLLBACK_IMAGE}")"
+ test "$(jq -r '.changes' <<< "${result}")" = 0
diff --git a/.github/workflows/cloud-publish-relay-production.yml b/.github/workflows/cloud-publish-relay-production.yml
new file mode 100644
index 00000000000..d7bb42b4c55
--- /dev/null
+++ b/.github/workflows/cloud-publish-relay-production.yml
@@ -0,0 +1,131 @@
+name: Publish Relay Production Image
+
+on:
+ workflow_dispatch:
+ inputs:
+ mode:
+ description: Publish a new production image or mirror an existing immutable image to staging
+ required: true
+ default: publish
+ type: choice
+ options: [publish, mirror-staging]
+ image-digest:
+ description: Exact existing production digest for mirror-staging mode
+ required: false
+ type: string
+ confirmation:
+ description: Enter MIRROR_RELAY_PRODUCTION_IMAGE_TO_STAGING for mirror-staging mode
+ required: false
+ type: string
+
+permissions:
+ contents: read
+ id-token: write
+
+concurrency:
+ group: publish-relay-production
+ cancel-in-progress: false
+
+defaults:
+ run:
+ working-directory: cloud
+
+jobs:
+ publish:
+ if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ environment: production
+ env:
+ GCP_PROJECT_ID: onorca-cloud
+ GCP_REGION: ${{ vars.PRODUCTION_GCP_REGION }}
+ REPOSITORY_ID: orca-cloud
+ IMAGE_NAME: relay
+ PUBLISH_MODE: ${{ inputs.mode }}
+ MIRROR_DIGEST: ${{ inputs.image-digest }}
+ MIRROR_CONFIRMATION: ${{ inputs.confirmation }}
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Validate the exact publish request before authentication
+ shell: bash
+ run: |
+ set -euo pipefail
+ if test "${PUBLISH_MODE}" = mirror-staging; then
+ [[ "${MIRROR_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
+ test "${MIRROR_CONFIRMATION}" = MIRROR_RELAY_PRODUCTION_IMAGE_TO_STAGING
+ else
+ test "${PUBLISH_MODE}" = publish
+ test -z "${MIRROR_DIGEST}"
+ test -z "${MIRROR_CONFIRMATION}"
+ fi
+
+ - uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+
+ - uses: google-github-actions/setup-gcloud@v2
+
+ - uses: docker/setup-buildx-action@v3
+
+ - name: Configure Docker auth
+ run: gcloud auth configure-docker "${GCP_REGION}-docker.pkg.dev" --quiet
+
+ - name: Build and publish immutable image
+ if: ${{ inputs.mode == 'publish' }}
+ run: |
+ IMAGE_TAG="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}:sha-${GITHUB_SHA}"
+ docker build -f apps/relay/Dockerfile -t "${IMAGE_TAG}" .
+ docker push "${IMAGE_TAG}"
+ DIGEST="$(gcloud artifacts docker images describe "${IMAGE_TAG}" --format='value(image_summary.digest)')"
+ test -n "${DIGEST}"
+ IMAGE="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}@${DIGEST}"
+ {
+ echo '### Terraform candidate image'
+ echo
+ echo "\`${IMAGE}\`"
+ echo
+ echo 'Declare this digest on a distinct disabled candidate cell in a reviewed Terraform PR.'
+ } >> "${GITHUB_STEP_SUMMARY}"
+
+ - name: Build and publish immutable fence broker
+ if: ${{ inputs.mode == 'publish' }}
+ run: |
+ IMAGE_TAG="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/relay-fence-broker:sha-${GITHUB_SHA}"
+ docker build \
+ --build-arg "ORCA_RELAY_FENCE_IMAGE_COMMIT=${GITHUB_SHA}" \
+ -f apps/relay-fence-broker/Dockerfile \
+ -t "${IMAGE_TAG}" .
+ docker push "${IMAGE_TAG}"
+ DIGEST="$(gcloud artifacts docker images describe "${IMAGE_TAG}" --format='value(image_summary.digest)')"
+ test -n "${DIGEST}"
+ IMAGE="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/relay-fence-broker@${DIGEST}"
+ {
+ echo
+ echo '### Terraform fence broker image'
+ echo
+ echo "\`${IMAGE}\`"
+ } >> "${GITHUB_STEP_SUMMARY}"
+
+ - name: Mirror the exact production manifest to staging
+ if: ${{ inputs.mode == 'mirror-staging' }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ source_image="${GCP_REGION}-docker.pkg.dev/${GCP_PROJECT_ID}/${REPOSITORY_ID}/${IMAGE_NAME}@${MIRROR_DIGEST}"
+ target_tag="${GCP_REGION}-docker.pkg.dev/onorca-cloud-staging/${REPOSITORY_ID}/${IMAGE_NAME}:production-${MIRROR_DIGEST#sha256:}"
+ source_digest="$(gcloud artifacts docker images describe "${source_image}" \
+ --project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')"
+ test "${source_digest}" = "${MIRROR_DIGEST}"
+ docker pull "${source_image}"
+ docker tag "${source_image}" "${target_tag}"
+ docker push "${target_tag}"
+ target_digest="$(gcloud artifacts docker images describe "${target_tag}" \
+ --project onorca-cloud-staging --format='value(image_summary.digest)')"
+ test "${target_digest}" = "${MIRROR_DIGEST}"
+ {
+ echo '### Mirrored Relay image'
+ echo
+ printf 'Production and staging now resolve the same immutable digest: %s.\n' \
+ "${MIRROR_DIGEST}"
+ } >> "${GITHUB_STEP_SUMMARY}"
diff --git a/.github/workflows/cloud-recover-relay-staging-c4-image.yml b/.github/workflows/cloud-recover-relay-staging-c4-image.yml
new file mode 100644
index 00000000000..244cee453f6
--- /dev/null
+++ b/.github/workflows/cloud-recover-relay-staging-c4-image.yml
@@ -0,0 +1,411 @@
+name: Recover Relay Staging C4 Image
+
+on:
+ workflow_run:
+ workflows: [Prove Relay Staging Capacity]
+ types: [completed]
+ workflow_dispatch:
+ inputs:
+ confirmation:
+ description: Enter RECOVER_STAGING_ASIA_C4_IMAGE
+ required: true
+ type: string
+
+permissions:
+ actions: read
+ contents: read
+ id-token: write
+
+defaults:
+ run:
+ working-directory: cloud
+
+jobs:
+ gate:
+ if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.event_name == 'workflow_dispatch' || (github.event.workflow_run.head_branch == 'main' && github.event.workflow_run.conclusion != 'success')) }}
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ timeout-minutes: 5
+ outputs:
+ recover: ${{ steps.trigger.outputs.recover }}
+ steps:
+ - name: Bind recovery to the exact failed C4 job
+ working-directory: .
+ id: trigger
+ env:
+ CONFIRMATION: ${{ inputs.confirmation }}
+ GH_TOKEN: ${{ github.token }}
+ SOURCE_RUN_ID: ${{ github.event.workflow_run.id }}
+ SOURCE_RUN_EVENT: ${{ github.event.workflow_run.event }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ if test "${GITHUB_EVENT_NAME}" = workflow_dispatch; then
+ test "${CONFIRMATION}" = RECOVER_STAGING_ASIA_C4_IMAGE
+ echo "recover=true" >> "${GITHUB_OUTPUT}"
+ exit 0
+ fi
+ test "${SOURCE_RUN_EVENT}" = workflow_dispatch
+ [[ "${SOURCE_RUN_ID}" =~ ^[1-9][0-9]*$ ]]
+ jobs="$(gh api --paginate \
+ "repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}/jobs?filter=latest")"
+ count="$(jq -s '[.[].jobs[] | select(.name == "refresh-asia-c4-image" and
+ (.conclusion == "failure" or .conclusion == "cancelled" or
+ .conclusion == "timed_out"))] | length' <<< "${jobs}")"
+ if test "${count}" = 0; then
+ echo "recover=false" >> "${GITHUB_OUTPUT}"
+ exit 0
+ fi
+ test "${count}" = 1
+ echo "recover=true" >> "${GITHUB_OUTPUT}"
+
+ recover:
+ needs: gate
+ if: ${{ needs.gate.outputs.recover == 'true' }}
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ timeout-minutes: 90
+ environment: staging
+ concurrency:
+ group: relay-staging-mutation
+ cancel-in-progress: false
+ env:
+ GCP_PROJECT_ID: onorca-cloud-staging
+ DIRECTOR_ORIGIN: https://relay-staging.onorca.dev
+ CELL_ORIGIN: https://c4.relay-staging.onorca.dev
+ TARGET_CELL_ID: staging-gce-c4
+ TARGET_ZONE: asia-east2-a
+ MIG_NAME: orca-cloud-staging-relay-gce-c4
+ PREDECESSOR_IMAGE_DIGEST: sha256:ce16d13ce6b633c6fbb1a2afdd6cdb8369645a329d42a8355efa7ad1e60a44f7
+ TARGET_IMAGE_DIGEST: sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563
+ steps:
+ - uses: actions/checkout@v4
+
+ - id: auth
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
+
+ - uses: google-github-actions/setup-gcloud@v2
+
+ - uses: ./.github/actions/cloud-sql-rollout-lease
+ with:
+ bucket: onorca-cloud-staging-terraform-state
+ object: terraform/state/cloud-sql-rollout/staging.lock
+
+ - uses: hashicorp/setup-terraform@v3
+ with:
+ terraform_version: 1.15.8
+ terraform_wrapper: false
+
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 24
+
+ - name: Initialize the exact staging backend
+ run: node dev/scripts/infra.mjs init --env staging
+
+ - id: preflight-auth
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - name: Inspect the exact C4 recovery state
+ id: preflight
+ timeout-minutes: 3
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.preflight-auth.outputs.id_token }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ inspect="$(node dev/scripts/operate-relay-asia-admission.mjs \
+ --environment staging --mode inspect --cell-ids "${TARGET_CELL_ID}" \
+ --expected-generation '' --expected-membership-sha256 '' --attempt-id '' \
+ --image-digest "${PREDECESSOR_IMAGE_DIGEST}")"
+ generation="$(jq -er '.generation' <<< "${inspect}")"
+ [[ "${generation}" =~ ^(0|[1-9][0-9]*)$ ]]
+ test "$(jq -r --arg cell "${TARGET_CELL_ID}" '.states[$cell]' <<< "${inspect}")" = \
+ migration-only
+ echo "selector_generation=${generation}" >> "${GITHUB_OUTPUT}"
+ if runtime="$(curl --silent --show-error --fail-with-body --max-time 30 --request POST \
+ "${CELL_ORIGIN}/v1/admin/runtime-status" \
+ --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
+ --header 'Content-Type: application/json' --data '{"v":1}')"; then
+ current_digest="$(jq -er '.imageDigest' <<< "${runtime}")"
+ [[ "${current_digest}" =~ ^sha256:[a-f0-9]{64}$ ]]
+ status="$(curl --fail-with-body --max-time 30 --request POST \
+ "${DIRECTOR_ORIGIN}/v1/admin/cell-status" \
+ --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
+ --header 'Content-Type: application/json' \
+ --data "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"
+ jq -e '.draining | type == "boolean"' <<< "${runtime}" >/dev/null
+ {
+ echo "runtime_available=true"
+ echo "current_digest=${current_digest}"
+ echo "draining=$(jq -r '.draining' <<< "${runtime}")"
+ echo "ready=$(jq -r '.status.runtime.ready == true' <<< "${status}")"
+ } >> "${GITHUB_OUTPUT}"
+ else
+ echo "runtime_available=false" >> "${GITHUB_OUTPUT}"
+ fi
+
+ - name: Classify both exact recovery end states
+ id: recovery-plan
+ timeout-minutes: 10
+ shell: bash
+ run: |
+ set -euo pipefail
+ image_repository="us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay"
+ predecessor_image="${image_repository}@${PREDECESSOR_IMAGE_DIGEST}"
+ target_image="${image_repository}@${TARGET_IMAGE_DIGEST}"
+ served_digest="$(gcloud artifacts docker images describe "${predecessor_image}" \
+ --project "${GCP_PROJECT_ID}" --format='value(image_summary.digest)')"
+ test "${served_digest}" = "${PREDECESSOR_IMAGE_DIGEST}"
+ cells="$(terraform -chdir=infra/terraform console \
+ -var-file=environments/staging.tfvars \
+ <<< 'jsonencode(var.relay_gce_cells)' | jq -er '.')"
+ test "$(jq -r --arg cell "${TARGET_CELL_ID}" '.[$cell].image' <<< "${cells}")" = \
+ "${target_image}"
+ target_plan="${RUNNER_TEMP}/relay-c4-image-target.tfplan"
+ terraform -chdir=infra/terraform plan \
+ -var-file=environments/staging.tfvars -lock-timeout=5m \
+ '-target=google_compute_instance_template.relay_gce_cell["staging-gce-c4"]' \
+ '-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]' \
+ -out="${target_plan}"
+ target_result="$(terraform -chdir=infra/terraform show -json "${target_plan}" \
+ | node dev/scripts/validate-relay-capacity-plan.mjs \
+ --mode same-cap-image --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 \
+ --unobserved-bound 60 --image "${target_image}" \
+ --rollback-image "${predecessor_image}")"
+ target_state="$(jq -r '[.changes,.changeKind] | join(":")' <<< "${target_result}")"
+ case "${target_state}" in
+ 0:none|2:replacement|*:obsolete-template-delete|*:manager-convergence|*:replacement-with-obsolete-template) ;;
+ *) exit 1 ;;
+ esac
+ recovery_cells="$(jq -ce --arg cell "${TARGET_CELL_ID}" \
+ --arg image "${predecessor_image}" '.[$cell].image = $image' <<< "${cells}")"
+ jq -n --argjson cells "${recovery_cells}" \
+ '{relay_gce_cells:$cells}' > "${RUNNER_TEMP}/relay-c4-recovery.tfvars.json"
+ plan="${RUNNER_TEMP}/relay-c4-image-recovery.tfplan"
+ terraform -chdir=infra/terraform plan \
+ -var-file=environments/staging.tfvars \
+ -var-file="${RUNNER_TEMP}/relay-c4-recovery.tfvars.json" \
+ -lock-timeout=5m \
+ '-target=google_compute_instance_template.relay_gce_cell["staging-gce-c4"]' \
+ '-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]' \
+ -out="${plan}"
+ result="$(terraform -chdir=infra/terraform show -json "${plan}" \
+ | node dev/scripts/validate-relay-capacity-plan.mjs \
+ --mode same-cap-image --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 \
+ --unobserved-bound 60 --image "${predecessor_image}" \
+ --rollback-image "${target_image}")"
+ changes="$(jq -r '.changes' <<< "${result}")"
+ change_kind="$(jq -r '.changeKind' <<< "${result}")"
+ case "${changes}:${change_kind}" in
+ 0:none|2:replacement|*:obsolete-template-delete|*:manager-convergence|*:replacement-with-obsolete-template) ;;
+ *) exit 1 ;;
+ esac
+ mig="$(gcloud compute instance-groups managed describe "${MIG_NAME}" \
+ --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --format=json)"
+ mig_stable="$(jq -r '.status.isStable == true and .status.versionTarget.isReached == true' \
+ <<< "${mig}")"
+ instances="$(gcloud compute instance-groups managed list-instances "${MIG_NAME}" \
+ --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --format=json)"
+ instance_stable="$(jq -r 'length == 1 and .[0].instanceStatus == "RUNNING" and
+ .[0].currentAction == "NONE"' <<< "${instances}")"
+ action=rollback-predecessor
+ recovery_digest="${PREDECESSOR_IMAGE_DIGEST}"
+ if test "${{ steps.preflight.outputs.runtime_available }}" = true && \
+ test "${{ steps.preflight.outputs.ready }}" = true && \
+ test "${{ steps.preflight.outputs.draining }}" = false && \
+ test "${mig_stable}" = true && test "${instance_stable}" = true; then
+ if test "${{ steps.preflight.outputs.current_digest }}" = "${TARGET_IMAGE_DIGEST}" && \
+ test "${target_state}" = 0:none; then
+ action=verify-target
+ recovery_digest="${TARGET_IMAGE_DIGEST}"
+ elif test "${{ steps.preflight.outputs.current_digest }}" = \
+ "${PREDECESSOR_IMAGE_DIGEST}" && test "${changes}:${change_kind}" = 0:none; then
+ action=verify-predecessor
+ fi
+ fi
+ {
+ echo "changes=${changes}"
+ echo "change_kind=${change_kind}"
+ echo "action=${action}"
+ echo "recovery_digest=${recovery_digest}"
+ } >> "${GITHUB_OUTPUT}"
+
+ - id: fence-auth
+ if: ${{ steps.recovery-plan.outputs.action == 'rollback-predecessor' }}
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - name: Fence C4 before predecessor recovery
+ if: ${{ steps.recovery-plan.outputs.action == 'rollback-predecessor' }}
+ timeout-minutes: 6
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.fence-auth.outputs.id_token }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ expected_digests="${PREDECESSOR_IMAGE_DIGEST},${TARGET_IMAGE_DIGEST}"
+ current_digest="${{ steps.preflight.outputs.current_digest }}"
+ if test -n "${current_digest}" && [[ ",${expected_digests}," != *",${current_digest},"* ]]; then
+ expected_digests="${expected_digests},${current_digest}"
+ fi
+ if curl --silent --show-error --fail-with-body --max-time 30 --request POST \
+ "${CELL_ORIGIN}/v1/admin/drain" \
+ --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
+ --header 'Content-Type: application/json' --data '{"v":1,"graceMs":0}' \
+ >/dev/null; then
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 --unobserved-bound 60 \
+ --heartbeat fresh --admission migration-only --draining required \
+ --activity quiescent \
+ --expected-image-digests "${expected_digests}" \
+ --timeout-ms 240000
+ else
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" --runtime unavailable --heartbeat stale \
+ --admission migration-only --draining either --activity restart-safe \
+ --timeout-ms 240000
+ fi
+
+ - name: Apply and stabilize the saved predecessor plan
+ id: apply
+ if: ${{ steps.recovery-plan.outputs.action == 'rollback-predecessor' }}
+ timeout-minutes: 20
+ env:
+ CHANGES: ${{ steps.recovery-plan.outputs.changes }}
+ CHANGE_KIND: ${{ steps.recovery-plan.outputs.change_kind }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ plan="${RUNNER_TEMP}/relay-c4-image-recovery.tfplan"
+ if test "${CHANGES}" != 0; then
+ terraform -chdir=infra/terraform apply -auto-approve "${plan}"
+ fi
+ gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
+ --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
+ echo "stable_at_ms=$(date -u +%s%3N)" >> "${GITHUB_OUTPUT}"
+
+ - name: Restart only when the plan did not replace C4
+ id: restore
+ if: ${{ steps.recovery-plan.outputs.action == 'rollback-predecessor' }}
+ timeout-minutes: 20
+ env:
+ APPLY_STABLE_AT_MS: ${{ steps.apply.outputs.stable_at_ms }}
+ CHANGE_KIND: ${{ steps.recovery-plan.outputs.change_kind }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ if [[ "${CHANGE_KIND}" =~ ^(replacement|replacement-with-obsolete-template|manager-convergence)$ ]]; then
+ echo "stable_at_ms=${APPLY_STABLE_AT_MS}" >> "${GITHUB_OUTPUT}"
+ exit 0
+ fi
+ instance="$(gcloud compute instance-groups managed list-instances "${MIG_NAME}" \
+ --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --format=json \
+ | jq -er 'if length == 1 and .[0].instanceStatus == "RUNNING" and
+ .[0].currentAction == "NONE" then .[0].instance | split("/") | last
+ else error("C4 is not one stable running instance") end')"
+ gcloud compute instance-groups managed recreate-instances "${MIG_NAME}" \
+ --instances "${instance}" --project "${GCP_PROJECT_ID}" \
+ --zone "${TARGET_ZONE}" --quiet
+ gcloud compute instance-groups managed wait-until "${MIG_NAME}" --stable \
+ --project "${GCP_PROJECT_ID}" --zone "${TARGET_ZONE}" --timeout 900
+ echo "stable_at_ms=$(date -u +%s%3N)" >> "${GITHUB_OUTPUT}"
+
+ - name: Require an empty selected-image recovery readback
+ timeout-minutes: 8
+ env:
+ RECOVERY_DIGEST: ${{ steps.recovery-plan.outputs.recovery_digest }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ image_repository="us-central1-docker.pkg.dev/${GCP_PROJECT_ID}/orca-cloud/relay"
+ recovery_image="${image_repository}@${RECOVERY_DIGEST}"
+ other_image="${image_repository}@${TARGET_IMAGE_DIGEST}"
+ if test "${RECOVERY_DIGEST}" = "${TARGET_IMAGE_DIGEST}"; then
+ other_image="${image_repository}@${PREDECESSOR_IMAGE_DIGEST}"
+ fi
+ cells="$(terraform -chdir=infra/terraform console \
+ -var-file=environments/staging.tfvars \
+ <<< 'jsonencode(var.relay_gce_cells)' | jq -er '.')"
+ recovery_cells="$(jq -ce --arg cell "${TARGET_CELL_ID}" --arg image "${recovery_image}" \
+ '.[$cell].image = $image' <<< "${cells}")"
+ jq -n --argjson cells "${recovery_cells}" \
+ '{relay_gce_cells:$cells}' > "${RUNNER_TEMP}/relay-c4-readback.tfvars.json"
+ readback="${RUNNER_TEMP}/relay-c4-image-recovery-readback.tfplan"
+ terraform -chdir=infra/terraform plan \
+ -var-file=environments/staging.tfvars \
+ -var-file="${RUNNER_TEMP}/relay-c4-readback.tfvars.json" \
+ -lock-timeout=5m \
+ '-target=google_compute_instance_template.relay_gce_cell["staging-gce-c4"]' \
+ '-target=google_compute_instance_group_manager.relay_gce_cell["staging-gce-c4"]' \
+ -out="${readback}"
+ readback_result="$(terraform -chdir=infra/terraform show -json "${readback}" \
+ | node dev/scripts/validate-relay-capacity-plan.mjs \
+ --mode same-cap-image --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 \
+ --unobserved-bound 60 --image "${recovery_image}" \
+ --rollback-image "${other_image}")"
+ test "$(jq -r '.changes' <<< "${readback_result}")" = 0
+
+ - id: verify-auth
+ if: ${{ always() }}
+ uses: google-github-actions/auth@v2
+ with:
+ workload_identity_provider: ${{ vars.STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER }}
+ service_account: ${{ vars.STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT }}
+ token_format: id_token
+ id_token_audience: https://relay-staging.onorca.dev/v1/admin/drain
+ id_token_include_email: true
+
+ - name: Verify the recovered image and unchanged isolation
+ if: ${{ always() && steps.verify-auth.outcome == 'success' }}
+ timeout-minutes: 8
+ env:
+ ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.verify-auth.outputs.id_token }}
+ SELECTOR_GENERATION: ${{ steps.preflight.outputs.selector_generation }}
+ STABLE_AT_MS: ${{ steps.restore.outputs.stable_at_ms }}
+ RECOVERY_DIGEST: ${{ steps.recovery-plan.outputs.recovery_digest }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ node dev/scripts/verify-relay-capacity-transition.mjs \
+ --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \
+ --cell-id "${TARGET_CELL_ID}" --hard-cap 3000 --unobserved-bound 60 \
+ --heartbeat fresh --admission migration-only --draining forbidden \
+ --activity quiescent --expected-image-digests "${RECOVERY_DIGEST}" \
+ --timeout-ms 240000
+ stable_at_ms="${STABLE_AT_MS:-0}"
+ for _ in $(seq 1 36); do
+ status="$(curl --fail-with-body --max-time 30 --request POST \
+ "${DIRECTOR_ORIGIN}/v1/admin/cell-status" \
+ --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \
+ --header 'Content-Type: application/json' \
+ --data "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"
+ if test "$(jq -r '.status.runtime.ready' <<< "${status}")" = true && \
+ test "$(jq -r '.status.runtime.lastHeartbeatAt' <<< "${status}")" \
+ -ge "${stable_at_ms}"; then break; fi
+ sleep 5
+ done
+ test "$(jq -r '.status.runtime.ready' <<< "${status}")" = true
+ test "$(jq -r '.status.runtime.lastHeartbeatAt' <<< "${status}")" \
+ -ge "${stable_at_ms}"
+ result="$(node dev/scripts/operate-relay-asia-admission.mjs \
+ --environment staging --mode verify --cell-ids "${TARGET_CELL_ID}" \
+ --expected-generation "${SELECTOR_GENERATION}" \
+ --expected-membership-sha256 '' --attempt-id '' \
+ --image-digest "${RECOVERY_DIGEST}")"
+ test "$(jq -r --arg cell "${TARGET_CELL_ID}" '.states[$cell]' <<< "${result}")" = \
+ migration-only
diff --git a/.github/workflows/cloud-requeue-relay-staging-c4-recovery.yml b/.github/workflows/cloud-requeue-relay-staging-c4-recovery.yml
new file mode 100644
index 00000000000..d92086a620a
--- /dev/null
+++ b/.github/workflows/cloud-requeue-relay-staging-c4-recovery.yml
@@ -0,0 +1,60 @@
+name: Requeue Relay Staging C4 Recovery
+
+on:
+ workflow_run:
+ workflows: [Recover Relay Staging C4 Image]
+ types: [completed]
+
+permissions:
+ actions: write
+ contents: read
+
+concurrency:
+ group: relay-staging-c4-recovery-requeue
+ cancel-in-progress: false
+
+defaults:
+ run:
+ working-directory: cloud
+
+jobs:
+ requeue:
+ if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && (github.event.workflow_run.head_branch == 'main' && github.event.workflow_run.conclusion == 'cancelled') }}
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ timeout-minutes: 5
+ steps:
+ - name: Requeue only a cancelled protected recovery job
+ working-directory: .
+ env:
+ GH_TOKEN: ${{ github.token }}
+ SOURCE_RUN_ID: ${{ github.event.workflow_run.id }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ [[ "${SOURCE_RUN_ID}" =~ ^[1-9][0-9]*$ ]]
+ jobs="$(gh api --paginate \
+ "repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}/jobs?filter=latest")"
+ count="$(jq -s '[.[].jobs[] | select(.name == "recover" and
+ .conclusion == "cancelled" and .started_at == null)] | length' <<< "${jobs}")"
+ if test "${count}" = 0; then exit 0; fi
+ test "${count}" = 1
+ runs="$(gh api \
+ "repos/${GITHUB_REPOSITORY}/actions/workflows/cloud-recover-relay-staging-c4-image.yml/runs?branch=main&per_page=100")"
+ active=0
+ while IFS= read -r run_id; do
+ active_jobs="$(gh api --paginate \
+ "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/jobs?filter=latest")"
+ if jq -se '
+ ([.[].jobs[] | select(.name == "gate" and .conclusion == "success")] | length) == 1 and
+ ([.[].jobs[] | select(.name == "recover" and .status != "completed")] | length) == 1
+ ' <<< "${active_jobs}" >/dev/null; then
+ active=1
+ break
+ fi
+ done < <(jq -r --arg source "${SOURCE_RUN_ID}" \
+ '.workflow_runs[] | select((.id | tostring) != $source and
+ .status != "completed") | .id' <<< "${runs}")
+ if test "${active}" != 0; then exit 0; fi
+ gh workflow run cloud-recover-relay-staging-c4-image.yml \
+ --repo "${GITHUB_REPOSITORY}" --ref main \
+ -f confirmation=RECOVER_STAGING_ASIA_C4_IMAGE
diff --git a/.github/workflows/cloud-verify.yml b/.github/workflows/cloud-verify.yml
new file mode 100644
index 00000000000..f0cc2df2bad
--- /dev/null
+++ b/.github/workflows/cloud-verify.yml
@@ -0,0 +1,120 @@
+name: Cloud Verify
+
+on:
+ pull_request:
+ paths:
+ - cloud/**
+ - .github/workflows/cloud-*.yml
+ - .github/actions/cloud-sql-rollout-lease/**
+ push:
+ branches: [main]
+ paths:
+ - cloud/**
+ - .github/workflows/cloud-*.yml
+ - .github/actions/cloud-sql-rollout-lease/**
+
+permissions:
+ contents: read
+
+concurrency:
+ group: cloud-verify-${{ github.ref }}
+ cancel-in-progress: true
+
+defaults:
+ run:
+ working-directory: cloud
+
+jobs:
+ security:
+ name: Secret scan
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - name: Scan Cloud history with Gitleaks
+ run: >-
+ docker run --rm
+ --volume "${GITHUB_WORKSPACE}:/repo:ro"
+ zricethezav/gitleaks@sha256:cdbb7c955abce02001a9f6c9f602fb195b7fadc1e812065883f695d1eeaba854
+ git /repo --config /repo/cloud/.gitleaks.toml
+ --log-opts="--all -- cloud :(glob).github/workflows/cloud-*.yml .github/actions/cloud-sql-rollout-lease"
+
+ - name: Scan the single-commit Cloud snapshot with TruffleHog
+ run: >-
+ docker run --rm
+ --volume "${GITHUB_WORKSPACE}:/repo:ro"
+ trufflesecurity/trufflehog@sha256:5dc064868ba7933601b5cbaea6954954d524ddd5dc6222a9667acea70068bf7d
+ filesystem /repo --no-verification --fail
+ --include-paths=/repo/cloud/.trufflehog-include-paths.txt
+ --exclude-paths=/repo/cloud/.trufflehog-exclude-paths.txt
+
+ # Compiles the workspace. No Postgres service: nothing here reaches a
+ # database, and the service container costs ~13s of startup.
+ build:
+ runs-on: blacksmith-4vcpu-ubuntu-2204
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: pnpm/action-setup@v4
+ with:
+ package_json_file: cloud/package.json
+
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 24
+
+ - run: pnpm install --frozen-lockfile
+ - run: pnpm build
+ - run: pnpm typecheck
+
+ # Runs in parallel with build. `pnpm test` compiles the one workspace
+ # package it needs through the relay pretest hook, so it does not depend on
+ # `pnpm build` having run.
+ test:
+ runs-on: blacksmith-4vcpu-ubuntu-2204
+ services:
+ postgres:
+ image: postgres:16-alpine
+ env:
+ POSTGRES_DB: orca_relay_test
+ POSTGRES_PASSWORD: relay_test
+ POSTGRES_USER: relay_test
+ ports:
+ - 5432:5432
+ options: >-
+ --health-cmd "pg_isready -U relay_test -d orca_relay_test"
+ --health-interval 5s
+ --health-timeout 5s
+ --health-retries 10
+ env:
+ ORCA_RELAY_TEST_POSTGRES_URL: postgres://relay_test:relay_test@127.0.0.1:5432/orca_relay_test
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: pnpm/action-setup@v4
+ with:
+ package_json_file: cloud/package.json
+
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 24
+
+ - run: pnpm install --frozen-lockfile
+ - run: pnpm test
+
+ # Fork pull requests reach this job, so it never configures a backend, never plans, and never
+ # holds a credential. Only the relay root ships here; foundation and apps stay private.
+ terraform:
+ runs-on: blacksmith-2vcpu-ubuntu-2204
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: hashicorp/setup-terraform@v3
+ with:
+ terraform_version: 1.15.8
+
+ - run: terraform -chdir=infra/terraform fmt -check -recursive
+ - run: terraform -chdir=infra/terraform init -backend=false -input=false
+ - run: terraform -chdir=infra/terraform validate
diff --git a/.github/workflows/hourly-mac-build.yml b/.github/workflows/hourly-mac-build.yml
index ac3af92a3bc..c300b2543b8 100644
--- a/.github/workflows/hourly-mac-build.yml
+++ b/.github/workflows/hourly-mac-build.yml
@@ -26,7 +26,7 @@ name: Hourly macOS Dev Build
# HOURLY_RELEASE_APP_ID the App's numeric id
# HOURLY_RELEASE_APP_PRIVATE_KEY the App's .pem private key
#
-# Installation tokens live one hour, which is why this mints twice. Install and
+# Installation tokens live one hour, so the build job mints twice. Install and
# build need no token at all, and notarization can hold the publish step for tens
# of minutes; minting again once the build is done starts the clock at the first
# call that actually uses it rather than burning a third of it on `pnpm install`.
@@ -60,33 +60,15 @@ env:
HOURLY_RETAIN_COUNT: 72
jobs:
- build-hourly-mac:
+ # Avoid occupying the limited Mac pool when main has not moved.
+ preflight:
if: github.repository == 'stablyai/orca'
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
outputs:
- tag: ${{ steps.release.outputs.tag }}
- version: ${{ steps.hourly.outputs.version }}
+ should_build: ${{ steps.freshness.outputs.should_build }}
head_sha: ${{ steps.freshness.outputs.head_sha }}
- published: ${{ steps.publish_live.outcome == 'success' && 'true' || 'false' }}
- runs-on: blacksmith-6vcpu-macos-15
- # Why 150: it must exceed the worst case the retry budgets below can produce
- # (install 3x10 + publish 2x45 = 120, plus ~25 for checkout/build/verify), or
- # the job is killed mid-retry and no cleanup step runs at all. A typical run
- # is far shorter — this is the notary queue's tail, not its median.
- timeout-minutes: 150
- env:
- NODE_OPTIONS: --max-old-space-size=4096
steps:
- - name: Checkout
- uses: actions/checkout@v6
- with:
- ref: main
- fetch-depth: 0
- # Why: this job only reads stablyai/orca and never pushes; every write
- # goes to the hourly repo through a minted App token passed by env.
- # Not persisting the checkout credential shrinks the blast radius if a
- # build step is compromised (zizmor: artipacked).
- persist-credentials: false
-
- name: Mint hourly repo token
id: app_token
uses: actions/create-github-app-token@v2
@@ -95,18 +77,19 @@ jobs:
private-key: ${{ secrets.HOURLY_RELEASE_APP_PRIVATE_KEY }}
owner: stablyai
repositories: orca-hourly
+ permission-contents: read
- # Why: main is often idle overnight. Rebuilding an unchanged commit burns a
- # runner hour and adds a redundant tag to the retention window.
- name: Check whether main moved since the last hourly
id: freshness
shell: bash
env:
GH_TOKEN: ${{ steps.app_token.outputs.token }}
+ MAIN_REPO_TOKEN: ${{ github.token }}
FORCED: ${{ github.event_name == 'workflow_dispatch' && inputs.force }}
run: |
set -euo pipefail
- head_sha="$(git rev-parse HEAD)"
+ head_sha="$(GH_TOKEN="$MAIN_REPO_TOKEN" gh api "repos/$GITHUB_REPOSITORY/commits/main" --jq .sha)"
+ [[ "$head_sha" =~ ^[0-9a-f]{40}$ ]] || { echo "::error::Could not resolve main"; exit 1; }
echo "head_sha=$head_sha" >>"$GITHUB_OUTPUT"
if [[ "$FORCED" == "true" ]]; then
echo "should_build=true" >>"$GITHUB_OUTPUT"
@@ -133,21 +116,55 @@ jobs:
echo "main moved to $head_sha (last hourly built $last_sha); building."
fi
+ build-hourly-mac:
+ needs: preflight
+ if: needs.preflight.outputs.should_build == 'true'
+ outputs:
+ tag: ${{ steps.release.outputs.tag }}
+ version: ${{ steps.hourly.outputs.version }}
+ head_sha: ${{ needs.preflight.outputs.head_sha }}
+ published: ${{ steps.publish_live.outcome == 'success' && 'true' || 'false' }}
+ runs-on: blacksmith-6vcpu-macos-15
+ # Why 150: it must exceed the worst case the retry budgets below can produce
+ # (install 3x10 + publish 2x45 = 120, plus ~25 for checkout/build/verify), or
+ # the job is killed mid-retry and no cleanup step runs at all. A typical run
+ # is far shorter — this is the notary queue's tail, not its median.
+ timeout-minutes: 150
+ env:
+ NODE_OPTIONS: --max-old-space-size=4096
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v6
+ with:
+ ref: ${{ needs.preflight.outputs.head_sha }}
+ fetch-depth: 0
+ # Why: this job only reads stablyai/orca and never pushes; every write
+ # goes to the hourly repo through a minted App token passed by env.
+ # Not persisting the checkout credential shrinks the blast radius if a
+ # build step is compromised (zizmor: artipacked).
+ persist-credentials: false
+
+ - name: Mint hourly repo token
+ id: app_token
+ uses: actions/create-github-app-token@v2
+ with:
+ app-id: ${{ secrets.HOURLY_RELEASE_APP_ID }}
+ private-key: ${{ secrets.HOURLY_RELEASE_APP_PRIVATE_KEY }}
+ owner: stablyai
+ repositories: orca-hourly
+
- name: Setup pnpm
- if: steps.freshness.outputs.should_build == 'true'
uses: pnpm/setup@v2
with:
install: false
- name: Setup Node.js
- if: steps.freshness.outputs.should_build == 'true'
uses: actions/setup-node@v6
with:
node-version-file: package.json
cache: pnpm
- name: Cache electron-builder downloads
- if: steps.freshness.outputs.should_build == 'true'
uses: actions/cache@v5
with:
path: |
@@ -158,7 +175,6 @@ jobs:
electron-builder-mac-
- name: Install dependencies
- if: steps.freshness.outputs.should_build == 'true'
uses: nick-fields/retry@v4
with:
timeout_minutes: 10
@@ -169,7 +185,6 @@ jobs:
# Why: signing is what makes an hourly installable over an existing Orca, so
# a missing cert must fail here rather than after a 20-minute build.
- name: Verify macOS signing environment
- if: steps.freshness.outputs.should_build == 'true'
run: node config/scripts/verify-macos-release-env.mjs
env:
CSC_LINK: ${{ secrets.MAC_CERTS }}
@@ -180,7 +195,6 @@ jobs:
- name: Compute hourly version
id: hourly
- if: steps.freshness.outputs.should_build == 'true'
shell: bash
env:
GH_TOKEN: ${{ steps.app_token.outputs.token }}
@@ -211,7 +225,7 @@ jobs:
node config/scripts/hourly-build-version.mjs \
>"$RUNNER_TEMP/hourly-identity.txt"
grep -E '^(version|build_number)=' "$RUNNER_TEMP/hourly-identity.txt"
- # Why check rather than trust: the checkout above pins `ref: main`, but a
+ # Why check rather than trust: the checkout above pins the resolved main commit, but a
# workflow_dispatch runs this file from whatever branch was dispatched. A
# branch that edits this step while main still has the old script yields
# an empty name and an untitled release — silent, and only visible once
@@ -223,7 +237,6 @@ jobs:
cat "$RUNNER_TEMP/hourly-identity.txt" >>"$GITHUB_OUTPUT"
- name: Build app
- if: steps.freshness.outputs.should_build == 'true'
run: pnpm build:release
env:
NODE_OPTIONS: --max-old-space-size=4096
@@ -239,7 +252,6 @@ jobs:
# part the full budget.
- name: Re-mint hourly repo token for publish
id: app_token_publish
- if: steps.freshness.outputs.should_build == 'true'
uses: actions/create-github-app-token@v2
with:
app-id: ${{ secrets.HOURLY_RELEASE_APP_ID }}
@@ -249,13 +261,12 @@ jobs:
- name: Create hourly release
id: release
- if: steps.freshness.outputs.should_build == 'true'
shell: bash
env:
GH_TOKEN: ${{ steps.app_token_publish.outputs.token }}
TAG: v${{ steps.hourly.outputs.version }}
NAME: ${{ steps.hourly.outputs.name }}
- SHA: ${{ steps.freshness.outputs.head_sha }}
+ SHA: ${{ needs.preflight.outputs.head_sha }}
run: |
set -euo pipefail
# Kept at 12 even though the title shows 7: the freshness check above
@@ -291,7 +302,6 @@ jobs:
echo "tag=$TAG" >>"$GITHUB_OUTPUT"
- name: Publish hourly macOS artifacts
- if: steps.freshness.outputs.should_build == 'true'
uses: nick-fields/retry@v4
with:
# Why 45 like the release pipeline: an attempt is pack + notarize +
@@ -322,7 +332,6 @@ jobs:
# release missing that manifest is a tag the picker offers and the download
# 404s on, so fail loudly instead of leaving a broken entry.
- name: Verify update manifest published
- if: steps.freshness.outputs.should_build == 'true'
shell: bash
env:
GH_TOKEN: ${{ steps.app_token_publish.outputs.token }}
@@ -352,7 +361,6 @@ jobs:
# means the picker can never offer a release whose assets are incomplete.
- name: Publish the verified release
id: publish_live
- if: steps.freshness.outputs.should_build == 'true'
shell: bash
env:
GH_TOKEN: ${{ steps.app_token_publish.outputs.token }}
diff --git a/.github/workflows/mobile.yml b/.github/workflows/mobile.yml
index 59f6cf20bf4..6dbfc02aa3c 100644
--- a/.github/workflows/mobile.yml
+++ b/.github/workflows/mobile.yml
@@ -15,8 +15,13 @@ on:
# Why: this job holds the only checks that load the Fastfile, so edits to
# it or to the release workflow it guards must re-run them.
- '.github/workflows/mobile.yml'
+ - '.github/actions/install-node-dependencies/**'
- '.github/workflows/mobile-ios-release.yml'
+concurrency:
+ group: mobile-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
jobs:
verify:
runs-on: ubuntu-latest
@@ -35,10 +40,7 @@ jobs:
- name: Checkout
uses: actions/checkout@v6
- - name: Setup Node.js
- uses: actions/setup-node@v6
- with:
- node-version-file: package.json
+ - uses: ./.github/actions/install-node-dependencies
# bundler-cache installs mobile/Gemfile.lock, so this job is also what
# proves the pinned fastlane the release workflow depends on still
@@ -50,23 +52,6 @@ jobs:
bundler-cache: true
working-directory: mobile
- - name: Setup pnpm
- uses: pnpm/setup@v2
- with:
- install: false
-
- # Why: the mobile typecheck imports shared types from ../src/shared, and
- # some of those files import runtime deps (tweetnacl, ws) resolved from
- # the repo-root node_modules. Without a root install, tsc fails with
- # "Cannot find module 'tweetnacl'/'ws'". Mobile is a separate pnpm project
- # (not in the root workspace), so this is a distinct install.
- # --ignore-scripts skips the root postinstall (Electron native-module
- # rebuild) which is irrelevant to a type-only check and would only add
- # time and failure surface on this ubuntu mobile runner.
- - name: Install root dependencies
- working-directory: .
- run: pnpm install --frozen-lockfile --ignore-scripts
-
- name: Install dependencies
run: pnpm install --frozen-lockfile
diff --git a/.github/workflows/performance-contracts.yml b/.github/workflows/performance-contracts.yml
new file mode 100644
index 00000000000..d45d8b8f45a
--- /dev/null
+++ b/.github/workflows/performance-contracts.yml
@@ -0,0 +1,63 @@
+name: Performance contracts
+
+on:
+ schedule:
+ - cron: '15 9 * * *'
+ workflow_dispatch:
+ pull_request:
+ paths:
+ - '.github/workflows/performance-contracts.yml'
+ - 'config/vitest.performance.config.ts'
+ - 'config/oxlint-performance-audit.json'
+ - 'config/oxlint-plugins/*performance.mjs'
+ - 'config/oxlint-plugins/quadratic-buffer-concat.mjs'
+ - 'config/scripts/*-plugin.test.mjs'
+ # Keep in sync with the contract list in config/vitest.performance.config.ts;
+ # without these a rename lands green and only breaks the next nightly.
+ - 'src/main/sqlite/sync-database.test.ts'
+ - 'src/main/runtime/orchestration/db/row-column-lists.test.ts'
+ - 'src/relay/fs-path-metadata-symlink-concurrency.test.ts'
+ - 'src/renderer/src/components/editor/rich-markdown-list-tokenizers.test.ts'
+ - 'src/renderer/src/components/editor/rich-markdown-lowlight-cache.test.ts'
+ - 'src/renderer/src/components/terminal-pane/agent-completion-coordinator-queued-inspection-disposal.test.ts'
+ - 'src/renderer/src/lib/pane-manager/pane-terminal-output-scheduler-queue-retention.test.ts'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: performance-contracts-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ contracts:
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ubuntu-latest, macos-latest, windows-latest]
+ runs-on: ${{ matrix.os }}
+ timeout-minutes: 20
+ steps:
+ - uses: actions/checkout@v6
+ with:
+ persist-credentials: false
+ - uses: ./.github/actions/install-node-dependencies
+ - name: Run operation-count and retention contracts
+ run: pnpm test:perf:contracts --reporter=default --reporter=json --outputFile=performance-contracts.json
+ # Source-only scan: identical on every OS, so run it once.
+ - name: Audit production performance patterns
+ if: always() && matrix.os == 'ubuntu-latest'
+ shell: bash
+ run: pnpm --silent audit:perf > performance-audit.json
+ - uses: actions/upload-artifact@v7
+ if: always()
+ with:
+ name: performance-contracts-${{ matrix.os }}
+ path: performance-contracts.json
+ if-no-files-found: error
+ - uses: actions/upload-artifact@v7
+ if: always() && matrix.os == 'ubuntu-latest'
+ with:
+ name: performance-audit
+ path: performance-audit.json
+ if-no-files-found: error
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 80d3d42a8bb..93bc4c0afc8 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -28,6 +28,7 @@ jobs:
outputs:
should_run: ${{ steps.filter.outputs.should_run }}
native_cache_changed: ${{ steps.filter.outputs.native_cache_changed }}
+ mobile_dependencies: ${{ steps.filter.outputs.mobile_dependencies }}
static_analysis: ${{ steps.filter.outputs.static_analysis }}
typecheck: ${{ steps.filter.outputs.typecheck }}
git_compatibility: ${{ steps.filter.outputs.git_compatibility }}
@@ -40,6 +41,10 @@ jobs:
managed_hook_node18: ${{ steps.filter.outputs.managed_hook_node18 }}
package: ${{ steps.filter.outputs.package }}
package_windows: ${{ steps.filter.outputs.package_windows }}
+ e2e_should_run: ${{ steps.e2e_filter.outputs.should_run }}
+ test_files: ${{ steps.e2e_filter.outputs.test_files }}
+ ssh_source_changed: ${{ steps.e2e_filter.outputs.ssh_source_changed }}
+ native_ime_source_changed: ${{ steps.e2e_filter.outputs.native_ime_source_changed }}
steps:
- name: Checkout
uses: actions/checkout@v6
@@ -65,6 +70,37 @@ jobs:
printf '%s\n' "$CHANGED"
printf '%s\n' "$CHANGED" | node config/scripts/pr-code-change-scope.mjs | tee -a "$GITHUB_OUTPUT"
+ # Reuse the path-detector checkout instead of queuing another runner.
+ - name: Filter changed E2E specs
+ id: e2e_filter
+ if: github.event.pull_request.draft != true && steps.filter.outputs.should_run == 'true'
+ run: |
+ set -euo pipefail
+ BASE="${{ github.event.pull_request.base.sha }}"
+ HEAD="${{ github.event.pull_request.head.sha }}"
+ CHANGED="$(git diff --name-only --diff-filter=AMCR --merge-base "$BASE" "$HEAD")"
+ # Source routes are executable contracts so a test can prove exact
+ # authorities, exclusions, and sentinels without evaluating workflow shell.
+ TEST_FILES_JSON="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs)"
+ echo "test_files=$TEST_FILES_JSON" >> "$GITHUB_OUTPUT"
+ # Why a separate signal: the Docker-SSH lane must trigger on SSH source, not on a
+ # spec name surviving in a route's list. Same routes, so the two cannot drift.
+ SSH_SOURCE_CHANGED="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --ssh-source)"
+ echo "ssh_source_changed=$SSH_SOURCE_CHANGED" >> "$GITHUB_OUTPUT"
+ echo "SSH source changed: $SSH_SOURCE_CHANGED"
+ # Why its own signal: the real-IME lane is a whole ibus session, not a spec, so it must
+ # trigger on IME source rather than on a spec name in some route's list.
+ NATIVE_IME_SOURCE_CHANGED="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --native-ime-source)"
+ echo "native_ime_source_changed=$NATIVE_IME_SOURCE_CHANGED" >> "$GITHUB_OUTPUT"
+ echo "Native IME source changed: $NATIVE_IME_SOURCE_CHANGED"
+ if [ "$TEST_FILES_JSON" != '[]' ]; then
+ echo "should_run=true" >> "$GITHUB_OUTPUT"
+ echo "Changed E2E specs: $TEST_FILES_JSON"
+ else
+ echo "should_run=false" >> "$GITHUB_OUTPUT"
+ echo "No changed E2E specs"
+ fi
+
static_analysis:
name: static analysis
needs: [code_paths]
@@ -95,6 +131,25 @@ jobs:
- name: Enforce type-aware code-quality baseline
run: pnpm run audit:code-quality:type-aware
+ # Why: the changed-code gate lints mobile files too, and its type-aware pass
+ # resolves types from mobile/node_modules. Mobile is a separate pnpm project,
+ # so the root install above leaves it empty and every mobile type degrades to
+ # an `error` type — reported as phantom findings against the changed lines.
+ # Why no --ignore-scripts, unlike the root install: mobile's postinstall generates
+ # the gitignored terminal/mermaid webview engine modules that tracked source imports,
+ # and skipping it degrades those very types the step exists to resolve. The drift
+ # guard mirrors the root install so a stale mobile lockfile fails by name — mobile's
+ # lockfile carries patchedDependencies that a silent rewrite would drop.
+ - name: Install mobile dependencies
+ if: needs.code_paths.outputs.mobile_dependencies == 'true'
+ working-directory: mobile
+ run: |
+ pnpm install --frozen-lockfile
+ if [ "$(git -C "$GITHUB_WORKSPACE" rev-parse --is-inside-work-tree 2>/dev/null)" = true ]; then
+ git -C "$GITHUB_WORKSPACE" diff --exit-code -- \
+ mobile/package.json mobile/pnpm-lock.yaml mobile/pnpm-workspace.yaml
+ fi
+
- name: Enforce changed-code quality
run: pnpm run check:code-quality:changed -- "${{ github.event.pull_request.base.sha }}"
@@ -152,6 +207,12 @@ jobs:
- name: Verify localization catalog
run: pnpm run verify:localization-catalog
+ # Why: the renderer ships only the English entries i18next cannot rebuild
+ # from each call site's inline default, so the generated subset has to
+ # track en.json and those defaults.
+ - name: Verify runtime-required localization catalog
+ run: pnpm run verify:localization-runtime-catalog
+
# Why: extraction writes sorted evidence to an isolated temporary path,
# so feature PRs need one normalized AST pass rather than a three-OS matrix.
- name: Verify localization extraction
@@ -354,7 +415,7 @@ jobs:
- uses: ./.github/actions/install-node-dependencies
# Why: the check rebuilds every package in the manifest from a pinned upstream
- # commit — @xterm/xterm and the two addons, each built twice (once unmodified to
+ # commit — @xterm/xterm and its three addons, each built twice (once unmodified to
# prove the toolchain still reproduces the published bundles, once patched). Caching
# the npm metadata and the shallow clone keeps the repeated cost to the builds
# themselves; the key is the manifest, so a commit, package or toolchain bump
@@ -623,6 +684,8 @@ jobs:
needs: [code_paths]
if: needs.code_paths.outputs.package == 'true'
runs-on: ubuntu-latest
+ # Let the serial Docker gates reach their own deadlines and report cleanup failures.
+ timeout-minutes: 90
steps:
- name: Checkout
@@ -678,14 +741,49 @@ jobs:
- name: Build native components
run: pnpm run build:native
+ - name: Install Linux package tooling
+ run: sudo apt-get update && sudo apt-get install -y cpio rpm
+
- name: Package unpacked app
env:
ORCA_REUSE_PREPARED_NATIVE_RUNTIME: '1'
- run: pnpm exec electron-builder --config config/electron-builder.config.cjs --linux AppImage --x64 --publish never
+ # PR artifacts are only inspected locally; gzip avoids release-size xz compression.
+ run: >-
+ pnpm exec electron-builder --config config/electron-builder.config.cjs
+ --linux AppImage deb rpm --x64 --publish never
+ --config.deb.compression=gz --config.rpm.compression=gzip
+
+ - name: Verify root-package marker payloads
+ run: |
+ set -euo pipefail
+ version="$(node -p "require('./package.json').version")"
+ deb="dist/orca-ide_${version}_amd64.deb"
+ rpm="dist/orca-ide-${version}.x86_64.rpm"
+ test -s "$deb"
+ test -s "$rpm"
+ deb_marker="$(dpkg-deb --fsys-tarfile "$deb" | tar -xOf - ./opt/Orca/resources/package-type)"
+ rpm_marker="$(rpm2cpio "$rpm" | cpio --quiet --extract --to-stdout ./opt/Orca/resources/package-type)"
+ [[ "$deb_marker" == deb ]] || { echo "Expected deb marker, got: $deb_marker"; exit 1; }
+ [[ "$rpm_marker" == rpm ]] || { echo "Expected rpm marker, got: $rpm_marker"; exit 1; }
- name: Verify headless serve signal shutdown
run: node config/scripts/run-headless-serve-shutdown-docker.mjs --appimage dist/orca-linux.AppImage
+ - name: Verify extracted launcher serve signal shutdown
+ run: >-
+ node config/scripts/run-headless-serve-shutdown-docker.mjs
+ --appimage dist/orca-linux.AppImage --entrypoint launcher
+
+ - name: Verify AppImage CLI registration and serve signal shutdown
+ run: >-
+ node config/scripts/run-headless-serve-shutdown-docker.mjs
+ --appimage dist/orca-linux.AppImage --entrypoint appimage
+ --signal-target serving-electron --int-delivery pid
+
+ # A default container reproduces the hostile AppImage launch environment.
+ - name: Verify Linux CLI launch contract
+ run: node config/scripts/run-linux-cli-launch-contract-docker.mjs --appimage dist/orca-linux.AppImage
+
- name: Smoke packaged CLI
run: node config/scripts/smoke-packaged-cli.mjs --app-dir=dist/linux-unpacked
@@ -746,8 +844,10 @@ jobs:
src/main/providers/pty-repaint-wide-char-buffer.node-pty.test.ts
src/shared/child-process/windows-command-line.win32.test.ts
src/main/agent-hooks/windows-hook-payload-delivery.test.ts
+ src/main/agent-hooks/windows-direct-cmd-hook-command.test.ts
src/main/windows/windows-pty-job.win32.test.ts
src/main/windows/windows-host-job.win32.test.ts
+ src/main/windows-live-tree-kill.win32.test.ts
src/main/wsl/wsl-runner.test.ts
src/main/wsl/wsl-guest-environment.test.ts
src/main/wsl/wsl-invocation-boundary.test.ts
@@ -757,6 +857,7 @@ jobs:
src/main/cli/wsl-cli-powershell-boundary.test.ts
src/main/cursor/hook-service.test.ts
src/main/orca-profiles/profile-index-store.test.ts
+ src/main/startup/windows-install-dir-acl-repair.win32.test.ts
src/main/runtime/repo-worktree-admin-fingerprint.test.ts
src/main/runtime/worktree-scan-admin-fingerprint-gate.test.ts
src/shared/secure-file-fsync-flags.test.ts
@@ -800,72 +901,20 @@ jobs:
- name: Smoke packaged CLI
run: node config/scripts/smoke-packaged-cli.mjs --app-dir=dist/win-unpacked
- # Why: PR E2E is advisory and only validates changed specs; scheduled and
- # release runs retain full-suite coverage.
- e2e-paths:
- name: detect changed e2e specs
- needs: [code_paths]
- runs-on: ubuntu-latest
- if: github.event.pull_request.draft != true && needs.code_paths.outputs.should_run == 'true'
- # Why: detector only needs to read the checkout; do not inherit repo defaults.
- permissions:
- contents: read
- outputs:
- should_run: ${{ steps.filter.outputs.should_run }}
- test_files: ${{ steps.filter.outputs.test_files }}
- ssh_source_changed: ${{ steps.filter.outputs.ssh_source_changed }}
- native_ime_source_changed: ${{ steps.filter.outputs.native_ime_source_changed }}
- steps:
- - name: Checkout
- uses: actions/checkout@v6
- with:
- # Why blob:none: full history is needed for the merge-base diff, but historical
- # file contents are not. Blobs are ~89% of this repo's pack, and Git fetches the
- # few this job actually reads on demand.
- fetch-depth: 0
- filter: blob:none
- persist-credentials: false
-
- - name: Filter changed E2E specs
- id: filter
- run: |
- set -euo pipefail
- BASE="${{ github.event.pull_request.base.sha }}"
- HEAD="${{ github.event.pull_request.head.sha }}"
- CHANGED="$(git diff --name-only --diff-filter=AMCR --merge-base "$BASE" "$HEAD")"
- # Source routes are executable contracts so a test can prove exact
- # authorities, exclusions, and sentinels without evaluating workflow shell.
- TEST_FILES_JSON="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs)"
- echo "test_files=$TEST_FILES_JSON" >> "$GITHUB_OUTPUT"
- # Why a separate signal: the Docker-SSH lane must trigger on SSH source, not on a
- # spec name surviving in a route's list. Same routes, so the two cannot drift.
- SSH_SOURCE_CHANGED="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --ssh-source)"
- echo "ssh_source_changed=$SSH_SOURCE_CHANGED" >> "$GITHUB_OUTPUT"
- echo "SSH source changed: $SSH_SOURCE_CHANGED"
- # Why its own signal: the real-IME lane is a whole ibus session, not a spec, so it must
- # trigger on IME source rather than on a spec name in some route's list.
- NATIVE_IME_SOURCE_CHANGED="$(printf '%s\n' "$CHANGED" | node config/scripts/pr-e2e-source-routing.mjs --native-ime-source)"
- echo "native_ime_source_changed=$NATIVE_IME_SOURCE_CHANGED" >> "$GITHUB_OUTPUT"
- echo "Native IME source changed: $NATIVE_IME_SOURCE_CHANGED"
- if [ "$TEST_FILES_JSON" != '[]' ]; then
- echo "should_run=true" >> "$GITHUB_OUTPUT"
- echo "Changed E2E specs: $TEST_FILES_JSON"
- else
- echo "should_run=false" >> "$GITHUB_OUTPUT"
- echo "No changed E2E specs"
- fi
-
e2e:
name: e2e
- needs: e2e-paths
- if: needs.e2e-paths.outputs.should_run == 'true'
+ needs: code_paths
+ if: needs.code_paths.outputs.e2e_should_run == 'true'
# Why: reusable e2e.yml only checkouts, builds, and uploads artifacts.
permissions:
contents: read
uses: ./.github/workflows/e2e.yml
with:
- test_files: ${{ needs.e2e-paths.outputs.test_files }}
- ssh_source_changed: ${{ needs.e2e-paths.outputs.ssh_source_changed }}
+ # The synthetic pull-request merge ref can disappear while this reusable
+ # workflow is queued. The head SHA is immutable and works for every PR.
+ ref: ${{ github.event.pull_request.head.sha }}
+ test_files: ${{ needs.code_paths.outputs.test_files }}
+ ssh_source_changed: ${{ needs.code_paths.outputs.ssh_source_changed }}
# Why this is not in verify's needs: it is the first PR-gate run of a harness whose reliability
# is only known from nightly main runs (20/20 green, 2026-08-09..2026-08-29, p50 3m25s). It
@@ -875,8 +924,8 @@ jobs:
# require `success || skipped` outside the strict loop — see the note on `e2e`.
terminal_ime_native:
name: real IME
- needs: e2e-paths
- if: needs.e2e-paths.outputs.native_ime_source_changed == 'true'
+ needs: code_paths
+ if: needs.code_paths.outputs.native_ime_source_changed == 'true'
# Why: the reusable workflow only checks out, builds, and uploads artifacts.
permissions:
contents: read
diff --git a/.github/workflows/release-cut.yml b/.github/workflows/release-cut.yml
index f25be00ba8f..eaccc25f14c 100644
--- a/.github/workflows/release-cut.yml
+++ b/.github/workflows/release-cut.yml
@@ -922,9 +922,7 @@ jobs:
run: |
$env:SKIP_BUILD = '1'
$env:ORCA_E2E_FORWARD_APP_LOGS = '1'
- pnpm run --if-present test:e2e:workspace-session-golden
pnpm run --if-present test:e2e:windows-fresh-startup-golden
- pnpm run --if-present test:e2e:source-control-golden
- name: Upload Playwright traces
if: failure()
@@ -940,6 +938,9 @@ jobs:
if: needs.cut.outputs.should_release == 'true'
name: skill sharing release gate ${{ matrix.platform }}
runs-on: ${{ matrix.os }}
+ # The full suite is release-blocking on macOS. Windows still produces the
+ # same evidence, but intermittent filesystem contention cannot block signing.
+ continue-on-error: ${{ matrix.platform == 'windows' }}
timeout-minutes: 20
strategy:
fail-fast: false
diff --git a/.github/workflows/skill-update-roundtrip.yml b/.github/workflows/skill-update-roundtrip.yml
index 71fcf264f69..239f1b2f27c 100644
--- a/.github/workflows/skill-update-roundtrip.yml
+++ b/.github/workflows/skill-update-roundtrip.yml
@@ -22,6 +22,10 @@ on:
- main
paths: *skill-roundtrip-paths
+concurrency:
+ group: skill-roundtrip-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
jobs:
roundtrip:
strategy:
diff --git a/.gitignore b/.gitignore
index e3fd07e45d1..8be3fc5b6f4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -110,6 +110,7 @@ docs/**
!docs/reference/macos-press-and-hold.md
!docs/reference/orcad-operations.md
!docs/reference/relay-grace-time-reconfiguration.md
+!docs/reference/windows-edr-posture.md
!docs/reference/windows-process-enumeration.md
!docs/reference/wsl-runner-verification.md
!docs/reference/remote-wire-compatibility.md
@@ -157,6 +158,7 @@ src/renderer/src/i18n/locales/.zh-catalog-cache.json
src/renderer/src/i18n/locales/.ko-catalog-cache.json
src/renderer/src/i18n/locales/.ja-catalog-cache.json
src/renderer/src/i18n/locales/.es-catalog-cache.json
+src/renderer/src/i18n/locales/.fr-catalog-cache.json
# Bench result JSONs are working artifacts
tests/tools/benchmarks/results/terminal-pipeline-*.json
diff --git a/.oxfmtrc.json b/.oxfmtrc.json
index b2aa0deabfa..0f27189d7cb 100644
--- a/.oxfmtrc.json
+++ b/.oxfmtrc.json
@@ -3,5 +3,6 @@
"singleQuote": true,
"semi": false,
"printWidth": 100,
- "trailingComma": "none"
+ "trailingComma": "none",
+ "ignorePatterns": ["cloud/**", ".github/actions/cloud-sql-rollout-lease/**"]
}
diff --git a/.oxlintrc.json b/.oxlintrc.json
index cae0d09dd58..03cc659f494 100644
--- a/.oxlintrc.json
+++ b/.oxlintrc.json
@@ -2,6 +2,10 @@
"$schema": "./node_modules/oxlint/configuration_schema.json",
"plugins": ["typescript", "react", "react-hooks", "react-perf", "unicorn"],
"jsPlugins": [
+ {
+ "name": "sort-comparator-performance",
+ "specifier": "./config/oxlint-plugins/sort-comparator-performance.mjs"
+ },
{
"name": "mobile-pairing",
"specifier": "./config/oxlint-plugins/mobile-pairing-qrcode-import.mjs"
@@ -23,6 +27,7 @@
"correctness": "error"
},
"rules": {
+ "sort-comparator-performance/no-repeated-collator": "warn",
"app-store-performance/require-selector": "error",
"app-store-performance/no-identity-selector": "error",
"app-store-performance/no-fresh-selector-result": "error",
@@ -174,5 +179,12 @@
}
}
],
- "ignorePatterns": ["**/node_modules", "**/dist", "**/out", "tests/e2e/.cross-version-checkouts"]
+ "ignorePatterns": [
+ "**/node_modules",
+ "**/dist",
+ "**/out",
+ "cloud/**",
+ ".github/actions/cloud-sql-rollout-lease/**",
+ "tests/e2e/.cross-version-checkouts"
+ ]
}
diff --git a/AGENTS.md b/AGENTS.md
index 9817cc41cc8..8b0156ba6b1 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -49,6 +49,7 @@ Orca targets macOS, Linux, and Windows. Keep all platform-dependent behavior beh
- **Windows setup scripts**: the setup/issue-command runner is a `.cmd` batch file unless the script starts with a `#!` line — never derive that from the user's terminal-shell preference, and never launch a `.cmd` runner with a bare `cmd.exe /c` from a Git Bash pane (MSYS rewrites the `/c`). See [`docs/reference/windows-setup-shell.md`](./docs/reference/windows-setup-shell.md).
- **Windows child processes**: start them through `runProcess`/`spawnProcess` in `src/shared/child-process/` — never `child_process` directly. It pins `windowsHide`, refuses `shell: true`, and encodes `.cmd`/`.bat` arguments so neither `CommandLineToArgvW` nor `cmd.exe` mangles them. A ratchet test fails on any new direct import.
- **Windows process enumeration**: read the table through `src/main/windows/windows-process-table.ts`, never by forking `powershell.exe`. See [`docs/reference/windows-process-enumeration.md`](./docs/reference/windows-process-enumeration.md).
+- **Windows EDR signal**: don't add `-ExecutionPolicy Bypass`, `-EncodedCommand`, `cmd.exe /c` with escaped free text, per-operation interpreter spawning, or runtime `Add-Type` compilation without reading [`docs/reference/windows-edr-posture.md`](./docs/reference/windows-edr-posture.md) first — behavioural EDR scores each of those, and being signed does not clear them.
- **WSL commands**: build argv with `buildWslExecArgs` (always `--exec` — under `--`, `wsl.exe` expands `$name` in every argument and silently rewrites the script), and fence anything whose stdout you parse with `buildWslCapturedLoginShellCommand`, because the interactive login shell prints the distro banner to stdout. See [`docs/reference/wsl-command-execution.md`](./docs/reference/wsl-command-execution.md).
- **Linux native modules**: keep the glibc floor at Ubuntu 20.04 / glibc 2.31. A module compiled from source on a newer runner can reference symbol versions absent on the floor and crash the app on startup. See [`docs/reference/linux-glibc-compatibility.md`](./docs/reference/linux-glibc-compatibility.md); packaging fails if a bundled native binary needs newer glibc.
diff --git a/README.md b/README.md
index dfb676bcef5..2ae59035da8 100644
--- a/README.md
+++ b/README.md
@@ -238,10 +238,9 @@ Pair with your desktop app to monitor and steer your agents from your phone.
- **Discord:** Join the community on **[Discord](https://discord.gg/fzjDKHxv8Q)**.
- **Twitter / X:** Follow **[@orca_build](https://x.com/orca_build)** for updates and announcements.
-- **WeChat:** Scan to join the Orca community WeChat group 7. If it is full, use group 8.
+- **WeChat:** Scan to join the Orca community WeChat group 8. Group 8 may be full; if so, scan the Group 9 QR code instead.
-
-
+
- **Feedback & Ideas:** We ship fast. Missing something? [Request a new feature](https://github.com/stablyai/orca/issues).
- **Privacy:** See the [privacy & telemetry docs](https://www.onorca.dev/docs/telemetry) for what anonymous usage data Orca collects and how to opt out.
@@ -253,6 +252,9 @@ Pair with your desktop app to monitor and steer your agents from your phone.
Want to contribute or run locally? See our [CONTRIBUTING.md](.github/CONTRIBUTING.md) guide.
+The relay that pairs the mobile app with a desktop host is also in this repository under
+[`cloud/`](cloud/README.md), with a separate pnpm workspace and setup guide.
+
@@ -262,6 +264,7 @@ Want to contribute or run locally? See our [CONTRIBUTING.md](.github/CONTRIBUTIN
## Signed Builds
+
Windows code signing sponored/provided by [SignPath.io](https://signpath.io), certificate by [SignPath Foundation](https://signpath.org).
## License
diff --git a/cloud/.dockerignore b/cloud/.dockerignore
new file mode 100644
index 00000000000..bc93f9a9f1b
--- /dev/null
+++ b/cloud/.dockerignore
@@ -0,0 +1,15 @@
+.git/
+.github/
+.local/
+.terraform/
+node_modules/
+dist/
+coverage/
+.env
+.env.local
+.env.local.generated
+*.log
+*.tfplan
+*.tfstate
+*.tfstate.*
+
diff --git a/cloud/.editorconfig b/cloud/.editorconfig
new file mode 100644
index 00000000000..c814d3002a4
--- /dev/null
+++ b/cloud/.editorconfig
@@ -0,0 +1,10 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+indent_style = space
+indent_size = 2
+trim_trailing_whitespace = true
+
diff --git a/cloud/.gitignore b/cloud/.gitignore
new file mode 100644
index 00000000000..8105ed487f9
--- /dev/null
+++ b/cloud/.gitignore
@@ -0,0 +1,24 @@
+node_modules/
+dist/
+coverage/
+.turbo/
+.local/
+.env
+.env.local
+.env.local.generated
+*.log
+
+# Terraform/OpenTofu local state and plans must stay out of git.
+**/.terraform/
+*.tfstate
+*.tfstate.*
+*.tfplan
+crash.log
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Local dev signing key + SQLite live under data/ — never commit (contains a
+# private key and dev PII). Prod supplies the key via ORCA_CLOUD_SIGNING_KEY_PEM.
+data/
diff --git a/cloud/.gitleaks.toml b/cloud/.gitleaks.toml
new file mode 100644
index 00000000000..0bb1f966fae
--- /dev/null
+++ b/cloud/.gitleaks.toml
@@ -0,0 +1,22 @@
+[extend]
+useDefault = true
+
+[[allowlists]]
+description = "Explicit Relay test signing key"
+regexTarget = "secret"
+regexes = ['''^test-assignment-key-with-at-least-32-bytes$''']
+
+# The Cloud SQL rollout lease records `owner/repo/run_id` as the holder of a lease. The action's
+# unit tests build fixture holders from that shape, which the generic key rule reads as a secret.
+[[allowlists]]
+description = "Cloud SQL rollout lease holder keys in the action's unit tests"
+regexTarget = "secret"
+paths = ['''\.github/actions/cloud-sql-rollout-lease/[a-z-]+\.test\.mjs$''']
+regexes = ['''^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+/[0-9]+$''']
+
+# RFC 6455 §1.3 example handshake nonce ("the sample nonce" in base64), sent by the raw-socket
+# upgrade tests; the generic key rule reads any base64 header value as a secret.
+[[allowlists]]
+description = "RFC 6455 example Sec-WebSocket-Key in upgrade tests"
+regexTarget = "secret"
+regexes = ['''^dGhlIHNhbXBsZSBub25jZQ==$''']
diff --git a/cloud/.node-version b/cloud/.node-version
new file mode 100644
index 00000000000..14db15d775c
--- /dev/null
+++ b/cloud/.node-version
@@ -0,0 +1,2 @@
+24
+
diff --git a/cloud/.npmrc b/cloud/.npmrc
new file mode 100644
index 00000000000..e05d6820ed5
--- /dev/null
+++ b/cloud/.npmrc
@@ -0,0 +1,3 @@
+engine-strict=false
+package-manager-strict=true
+
diff --git a/cloud/.trufflehog-exclude-paths.txt b/cloud/.trufflehog-exclude-paths.txt
new file mode 100644
index 00000000000..ea20399af05
--- /dev/null
+++ b/cloud/.trufflehog-exclude-paths.txt
@@ -0,0 +1 @@
+(^|/)cloud/apps/relay/src/postgres-idle-client-error\.test\.ts$
diff --git a/cloud/.trufflehog-include-paths.txt b/cloud/.trufflehog-include-paths.txt
new file mode 100644
index 00000000000..abd82ac6354
--- /dev/null
+++ b/cloud/.trufflehog-include-paths.txt
@@ -0,0 +1,2 @@
+(^|/)cloud/
+(^|/)\.github/workflows/cloud-[^/]+\.yml$
diff --git a/cloud/README.md b/cloud/README.md
new file mode 100644
index 00000000000..8ffcd9fa6b3
--- /dev/null
+++ b/cloud/README.md
@@ -0,0 +1,92 @@
+# Orca Relay
+
+The relay that connects the Orca mobile app to a desktop host. Phones and
+desktops never talk to each other directly: each opens an outbound WebSocket
+to a relay cell, the relay pairs the two sessions, and it splices frames
+between them. A director assigns hosts to cells and coordinates migrations;
+cells carry the user connections.
+
+This directory is an independent pnpm workspace inside the Orca monorepo. Run
+its commands from `cloud/`, not the repository root. The source is covered by
+the repository's root [MIT license](../LICENSE).
+
+## Packages
+
+- `packages/relay-contract`: the wire contract shared by the relay, the
+ desktop app, and the mobile app (frame shapes, close codes, admission budgets,
+ splice state machine).
+- `apps/relay`: the relay server. The same image runs as a director or a cell
+ depending on `ORCA_RELAY_ROLE`.
+- `apps/relay-fence-broker`: a private, IAM-only service that owns the durable
+ mutation lease, the Terraform checkout, and the narrow Compute mutation used
+ when a registered target is superseded. The workflow that calls it holds read
+ and invoke rights only, never those mutation permissions.
+- `apps/relay-ops`: the relay operations console and the incident monitor
+ behind `pnpm ops:relay`, `pnpm incident:relay`, and
+ `pnpm incident:relay-preflight`.
+
+## Infrastructure and operations
+
+- `infra/terraform`: the relay Terraform root. It owns the cells, the director,
+ the shared Cloud SQL instance, DNS, observability, and every GitHub Workload
+ Identity provider the relay workflows authenticate through. `backend/` holds
+ the per-environment backend configuration and `environments/` the tfvars.
+ Drive it through `pnpm infra:init`, `pnpm infra:plan`, and `pnpm infra:apply`.
+- `dev/scripts`: the deploy, capacity, admission, rehome, monitoring, and load
+ scripts the workflows call, plus the contract tests that pin each workflow
+ and Terraform surface. Run them with `pnpm test`.
+- `dev/contracts` and `dev/fixtures`: the checked-in data those contract tests
+ read, including the Terraform root partition.
+- `docs/`: the relay runbooks, capacity-testing guide, incident-monitor
+ reference, and the workflow variable reference in `docs/relay-workflows.md`.
+
+## Workflows
+
+The 24 `.github/workflows/cloud-*.yml` workflows are the relay's deploy and
+operate surface: publish and deploy the director, roll GCE cell capacity,
+operate Asia admission and regional rehoming, prove staging capacity, monitor
+production, and power staging up and down. `.github/actions/cloud-sql-rollout-lease`
+is the compare-and-swap lease that serializes every rollout against the shared
+Cloud SQL instance.
+
+Every one of them is inert. Each top-level job is gated on
+`vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true'`, a repository variable that is
+unset here, so the two scheduled triggers and every manual dispatch skip
+without running a step. Only the repository owner, holding the GCP identities
+these workflows authenticate as, can turn them on.
+
+`Cloud Verify` is not gated. It builds, typechecks, lints, tests, secret-scans,
+and validates the relay Terraform on every change under `cloud/`, and it runs
+on fork pull requests, so it configures no backend and holds no credential.
+
+## What is not here
+
+The `terraform-foundation` and `terraform-apps` roots and the API and auth
+services live in the private `stablyai/orca-cloud` repository. Scripts and
+tests that spanned both trees were narrowed to the relay side rather than
+carrying a dangling reference.
+
+## Local development
+
+```sh
+cd cloud
+pnpm install
+pnpm build
+pnpm test
+```
+
+`pnpm test` runs the SQLite-backed suites. Tests that need PostgreSQL run only
+when `ORCA_RELAY_TEST_POSTGRES_URL` points at a disposable PostgreSQL 16 or 17
+database, for example:
+
+```sh
+docker run --rm -d --name orca-relay-pg -e POSTGRES_HOST_AUTH_METHOD=trust \
+ -e POSTGRES_DB=orca_relay_test -p 55440:5432 postgres:16-alpine
+ORCA_RELAY_TEST_POSTGRES_URL=postgres://postgres@127.0.0.1:55440/orca_relay_test \
+ pnpm --filter @orca-cloud/relay test
+docker rm -f orca-relay-pg
+```
+
+Configuration is read from environment variables validated in
+`apps/relay/src/config.ts`. `ORCA_RELAY_ASSIGNMENT_SIGNING_KEY` (at least 32
+bytes) is the only required value; everything else has a local default.
diff --git a/cloud/apps/relay-fence-broker/Dockerfile b/cloud/apps/relay-fence-broker/Dockerfile
new file mode 100644
index 00000000000..6c452dac393
--- /dev/null
+++ b/cloud/apps/relay-fence-broker/Dockerfile
@@ -0,0 +1,33 @@
+FROM node:24-bookworm-slim AS build
+WORKDIR /workspace
+RUN corepack enable
+COPY package.json pnpm-lock.yaml pnpm-workspace.yaml tsconfig.base.json ./
+COPY apps/relay-fence-broker/package.json apps/relay-fence-broker/package.json
+RUN pnpm install --frozen-lockfile
+COPY apps/relay-fence-broker apps/relay-fence-broker
+RUN pnpm --filter @orca-cloud/relay-fence-broker build
+
+FROM hashicorp/terraform:1.15.8 AS terraform
+
+FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:slim
+ARG ORCA_RELAY_FENCE_IMAGE_COMMIT
+RUN test "$(printf '%s' "${ORCA_RELAY_FENCE_IMAGE_COMMIT}" | grep -E '^[a-f0-9]{40}$')"
+ENV NODE_ENV=production
+ENV PORT=8080
+ENV ORCA_RELAY_FENCE_IMAGE_COMMIT=${ORCA_RELAY_FENCE_IMAGE_COMMIT}
+ENV IAC_TOOL=terraform
+WORKDIR /workspace
+COPY --from=build /usr/local /usr/local
+COPY --from=terraform /bin/terraform /usr/local/bin/terraform
+COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
+COPY apps/relay-fence-broker/package.json apps/relay-fence-broker/package.json
+COPY --from=build /workspace/apps/relay-fence-broker/dist apps/relay-fence-broker/dist
+COPY dev/scripts dev/scripts
+COPY infra/terraform infra/terraform
+RUN corepack enable \
+ && pnpm install --prod --frozen-lockfile --filter @orca-cloud/relay-fence-broker... \
+ && useradd --create-home --uid 10001 broker \
+ && chown -R broker:broker /workspace
+USER broker
+EXPOSE 8080
+CMD ["node", "apps/relay-fence-broker/dist/index.js"]
diff --git a/cloud/apps/relay-fence-broker/package.json b/cloud/apps/relay-fence-broker/package.json
new file mode 100644
index 00000000000..c9bf65c2cf3
--- /dev/null
+++ b/cloud/apps/relay-fence-broker/package.json
@@ -0,0 +1,27 @@
+{
+ "name": "@orca-cloud/relay-fence-broker",
+ "private": true,
+ "version": "0.0.0",
+ "type": "module",
+ "main": "dist/index.js",
+ "scripts": {
+ "build": "pnpm clean && tsc -p tsconfig.build.json",
+ "clean": "node -e \"require('fs').rmSync('dist', { recursive: true, force: true })\"",
+ "dev": "tsx watch src/index.ts",
+ "lint": "tsc -p tsconfig.json --noEmit",
+ "start": "node dist/index.js",
+ "test": "vitest run",
+ "typecheck": "tsc -p tsconfig.json --noEmit"
+ },
+ "dependencies": {
+ "@hono/node-server": "^1.19.14",
+ "hono": "^4.12.27",
+ "zod": "^3.25.76"
+ },
+ "devDependencies": {
+ "@types/node": "^24.10.0",
+ "tsx": "^4.21.0",
+ "typescript": "^5.9.3",
+ "vitest": "^4.0.8"
+ }
+}
diff --git a/cloud/apps/relay-fence-broker/src/app.test.ts b/cloud/apps/relay-fence-broker/src/app.test.ts
new file mode 100644
index 00000000000..0047c95b07b
--- /dev/null
+++ b/cloud/apps/relay-fence-broker/src/app.test.ts
@@ -0,0 +1,316 @@
+import { describe, expect, it, vi } from 'vitest'
+import { createApp } from './app.js'
+import type { RelayFenceBrokerConfig } from './config.js'
+import type {
+ GoogleStorageMutationLease,
+ MutationLease
+} from './mutation-lease.js'
+
+const commit = 'a'.repeat(40)
+const config: RelayFenceBrokerConfig = {
+ port: 8080,
+ project: 'onorca-cloud',
+ stateBucket: 'onorca-cloud-terraform-state',
+ leaseObject: 'terraform/state/relay-fence-broker/production.lock',
+ directorOrigin: 'https://relay.onorca.dev',
+ adminAudience: 'https://relay.onorca.dev/v1/admin/drain',
+ requesterServiceAccount: 'requester@example.com',
+ runtimeServiceAccount: 'runtime@example.com',
+ sourceCellId: 'production-gce-c3',
+ failedTargetCellId: 'production-gce-c11',
+ replacementTargetCellId: 'production-gce-c12',
+ imageCommit: commit,
+ terraformDir: 'infra/terraform',
+ unobservedConnectionBound: 10,
+ connectionCeiling: 600
+}
+
+function request(fenceCommit = commit): Request {
+ return new Request('http://broker/v1/supersede-target', {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({
+ v: 1,
+ operationId: 'c11-to-c12-forward',
+ fenceCommit,
+ confirmation: 'SUPERSEDE_TARGET'
+ })
+ })
+}
+
+function recoveryRequest(): Request {
+ return new Request('http://broker/v1/supersede-target', {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({
+ v: 1,
+ operationId: 'c11-to-c12-forward',
+ fenceCommit: commit,
+ completedFenceRecovery: {
+ attemptId: '11111111-1111-4111-8111-111111111111',
+ fenceCommit: 'b'.repeat(40),
+ gceOperation: 'operation-1',
+ terraformStateSerial: 61,
+ planObjectGeneration: '123',
+ terraformStateObjectGeneration: '456',
+ terraformStateObjectSha256: 'c'.repeat(64)
+ },
+ expectedLease: {
+ generation: '7',
+ operationId: 'c11-to-c12-forward',
+ requestDigest: 'd'.repeat(64)
+ },
+ confirmation: 'SUPERSEDE_TARGET'
+ })
+ })
+}
+
+function leaseTakeoverRequest(): Request {
+ return new Request('http://broker/v1/supersede-target', {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({
+ v: 1,
+ operationId: 'c11-to-c12-forward',
+ fenceCommit: commit,
+ expectedLease: {
+ generation: '7',
+ operationId: 'c11-to-c12-forward',
+ requestDigest: 'd'.repeat(64)
+ },
+ confirmation: 'SUPERSEDE_TARGET'
+ })
+ })
+}
+
+function sourceFenceRequest(
+ overrides: Record = {}
+): Request {
+ return new Request('http://broker/v1/fence-source', {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({
+ v: 1,
+ operationId: 'c3-final-fence',
+ fenceCommit: commit,
+ targetCellIds: ['production-gce-c7', 'production-gce-c12'],
+ confirmation: 'FENCE_SOURCE',
+ ...overrides
+ })
+ })
+}
+
+describe('relay fence broker', () => {
+ it('runs only after acquiring the durable lease', async () => {
+ const events: string[] = []
+ const lease = {
+ acquire: vi.fn(async () => {
+ events.push('acquire')
+ return {
+ generation: '7',
+ record: {}
+ } as MutationLease
+ }),
+ release: vi.fn(async () => {
+ events.push('release')
+ })
+ } as unknown as GoogleStorageMutationLease
+ const app = createApp(config, {
+ lease,
+ supersede: async () => {
+ events.push('supersede')
+ }
+ })
+
+ const response = await app.request(request())
+
+ expect(response.status).toBe(200)
+ expect(events).toEqual(['acquire', 'supersede', 'release'])
+ })
+
+ it('rejects a commit not bound to the immutable image', async () => {
+ const lease = {
+ acquire: vi.fn()
+ } as unknown as GoogleStorageMutationLease
+ const app = createApp(config, { lease })
+
+ const response = await app.request(request('b'.repeat(40)))
+
+ expect(response.status).toBe(409)
+ expect(lease.acquire).not.toHaveBeenCalled()
+ })
+
+ it('retains the lease when supersession fails', async () => {
+ const lease = {
+ acquire: vi.fn(async () => ({ generation: '7', record: {} })),
+ release: vi.fn()
+ } as unknown as GoogleStorageMutationLease
+ const app = createApp(config, {
+ lease,
+ supersede: async () => {
+ throw new Error('stopped safely')
+ }
+ })
+
+ const response = await app.request(request())
+
+ expect(response.status).toBe(500)
+ expect(lease.release).not.toHaveBeenCalled()
+ })
+
+ it('passes exact completed-attempt and live-lease recovery pins', async () => {
+ const lease = {
+ acquire: vi.fn(async () => ({ generation: '8', record: {} })),
+ release: vi.fn()
+ } as unknown as GoogleStorageMutationLease
+ const supersede = vi.fn(async () => {})
+ const app = createApp(config, { lease, supersede })
+
+ const response = await app.request(recoveryRequest())
+
+ expect(response.status).toBe(200)
+ expect(lease.acquire).toHaveBeenCalledWith(
+ 'c11-to-c12-forward',
+ expect.objectContaining({
+ completedFenceRecovery: expect.objectContaining({
+ terraformStateSerial: 61
+ })
+ }),
+ {
+ generation: '7',
+ operationId: 'c11-to-c12-forward',
+ requestDigest: 'd'.repeat(64)
+ }
+ )
+ expect(supersede).toHaveBeenCalledWith(
+ config,
+ expect.objectContaining({
+ attemptId: '11111111-1111-4111-8111-111111111111'
+ })
+ )
+ })
+
+ it('conditionally resumes the exact live supersession lease', async () => {
+ const lease = {
+ acquire: vi.fn(async () => ({ generation: '8', record: {} })),
+ release: vi.fn()
+ } as unknown as GoogleStorageMutationLease
+ const supersede = vi.fn(async () => {})
+ const app = createApp(config, { lease, supersede })
+
+ const response = await app.request(leaseTakeoverRequest())
+
+ expect(response.status).toBe(200)
+ expect(lease.acquire).toHaveBeenCalledWith(
+ 'c11-to-c12-forward',
+ expect.objectContaining({
+ expectedLease: {
+ generation: '7',
+ operationId: 'c11-to-c12-forward',
+ requestDigest: 'd'.repeat(64)
+ }
+ }),
+ {
+ generation: '7',
+ operationId: 'c11-to-c12-forward',
+ requestDigest: 'd'.repeat(64)
+ }
+ )
+ expect(supersede).toHaveBeenCalledWith(config, undefined)
+ })
+
+ it('rejects a live supersession lease for another operation', async () => {
+ const lease = {
+ acquire: vi.fn()
+ } as unknown as GoogleStorageMutationLease
+ const app = createApp(config, { lease })
+ const request = leaseTakeoverRequest()
+ const body = (await request.json()) as {
+ expectedLease: { operationId: string }
+ }
+ body.expectedLease.operationId = 'another-operation'
+
+ const response = await app.request(
+ new Request(request.url, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify(body)
+ })
+ )
+
+ expect(response.status).toBe(400)
+ expect(lease.acquire).not.toHaveBeenCalled()
+ })
+
+ it('fences the configured source only after acquiring the durable lease', async () => {
+ const events: string[] = []
+ const expectedLease = {
+ generation: '6',
+ operationId: 'c3-final-fence',
+ requestDigest: 'd'.repeat(64)
+ }
+ const lease = {
+ acquire: vi.fn(async () => {
+ events.push('acquire')
+ return { generation: '7', record: {} } as MutationLease
+ }),
+ release: vi.fn(async () => {
+ events.push('release')
+ })
+ } as unknown as GoogleStorageMutationLease
+ const fenceSource = vi.fn(async () => {
+ events.push('fence')
+ })
+ const app = createApp(config, { lease, fenceSource })
+
+ const response = await app.request(sourceFenceRequest({ expectedLease }))
+
+ expect(response.status).toBe(200)
+ expect(events).toEqual(['acquire', 'fence', 'release'])
+ expect(lease.acquire).toHaveBeenCalledWith(
+ 'c3-final-fence',
+ expect.objectContaining({ targetCellIds: expect.any(Array) }),
+ expectedLease
+ )
+ expect(fenceSource).toHaveBeenCalledWith(config, [
+ 'production-gce-c7',
+ 'production-gce-c12'
+ ])
+ })
+
+ it('rejects duplicate targets and the configured source', async () => {
+ const lease = { acquire: vi.fn() } as unknown as GoogleStorageMutationLease
+ const app = createApp(config, { lease })
+
+ const duplicate = await app.request(
+ sourceFenceRequest({
+ targetCellIds: ['production-gce-c7', 'production-gce-c7']
+ })
+ )
+ const source = await app.request(
+ sourceFenceRequest({ targetCellIds: [config.sourceCellId] })
+ )
+
+ expect(duplicate.status).toBe(400)
+ expect(source.status).toBe(400)
+ expect(lease.acquire).not.toHaveBeenCalled()
+ })
+
+ it('retains the source-fence lease after a controlled failure', async () => {
+ const lease = {
+ acquire: vi.fn(async () => ({ generation: '7', record: {} })),
+ release: vi.fn()
+ } as unknown as GoogleStorageMutationLease
+ const app = createApp(config, {
+ lease,
+ fenceSource: async () => {
+ throw new Error('stopped safely')
+ }
+ })
+
+ const response = await app.request(sourceFenceRequest())
+
+ expect(response.status).toBe(500)
+ expect(lease.release).not.toHaveBeenCalled()
+ })
+})
diff --git a/cloud/apps/relay-fence-broker/src/app.ts b/cloud/apps/relay-fence-broker/src/app.ts
new file mode 100644
index 00000000000..76d90ae205c
--- /dev/null
+++ b/cloud/apps/relay-fence-broker/src/app.ts
@@ -0,0 +1,162 @@
+import { Hono } from 'hono'
+import { z } from 'zod'
+import type { RelayFenceBrokerConfig } from './config.js'
+import {
+ GoogleStorageMutationLease,
+ MutationLeaseConflict
+} from './mutation-lease.js'
+import {
+ runSourceFence,
+ runTargetSupersession
+} from './fence-operation-runner.js'
+
+const expectedLeaseSchema = z
+ .object({
+ generation: z.string().regex(/^[1-9][0-9]{0,30}$/),
+ operationId: z.string().regex(/^[A-Za-z0-9_-]{8,128}$/),
+ requestDigest: z.string().regex(/^[a-f0-9]{64}$/)
+ })
+ .strict()
+
+const supersessionRequestSchema = z.object({
+ v: z.literal(1),
+ operationId: z.string().regex(/^[A-Za-z0-9_-]{8,128}$/),
+ fenceCommit: z.string().regex(/^[a-f0-9]{40}$/),
+ completedFenceRecovery: z
+ .object({
+ attemptId: z.string().uuid(),
+ fenceCommit: z.string().regex(/^[a-f0-9]{40}$/),
+ gceOperation: z.string().min(1).max(256),
+ terraformStateSerial: z.number().int().nonnegative().safe(),
+ planObjectGeneration: z.string().regex(/^[1-9][0-9]{0,30}$/),
+ terraformStateObjectGeneration: z.string().regex(/^[1-9][0-9]{0,30}$/),
+ terraformStateObjectSha256: z.string().regex(/^[a-f0-9]{64}$/)
+ })
+ .strict()
+ .optional(),
+ expectedLease: expectedLeaseSchema.optional(),
+ confirmation: z.literal('SUPERSEDE_TARGET')
+})
+ .strict()
+ .refine(
+ (value) =>
+ (!value.completedFenceRecovery || Boolean(value.expectedLease)) &&
+ (!value.expectedLease || value.expectedLease.operationId === value.operationId)
+ )
+
+const cellIdSchema = z.string().regex(/^[a-z][a-z0-9-]{0,127}$/)
+const sourceFenceRequestSchema = z
+ .object({
+ v: z.literal(1),
+ operationId: z.string().regex(/^[A-Za-z0-9_-]{8,128}$/),
+ fenceCommit: z.string().regex(/^[a-f0-9]{40}$/),
+ targetCellIds: z.array(cellIdSchema).min(1).max(16),
+ expectedLease: expectedLeaseSchema.optional(),
+ confirmation: z.literal('FENCE_SOURCE')
+ })
+ .strict()
+ .refine(
+ (value) =>
+ new Set(value.targetCellIds).size === value.targetCellIds.length &&
+ (!value.expectedLease || value.expectedLease.operationId === value.operationId)
+ )
+
+type AppDependencies = {
+ lease?: GoogleStorageMutationLease
+ supersede?: (
+ config: RelayFenceBrokerConfig,
+ recovery?: z.infer['completedFenceRecovery']
+ ) => Promise
+ fenceSource?: (
+ config: RelayFenceBrokerConfig,
+ targetCellIds: string[]
+ ) => Promise
+}
+
+export function createApp(
+ config: RelayFenceBrokerConfig,
+ dependencies: AppDependencies = {}
+): Hono {
+ const app = new Hono()
+ const lease =
+ dependencies.lease ??
+ new GoogleStorageMutationLease(
+ config.stateBucket,
+ config.leaseObject,
+ config.imageCommit
+ )
+ const supersede = dependencies.supersede ?? runTargetSupersession
+ const fenceSource = dependencies.fenceSource ?? runSourceFence
+
+ app.get('/healthz', (context) =>
+ context.json({ ok: true, fenceCommit: config.imageCommit })
+ )
+ app.post('/v1/supersede-target', async (context) => {
+ const parsed = supersessionRequestSchema.safeParse(
+ await context.req.json().catch(() => null)
+ )
+ if (!parsed.success) return context.json({ error: 'invalid_request' }, 400)
+ if (parsed.data.fenceCommit !== config.imageCommit) {
+ return context.json({ error: 'fence_commit_mismatch' }, 409)
+ }
+ let acquired
+ try {
+ acquired = await lease.acquire(
+ parsed.data.operationId,
+ parsed.data,
+ parsed.data.expectedLease
+ )
+ } catch (error) {
+ if (error instanceof MutationLeaseConflict) {
+ return context.json({ error: 'mutation_lease_conflict' }, 409)
+ }
+ throw error
+ }
+ await supersede(config, parsed.data.completedFenceRecovery)
+ await lease.release(acquired)
+ return context.json({
+ ok: true,
+ operationId: parsed.data.operationId,
+ fenceCommit: config.imageCommit
+ })
+ })
+ app.post('/v1/fence-source', async (context) => {
+ const parsed = sourceFenceRequestSchema.safeParse(
+ await context.req.json().catch(() => null)
+ )
+ if (
+ !parsed.success ||
+ parsed.data.targetCellIds.includes(config.sourceCellId)
+ ) {
+ return context.json({ error: 'invalid_request' }, 400)
+ }
+ if (parsed.data.fenceCommit !== config.imageCommit) {
+ return context.json({ error: 'fence_commit_mismatch' }, 409)
+ }
+ let acquired
+ try {
+ acquired = await lease.acquire(
+ parsed.data.operationId,
+ parsed.data,
+ parsed.data.expectedLease
+ )
+ } catch (error) {
+ if (error instanceof MutationLeaseConflict) {
+ return context.json({ error: 'mutation_lease_conflict' }, 409)
+ }
+ throw error
+ }
+ await fenceSource(config, parsed.data.targetCellIds)
+ await lease.release(acquired)
+ return context.json({
+ ok: true,
+ operationId: parsed.data.operationId,
+ fenceCommit: config.imageCommit
+ })
+ })
+ app.onError((error, context) => {
+ console.error(error)
+ return context.json({ error: 'broker_operation_failed' }, 500)
+ })
+ return app
+}
diff --git a/cloud/apps/relay-fence-broker/src/config.ts b/cloud/apps/relay-fence-broker/src/config.ts
new file mode 100644
index 00000000000..baf8340cd65
--- /dev/null
+++ b/cloud/apps/relay-fence-broker/src/config.ts
@@ -0,0 +1,92 @@
+import { z } from 'zod'
+
+const environmentSchema = z.object({
+ PORT: z.coerce.number().int().positive().max(65_535).default(8080),
+ ORCA_RELAY_FENCE_PROJECT: z.string().regex(/^[a-z][a-z0-9-]{4,29}$/),
+ ORCA_RELAY_FENCE_STATE_BUCKET: z.string().min(3).max(222),
+ ORCA_RELAY_FENCE_LEASE_OBJECT: z.string().min(1).max(512),
+ ORCA_RELAY_FENCE_DIRECTOR_ORIGIN: z.string().url(),
+ ORCA_RELAY_FENCE_ADMIN_AUDIENCE: z.string().url(),
+ ORCA_RELAY_FENCE_REQUESTER_SERVICE_ACCOUNT: z.string().email(),
+ ORCA_RELAY_FENCE_RUNTIME_SERVICE_ACCOUNT: z.string().email(),
+ ORCA_RELAY_FENCE_SOURCE_CELL_ID: z.string().regex(/^[a-z][a-z0-9-]{0,127}$/),
+ ORCA_RELAY_FENCE_FAILED_TARGET_CELL_ID: z
+ .string()
+ .regex(/^[a-z][a-z0-9-]{0,127}$/),
+ ORCA_RELAY_FENCE_REPLACEMENT_TARGET_CELL_ID: z
+ .string()
+ .regex(/^[a-z][a-z0-9-]{0,127}$/),
+ ORCA_RELAY_FENCE_IMAGE_COMMIT: z.string().regex(/^[a-f0-9]{40}$/),
+ // Resolved against the broker's working directory, which the image sets to the copied tree root.
+ ORCA_RELAY_FENCE_TERRAFORM_DIR: z.string().min(1).default('infra/terraform'),
+ ORCA_RELAY_FENCE_UNOBSERVED_CONNECTION_BOUND: z.coerce
+ .number()
+ .int()
+ .nonnegative()
+ .max(499),
+ ORCA_RELAY_FENCE_CONNECTION_CEILING: z.coerce
+ .number()
+ .int()
+ .positive()
+ .max(600)
+ .default(600)
+})
+
+export type RelayFenceBrokerConfig = {
+ port: number
+ project: string
+ stateBucket: string
+ leaseObject: string
+ directorOrigin: string
+ adminAudience: string
+ requesterServiceAccount: string
+ runtimeServiceAccount: string
+ sourceCellId: string
+ failedTargetCellId: string
+ replacementTargetCellId: string
+ imageCommit: string
+ terraformDir: string
+ unobservedConnectionBound: number
+ connectionCeiling: number
+}
+
+export function loadConfig(
+ environment: NodeJS.ProcessEnv = process.env
+): RelayFenceBrokerConfig {
+ const parsed = environmentSchema.parse(environment)
+ const director = new URL(parsed.ORCA_RELAY_FENCE_DIRECTOR_ORIGIN)
+ const audience = new URL(parsed.ORCA_RELAY_FENCE_ADMIN_AUDIENCE)
+ if (
+ director.origin !== parsed.ORCA_RELAY_FENCE_DIRECTOR_ORIGIN ||
+ audience.origin !== director.origin ||
+ audience.pathname !== '/v1/admin/drain' ||
+ audience.search ||
+ audience.hash
+ ) {
+ throw new Error('broker director and admin audience must use the canonical drain origin')
+ }
+ const cells = new Set([
+ parsed.ORCA_RELAY_FENCE_SOURCE_CELL_ID,
+ parsed.ORCA_RELAY_FENCE_FAILED_TARGET_CELL_ID,
+ parsed.ORCA_RELAY_FENCE_REPLACEMENT_TARGET_CELL_ID
+ ])
+ if (cells.size !== 3) throw new Error('broker cells must be distinct')
+ return {
+ port: parsed.PORT,
+ project: parsed.ORCA_RELAY_FENCE_PROJECT,
+ stateBucket: parsed.ORCA_RELAY_FENCE_STATE_BUCKET,
+ leaseObject: parsed.ORCA_RELAY_FENCE_LEASE_OBJECT,
+ directorOrigin: parsed.ORCA_RELAY_FENCE_DIRECTOR_ORIGIN,
+ adminAudience: parsed.ORCA_RELAY_FENCE_ADMIN_AUDIENCE,
+ requesterServiceAccount: parsed.ORCA_RELAY_FENCE_REQUESTER_SERVICE_ACCOUNT,
+ runtimeServiceAccount: parsed.ORCA_RELAY_FENCE_RUNTIME_SERVICE_ACCOUNT,
+ sourceCellId: parsed.ORCA_RELAY_FENCE_SOURCE_CELL_ID,
+ failedTargetCellId: parsed.ORCA_RELAY_FENCE_FAILED_TARGET_CELL_ID,
+ replacementTargetCellId: parsed.ORCA_RELAY_FENCE_REPLACEMENT_TARGET_CELL_ID,
+ imageCommit: parsed.ORCA_RELAY_FENCE_IMAGE_COMMIT,
+ terraformDir: parsed.ORCA_RELAY_FENCE_TERRAFORM_DIR,
+ unobservedConnectionBound:
+ parsed.ORCA_RELAY_FENCE_UNOBSERVED_CONNECTION_BOUND,
+ connectionCeiling: parsed.ORCA_RELAY_FENCE_CONNECTION_CEILING
+ }
+}
diff --git a/cloud/apps/relay-fence-broker/src/fence-operation-runner.test.ts b/cloud/apps/relay-fence-broker/src/fence-operation-runner.test.ts
new file mode 100644
index 00000000000..b48898977cd
--- /dev/null
+++ b/cloud/apps/relay-fence-broker/src/fence-operation-runner.test.ts
@@ -0,0 +1,57 @@
+import { describe, expect, it } from 'vitest'
+import type { RelayFenceBrokerConfig } from './config.js'
+import {
+ fenceChildEnvironment,
+ sourceFenceArguments
+} from './fence-operation-runner.js'
+
+const config = {
+ project: 'onorca-cloud',
+ directorOrigin: 'https://relay.onorca.dev',
+ adminAudience: 'https://relay.onorca.dev/v1/admin/drain',
+ sourceCellId: 'production-gce-c3',
+ runtimeServiceAccount: 'runtime@example.com',
+ imageCommit: 'a'.repeat(40),
+ terraformDir: 'infra/terraform',
+ unobservedConnectionBound: 10,
+ connectionCeiling: 600
+} as RelayFenceBrokerConfig
+
+describe('fence operation runner', () => {
+ it('passes distinct read and mutation identity tokens', () => {
+ expect(
+ fenceChildEnvironment(
+ config,
+ 'read.token.value',
+ 'mutation.token.value',
+ { PRESERVED: 'yes' }
+ )
+ ).toMatchObject({
+ PRESERVED: 'yes',
+ IAC_TOOL: 'terraform',
+ ORCA_RELAY_ADMIN_ID_TOKEN: 'read.token.value',
+ ORCA_RELAY_FENCE_MUTATION_ID_TOKEN: 'mutation.token.value',
+ ORCA_RELAY_FENCE_IMAGE_COMMIT: 'a'.repeat(40)
+ })
+ })
+
+ it('binds a source fence to deterministic targets and the production var file', () => {
+ const args = sourceFenceArguments(config, '/tmp/topology.json', [
+ 'production-gce-c12',
+ 'production-gce-c7'
+ ])
+
+ expect(args).toEqual(
+ expect.arrayContaining([
+ '--source-cell-id',
+ 'production-gce-c3',
+ '--target-cell-ids',
+ 'production-gce-c12,production-gce-c7',
+ '--terraform-var-file',
+ 'environments/production.tfvars',
+ '--mode',
+ 'fence-source'
+ ])
+ )
+ })
+})
diff --git a/cloud/apps/relay-fence-broker/src/fence-operation-runner.ts b/cloud/apps/relay-fence-broker/src/fence-operation-runner.ts
new file mode 100644
index 00000000000..e7fbff5a6ab
--- /dev/null
+++ b/cloud/apps/relay-fence-broker/src/fence-operation-runner.ts
@@ -0,0 +1,185 @@
+import { execFile } from 'node:child_process'
+import { mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { promisify } from 'node:util'
+import type { RelayFenceBrokerConfig } from './config.js'
+import {
+ metadataIdentityToken,
+ metadataServiceAccountEmail
+} from './google-metadata.js'
+
+const exec = promisify(execFile)
+const MAX_OUTPUT_BYTES = 10 * 1024 * 1024
+type CompletedFenceRecovery = {
+ attemptId: string
+ fenceCommit: string
+ gceOperation: string
+ terraformStateSerial: number
+ planObjectGeneration: string
+ terraformStateObjectGeneration: string
+ terraformStateObjectSha256: string
+}
+
+async function run(file: string, args: string[], environment?: NodeJS.ProcessEnv) {
+ return await exec(file, args, {
+ env: environment,
+ maxBuffer: MAX_OUTPUT_BYTES
+ })
+}
+
+export function fenceChildEnvironment(
+ config: RelayFenceBrokerConfig,
+ readToken: string,
+ mutationToken: string,
+ environment: NodeJS.ProcessEnv = process.env
+): NodeJS.ProcessEnv {
+ return {
+ ...environment,
+ IAC_TOOL: 'terraform',
+ ORCA_RELAY_ADMIN_ID_TOKEN: readToken,
+ ORCA_RELAY_FENCE_MUTATION_ID_TOKEN: mutationToken,
+ ORCA_RELAY_FENCE_IMAGE_COMMIT: config.imageCommit
+ }
+}
+
+function relayFenceArguments(
+ config: RelayFenceBrokerConfig,
+ topologyFile: string,
+ targetCellIds: string[]
+): string[] {
+ return [
+ 'dev/scripts/deploy-relay-gce-multi-target.mjs',
+ '--project',
+ config.project,
+ '--director-origin',
+ config.directorOrigin,
+ '--admin-audience',
+ config.adminAudience,
+ '--topology-file',
+ topologyFile,
+ '--source-cell-id',
+ config.sourceCellId,
+ '--target-cell-ids',
+ [...targetCellIds].sort().join(','),
+ '--unobserved-connection-bound',
+ String(config.unobservedConnectionBound),
+ '--runtime-service-account',
+ config.runtimeServiceAccount,
+ '--environment',
+ 'production',
+ '--fence-commit',
+ config.imageCommit,
+ '--terraform-dir',
+ config.terraformDir,
+ '--terraform-var-file',
+ 'environments/production.tfvars',
+ '--connection-ceiling',
+ String(config.connectionCeiling),
+ '--minimum-lease-remaining-ms',
+ '600000'
+ ]
+}
+
+async function runRelayFenceCommand(
+ config: RelayFenceBrokerConfig,
+ argumentsForTopology: (
+ topologyFile: string,
+ brokerServiceAccount: string
+ ) => string[]
+): Promise {
+ const directory = await mkdtemp(join(tmpdir(), 'orca-relay-fence-operation-'))
+ const topologyFile = join(directory, 'topology.json')
+ try {
+ await run('terraform', [
+ `-chdir=${config.terraformDir}`,
+ 'init',
+ '-input=false',
+ '-backend-config=backend/production.hcl'
+ ])
+ const topology = await run('terraform', [
+ `-chdir=${config.terraformDir}`,
+ 'output',
+ '-json',
+ 'relay_gce_cell_deployments'
+ ])
+ await writeFile(topologyFile, topology.stdout, { mode: 0o600 })
+ const brokerToken = await metadataIdentityToken(config.adminAudience)
+ const brokerServiceAccount = await metadataServiceAccountEmail()
+ const environment = fenceChildEnvironment(
+ config,
+ brokerToken,
+ brokerToken
+ )
+ await run(
+ 'node',
+ argumentsForTopology(topologyFile, brokerServiceAccount),
+ environment
+ )
+ } finally {
+ await rm(directory, { recursive: true, force: true })
+ }
+}
+
+export function sourceFenceArguments(
+ config: RelayFenceBrokerConfig,
+ topologyFile: string,
+ targetCellIds: string[]
+): string[] {
+ return [
+ ...relayFenceArguments(config, topologyFile, targetCellIds),
+ '--mode',
+ 'fence-source'
+ ]
+}
+
+export async function runSourceFence(
+ config: RelayFenceBrokerConfig,
+ targetCellIds: string[]
+): Promise {
+ await runRelayFenceCommand(config, (topologyFile) =>
+ sourceFenceArguments(config, topologyFile, targetCellIds)
+ )
+}
+
+export async function runTargetSupersession(
+ config: RelayFenceBrokerConfig,
+ recovery?: CompletedFenceRecovery
+): Promise {
+ await runRelayFenceCommand(config, (topologyFile, brokerServiceAccount) => {
+ const targets = [
+ config.failedTargetCellId,
+ config.replacementTargetCellId
+ ]
+ const args = [
+ ...relayFenceArguments(config, topologyFile, targets),
+ '--failed-target-cell-id',
+ config.failedTargetCellId,
+ '--replacement-target-cell-id',
+ config.replacementTargetCellId,
+ '--mode',
+ 'supersede-target'
+ ]
+ if (recovery) {
+ args.push(
+ '--completed-fence-attempt-id',
+ recovery.attemptId,
+ '--completed-fence-commit',
+ recovery.fenceCommit,
+ '--completed-fence-operation',
+ recovery.gceOperation,
+ '--completed-fence-state-serial',
+ String(recovery.terraformStateSerial),
+ '--completed-fence-plan-generation',
+ recovery.planObjectGeneration,
+ '--completed-fence-state-generation',
+ recovery.terraformStateObjectGeneration,
+ '--completed-fence-state-sha256',
+ recovery.terraformStateObjectSha256,
+ '--fence-broker-service-account',
+ brokerServiceAccount
+ )
+ }
+ return args
+ })
+}
diff --git a/cloud/apps/relay-fence-broker/src/google-metadata.ts b/cloud/apps/relay-fence-broker/src/google-metadata.ts
new file mode 100644
index 00000000000..ffae11e392c
--- /dev/null
+++ b/cloud/apps/relay-fence-broker/src/google-metadata.ts
@@ -0,0 +1,44 @@
+const METADATA_ROOT =
+ 'http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default'
+
+async function metadataText(path: string, fetcher: typeof fetch): Promise {
+ const response = await fetcher(`${METADATA_ROOT}/${path}`, {
+ headers: { 'Metadata-Flavor': 'Google' }
+ })
+ if (!response.ok) throw new Error(`metadata request failed: ${response.status}`)
+ return await response.text()
+}
+
+export async function metadataAccessToken(fetcher: typeof fetch = fetch): Promise {
+ const body = JSON.parse(await metadataText('token', fetcher)) as {
+ access_token?: unknown
+ }
+ if (typeof body.access_token !== 'string' || body.access_token.length < 20) {
+ throw new Error('metadata access token is missing')
+ }
+ return body.access_token
+}
+
+export async function metadataServiceAccountEmail(
+ fetcher: typeof fetch = fetch
+): Promise {
+ const email = (await metadataText('email', fetcher)).trim()
+ if (!/^[^@\s]+@[^@\s]+\.gserviceaccount\.com$/.test(email)) {
+ throw new Error('metadata service account email is invalid')
+ }
+ return email
+}
+
+export async function metadataIdentityToken(
+ audience: string,
+ fetcher: typeof fetch = fetch
+): Promise {
+ const token = await metadataText(
+ `identity?audience=${encodeURIComponent(audience)}&format=full`,
+ fetcher
+ )
+ if (!/^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/.test(token)) {
+ throw new Error('metadata identity token is invalid')
+ }
+ return token
+}
diff --git a/cloud/apps/relay-fence-broker/src/index.ts b/cloud/apps/relay-fence-broker/src/index.ts
new file mode 100644
index 00000000000..cd237b964f1
--- /dev/null
+++ b/cloud/apps/relay-fence-broker/src/index.ts
@@ -0,0 +1,10 @@
+import { serve } from '@hono/node-server'
+import { createApp } from './app.js'
+import { loadConfig } from './config.js'
+
+const config = loadConfig()
+const app = createApp(config)
+
+serve({ fetch: app.fetch, port: config.port }, (info) => {
+ console.log(`[relay-fence-broker] listening on port ${info.port}`)
+})
diff --git a/cloud/apps/relay-fence-broker/src/mutation-lease.test.ts b/cloud/apps/relay-fence-broker/src/mutation-lease.test.ts
new file mode 100644
index 00000000000..6006e207ce7
--- /dev/null
+++ b/cloud/apps/relay-fence-broker/src/mutation-lease.test.ts
@@ -0,0 +1,164 @@
+import { describe, expect, it, vi } from 'vitest'
+import {
+ GoogleStorageMutationLease,
+ MutationLeaseConflict
+} from './mutation-lease.js'
+
+function json(body: unknown, status = 200): Response {
+ return new Response(JSON.stringify(body), {
+ status,
+ headers: { 'Content-Type': 'application/json' }
+ })
+}
+
+const accessToken = () => json({ access_token: 'a'.repeat(40) })
+const request = {
+ v: 1,
+ operationId: 'c11-to-c12-forward',
+ fenceCommit: 'a'.repeat(40),
+ confirmation: 'SUPERSEDE_TARGET'
+}
+
+describe('GoogleStorageMutationLease', () => {
+ it('creates and conditionally releases a new lease', async () => {
+ const fetcher = vi
+ .fn()
+ .mockResolvedValueOnce(accessToken())
+ .mockResolvedValueOnce(new Response(null, { status: 404 }))
+ .mockResolvedValueOnce(json({ generation: '7' }))
+ .mockResolvedValueOnce(accessToken())
+ .mockResolvedValueOnce(new Response(null, { status: 204 }))
+ const leaseStore = new GoogleStorageMutationLease(
+ 'state-bucket',
+ 'fence/production.lock',
+ 'a'.repeat(40),
+ fetcher,
+ () => 1_000
+ )
+
+ const lease = await leaseStore.acquire(request.operationId, request)
+ await leaseStore.release(lease)
+
+ expect(lease.generation).toBe('7')
+ expect(fetcher.mock.calls[2]?.[0]).toContain('ifGenerationMatch=0')
+ expect(fetcher.mock.calls[4]?.[0]).toContain('ifGenerationMatch=7')
+ })
+
+ it('rejects a different request while the durable lease is live', async () => {
+ const existing = {
+ operationId: 'other-operation',
+ requestDigest: 'b'.repeat(64),
+ imageCommit: 'a'.repeat(40),
+ acquiredAt: 500,
+ expiresAt: 2_000
+ }
+ const fetcher = vi
+ .fn()
+ .mockResolvedValueOnce(accessToken())
+ .mockResolvedValueOnce(json({ generation: '7' }))
+ .mockResolvedValueOnce(json(existing))
+ const leaseStore = new GoogleStorageMutationLease(
+ 'state-bucket',
+ 'fence/production.lock',
+ 'a'.repeat(40),
+ fetcher,
+ () => 1_000
+ )
+
+ await expect(
+ leaseStore.acquire(request.operationId, request)
+ ).rejects.toBeInstanceOf(MutationLeaseConflict)
+ expect(fetcher).toHaveBeenCalledTimes(3)
+ })
+
+ it('takes over an expired lease with an exact generation precondition', async () => {
+ const existing = {
+ operationId: 'other-operation',
+ requestDigest: 'b'.repeat(64),
+ imageCommit: 'a'.repeat(40),
+ acquiredAt: 500,
+ expiresAt: 999
+ }
+ const fetcher = vi
+ .fn()
+ .mockResolvedValueOnce(accessToken())
+ .mockResolvedValueOnce(json({ generation: '7' }))
+ .mockResolvedValueOnce(json(existing))
+ .mockResolvedValueOnce(json({ generation: '8' }))
+ const leaseStore = new GoogleStorageMutationLease(
+ 'state-bucket',
+ 'fence/production.lock',
+ 'a'.repeat(40),
+ fetcher,
+ () => 1_000
+ )
+
+ const lease = await leaseStore.acquire(request.operationId, request)
+
+ expect(lease.generation).toBe('8')
+ expect(fetcher.mock.calls[3]?.[0]).toContain('ifGenerationMatch=7')
+ })
+
+ it('conditionally replaces the exact authorized live lease', async () => {
+ const existing = {
+ operationId: request.operationId,
+ requestDigest: 'b'.repeat(64),
+ imageCommit: 'b'.repeat(40),
+ acquiredAt: 500,
+ expiresAt: 2_000
+ }
+ const fetcher = vi
+ .fn()
+ .mockResolvedValueOnce(accessToken())
+ .mockResolvedValueOnce(json({ generation: '7' }))
+ .mockResolvedValueOnce(json(existing))
+ .mockResolvedValueOnce(json({ generation: '8' }))
+ const leaseStore = new GoogleStorageMutationLease(
+ 'state-bucket',
+ 'fence/production.lock',
+ 'a'.repeat(40),
+ fetcher,
+ () => 1_000
+ )
+
+ const lease = await leaseStore.acquire(request.operationId, request, {
+ generation: '7',
+ operationId: existing.operationId,
+ requestDigest: existing.requestDigest
+ })
+
+ expect(lease.generation).toBe('8')
+ expect(fetcher.mock.calls[3]?.[0]).toContain('ifGenerationMatch=7')
+ })
+
+ it('rejects a live-lease takeover when any expected field differs', async () => {
+ const existing = {
+ operationId: request.operationId,
+ requestDigest: 'b'.repeat(64),
+ imageCommit: 'b'.repeat(40),
+ acquiredAt: 500,
+ expiresAt: 2_000
+ }
+ const fetcher = vi
+ .fn()
+ .mockResolvedValueOnce(accessToken())
+ .mockResolvedValueOnce(json({ generation: '7' }))
+ .mockResolvedValueOnce(json(existing))
+ const leaseStore = new GoogleStorageMutationLease(
+ 'state-bucket',
+ 'fence/production.lock',
+ 'a'.repeat(40),
+ fetcher,
+ () => 1_000
+ )
+
+ await expect(
+ leaseStore.acquire(request.operationId, request, {
+ generation: '8',
+ operationId: existing.operationId,
+ requestDigest: existing.requestDigest
+ })
+ ).rejects.toBeInstanceOf(MutationLeaseConflict)
+ expect(fetcher).toHaveBeenCalledTimes(3)
+ })
+})
diff --git a/cloud/apps/relay-fence-broker/src/mutation-lease.ts b/cloud/apps/relay-fence-broker/src/mutation-lease.ts
new file mode 100644
index 00000000000..38bd4a1cdfe
--- /dev/null
+++ b/cloud/apps/relay-fence-broker/src/mutation-lease.ts
@@ -0,0 +1,152 @@
+import { createHash } from 'node:crypto'
+import { metadataAccessToken } from './google-metadata.js'
+
+const LEASE_TTL_MS = 35 * 60 * 1_000
+
+type LeaseRecord = {
+ operationId: string
+ requestDigest: string
+ imageCommit: string
+ acquiredAt: number
+ expiresAt: number
+}
+
+type ObjectMetadata = {
+ generation: string
+}
+
+export class MutationLeaseConflict extends Error {}
+
+export type MutationLease = {
+ generation: string
+ record: LeaseRecord
+}
+
+export type ExpectedMutationLease = {
+ generation: string
+ operationId: string
+ requestDigest: string
+}
+
+export class GoogleStorageMutationLease {
+ constructor(
+ private readonly bucket: string,
+ private readonly objectName: string,
+ private readonly imageCommit: string,
+ private readonly fetcher: typeof fetch = fetch,
+ private readonly now: () => number = Date.now
+ ) {}
+
+ async acquire(
+ operationId: string,
+ request: unknown,
+ expectedExisting?: ExpectedMutationLease
+ ): Promise {
+ const token = await metadataAccessToken(this.fetcher)
+ const existing = await this.read(token)
+ const requestDigest = createHash('sha256')
+ .update(JSON.stringify(request))
+ .digest('hex')
+ const exactTakeover =
+ existing &&
+ expectedExisting?.generation === existing.metadata.generation &&
+ expectedExisting.operationId === existing.record.operationId &&
+ expectedExisting.requestDigest === existing.record.requestDigest
+ if (
+ (!existing && expectedExisting) ||
+ (existing &&
+ existing.record.requestDigest !== requestDigest &&
+ existing.record.expiresAt > this.now() &&
+ !exactTakeover)
+ ) {
+ throw new MutationLeaseConflict('another relay mutation owns the durable lease')
+ }
+ const acquiredAt = this.now()
+ const record: LeaseRecord = {
+ operationId,
+ requestDigest,
+ imageCommit: this.imageCommit,
+ acquiredAt,
+ expiresAt: acquiredAt + LEASE_TTL_MS
+ }
+ const generation = existing?.metadata.generation ?? '0'
+ const response = await this.fetcher(
+ `${this.uploadUrl()}&ifGenerationMatch=${encodeURIComponent(generation)}`,
+ {
+ method: 'POST',
+ headers: {
+ Authorization: `Bearer ${token}`,
+ 'Content-Type': 'application/json'
+ },
+ body: JSON.stringify(record)
+ }
+ )
+ if (response.status === 412) {
+ throw new MutationLeaseConflict('relay mutation lease changed concurrently')
+ }
+ if (!response.ok) throw new Error(`mutation lease acquisition failed: ${response.status}`)
+ const metadata = (await response.json()) as Partial
+ if (!/^[1-9][0-9]{0,30}$/.test(metadata.generation ?? '')) {
+ throw new Error('mutation lease has no valid generation')
+ }
+ return { generation: metadata.generation!, record }
+ }
+
+ async release(lease: MutationLease): Promise {
+ const token = await metadataAccessToken(this.fetcher)
+ const response = await this.fetcher(
+ `${this.metadataUrl()}?ifGenerationMatch=${encodeURIComponent(lease.generation)}`,
+ {
+ method: 'DELETE',
+ headers: { Authorization: `Bearer ${token}` }
+ }
+ )
+ if (!response.ok && response.status !== 404) {
+ throw new Error(`mutation lease release failed: ${response.status}`)
+ }
+ }
+
+ private async read(
+ token: string
+ ): Promise<{ metadata: ObjectMetadata; record: LeaseRecord } | null> {
+ const metadataResponse = await this.fetcher(this.metadataUrl(), {
+ headers: { Authorization: `Bearer ${token}` }
+ })
+ if (metadataResponse.status === 404) return null
+ if (!metadataResponse.ok) {
+ throw new Error(`mutation lease inspection failed: ${metadataResponse.status}`)
+ }
+ const metadata = (await metadataResponse.json()) as Partial
+ if (!/^[1-9][0-9]{0,30}$/.test(metadata.generation ?? '')) {
+ throw new Error('existing mutation lease has no valid generation')
+ }
+ const bodyResponse = await this.fetcher(`${this.metadataUrl()}?alt=media`, {
+ headers: { Authorization: `Bearer ${token}` }
+ })
+ if (!bodyResponse.ok) {
+ throw new Error(`mutation lease body read failed: ${bodyResponse.status}`)
+ }
+ const record = (await bodyResponse.json()) as Partial
+ if (
+ typeof record.operationId !== 'string' ||
+ !/^[a-f0-9]{64}$/.test(record.requestDigest ?? '') ||
+ !/^[a-f0-9]{40}$/.test(record.imageCommit ?? '') ||
+ !Number.isSafeInteger(record.acquiredAt) ||
+ !Number.isSafeInteger(record.expiresAt)
+ ) {
+ throw new Error('existing mutation lease is invalid')
+ }
+ return {
+ metadata: { generation: metadata.generation! },
+ record: record as LeaseRecord
+ }
+ }
+
+ private metadataUrl(): string {
+ return `https://storage.googleapis.com/storage/v1/b/${encodeURIComponent(this.bucket)}/o/${encodeURIComponent(this.objectName)}`
+ }
+
+ private uploadUrl(): string {
+ return `https://storage.googleapis.com/upload/storage/v1/b/${encodeURIComponent(this.bucket)}/o?uploadType=media&name=${encodeURIComponent(this.objectName)}`
+ }
+}
diff --git a/cloud/apps/relay-fence-broker/tsconfig.build.json b/cloud/apps/relay-fence-broker/tsconfig.build.json
new file mode 100644
index 00000000000..489ddfd34d6
--- /dev/null
+++ b/cloud/apps/relay-fence-broker/tsconfig.build.json
@@ -0,0 +1,10 @@
+{
+ "extends": "./tsconfig.json",
+ "compilerOptions": {
+ "declaration": true,
+ "noEmit": false,
+ "outDir": "dist",
+ "rootDir": "src"
+ },
+ "exclude": ["src/**/*.test.ts"]
+}
diff --git a/cloud/apps/relay-fence-broker/tsconfig.json b/cloud/apps/relay-fence-broker/tsconfig.json
new file mode 100644
index 00000000000..a552e34dbe9
--- /dev/null
+++ b/cloud/apps/relay-fence-broker/tsconfig.json
@@ -0,0 +1,5 @@
+{
+ "extends": "../../tsconfig.base.json",
+ "compilerOptions": { "noEmit": true },
+ "include": ["src/**/*.ts"]
+}
diff --git a/cloud/apps/relay-ops/README.md b/cloud/apps/relay-ops/README.md
new file mode 100644
index 00000000000..7dd16307dc0
--- /dev/null
+++ b/cloud/apps/relay-ops/README.md
@@ -0,0 +1,66 @@
+# Orca Relay Operations
+
+A private, aggregate dashboard for the Orca Relay control and data planes. It reads local `gcloud` and `gh` credentials on the server; credentials and per-user Relay state never enter the browser. One cached `gcloud auth print-access-token` refresh feeds concurrent read-only Google APIs so the collector does not stampede the local credential store.
+
+## Run locally
+
+Prerequisites:
+
+- Node 24 and pnpm 10
+- `gcloud` authenticated for `onorca-cloud` and `onorca-cloud-staging`
+- `gh` authenticated with read access to `stablyai/orca-cloud`
+
+From the repository root:
+
+```sh
+pnpm install
+pnpm ops:relay
+```
+
+Open . The server binds only to loopback and refreshes aggregate data every minute. Production and staging are read-only by default.
+
+The cost panel is a labeled planning estimate. There is currently no Cloud Billing export in either project, so the dashboard cannot claim exact billed spend. GCP Billing remains authoritative.
+
+## Share through Tailscale
+
+Keep the dashboard bound to loopback and let Tailscale provide identity, TLS, and tailnet ACL enforcement:
+
+```sh
+tailscale serve --bg http://127.0.0.1:2455
+tailscale serve status
+```
+
+Share the HTTPS URL printed by `tailscale serve status` with the team. Limit access to the intended operator group in the tailnet ACL. Do not use a public funnel. Stop sharing with:
+
+```sh
+tailscale serve reset
+```
+
+For a persistent host, run `pnpm --filter @orca-cloud/relay-ops build` and supervise `pnpm --filter @orca-cloud/relay-ops start` with the host's normal process manager. The process needs the same non-interactive `gcloud` and `gh` identities.
+
+## Optional staging controls
+
+Controls are intentionally local-only and disabled unless explicitly enabled:
+
+```sh
+RELAY_OPS_ENABLE_STAGING_CONTROLS=1 pnpm ops:relay
+```
+
+Even in this mode the service never changes GCP directly. It dispatches `.github/workflows/power-relay-staging.yml`, preserves the workflow's typed `WAKE_STAGING` / `SLEEP_STAGING` confirmation, and always wakes only configured-admission cells. Requests require the loopback origin and a per-process CSRF token, so controls stay unavailable through the Tailscale view.
+
+## Data and security boundaries
+
+- Browser payloads contain aggregate Monitoring points, resource health, immutable image digests, alert-policy metadata, and workflow metadata.
+- Account IDs, host IDs, device IDs, pairing state, assignment rows, bearer tokens, service-account tokens, startup scripts, secret values, and individual Relay-admin state are excluded.
+- Sleeping staging is inventory-only. Viewing it does not probe or cold-start Cloud Run services and cannot resize empty MIGs.
+- Partial GCP or GitHub failures degrade the affected panel and produce a sanitized warning.
+- Missing cell inventory renders as `Unknown`, never `Sleeping`. After one successful read, transient credential or collector failures retain the last good snapshot and mark it stale.
+- Responses use `no-store`, a restrictive CSP, frame denial, and no-referrer headers.
+
+## Verification
+
+```sh
+pnpm --filter @orca-cloud/relay-ops test
+pnpm --filter @orca-cloud/relay-ops typecheck
+pnpm --filter @orca-cloud/relay-ops build
+```
diff --git a/cloud/apps/relay-ops/package.json b/cloud/apps/relay-ops/package.json
new file mode 100644
index 00000000000..da4f73f8672
--- /dev/null
+++ b/cloud/apps/relay-ops/package.json
@@ -0,0 +1,29 @@
+{
+ "name": "@orca-cloud/relay-ops",
+ "private": true,
+ "version": "0.0.0",
+ "type": "module",
+ "main": "dist/index.js",
+ "scripts": {
+ "build": "pnpm clean && tsc -p tsconfig.build.json && node -e \"require('fs').cpSync('public','dist/public',{recursive:true})\"",
+ "clean": "node -e \"require('fs').rmSync('dist', { recursive: true, force: true })\"",
+ "dev": "tsx watch src/index.ts",
+ "incident:monitor": "tsx src/incident-monitor-cli.ts",
+ "incident:preflight": "tsx src/incident-live-preflight-cli.ts",
+ "lint": "tsc -p tsconfig.json --noEmit",
+ "start": "node dist/index.js",
+ "test": "vitest run",
+ "typecheck": "tsc -p tsconfig.json --noEmit"
+ },
+ "dependencies": {
+ "@hono/node-server": "^1.19.14",
+ "hono": "^4.12.27",
+ "zod": "^3.25.76"
+ },
+ "devDependencies": {
+ "@types/node": "^24.10.0",
+ "tsx": "^4.21.0",
+ "typescript": "^5.9.3",
+ "vitest": "^4.0.8"
+ }
+}
diff --git a/cloud/apps/relay-ops/public/app.js b/cloud/apps/relay-ops/public/app.js
new file mode 100644
index 00000000000..83c072662fb
--- /dev/null
+++ b/cloud/apps/relay-ops/public/app.js
@@ -0,0 +1,273 @@
+const state = { environment: 'production', window: 360, snapshot: null, config: null, loading: false }
+
+const $ = (selector) => document.querySelector(selector)
+const all = (selector) => [...document.querySelectorAll(selector)]
+const escapeHtml = (value) => String(value).replace(/[&<>'"]/g, (character) => ({
+ '&': '&', '<': '<', '>': '>', "'": ''', '"': '"'
+})[character])
+
+function formatNumber(value, maximumFractionDigits = 0) {
+ if (value === null || value === undefined) return '—'
+ return new Intl.NumberFormat('en-US', { notation: value >= 10_000 ? 'compact' : 'standard', maximumFractionDigits }).format(value)
+}
+
+function formatBytes(value) {
+ if (value === null || value === undefined) return '—'
+ const units = ['B', 'KB', 'MB', 'GB', 'TB']
+ let amount = value
+ let unit = 0
+ while (Math.abs(amount) >= 1000 && unit < units.length - 1) { amount /= 1000; unit += 1 }
+ return `${formatNumber(amount, amount < 10 ? 1 : 0)} ${units[unit]}`
+}
+
+function formatMetric(metric, value) {
+ if (value === null || value === undefined) return '—'
+ if (metric.unit === 'bytes') return formatBytes(value)
+ if (metric.unit === 'milliseconds') return `${formatNumber(value, 1)} ms`
+ return formatNumber(value, value < 10 ? 1 : 0)
+}
+
+function timeAgo(value) {
+ const seconds = Math.max(0, Math.round((Date.now() - Date.parse(value)) / 1000))
+ if (seconds < 60) return `${seconds}s ago`
+ if (seconds < 3600) return `${Math.floor(seconds / 60)}m ago`
+ if (seconds < 86400) return `${Math.floor(seconds / 3600)}h ago`
+ return `${Math.floor(seconds / 86400)}d ago`
+}
+
+function shortImage(value) {
+ const digest = value?.match(/sha256:([a-f0-9]{64})/)?.[1]
+ if (digest) return digest.slice(0, 10)
+ const tag = value?.split(':').at(-1)
+ return tag?.slice(0, 16) ?? '—'
+}
+
+function badge(label, healthy) {
+ return `${escapeHtml(label)}`
+}
+
+function renderSummary(snapshot) {
+ const { summary } = snapshot
+ const utilization = summary.poweredCapacity === null
+ ? null
+ : summary.poweredCapacity > 0
+ ? summary.observedConnections / summary.poweredCapacity * 100
+ : 0
+ const items = [
+ ['Observed connections', formatNumber(summary.observedConnections, 1), 'Latest 1-minute aggregate mean'],
+ ['Active relay sessions', formatNumber(summary.observedSplices, 1), `${formatNumber(summary.observedControls, 1)} desktop-control mean`],
+ ['Healthy cells', `${summary.activeCells ?? '—'} / ${summary.totalCells}`, summary.poweredCapacity === null ? 'Cell inventory unavailable' : `${formatNumber(summary.poweredCapacity)} powered request units`],
+ ['Capacity signal', utilization === null ? '—' : `${formatNumber(utilization, 2)}%`, `${formatNumber(summary.configuredCapacity)} configured admission units`]
+ ]
+ $('#summary').innerHTML = items.map(([label, value, caption]) => `
+
+ ${escapeHtml(label)}
+ ${escapeHtml(value)}
+ ${escapeHtml(caption)}
+ `).join('')
+}
+
+function cellState(cell) {
+ if (cell.targetSize === null) return ['Unknown', null]
+ if (cell.targetSize === 0) return ['Sleeping', null]
+ if (cell.backendHealth === 'healthy' && cell.endpoint.ready) return ['Healthy', true]
+ if (cell.stable && cell.backendHealth === 'empty') return ['Starting', null]
+ return ['Attention', false]
+}
+
+function renderTopology(snapshot) {
+ const director = snapshot.resources.director
+ const sleeping = snapshot.resources.cells.every((cell) => cell.targetSize === 0)
+ && snapshot.resources.sql?.activationPolicy === 'NEVER'
+ const directorHealthy = director?.ready && snapshot.resources.directorEndpoint.health
+ $('#topology-meta').textContent = `${snapshot.environment.project} · ${snapshot.environment.region}`
+ const cellNodes = snapshot.resources.cells.map((cell) => {
+ const [label, healthy] = cellState(cell)
+ return `
+ ${escapeHtml(cell.hostname.toUpperCase())} ${badge(label, healthy)}
+ ${escapeHtml(cell.region)} · ${escapeHtml(cell.zone)} · ${formatNumber(cell.capacityRequests)} units
+
`
+ }).join('')
+ $('#topology').innerHTML = `
+ Desktop + phoneEncrypted Relay traffic
+
+ Director ${badge(sleeping ? 'Sleeping' : directorHealthy ? 'Ready' : 'Attention', sleeping ? null : Boolean(directorHealthy))}${escapeHtml(snapshot.environment.directorOrigin)}
+
+ ${cellNodes}
`
+}
+
+function chartPaths(points, width = 400, height = 112) {
+ if (points.length === 0) return null
+ const values = points.map((point) => point.value)
+ const minimum = Math.min(0, ...values)
+ const maximum = Math.max(...values)
+ const spread = maximum - minimum || 1
+ const coordinates = points.map((point, index) => {
+ const x = points.length === 1 ? width : index / (points.length - 1) * width
+ const y = height - ((point.value - minimum) / spread * (height - 10) + 5)
+ return [x, y]
+ })
+ const line = coordinates.map(([x, y], index) => `${index === 0 ? 'M' : 'L'}${x.toFixed(1)},${y.toFixed(1)}`).join(' ')
+ const area = `${line} L${width},${height} L0,${height} Z`
+ return { line, area }
+}
+
+function renderCharts(snapshot) {
+ const names = ['controls', 'splices', 'assignment_5xx', 'postgres_retries', 'auth_failures', 'event_loop_ms_p99']
+ $('#charts').innerHTML = names.map((name) => {
+ const metric = snapshot.monitoring.metrics[name]
+ const paths = chartPaths(metric.points)
+ const graph = paths ? `` : 'No samples in this window
'
+ return `${escapeHtml(metric.label)}
${escapeHtml(formatMetric(metric, metric.latest))}
${escapeHtml(metric.unit)} ${graph}`
+ }).join('')
+}
+
+function renderCells(snapshot) {
+ const controlsByCell = snapshot.monitoring.metrics.controls.latestByCell
+ const splicesByCell = snapshot.monitoring.metrics.splices.latestByCell
+ $('#cells').innerHTML = snapshot.resources.cells.map((cell) => {
+ const [label, healthy] = cellState(cell)
+ const observed = (controlsByCell[cell.cellId] ?? 0) + (splicesByCell[cell.cellId] ?? 0)
+ return `
+ | ${escapeHtml(cell.hostname.toUpperCase())} ${escapeHtml(cell.region)} · ${escapeHtml(cell.zone)} |
+ ${badge(label, healthy)} MIG ${cell.runningInstances ?? '—'}/${cell.targetSize ?? '—'} |
+ ${formatNumber(observed, 1)} 1-minute mean |
+ ${formatNumber(cell.capacityRequests)} DB pool ${formatNumber(cell.databasePoolMax)} · ${cell.configuredAdmission ? 'configured' : 'candidate only'} |
+ ${escapeHtml(shortImage(cell.imageDigest))} |
+
`
+ }).join('')
+}
+
+function serviceRow(name, healthy, detail, suffix = '') {
+ return `${escapeHtml(name)}
${escapeHtml(detail)}
${badge(suffix || (healthy ? 'Ready' : 'Attention'), healthy)}
`
+}
+
+function renderServices(snapshot) {
+ const { resources } = snapshot
+ const sleeping = resources.cells.every((cell) => cell.targetSize === 0)
+ && resources.sql?.activationPolicy === 'NEVER'
+ const certDays = resources.certificate?.expireTime
+ ? Math.floor((Date.parse(resources.certificate.expireTime) - Date.now()) / 86400000)
+ : null
+ $('#services').innerHTML = [
+ serviceRow('Director', sleeping ? null : Boolean(resources.director?.ready && resources.directorEndpoint.health), `${resources.director?.revision ?? 'Revision unavailable'} · ${shortImage(resources.director?.image)}`, sleeping ? 'Sleeping' : ''),
+ serviceRow('Authentication', sleeping ? null : Boolean(resources.auth?.ready && resources.authEndpoint.health), `${resources.auth?.revision ?? 'Revision unavailable'} · ${shortImage(resources.auth?.image)}`, sleeping ? 'Sleeping' : ''),
+ serviceRow('Cloud SQL', resources.sql?.state === 'RUNNABLE' || resources.sql?.activationPolicy === 'NEVER', `${resources.sql?.tier ?? 'unknown'} · ${resources.sql?.activationPolicy ?? 'unknown'}`, resources.sql?.state ?? 'Unknown'),
+ serviceRow('Wildcard TLS', resources.certificate?.state === 'ACTIVE', resources.certificate?.domains.join(', ') ?? 'Certificate unavailable', certDays === null ? 'Unknown' : `${certDays}d`)
+ ].join('')
+}
+
+function renderAlerts(snapshot) {
+ const policies = snapshot.monitoring.alertPolicies
+ $('#alerts').innerHTML = policies.length ? policies.map((policy) => `${escapeHtml(policy.displayName.replace('Orca Relay: ', ''))}
Cloud Monitoring policy
${badge(policy.enabled ? 'Enabled' : 'Disabled', policy.enabled)}
`).join('') : 'No Relay alert policies returned.
'
+}
+
+function renderWorkflows(snapshot) {
+ $('#workflows').innerHTML = snapshot.workflows.length ? snapshot.workflows.slice(0, 6).map((run) => {
+ const healthy = run.conclusion === 'success'
+ const stateLabel = run.status === 'completed' ? (run.conclusion ?? 'completed') : run.status
+ return `${escapeHtml(run.name)}
${escapeHtml(run.headSha)} · ${timeAgo(run.updatedAt)}
${badge(stateLabel, healthy)}`
+ }).join('') : 'Workflow history unavailable.
'
+}
+
+function renderCost(snapshot) {
+ const cost = snapshot.cost
+ $('#cost').innerHTML = `$${formatNumber(cost.monthlyUsd)}/ month
+ Modeled range $${formatNumber(cost.rangeUsd[0])}–$${formatNumber(cost.rangeUsd[1])}. Not billed cost.
+ ${cost.lines.map((line) => `
${escapeHtml(line.label)}$${formatNumber(line.monthlyUsd, 2)}
`).join('')}
+ Exact billing: ${escapeHtml(cost.actualBilling.reason)}
${escapeHtml(cost.caveats[1])}
`
+}
+
+function renderWarnings(snapshot) {
+ const warning = $('#warnings')
+ warning.classList.toggle('hidden', snapshot.warnings.length === 0)
+ warning.textContent = snapshot.warnings.length ? `Partial data: ${snapshot.warnings.join(' ')}` : ''
+}
+
+function render(snapshot) {
+ state.snapshot = snapshot
+ $('#environment-label').textContent = `${snapshot.environment.label} · ${snapshot.environment.region}`
+ $('#freshness').textContent = snapshot.stale
+ ? `Last good update ${timeAgo(snapshot.generatedAt)} · refresh degraded`
+ : `Updated ${timeAgo(snapshot.generatedAt)}`
+ $('#freshness-dot').className = snapshot.stale ? 'status-dot neutral' : 'status-dot healthy'
+ renderWarnings(snapshot)
+ renderSummary(snapshot)
+ renderTopology(snapshot)
+ renderCharts(snapshot)
+ renderCells(snapshot)
+ renderServices(snapshot)
+ renderAlerts(snapshot)
+ renderWorkflows(snapshot)
+ renderCost(snapshot)
+ $('#power-panel').classList.toggle('hidden', !(state.environment === 'staging' && state.config?.stagingControlsEnabled))
+ $('#compute-link').href = snapshot.environment.consoleLinks.compute
+ $('#alert-link').href = snapshot.environment.consoleLinks.alerts
+}
+
+async function loadSnapshot() {
+ if (state.loading) return
+ state.loading = true
+ $('#refresh').disabled = true
+ $('#error').classList.add('hidden')
+ $('#freshness-dot').className = 'status-dot neutral'
+ $('#freshness').textContent = 'Refreshing current state…'
+ try {
+ const response = await fetch(`/api/snapshot?environment=${state.environment}&window=${state.window}`)
+ const body = await response.json()
+ if (!response.ok) throw new Error(body.error ?? 'Snapshot failed')
+ render(body)
+ } catch (error) {
+ $('#freshness-dot').className = 'status-dot unhealthy'
+ $('#freshness').textContent = 'Refresh failed'
+ $('#error').textContent = error instanceof Error ? error.message : 'Relay operations data is unavailable.'
+ $('#error').classList.remove('hidden')
+ } finally {
+ state.loading = false
+ $('#refresh').disabled = false
+ }
+}
+
+async function dispatchPower(mode) {
+ const confirmation = mode === 'wake' ? 'WAKE_STAGING' : mode === 'sleep' ? 'SLEEP_STAGING' : ''
+ if (confirmation) {
+ const entered = window.prompt(`Type ${confirmation} to dispatch the guarded workflow.`) ?? ''
+ if (entered !== confirmation) return
+ }
+ all('[data-power]').forEach((button) => { button.disabled = true })
+ $('#power-result').textContent = 'Dispatching…'
+ try {
+ const response = await fetch('/api/staging/power', {
+ method: 'POST',
+ headers: { 'content-type': 'application/json', 'x-csrf-token': state.config.csrfToken },
+ body: JSON.stringify({ mode, confirmation })
+ })
+ const body = await response.json()
+ if (!response.ok) throw new Error(body.error)
+ $('#power-result').textContent = 'Workflow accepted. Refresh after it completes.'
+ } catch (error) {
+ $('#power-result').textContent = error instanceof Error ? error.message : 'Dispatch failed.'
+ } finally {
+ all('[data-power]').forEach((button) => { button.disabled = false })
+ }
+}
+
+all('[data-environment]').forEach((button) => button.addEventListener('click', () => {
+ state.environment = button.dataset.environment
+ all('[data-environment]').forEach((candidate) => candidate.setAttribute('aria-pressed', String(candidate === button)))
+ loadSnapshot()
+}))
+$('#window').addEventListener('change', (event) => { state.window = Number(event.target.value); loadSnapshot() })
+$('#refresh').addEventListener('click', loadSnapshot)
+all('[data-power]').forEach((button) => button.addEventListener('click', () => dispatchPower(button.dataset.power)))
+
+async function start() {
+ try { state.config = await fetch('/api/config').then((response) => response.json()) } catch { state.config = {} }
+ await loadSnapshot()
+ window.setInterval(loadSnapshot, 60_000)
+}
+
+start()
diff --git a/cloud/apps/relay-ops/public/index.html b/cloud/apps/relay-ops/public/index.html
new file mode 100644
index 00000000000..c49c3bee579
--- /dev/null
+++ b/cloud/apps/relay-ops/public/index.html
@@ -0,0 +1,115 @@
+
+
+
+
+
+
+ Orca Relay Operations
+
+
+
+
+
+
+
+
+
+
+
Production · us-central1
+
Relay data plane
+
Private, aggregate operations view. No account, host, device, or pairing identifiers.
+
+
+
+ Loading current state…
+
+
+
+
+
+
+
+
+
+
+ Traffic
Signals in selected window
+
+
+
+
+
+ Configured admission is shown below. Live director admission and heartbeat state stays unavailable because this dashboard has no Relay-admin identity.
+
+ | Cell | State | Observed | Capacity | Image |
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Staging only
Power workflow
+
GitHub guarded
+
+ Dispatches the reviewed GitHub workflow. It never changes GCP resources directly from this process.
+
+
+
+
+
+
+
+
+
+
+
diff --git a/cloud/apps/relay-ops/public/styles.css b/cloud/apps/relay-ops/public/styles.css
new file mode 100644
index 00000000000..17a974c3d9a
--- /dev/null
+++ b/cloud/apps/relay-ops/public/styles.css
@@ -0,0 +1,165 @@
+:root {
+ --radius: 10px;
+ --background: #fff;
+ --foreground: #0a0a0a;
+ --card: #fff;
+ --primary: #171717;
+ --primary-foreground: #fafafa;
+ --secondary: #f5f5f5;
+ --muted: #f5f5f5;
+ --muted-foreground: #737373;
+ --accent: #f5f5f5;
+ --border: #e5e5e5;
+ --ring: #a1a1a1;
+ --destructive: #e40014;
+ --success: #15803d;
+ --warning: #895503;
+ --font-mono: 'SF Mono', SFMono-Regular, ui-monospace, Menlo, Consolas, monospace;
+}
+@media (prefers-color-scheme: dark) {
+ :root {
+ --background: #0a0a0a;
+ --foreground: #fafafa;
+ --card: #171717;
+ --primary: #e5e5e5;
+ --primary-foreground: #171717;
+ --secondary: #262626;
+ --muted: #262626;
+ --muted-foreground: #a1a1a1;
+ --accent: #262626;
+ --border: rgb(255 255 255 / 0.07);
+ --ring: #737373;
+ --destructive: #ff6568;
+ --success: #4ade80;
+ --warning: #fbbf24;
+ }
+}
+* { box-sizing: border-box; }
+body {
+ margin: 0;
+ background: var(--background);
+ color: var(--foreground);
+ font-family: 'Geist', -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
+ font-size: 14px;
+ letter-spacing: .01em;
+}
+button, select { font: inherit; }
+button:focus-visible, select:focus-visible { outline: 2px solid var(--ring); outline-offset: 2px; }
+.topbar {
+ height: 64px;
+ border-bottom: 1px solid var(--border);
+ padding: 0 max(24px, calc((100vw - 1440px) / 2));
+ display: flex;
+ align-items: center;
+ justify-content: space-between;
+ position: sticky;
+ top: 0;
+ background: color-mix(in srgb, var(--background) 92%, transparent);
+ backdrop-filter: blur(16px);
+ z-index: 10;
+}
+.brand { display: flex; align-items: center; gap: 10px; }
+.brand-mark {
+ width: 30px; height: 30px; border-radius: 9px; background: var(--primary);
+ color: var(--primary-foreground); display: grid; place-items: center; font-weight: 700;
+}
+.brand div { display: flex; align-items: baseline; gap: 7px; }
+.brand span:last-child { color: var(--muted-foreground); font-size: 12px; }
+.toolbar { display: flex; align-items: center; gap: 8px; }
+.segmented { display: flex; padding: 3px; gap: 2px; border-radius: 8px; background: var(--secondary); }
+.segmented button { border: 0; background: transparent; color: var(--muted-foreground); padding: 5px 10px; border-radius: 6px; cursor: pointer; }
+.segmented button[aria-pressed="true"] { background: var(--card); color: var(--foreground); box-shadow: 0 1px 2px rgb(0 0 0 / .08); }
+select, .button { height: 32px; border: 1px solid var(--border); border-radius: 8px; background: var(--card); color: var(--foreground); padding: 0 10px; }
+.button { cursor: pointer; font-weight: 500; }
+.button:hover { background: var(--accent); }
+.button:disabled { opacity: .5; cursor: wait; }
+main { max-width: 1440px; margin: 0 auto; padding: 42px 24px 72px; }
+.page-heading { display: flex; justify-content: space-between; align-items: end; gap: 24px; margin-bottom: 28px; }
+h1, h2, p { margin: 0; }
+h1 { font-size: 28px; line-height: 1.2; letter-spacing: -.025em; margin-top: 5px; }
+h2 { font-size: 16px; line-height: 1.25; letter-spacing: -.01em; margin-top: 3px; }
+.lede { color: var(--muted-foreground); margin-top: 8px; max-width: 680px; }
+.eyebrow { color: var(--muted-foreground); font-size: 11px; line-height: 1; font-weight: 600; text-transform: uppercase; letter-spacing: .05em; }
+.freshness { color: var(--muted-foreground); display: flex; align-items: center; gap: 8px; font-size: 12px; white-space: nowrap; }
+.status-dot { width: 7px; height: 7px; border-radius: 999px; background: var(--ring); display: inline-block; }
+.status-dot.healthy { background: var(--success); }
+.status-dot.unhealthy { background: var(--destructive); }
+.summary-grid { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 12px; margin-bottom: 12px; }
+.metric-card, .panel { border: 1px solid var(--border); border-radius: 14px; background: var(--card); }
+.metric-card { padding: 18px; min-height: 116px; }
+.metric-card .value { font-size: 29px; font-weight: 600; letter-spacing: -.035em; margin-top: 16px; }
+.metric-card .caption { color: var(--muted-foreground); font-size: 12px; margin-top: 3px; }
+.skeleton { background: linear-gradient(90deg, var(--card), var(--muted), var(--card)); background-size: 200% 100%; animation: shimmer 1.6s infinite; }
+@keyframes shimmer { to { background-position: -200% 0; } }
+.panel { padding: 18px; }
+.topology-panel { margin-bottom: 36px; }
+.panel-heading, .section-heading { display: flex; align-items: center; justify-content: space-between; gap: 16px; margin-bottom: 18px; }
+.section-heading { margin-top: 34px; }
+.topology { display: grid; grid-template-columns: 1fr 48px 1fr 48px 2fr; align-items: stretch; gap: 8px; }
+.topology-node { border: 1px solid var(--border); background: var(--background); border-radius: 10px; padding: 14px; min-width: 0; }
+.topology-node strong { display: block; font-size: 13px; }
+.topology-node span { display: block; color: var(--muted-foreground); font-size: 12px; margin-top: 4px; overflow: hidden; text-overflow: ellipsis; }
+.topology-cells { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 8px; }
+.connector { display: grid; place-items: center; color: var(--muted-foreground); }
+.connector::before { content: ''; width: 100%; height: 1px; background: var(--border); }
+.chart-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 12px; margin-bottom: 36px; }
+.chart-card { min-height: 220px; }
+.chart-head { display: flex; justify-content: space-between; align-items: start; }
+.chart-value { font-size: 22px; font-weight: 600; letter-spacing: -.03em; }
+.chart-unit { color: var(--muted-foreground); font-size: 11px; }
+.chart { width: 100%; height: 122px; margin-top: 18px; overflow: visible; }
+.chart path.area { fill: color-mix(in srgb, var(--foreground) 5%, transparent); }
+.chart path.line { fill: none; stroke: var(--foreground); stroke-width: 1.5; vector-effect: non-scaling-stroke; }
+.chart line { stroke: var(--border); stroke-width: 1; }
+.chart-empty { color: var(--muted-foreground); height: 120px; display: grid; place-items: center; font-size: 12px; }
+.two-column { display: grid; grid-template-columns: 1.7fr 1fr; gap: 12px; margin-bottom: 12px; }
+.three-column { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 12px; margin-bottom: 12px; }
+.two-column > *, .three-column > *, .chart-grid > * { min-width: 0; }
+.table-wrap { overflow-x: auto; }
+table { width: 100%; border-collapse: collapse; font-size: 12px; }
+th { color: var(--muted-foreground); font-weight: 500; text-align: left; padding: 0 10px 10px; }
+td { padding: 12px 10px; border-top: 1px solid var(--border); vertical-align: middle; }
+td:first-child, th:first-child { padding-left: 0; }
+td:last-child, th:last-child { padding-right: 0; }
+.mono { font-family: var(--font-mono); font-size: 11px; }
+.badge { display: inline-flex; align-items: center; border: 1px solid var(--border); border-radius: 999px; padding: 2px 7px; font-size: 11px; color: var(--muted-foreground); }
+.badge.healthy { color: var(--success); border-color: color-mix(in srgb, var(--success) 25%, var(--border)); }
+.badge.unhealthy { color: var(--destructive); border-color: color-mix(in srgb, var(--destructive) 25%, var(--border)); }
+.service-list, .list { display: grid; }
+.service-row, .list-row { padding: 11px 0; border-top: 1px solid var(--border); display: flex; align-items: center; justify-content: space-between; gap: 12px; min-width: 0; }
+.service-row:first-child, .list-row:first-child { border-top: 0; padding-top: 0; }
+.service-row:last-child, .list-row:last-child { padding-bottom: 0; }
+.row-title { font-size: 12px; font-weight: 500; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
+.row-caption, .meta, .muted { color: var(--muted-foreground); font-size: 11px; }
+.panel-note { color: var(--muted-foreground); font-size: 11px; line-height: 1.45; margin: -8px 0 14px; }
+.panel-link:hover { color: var(--foreground); text-decoration: underline; }
+.row-caption { margin-top: 3px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
+a { color: inherit; text-decoration: none; }
+a:hover .row-title { text-decoration: underline; }
+.cost-total { display: flex; align-items: baseline; gap: 7px; margin-bottom: 12px; }
+.cost-total strong { font-size: 28px; letter-spacing: -.035em; }
+.cost-line { display: flex; justify-content: space-between; gap: 10px; padding: 7px 0; border-top: 1px solid var(--border); font-size: 12px; }
+.cost-note { color: var(--muted-foreground); font-size: 11px; line-height: 1.5; margin-top: 12px; }
+.banner { border: 1px solid var(--border); border-radius: 10px; padding: 11px 13px; margin-bottom: 12px; font-size: 12px; }
+.banner.error { color: var(--destructive); }
+.banner.warning { color: var(--warning); }
+.hidden { display: none !important; }
+.power-actions { display: flex; align-items: center; gap: 8px; margin-top: 14px; flex-wrap: wrap; }
+footer { border-top: 1px solid var(--border); padding: 24px; color: var(--muted-foreground); font-size: 11px; text-align: center; }
+@media (max-width: 1050px) {
+ .summary-grid, .chart-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); }
+ .three-column { grid-template-columns: 1fr; }
+ .topology { grid-template-columns: 1fr; }
+ .connector { height: 20px; }
+ .connector::before { height: 100%; width: 1px; }
+}
+@media (max-width: 760px) {
+ .topbar { height: auto; min-height: 64px; padding: 12px 16px; align-items: stretch; flex-direction: column; gap: 12px; }
+ .brand div { display: grid; gap: 0; }
+ .toolbar { flex-wrap: wrap; justify-content: flex-start; }
+ main { padding: 28px 16px 56px; }
+ .page-heading { align-items: flex-start; flex-direction: column; }
+ .summary-grid, .chart-grid, .two-column { grid-template-columns: 1fr; }
+ .topology-cells { grid-template-columns: 1fr; }
+ table { min-width: 600px; }
+}
diff --git a/cloud/apps/relay-ops/src/cost-model.test.ts b/cloud/apps/relay-ops/src/cost-model.test.ts
new file mode 100644
index 00000000000..9ee6f6adf9d
--- /dev/null
+++ b/cloud/apps/relay-ops/src/cost-model.test.ts
@@ -0,0 +1,85 @@
+import { describe, expect, it } from 'vitest'
+import { buildCostModel } from './cost-model.js'
+import {
+ RELAY_OPS_ENVIRONMENTS,
+ relayOpsCellsFromTerraform,
+ type RelayOpsCellConfig
+} from './environment-config.js'
+import type { ResourceInventory } from './resource-inventory.js'
+
+const regionalCellsSource = `
+relay_gce_cells = {
+ "staging-gce-c3" = {
+ hostname = "c3"
+ zone = "us-central1-a"
+ machine_type = "e2-standard-2"
+ capacity_requests = 4000
+ }
+ "staging-gce-c4" = {
+ hostname = "c4"
+ region = "asia-east2"
+ zone = "asia-east2-a"
+ machine_type = "e2-standard-4"
+ capacity_requests = 6000
+ database_pool_max = 10
+ initially_enabled = false
+ }
+}
+`
+
+function inventory(
+ targetSize: number,
+ activationPolicy: string,
+ cells: RelayOpsCellConfig[] = RELAY_OPS_ENVIRONMENTS.staging.cells
+): ResourceInventory {
+ return {
+ director: null,
+ auth: null,
+ sql: { state: targetSize ? 'RUNNABLE' : 'STOPPED', activationPolicy, tier: 'db-custom-1-3840', availabilityType: 'ZONAL', databaseVersion: 'POSTGRES_17' },
+ certificate: null,
+ directorEndpoint: { health: null, ready: null, latencyMs: null },
+ authEndpoint: { health: null, ready: null, latencyMs: null },
+ cells: cells.map((cell) => ({
+ ...cell, migName: `mig-${cell.hostname}`, targetSize, runningInstances: targetSize,
+ stable: true, template: 'template', imageDigest: null,
+ backendHealth: targetSize ? 'healthy' : 'empty',
+ endpoint: { health: null, ready: null, latencyMs: null }
+ })),
+ warnings: []
+ }
+}
+
+describe('buildCostModel', () => {
+ it('shows the sleeping staging floor without VM or SQL compute', () => {
+ const result = buildCostModel(RELAY_OPS_ENVIRONMENTS.staging, inventory(0, 'NEVER'))
+ expect(result.kind).toBe('planning-estimate')
+ expect(result.monthlyUsd).toBe(34)
+ expect(result.lines.find((line) => line.label === 'Relay cell VMs')?.monthlyUsd).toBe(0)
+ expect(result.actualBilling.available).toBe(false)
+ expect(result.caveats[0]).toContain('not the Cloud Billing invoice')
+ })
+
+ it('prices durable machine inventory and network floors by region', () => {
+ const cells = relayOpsCellsFromTerraform({
+ environment: 'staging',
+ domain: 'relay-staging.onorca.dev',
+ source: regionalCellsSource
+ })
+ const environment = { ...RELAY_OPS_ENVIRONMENTS.staging, cells }
+ const result = buildCostModel(environment, inventory(1, 'NEVER', cells))
+ const machines = result.lines.find((line) => line.label === 'Relay cell VMs')
+ const network = result.lines.find((line) => line.label === 'Load balancer and network floor')
+
+ expect(machines).toEqual({
+ label: 'Relay cell VMs',
+ monthlyUsd: 185.79,
+ basis: '2 configured VM cells at regional machine rates × 730 hours'
+ })
+
+ expect(network).toEqual({
+ label: 'Load balancer and network floor',
+ monthlyUsd: 29,
+ basis: 'shared HTTPS foundation plus NAT floor in 2 configured regions'
+ })
+ })
+})
diff --git a/cloud/apps/relay-ops/src/cost-model.ts b/cloud/apps/relay-ops/src/cost-model.ts
new file mode 100644
index 00000000000..39e062597a8
--- /dev/null
+++ b/cloud/apps/relay-ops/src/cost-model.ts
@@ -0,0 +1,115 @@
+import type {
+ RelayOpsEnvironment,
+ RelayOpsMachineType,
+ RelayOpsRegion
+} from './environment-config.js'
+import type { ResourceInventory } from './resource-inventory.js'
+
+export type CostLine = {
+ label: string
+ monthlyUsd: number
+ basis: string
+}
+
+export type CostModel = {
+ kind: 'planning-estimate'
+ monthlyUsd: number
+ rangeUsd: [number, number]
+ actualBilling: { available: false; reason: string }
+ lines: CostLine[]
+ caveats: string[]
+}
+
+const HOURS_PER_MONTH = 730
+const MACHINE_HOURLY_USD: Record<
+ RelayOpsRegion,
+ Record
+> = {
+ 'us-central1': {
+ 'e2-standard-2': 0.06701142,
+ 'e2-standard-4': 0.13402284
+ },
+ 'asia-east2': {
+ 'e2-standard-2': 0.0938,
+ 'e2-standard-4': 0.1875
+ }
+}
+const SHARED_NETWORK_FLOOR_USD = 19
+const REGIONAL_NAT_FLOOR_USD = 5
+
+function round(value: number): number {
+ return Math.round(value * 100) / 100
+}
+
+export function buildCostModel(
+ environment: RelayOpsEnvironment,
+ resources: ResourceInventory
+): CostModel {
+ const runningCells = resources.cells.filter((cell) => (cell.targetSize ?? 0) > 0)
+ const runningCellCount = runningCells.reduce((sum, cell) => sum + (cell.targetSize ?? 0), 0)
+ const compute = runningCells.reduce(
+ (sum, cell) =>
+ sum + (cell.targetSize ?? 0) * MACHINE_HOURLY_USD[cell.region][cell.machineType],
+ 0
+ ) * HOURS_PER_MONTH
+ const disks = runningCellCount * 30 * 0.1
+ const sqlRunning = resources.sql?.activationPolicy === 'ALWAYS'
+ const sql = sqlRunning ? (environment.id === 'production' ? 105 : 52) : 0
+ const cloudRunMinimums =
+ (resources.director?.minInstances ?? 0) + (resources.auth?.minInstances ?? 0)
+ const cloudRun = cloudRunMinimums * 10
+ const configuredRegions = new Set(
+ (resources.cells.length > 0 ? resources.cells : environment.cells).map((cell) => cell.region)
+ )
+ const networkFoundation =
+ SHARED_NETWORK_FLOOR_USD + configuredRegions.size * REGIONAL_NAT_FLOOR_USD
+ const observability = environment.id === 'production' ? 12 : 5
+ const lines: CostLine[] = [
+ {
+ label: 'Relay cell VMs',
+ monthlyUsd: round(compute),
+ basis: `${runningCellCount} configured VM cells at regional machine rates × 730 hours`
+ },
+ {
+ label: 'Cell boot disks',
+ monthlyUsd: round(disks),
+ basis: `${runningCellCount} × 30 GB balanced persistent disk`
+ },
+ {
+ label: 'Cloud SQL',
+ monthlyUsd: sql,
+ basis: sqlRunning ? `${resources.sql?.tier ?? 'configured tier'} active` : 'stopped'
+ },
+ {
+ label: 'Cloud Run minimums',
+ monthlyUsd: cloudRun,
+ basis: `${cloudRunMinimums} configured minimum instances`
+ },
+ {
+ label: 'Load balancer and network floor',
+ monthlyUsd: networkFoundation,
+ basis: `shared HTTPS foundation plus NAT floor in ${configuredRegions.size} configured region${configuredRegions.size === 1 ? '' : 's'}`
+ },
+ {
+ label: 'Logs and monitoring allowance',
+ monthlyUsd: observability,
+ basis: 'planning allowance; varies with traffic and retention'
+ }
+ ]
+ const monthlyUsd = round(lines.reduce((sum, line) => sum + line.monthlyUsd, 0))
+ return {
+ kind: 'planning-estimate',
+ monthlyUsd,
+ rangeUsd: [round(monthlyUsd * 0.85), round(monthlyUsd * 1.3)],
+ actualBilling: {
+ available: false,
+ reason: 'Cloud Billing export is not configured in either Relay project.'
+ },
+ lines,
+ caveats: [
+ 'This is a modeled run-rate, not the Cloud Billing invoice.',
+ 'Network egress, actual request traffic, credits, discounts, taxes, and free tiers are excluded.',
+ 'Use the GCP Billing report for authoritative spend and forecasts.'
+ ]
+ }
+}
diff --git a/cloud/apps/relay-ops/src/dashboard-snapshot.test.ts b/cloud/apps/relay-ops/src/dashboard-snapshot.test.ts
new file mode 100644
index 00000000000..247b4b53ed0
--- /dev/null
+++ b/cloud/apps/relay-ops/src/dashboard-snapshot.test.ts
@@ -0,0 +1,62 @@
+import { describe, expect, it } from 'vitest'
+import { DashboardSnapshotCache } from './dashboard-snapshot.js'
+import type { DashboardSnapshot } from './dashboard-snapshot.js'
+import type { GcloudClient } from './gcloud-client.js'
+
+const gcloud: GcloudClient = { accessToken: async () => 'a'.repeat(40) }
+
+function snapshot(kind: 'good' | 'unavailable'): DashboardSnapshot {
+ const good = kind === 'good'
+ return {
+ generatedAt: '2026-07-15T12:00:00.000Z',
+ resources: {
+ director: good ? {} : null,
+ auth: good ? {} : null,
+ sql: good ? {} : null,
+ cells: [{ targetSize: good ? 1 : null }]
+ },
+ monitoring: {
+ warnings: good
+ ? []
+ : ['Cloud Monitoring credentials are unavailable. Run gcloud auth login.']
+ },
+ summary: { observedConnections: good ? 7 : 0 },
+ warnings: good ? [] : ['Google Cloud credentials are unavailable. Run gcloud auth login.'],
+ stale: false,
+ staleReason: null
+ } as unknown as DashboardSnapshot
+}
+
+describe('DashboardSnapshotCache', () => {
+ it('keeps the last good view when a later credential refresh fails', async () => {
+ let calls = 0
+ const cache = new DashboardSnapshotCache(gcloud, 0, async () => {
+ calls += 1
+ return snapshot(calls === 1 ? 'good' : 'unavailable')
+ })
+
+ const first = await cache.read('production', 30)
+ const second = await cache.read('production', 31)
+
+ expect(first.stale).toBe(false)
+ expect(second.stale).toBe(true)
+ expect(second.summary.observedConnections).toBe(7)
+ expect(second.staleReason).toContain('credentials')
+ })
+
+ it('keeps the last good view when a later collector throws', async () => {
+ let calls = 0
+ const cache = new DashboardSnapshotCache(gcloud, 0, async () => {
+ calls += 1
+ if (calls > 1) throw new Error('sensitive collector context')
+ return snapshot('good')
+ })
+
+ await cache.read('production', 30)
+ const second = await cache.read('production', 31)
+
+ expect(second.stale).toBe(true)
+ expect(second.summary.observedConnections).toBe(7)
+ expect(JSON.stringify(second)).not.toContain('sensitive collector context')
+ })
+})
diff --git a/cloud/apps/relay-ops/src/dashboard-snapshot.ts b/cloud/apps/relay-ops/src/dashboard-snapshot.ts
new file mode 100644
index 00000000000..0d6f0c1151b
--- /dev/null
+++ b/cloud/apps/relay-ops/src/dashboard-snapshot.ts
@@ -0,0 +1,167 @@
+import type { RelayOpsEnvironmentId } from './environment-config.js'
+import { relayOpsEnvironment } from './environment-config.js'
+import type { GcloudClient } from './gcloud-client.js'
+import { readRelayWorkflowRuns } from './github-runs.js'
+import { readMonitoringSnapshot } from './monitoring-snapshot.js'
+import { readResourceInventory } from './resource-inventory.js'
+import { buildCostModel } from './cost-model.js'
+
+export type DashboardSnapshot = Awaited>
+
+export async function buildDashboardSnapshot(
+ environmentId: RelayOpsEnvironmentId,
+ gcloud: GcloudClient,
+ options: { windowMinutes?: number; fetchImpl?: typeof fetch; now?: Date } = {}
+) {
+ const generatedAt = (options.now ?? new Date()).toISOString()
+ const environment = relayOpsEnvironment(environmentId)
+ const [monitoringResult, resourceResult, workflowResult] = await Promise.allSettled([
+ readMonitoringSnapshot(environment, gcloud, {
+ ...(options.now === undefined ? {} : { now: options.now }),
+ ...(options.windowMinutes === undefined ? {} : { windowMinutes: options.windowMinutes }),
+ ...(options.fetchImpl === undefined ? {} : { fetchImpl: options.fetchImpl })
+ }),
+ readResourceInventory(environment, gcloud, options.fetchImpl),
+ readRelayWorkflowRuns()
+ ])
+ if (monitoringResult.status === 'rejected' || resourceResult.status === 'rejected') {
+ const failed = [
+ monitoringResult.status === 'rejected' ? 'Monitoring snapshot' : null,
+ resourceResult.status === 'rejected' ? 'Resource inventory' : null
+ ].filter(Boolean)
+ throw new Error(`${failed.join(' and ')} unavailable`)
+ }
+ const resources = resourceResult.value
+ const monitoring = monitoringResult.value
+ const warnings = [...resources.warnings, ...monitoring.warnings]
+ const poweredDigests = new Set(
+ resources.cells.filter((cell) => (cell.targetSize ?? 0) > 0).map((cell) => cell.imageDigest)
+ )
+ if (poweredDigests.has(null)) warnings.push('A powered cell image digest is unavailable.')
+ if (poweredDigests.size > 1) warnings.push('Powered cells are not serving one immutable digest.')
+ const expectedCertificateDomain = `*.${new URL(environment.cells[0]!.origin).hostname
+ .split('.').slice(1).join('.')}`
+ if (resources.certificate && !resources.certificate.domains.includes(expectedCertificateDomain)) {
+ warnings.push('The Relay certificate domain does not match the configured cell domain.')
+ }
+ if (workflowResult.status === 'rejected') warnings.push('GitHub workflow history is unavailable.')
+ const observedConnections = monitoring.metrics.total_connections.latest ?? 0
+ const observedControls = monitoring.metrics.controls.latest ?? 0
+ const observedSplices = monitoring.metrics.splices.latest ?? 0
+ const configuredCapacity = environment.cells
+ .filter((cell) => cell.configuredAdmission)
+ .reduce((sum, cell) => sum + cell.capacityRequests, 0)
+ const cellInventoryAvailable = resources.cells.every((cell) => cell.targetSize !== null)
+ const poweredCapacity = cellInventoryAvailable
+ ? resources.cells
+ .filter((cell) => (cell.targetSize ?? 0) > 0)
+ .reduce((sum, cell) => sum + cell.capacityRequests, 0)
+ : null
+ return {
+ schemaVersion: 1,
+ generatedAt,
+ environment: {
+ id: environment.id,
+ label: environment.label,
+ project: environment.project,
+ region: environment.region,
+ directorOrigin: environment.directorOrigin,
+ authOrigin: environment.authOrigin,
+ consoleLinks: {
+ project: `https://console.cloud.google.com/home/dashboard?project=${environment.project}`,
+ alerts: `https://console.cloud.google.com/monitoring/alerting?project=${environment.project}`,
+ compute: `https://console.cloud.google.com/compute/instanceGroups/list?project=${environment.project}`
+ }
+ },
+ summary: {
+ observedConnections,
+ observedControls,
+ observedSplices,
+ configuredCapacity,
+ poweredCapacity,
+ activeCells: cellInventoryAvailable
+ ? resources.cells.filter(
+ (cell) => (cell.targetSize ?? 0) > 0 && cell.backendHealth === 'healthy'
+ ).length
+ : null,
+ totalCells: resources.cells.length
+ },
+ resources,
+ monitoring,
+ workflows: workflowResult.status === 'fulfilled' ? workflowResult.value : [],
+ cost: buildCostModel(environment, resources),
+ warnings,
+ stale: false,
+ staleReason: null as string | null
+ }
+}
+
+type SnapshotCacheEntry = { snapshot: DashboardSnapshot; expiresAt: number }
+type SnapshotBuilder = (
+ environment: RelayOpsEnvironmentId,
+ gcloud: GcloudClient,
+ options: { windowMinutes?: number }
+) => Promise
+
+export class DashboardSnapshotCache {
+ private readonly entries = new Map()
+ private readonly pending = new Map>()
+ private readonly lastGood = new Map()
+
+ constructor(
+ private readonly gcloud: GcloudClient,
+ private readonly ttlMs = 30_000,
+ private readonly builder: SnapshotBuilder = buildDashboardSnapshot
+ ) {}
+
+ async read(environment: RelayOpsEnvironmentId, windowMinutes: number): Promise {
+ const key = `${environment}:${windowMinutes}`
+ const cached = this.entries.get(key)
+ if (cached && cached.expiresAt > Date.now()) return cached.snapshot
+ const existing = this.pending.get(key)
+ if (existing) return await existing
+ const request = this.builder(environment, this.gcloud, { windowMinutes })
+ .then((snapshot) => {
+ const coreInventoryUnavailable =
+ snapshot.resources.director === null &&
+ snapshot.resources.auth === null &&
+ snapshot.resources.sql === null &&
+ snapshot.resources.cells.every((cell) => cell.targetSize === null)
+ const monitoringCredentialsUnavailable = snapshot.monitoring.warnings.some(
+ (warning) => warning.includes('credentials are unavailable')
+ )
+ const lastGood = this.lastGood.get(environment)
+ const result = (coreInventoryUnavailable || monitoringCredentialsUnavailable) && lastGood
+ ? {
+ ...lastGood,
+ stale: true,
+ staleReason: 'Local Google Cloud credentials are temporarily unavailable.',
+ warnings: [...new Set([...lastGood.warnings, ...snapshot.warnings])]
+ }
+ : snapshot
+ if (!coreInventoryUnavailable && !monitoringCredentialsUnavailable) {
+ this.lastGood.set(environment, snapshot)
+ }
+ this.entries.set(key, { snapshot: result, expiresAt: Date.now() + this.ttlMs })
+ return result
+ })
+ .catch((error: unknown) => {
+ const lastGood = this.lastGood.get(environment)
+ if (!lastGood) throw error
+ const stale = {
+ ...lastGood,
+ stale: true,
+ staleReason: 'The latest operations refresh failed; showing the last good snapshot.',
+ warnings: [...new Set([
+ ...lastGood.warnings,
+ 'The latest operations refresh failed before a complete snapshot was available.'
+ ])]
+ }
+ this.entries.set(key, { snapshot: stale, expiresAt: Date.now() + this.ttlMs })
+ return stale
+ })
+ .finally(() => this.pending.delete(key))
+ this.pending.set(key, request)
+ return await request
+ }
+}
diff --git a/cloud/apps/relay-ops/src/environment-config.test.ts b/cloud/apps/relay-ops/src/environment-config.test.ts
new file mode 100644
index 00000000000..321167d7cc8
--- /dev/null
+++ b/cloud/apps/relay-ops/src/environment-config.test.ts
@@ -0,0 +1,148 @@
+import { readFileSync } from 'node:fs'
+import { fileURLToPath } from 'node:url'
+import { describe, expect, it } from 'vitest'
+import {
+ RELAY_OPS_ENVIRONMENTS,
+ relayOpsCellsFromTerraform
+} from './environment-config.js'
+
+const durableUsCell = `
+ "production-gce-c26" = {
+ hostname = "c26"
+ zone = "us-central1-a"
+ machine_type = "e2-standard-4"
+ capacity_requests = 4000
+ initially_enabled = false
+ }
+`
+
+const durableAsiaCells = `
+ "production-gce-c27" = {
+ hostname = "c27"
+ region = "asia-east2"
+ zone = "asia-east2-a"
+ machine_type = "e2-standard-4"
+ capacity_requests = 6000
+ database_pool_max = 10
+ initially_enabled = false
+ }
+ "production-gce-c28" = {
+ hostname = "c28"
+ region = "asia-east2"
+ zone = "asia-east2-b"
+ machine_type = "e2-standard-4"
+ capacity_requests = 6000
+ database_pool_max = 10
+ initially_enabled = false
+ }
+ "production-gce-c29" = {
+ hostname = "c29"
+ region = "asia-east2"
+ zone = "asia-east2-c"
+ machine_type = "e2-standard-4"
+ capacity_requests = 6000
+ database_pool_max = 10
+ initially_enabled = false
+ }
+`
+
+const durableAsiaSource = `
+relay_gce_cells = {${durableUsCell}${durableAsiaCells}}
+`
+
+const durableUsOnlySource = `
+relay_gce_cells = {${durableUsCell}}
+`
+
+// Why: relay-ops sat at 18 cells for four days after C19-C22 shipped, which threw
+// `selector membership must contain every configured cell exactly once` and blocked
+// every production mutation. Reading Terraform here makes that drift fail the build.
+function terraformCells(environment: 'production' | 'staging'): Array<{
+ cellId: string
+ region: string
+ zone: string
+ machineType: string
+ capacityRequests: number
+ databasePoolMax: number
+}> {
+ const tfvars = readFileSync(
+ fileURLToPath(new URL(`../../../infra/terraform/environments/${environment}.tfvars`, import.meta.url)),
+ 'utf8'
+ )
+ const block = /relay_gce_cells\s*=\s*\{([\s\S]*)\n\}/.exec(tfvars)?.[1] ?? ''
+ return [...block.matchAll(/"([a-z]+-gce-c\d+)"\s*=\s*\{([\s\S]*?)\n {2}\}/g)]
+ .map((match) => ({
+ cellId: match[1] ?? '',
+ region: /region\s*=\s*"([^"]+)"/.exec(match[2] ?? '')?.[1] ?? 'us-central1',
+ zone: /zone\s*=\s*"([^"]+)"/.exec(match[2] ?? '')?.[1] ?? '',
+ machineType: /machine_type\s*=\s*"([^"]+)"/.exec(match[2] ?? '')?.[1] ?? '',
+ capacityRequests: Number(/capacity_requests\s*=\s*(\d+)/.exec(match[2] ?? '')?.[1]),
+ databasePoolMax: Number(/database_pool_max\s*=\s*(\d+)/.exec(match[2] ?? '')?.[1] ?? 10)
+ }))
+ .sort((left, right) => cellOrdinal(left.cellId) - cellOrdinal(right.cellId))
+}
+
+const cellOrdinal = (cellId: string): number => Number(/c(\d+)$/.exec(cellId)?.[1] ?? 0)
+
+describe('relay operations environment config', () => {
+ it.each(['production', 'staging'] as const)(
+ 'matches the %s cells Terraform actually provisions',
+ (environment) => {
+ const expected = terraformCells(environment)
+ expect(expected.length).toBeGreaterThan(0)
+ expect(
+ RELAY_OPS_ENVIRONMENTS[environment].cells.map((cell) => ({
+ cellId: cell.cellId,
+ region: cell.region,
+ zone: cell.zone,
+ machineType: cell.machineType,
+ capacityRequests: cell.capacityRequests,
+ databasePoolMax: cell.databasePoolMax
+ }))
+ ).toEqual(expected)
+ }
+ )
+
+ it('derives hostname and origin from the cell ordinal', () => {
+ const cells = RELAY_OPS_ENVIRONMENTS.production.cells
+ expect(cells.at(-1)).toMatchObject({
+ hostname: `c${cells.length}`,
+ origin: `https://c${cells.length}.relay.onorca.dev`
+ })
+ })
+
+ it('inventories Asia cells only when they exist in durable Terraform', () => {
+ const cells = relayOpsCellsFromTerraform({
+ environment: 'production',
+ domain: 'relay.onorca.dev',
+ source: durableAsiaSource
+ })
+
+ expect(cells.slice(1)).toEqual([
+ {
+ cellId: 'production-gce-c27', hostname: 'c27', origin: 'https://c27.relay.onorca.dev',
+ region: 'asia-east2', zone: 'asia-east2-a', machineType: 'e2-standard-4',
+ capacityRequests: 6000, databasePoolMax: 10, configuredAdmission: false
+ },
+ {
+ cellId: 'production-gce-c28', hostname: 'c28', origin: 'https://c28.relay.onorca.dev',
+ region: 'asia-east2', zone: 'asia-east2-b', machineType: 'e2-standard-4',
+ capacityRequests: 6000, databasePoolMax: 10, configuredAdmission: false
+ },
+ {
+ cellId: 'production-gce-c29', hostname: 'c29', origin: 'https://c29.relay.onorca.dev',
+ region: 'asia-east2', zone: 'asia-east2-c', machineType: 'e2-standard-4',
+ capacityRequests: 6000, databasePoolMax: 10, configuredAdmission: false
+ }
+ ])
+
+ const usOnlyCells = relayOpsCellsFromTerraform({
+ environment: 'production',
+ domain: 'relay.onorca.dev',
+ source: durableUsOnlySource
+ })
+ expect(usOnlyCells).toHaveLength(1)
+ expect(usOnlyCells.some((cell) => cell.region === 'asia-east2')).toBe(false)
+ expect(usOnlyCells.some((cell) => cell.cellId === 'production-gce-c27')).toBe(false)
+ })
+})
diff --git a/cloud/apps/relay-ops/src/environment-config.ts b/cloud/apps/relay-ops/src/environment-config.ts
new file mode 100644
index 00000000000..da3de7765ed
--- /dev/null
+++ b/cloud/apps/relay-ops/src/environment-config.ts
@@ -0,0 +1,125 @@
+import { readFileSync } from 'node:fs'
+import { z } from 'zod'
+
+export type RelayOpsEnvironmentId = 'production' | 'staging'
+export type RelayOpsRegion = 'us-central1' | 'asia-east2'
+export type RelayOpsMachineType = 'e2-standard-2' | 'e2-standard-4'
+
+export type RelayOpsCellConfig = {
+ cellId: string
+ hostname: string
+ origin: string
+ region: RelayOpsRegion
+ zone: string
+ machineType: RelayOpsMachineType
+ capacityRequests: number
+ databasePoolMax: number
+ configuredAdmission: boolean
+}
+
+export type RelayOpsEnvironment = {
+ id: RelayOpsEnvironmentId
+ label: string
+ project: string
+ region: string
+ directorOrigin: string
+ authOrigin: string
+ directorService: string
+ authService: string
+ sqlInstance: string
+ migPrefix: string
+ certificateName: string
+ cells: RelayOpsCellConfig[]
+}
+
+const RegionSchema = z.enum(['us-central1', 'asia-east2'])
+const MachineTypeSchema = z.enum(['e2-standard-2', 'e2-standard-4'])
+const EnvironmentSchema = z.enum(['production', 'staging'])
+
+function cellOrdinal(cellId: string): number {
+ return Number(/c(\d+)$/.exec(cellId)?.[1] ?? 0)
+}
+
+function required(body: string, pattern: RegExp, label: string): string {
+ const value = pattern.exec(body)?.[1]
+ if (!value) throw new Error(`Relay Ops could not read ${label} from durable Terraform config`)
+ return value
+}
+
+export function relayOpsCellsFromTerraform(input: {
+ environment: RelayOpsEnvironmentId
+ domain: string
+ source: string
+}): RelayOpsCellConfig[] {
+ const block = /relay_gce_cells\s*=\s*\{([\s\S]*)\n\}/.exec(input.source)?.[1]
+ if (!block) throw new Error('Relay Ops could not read durable Relay cells')
+ return [...block.matchAll(/"([a-z]+-gce-c\d+)"\s*=\s*\{([\s\S]*?)\n {2}\}/g)]
+ .map((match) => {
+ const cellId = match[1]!
+ const body = match[2]!
+ const hostname = required(body, /\bhostname\s*=\s*"([^"]+)"/, `${cellId} hostname`)
+ const configuredAdmission = /\binitially_enabled\s*=\s*(true|false)/.exec(body)?.[1]
+ return {
+ cellId,
+ hostname,
+ origin: `https://${hostname}.${input.domain}`,
+ region: RegionSchema.parse(
+ /\bregion\s*=\s*"([^"]+)"/.exec(body)?.[1] ?? 'us-central1'
+ ),
+ zone: required(body, /\bzone\s*=\s*"([^"]+)"/, `${cellId} zone`),
+ machineType: MachineTypeSchema.parse(
+ required(body, /\bmachine_type\s*=\s*"([^"]+)"/, `${cellId} machine type`)
+ ),
+ capacityRequests: Number(
+ required(body, /\bcapacity_requests\s*=\s*(\d+)/, `${cellId} capacity`)
+ ),
+ databasePoolMax: Number(/\bdatabase_pool_max\s*=\s*(\d+)/.exec(body)?.[1] ?? 10),
+ configuredAdmission: configuredAdmission === undefined || configuredAdmission === 'true'
+ }
+ })
+ .sort((left, right) => cellOrdinal(left.cellId) - cellOrdinal(right.cellId))
+}
+
+function durableCells(environment: RelayOpsEnvironmentId, domain: string): RelayOpsCellConfig[] {
+ const source = readFileSync(
+ // Repository root; infra/terraform moves with this tree, so the relative depth holds.
+ new URL(`../../../infra/terraform/environments/${environment}.tfvars`, import.meta.url),
+ 'utf8'
+ )
+ return relayOpsCellsFromTerraform({ environment, domain, source })
+}
+
+export const RELAY_OPS_ENVIRONMENTS: Record = {
+ production: {
+ id: 'production',
+ label: 'Production',
+ project: 'onorca-cloud',
+ region: 'us-central1',
+ directorOrigin: 'https://relay.onorca.dev',
+ authOrigin: 'https://login.onorca.dev',
+ directorService: 'orca-cloud-relay',
+ authService: 'orca-cloud-auth',
+ sqlInstance: 'orca-cloud-auth-db',
+ migPrefix: 'orca-cloud-relay-gce-',
+ certificateName: 'orca-cloud-relay-gce',
+ cells: durableCells('production', 'relay.onorca.dev')
+ },
+ staging: {
+ id: 'staging',
+ label: 'Staging',
+ project: 'onorca-cloud-staging',
+ region: 'us-central1',
+ directorOrigin: 'https://relay-staging.onorca.dev',
+ authOrigin: 'https://auth-staging.onorca.dev',
+ directorService: 'orca-cloud-relay-staging',
+ authService: 'orca-cloud-auth-staging',
+ sqlInstance: 'orca-cloud-staging-auth-db',
+ migPrefix: 'orca-cloud-staging-relay-gce-',
+ certificateName: 'orca-cloud-staging-relay-gce',
+ cells: durableCells('staging', 'relay-staging.onorca.dev')
+ }
+}
+
+export function relayOpsEnvironment(value: unknown): RelayOpsEnvironment {
+ return RELAY_OPS_ENVIRONMENTS[EnvironmentSchema.parse(value)]
+}
diff --git a/cloud/apps/relay-ops/src/gcloud-client.test.ts b/cloud/apps/relay-ops/src/gcloud-client.test.ts
new file mode 100644
index 00000000000..3347c0cb367
--- /dev/null
+++ b/cloud/apps/relay-ops/src/gcloud-client.test.ts
@@ -0,0 +1,43 @@
+import { describe, expect, it } from 'vitest'
+import { createGcloudClient } from './gcloud-client.js'
+
+describe('createGcloudClient', () => {
+ it('shares and caches one credential refresh across concurrent readers', async () => {
+ let calls = 0
+ const token = 'a'.repeat(40)
+ const client = createGcloudClient(async () => {
+ calls += 1
+ await Promise.resolve()
+ return token
+ })
+
+ const values = await Promise.all([
+ client.accessToken(),
+ client.accessToken(),
+ client.accessToken()
+ ])
+
+ expect(values).toEqual([token, token, token])
+ expect(await client.accessToken()).toBe(token)
+ expect(calls).toBe(1)
+ })
+
+ it('caches bounded identity tokens by audience', async () => {
+ const commands: string[][] = []
+ const token = 'aaa.bbb.ccc'
+ const client = createGcloudClient(async (args) => {
+ commands.push(args)
+ return token
+ })
+ await expect(client.identityToken?.('https://relay.example/admin')).resolves.toBe(token)
+ await expect(client.identityToken?.('https://relay.example/admin')).resolves.toBe(token)
+ expect(commands).toEqual([
+ [
+ 'auth',
+ 'print-identity-token',
+ '--audiences=https://relay.example/admin',
+ '--include-email'
+ ]
+ ])
+ })
+})
diff --git a/cloud/apps/relay-ops/src/gcloud-client.ts b/cloud/apps/relay-ops/src/gcloud-client.ts
new file mode 100644
index 00000000000..b565242ec0e
--- /dev/null
+++ b/cloud/apps/relay-ops/src/gcloud-client.ts
@@ -0,0 +1,77 @@
+import { execFile } from 'node:child_process'
+import { promisify } from 'node:util'
+
+const execFileAsync = promisify(execFile)
+const TOKEN_PATTERN = /^[A-Za-z0-9._~+\/-]{32,8192}$/
+
+export type GcloudClient = {
+ accessToken(): Promise
+ identityToken?(audience: string): Promise
+}
+
+export class GcloudCommandError extends Error {
+ constructor(readonly operation: string) {
+ super(`${operation} is unavailable`)
+ }
+}
+
+async function runGcloud(args: string[]): Promise {
+ try {
+ const result = await execFileAsync('gcloud', args, {
+ encoding: 'utf8',
+ timeout: 90_000,
+ maxBuffer: 8 * 1024 * 1024,
+ env: {
+ ...process.env,
+ CLOUDSDK_COMPONENT_MANAGER_DISABLE_UPDATE_CHECK: '1',
+ CLOUDSDK_CORE_DISABLE_PROMPTS: '1',
+ CLOUDSDK_CORE_DISABLE_USAGE_REPORTING: '1'
+ }
+ })
+ return result.stdout.trim()
+ } catch {
+ // Gcloud stderr can echo command context; keep dashboard errors intentionally non-sensitive.
+ throw new GcloudCommandError(`gcloud ${args.slice(0, 3).join(' ')}`)
+ }
+}
+
+export function createGcloudClient(
+ tokenCommand: (args: string[]) => Promise = runGcloud
+): GcloudClient {
+ let cachedToken: { value: string; expiresAt: number } | null = null
+ const identityTokens = new Map()
+ let pendingToken: Promise | null = null
+ return {
+ async accessToken(): Promise {
+ if (cachedToken && cachedToken.expiresAt > Date.now()) return cachedToken.value
+ if (pendingToken) return await pendingToken
+ // One refresh avoids concurrent gcloud processes contending on the local credential store.
+ pendingToken = tokenCommand(['auth', 'print-access-token'])
+ .then((token) => {
+ if (!TOKEN_PATTERN.test(token)) throw new GcloudCommandError('gcloud access token')
+ cachedToken = { value: token, expiresAt: Date.now() + 5 * 60_000 }
+ return token
+ })
+ .finally(() => { pendingToken = null })
+ return await pendingToken
+ },
+ async identityToken(audience: string): Promise {
+ const cached = identityTokens.get(audience)
+ if (cached && cached.expiresAt > Date.now()) return cached.value
+ const token = await tokenCommand([
+ 'auth',
+ 'print-identity-token',
+ `--audiences=${audience}`,
+ '--include-email'
+ ])
+ if (
+ token.length > 8_192 ||
+ !/^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/.test(token)
+ ) {
+ throw new GcloudCommandError('gcloud identity token')
+ }
+ identityTokens.set(audience, { value: token, expiresAt: Date.now() + 5 * 60_000 })
+ return token
+ }
+ }
+}
diff --git a/cloud/apps/relay-ops/src/github-runs.ts b/cloud/apps/relay-ops/src/github-runs.ts
new file mode 100644
index 00000000000..47ef8469fa9
--- /dev/null
+++ b/cloud/apps/relay-ops/src/github-runs.ts
@@ -0,0 +1,77 @@
+import { execFile } from 'node:child_process'
+import { promisify } from 'node:util'
+import { z } from 'zod'
+import { relayRepositoryApiPath } from './relay-repository.js'
+
+const execFileAsync = promisify(execFile)
+
+const WorkflowRunSchema = z.object({
+ id: z.number().int().positive(),
+ name: z.string(),
+ event: z.string(),
+ status: z.string(),
+ conclusion: z.string().nullable(),
+ head_sha: z.string(),
+ html_url: z.string().url(),
+ created_at: z.string(),
+ updated_at: z.string()
+})
+
+const WorkflowRunsSchema = z.object({
+ workflow_runs: z.array(WorkflowRunSchema)
+})
+
+export type RelayWorkflowRun = {
+ id: number
+ name: string
+ status: string
+ conclusion: string | null
+ headSha: string
+ url: string
+ createdAt: string
+ updatedAt: string
+}
+
+const RELAY_WORKFLOW_PATTERN = /(Relay|Auth|Power)/i
+
+export async function readRelayWorkflowRuns(): Promise {
+ let stdout: string
+ try {
+ const result = await execFileAsync(
+ 'gh',
+ [
+ 'api',
+ '--method',
+ 'GET',
+ relayRepositoryApiPath('actions/runs'),
+ '-f',
+ 'per_page=100'
+ ],
+ { encoding: 'utf8', timeout: 30_000, maxBuffer: 4 * 1024 * 1024 }
+ )
+ stdout = result.stdout
+ } catch {
+ throw new Error('GitHub workflow history is unavailable')
+ }
+ const runs = WorkflowRunsSchema.parse(JSON.parse(stdout) as unknown).workflow_runs
+ const counts = new Map()
+ return runs
+ .filter((run) => {
+ if (!RELAY_WORKFLOW_PATTERN.test(run.name)) return false
+ const count = counts.get(run.name) ?? 0
+ if (count >= 2) return false
+ counts.set(run.name, count + 1)
+ return true
+ })
+ .slice(0, 12)
+ .map((run) => ({
+ id: run.id,
+ name: run.name,
+ status: run.status,
+ conclusion: run.conclusion,
+ headSha: run.head_sha.slice(0, 8),
+ url: run.html_url,
+ createdAt: run.created_at,
+ updatedAt: run.updated_at
+ }))
+}
diff --git a/cloud/apps/relay-ops/src/incident-live-preflight-cli.test.ts b/cloud/apps/relay-ops/src/incident-live-preflight-cli.test.ts
new file mode 100644
index 00000000000..b642d3cc3e1
--- /dev/null
+++ b/cloud/apps/relay-ops/src/incident-live-preflight-cli.test.ts
@@ -0,0 +1,434 @@
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import {
+ livePreflightGcloud,
+ runIncidentLivePreflight
+} from './incident-live-preflight-cli.js'
+import {
+ INCIDENT_MONITOR_THRESHOLDS,
+ type IncidentSample
+} from './incident-monitor.js'
+import type { AdmissionSelector } from './incident-selector.js'
+
+const directories: string[] = []
+const now = Date.parse('2026-07-28T12:00:00.000Z')
+const selector = {
+ generation: 1,
+ membership: {
+ existingOnly: ['production-gce-c1'],
+ migrationOnly: [],
+ general: []
+ }
+}
+
+function stateFile(
+ migrationPolicy: 'strict' | 'recover-forward' | 'capacity-transition' = 'strict',
+ overrides: Record = {}
+): string {
+ const directory = mkdtempSync(join(tmpdir(), 'relay-live-preflight-'))
+ directories.push(directory)
+ const path = join(directory, 'state.json')
+ const expectedSelector = migrationPolicy === 'capacity-transition'
+ ? {
+ generation: 1,
+ membership: {
+ existingOnly: [],
+ migrationOnly: [],
+ general: ['production-gce-c1']
+ }
+ }
+ : selector
+ writeFileSync(path, JSON.stringify({
+ schemaVersion: 4,
+ environment: 'production',
+ expectedSelector,
+ migrationPolicy,
+ recoverySourceCellId:
+ migrationPolicy === 'recover-forward' ? 'production-gce-c1' : null,
+ capacityCellId:
+ migrationPolicy === 'capacity-transition' ? 'production-gce-c1' : null,
+ preDrainDryRun: true,
+ startedAt: new Date(now - 17 * 60_000).toISOString(),
+ windowStartedAt: new Date(now - 16 * 60_000).toISOString(),
+ durationMinutes: 15,
+ intervalMs: 60_000,
+ sampleCount: 16,
+ lastSampleAt: new Date(now - 60_007).toISOString(),
+ frozenAt: null,
+ completedAt: new Date(now - 60_000).toISOString(),
+ ...overrides
+ }))
+ return path
+}
+
+function sample(): IncidentSample {
+ const observedAt = new Date(now).toISOString()
+ const signal = (value: number) => ({ value, observedAt })
+ return {
+ collectedAt: observedAt,
+ selector,
+ expectedSelector: selector,
+ cells: [{
+ cellId: 'production-gce-c1',
+ region: 'us-central1',
+ runtimeKnown: true,
+ powered: true,
+ expectedAdmissionState: 'existing-only'
+ }],
+ sources: {
+ 'active-probe': {
+ observedAt,
+ signals: {
+ 'director.health': signal(1),
+ 'director.ready': signal(1),
+ 'director.latency_ms': signal(1),
+ 'auth.health': signal(1),
+ 'auth.ready': signal(1),
+ 'auth.latency_ms': signal(1),
+ 'cell.production-gce-c1.health': signal(1),
+ 'cell.production-gce-c1.ready': signal(1),
+ 'cell.production-gce-c1.latency_ms': signal(1)
+ }
+ },
+ 'cloud-monitoring': {
+ observedAt,
+ signals: {
+ 'cloud_sql.cpu': signal(0.1),
+ 'cloud_sql.memory': signal(0.1),
+ 'cloud_sql.backends': signal(1),
+ 'cloud_sql.lock_waits': signal(0),
+ 'cloud_sql.deadlocks': signal(0),
+ 'director.instances': signal(5),
+ 'director.cpu': signal(0.1),
+ 'director.memory': signal(0.1),
+ 'director.concurrency': signal(1),
+ 'director.errors': signal(0),
+ 'auth.errors': signal(0)
+ }
+ },
+ 'relay-logs': {
+ observedAt,
+ signals: {
+ 'relay.pool_waiting': signal(0),
+ 'relay.pool_wait_ms': signal(0),
+ 'relay.postgres_retries': signal(0),
+ 'relay.postgres_retry_exhausted': signal(0),
+ 'cell.production-gce-c1.connections': signal(1),
+ 'cell.production-gce-c1.queued_bytes': signal(0)
+ }
+ },
+ 'director-admin': {
+ observedAt,
+ signals: {
+ 'cell.production-gce-c1.admission_state': signal(0),
+ 'cell.production-gce-c1.heartbeat_fresh': signal(1),
+ 'cell.production-gce-c1.heartbeat_age_ms': signal(1),
+ 'cell.production-gce-c1.migration_blocked': signal(0),
+ 'cell.production-gce-c1.migration_target_inactive': signal(0)
+ }
+ }
+ }
+ }
+}
+
+afterEach(() => {
+ for (const directory of directories.splice(0)) {
+ rmSync(directory, { recursive: true, force: true })
+ }
+})
+
+describe('relay incident live preflight', () => {
+ it('accepts the package-manager argument separator', async () => {
+ await expect(runIncidentLivePreflight(
+ ['--', '--state-file', stateFile()],
+ { now: () => now, collect: async () => sample() }
+ )).resolves.toBeUndefined()
+ })
+
+ it('accepts one complete fresh green sample', async () => {
+ await expect(runIncidentLivePreflight(
+ ['--state-file', stateFile()],
+ { now: () => now, collect: async () => sample() }
+ )).resolves.toBeUndefined()
+ })
+
+ it('rejects monitor evidence beyond the 25-minute lineage bound', async () => {
+ const path = stateFile('strict', {
+ startedAt: new Date(now - 26 * 60_000 - 1).toISOString()
+ })
+ await expect(runIncidentLivePreflight(
+ ['--state-file', path],
+ { now: () => now, collect: async () => sample() }
+ )).rejects.toThrow('monitor evidence is incomplete or stale')
+ })
+
+ it('scales the evidence age bound by same-cap wave index', async () => {
+ const agedState = (ageMs: number) => stateFile('strict', {
+ startedAt: new Date(now - ageMs - 17 * 60_000).toISOString(),
+ windowStartedAt: new Date(now - ageMs - 16 * 60_000).toISOString(),
+ lastSampleAt: new Date(now - ageMs - 7).toISOString(),
+ completedAt: new Date(now - ageMs).toISOString()
+ })
+ const deps = { now: () => now, collect: async () => sample() }
+ // One predecessor cell roll (~16 min) exceeds wave 0 but fits wave 1.
+ const oneRollOld = agedState(17 * 60_000)
+ await expect(runIncidentLivePreflight(
+ ['--state-file', oneRollOld], deps
+ )).rejects.toThrow('monitor evidence is incomplete or stale')
+ await expect(runIncidentLivePreflight(
+ ['--state-file', oneRollOld, '--wave-index', '0'], deps
+ )).rejects.toThrow('monitor evidence is incomplete or stale')
+ await expect(runIncidentLivePreflight(
+ ['--state-file', oneRollOld, '--wave-index', '1'], deps
+ )).resolves.toBeUndefined()
+ // Both edges of one predecessor job timeout: 5min + 75min exactly.
+ await expect(runIncidentLivePreflight(
+ ['--state-file', agedState(80 * 60_000), '--wave-index', '1'], deps
+ )).resolves.toBeUndefined()
+ await expect(runIncidentLivePreflight(
+ ['--state-file', agedState(80 * 60_000 + 1), '--wave-index', '1'], deps
+ )).rejects.toThrow('monitor evidence is incomplete or stale')
+ await expect(runIncidentLivePreflight(
+ ['--state-file', agedState(155 * 60_000), '--wave-index', '2'], deps
+ )).resolves.toBeUndefined()
+ await expect(runIncidentLivePreflight(
+ ['--state-file', agedState(155 * 60_000 + 1), '--wave-index', '2'], deps
+ )).rejects.toThrow('monitor evidence is incomplete or stale')
+ await expect(runIncidentLivePreflight(
+ ['--state-file', agedState(230 * 60_000), '--wave-index', '3'], deps
+ )).resolves.toBeUndefined()
+ await expect(runIncidentLivePreflight(
+ ['--state-file', agedState(230 * 60_000 + 1), '--wave-index', '3'], deps
+ )).rejects.toThrow('monitor evidence is incomplete or stale')
+ // The wave index is a strict single-use 0-3 argument.
+ await expect(runIncidentLivePreflight(
+ ['--state-file', stateFile(), '--wave-index', '4'], deps
+ )).rejects.toThrow('usage:')
+ await expect(runIncidentLivePreflight(
+ ['--state-file', stateFile(), '--wave-index', ''], deps
+ )).rejects.toThrow('usage:')
+ await expect(runIncidentLivePreflight(
+ ['--state-file', stateFile(), '--wave-index', '1', '--wave-index', '1'],
+ deps
+ )).rejects.toThrow('usage:')
+ })
+
+ it('expects the wave-adjusted live selector generation', async () => {
+ const agedPath = stateFile('strict', {
+ startedAt: new Date(now - 34 * 60_000).toISOString(),
+ windowStartedAt: new Date(now - 33 * 60_000).toISOString(),
+ lastSampleAt: new Date(now - 17 * 60_000 - 7).toISOString(),
+ completedAt: new Date(now - 17 * 60_000).toISOString()
+ })
+ const liveAt = (generation: number) =>
+ async (expectedSelector: AdmissionSelector) => ({
+ ...sample(),
+ selector: { ...selector, generation },
+ expectedSelector
+ })
+ // One predecessor roll advanced the live selector by exactly 2.
+ await expect(runIncidentLivePreflight(
+ ['--state-file', agedPath, '--wave-index', '1'],
+ { now: () => now, collect: liveAt(selector.generation + 2) }
+ )).resolves.toBeUndefined()
+ // The sealed pre-roll generation must no longer satisfy wave 1.
+ await expect(runIncidentLivePreflight(
+ ['--state-file', agedPath, '--wave-index', '1'],
+ { now: () => now, collect: liveAt(selector.generation) }
+ )).rejects.toThrow('director-admin/selector_mismatch')
+ // Wave 0 still expects the sealed generation itself.
+ await expect(runIncidentLivePreflight(
+ ['--state-file', stateFile()],
+ { now: () => now, collect: liveAt(selector.generation) }
+ )).resolves.toBeUndefined()
+ })
+
+ it('fails closed on a live threshold breach', async () => {
+ const unhealthy = sample()
+ unhealthy.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.value = 0.9
+ await expect(runIncidentLivePreflight(
+ ['--state-file', stateFile()],
+ { now: () => now, collect: async () => unhealthy }
+ )).rejects.toThrow('cloud-monitoring/threshold_max')
+ })
+
+ // Why: a frozen wave has to name what froze it without re-reading the sample.
+ it('names the signal and its numbers in the failure message', async () => {
+ const slowCell = sample()
+ slowCell.sources['active-probe']!.signals[
+ 'cell.production-gce-c1.latency_ms'
+ ]!.value = 2_568
+ await expect(runIncidentLivePreflight(
+ ['--state-file', stateFile()],
+ { now: () => now, collect: async () => slowCell }
+ )).rejects.toThrow(
+ 'relay live preflight failed: active-probe/threshold_max cell.production-gce-c1.latency_ms observed=2568 threshold=2000'
+ )
+
+ // A failure with no signal keeps the source/code token and drops the rest.
+ const stale = sample()
+ stale.sources['active-probe']!.observedAt = new Date(now - 60_001).toISOString()
+ await expect(runIncidentLivePreflight(
+ ['--state-file', stateFile()],
+ { now: () => now, collect: async () => stale }
+ )).rejects.toThrow(
+ 'relay live preflight failed: active-probe/source_stale observed=60001 threshold=60000'
+ )
+ })
+
+ it('enforces the signed migration policy', async () => {
+ const inactiveTarget = sample()
+ inactiveTarget.sources['director-admin']!.signals[
+ 'cell.production-gce-c1.migration_target_inactive'
+ ]!.value = 30
+ await expect(runIncidentLivePreflight(
+ ['--state-file', stateFile()],
+ { now: () => now, collect: async () => inactiveTarget }
+ )).rejects.toThrow('director-admin/threshold_max')
+ await expect(runIncidentLivePreflight(
+ ['--state-file', stateFile('recover-forward')],
+ { now: () => now, collect: async () => inactiveTarget }
+ )).resolves.toBeUndefined()
+ inactiveTarget.sources['director-admin']!.signals[
+ 'cell.production-gce-c1.migration_blocked'
+ ]!.value = 1
+ await expect(runIncidentLivePreflight(
+ ['--state-file', stateFile('recover-forward')],
+ { now: () => now, collect: async () => inactiveTarget }
+ )).rejects.toThrow('director-admin/threshold_max')
+ })
+
+ it('binds capacity-transition evidence to its general cell', async () => {
+ const capacitySample = sample()
+ const capacitySelector = {
+ generation: 1,
+ membership: {
+ existingOnly: [],
+ migrationOnly: [],
+ general: ['production-gce-c1']
+ }
+ }
+ capacitySample.selector = capacitySelector
+ capacitySample.expectedSelector = capacitySelector
+ capacitySample.cells[0]!.expectedAdmissionState = 'general'
+ capacitySample.sources['director-admin']!.signals[
+ 'cell.production-gce-c1.admission_state'
+ ]!.value = 2
+ await expect(runIncidentLivePreflight(
+ ['--state-file', stateFile('capacity-transition')],
+ { now: () => now, collect: async () => capacitySample }
+ )).resolves.toBeUndefined()
+ capacitySample.sources['director-admin']!.signals[
+ 'cell.production-gce-c1.migration_target_inactive'
+ ]!.value = 1
+ await expect(runIncidentLivePreflight(
+ ['--state-file', stateFile('capacity-transition')],
+ { now: () => now, collect: async () => capacitySample }
+ )).rejects.toThrow('director-admin/threshold_max')
+ })
+
+ it('rejects stale live evidence', async () => {
+ const stale = sample()
+ stale.sources['active-probe']!.observedAt = new Date(now - 60_001).toISOString()
+ await expect(runIncidentLivePreflight(
+ ['--state-file', stateFile()],
+ { now: () => now, collect: async () => stale }
+ )).rejects.toThrow('active-probe/source_stale')
+ })
+
+ it('retries freshness-only failures when explicitly requested', async () => {
+ const stale = sample()
+ stale.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.observedAt =
+ new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString()
+ const missing = sample()
+ delete missing.sources['relay-logs']
+ const collect = vi.fn()
+ .mockResolvedValueOnce(stale)
+ .mockResolvedValueOnce(missing)
+ .mockResolvedValueOnce(sample())
+ const wait = vi.fn(async () => undefined)
+ await expect(runIncidentLivePreflight(
+ ['--state-file', stateFile(), '--retry-freshness'],
+ { now: () => now, collect, wait }
+ )).resolves.toBeUndefined()
+ expect(collect).toHaveBeenCalledTimes(3)
+ expect(wait).toHaveBeenCalledTimes(2)
+ expect(wait).toHaveBeenNthCalledWith(1, 15_000)
+ expect(wait).toHaveBeenNthCalledWith(2, 15_000)
+ })
+
+ it('retries a first-wave stale sample and passes on the fresh one', async () => {
+ const stale = sample()
+ stale.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.observedAt =
+ new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString()
+ const collect = vi.fn().mockResolvedValueOnce(stale).mockResolvedValueOnce(sample())
+ const wait = vi.fn(async () => undefined)
+ await expect(runIncidentLivePreflight(
+ ['--state-file', stateFile(), '--wave-index', '0', '--retry-freshness'],
+ { now: () => now, collect, wait }
+ )).resolves.toBeUndefined()
+ expect(collect).toHaveBeenCalledTimes(2)
+ expect(wait).toHaveBeenCalledOnce()
+ })
+
+ it('stops retrying when the next wait would exceed the evidence-age bound', async () => {
+ const completedAt = now - 290_000
+ const stale = sample()
+ stale.sources['cloud-monitoring']!.observedAt = new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString()
+ const collect = vi.fn(async () => stale)
+ const wait = vi.fn(async () => undefined)
+ await expect(runIncidentLivePreflight(
+ ['--state-file', stateFile('strict', {
+ startedAt: new Date(completedAt - 17 * 60_000).toISOString(),
+ windowStartedAt: new Date(completedAt - 16 * 60_000).toISOString(),
+ lastSampleAt: new Date(completedAt - 30_000).toISOString(),
+ completedAt: new Date(completedAt).toISOString()
+ }), '--retry-freshness'],
+ { now: () => now, collect, wait }
+ )).rejects.toThrow('cloud-monitoring/source_stale')
+ expect(collect).toHaveBeenCalledOnce()
+ expect(wait).not.toHaveBeenCalled()
+ })
+
+ it('does not retry a threshold failure', async () => {
+ const unhealthy = sample()
+ unhealthy.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.value = 0.9
+ unhealthy.sources['cloud-monitoring']!.signals['cloud_sql.cpu']!.observedAt =
+ new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString()
+ const collect = vi.fn(async () => unhealthy)
+ const wait = vi.fn(async () => undefined)
+ await expect(runIncidentLivePreflight(
+ ['--state-file', stateFile(), '--retry-freshness'],
+ { now: () => now, collect, wait }
+ )).rejects.toThrow('cloud-monitoring/threshold_max')
+ expect(collect).toHaveBeenCalledOnce()
+ expect(wait).not.toHaveBeenCalled()
+ })
+
+ it('fails closed after the bounded freshness retry window', async () => {
+ const stale = sample()
+ stale.sources['cloud-monitoring']!.observedAt = new Date(now - (INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs + 1)).toISOString()
+ const collect = vi.fn(async () => stale)
+ const wait = vi.fn(async () => undefined)
+ await expect(runIncidentLivePreflight(
+ ['--state-file', stateFile(), '--retry-freshness'],
+ { now: () => now, collect, wait }
+ )).rejects.toThrow('cloud-monitoring/source_stale')
+ expect(collect).toHaveBeenCalledTimes(5)
+ expect(wait).toHaveBeenCalledTimes(4)
+ })
+
+ it('uses the supplied admin token without minting through gcloud', async () => {
+ const identityToken = vi.fn(async () => 'minted.token.value')
+ const gcloud = livePreflightGcloud(
+ { accessToken: async () => 'access-token', identityToken },
+ { ORCA_RELAY_ADMIN_ID_TOKEN: 'supplied.token.value' }
+ )
+ await expect(gcloud.identityToken!('audience')).resolves.toBe(
+ 'supplied.token.value'
+ )
+ expect(identityToken).not.toHaveBeenCalled()
+ })
+})
diff --git a/cloud/apps/relay-ops/src/incident-live-preflight-cli.ts b/cloud/apps/relay-ops/src/incident-live-preflight-cli.ts
new file mode 100644
index 00000000000..c277325ed84
--- /dev/null
+++ b/cloud/apps/relay-ops/src/incident-live-preflight-cli.ts
@@ -0,0 +1,206 @@
+import { readFile } from 'node:fs/promises'
+import { resolve } from 'node:path'
+import { pathToFileURL } from 'node:url'
+import { z } from 'zod'
+import { createGcloudClient } from './gcloud-client.js'
+import { suppliedIdentityToken } from './incident-monitor-cli.js'
+import { AdmissionSelectorSchema, type AdmissionSelector } from './incident-selector.js'
+import {
+ evaluateIncidentSample,
+ FRESHNESS_FAILURE_CODES,
+ preDrainDryRunPassed,
+ type IncidentFailure,
+ type IncidentSample
+} from './incident-monitor.js'
+import { createIncidentSampleCollector } from './incident-monitor-sources.js'
+
+const FRESHNESS_RETRY_ATTEMPTS = 5
+const FRESHNESS_RETRY_INTERVAL_MS = 15_000
+const MONITOR_EVIDENCE_MAX_AGE_MS = 5 * 60_000
+// Matches the same-cap cell job timeout-minutes; bounds each predecessor wave.
+const WAVE_PREDECESSOR_TIMEOUT_MS = 75 * 60_000
+const WAVE_INDEX_PATTERN = /^[0-3]$/
+
+export function livePreflightGcloud(
+ gcloud: ReturnType,
+ environment: NodeJS.ProcessEnv = process.env
+): ReturnType {
+ const token = suppliedIdentityToken(environment.ORCA_RELAY_ADMIN_ID_TOKEN)
+ return token ? { ...gcloud, identityToken: async () => token } : gcloud
+}
+
+const PreflightStateSchema = z.object({
+ schemaVersion: z.literal(4),
+ environment: z.literal('production'),
+ expectedSelector: AdmissionSelectorSchema,
+ migrationPolicy: z.enum(['strict', 'recover-forward', 'capacity-transition']),
+ recoverySourceCellId: z.string().nullable(),
+ capacityCellId: z.string().nullable(),
+ preDrainDryRun: z.literal(true),
+ startedAt: z.string(),
+ windowStartedAt: z.string(),
+ durationMinutes: z.literal(15),
+ intervalMs: z.literal(60_000),
+ sampleCount: z.number().int().min(16),
+ lastSampleAt: z.string(),
+ frozenAt: z.null(),
+ completedAt: z.string()
+}).superRefine((state, context) => {
+ const validRecovery =
+ state.migrationPolicy === 'recover-forward' &&
+ state.capacityCellId === null &&
+ state.recoverySourceCellId !== null &&
+ state.expectedSelector.membership.existingOnly.includes(
+ state.recoverySourceCellId
+ )
+ const validStrict =
+ state.migrationPolicy === 'strict' &&
+ state.recoverySourceCellId === null &&
+ state.capacityCellId === null
+ const validCapacity =
+ state.migrationPolicy === 'capacity-transition' &&
+ state.recoverySourceCellId === null &&
+ state.capacityCellId !== null &&
+ state.expectedSelector.membership.general.includes(state.capacityCellId)
+ if (!validRecovery && !validStrict && !validCapacity) {
+ context.addIssue({
+ code: 'custom',
+ message: 'relay live preflight migration policy is invalid'
+ })
+ }
+})
+
+// Keep the source/code prefix other tooling matches on, then name the signal and
+// its numbers so a frozen wave is attributable without re-reading the sample.
+function describeFailure(failure: IncidentFailure): string {
+ const detail = [
+ failure.signal,
+ failure.observed === undefined ? null : `observed=${failure.observed}`,
+ failure.threshold === undefined ? null : `threshold=${failure.threshold}`
+ ].filter((part): part is string => part !== null && part !== undefined)
+ return [`${failure.source}/${failure.code}`, ...detail].join(' ')
+}
+
+export async function runIncidentLivePreflight(
+ argv: string[],
+ dependencies: {
+ now?: () => number
+ wait?: (ms: number) => Promise
+ collect?: (expectedSelector: AdmissionSelector) => Promise
+ gcloud?: ReturnType
+ environment?: NodeJS.ProcessEnv
+ } = {}
+): Promise {
+ const args = argv[0] === '--' ? argv.slice(1) : argv
+ const freshnessRetryCount = args.filter((arg) => arg === '--retry-freshness').length
+ const rest = args.filter((arg) => arg !== '--retry-freshness')
+ const stateArgs: string[] = []
+ let waveIndex = '0'
+ let waveIndexCount = 0
+ for (let index = 0; index < rest.length; index += 1) {
+ if (rest[index] === '--wave-index') {
+ waveIndexCount += 1
+ waveIndex = rest[index + 1] ?? ''
+ index += 1
+ } else {
+ stateArgs.push(rest[index] as string)
+ }
+ }
+ if (
+ freshnessRetryCount > 1 ||
+ waveIndexCount > 1 ||
+ !WAVE_INDEX_PATTERN.test(waveIndex) ||
+ stateArgs.length !== 2 ||
+ stateArgs[0] !== '--state-file' ||
+ !stateArgs[1]
+ ) {
+ throw new Error(
+ 'usage: --state-file [--wave-index <0-3>] [--retry-freshness]'
+ )
+ }
+ const state = PreflightStateSchema.parse(
+ JSON.parse(await readFile(resolve(stateArgs[1]), 'utf8'))
+ )
+ const now = dependencies.now ?? Date.now
+ const completedAt = Date.parse(state.completedAt)
+ const windowStartedAt = Date.parse(state.windowStartedAt)
+ const lastSampleAt = Date.parse(state.lastSampleAt)
+ const evidenceAgeMs = now() - completedAt
+ // Later same-cap waves start after sequential predecessor cell rolls, so the
+ // freshness bound grows by one cell-job timeout per predecessor; the live
+ // samples collected below still hold every wave to current health.
+ const maxEvidenceAgeMs =
+ MONITOR_EVIDENCE_MAX_AGE_MS + Number(waveIndex) * WAVE_PREDECESSOR_TIMEOUT_MS
+ if (
+ !preDrainDryRunPassed(state) ||
+ !Number.isFinite(windowStartedAt) ||
+ completedAt - windowStartedAt < 15 * 60_000 ||
+ !Number.isFinite(lastSampleAt) ||
+ lastSampleAt > completedAt ||
+ completedAt - lastSampleAt > state.intervalMs ||
+ !Number.isFinite(completedAt) ||
+ evidenceAgeMs < 0 ||
+ evidenceAgeMs > maxEvidenceAgeMs
+ ) {
+ throw new Error('relay live preflight monitor evidence is incomplete or stale')
+ }
+ const gcloud = livePreflightGcloud(
+ dependencies.gcloud ?? createGcloudClient(),
+ dependencies.environment
+ )
+ // Each predecessor same-cap apply wave reversibly isolates and restores its
+ // cell, advancing the selector generation by exactly 2 with membership
+ // unchanged (rollback is single-cell, so it never reaches a later wave), so
+ // the live selector comparison must expect the wave-adjusted generation.
+ const collectOptions = {
+ environment: state.environment,
+ expectedSelector: {
+ ...state.expectedSelector,
+ generation: state.expectedSelector.generation + 2 * Number(waveIndex)
+ },
+ ...(dependencies.now ? { now: dependencies.now } : {})
+ }
+ const injected = dependencies.collect
+ const collect = injected
+ ? () => injected(collectOptions.expectedSelector)
+ : createIncidentSampleCollector(gcloud, collectOptions)
+ const wait = dependencies.wait ?? ((ms: number) => new Promise((resolveWait) => {
+ setTimeout(resolveWait, ms)
+ }))
+ const attempts = freshnessRetryCount === 1 ? FRESHNESS_RETRY_ATTEMPTS : 1
+ for (let attempt = 1; attempt <= attempts; attempt++) {
+ const evaluation = evaluateIncidentSample(
+ await collect(),
+ now(),
+ state.migrationPolicy,
+ state.recoverySourceCellId,
+ state.capacityCellId
+ )
+ if (evaluation.status === 'green') return
+ const freshnessOnly = evaluation.failures.every((failure) =>
+ FRESHNESS_FAILURE_CODES.has(failure.code)
+ )
+ // Waiting must never carry the mutation past the same evidence-age bound
+ // the entry check enforces, so the wave budget also caps the retry window.
+ const budgetExhausted =
+ now() + FRESHNESS_RETRY_INTERVAL_MS - completedAt > maxEvidenceAgeMs
+ if (!freshnessOnly || attempt === attempts || budgetExhausted) {
+ throw new Error(
+ `relay live preflight failed: ${evaluation.failures
+ .map(describeFailure)
+ .join(',')}`
+ )
+ }
+ console.warn(
+ `relay live preflight awaiting fresh evidence (${attempt}/${attempts - 1})`
+ )
+ await wait(FRESHNESS_RETRY_INTERVAL_MS)
+ }
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+ runIncidentLivePreflight(process.argv.slice(2)).catch((error: unknown) => {
+ console.error(error instanceof Error ? error.message : 'relay live preflight failed')
+ process.exitCode = 1
+ })
+}
diff --git a/cloud/apps/relay-ops/src/incident-monitor-cli.test.ts b/cloud/apps/relay-ops/src/incident-monitor-cli.test.ts
new file mode 100644
index 00000000000..31e5a131d35
--- /dev/null
+++ b/cloud/apps/relay-ops/src/incident-monitor-cli.test.ts
@@ -0,0 +1,455 @@
+import { mkdtempSync, readFileSync, rmSync, statSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, describe, expect, it } from 'vitest'
+import {
+ parseIncidentMonitorArguments,
+ runIncidentMonitorCli
+} from './incident-monitor-cli.js'
+import type { IncidentSample } from './incident-monitor.js'
+import { RELAY_OPS_ENVIRONMENTS } from './environment-config.js'
+
+const directories: string[] = []
+const startedAt = Date.parse('2026-07-28T00:00:00.000Z')
+const productionCells = RELAY_OPS_ENVIRONMENTS.production.cells.map((cell) => cell.cellId)
+const selector = {
+ generation: 1,
+ membership: {
+ existingOnly: productionCells.slice(1),
+ migrationOnly: [],
+ general: [productionCells[0]!]
+ }
+}
+const selectorArguments = [
+ '--expected-selector-generation',
+ '1',
+ '--expected-existing-only-cells',
+ productionCells.slice(1).join(','),
+ '--expected-migration-only-cells',
+ 'none',
+ '--expected-general-cells',
+ productionCells[0]!
+]
+const signal = (value: number, at: number) => ({
+ value,
+ observedAt: new Date(at).toISOString()
+})
+
+function sample(at: number): IncidentSample {
+ const observedAt = new Date(at).toISOString()
+ const cellId = 'production-gce-c1'
+ return {
+ collectedAt: observedAt,
+ selector,
+ expectedSelector: selector,
+ cells: [{
+ cellId,
+ region: 'us-central1',
+ runtimeKnown: true,
+ powered: true,
+ expectedAdmissionState: 'general'
+ }],
+ sources: {
+ 'active-probe': {
+ observedAt,
+ signals: {
+ 'director.health': signal(1, at),
+ 'director.ready': signal(1, at),
+ 'director.latency_ms': signal(1, at),
+ 'auth.health': signal(1, at),
+ 'auth.ready': signal(1, at),
+ 'auth.latency_ms': signal(1, at),
+ [`cell.${cellId}.health`]: signal(1, at),
+ [`cell.${cellId}.ready`]: signal(1, at),
+ [`cell.${cellId}.latency_ms`]: signal(1, at)
+ }
+ },
+ 'cloud-monitoring': {
+ observedAt,
+ signals: {
+ 'cloud_sql.cpu': signal(0.1, at),
+ 'cloud_sql.memory': signal(0.1, at),
+ 'cloud_sql.backends': signal(1, at),
+ 'cloud_sql.lock_waits': signal(0, at),
+ 'cloud_sql.deadlocks': signal(0, at),
+ 'director.instances': signal(5, at),
+ 'director.cpu': signal(0.1, at),
+ 'director.memory': signal(0.1, at),
+ 'director.concurrency': signal(1, at),
+ 'director.errors': signal(0, at),
+ 'auth.errors': signal(0, at)
+ }
+ },
+ 'relay-logs': {
+ observedAt,
+ signals: {
+ 'relay.pool_waiting': signal(0, at),
+ 'relay.pool_wait_ms': signal(0, at),
+ 'relay.postgres_retries': signal(0, at),
+ 'relay.postgres_retry_exhausted': signal(0, at),
+ [`cell.${cellId}.connections`]: signal(1, at),
+ [`cell.${cellId}.queued_bytes`]: signal(0, at)
+ }
+ },
+ 'director-admin': {
+ observedAt,
+ signals: {
+ [`cell.${cellId}.admission_state`]: signal(2, at),
+ [`cell.${cellId}.heartbeat_fresh`]: signal(1, at),
+ [`cell.${cellId}.heartbeat_age_ms`]: signal(1, at),
+ [`cell.${cellId}.migration_blocked`]: signal(0, at),
+ [`cell.${cellId}.migration_target_inactive`]: signal(0, at)
+ }
+ }
+ }
+ }
+}
+
+afterEach(() => {
+ for (const directory of directories.splice(0)) {
+ rmSync(directory, { recursive: true, force: true })
+ }
+})
+
+describe('incident monitor CLI', () => {
+ it('requires an exact selector and rejects invalid membership', () => {
+ expect(() => parseIncidentMonitorArguments([])).toThrow(
+ '--expected-selector-generation'
+ )
+ expect(() =>
+ parseIncidentMonitorArguments([
+ '--incident-id',
+ 'incident-1',
+ '--expected-selector-generation',
+ '1',
+ '--expected-existing-only-cells',
+ productionCells.slice(1).join(','),
+ '--expected-migration-only-cells',
+ 'none',
+ '--expected-general-cells',
+ 'production-gce-c99'
+ ])
+ ).toThrow('every configured cell exactly once')
+ expect(() =>
+ parseIncidentMonitorArguments([
+ '--incident-id',
+ 'incident-1',
+ ...selectorArguments,
+ '--interval-seconds',
+ '61'
+ ])
+ ).toThrow('between 1 and 60')
+ expect(() =>
+ parseIncidentMonitorArguments([
+ '--incident-id',
+ 'incident-1',
+ ...selectorArguments,
+ '--duration-minutes',
+ '14'
+ ])
+ ).toThrow('between 15 and 90')
+ expect(() =>
+ parseIncidentMonitorArguments([
+ '--incident-id',
+ 'incident-1',
+ '--expected-selector-generation',
+ '0',
+ '--expected-existing-only-cells',
+ productionCells.slice(1).join(','),
+ '--expected-migration-only-cells',
+ productionCells[0]!,
+ '--expected-general-cells',
+ 'none'
+ ])
+ ).toThrow('generation 0 cannot represent migration-only')
+ expect(() =>
+ parseIncidentMonitorArguments([
+ '--incident-id',
+ 'incident-1',
+ ...selectorArguments,
+ '--migration-policy',
+ 'recover-forward',
+ '--pre-drain-dry-run'
+ ])
+ ).toThrow('requires an existing-only --recovery-source-cell-id')
+ expect(() =>
+ parseIncidentMonitorArguments([
+ '--incident-id',
+ 'incident-1',
+ ...selectorArguments,
+ '--migration-policy',
+ 'capacity-transition',
+ '--pre-drain-dry-run'
+ ])
+ ).toThrow('requires a general --capacity-cell-id')
+ expect(
+ parseIncidentMonitorArguments([
+ '--incident-id',
+ 'incident-1',
+ ...selectorArguments,
+ '--migration-policy',
+ 'capacity-transition',
+ '--capacity-cell-id',
+ productionCells[0]!,
+ '--pre-drain-dry-run'
+ ])
+ ).toMatchObject({
+ migrationPolicy: 'capacity-transition',
+ capacityCellId: productionCells[0]!,
+ recoverySourceCellId: null
+ })
+ })
+
+ it('writes private durable checkpoints for a green pre-drain dry run', async () => {
+ const directory = mkdtempSync(join(tmpdir(), 'relay-incident-cli-'))
+ directories.push(directory)
+ let now = startedAt
+ const output: string[] = []
+ const code = await runIncidentMonitorCli(
+ [
+ '--incident-id',
+ 'incident-1',
+ ...selectorArguments,
+ '--pre-drain-dry-run',
+ '--output-directory',
+ directory
+ ],
+ {
+ cwd: directory,
+ now: () => now,
+ wait: async (ms) => {
+ now += ms
+ },
+ collect: async () => sample(now),
+ writeOutput: (value) => output.push(value)
+ }
+ )
+ expect(code).toBe(0)
+ const statePath = join(directory, 'incident-1.state.json')
+ const summaryPath = join(directory, 'incident-1.summaries.jsonl')
+ const markdownPath = join(directory, 'incident-1.summary.md')
+ expect(statSync(statePath).mode & 0o077).toBe(0)
+ expect(statSync(summaryPath).mode & 0o077).toBe(0)
+ expect(statSync(markdownPath).mode & 0o077).toBe(0)
+ const summaries = readFileSync(summaryPath, 'utf8')
+ .trim()
+ .split('\n')
+ .map((line) => JSON.parse(line))
+ expect(summaries.map((entry) => entry.checkpointMinute)).toEqual([0, 5, 15])
+ expect(output.join('')).not.toContain('token')
+ expect(readFileSync(markdownPath, 'utf8')).toContain(
+ '| 0 | 15 | green | 16 | none |'
+ )
+ expect(JSON.parse(readFileSync(statePath, 'utf8'))).toMatchObject({
+ completedAt: new Date(startedAt + 15 * 60_000).toISOString(),
+ frozenAt: null,
+ migrationPolicy: 'strict',
+ recoverySourceCellId: null,
+ capacityCellId: null,
+ sampleCount: 16
+ })
+ })
+
+ it('runs a recovery dry run without masking blocked migrations', async () => {
+ const directory = mkdtempSync(join(tmpdir(), 'relay-incident-cli-'))
+ directories.push(directory)
+ let now = startedAt
+ const recoverySelector = {
+ generation: 1,
+ membership: {
+ existingOnly: ['production-gce-c1'],
+ migrationOnly: [],
+ general: productionCells.slice(1)
+ }
+ }
+ const recoverySample = (): IncidentSample => {
+ const current = sample(now)
+ current.selector = recoverySelector
+ current.expectedSelector = recoverySelector
+ current.cells[0]!.expectedAdmissionState = 'existing-only'
+ current.sources['director-admin']!.signals[
+ 'cell.production-gce-c1.admission_state'
+ ] = signal(0, now)
+ current.sources['director-admin']!.signals[
+ 'cell.production-gce-c1.migration_target_inactive'
+ ] = signal(30, now)
+ return current
+ }
+ const args = [
+ '--incident-id',
+ 'incident-1',
+ '--expected-selector-generation',
+ '1',
+ '--expected-existing-only-cells',
+ 'production-gce-c1',
+ '--expected-migration-only-cells',
+ 'none',
+ '--expected-general-cells',
+ productionCells.slice(1).join(','),
+ '--pre-drain-dry-run',
+ '--migration-policy',
+ 'recover-forward',
+ '--recovery-source-cell-id',
+ 'production-gce-c1',
+ '--output-directory',
+ directory
+ ]
+ const dependencies = {
+ cwd: directory,
+ now: () => now,
+ wait: async (ms: number) => {
+ now += ms
+ },
+ collect: async () => recoverySample(),
+ writeOutput: () => {}
+ }
+ await expect(runIncidentMonitorCli(args, dependencies)).resolves.toBe(0)
+ expect(
+ JSON.parse(readFileSync(join(directory, 'incident-1.state.json'), 'utf8'))
+ ).toMatchObject({
+ migrationPolicy: 'recover-forward',
+ recoverySourceCellId: 'production-gce-c1',
+ capacityCellId: null,
+ frozenAt: null
+ })
+ })
+
+ it('fails a frozen pre-drain gate after its first sample', async () => {
+ const directory = mkdtempSync(join(tmpdir(), 'relay-incident-cli-'))
+ directories.push(directory)
+ let now = startedAt
+ let collections = 0
+ let waits = 0
+ const code = await runIncidentMonitorCli(
+ [
+ '--incident-id',
+ 'incident-1',
+ ...selectorArguments,
+ '--pre-drain-dry-run',
+ '--output-directory',
+ directory
+ ],
+ {
+ cwd: directory,
+ now: () => now,
+ wait: async (ms) => {
+ waits++
+ now += ms
+ },
+ collect: async () => {
+ collections++
+ const unhealthy = sample(now)
+ unhealthy.sources['relay-logs']!.signals['relay.pool_waiting'] =
+ signal(801, now)
+ return unhealthy
+ },
+ writeOutput: () => {}
+ }
+ )
+ expect(code).toBe(2)
+ expect(collections).toBe(1)
+ expect(waits).toBe(0)
+ expect(
+ JSON.parse(readFileSync(join(directory, 'incident-1.state.json'), 'utf8'))
+ ).toMatchObject({
+ completedAt: new Date(startedAt).toISOString(),
+ frozenAt: new Date(startedAt).toISOString(),
+ sampleCount: 1
+ })
+ })
+
+ it('requires --restart and preserves a latched freeze', async () => {
+ const directory = mkdtempSync(join(tmpdir(), 'relay-incident-cli-'))
+ directories.push(directory)
+ let now = startedAt
+ const args = [
+ '--incident-id',
+ 'incident-1',
+ ...selectorArguments,
+ '--duration-minutes',
+ '15',
+ '--output-directory',
+ directory
+ ]
+ await runIncidentMonitorCli(args, {
+ cwd: directory,
+ now: () => now,
+ wait: async (ms) => {
+ now += ms
+ },
+ collect: async () => {
+ const unhealthy = sample(now)
+ unhealthy.sources['relay-logs']!.signals['relay.pool_waiting'] = signal(801, now)
+ return unhealthy
+ },
+ writeOutput: () => {}
+ })
+ await expect(
+ runIncidentMonitorCli(args, {
+ cwd: directory,
+ now: () => now,
+ collect: async () => sample(now),
+ writeOutput: () => {}
+ })
+ ).rejects.toThrow('pass --restart')
+ const changedAdmission = [...args]
+ changedAdmission[changedAdmission.indexOf('--expected-selector-generation') + 1] = '2'
+ await expect(
+ runIncidentMonitorCli([...changedAdmission, '--restart'], {
+ cwd: directory,
+ now: () => now,
+ collect: async () => sample(now),
+ writeOutput: () => {}
+ })
+ ).rejects.toThrow('do not match')
+ await expect(
+ runIncidentMonitorCli([...args, '--restart'], {
+ cwd: directory,
+ now: () => now,
+ collect: async () => sample(now),
+ writeOutput: () => {}
+ })
+ ).resolves.toBe(2)
+ })
+
+ it('resumes a gracefully segmented monitor without resetting continuity', async () => {
+ const directory = mkdtempSync(join(tmpdir(), 'relay-incident-cli-'))
+ directories.push(directory)
+ let now = startedAt
+ const args = [
+ '--incident-id',
+ 'incident-1',
+ ...selectorArguments,
+ '--duration-minutes',
+ '15',
+ '--output-directory',
+ directory
+ ]
+ const dependencies = {
+ cwd: directory,
+ now: () => now,
+ wait: async (ms: number) => {
+ now += ms
+ },
+ collect: async () => sample(now),
+ writeOutput: () => {}
+ }
+ await expect(
+ runIncidentMonitorCli([...args, '--max-samples-this-run', '2'], dependencies)
+ ).resolves.toBe(0)
+ const statePath = join(directory, 'incident-1.state.json')
+ expect(JSON.parse(readFileSync(statePath, 'utf8'))).toMatchObject({
+ completedAt: null,
+ sampleCount: 2,
+ lastSampleAt: new Date(startedAt + 60_000).toISOString()
+ })
+ await expect(
+ runIncidentMonitorCli([...args, '--restart'], dependencies)
+ ).resolves.toBe(0)
+ expect(JSON.parse(readFileSync(statePath, 'utf8'))).toMatchObject({
+ completedAt: new Date(startedAt + 15 * 60_000).toISOString(),
+ windowSequence: 0,
+ sampleCount: 17
+ })
+ })
+})
diff --git a/cloud/apps/relay-ops/src/incident-monitor-cli.ts b/cloud/apps/relay-ops/src/incident-monitor-cli.ts
new file mode 100644
index 00000000000..adfe6cad480
--- /dev/null
+++ b/cloud/apps/relay-ops/src/incident-monitor-cli.ts
@@ -0,0 +1,495 @@
+import { randomUUID } from 'node:crypto'
+import {
+ appendFile,
+ chmod,
+ mkdir,
+ open,
+ readFile,
+ rename,
+ stat,
+ writeFile
+} from 'node:fs/promises'
+import { dirname, resolve } from 'node:path'
+import { pathToFileURL } from 'node:url'
+import { z } from 'zod'
+import { relayOpsEnvironment } from './environment-config.js'
+import { createGcloudClient } from './gcloud-client.js'
+import {
+ AdmissionSelectorSchema,
+ normalizeSelectorMembership,
+ type AdmissionSelector
+} from './incident-selector.js'
+import {
+ initialIncidentMonitorState,
+ preDrainDryRunPassed,
+ runIncidentMonitor,
+ type IncidentCheckpoint,
+ type IncidentSample,
+ type IncidentMonitorState
+} from './incident-monitor.js'
+import { createIncidentSampleCollector } from './incident-monitor-sources.js'
+
+const StateSchema = z.object({
+ schemaVersion: z.literal(4),
+ incidentId: z.string(),
+ environment: z.enum(['production', 'staging']),
+ expectedSelector: AdmissionSelectorSchema,
+ preDrainDryRun: z.boolean(),
+ migrationPolicy: z.enum(['strict', 'recover-forward', 'capacity-transition']),
+ recoverySourceCellId: z.string().nullable(),
+ capacityCellId: z.string().nullable(),
+ startedAt: z.string(),
+ windowStartedAt: z.string().nullable(),
+ windowSequence: z.number().int().nonnegative(),
+ durationMinutes: z.number().int(),
+ intervalMs: z.number().int(),
+ nextCheckpointIndex: z.number().int().nonnegative(),
+ sampleCount: z.number().int().nonnegative(),
+ totalSampleCount: z.number().int().nonnegative(),
+ lastSampleAt: z.string().nullable(),
+ continuityEvents: z.array(z.object({
+ recordedAt: z.string(),
+ windowSequence: z.number().int().nonnegative(),
+ // Pre-2026-09-05 state files predate tolerated freshness gaps.
+ tolerated: z.boolean().default(false),
+ failures: z.array(z.object({
+ code: z.string(),
+ source: z.enum(['active-probe', 'cloud-monitoring', 'relay-logs', 'director-admin']),
+ signal: z.string().optional(),
+ observed: z.number().optional(),
+ threshold: z.number().optional()
+ }))
+ })),
+ frozenAt: z.string().nullable(),
+ failures: z.array(z.object({
+ code: z.string(),
+ source: z.enum(['active-probe', 'cloud-monitoring', 'relay-logs', 'director-admin']),
+ signal: z.string().optional(),
+ observed: z.number().optional(),
+ threshold: z.number().optional()
+ })),
+ completedAt: z.string().nullable()
+})
+
+type CliOptions = {
+ environment: 'production' | 'staging'
+ incidentId: string
+ durationMinutes: number
+ intervalMs: number
+ expectedSelector: AdmissionSelector
+ stateFile: string
+ summaryFile: string
+ markdownFile: string
+ restart: boolean
+ preDrainDryRun: boolean
+ migrationPolicy: 'strict' | 'recover-forward' | 'capacity-transition'
+ recoverySourceCellId: string | null
+ capacityCellId: string | null
+ maxSamplesThisRun: number | null
+}
+
+function parsePositiveInteger(value: string | undefined, name: string): number {
+ const parsed = Number(value)
+ if (!Number.isSafeInteger(parsed) || parsed <= 0) {
+ throw new Error(`${name} must be a positive integer`)
+ }
+ return parsed
+}
+
+export function parseIncidentMonitorArguments(argv: string[], cwd = process.cwd()): CliOptions {
+ const flags = new Set(['restart', 'pre-drain-dry-run'])
+ const valueArguments = new Set([
+ 'duration-minutes',
+ 'environment',
+ 'expected-selector-generation',
+ 'expected-existing-only-cells',
+ 'expected-migration-only-cells',
+ 'expected-general-cells',
+ 'incident-id',
+ 'interval-seconds',
+ 'max-samples-this-run',
+ 'migration-policy',
+ 'output-directory',
+ 'recovery-source-cell-id',
+ 'capacity-cell-id'
+ ])
+ const values: Record = {}
+ const enabledFlags = new Set()
+ for (let index = 0; index < argv.length; index++) {
+ const argument = argv[index]
+ if (!argument?.startsWith('--')) throw new Error(`invalid argument ${argument ?? ''}`)
+ const name = argument.slice(2)
+ if (flags.has(name)) {
+ enabledFlags.add(name)
+ continue
+ }
+ if (!valueArguments.has(name)) throw new Error(`unknown argument --${name}`)
+ const value = argv[++index]
+ if (!value || value.startsWith('--')) throw new Error(`missing --${name} value`)
+ values[name] = value
+ }
+ const environment = z.enum(['production', 'staging']).parse(
+ values.environment ?? 'production'
+ )
+ const incidentId = values['incident-id'] ?? randomUUID()
+ if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]{7,127}$/.test(incidentId)) {
+ throw new Error('--incident-id must be 8-128 safe characters')
+ }
+ const selectorGeneration = Number(values['expected-selector-generation'])
+ if (!Number.isSafeInteger(selectorGeneration) || selectorGeneration < 0) {
+ throw new Error('--expected-selector-generation must be a nonnegative integer')
+ }
+ const configuredCells = new Set(
+ relayOpsEnvironment(environment).cells.map((cell) => cell.cellId)
+ )
+ const cellList = (name: string): string[] => {
+ const value = values[name]
+ if (value === undefined) throw new Error(`--${name} is required; use none for an empty set`)
+ return value === 'none' ? [] : value.split(',').map((cellId) => cellId.trim())
+ }
+ const expectedSelector = {
+ generation: selectorGeneration,
+ membership: normalizeSelectorMembership(
+ {
+ existingOnly: cellList('expected-existing-only-cells'),
+ migrationOnly: cellList('expected-migration-only-cells'),
+ general: cellList('expected-general-cells')
+ },
+ configuredCells
+ )
+ }
+ if (selectorGeneration === 0 && expectedSelector.membership.migrationOnly.length > 0) {
+ throw new Error('generation 0 cannot represent migration-only admission')
+ }
+ const preDrainDryRun = enabledFlags.has('pre-drain-dry-run')
+ const migrationPolicy = z.enum([
+ 'strict',
+ 'recover-forward',
+ 'capacity-transition'
+ ]).parse(
+ values['migration-policy'] ?? 'strict'
+ )
+ const recoverySourceCellId =
+ values['recovery-source-cell-id'] === undefined ||
+ values['recovery-source-cell-id'] === 'none'
+ ? null
+ : values['recovery-source-cell-id']
+ const capacityCellId =
+ values['capacity-cell-id'] === undefined || values['capacity-cell-id'] === 'none'
+ ? null
+ : values['capacity-cell-id']
+ const durationMinutes = parsePositiveInteger(
+ values['duration-minutes'] ?? (preDrainDryRun ? '15' : '90'),
+ '--duration-minutes'
+ )
+ const intervalMs =
+ parsePositiveInteger(values['interval-seconds'] ?? '60', '--interval-seconds') * 1_000
+ if (durationMinutes < 15 || durationMinutes > 90) {
+ throw new Error('--duration-minutes must be between 15 and 90')
+ }
+ if (intervalMs > 60_000) {
+ throw new Error('--interval-seconds must be between 1 and 60')
+ }
+ if (preDrainDryRun && durationMinutes !== 15) {
+ throw new Error('--pre-drain-dry-run requires --duration-minutes 15')
+ }
+ if (migrationPolicy !== 'strict' && !preDrainDryRun) {
+ throw new Error(`--migration-policy ${migrationPolicy} requires --pre-drain-dry-run`)
+ }
+ if (
+ migrationPolicy === 'recover-forward' &&
+ (
+ recoverySourceCellId === null ||
+ !configuredCells.has(recoverySourceCellId) ||
+ !expectedSelector.membership.existingOnly.includes(recoverySourceCellId)
+ )
+ ) {
+ throw new Error(
+ '--migration-policy recover-forward requires an existing-only --recovery-source-cell-id'
+ )
+ }
+ if (migrationPolicy !== 'recover-forward' && recoverySourceCellId !== null) {
+ throw new Error('--recovery-source-cell-id requires --migration-policy recover-forward')
+ }
+ if (
+ migrationPolicy === 'capacity-transition' &&
+ (
+ capacityCellId === null ||
+ !configuredCells.has(capacityCellId) ||
+ !expectedSelector.membership.general.includes(capacityCellId)
+ )
+ ) {
+ throw new Error(
+ '--migration-policy capacity-transition requires a general --capacity-cell-id'
+ )
+ }
+ if (migrationPolicy !== 'capacity-transition' && capacityCellId !== null) {
+ throw new Error('--capacity-cell-id requires --migration-policy capacity-transition')
+ }
+ const directory = resolve(cwd, values['output-directory'] ?? '.relay-incidents')
+ const maxSamplesThisRun = values['max-samples-this-run']
+ ? parsePositiveInteger(values['max-samples-this-run'], '--max-samples-this-run')
+ : null
+ return {
+ environment,
+ incidentId,
+ durationMinutes,
+ intervalMs,
+ expectedSelector,
+ stateFile: resolve(directory, `${incidentId}.state.json`),
+ summaryFile: resolve(directory, `${incidentId}.summaries.jsonl`),
+ markdownFile: resolve(directory, `${incidentId}.summary.md`),
+ restart: enabledFlags.has('restart'),
+ preDrainDryRun,
+ migrationPolicy,
+ recoverySourceCellId,
+ capacityCellId,
+ maxSamplesThisRun
+ }
+}
+
+async function fileExists(path: string): Promise {
+ try {
+ await stat(path)
+ return true
+ } catch {
+ return false
+ }
+}
+
+async function syncFile(path: string): Promise {
+ const handle = await open(path, 'r')
+ try {
+ await handle.sync()
+ } finally {
+ await handle.close()
+ }
+}
+
+async function persistState(path: string, state: IncidentMonitorState): Promise {
+ await mkdir(dirname(path), { recursive: true, mode: 0o700 })
+ await chmod(dirname(path), 0o700)
+ const temporaryPath = `${path}.tmp`
+ await writeFile(temporaryPath, `${JSON.stringify(state)}\n`, { mode: 0o600 })
+ await chmod(temporaryPath, 0o600)
+ await syncFile(temporaryPath)
+ await rename(temporaryPath, path)
+ await syncFile(path)
+}
+
+async function appendCheckpoint(path: string, checkpoint: IncidentCheckpoint): Promise {
+ await mkdir(dirname(path), { recursive: true, mode: 0o700 })
+ await chmod(dirname(path), 0o700)
+ if (await fileExists(path)) {
+ const existing = (await readFile(path, 'utf8'))
+ .trim()
+ .split('\n')
+ .filter(Boolean)
+ .map((line) => JSON.parse(line) as IncidentCheckpoint)
+ if (
+ existing.some((entry) =>
+ entry.windowSequence === checkpoint.windowSequence &&
+ entry.checkpointMinute === checkpoint.checkpointMinute
+ )
+ ) return
+ }
+ await appendFile(path, `${JSON.stringify(checkpoint)}\n`, { mode: 0o600 })
+ await chmod(path, 0o600)
+ await syncFile(path)
+}
+
+function markdownFailure(checkpoint: IncidentCheckpoint): string {
+ if (checkpoint.failures.length === 0) return 'none'
+ return checkpoint.failures
+ .map((failure) => {
+ const signal = failure.signal ? `/${failure.signal}` : ''
+ return `${failure.source}/${failure.code}${signal}`
+ })
+ .join(', ')
+}
+
+async function writeMarkdownSummary(
+ path: string,
+ summaryPath: string,
+ incidentId: string,
+ environment: string
+): Promise {
+ const checkpoints = (await readFile(summaryPath, 'utf8'))
+ .trim()
+ .split('\n')
+ .filter(Boolean)
+ .map((line) => JSON.parse(line) as IncidentCheckpoint)
+ const rows = checkpoints.map((checkpoint) => [
+ `| ${checkpoint.windowSequence}`,
+ checkpoint.checkpointMinute,
+ checkpoint.status,
+ checkpoint.sampleCount,
+ `${markdownFailure(checkpoint)} |`
+ ].join(' | '))
+ const markdown = [
+ '# Relay incident monitor',
+ '',
+ `Incident: \`${incidentId}\``,
+ '',
+ `Environment: \`${environment}\``,
+ '',
+ `Expected selector: \`${JSON.stringify(checkpoints[0]?.expectedSelector ?? null)}\``,
+ '',
+ `Migration policy: \`${checkpoints[0]?.migrationPolicy ?? 'unknown'}\``,
+ '',
+ `Recovery source: \`${checkpoints[0]?.recoverySourceCellId ?? 'none'}\``,
+ '',
+ `Capacity cell: \`${checkpoints[0]?.capacityCellId ?? 'none'}\``,
+ '',
+ '| Window | Minute | Status | Samples | Failures |',
+ '| ---: | ---: | --- | ---: | --- |',
+ ...rows,
+ ''
+ ].join('\n')
+ const temporaryPath = `${path}.tmp`
+ await writeFile(temporaryPath, markdown, { mode: 0o600 })
+ await chmod(temporaryPath, 0o600)
+ await syncFile(temporaryPath)
+ await rename(temporaryPath, path)
+ await syncFile(path)
+}
+
+export function suppliedIdentityToken(value: string | undefined): string | null {
+ if (value === undefined) return null
+ if (
+ value.length > 8_192 ||
+ !/^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/.test(value)
+ ) {
+ throw new Error('ORCA_RELAY_ADMIN_ID_TOKEN is invalid')
+ }
+ return value
+}
+
+async function readInitialState(
+ options: CliOptions,
+ now: () => number = Date.now
+): Promise {
+ const exists = await fileExists(options.stateFile)
+ if (exists && !options.restart) {
+ throw new Error('incident state already exists; pass --restart to resume it')
+ }
+ if (!exists && options.restart) throw new Error('no incident state exists to restart')
+ if (!exists) {
+ return initialIncidentMonitorState({
+ incidentId: options.incidentId,
+ environment: options.environment,
+ expectedSelector: options.expectedSelector,
+ preDrainDryRun: options.preDrainDryRun,
+ migrationPolicy: options.migrationPolicy,
+ recoverySourceCellId: options.recoverySourceCellId,
+ capacityCellId: options.capacityCellId,
+ startedAt: new Date(now()).toISOString(),
+ durationMinutes: options.durationMinutes,
+ intervalMs: options.intervalMs
+ })
+ }
+ const state = StateSchema.parse(
+ JSON.parse(await readFile(options.stateFile, 'utf8'))
+ ) as IncidentMonitorState
+ if (
+ state.incidentId !== options.incidentId ||
+ state.environment !== options.environment ||
+ JSON.stringify(state.expectedSelector) !== JSON.stringify(options.expectedSelector) ||
+ state.preDrainDryRun !== options.preDrainDryRun ||
+ state.migrationPolicy !== options.migrationPolicy ||
+ state.recoverySourceCellId !== options.recoverySourceCellId ||
+ state.capacityCellId !== options.capacityCellId ||
+ state.durationMinutes !== options.durationMinutes ||
+ state.intervalMs !== options.intervalMs
+ ) {
+ throw new Error('restart arguments do not match durable incident state')
+ }
+ return state
+}
+
+export async function runIncidentMonitorCli(
+ argv: string[],
+ dependencies: {
+ cwd?: string
+ now?: () => number
+ wait?: (ms: number) => Promise
+ gcloud?: ReturnType
+ collect?: () => Promise
+ writeOutput?: (value: string) => void
+ environment?: NodeJS.ProcessEnv
+ } = {}
+): Promise {
+ const options = parseIncidentMonitorArguments(argv, dependencies.cwd)
+ const state = await readInitialState(options, dependencies.now)
+ const baseGcloud = dependencies.gcloud ?? createGcloudClient()
+ const token = suppliedIdentityToken(
+ (dependencies.environment ?? process.env).ORCA_RELAY_ADMIN_ID_TOKEN
+ )
+ await persistState(options.stateFile, state)
+ const gcloud = token
+ ? { ...baseGcloud, identityToken: async () => token }
+ : baseGcloud
+ const collect =
+ dependencies.collect ??
+ createIncidentSampleCollector(gcloud, {
+ environment: options.environment,
+ expectedSelector: options.expectedSelector,
+ ...(dependencies.now ? { now: dependencies.now } : {})
+ })
+ let samplesThisRun = 0
+ const segmentedCollect = async (): Promise => {
+ try {
+ return await collect()
+ } finally {
+ samplesThisRun++
+ }
+ }
+ const wait =
+ dependencies.wait ??
+ ((ms: number) => new Promise((resolvePromise) => setTimeout(resolvePromise, ms)))
+ const segmentComplete = Symbol('segment-complete')
+ const output = dependencies.writeOutput ?? ((value) => process.stdout.write(`${value}\n`))
+ let result: IncidentMonitorState
+ try {
+ result = await runIncidentMonitor(state, {
+ now: dependencies.now ?? Date.now,
+ wait: async (ms) => {
+ if (
+ options.maxSamplesThisRun !== null &&
+ samplesThisRun >= options.maxSamplesThisRun
+ ) {
+ throw segmentComplete
+ }
+ await wait(ms)
+ },
+ collect: segmentedCollect,
+ persist: async (nextState) => await persistState(options.stateFile, nextState),
+ checkpoint: async (checkpoint) => {
+ await appendCheckpoint(options.summaryFile, checkpoint)
+ await writeMarkdownSummary(
+ options.markdownFile,
+ options.summaryFile,
+ options.incidentId,
+ options.environment
+ )
+ output(JSON.stringify(checkpoint))
+ }
+ })
+ } catch (error) {
+ if (error !== segmentComplete) throw error
+ return 0
+ }
+ if (options.preDrainDryRun && !preDrainDryRunPassed(result)) return 2
+ return result.frozenAt ? 2 : 0
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+ runIncidentMonitorCli(process.argv.slice(2))
+ .then((code) => {
+ process.exitCode = code
+ })
+ .catch((error: unknown) => {
+ console.error(error instanceof Error ? error.message : 'incident monitor failed')
+ process.exitCode = 1
+ })
+}
diff --git a/cloud/apps/relay-ops/src/incident-monitor-sources.test.ts b/cloud/apps/relay-ops/src/incident-monitor-sources.test.ts
new file mode 100644
index 00000000000..93000131327
--- /dev/null
+++ b/cloud/apps/relay-ops/src/incident-monitor-sources.test.ts
@@ -0,0 +1,461 @@
+import { describe, expect, it } from 'vitest'
+import { RELAY_OPS_ENVIRONMENTS } from './environment-config.js'
+import type { GcloudClient } from './gcloud-client.js'
+import { effectiveAdmissionState } from './incident-selector.js'
+import { INCIDENT_MONITOR_THRESHOLDS } from './incident-monitor.js'
+import {
+ directorSignals,
+ GOOGLE_METRICS,
+ readGoogleMetric,
+ readGoogleMetricWithEmptyRetry,
+ relayFiveMinuteDeltaSignal
+} from './incident-monitor-sources.js'
+
+const now = Date.parse('2026-07-28T10:00:00.000Z')
+const startAt = new Date(now - 5 * 60_000).toISOString()
+const endAt = new Date(now).toISOString()
+const productionCells = RELAY_OPS_ENVIRONMENTS.production.cells.map(
+ ({ cellId }) => cellId
+)
+
+describe('incident monitor sources', () => {
+ it('uses legacy booleans only at selector generation zero', () => {
+ const membership = {
+ existingOnly: ['c1'],
+ migrationOnly: [],
+ general: ['c2']
+ }
+ expect(
+ effectiveAdmissionState({ generation: 0, membership }, true, 'c1')
+ ).toBe('general')
+ expect(
+ effectiveAdmissionState({ generation: 1, membership }, true, 'c1')
+ ).toBe('existing-only')
+ })
+
+ it('sums DELTA metrics across the window and scopes every target exactly', async () => {
+ const filters: string[] = []
+ const fetchImpl: typeof fetch = async (input) => {
+ const url = new URL(String(input))
+ filters.push(url.searchParams.get('filter') ?? '')
+ return Response.json({
+ timeSeries: [
+ {
+ points: [
+ {
+ interval: { endTime: new Date(now - 120_000).toISOString() },
+ value: { int64Value: '2' }
+ },
+ {
+ interval: { endTime: new Date(now - 60_000).toISOString() },
+ value: { int64Value: '3' }
+ }
+ ]
+ }
+ ]
+ })
+ }
+ const environment = RELAY_OPS_ENVIRONMENTS.production
+ const directorErrors = GOOGLE_METRICS.find(
+ (definition) => definition.signal === 'director.errors'
+ )!
+ const deadlocks = GOOGLE_METRICS.find(
+ (definition) => definition.signal === 'cloud_sql.deadlocks'
+ )!
+ await expect(
+ readGoogleMetric(
+ environment,
+ directorErrors,
+ 'secret-access-token',
+ startAt,
+ endAt,
+ fetchImpl
+ )
+ ).resolves.toEqual({
+ value: 5,
+ observedAt: new Date(now - 60_000).toISOString()
+ })
+ await readGoogleMetric(
+ environment,
+ deadlocks,
+ 'secret-access-token',
+ startAt,
+ endAt,
+ fetchImpl
+ )
+ expect(filters[0]).toContain(
+ 'resource.label."service_name"="orca-cloud-relay"'
+ )
+ expect(filters[0]).toContain('metric.label."response_code"!="503"')
+ expect(filters[1]).toContain(
+ 'resource.label."database_id"="onorca-cloud:orca-cloud-auth-db"'
+ )
+ })
+
+ it('zero-fills an expired sparse lock-wait point', async () => {
+ let pointAt = now - INCIDENT_MONITOR_THRESHOLDS.cloudLockWaitCarryMs
+ const fetchImpl: typeof fetch = async () => Response.json({
+ timeSeries: [{
+ points: [{
+ interval: { endTime: new Date(pointAt).toISOString() },
+ value: { int64Value: '1' }
+ }]
+ }]
+ })
+ const definition = GOOGLE_METRICS.find(
+ ({ signal }) => signal === 'cloud_sql.lock_waits'
+ )!
+ await expect(readGoogleMetric(
+ RELAY_OPS_ENVIRONMENTS.production,
+ definition,
+ 'secret-access-token',
+ startAt,
+ endAt,
+ fetchImpl
+ )).resolves.toEqual({
+ value: 1,
+ observedAt: new Date(pointAt).toISOString()
+ })
+ await expect(readGoogleMetric(
+ RELAY_OPS_ENVIRONMENTS.production,
+ definition,
+ 'secret-access-token',
+ startAt,
+ endAt,
+ fetchImpl,
+ () => now + 3_000
+ )).resolves.toEqual({
+ value: 1,
+ observedAt: new Date(pointAt).toISOString()
+ })
+ pointAt--
+ await expect(readGoogleMetric(
+ RELAY_OPS_ENVIRONMENTS.production,
+ definition,
+ 'secret-access-token',
+ startAt,
+ endAt,
+ fetchImpl
+ )).resolves.toEqual({ value: 0, observedAt: endAt })
+ })
+
+ it('freshens a sparse zero without masking a recent nonzero lock wait', async () => {
+ let value = 0
+ const pointAt = now - INCIDENT_MONITOR_THRESHOLDS.cloudLockWaitCarryMs
+ const readAt = now + 11_879
+ const fetchImpl: typeof fetch = async () => Response.json({
+ timeSeries: [{
+ points: [{
+ interval: { endTime: new Date(pointAt).toISOString() },
+ value: { int64Value: String(value) }
+ }]
+ }]
+ })
+ const definition = GOOGLE_METRICS.find(
+ ({ signal }) => signal === 'cloud_sql.lock_waits'
+ )!
+
+ const sparseZero = await readGoogleMetric(
+ RELAY_OPS_ENVIRONMENTS.production,
+ definition,
+ 'secret-access-token',
+ startAt,
+ endAt,
+ fetchImpl,
+ () => readAt
+ )
+ expect(sparseZero).toEqual({
+ value: 0,
+ observedAt: new Date(readAt).toISOString()
+ })
+ expect(readAt - pointAt).toBe(191_879)
+ expect(readAt - Date.parse(sparseZero!.observedAt)).toBe(0)
+
+ value = 20
+ await expect(readGoogleMetric(
+ RELAY_OPS_ENVIRONMENTS.production,
+ definition,
+ 'secret-access-token',
+ startAt,
+ endAt,
+ fetchImpl,
+ () => readAt
+ )).resolves.toEqual({
+ value: 20,
+ observedAt: new Date(pointAt).toISOString()
+ })
+ })
+
+ it('preserves a recent nonzero lock wait across staggered series', async () => {
+ const nonzeroAt = now - 179_000
+ const zeroAt = now - 178_000
+ const definition = GOOGLE_METRICS.find(
+ ({ signal }) => signal === 'cloud_sql.lock_waits'
+ )!
+ const metric = await readGoogleMetric(
+ RELAY_OPS_ENVIRONMENTS.production,
+ definition,
+ 'secret-access-token',
+ startAt,
+ endAt,
+ async () => Response.json({
+ timeSeries: [
+ {
+ points: [{
+ interval: { endTime: new Date(nonzeroAt).toISOString() },
+ value: { int64Value: '7' }
+ }]
+ },
+ {
+ points: [{
+ interval: { endTime: new Date(zeroAt).toISOString() },
+ value: { int64Value: '0' }
+ }]
+ }
+ ]
+ })
+ )
+
+ expect(metric).toEqual({
+ value: 7,
+ observedAt: new Date(nonzeroAt).toISOString()
+ })
+
+ await expect(readGoogleMetric(
+ RELAY_OPS_ENVIRONMENTS.production,
+ definition,
+ 'secret-access-token',
+ startAt,
+ endAt,
+ async () => Response.json({
+ timeSeries: [{
+ points: [
+ {
+ interval: { endTime: new Date(nonzeroAt).toISOString() },
+ value: { int64Value: '7' }
+ },
+ {
+ interval: { endTime: new Date(zeroAt).toISOString() },
+ value: { int64Value: '0' }
+ }
+ ]
+ }]
+ })
+ )).resolves.toEqual({ value: 0, observedAt: endAt })
+ })
+
+ it('retries an empty required metric without weakening its value', async () => {
+ let calls = 0
+ const waits: number[] = []
+ const definition = GOOGLE_METRICS.find(
+ ({ signal }) => signal === 'cloud_sql.cpu'
+ )!
+ await expect(readGoogleMetricWithEmptyRetry(
+ RELAY_OPS_ENVIRONMENTS.production,
+ definition,
+ 'secret-access-token',
+ startAt,
+ endAt,
+ async () => Response.json(calls++ === 0
+ ? { timeSeries: [] }
+ : {
+ timeSeries: [{
+ points: [{
+ interval: { endTime: new Date(now - 60_000).toISOString() },
+ value: { doubleValue: 0.81 }
+ }]
+ }]
+ }),
+ () => now,
+ async (ms) => { waits.push(ms) }
+ )).resolves.toEqual({
+ value: 0.81,
+ observedAt: new Date(now - 60_000).toISOString()
+ })
+ expect(calls).toBe(2)
+ expect(waits).toEqual([2_000])
+ })
+
+ it('still reports a required metric missing after bounded retries', async () => {
+ let calls = 0
+ const waits: number[] = []
+ const definition = GOOGLE_METRICS.find(
+ ({ signal }) => signal === 'director.concurrency'
+ )!
+ await expect(readGoogleMetricWithEmptyRetry(
+ RELAY_OPS_ENVIRONMENTS.production,
+ definition,
+ 'secret-access-token',
+ startAt,
+ endAt,
+ async () => {
+ calls++
+ return Response.json({ timeSeries: [] })
+ },
+ () => now,
+ async (ms) => { waits.push(ms) }
+ )).resolves.toBeNull()
+ expect(calls).toBe(3)
+ expect(waits).toEqual([2_000, 2_000])
+ })
+
+ it('timestamps sparse retry aggregates at query completion', () => {
+ expect(relayFiveMinuteDeltaSignal({
+ available: true,
+ points: [
+ { at: new Date(now - 22 * 60_000).toISOString(), value: 7 },
+ { at: new Date(now - 4 * 60_000).toISOString(), value: 2 }
+ ]
+ }, endAt)).toEqual({ value: 2, observedAt: endAt })
+ expect(relayFiveMinuteDeltaSignal({
+ available: true,
+ points: [{ at: new Date(now - 22 * 60_000).toISOString(), value: 7 }]
+ }, endAt)).toEqual({ value: 0, observedAt: endAt })
+ expect(relayFiveMinuteDeltaSignal({
+ available: false,
+ points: []
+ }, endAt)).toBeNull()
+ })
+
+ // Why: the per-region cell latency bar is only correct if the tfvars region
+ // reaches the evaluator on every cell expectation.
+ it('carries the configured region onto every cell expectation', async () => {
+ const gcloud: GcloudClient = {
+ accessToken: async () => 'unused',
+ identityToken: async () => 'unused'
+ }
+ const selector = {
+ generation: 1,
+ membership: {
+ existingOnly: [],
+ migrationOnly: [],
+ general: productionCells
+ }
+ }
+ const fetchImpl: typeof fetch = async (_input, init) => {
+ const body = JSON.parse(String(init?.body)) as { cellId?: string; sourceCellId?: string }
+ if (!body.cellId && !body.sourceCellId) return Response.json({ selector })
+ if (body.cellId) {
+ return Response.json({
+ status: {
+ enabled: true,
+ connectionCapacity: { hardCap: 600 },
+ runtime: { lastHeartbeatAt: now - 1_000, heartbeatFresh: true }
+ }
+ })
+ }
+ return Response.json({
+ blocked: 0,
+ blockedExpiredUnregistered: 0,
+ registeredTargetInactive: 0
+ })
+ }
+ const result = await directorSignals('production', selector, gcloud, now, fetchImpl)
+ const regionById = new Map(result.cells.map((cell) => [cell.cellId, cell.region]))
+ expect(regionById.get('production-gce-c1')).toBe('us-central1')
+ expect(regionById.get('production-gce-c27')).toBe('asia-east2')
+ expect(result.cells).toHaveLength(productionCells.length)
+ for (const cell of RELAY_OPS_ENVIRONMENTS.production.cells) {
+ expect(regionById.get(cell.cellId)).toBe(cell.region)
+ }
+ })
+
+ it('aggregates admin state without returning tokens or response identities', async () => {
+ const identityToken = 'secret.header.signature'
+ const sensitiveIdentity = 'user@example.test'
+ const gcloud: GcloudClient = {
+ accessToken: async () => 'unused',
+ identityToken: async () => identityToken
+ }
+ let activeRequests = 0
+ let maximumActiveRequests = 0
+ let requestCount = 0
+ const fetchImpl: typeof fetch = async (_input, init) => {
+ requestCount++
+ activeRequests++
+ maximumActiveRequests = Math.max(maximumActiveRequests, activeRequests)
+ expect(new Headers(init?.headers).get('authorization')).toBe(
+ `Bearer ${identityToken}`
+ )
+ const body = JSON.parse(String(init?.body)) as {
+ cellId?: string
+ sourceCellId?: string
+ targetCellId?: string
+ }
+ await Promise.resolve()
+ activeRequests--
+ if (!body.cellId && !body.sourceCellId) {
+ return Response.json({
+ selector: {
+ generation: 1,
+ membership: {
+ existingOnly: productionCells.slice(2),
+ migrationOnly: ['production-gce-c2'],
+ general: ['production-gce-c1']
+ }
+ }
+ })
+ }
+ if (body.cellId) {
+ return Response.json({
+ status: {
+ // Selector-era monitoring must ignore this legacy compatibility bit.
+ enabled: body.cellId !== 'production-gce-c1',
+ connectionCapacity: {
+ hardCap: body.cellId === 'production-gce-c1' ? 1_000 : 600
+ },
+ runtime: {
+ lastHeartbeatAt: now - 1_000,
+ heartbeatFresh: true
+ },
+ userId: sensitiveIdentity
+ }
+ })
+ }
+ return Response.json({
+ blocked: 0,
+ blockedExpiredUnregistered:
+ body.sourceCellId === 'production-gce-c1' &&
+ body.targetCellId === 'production-gce-c2'
+ ? 1
+ : 0,
+ registeredTargetInactive: 0,
+ userId: sensitiveIdentity
+ })
+ }
+ const result = await directorSignals(
+ 'production',
+ {
+ generation: 1,
+ membership: {
+ existingOnly: productionCells.slice(2),
+ migrationOnly: ['production-gce-c2'],
+ general: ['production-gce-c1']
+ }
+ },
+ gcloud,
+ now,
+ fetchImpl
+ )
+ expect(requestCount).toBe(productionCells.length * 2)
+ expect(maximumActiveRequests).toBe(1)
+ expect(
+ result.source.signals['cell.production-gce-c1.migration_blocked']
+ ).toMatchObject({ value: 1 })
+ expect(result.cells.find((cell) => cell.cellId === 'production-gce-c1')).toMatchObject({
+ expectedAdmissionState: 'general'
+ })
+ expect(
+ result.source.signals['cell.production-gce-c1.admission_state']
+ ).toMatchObject({ value: 2 })
+ expect(
+ result.source.signals['cell.production-gce-c1.connection_hard_cap']
+ ).toMatchObject({ value: 1_000 })
+ expect(
+ result.source.signals['cell.production-gce-c2.admission_state']
+ ).toMatchObject({ value: 1 })
+ const serialized = JSON.stringify(result)
+ expect(serialized).not.toContain(identityToken)
+ expect(serialized).not.toContain(sensitiveIdentity)
+ })
+})
diff --git a/cloud/apps/relay-ops/src/incident-monitor-sources.ts b/cloud/apps/relay-ops/src/incident-monitor-sources.ts
new file mode 100644
index 00000000000..a93c01b6099
--- /dev/null
+++ b/cloud/apps/relay-ops/src/incident-monitor-sources.ts
@@ -0,0 +1,647 @@
+import { z } from 'zod'
+import { buildDashboardSnapshot } from './dashboard-snapshot.js'
+import type {
+ RelayOpsEnvironment,
+ RelayOpsEnvironmentId
+} from './environment-config.js'
+import { relayOpsEnvironment } from './environment-config.js'
+import type { GcloudClient } from './gcloud-client.js'
+import type { RelayMetricSnapshot } from './monitoring-snapshot.js'
+import {
+ AdmissionSelectorSchema,
+ effectiveAdmissionState,
+ normalizeSelectorMembership,
+ selectorCellState,
+ type AdmissionSelector,
+} from './incident-selector.js'
+import {
+ INCIDENT_MONITOR_THRESHOLDS,
+ type IncidentSample,
+ type IncidentSignal,
+ type IncidentSource
+} from './incident-monitor.js'
+
+const NumericSchema = z.union([z.number(), z.string()])
+ .transform(Number)
+ .pipe(z.number().finite())
+const MonitoringPointSchema = z.object({
+ interval: z.object({ endTime: z.string() }),
+ value: z.object({
+ doubleValue: NumericSchema.optional(),
+ int64Value: NumericSchema.optional(),
+ distributionValue: z.object({
+ mean: NumericSchema.optional(),
+ range: z.object({ max: NumericSchema.optional() }).optional()
+ }).optional()
+ })
+})
+const MonitoringResponseSchema = z.object({
+ timeSeries: z.array(z.object({ points: z.array(MonitoringPointSchema) })).default([]),
+ nextPageToken: z.string().optional()
+})
+const CellStatusSchema = z.object({
+ status: z.object({
+ enabled: z.boolean(),
+ connectionCapacity: z
+ .object({ hardCap: z.number().int().positive() })
+ .nullable(),
+ runtime: z.object({
+ lastHeartbeatAt: z.number(),
+ heartbeatFresh: z.boolean()
+ }).nullable()
+ })
+})
+const SelectorStatusSchema = z.object({
+ selector: AdmissionSelectorSchema
+})
+const MigrationStatusSchema = z.object({
+ blocked: z.number().int().nonnegative(),
+ registeredTargetInactive: z.number().int().nonnegative(),
+ blockedExpiredUnregistered: z.number().int().nonnegative()
+})
+
+export type GoogleMetricDefinition = {
+ signal: string
+ type: string
+ resourceFilter: string
+ aggregation: 'latest-max' | 'latest-sum' | 'window-sum'
+ emptyIsZero?: boolean
+ zeroAfterMs?: number
+}
+
+export const GOOGLE_METRICS: GoogleMetricDefinition[] = [
+ {
+ signal: 'cloud_sql.cpu',
+ type: 'cloudsql.googleapis.com/database/cpu/utilization',
+ resourceFilter: 'resource.type="cloudsql_database"',
+ aggregation: 'latest-max'
+ },
+ {
+ signal: 'cloud_sql.memory',
+ type: 'cloudsql.googleapis.com/database/memory/utilization',
+ resourceFilter: 'resource.type="cloudsql_database"',
+ aggregation: 'latest-max'
+ },
+ {
+ signal: 'cloud_sql.backends',
+ type: 'cloudsql.googleapis.com/database/postgresql/num_backends',
+ resourceFilter: 'resource.type="cloudsql_database"',
+ aggregation: 'latest-sum'
+ },
+ {
+ signal: 'cloud_sql.lock_waits',
+ type: 'cloudsql.googleapis.com/database/postgresql/backends_in_wait',
+ resourceFilter:
+ 'resource.type="cloudsql_database" AND metric.label."wait_event_type"="Lock"',
+ aggregation: 'latest-max',
+ emptyIsZero: true,
+ zeroAfterMs: INCIDENT_MONITOR_THRESHOLDS.cloudLockWaitCarryMs
+ },
+ {
+ signal: 'cloud_sql.deadlocks',
+ type: 'cloudsql.googleapis.com/database/postgresql/deadlock_count',
+ resourceFilter: 'resource.type="cloudsql_database"',
+ aggregation: 'window-sum',
+ emptyIsZero: true
+ },
+ {
+ signal: 'director.instances',
+ type: 'run.googleapis.com/container/instance_count',
+ resourceFilter: 'resource.type="cloud_run_revision"',
+ aggregation: 'latest-sum'
+ },
+ {
+ signal: 'director.cpu',
+ type: 'run.googleapis.com/container/cpu/utilizations',
+ resourceFilter: 'resource.type="cloud_run_revision"',
+ aggregation: 'latest-max'
+ },
+ {
+ signal: 'director.memory',
+ type: 'run.googleapis.com/container/memory/utilizations',
+ resourceFilter: 'resource.type="cloud_run_revision"',
+ aggregation: 'latest-max'
+ },
+ {
+ signal: 'director.concurrency',
+ type: 'run.googleapis.com/container/max_request_concurrencies',
+ resourceFilter:
+ 'resource.type="cloud_run_revision" AND metric.label."state"="active"',
+ aggregation: 'latest-max'
+ },
+ {
+ signal: 'director.errors',
+ type: 'run.googleapis.com/request_count',
+ resourceFilter:
+ 'resource.type="cloud_run_revision" AND metric.label."response_code_class"="5xx" AND metric.label."response_code"!="503"',
+ aggregation: 'window-sum',
+ emptyIsZero: true
+ },
+ {
+ signal: 'auth.errors',
+ type: 'run.googleapis.com/request_count',
+ resourceFilter:
+ 'resource.type="cloud_run_revision" AND metric.label."response_code_class"="5xx"',
+ aggregation: 'window-sum',
+ emptyIsZero: true
+ }
+]
+
+function pointValue(point: z.infer): number {
+ return (
+ point.value.doubleValue ??
+ point.value.int64Value ??
+ point.value.distributionValue?.range?.max ??
+ point.value.distributionValue?.mean ??
+ 0
+ )
+}
+
+function serviceFilter(signal: string, directorService: string, authService: string): string {
+ if (signal.startsWith('director.')) {
+ return `resource.label."service_name"="${directorService}"`
+ }
+ if (signal.startsWith('auth.')) {
+ return `resource.label."service_name"="${authService}"`
+ }
+ return ''
+}
+
+function targetFilter(
+ definition: GoogleMetricDefinition,
+ environment: RelayOpsEnvironment
+): string {
+ if (definition.signal.startsWith('cloud_sql.')) {
+ return `resource.label."database_id"="${environment.project}:${environment.sqlInstance}"`
+ }
+ return serviceFilter(
+ definition.signal,
+ environment.directorService,
+ environment.authService
+ )
+}
+
+async function googleJson(
+ fetchImpl: typeof fetch,
+ token: string,
+ url: URL | string,
+ init: RequestInit = {}
+): Promise {
+ const response = await fetchImpl(url, {
+ ...init,
+ headers: {
+ authorization: `Bearer ${token}`,
+ ...(init.body ? { 'content-type': 'application/json' } : {})
+ },
+ signal: AbortSignal.timeout(30_000)
+ })
+ if (!response.ok) throw new Error(`Google telemetry returned ${response.status}`)
+ return await response.json()
+}
+
+export async function readGoogleMetric(
+ environment: RelayOpsEnvironment,
+ definition: GoogleMetricDefinition,
+ token: string,
+ startAt: string,
+ endAt: string,
+ fetchImpl: typeof fetch,
+ now: () => number = () => Date.parse(endAt)
+): Promise {
+ const url = new URL(
+ `https://monitoring.googleapis.com/v3/projects/${environment.project}/timeSeries`
+ )
+ const filters = [
+ `metric.type="${definition.type}"`,
+ definition.resourceFilter,
+ targetFilter(definition, environment)
+ ].filter(Boolean)
+ url.searchParams.set('filter', filters.join(' AND '))
+ url.searchParams.set('interval.startTime', startAt)
+ url.searchParams.set('interval.endTime', endAt)
+ url.searchParams.set('view', 'FULL')
+ url.searchParams.set('pageSize', '1000')
+ const parsed = MonitoringResponseSchema.parse(
+ await googleJson(fetchImpl, token, url)
+ )
+ if (parsed.nextPageToken) throw new Error('Google metric pagination is incomplete')
+ const queryEndMs = Date.parse(endAt)
+ const readAtMs = Math.max(queryEndMs, now())
+ const readAt = new Date(readAtMs).toISOString()
+ const points = parsed.timeSeries.flatMap((series) => series.points)
+ if (points.length === 0) {
+ return definition.emptyIsZero ? { value: 0, observedAt: readAt } : null
+ }
+ const zeroAfterMs = definition.zeroAfterMs
+ if (definition.emptyIsZero && zeroAfterMs !== undefined) {
+ const latestSeriesPoints = parsed.timeSeries.flatMap((series) => {
+ const seriesNewestAt = Math.max(
+ ...series.points.map((point) => Date.parse(point.interval.endTime))
+ )
+ return series.points.filter(
+ (point) => Date.parse(point.interval.endTime) === seriesNewestAt
+ )
+ })
+ const futurePoints = latestSeriesPoints.filter(
+ (point) => Date.parse(point.interval.endTime) > queryEndMs
+ )
+ if (futurePoints.length > 0) {
+ return {
+ value: Math.max(...futurePoints.map(pointValue)),
+ observedAt: new Date(Math.max(
+ ...futurePoints.map((point) => Date.parse(point.interval.endTime))
+ )).toISOString()
+ }
+ }
+ const recentNonzero = latestSeriesPoints.filter((point) => {
+ const pointAt = Date.parse(point.interval.endTime)
+ return pointValue(point) > 0 && queryEndMs - pointAt <= zeroAfterMs
+ })
+ if (recentNonzero.length === 0) return { value: 0, observedAt: readAt }
+ return {
+ value: Math.max(...recentNonzero.map(pointValue)),
+ observedAt: new Date(Math.min(
+ ...recentNonzero.map((point) => Date.parse(point.interval.endTime))
+ )).toISOString()
+ }
+ }
+ const newestAt = Math.max(...points.map((point) => Date.parse(point.interval.endTime)))
+ const selected = definition.aggregation === 'window-sum'
+ ? points
+ : points.filter((point) => Date.parse(point.interval.endTime) === newestAt)
+ const values = selected.map(pointValue)
+ const value =
+ definition.aggregation !== 'latest-max'
+ ? values.reduce((total, entry) => total + entry, 0)
+ : Math.max(...values)
+ return {
+ value,
+ observedAt: new Date(newestAt).toISOString()
+ }
+}
+
+export async function readGoogleMetricWithEmptyRetry(
+ environment: RelayOpsEnvironment,
+ definition: GoogleMetricDefinition,
+ token: string,
+ startAt: string,
+ endAt: string,
+ fetchImpl: typeof fetch,
+ now: () => number = () => Date.parse(endAt),
+ wait: (ms: number) => Promise = async (ms) =>
+ await new Promise((resolve) => setTimeout(resolve, ms))
+): Promise {
+ for (let attempt = 0; attempt < 3; attempt++) {
+ const signal = await readGoogleMetric(
+ environment,
+ definition,
+ token,
+ startAt,
+ endAt,
+ fetchImpl,
+ now
+ )
+ if (signal !== null || definition.emptyIsZero) return signal
+ if (attempt < 2) await wait(2_000)
+ }
+ return null
+}
+
+function addSignal(
+ signals: Record,
+ name: string,
+ value: number | null,
+ observedAt: string | null
+): void {
+ if (value === null || observedAt === null) return
+ signals[name] = { value, observedAt }
+}
+
+function endpointSignals(
+ snapshot: Awaited>,
+ nowAt: string
+): IncidentSource {
+ const signals: Record = {}
+ const addEndpoint = (
+ prefix: string,
+ endpoint: { health: boolean | null; ready: boolean | null; latencyMs: number | null },
+ unavailableIsZero = false
+ ) => {
+ addSignal(
+ signals,
+ `${prefix}.health`,
+ endpoint.health === null ? (unavailableIsZero ? 0 : null) : Number(endpoint.health),
+ nowAt
+ )
+ addSignal(
+ signals,
+ `${prefix}.ready`,
+ endpoint.ready === null ? (unavailableIsZero ? 0 : null) : Number(endpoint.ready),
+ nowAt
+ )
+ addSignal(signals, `${prefix}.latency_ms`, endpoint.latencyMs, nowAt)
+ }
+ addEndpoint('director', snapshot.resources.directorEndpoint)
+ addEndpoint('auth', snapshot.resources.authEndpoint)
+ for (const cell of snapshot.resources.cells) {
+ addEndpoint(`cell.${cell.cellId}`, cell.endpoint, true)
+ }
+ return { observedAt: nowAt, signals }
+}
+
+export function relayFiveMinuteDeltaSignal(
+ metric: Pick,
+ endAt: string
+): IncidentSignal | null {
+ if (!metric.available) return null
+ const endMs = Date.parse(endAt)
+ if (!Number.isFinite(endMs)) throw new Error('Relay telemetry end time is invalid')
+ return {
+ value: metric.points
+ .filter((point) => {
+ const pointMs = Date.parse(point.at)
+ return pointMs >= endMs - 300_000 && pointMs <= endMs
+ })
+ .reduce((total, point) => total + point.value, 0),
+ observedAt: endAt
+ }
+}
+
+function relaySignals(
+ snapshot: Awaited>
+): IncidentSource {
+ const metrics = snapshot.monitoring.metrics
+ const signals: Record = {}
+ addSignal(
+ signals,
+ 'relay.pool_waiting',
+ metrics.db_waiters_max.latest,
+ metrics.db_waiters_max.latestAt
+ )
+ addSignal(
+ signals,
+ 'relay.pool_wait_ms',
+ metrics.db_wait_ms_max.latest,
+ metrics.db_wait_ms_max.latestAt
+ )
+ const retries = relayFiveMinuteDeltaSignal(
+ metrics.postgres_retries,
+ snapshot.monitoring.endAt
+ )
+ if (retries) signals['relay.postgres_retries'] = retries
+ const retryExhausted = relayFiveMinuteDeltaSignal(
+ metrics.postgres_retry_exhausted,
+ snapshot.monitoring.endAt
+ )
+ if (retryExhausted) signals['relay.postgres_retry_exhausted'] = retryExhausted
+ for (const cell of snapshot.resources.cells) {
+ addSignal(
+ signals,
+ `cell.${cell.cellId}.connections`,
+ metrics.total_connections.latestByCell[cell.cellId] ?? null,
+ metrics.total_connections.latestAt
+ )
+ addSignal(
+ signals,
+ `cell.${cell.cellId}.queued_bytes`,
+ metrics.queued_bytes.latestByCell[cell.cellId] ?? null,
+ metrics.queued_bytes.latestAt
+ )
+ }
+ const observedTimes = Object.values(signals).map((entry) => Date.parse(entry.observedAt))
+ if (observedTimes.length === 0) throw new Error('Relay telemetry is unavailable')
+ const observedAt = new Date(Math.max(...observedTimes)).toISOString()
+ return { observedAt, signals }
+}
+
+async function adminPost(
+ fetchImpl: typeof fetch,
+ origin: string,
+ token: string,
+ path: string,
+ body: unknown
+): Promise {
+ const response = await fetchImpl(`${origin}${path}`, {
+ method: 'POST',
+ headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
+ body: JSON.stringify(body),
+ signal: AbortSignal.timeout(30_000)
+ })
+ if (!response.ok) throw new Error(`Relay admin telemetry returned ${response.status}`)
+ return await response.json()
+}
+
+export async function directorSignals(
+ environmentId: RelayOpsEnvironmentId,
+ expectedSelector: AdmissionSelector,
+ gcloud: GcloudClient,
+ nowMs: number,
+ fetchImpl: typeof fetch
+): Promise<{
+ source: IncidentSource
+ selector: AdmissionSelector
+ cells: IncidentSample['cells']
+}> {
+ const environment = relayOpsEnvironment(environmentId)
+ if (!gcloud.identityToken) throw new Error('gcloud identity-token support is unavailable')
+ const token = await gcloud.identityToken(`${environment.directorOrigin}/v1/admin/drain`)
+ const configuredCellIds = new Set(environment.cells.map((cell) => cell.cellId))
+ const rawSelector = SelectorStatusSchema.parse(
+ await adminPost(
+ fetchImpl,
+ environment.directorOrigin,
+ token,
+ '/v1/admin/admission-selector/status',
+ { v: 1 }
+ )
+ ).selector
+ const selector = {
+ generation: rawSelector.generation,
+ membership: normalizeSelectorMembership(rawSelector.membership, configuredCellIds)
+ }
+ const statuses: Array<{
+ cell: RelayOpsEnvironment['cells'][number]
+ status: z.infer['status']
+ }> = []
+ for (const cell of environment.cells) {
+ statuses.push({
+ cell,
+ status: CellStatusSchema.parse(
+ await adminPost(fetchImpl, environment.directorOrigin, token, '/v1/admin/cell-status', {
+ v: 1,
+ cellId: cell.cellId
+ })
+ ).status
+ })
+ }
+ const migrationEntries: Array<{
+ sourceCellId: string
+ migration: z.infer
+ }> = []
+ const migrationTargets = new Set(expectedSelector.membership.migrationOnly)
+ for (const source of environment.cells) {
+ for (const target of environment.cells) {
+ if (source.cellId === target.cellId || !migrationTargets.has(target.cellId)) continue
+ migrationEntries.push({
+ sourceCellId: source.cellId,
+ migration: MigrationStatusSchema.parse(
+ await adminPost(
+ fetchImpl,
+ environment.directorOrigin,
+ token,
+ '/v1/admin/evacuation-status',
+ {
+ v: 1,
+ sourceCellId: source.cellId,
+ targetCellId: target.cellId,
+ completeReady: false
+ }
+ )
+ )
+ })
+ }
+ }
+ const migrationBySource = new Map()
+ for (const { sourceCellId, migration } of migrationEntries) {
+ const aggregate = migrationBySource.get(sourceCellId) ?? { blocked: 0, targetInactive: 0 }
+ aggregate.blocked += migration.blocked + migration.blockedExpiredUnregistered
+ aggregate.targetInactive += migration.registeredTargetInactive
+ migrationBySource.set(sourceCellId, aggregate)
+ }
+ const nowAt = new Date(nowMs).toISOString()
+ const signals: Record = {}
+ for (const { cell, status } of statuses) {
+ const prefix = `cell.${cell.cellId}`
+ const admissionState = effectiveAdmissionState(
+ selector,
+ status.enabled,
+ cell.cellId
+ )
+ addSignal(
+ signals,
+ `${prefix}.admission_state`,
+ ['existing-only', 'migration-only', 'general'].indexOf(admissionState),
+ nowAt
+ )
+ addSignal(
+ signals,
+ `${prefix}.connection_hard_cap`,
+ status.connectionCapacity?.hardCap ??
+ INCIDENT_MONITOR_THRESHOLDS.cellConnections,
+ nowAt
+ )
+ if (status.runtime) {
+ addSignal(
+ signals,
+ `${prefix}.heartbeat_fresh`,
+ Number(status.runtime.heartbeatFresh),
+ nowAt
+ )
+ addSignal(
+ signals,
+ `${prefix}.heartbeat_age_ms`,
+ Math.max(0, nowMs - status.runtime.lastHeartbeatAt),
+ nowAt
+ )
+ }
+ const migration = migrationBySource.get(cell.cellId) ?? { blocked: 0, targetInactive: 0 }
+ addSignal(signals, `${prefix}.migration_blocked`, migration.blocked, nowAt)
+ addSignal(
+ signals,
+ `${prefix}.migration_target_inactive`,
+ migration.targetInactive,
+ nowAt
+ )
+ }
+ return {
+ source: { observedAt: nowAt, signals },
+ selector,
+ cells: statuses.map(({ cell }) => ({
+ cellId: cell.cellId,
+ region: cell.region,
+ runtimeKnown: true,
+ powered: true,
+ expectedAdmissionState: selectorCellState(expectedSelector, cell.cellId)
+ }))
+ }
+}
+
+export type IncidentSampleCollectorOptions = {
+ environment: RelayOpsEnvironmentId
+ expectedSelector: AdmissionSelector
+ fetchImpl?: typeof fetch
+ now?: () => number
+}
+
+export function createIncidentSampleCollector(
+ gcloud: GcloudClient,
+ options: IncidentSampleCollectorOptions
+): () => Promise {
+ const fetchImpl = options.fetchImpl ?? fetch
+ const now = options.now ?? Date.now
+ return async () => {
+ const nowMs = now()
+ const nowAt = new Date(nowMs).toISOString()
+ const startAt = new Date(nowMs - 5 * 60_000).toISOString()
+ const environment = relayOpsEnvironment(options.environment)
+ const accessToken = gcloud.accessToken()
+ const cloudMetricEntries = accessToken.then(async (token) => await Promise.all(
+ GOOGLE_METRICS.map(async (definition) => [
+ definition.signal,
+ await readGoogleMetricWithEmptyRetry(
+ environment,
+ definition,
+ token,
+ startAt,
+ nowAt,
+ fetchImpl,
+ now
+ )
+ ] as const)
+ ))
+ const [snapshot, director, metricEntries] = await Promise.all([
+ buildDashboardSnapshot(options.environment, gcloud, {
+ windowMinutes: 5,
+ now: new Date(nowMs),
+ fetchImpl
+ }),
+ directorSignals(
+ options.environment,
+ options.expectedSelector,
+ gcloud,
+ nowMs,
+ fetchImpl
+ ),
+ cloudMetricEntries
+ ])
+ const cloudSignals = Object.fromEntries(
+ metricEntries.filter((entry): entry is [string, IncidentSignal] => entry[1] !== null)
+ )
+ const relay = relaySignals(snapshot)
+ const poweredByCell = new Map(
+ snapshot.resources.cells.map((cell) => [
+ cell.cellId,
+ {
+ runtimeKnown: cell.targetSize !== null,
+ powered: (cell.targetSize ?? 0) > 0
+ }
+ ])
+ )
+ return {
+ collectedAt: nowAt,
+ selector: director.selector,
+ expectedSelector: options.expectedSelector,
+ sources: {
+ 'active-probe': endpointSignals(snapshot, nowAt),
+ 'cloud-monitoring': { observedAt: nowAt, signals: cloudSignals },
+ 'relay-logs': relay,
+ 'director-admin': director.source
+ },
+ cells: director.cells.map((cell) => ({
+ ...cell,
+ runtimeKnown: poweredByCell.get(cell.cellId)?.runtimeKnown ?? false,
+ powered: poweredByCell.get(cell.cellId)?.powered ?? false
+ }))
+ }
+ }
+}
diff --git a/cloud/apps/relay-ops/src/incident-monitor.test.ts b/cloud/apps/relay-ops/src/incident-monitor.test.ts
new file mode 100644
index 00000000000..c1a073cde4a
--- /dev/null
+++ b/cloud/apps/relay-ops/src/incident-monitor.test.ts
@@ -0,0 +1,1096 @@
+import { describe, expect, it } from 'vitest'
+import {
+ evaluateIncidentSample,
+ INCIDENT_CHECKPOINT_MINUTES,
+ INCIDENT_FRESHNESS_TOLERANCE_SAMPLES,
+ INCIDENT_MONITOR_THRESHOLDS,
+ INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS,
+ initialIncidentMonitorState,
+ preDrainDryRunPassed,
+ runIncidentMonitor,
+ type IncidentSample
+} from './incident-monitor.js'
+
+const startedAt = Date.parse('2026-07-28T00:00:00.000Z')
+const selector = {
+ generation: 1,
+ membership: {
+ existingOnly: [],
+ migrationOnly: [],
+ general: ['production-gce-c1']
+ }
+}
+const signal = (value: number, at = startedAt) => ({
+ value,
+ observedAt: new Date(at).toISOString()
+})
+
+function healthySample(at = startedAt): IncidentSample {
+ const observedAt = new Date(at).toISOString()
+ return {
+ collectedAt: observedAt,
+ selector,
+ expectedSelector: selector,
+ cells: [{
+ cellId: 'production-gce-c1',
+ region: 'us-central1',
+ runtimeKnown: true,
+ powered: true,
+ expectedAdmissionState: 'general'
+ }],
+ sources: {
+ 'active-probe': {
+ observedAt,
+ signals: {
+ 'director.health': signal(1, at),
+ 'director.ready': signal(1, at),
+ 'director.latency_ms': signal(100, at),
+ 'auth.health': signal(1, at),
+ 'auth.ready': signal(1, at),
+ 'auth.latency_ms': signal(100, at),
+ 'cell.production-gce-c1.health': signal(1, at),
+ 'cell.production-gce-c1.ready': signal(1, at),
+ 'cell.production-gce-c1.latency_ms': signal(100, at)
+ }
+ },
+ 'cloud-monitoring': {
+ observedAt,
+ signals: {
+ 'cloud_sql.cpu': signal(0.2, at),
+ 'cloud_sql.memory': signal(0.3, at),
+ 'cloud_sql.backends': signal(12, at),
+ 'cloud_sql.lock_waits': signal(0, at),
+ 'cloud_sql.deadlocks': signal(0, at),
+ 'director.instances': signal(5, at),
+ 'director.cpu': signal(0.2, at),
+ 'director.memory': signal(0.3, at),
+ 'director.concurrency': signal(5, at),
+ 'director.errors': signal(0, at),
+ 'auth.errors': signal(0, at)
+ }
+ },
+ 'relay-logs': {
+ observedAt,
+ signals: {
+ 'relay.pool_waiting': signal(0, at),
+ 'relay.pool_wait_ms': signal(1, at),
+ 'relay.postgres_retries': signal(0, at),
+ 'relay.postgres_retry_exhausted': signal(0, at),
+ 'cell.production-gce-c1.connections': signal(100, at),
+ 'cell.production-gce-c1.queued_bytes': signal(0, at)
+ }
+ },
+ 'director-admin': {
+ observedAt,
+ signals: {
+ 'cell.production-gce-c1.admission_state': signal(2, at),
+ 'cell.production-gce-c1.heartbeat_fresh': signal(1, at),
+ 'cell.production-gce-c1.heartbeat_age_ms': signal(1_000, at),
+ 'cell.production-gce-c1.migration_blocked': signal(0, at),
+ 'cell.production-gce-c1.migration_target_inactive': signal(0, at)
+ }
+ }
+ }
+ }
+}
+
+describe('incident monitor evaluator', () => {
+ it('accepts a complete fresh sample at every exact boundary', () => {
+ const sample = healthySample()
+ sample.sources['active-probe']!.signals['director.latency_ms'] =
+ signal(INCIDENT_MONITOR_THRESHOLDS.endpointLatencyMs)
+ sample.sources['cloud-monitoring']!.signals['cloud_sql.cpu'] =
+ signal(INCIDENT_MONITOR_THRESHOLDS.cloudSqlCpuUtilization)
+ sample.sources['relay-logs']!.signals['relay.pool_wait_ms'] =
+ signal(INCIDENT_MONITOR_THRESHOLDS.relayPoolWaitMs)
+ sample.sources['relay-logs']!.signals['relay.postgres_retries'] =
+ signal(INCIDENT_MONITOR_THRESHOLDS.relayPostgresRetries)
+ sample.sources['cloud-monitoring']!.signals['cloud_sql.backends'] =
+ signal(INCIDENT_MONITOR_THRESHOLDS.cloudSqlBackends)
+ expect(evaluateIncidentSample(sample, startedAt)).toMatchObject({
+ status: 'green',
+ failures: []
+ })
+ })
+
+ // Why: the global relay_cells lock made retries a steady-state rate (24 h p99
+ // 1320/5min on 2026-09-04); the bar fences only unbounded growth beyond that.
+ it('tolerates the measured healthy retry rate and freezes above the bar', () => {
+ const healthy = healthySample()
+ healthy.sources['relay-logs']!.signals['relay.postgres_retries'] = signal(1504)
+ expect(evaluateIncidentSample(healthy, startedAt).status).toBe('green')
+
+ const incident = healthySample()
+ incident.sources['relay-logs']!.signals['relay.postgres_retries'] = signal(2001)
+ expect(evaluateIncidentSample(incident, startedAt)).toMatchObject({
+ status: 'freeze',
+ failures: [
+ expect.objectContaining({ signal: 'relay.postgres_retries', threshold: 2000 })
+ ]
+ })
+ })
+
+ // Why: since #18521 the request path fails fast on the cell-inventory lock, so
+ // exhaustion is a steady contention rate (post-#18521 p90 147/5min, max 220),
+ // not an anomaly. The bar bounds it below the 2026-08-23 incident peak of 467.
+ it('tolerates the measured healthy exhaustion rate and freezes above the bar', () => {
+ const healthy = healthySample()
+ healthy.sources['relay-logs']!.signals['relay.postgres_retry_exhausted'] = signal(220)
+ expect(evaluateIncidentSample(healthy, startedAt).status).toBe('green')
+
+ const atLimit = healthySample()
+ atLimit.sources['relay-logs']!.signals['relay.postgres_retry_exhausted'] = signal(300)
+ expect(evaluateIncidentSample(atLimit, startedAt).status).toBe('green')
+
+ const incident = healthySample()
+ incident.sources['relay-logs']!.signals['relay.postgres_retry_exhausted'] = signal(301)
+ expect(evaluateIncidentSample(incident, startedAt)).toMatchObject({
+ status: 'freeze',
+ failures: [
+ expect.objectContaining({ signal: 'relay.postgres_retry_exhausted', threshold: 300 })
+ ]
+ })
+ })
+
+ // Why: an asia-east2 cell's /ready reaches auth and Cloud SQL in us-central1, so
+ // from the US runner it measures p50 0.88 s / max 2.7 s and the flat 2 000 bar
+ // froze three healthy gates on 2026-09-05 (c27 at 2568/2668/2685 ms).
+ it('holds cell endpoint latency to a per-region bar', () => {
+ const asiaTail = healthySample()
+ asiaTail.cells[0]!.region = 'asia-east2'
+ asiaTail.sources['active-probe']!.signals['cell.production-gce-c1.latency_ms'] =
+ signal(2_685)
+ expect(evaluateIncidentSample(asiaTail, startedAt)).toMatchObject({
+ status: 'green',
+ failures: []
+ })
+
+ const asiaIncident = healthySample()
+ asiaIncident.cells[0]!.region = 'asia-east2'
+ asiaIncident.sources['active-probe']!.signals['cell.production-gce-c1.latency_ms'] =
+ signal(4_001)
+ expect(evaluateIncidentSample(asiaIncident, startedAt)).toMatchObject({
+ status: 'freeze',
+ failures: [
+ expect.objectContaining({
+ code: 'threshold_max',
+ source: 'active-probe',
+ signal: 'cell.production-gce-c1.latency_ms',
+ observed: 4_001,
+ threshold: 4_000
+ })
+ ]
+ })
+
+ const usIncident = healthySample()
+ usIncident.sources['active-probe']!.signals['cell.production-gce-c1.latency_ms'] =
+ signal(2_001)
+ expect(evaluateIncidentSample(usIncident, startedAt)).toMatchObject({
+ status: 'freeze',
+ failures: [
+ expect.objectContaining({
+ code: 'threshold_max',
+ signal: 'cell.production-gce-c1.latency_ms',
+ observed: 2_001,
+ threshold: 2_000
+ })
+ ]
+ })
+ })
+
+ it('allows missing auth readiness and legacy existing-only connections', () => {
+ const sample = healthySample()
+ const legacySelector = {
+ generation: 1,
+ membership: {
+ existingOnly: ['production-gce-c1'],
+ migrationOnly: [],
+ general: []
+ }
+ }
+ sample.selector = legacySelector
+ sample.expectedSelector = legacySelector
+ sample.cells[0]!.expectedAdmissionState = 'existing-only'
+ delete sample.sources['active-probe']!.signals['auth.ready']
+ sample.sources['relay-logs']!.signals['cell.production-gce-c1.connections'] =
+ signal(900)
+ sample.sources['director-admin']!.signals[
+ 'cell.production-gce-c1.admission_state'
+ ] = signal(0)
+ expect(evaluateIncidentSample(sample, startedAt)).toMatchObject({
+ status: 'green',
+ failures: []
+ })
+ })
+
+ it('fails loudly on every missing or stale source', () => {
+ const missing = healthySample()
+ delete missing.sources['relay-logs']
+ expect(evaluateIncidentSample(missing, startedAt).failures).toContainEqual({
+ code: 'source_missing',
+ source: 'relay-logs'
+ })
+ const stale = healthySample(
+ startedAt - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1
+ )
+ const failures = evaluateIncidentSample(stale, startedAt).failures
+ expect(failures.some((failure) => failure.source === 'cloud-monitoring')).toBe(true)
+ expect(failures.some((failure) => failure.source === 'active-probe')).toBe(true)
+ })
+
+ // Why: production run 33944873727 at 2026-09-05T04:46:09Z read
+ // cloud_sql.lock_waits 189 286 ms old and restarted a 15-minute window on
+ // Google's publish lag. Cloud SQL documents 60 s sampling plus up to 165 s of
+ // invisibility, so that age is Google's clock, not our fleet.
+ it('reads a 189-second cloud signal as fresh and holds the other sources at 180 s', () => {
+ const lagged = healthySample()
+ lagged.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] =
+ signal(0, startedAt - 189_286)
+ expect(evaluateIncidentSample(lagged, startedAt)).toMatchObject({
+ status: 'green',
+ failures: []
+ })
+ const laggedDirector = healthySample()
+ laggedDirector.sources['director-admin']!.observedAt =
+ new Date(startedAt - 189_286).toISOString()
+ expect(evaluateIncidentSample(laggedDirector, startedAt).failures).toContainEqual(
+ expect.objectContaining({ code: 'source_stale', source: 'director-admin' })
+ )
+ })
+
+ it('still fails a cloud signal past the documented publish lag', () => {
+ const dark = healthySample()
+ dark.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] = signal(
+ 0,
+ startedAt - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1
+ )
+ expect(evaluateIncidentSample(dark, startedAt).failures).toContainEqual(
+ expect.objectContaining({
+ code: 'signal_stale',
+ source: 'cloud-monitoring',
+ signal: 'cloud_sql.lock_waits'
+ })
+ )
+ })
+
+ it('freezes on SQL, director, relay pool, heartbeat, and migration breaches', () => {
+ const sample = healthySample()
+ sample.sources['cloud-monitoring']!.signals['cloud_sql.cpu'] = signal(0.81)
+ sample.sources['cloud-monitoring']!.signals['director.instances'] = signal(7)
+ sample.sources['relay-logs']!.signals['relay.pool_waiting'] = signal(801)
+ sample.sources['director-admin']!.signals[
+ 'cell.production-gce-c1.heartbeat_age_ms'
+ ] = signal(45_001)
+ sample.sources['director-admin']!.signals[
+ 'cell.production-gce-c1.migration_blocked'
+ ] = signal(1)
+ sample.sources['relay-logs']!.signals['cell.production-gce-c1.connections'] =
+ signal(501)
+ const evaluation = evaluateIncidentSample(sample, startedAt)
+ expect(evaluation.status).toBe('freeze')
+ expect(evaluation.failures.map((failure) => failure.signal)).toEqual(
+ expect.arrayContaining([
+ 'cloud_sql.cpu',
+ 'director.instances',
+ 'relay.pool_waiting',
+ 'cell.production-gce-c1.connections',
+ 'cell.production-gce-c1.heartbeat_age_ms',
+ 'cell.production-gce-c1.migration_blocked'
+ ])
+ )
+ })
+
+ it('uses each cell reported physical connection cap', () => {
+ const sample = healthySample()
+ sample.sources['director-admin']!.signals[
+ 'cell.production-gce-c1.connection_hard_cap'
+ ] = signal(1_000)
+ sample.sources['relay-logs']!.signals['cell.production-gce-c1.connections'] =
+ signal(999)
+
+ expect(evaluateIncidentSample(sample, startedAt).status).toBe('green')
+ sample.sources['relay-logs']!.signals['cell.production-gce-c1.connections'] =
+ signal(1_000)
+ expect(evaluateIncidentSample(sample, startedAt).status).toBe('freeze')
+ })
+
+ it('allows five active directors plus the warm rollback', () => {
+ const sample = healthySample()
+ sample.sources['cloud-monitoring']!.signals['director.instances'] = signal(6)
+ expect(evaluateIncidentSample(sample, startedAt).status).toBe('green')
+ })
+
+ it('allows bounded relay pool waiting below the latency ceiling', () => {
+ const sample = healthySample()
+ sample.sources['relay-logs']!.signals['relay.pool_waiting'] = signal(800)
+ sample.sources['relay-logs']!.signals['relay.pool_wait_ms'] = signal(2_500)
+ expect(evaluateIncidentSample(sample, startedAt)).toMatchObject({
+ status: 'green',
+ failures: []
+ })
+ sample.sources['relay-logs']!.signals['relay.pool_wait_ms'] = signal(2_501)
+ expect(evaluateIncidentSample(sample, startedAt).failures).toContainEqual({
+ code: 'threshold_max',
+ source: 'relay-logs',
+ signal: 'relay.pool_wait_ms',
+ observed: 2_501,
+ threshold: 2_500
+ })
+ })
+
+ it('bounds Cloud SQL backends above measured healthy peaks', () => {
+ const sample = healthySample()
+ sample.sources['cloud-monitoring']!.signals['cloud_sql.backends'] = signal(250)
+ expect(evaluateIncidentSample(sample, startedAt).status).toBe('green')
+ sample.sources['cloud-monitoring']!.signals['cloud_sql.backends'] = signal(251)
+ expect(evaluateIncidentSample(sample, startedAt).failures).toContainEqual({
+ code: 'threshold_max',
+ source: 'cloud-monitoring',
+ signal: 'cloud_sql.backends',
+ observed: 251,
+ threshold: 250
+ })
+ })
+
+ it('bounds SQL lock waiters and keeps deadlocks zero-tolerance', () => {
+ const sample = healthySample()
+ sample.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] = signal(20)
+ expect(evaluateIncidentSample(sample, startedAt)).toMatchObject({
+ status: 'green',
+ failures: []
+ })
+ sample.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] = signal(21)
+ expect(evaluateIncidentSample(sample, startedAt).failures).toContainEqual({
+ code: 'threshold_max',
+ source: 'cloud-monitoring',
+ signal: 'cloud_sql.lock_waits',
+ observed: 21,
+ threshold: 20
+ })
+ sample.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] = signal(0)
+ sample.sources['cloud-monitoring']!.signals['cloud_sql.deadlocks'] = signal(1)
+ expect(evaluateIncidentSample(sample, startedAt).failures).toContainEqual({
+ code: 'threshold_max',
+ source: 'cloud-monitoring',
+ signal: 'cloud_sql.deadlocks',
+ observed: 1,
+ threshold: 0
+ })
+ })
+
+ it('allows only registered target inactivity during forward recovery', () => {
+ const sample = healthySample()
+ sample.sources['director-admin']!.signals[
+ 'cell.production-gce-c1.migration_target_inactive'
+ ] = signal(40)
+ expect(evaluateIncidentSample(sample, startedAt).failures).toContainEqual({
+ code: 'threshold_max',
+ source: 'director-admin',
+ signal: 'cell.production-gce-c1.migration_target_inactive',
+ observed: 40,
+ threshold: 0
+ })
+ expect(
+ evaluateIncidentSample(
+ sample,
+ startedAt,
+ 'recover-forward',
+ 'production-gce-c1'
+ )
+ ).toMatchObject({
+ status: 'green',
+ failures: []
+ })
+ expect(
+ evaluateIncidentSample(
+ sample,
+ startedAt,
+ 'recover-forward',
+ 'production-gce-c2'
+ ).failures
+ ).toContainEqual(expect.objectContaining({
+ signal: 'cell.production-gce-c1.migration_target_inactive'
+ }))
+ sample.sources['director-admin']!.signals[
+ 'cell.production-gce-c1.migration_blocked'
+ ] = signal(1)
+ expect(
+ evaluateIncidentSample(
+ sample,
+ startedAt,
+ 'recover-forward',
+ 'production-gce-c1'
+ ).failures
+ ).toContainEqual({
+ code: 'threshold_max',
+ source: 'director-admin',
+ signal: 'cell.production-gce-c1.migration_blocked',
+ observed: 1,
+ threshold: 0
+ })
+ })
+
+ it('scopes registered target inactivity to the capacity cell', () => {
+ const sample = healthySample()
+ const scopedSelector = {
+ generation: 1,
+ membership: {
+ existingOnly: ['production-gce-c1'],
+ migrationOnly: [],
+ general: ['production-gce-c2', 'production-gce-c3']
+ }
+ }
+ sample.selector = scopedSelector
+ sample.expectedSelector = scopedSelector
+ sample.cells[0]!.expectedAdmissionState = 'existing-only'
+ sample.sources['director-admin']!.signals[
+ 'cell.production-gce-c1.admission_state'
+ ] = signal(0)
+ sample.cells.push({
+ cellId: 'production-gce-c2',
+ region: 'us-central1',
+ runtimeKnown: true,
+ powered: true,
+ expectedAdmissionState: 'general'
+ })
+ sample.cells.push({
+ cellId: 'production-gce-c3',
+ region: 'us-central1',
+ runtimeKnown: true,
+ powered: true,
+ expectedAdmissionState: 'general'
+ })
+ for (const sourceName of ['active-probe', 'relay-logs', 'director-admin'] as const) {
+ const signals = sample.sources[sourceName]!.signals
+ for (const [name, value] of Object.entries(signals)) {
+ if (name.includes('production-gce-c1')) {
+ for (const cellId of ['production-gce-c2', 'production-gce-c3']) {
+ signals[name.replace('production-gce-c1', cellId)] = value
+ }
+ }
+ }
+ }
+ for (const cellId of ['production-gce-c2', 'production-gce-c3']) {
+ sample.sources['director-admin']!.signals[
+ `cell.${cellId}.admission_state`
+ ] = signal(2)
+ }
+ sample.sources['director-admin']!.signals[
+ 'cell.production-gce-c1.migration_target_inactive'
+ ] = signal(40)
+ expect(
+ evaluateIncidentSample(
+ sample,
+ startedAt,
+ 'capacity-transition',
+ null,
+ 'production-gce-c2'
+ )
+ ).toMatchObject({ status: 'green', failures: [] })
+ expect(
+ evaluateIncidentSample(
+ sample,
+ startedAt,
+ 'capacity-transition',
+ null,
+ 'production-gce-c3'
+ )
+ ).toMatchObject({ status: 'green', failures: [] })
+ sample.sources['director-admin']!.signals[
+ 'cell.production-gce-c2.migration_target_inactive'
+ ] = signal(1)
+ expect(
+ evaluateIncidentSample(
+ sample,
+ startedAt,
+ 'capacity-transition',
+ null,
+ 'production-gce-c2'
+ ).failures
+ ).toContainEqual(expect.objectContaining({
+ signal: 'cell.production-gce-c2.migration_target_inactive'
+ }))
+ })
+
+ it('freezes when expected admission has no powered runtime', () => {
+ const sample = healthySample()
+ sample.cells[0]!.powered = false
+ const evaluation = evaluateIncidentSample(sample, startedAt)
+ expect(evaluation.failures).toContainEqual({
+ code: 'expected_admission_without_runtime',
+ source: 'director-admin',
+ signal: 'cell.production-gce-c1.powered',
+ observed: 0,
+ threshold: 1
+ })
+ })
+
+ it('ignores stale runtime signals for an expected offline existing-only cell', () => {
+ const sample = healthySample()
+ sample.cells[0] = {
+ ...sample.cells[0]!,
+ powered: false,
+ expectedAdmissionState: 'existing-only'
+ }
+ sample.expectedSelector = {
+ generation: sample.expectedSelector.generation,
+ membership: {
+ existingOnly: ['production-gce-c1'],
+ migrationOnly: [],
+ general: []
+ }
+ }
+ sample.selector = sample.expectedSelector
+ sample.sources['active-probe']!.signals['cell.production-gce-c1.health'] = signal(0)
+ sample.sources['active-probe']!.signals['cell.production-gce-c1.ready'] = signal(0)
+ sample.sources['director-admin']!.signals[
+ 'cell.production-gce-c1.admission_state'
+ ] = signal(0)
+ sample.sources['director-admin']!.signals[
+ 'cell.production-gce-c1.heartbeat_fresh'
+ ] = signal(0)
+ sample.sources['director-admin']!.signals[
+ 'cell.production-gce-c1.heartbeat_age_ms'
+ ] = signal(9_000_000)
+
+ expect(evaluateIncidentSample(sample, startedAt)).toMatchObject({
+ status: 'green',
+ failures: []
+ })
+ })
+
+ it('freezes when cell power inventory is unavailable', () => {
+ const sample = healthySample()
+ sample.cells[0]!.runtimeKnown = false
+ expect(evaluateIncidentSample(sample, startedAt).failures).toContainEqual({
+ code: 'runtime_power_unknown',
+ source: 'cloud-monitoring',
+ signal: 'cell.production-gce-c1.powered'
+ })
+ })
+
+ it('freezes on selector generation or tri-state membership drift', () => {
+ const generation = healthySample()
+ generation.selector = { ...generation.selector, generation: 2 }
+ expect(evaluateIncidentSample(generation, startedAt).failures).toContainEqual(
+ expect.objectContaining({ code: 'selector_mismatch' })
+ )
+
+ const membership = healthySample()
+ membership.selector = {
+ generation: 1,
+ membership: {
+ existingOnly: ['production-gce-c1'],
+ migrationOnly: [],
+ general: []
+ }
+ }
+ expect(evaluateIncidentSample(membership, startedAt).failures).toContainEqual(
+ expect.objectContaining({ code: 'selector_mismatch' })
+ )
+ })
+})
+
+describe('incident monitor lifecycle', () => {
+ it('persists the exact 90-minute checkpoints while polling every minute', async () => {
+ let now = startedAt
+ const checkpoints: number[] = []
+ const state = initialIncidentMonitorState({
+ incidentId: 'incident-1',
+ environment: 'production',
+ expectedSelector: selector,
+ preDrainDryRun: false,
+ migrationPolicy: 'strict',
+ recoverySourceCellId: null,
+ capacityCellId: null,
+ startedAt: new Date(startedAt).toISOString(),
+ durationMinutes: 90,
+ intervalMs: 60_000
+ })
+ const result = await runIncidentMonitor(state, {
+ now: () => now,
+ wait: async (ms) => {
+ now += ms
+ },
+ collect: async () => healthySample(now),
+ persist: async () => {},
+ checkpoint: async (summary) => {
+ checkpoints.push(summary.checkpointMinute)
+ }
+ })
+ expect(checkpoints).toEqual([...INCIDENT_CHECKPOINT_MINUTES])
+ expect(result.sampleCount).toBe(91)
+ expect(result.completedAt).not.toBeNull()
+ expect(result.frozenAt).toBeNull()
+ })
+
+ it('latches monitor freeze across a restart without rewriting its time', async () => {
+ let now = startedAt
+ let unhealthy = true
+ let persisted = initialIncidentMonitorState({
+ incidentId: 'incident-1',
+ environment: 'production',
+ expectedSelector: selector,
+ preDrainDryRun: false,
+ migrationPolicy: 'strict',
+ recoverySourceCellId: null,
+ capacityCellId: null,
+ startedAt: new Date(startedAt).toISOString(),
+ durationMinutes: 15,
+ intervalMs: 60_000
+ })
+ const stop = new Error('stop after first persistence')
+ await expect(
+ runIncidentMonitor(persisted, {
+ now: () => now,
+ wait: async () => {
+ throw stop
+ },
+ collect: async () => {
+ const sample = healthySample(now)
+ if (unhealthy) {
+ sample.sources['relay-logs']!.signals['relay.pool_waiting'] = signal(31, now)
+ }
+ return sample
+ },
+ persist: async (state) => {
+ persisted = structuredClone(state)
+ },
+ checkpoint: async () => {}
+ })
+ ).rejects.toThrow('stop after first persistence')
+ const frozenAt = persisted.frozenAt
+ unhealthy = false
+ now += 60_000
+ const result = await runIncidentMonitor(persisted, {
+ now: () => now,
+ wait: async (ms) => {
+ now += ms
+ },
+ collect: async () => healthySample(now),
+ persist: async () => {},
+ checkpoint: async () => {}
+ })
+ expect(result.frozenAt).toBe(frozenAt)
+ expect(preDrainDryRunPassed(result)).toBe(false)
+ })
+
+ it.each([15, 90])(
+ 'restarts a %i-minute continuous window after stale telemetry',
+ async (durationMinutes) => {
+ let now = startedAt
+ let staleSamples = INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1
+ const checkpoints: Array<[number, number]> = []
+ const state = initialIncidentMonitorState({
+ incidentId: 'incident-1',
+ environment: 'production',
+ expectedSelector: selector,
+ preDrainDryRun: durationMinutes === 15,
+ migrationPolicy: 'strict',
+ recoverySourceCellId: null,
+ capacityCellId: null,
+ startedAt: new Date(startedAt).toISOString(),
+ durationMinutes,
+ intervalMs: 60_000
+ })
+ const result = await runIncidentMonitor(state, {
+ now: () => now,
+ wait: async (ms) => {
+ now += ms
+ },
+ collect: async () => {
+ if (staleSamples > 0 && now >= startedAt + 5 * 60_000) {
+ staleSamples--
+ return healthySample(
+ now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1
+ )
+ }
+ return healthySample(now)
+ },
+ persist: async () => {},
+ checkpoint: async (summary) => {
+ checkpoints.push([summary.windowSequence, summary.checkpointMinute])
+ }
+ })
+ const restartMinute = 5 + INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1
+ expect(result.windowSequence).toBe(1)
+ expect(result.windowStartedAt).toBe(
+ new Date(startedAt + restartMinute * 60_000).toISOString()
+ )
+ expect(result.completedAt).toBe(
+ new Date(startedAt + (durationMinutes + restartMinute) * 60_000).toISOString()
+ )
+ expect(result.sampleCount).toBe(durationMinutes + 1)
+ expect(result.continuityEvents.map((event) => event.tolerated)).toEqual([
+ ...Array(INCIDENT_FRESHNESS_TOLERANCE_SAMPLES).fill(true),
+ false
+ ])
+ expect(result.continuityEvents.at(-1)!.failures).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ code: 'source_stale' })
+ ])
+ )
+ expect(checkpoints).toContainEqual([1, durationMinutes])
+ expect(result.frozenAt).toBeNull()
+ }
+ )
+
+ // Why: run 33944873727 on 2026-09-05 restarted at 04:46:09Z on a single
+ // 189-second cloud reading and then blew the 25-minute lineage cap, so a
+ // green fleet produced no verdict at all. One unread sample now continues the
+ // window; the sample is still checked against every threshold it can read.
+ it('carries a 15-minute window through a single stale cloud sample', async () => {
+ let now = startedAt
+ const state = initialIncidentMonitorState({
+ incidentId: 'incident-1',
+ environment: 'production',
+ expectedSelector: selector,
+ preDrainDryRun: true,
+ migrationPolicy: 'strict',
+ recoverySourceCellId: null,
+ capacityCellId: null,
+ startedAt: new Date(startedAt).toISOString(),
+ durationMinutes: 15,
+ intervalMs: 60_000
+ })
+ const result = await runIncidentMonitor(state, {
+ now: () => now,
+ wait: async (ms) => {
+ now += ms
+ },
+ collect: async () => {
+ const sample = healthySample(now)
+ if (now === startedAt + 10 * 60_000) {
+ sample.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] =
+ signal(0, now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1)
+ }
+ return sample
+ },
+ persist: async () => {},
+ checkpoint: async () => {}
+ })
+ expect(result.windowSequence).toBe(0)
+ expect(result.windowStartedAt).toBe(new Date(startedAt).toISOString())
+ expect(result.completedAt).toBe(new Date(startedAt + 15 * 60_000).toISOString())
+ expect(result.sampleCount).toBe(16)
+ expect(result.frozenAt).toBeNull()
+ expect(result.continuityEvents).toEqual([{
+ recordedAt: new Date(startedAt + 10 * 60_000).toISOString(),
+ windowSequence: 0,
+ tolerated: true,
+ failures: [expect.objectContaining({
+ code: 'signal_stale',
+ source: 'cloud-monitoring',
+ signal: 'cloud_sql.lock_waits'
+ })]
+ }])
+ expect(preDrainDryRunPassed(result)).toBe(true)
+ })
+
+ it('gives a signal a fresh budget only after it reads fresh again', async () => {
+ let now = startedAt
+ const staleMinutes = new Set([3, 5, 6, 9, 10])
+ const state = initialIncidentMonitorState({
+ incidentId: 'incident-1',
+ environment: 'production',
+ expectedSelector: selector,
+ preDrainDryRun: true,
+ migrationPolicy: 'strict',
+ recoverySourceCellId: null,
+ capacityCellId: null,
+ startedAt: new Date(startedAt).toISOString(),
+ durationMinutes: 15,
+ intervalMs: 60_000
+ })
+ const result = await runIncidentMonitor(state, {
+ now: () => now,
+ wait: async (ms) => {
+ now += ms
+ },
+ collect: async () => {
+ const sample = healthySample(now)
+ if (staleMinutes.has((now - startedAt) / 60_000)) {
+ sample.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] =
+ signal(0, now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1)
+ }
+ return sample
+ },
+ persist: async () => {},
+ checkpoint: async () => {}
+ })
+ expect(result.windowSequence).toBe(0)
+ expect(result.continuityEvents).toHaveLength(staleMinutes.size)
+ expect(result.continuityEvents.every((event) => event.tolerated)).toBe(true)
+ expect(preDrainDryRunPassed(result)).toBe(true)
+ })
+
+ it('does not hand a resumed monitor a fresh tolerance budget', async () => {
+ let now = startedAt + 3 * 60_000
+ const resumed = {
+ ...initialIncidentMonitorState({
+ incidentId: 'incident-1',
+ environment: 'production',
+ expectedSelector: selector,
+ preDrainDryRun: true,
+ migrationPolicy: 'strict',
+ recoverySourceCellId: null,
+ capacityCellId: null,
+ startedAt: new Date(startedAt).toISOString(),
+ durationMinutes: 15,
+ intervalMs: 60_000
+ }),
+ windowStartedAt: new Date(startedAt).toISOString(),
+ lastSampleAt: new Date(startedAt + 2 * 60_000).toISOString(),
+ sampleCount: 3,
+ totalSampleCount: 3,
+ continuityEvents: Array.from(
+ { length: INCIDENT_FRESHNESS_TOLERANCE_SAMPLES },
+ (_, index) => ({
+ recordedAt: new Date(startedAt + (index + 1) * 60_000).toISOString(),
+ windowSequence: 0,
+ tolerated: true,
+ failures: [{
+ code: 'signal_stale',
+ source: 'cloud-monitoring' as const,
+ signal: 'cloud_sql.lock_waits'
+ }]
+ })
+ )
+ }
+ const stop = new Error('stop after the resumed sample')
+ await expect(runIncidentMonitor(resumed, {
+ now: () => now,
+ wait: async () => {
+ throw stop
+ },
+ collect: async () => {
+ const sample = healthySample(now)
+ sample.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] =
+ signal(0, now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1)
+ return sample
+ },
+ persist: async (state) => {
+ expect(state.windowSequence).toBe(1)
+ expect(state.windowStartedAt).toBeNull()
+ expect(state.continuityEvents.at(-1)!.tolerated).toBe(false)
+ },
+ checkpoint: async () => {}
+ })).rejects.toThrow(stop)
+ })
+
+ it('freezes on a threshold breach that arrives with a tolerated stale signal', async () => {
+ let now = startedAt
+ const state = initialIncidentMonitorState({
+ incidentId: 'incident-1',
+ environment: 'production',
+ expectedSelector: selector,
+ preDrainDryRun: true,
+ migrationPolicy: 'strict',
+ recoverySourceCellId: null,
+ capacityCellId: null,
+ startedAt: new Date(startedAt).toISOString(),
+ durationMinutes: 15,
+ intervalMs: 60_000
+ })
+ const result = await runIncidentMonitor(state, {
+ now: () => now,
+ wait: async (ms) => {
+ now += ms
+ },
+ collect: async () => {
+ const sample = healthySample(now)
+ if (now === startedAt + 2 * 60_000) {
+ sample.sources['cloud-monitoring']!.signals['cloud_sql.lock_waits'] =
+ signal(0, now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1)
+ sample.sources['cloud-monitoring']!.signals['cloud_sql.cpu'] = signal(0.81, now)
+ }
+ return sample
+ },
+ persist: async () => {},
+ checkpoint: async () => {}
+ })
+ expect(result.frozenAt).toBe(new Date(startedAt + 2 * 60_000).toISOString())
+ expect(result.failures).toContainEqual(expect.objectContaining({
+ code: 'threshold_max',
+ signal: 'cloud_sql.cpu'
+ }))
+ expect(preDrainDryRunPassed(result)).toBe(false)
+ })
+
+ it('resets at the next fresh sample after a runner gap', async () => {
+ let now = startedAt + 10 * 60_000
+ const state = {
+ ...initialIncidentMonitorState({
+ incidentId: 'incident-1',
+ environment: 'production',
+ expectedSelector: selector,
+ preDrainDryRun: true,
+ migrationPolicy: 'strict',
+ recoverySourceCellId: null,
+ capacityCellId: null,
+ startedAt: new Date(startedAt).toISOString(),
+ durationMinutes: 15,
+ intervalMs: 60_000
+ }),
+ lastSampleAt: new Date(startedAt).toISOString(),
+ sampleCount: 1,
+ totalSampleCount: 1
+ }
+ const result = await runIncidentMonitor(state, {
+ now: () => now,
+ wait: async (ms) => {
+ now += ms
+ },
+ collect: async () => healthySample(now),
+ persist: async () => {},
+ checkpoint: async () => {}
+ })
+ expect(result.windowSequence).toBe(1)
+ expect(result.windowStartedAt).toBe(new Date(startedAt + 10 * 60_000).toISOString())
+ expect(result.continuityEvents[0]!.failures[0]!.code).toBe('monitor_gap')
+ expect(result.sampleCount).toBe(16)
+ })
+
+ it('fails a dry run after 25 total minutes of continuity resets', async () => {
+ let now = startedAt
+ const state = initialIncidentMonitorState({
+ incidentId: 'incident-1',
+ environment: 'production',
+ expectedSelector: selector,
+ preDrainDryRun: true,
+ migrationPolicy: 'strict',
+ recoverySourceCellId: null,
+ capacityCellId: null,
+ startedAt: new Date(startedAt).toISOString(),
+ durationMinutes: 15,
+ intervalMs: 60_000
+ })
+ let staleSamples = INCIDENT_FRESHNESS_TOLERANCE_SAMPLES + 1
+ const result = await runIncidentMonitor(state, {
+ now: () => now,
+ wait: async (ms) => {
+ now += ms
+ },
+ collect: async () => {
+ if (staleSamples > 0 && now >= startedAt + 10 * 60_000) {
+ staleSamples--
+ return healthySample(
+ now - INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs - 1
+ )
+ }
+ return healthySample(now)
+ },
+ persist: async () => {},
+ checkpoint: async () => {}
+ })
+
+ expect(result.completedAt).toBe(
+ new Date(startedAt + INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS).toISOString()
+ )
+ expect(result.frozenAt).not.toBeNull()
+ expect(result.windowSequence).toBe(1)
+ expect(result.sampleCount).toBe(13)
+ expect(result.failures).toContainEqual({
+ code: 'continuity_deadline_exceeded',
+ source: 'active-probe',
+ observed: INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS,
+ threshold: INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS
+ })
+ expect(preDrainDryRunPassed(result)).toBe(false)
+ })
+
+ it('fails an overdue resumed dry run before collecting again', async () => {
+ const now = startedAt + INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS + 1
+ let collections = 0
+ const state = initialIncidentMonitorState({
+ incidentId: 'incident-1',
+ environment: 'production',
+ expectedSelector: selector,
+ preDrainDryRun: true,
+ migrationPolicy: 'strict',
+ recoverySourceCellId: null,
+ capacityCellId: null,
+ startedAt: new Date(startedAt).toISOString(),
+ durationMinutes: 15,
+ intervalMs: 60_000
+ })
+ const result = await runIncidentMonitor(state, {
+ now: () => now,
+ wait: async () => {},
+ collect: async () => {
+ collections++
+ return healthySample(now)
+ },
+ persist: async () => {},
+ checkpoint: async () => {}
+ })
+
+ expect(collections).toBe(0)
+ expect(result.failures).toContainEqual(expect.objectContaining({
+ code: 'continuity_deadline_exceeded'
+ }))
+ })
+
+ it('requires a completed green 15-minute dry run', () => {
+ const state = {
+ ...initialIncidentMonitorState({
+ incidentId: 'incident-1',
+ environment: 'production',
+ expectedSelector: selector,
+ preDrainDryRun: true,
+ migrationPolicy: 'strict',
+ recoverySourceCellId: null,
+ capacityCellId: null,
+ startedAt: new Date(startedAt).toISOString(),
+ durationMinutes: 15,
+ intervalMs: 60_000
+ }),
+ sampleCount: 16,
+ completedAt: new Date(startedAt + 15 * 60_000).toISOString()
+ }
+ expect(preDrainDryRunPassed(state)).toBe(true)
+ expect(preDrainDryRunPassed({ ...state, frozenAt: state.startedAt })).toBe(false)
+ expect(preDrainDryRunPassed({
+ ...state,
+ completedAt: new Date(startedAt + INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS + 1).toISOString()
+ })).toBe(false)
+ })
+
+ it('keeps poll starts on the configured cadence after collection time', async () => {
+ let now = startedAt
+ const starts: number[] = []
+ const waits: number[] = []
+ const state = initialIncidentMonitorState({
+ incidentId: 'incident-1',
+ environment: 'production',
+ expectedSelector: selector,
+ preDrainDryRun: true,
+ migrationPolicy: 'strict',
+ recoverySourceCellId: null,
+ capacityCellId: null,
+ startedAt: new Date(startedAt).toISOString(),
+ durationMinutes: 15,
+ intervalMs: 60_000
+ })
+ await runIncidentMonitor(state, {
+ now: () => now,
+ wait: async (ms) => {
+ waits.push(ms)
+ now += ms
+ },
+ collect: async () => {
+ starts.push(now)
+ now += 15_000
+ return healthySample(now)
+ },
+ persist: async () => {},
+ checkpoint: async () => {}
+ })
+ expect(starts.slice(0, 3)).toEqual([
+ startedAt,
+ startedAt + 60_000,
+ startedAt + 120_000
+ ])
+ expect(waits[0]).toBe(45_000)
+ })
+})
diff --git a/cloud/apps/relay-ops/src/incident-monitor.ts b/cloud/apps/relay-ops/src/incident-monitor.ts
new file mode 100644
index 00000000000..0887bb2d1ee
--- /dev/null
+++ b/cloud/apps/relay-ops/src/incident-monitor.ts
@@ -0,0 +1,926 @@
+import type { RelayOpsRegion } from './environment-config.js'
+import {
+ exactAdmissionSelector,
+ type AdmissionSelector,
+ type AdmissionState
+} from './incident-selector.js'
+
+export const INCIDENT_MONITOR_THRESHOLDS = {
+ activeProbeMaxAgeMs: 60_000,
+ // Why: Cloud Monitoring publishes on Google's clock, not ours. Per the metric
+ // list read 2026-09-05, Cloud Run instance_count / cpu / memory /
+ // max_request_concurrencies / request_count are "Sampled every 60 seconds.
+ // After sampling, data is not visible for up to 120 seconds" (60+120=180 s),
+ // and Cloud SQL cpu / memory / num_backends / backends_in_wait /
+ // deadlock_count say "up to 165 seconds" (60+165=225 s). Window-sum signals
+ // age differently: observedAt is the newest point in the 5-minute query
+ // window, so a label series that stops emitting reads as 300 s old while its
+ // summed value is still complete. 330 s clears the worst of the three (the
+ // 300 s query window) plus ~30 s of collect-to-evaluate latency. The old
+ // 180 s bar restarted healthy 15-minute windows at 181 s, 189 s and 255 s on
+ // 2026-09-04/05, once burning the whole 25-minute lineage with no verdict.
+ cloudDataMaxAgeMs: 330_000,
+ // Why: the director admin API answers live on our own request, so hold its
+ // freshness bar where it sat while it shared cloudDataMaxAgeMs.
+ directorAdminMaxAgeMs: 180_000,
+ // Why: how long a nonzero backends-in-wait point is carried before it reads as
+ // zero. Held at the pre-2026-09-05 cloud bar: carrying it for the full
+ // cloudDataMaxAgeMs would hand the evaluator a point older than its own
+ // freshness bar as soon as collection latency is added.
+ cloudLockWaitCarryMs: 180_000,
+ relayLogMaxAgeMs: 180_000,
+ heartbeatMaxAgeMs: 45_000,
+ endpointLatencyMs: 2_000,
+ // Why: a cell's /ready fetches the auth JWKS and runs SELECT 1 against Cloud SQL,
+ // both in us-central1, so from the US runner asia-east2 cells measure p50 0.88 s /
+ // max 2.7 s against 0.08-0.5 s for us-central1. The flat 2 000 bar froze three
+ // healthy 15-minute gates on 2026-09-05 (c27 at 2568/2668/2685 ms); hard faults
+ // are still caught by the .health/.ready equal-1 checks and the 8 s fetch timeout.
+ cellEndpointLatencyMs: {
+ 'us-central1': 2_000,
+ 'asia-east2': 4_000
+ } as const satisfies Record,
+ cloudSqlCpuUtilization: 0.8,
+ cloudSqlMemoryUtilization: 0.9,
+ // Why: healthy latest-sum backends idle near 100 but spike to 216 in 1-minute
+ // bursts (~10 min/day exceeded the old bar of 160 on 2026-08-26, freezing a
+ // pre-drain gate on baseline noise). 250 clears measured healthy peaks while
+ // firing well before the verified 400-connection ceiling; the retry signals
+ // below discriminate incident-class contention.
+ cloudSqlBackends: 250,
+ // Bound the observed recovery load; deadlocks remain zero-tolerance.
+ cloudSqlLockWaits: 20,
+ cloudSqlDeadlocks: 0,
+ // Why: pool amplitude cannot discriminate the 2026-08-23 incident. Healthy
+ // fleet-wide bursts reach 43 waiters / 2.03s waits several times an hour,
+ // and a cell roll's reconnect surge peaks at 676 waiters, while the real
+ // incident peaked at 356 waiters and never crossed 2.5s (waits cap ~2s
+ // structurally). The old bars of 30/1000 froze pre-drain gates on baseline
+ // noise (~17% per 15-minute window). Incident-class contention is caught by
+ // the retry signals below at ~10x separation; these bars now fence only
+ // genuinely unbounded queueing, which grows past both.
+ relayPoolWaiting: 800,
+ relayPoolWaitMs: 2_500,
+ // Why: successful lock retries are the contention machinery working, not harm.
+ // Recalibrated 2026-09-04 from 300, which was set 2026-08-26 when healthy bursts
+ // reached 234/5min. The global relay_cells FOR UPDATE lock has since become the
+ // fleet's steady state: measured fleet-wide (director + cells, summed per five
+ // minutes) 2026-09-03T05Z..2026-09-04T05Z p50 430 / p90 924 / p99 1320 / max
+ // 1504, with 55% of windows over 300 and only 22% of 15-minute gates clean, so
+ // the bar blocked the very cell roll that carries the 500 ms lock wait (#18521)
+ // and the beginProof crash guard to the cells. The 2026-08-23 lock incident on
+ // this same metric peaked at 1510 in one window and 646 in the next, so it is
+ // not separable from today's contention by retries alone; it is caught by
+ // relayPostgresRetryExhausted (467 at the peak vs a 300 bar), director
+ // concurrency, and the pool bars. 2000 passes every healthy 15-minute window
+ // measured in the last 24 h and still fences unbounded growth. Re-tighten once
+ // the fleet is on the 500 ms lock wait and the baseline is re-measured.
+ relayPostgresRetries: 2000,
+ // Why: 300 per five minutes, recalibrated 2026-09-04 from a bar of zero that no
+ // production window has cleared since #18521 shipped to the director. That
+ // change cut the request-path cell-inventory wait from the 1 s pool lock_timeout
+ // to 500 ms, so a contended waiter now fails fast (one /v1/assign 503 with
+ // Retry-After, which the client retries) instead of succeeding slowly, and the
+ // exhaustion count became a steady-state contention rate rather than an
+ // anomaly. Measured fleet-wide (director + cells) per five minutes over
+ // 2026-09-03T03Z..2026-09-04T02Z: every one of 236 windows was non-zero;
+ // quiet hours p50 2 / max 36; pre-#18521 daytime p50 10 / p90 25 / max 87;
+ // post-#18521 p50 42 / p90 147 / max 220. The 2026-08-23 lock incident peaked
+ // at 467. 300 clears every measured healthy window and still sits below the
+ // incident shape; retries above fence only unbounded growth.
+ // User-facing /v1/assign 503 share did not move with #18521 (13.9% old image
+ // vs 12.3% new, same evening), so exhaustion is not a proxy for user harm.
+ relayPostgresRetryExhausted: 300,
+ // Why: public admission is a per-instance semaphore, so fleet assignment capacity is
+ // concurrency x instances. A floor of 1 let the 2026-08-04 collapse from five instances
+ // to two pass unnoticed, which is the exact failure this monitor exists to catch. Keep in
+ // step with relay_min_instances in infra/terraform/environments/production.tfvars.
+ directorInstancesMin: 5,
+ // Five serving instances plus one warm scale-to-zero rollback during recovery.
+ directorInstancesMax: 6,
+ directorCpuUtilization: 0.8,
+ directorMemoryUtilization: 0.8,
+ directorConcurrency: 64,
+ directorErrors: 0,
+ authErrors: 0,
+ // Why: 800 exceeded the 600 hard cap, so this could never trigger on a capped cell. 500 is
+ // the ordinary admission limit a cell actually stops at (600 cap - 100 control-rebind reserve).
+ cellConnections: 500,
+ cellQueuedBytes: 48 * 1024 * 1024,
+ migrationBlocked: 0
+} as const
+
+export const INCIDENT_CHECKPOINT_MINUTES = [0, 5, 15, 30, 45, 60, 75, 90] as const
+export const INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS = 25 * 60_000
+
+export type IncidentSourceName =
+ | 'active-probe'
+ | 'cloud-monitoring'
+ | 'relay-logs'
+ | 'director-admin'
+
+export type IncidentMigrationPolicy =
+ | 'strict'
+ | 'recover-forward'
+ | 'capacity-transition'
+
+export type IncidentSignal = {
+ value: number
+ observedAt: string
+}
+
+export type IncidentSource = {
+ observedAt: string
+ signals: Record
+}
+
+export type IncidentCellExpectation = {
+ cellId: string
+ region: RelayOpsRegion
+ runtimeKnown: boolean
+ powered: boolean
+ expectedAdmissionState: AdmissionState
+}
+
+export type IncidentSample = {
+ collectedAt: string
+ selector: AdmissionSelector
+ expectedSelector: AdmissionSelector
+ sources: Partial>
+ cells: IncidentCellExpectation[]
+}
+
+export type IncidentFailure = {
+ code: string
+ source: IncidentSourceName
+ signal?: string
+ observed?: number
+ threshold?: number
+}
+
+export type IncidentEvaluation = {
+ status: 'green' | 'freeze'
+ evaluatedAt: string
+ failures: IncidentFailure[]
+}
+
+export type IncidentCheckpoint = {
+ schemaVersion: 4
+ incidentId: string
+ environment: 'production' | 'staging'
+ expectedSelector: AdmissionSelector
+ preDrainDryRun: boolean
+ migrationPolicy: IncidentMigrationPolicy
+ recoverySourceCellId: string | null
+ capacityCellId: string | null
+ windowSequence: number
+ windowStartedAt: string
+ checkpointMinute: number
+ scheduledAt: string
+ recordedAt: string
+ status: 'green' | 'freeze'
+ frozenAt: string | null
+ sampleCount: number
+ failures: IncidentFailure[]
+ thresholds: typeof INCIDENT_MONITOR_THRESHOLDS
+}
+
+export type IncidentMonitorState = {
+ schemaVersion: 4
+ incidentId: string
+ environment: 'production' | 'staging'
+ expectedSelector: AdmissionSelector
+ preDrainDryRun: boolean
+ migrationPolicy: IncidentMigrationPolicy
+ recoverySourceCellId: string | null
+ capacityCellId: string | null
+ startedAt: string
+ windowStartedAt: string | null
+ windowSequence: number
+ durationMinutes: number
+ intervalMs: number
+ nextCheckpointIndex: number
+ sampleCount: number
+ totalSampleCount: number
+ lastSampleAt: string | null
+ continuityEvents: {
+ recordedAt: string
+ windowSequence: number
+ tolerated: boolean
+ failures: IncidentFailure[]
+ }[]
+ frozenAt: string | null
+ failures: IncidentFailure[]
+ completedAt: string | null
+}
+
+type NumericRule = {
+ source: IncidentSourceName
+ signal: string
+ comparison: 'max' | 'min' | 'equal'
+ threshold: number
+}
+
+const NUMERIC_RULES: NumericRule[] = [
+ { source: 'active-probe', signal: 'director.health', comparison: 'equal', threshold: 1 },
+ { source: 'active-probe', signal: 'director.ready', comparison: 'equal', threshold: 1 },
+ {
+ source: 'active-probe',
+ signal: 'director.latency_ms',
+ comparison: 'max',
+ threshold: INCIDENT_MONITOR_THRESHOLDS.endpointLatencyMs
+ },
+ { source: 'active-probe', signal: 'auth.health', comparison: 'equal', threshold: 1 },
+ {
+ source: 'active-probe',
+ signal: 'auth.latency_ms',
+ comparison: 'max',
+ threshold: INCIDENT_MONITOR_THRESHOLDS.endpointLatencyMs
+ },
+ {
+ source: 'cloud-monitoring',
+ signal: 'cloud_sql.cpu',
+ comparison: 'max',
+ threshold: INCIDENT_MONITOR_THRESHOLDS.cloudSqlCpuUtilization
+ },
+ {
+ source: 'cloud-monitoring',
+ signal: 'cloud_sql.memory',
+ comparison: 'max',
+ threshold: INCIDENT_MONITOR_THRESHOLDS.cloudSqlMemoryUtilization
+ },
+ {
+ source: 'cloud-monitoring',
+ signal: 'cloud_sql.backends',
+ comparison: 'max',
+ threshold: INCIDENT_MONITOR_THRESHOLDS.cloudSqlBackends
+ },
+ {
+ source: 'cloud-monitoring',
+ signal: 'director.instances',
+ comparison: 'min',
+ threshold: INCIDENT_MONITOR_THRESHOLDS.directorInstancesMin
+ },
+ {
+ source: 'cloud-monitoring',
+ signal: 'director.instances',
+ comparison: 'max',
+ threshold: INCIDENT_MONITOR_THRESHOLDS.directorInstancesMax
+ },
+ {
+ source: 'cloud-monitoring',
+ signal: 'director.cpu',
+ comparison: 'max',
+ threshold: INCIDENT_MONITOR_THRESHOLDS.directorCpuUtilization
+ },
+ {
+ source: 'cloud-monitoring',
+ signal: 'director.memory',
+ comparison: 'max',
+ threshold: INCIDENT_MONITOR_THRESHOLDS.directorMemoryUtilization
+ },
+ {
+ source: 'cloud-monitoring',
+ signal: 'director.concurrency',
+ comparison: 'max',
+ threshold: INCIDENT_MONITOR_THRESHOLDS.directorConcurrency
+ },
+ {
+ source: 'cloud-monitoring',
+ signal: 'director.errors',
+ comparison: 'max',
+ threshold: INCIDENT_MONITOR_THRESHOLDS.directorErrors
+ },
+ {
+ source: 'cloud-monitoring',
+ signal: 'auth.errors',
+ comparison: 'max',
+ threshold: INCIDENT_MONITOR_THRESHOLDS.authErrors
+ },
+ {
+ source: 'cloud-monitoring',
+ signal: 'cloud_sql.lock_waits',
+ comparison: 'max',
+ threshold: INCIDENT_MONITOR_THRESHOLDS.cloudSqlLockWaits
+ },
+ {
+ source: 'cloud-monitoring',
+ signal: 'cloud_sql.deadlocks',
+ comparison: 'max',
+ threshold: INCIDENT_MONITOR_THRESHOLDS.cloudSqlDeadlocks
+ },
+ {
+ source: 'relay-logs',
+ signal: 'relay.pool_waiting',
+ comparison: 'max',
+ threshold: INCIDENT_MONITOR_THRESHOLDS.relayPoolWaiting
+ },
+ {
+ source: 'relay-logs',
+ signal: 'relay.pool_wait_ms',
+ comparison: 'max',
+ threshold: INCIDENT_MONITOR_THRESHOLDS.relayPoolWaitMs
+ },
+ {
+ source: 'relay-logs',
+ signal: 'relay.postgres_retries',
+ comparison: 'max',
+ threshold: INCIDENT_MONITOR_THRESHOLDS.relayPostgresRetries
+ },
+ {
+ source: 'relay-logs',
+ signal: 'relay.postgres_retry_exhausted',
+ comparison: 'max',
+ threshold: INCIDENT_MONITOR_THRESHOLDS.relayPostgresRetryExhausted
+ }
+]
+
+const SOURCE_MAX_AGE: Record = {
+ 'active-probe': INCIDENT_MONITOR_THRESHOLDS.activeProbeMaxAgeMs,
+ 'cloud-monitoring': INCIDENT_MONITOR_THRESHOLDS.cloudDataMaxAgeMs,
+ 'relay-logs': INCIDENT_MONITOR_THRESHOLDS.relayLogMaxAgeMs,
+ 'director-admin': INCIDENT_MONITOR_THRESHOLDS.directorAdminMaxAgeMs
+}
+
+function ageMs(timestamp: string, nowMs: number): number {
+ const parsed = Date.parse(timestamp)
+ return Number.isFinite(parsed) ? nowMs - parsed : Number.POSITIVE_INFINITY
+}
+
+function addMissingSignal(
+ failures: IncidentFailure[],
+ source: IncidentSourceName,
+ signal: string
+): void {
+ failures.push({ code: 'signal_missing', source, signal })
+}
+
+function checkRule(
+ failures: IncidentFailure[],
+ source: IncidentSourceName,
+ signals: Record,
+ rule: NumericRule
+): void {
+ const signal = signals[rule.signal]
+ if (!signal) return addMissingSignal(failures, source, rule.signal)
+ const failed =
+ (rule.comparison === 'max' && signal.value > rule.threshold) ||
+ (rule.comparison === 'min' && signal.value < rule.threshold) ||
+ (rule.comparison === 'equal' && signal.value !== rule.threshold)
+ if (failed) {
+ failures.push({
+ code: `threshold_${rule.comparison}`,
+ source,
+ signal: rule.signal,
+ observed: signal.value,
+ threshold: rule.threshold
+ })
+ }
+}
+
+function checkCell(
+ failures: IncidentFailure[],
+ sample: IncidentSample,
+ cell: IncidentCellExpectation,
+ migrationPolicy: IncidentMigrationPolicy,
+ recoverySourceCellId: string | null,
+ capacityCellId: string | null
+): void {
+ const probe = sample.sources['active-probe']?.signals
+ const relay = sample.sources['relay-logs']?.signals
+ const admin = sample.sources['director-admin']?.signals
+ if (!cell.runtimeKnown) {
+ failures.push({
+ code: 'runtime_power_unknown',
+ source: 'cloud-monitoring',
+ signal: `cell.${cell.cellId}.powered`
+ })
+ }
+ if (
+ cell.runtimeKnown &&
+ cell.expectedAdmissionState !== 'existing-only' &&
+ !cell.powered
+ ) {
+ failures.push({
+ code: 'expected_admission_without_runtime',
+ source: 'director-admin',
+ signal: `cell.${cell.cellId}.powered`,
+ observed: 0,
+ threshold: 1
+ })
+ }
+ const checks = [
+ ['active-probe', probe, `cell.${cell.cellId}.health`, cell.powered ? 1 : 0, 'equal'],
+ ['active-probe', probe, `cell.${cell.cellId}.ready`, cell.powered ? 1 : 0, 'equal'],
+ [
+ 'active-probe',
+ probe,
+ `cell.${cell.cellId}.latency_ms`,
+ INCIDENT_MONITOR_THRESHOLDS.cellEndpointLatencyMs[cell.region],
+ 'max'
+ ],
+ [
+ 'director-admin',
+ admin,
+ `cell.${cell.cellId}.admission_state`,
+ ['existing-only', 'migration-only', 'general'].indexOf(
+ cell.expectedAdmissionState
+ ),
+ 'equal'
+ ],
+ [
+ 'director-admin',
+ admin,
+ `cell.${cell.cellId}.heartbeat_fresh`,
+ 1,
+ 'equal'
+ ],
+ [
+ 'director-admin',
+ admin,
+ `cell.${cell.cellId}.heartbeat_age_ms`,
+ INCIDENT_MONITOR_THRESHOLDS.heartbeatMaxAgeMs,
+ 'max'
+ ],
+ [
+ 'director-admin',
+ admin,
+ `cell.${cell.cellId}.migration_blocked`,
+ INCIDENT_MONITOR_THRESHOLDS.migrationBlocked,
+ 'max'
+ ],
+ [
+ 'director-admin',
+ admin,
+ `cell.${cell.cellId}.migration_target_inactive`,
+ INCIDENT_MONITOR_THRESHOLDS.migrationBlocked,
+ 'max'
+ ],
+ [
+ 'relay-logs',
+ relay,
+ `cell.${cell.cellId}.connections`,
+ (admin?.[`cell.${cell.cellId}.connection_hard_cap`]?.value ??
+ INCIDENT_MONITOR_THRESHOLDS.cellConnections + 1) - 1,
+ 'max'
+ ],
+ [
+ 'relay-logs',
+ relay,
+ `cell.${cell.cellId}.queued_bytes`,
+ INCIDENT_MONITOR_THRESHOLDS.cellQueuedBytes,
+ 'max'
+ ]
+ ] as const
+ for (const [source, signals, signalName, threshold, comparison] of checks) {
+ if (
+ migrationPolicy === 'recover-forward' &&
+ cell.cellId === recoverySourceCellId &&
+ signalName.endsWith('.migration_target_inactive')
+ ) {
+ if (!signals?.[signalName]) addMissingSignal(failures, source, signalName)
+ continue
+ }
+ if (
+ migrationPolicy === 'capacity-transition' &&
+ capacityCellId !== null &&
+ cell.cellId !== capacityCellId &&
+ cell.expectedAdmissionState === 'existing-only' &&
+ signalName.endsWith('.migration_target_inactive')
+ ) {
+ if (!signals?.[signalName]) addMissingSignal(failures, source, signalName)
+ continue
+ }
+ if (
+ cell.expectedAdmissionState === 'existing-only' &&
+ signalName.endsWith('.connections')
+ ) {
+ continue
+ }
+ if (
+ !cell.powered &&
+ [
+ 'latency_ms',
+ 'heartbeat_fresh',
+ 'heartbeat_age_ms',
+ 'connections',
+ 'queued_bytes'
+ ].some((suffix) => signalName.endsWith(suffix))
+ ) {
+ continue
+ }
+ if (!signals?.[signalName]) {
+ addMissingSignal(failures, source, signalName)
+ continue
+ }
+ const value = signals[signalName].value
+ const failed = comparison === 'equal' ? value !== threshold : value > threshold
+ if (failed) {
+ failures.push({
+ code: `threshold_${comparison}`,
+ source,
+ signal: signalName,
+ observed: value,
+ threshold
+ })
+ }
+ }
+}
+
+export function evaluateIncidentSample(
+ sample: IncidentSample,
+ nowMs = Date.now(),
+ migrationPolicy: IncidentMigrationPolicy = 'strict',
+ recoverySourceCellId: string | null = null,
+ capacityCellId: string | null = null
+): IncidentEvaluation {
+ const failures: IncidentFailure[] = []
+ if (!exactAdmissionSelector(sample.selector, sample.expectedSelector)) {
+ failures.push({
+ code: 'selector_mismatch',
+ source: 'director-admin',
+ signal: 'selector.generation',
+ observed: sample.selector.generation,
+ threshold: sample.expectedSelector.generation
+ })
+ }
+ for (const [sourceName, maxAge] of Object.entries(SOURCE_MAX_AGE) as [
+ IncidentSourceName,
+ number
+ ][]) {
+ const source = sample.sources[sourceName]
+ if (!source) {
+ failures.push({ code: 'source_missing', source: sourceName })
+ continue
+ }
+ if (ageMs(source.observedAt, nowMs) < 0 || ageMs(source.observedAt, nowMs) > maxAge) {
+ failures.push({
+ code: 'source_stale',
+ source: sourceName,
+ observed: ageMs(source.observedAt, nowMs),
+ threshold: maxAge
+ })
+ }
+ for (const [signalName, signal] of Object.entries(source.signals)) {
+ if (ageMs(signal.observedAt, nowMs) < 0 || ageMs(signal.observedAt, nowMs) > maxAge) {
+ failures.push({
+ code: 'signal_stale',
+ source: sourceName,
+ signal: signalName,
+ observed: ageMs(signal.observedAt, nowMs),
+ threshold: maxAge
+ })
+ }
+ }
+ }
+ for (const rule of NUMERIC_RULES) {
+ const source = sample.sources[rule.source]
+ if (source) checkRule(failures, rule.source, source.signals, rule)
+ }
+ for (const cell of sample.cells) {
+ checkCell(
+ failures,
+ sample,
+ cell,
+ migrationPolicy,
+ recoverySourceCellId,
+ capacityCellId
+ )
+ }
+ return {
+ status: failures.length === 0 ? 'green' : 'freeze',
+ evaluatedAt: new Date(nowMs).toISOString(),
+ failures
+ }
+}
+
+export function initialIncidentMonitorState(input: {
+ incidentId: string
+ environment: 'production' | 'staging'
+ expectedSelector: AdmissionSelector
+ preDrainDryRun: boolean
+ migrationPolicy: IncidentMigrationPolicy
+ recoverySourceCellId: string | null
+ capacityCellId: string | null
+ startedAt: string
+ durationMinutes: number
+ intervalMs: number
+}): IncidentMonitorState {
+ if (input.intervalMs < 1_000 || input.intervalMs > 60_000) {
+ throw new Error('incident monitor interval must be between 1 and 60 seconds')
+ }
+ if (input.durationMinutes < 15 || input.durationMinutes > 90) {
+ throw new Error('incident monitor duration must be between 15 and 90 minutes')
+ }
+ return {
+ schemaVersion: 4,
+ ...input,
+ windowStartedAt: input.startedAt,
+ windowSequence: 0,
+ nextCheckpointIndex: 0,
+ sampleCount: 0,
+ totalSampleCount: 0,
+ lastSampleAt: null,
+ continuityEvents: [],
+ frozenAt: null,
+ failures: [],
+ completedAt: null
+ }
+}
+
+export type IncidentMonitorDependencies = {
+ now(): number
+ wait(ms: number): Promise
+ collect(): Promise
+ persist(state: IncidentMonitorState): Promise
+ checkpoint(summary: IncidentCheckpoint): Promise
+}
+
+function checkpointMinutes(durationMinutes: number): number[] {
+ return INCIDENT_CHECKPOINT_MINUTES.filter((minute) => minute <= durationMinutes)
+}
+
+// Freshness-only failures: we could not read a signal this sample. Distinct from
+// collector_failed / monitor_gap, where the whole sample is absent.
+export const FRESHNESS_FAILURE_CODES = new Set([
+ 'signal_missing',
+ 'signal_stale',
+ 'source_missing',
+ 'source_stale'
+])
+
+const CONTINUITY_FAILURE_CODES = new Set([
+ 'collector_failed',
+ 'monitor_gap',
+ ...FRESHNESS_FAILURE_CODES
+])
+
+// Why: Cloud Monitoring overshoots its own publish bar, and one unread sample is
+// not evidence of an unhealthy fleet. Under the 25-minute lineage cap a restart
+// past minute 10 costs the entire verdict, so a healthy fleet produced none on
+// 2026-09-05. A signal may miss this many consecutive samples before the window
+// restarts; the sample is still evaluated against every threshold it can read,
+// and a threshold breach still freezes the run outright.
+export const INCIDENT_FRESHNESS_TOLERANCE_SAMPLES = 2
+
+function freshnessKey(failure: IncidentFailure): string {
+ return `${failure.source}/${failure.signal ?? '*'}`
+}
+
+// Rebuild the per-signal tolerated streak from the trailing continuity events so a
+// resumed monitor cannot hand a signal a fresh budget.
+function resumeFreshnessStreaks(
+ state: IncidentMonitorState
+): Map {
+ const events = state.continuityEvents
+ const streaks = new Map()
+ const last = events[events.length - 1]
+ if (!last?.tolerated) return streaks
+ for (const key of new Set(last.failures.map(freshnessKey))) {
+ let streak = 0
+ let laterAt: number | null = null
+ for (let index = events.length - 1; index >= 0; index--) {
+ const event = events[index]!
+ const recordedAt = Date.parse(event.recordedAt)
+ if (!event.tolerated) break
+ if (laterAt !== null && laterAt - recordedAt > state.intervalMs * 1.5) break
+ if (!event.failures.some((failure) => freshnessKey(failure) === key)) break
+ streak++
+ laterAt = recordedAt
+ }
+ streaks.set(key, streak)
+ }
+ return streaks
+}
+
+function resetContinuousWindow(
+ state: IncidentMonitorState,
+ recordedAt: string,
+ failures: IncidentFailure[]
+): void {
+ if (state.windowStartedAt !== null) {
+ state.windowSequence++
+ state.windowStartedAt = null
+ state.nextCheckpointIndex = 0
+ state.sampleCount = 0
+ state.completedAt = null
+ }
+ state.continuityEvents.push({
+ recordedAt,
+ windowSequence: state.windowSequence,
+ tolerated: false,
+ failures
+ })
+}
+
+function completeContinuityDeadline(
+ state: IncidentMonitorState,
+ nowMs: number,
+ lineageStartMs: number
+): void {
+ const recordedAt = new Date(nowMs).toISOString()
+ state.frozenAt ??= recordedAt
+ state.failures.push({
+ code: 'continuity_deadline_exceeded',
+ source: 'active-probe',
+ observed: nowMs - lineageStartMs,
+ threshold: INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS
+ })
+ state.completedAt = recordedAt
+}
+
+export async function runIncidentMonitor(
+ initialState: IncidentMonitorState,
+ dependencies: IncidentMonitorDependencies
+): Promise {
+ const state = structuredClone(initialState)
+ const lineageStartMs = Date.parse(state.startedAt)
+ if (!Number.isFinite(lineageStartMs)) {
+ throw new Error('incident monitor start time is invalid')
+ }
+ const checkpoints = checkpointMinutes(state.durationMinutes)
+ const lineageDeadlineMs = state.preDrainDryRun
+ ? lineageStartMs + INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS
+ : Number.POSITIVE_INFINITY
+ const resumedAt = dependencies.now()
+ const priorSampleMs = state.lastSampleAt
+ ? Date.parse(state.lastSampleAt)
+ : lineageStartMs
+ const gapThreshold = state.intervalMs
+ if (resumedAt - priorSampleMs > gapThreshold) {
+ resetContinuousWindow(state, new Date(resumedAt).toISOString(), [{
+ code: 'monitor_gap',
+ source: 'active-probe',
+ observed: resumedAt - priorSampleMs,
+ threshold: gapThreshold
+ }])
+ }
+ if (state.completedAt !== null) {
+ await dependencies.persist(state)
+ return state
+ }
+ const freshnessStreaks = resumeFreshnessStreaks(state)
+ while (state.completedAt === null) {
+ if (dependencies.now() > lineageDeadlineMs) {
+ completeContinuityDeadline(state, dependencies.now(), lineageStartMs)
+ await dependencies.persist(state)
+ break
+ }
+ const sampleStartedAt = dependencies.now()
+ let evaluation: IncidentEvaluation
+ try {
+ evaluation = evaluateIncidentSample(
+ await dependencies.collect(),
+ dependencies.now(),
+ state.migrationPolicy,
+ state.recoverySourceCellId,
+ state.capacityCellId
+ )
+ } catch {
+ evaluation = {
+ status: 'freeze',
+ evaluatedAt: new Date(dependencies.now()).toISOString(),
+ failures: [{
+ code: 'collector_failed',
+ source: 'cloud-monitoring'
+ }]
+ }
+ }
+ state.totalSampleCount++
+ state.lastSampleAt = evaluation.evaluatedAt
+ const continuityFailures = evaluation.failures.filter((failure) =>
+ CONTINUITY_FAILURE_CODES.has(failure.code)
+ )
+ const thresholdFailures = evaluation.failures.filter((failure) =>
+ !CONTINUITY_FAILURE_CODES.has(failure.code)
+ )
+ const toleratedKeys = new Set(
+ state.windowStartedAt !== null &&
+ continuityFailures.length > 0 &&
+ continuityFailures.every((failure) => FRESHNESS_FAILURE_CODES.has(failure.code))
+ ? continuityFailures.map(freshnessKey)
+ : []
+ )
+ for (const key of [...freshnessStreaks.keys()]) {
+ if (!toleratedKeys.has(key)) freshnessStreaks.delete(key)
+ }
+ let tolerated = toleratedKeys.size > 0
+ for (const key of toleratedKeys) {
+ const streak = (freshnessStreaks.get(key) ?? 0) + 1
+ freshnessStreaks.set(key, streak)
+ if (streak > INCIDENT_FRESHNESS_TOLERANCE_SAMPLES) tolerated = false
+ }
+ if (continuityFailures.length > 0 && !tolerated) {
+ freshnessStreaks.clear()
+ resetContinuousWindow(state, evaluation.evaluatedAt, continuityFailures)
+ } else {
+ if (tolerated) {
+ state.continuityEvents.push({
+ recordedAt: evaluation.evaluatedAt,
+ windowSequence: state.windowSequence,
+ tolerated: true,
+ failures: continuityFailures
+ })
+ }
+ if (state.windowStartedAt === null) {
+ state.windowStartedAt = evaluation.evaluatedAt
+ }
+ state.sampleCount++
+ }
+ if (thresholdFailures.length > 0) {
+ state.frozenAt ??= evaluation.evaluatedAt
+ state.failures = [...state.failures, ...thresholdFailures]
+ }
+ if (state.windowStartedAt === null) {
+ if (dependencies.now() >= lineageDeadlineMs) {
+ completeContinuityDeadline(state, dependencies.now(), lineageStartMs)
+ await dependencies.persist(state)
+ break
+ }
+ await dependencies.persist(state)
+ await dependencies.wait(
+ Math.max(0, Math.min(state.intervalMs, lineageDeadlineMs - dependencies.now()))
+ )
+ continue
+ }
+ const startMs = Date.parse(state.windowStartedAt)
+ const endMs = startMs + state.durationMinutes * 60_000
+ const elapsedMinutes = (dependencies.now() - startMs) / 60_000
+ while (
+ state.nextCheckpointIndex < checkpoints.length &&
+ elapsedMinutes >= checkpoints[state.nextCheckpointIndex]!
+ ) {
+ const minute = checkpoints[state.nextCheckpointIndex]!
+ await dependencies.checkpoint({
+ schemaVersion: 4,
+ incidentId: state.incidentId,
+ environment: state.environment,
+ expectedSelector: state.expectedSelector,
+ preDrainDryRun: state.preDrainDryRun,
+ migrationPolicy: state.migrationPolicy,
+ recoverySourceCellId: state.recoverySourceCellId,
+ capacityCellId: state.capacityCellId,
+ windowSequence: state.windowSequence,
+ windowStartedAt: state.windowStartedAt,
+ checkpointMinute: minute,
+ scheduledAt: new Date(startMs + minute * 60_000).toISOString(),
+ recordedAt: new Date(dependencies.now()).toISOString(),
+ status: state.frozenAt ? 'freeze' : 'green',
+ frozenAt: state.frozenAt,
+ sampleCount: state.sampleCount,
+ failures: state.failures,
+ thresholds: INCIDENT_MONITOR_THRESHOLDS
+ })
+ state.nextCheckpointIndex++
+ }
+ if (state.preDrainDryRun && state.frozenAt !== null) {
+ state.completedAt = new Date(dependencies.now()).toISOString()
+ await dependencies.persist(state)
+ break
+ }
+ if (dependencies.now() >= endMs) {
+ state.completedAt = new Date(dependencies.now()).toISOString()
+ await dependencies.persist(state)
+ break
+ }
+ if (dependencies.now() >= lineageDeadlineMs) {
+ completeContinuityDeadline(state, dependencies.now(), lineageStartMs)
+ await dependencies.persist(state)
+ break
+ }
+ await dependencies.persist(state)
+ await dependencies.wait(
+ Math.max(
+ 0,
+ Math.min(sampleStartedAt + state.intervalMs, endMs, lineageDeadlineMs) - dependencies.now()
+ )
+ )
+ }
+ return state
+}
+
+export function preDrainDryRunPassed(state: Pick<
+ IncidentMonitorState,
+ | 'completedAt'
+ | 'durationMinutes'
+ | 'frozenAt'
+ | 'intervalMs'
+ | 'preDrainDryRun'
+ | 'sampleCount'
+ | 'startedAt'
+>): boolean {
+ const minimumSamples =
+ Math.ceil((state.durationMinutes * 60_000) / state.intervalMs) + 1
+ const lineageElapsedMs = state.completedAt === null
+ ? Number.POSITIVE_INFINITY
+ : Date.parse(state.completedAt) - Date.parse(state.startedAt)
+ return (
+ state.preDrainDryRun &&
+ state.durationMinutes === 15 &&
+ state.completedAt !== null &&
+ lineageElapsedMs >= 0 &&
+ lineageElapsedMs <= INCIDENT_PRE_DRAIN_MAX_LINEAGE_MS &&
+ state.frozenAt === null &&
+ state.sampleCount >= minimumSamples
+ )
+}
diff --git a/cloud/apps/relay-ops/src/incident-selector.ts b/cloud/apps/relay-ops/src/incident-selector.ts
new file mode 100644
index 00000000000..7e7ddd91e59
--- /dev/null
+++ b/cloud/apps/relay-ops/src/incident-selector.ts
@@ -0,0 +1,77 @@
+import { z } from 'zod'
+
+export const AdmissionStateSchema = z.enum([
+ 'existing-only',
+ 'migration-only',
+ 'general'
+])
+
+export type AdmissionState = z.infer
+
+export const SelectorMembershipSchema = z.object({
+ existingOnly: z.array(z.string()),
+ migrationOnly: z.array(z.string()),
+ general: z.array(z.string())
+})
+
+export type SelectorMembership = z.infer
+
+export const AdmissionSelectorSchema = z.object({
+ generation: z.number().int().nonnegative(),
+ membership: SelectorMembershipSchema
+})
+
+export type AdmissionSelector = z.infer
+
+export function normalizeSelectorMembership(
+ membership: SelectorMembership,
+ configuredCellIds: ReadonlySet
+): SelectorMembership {
+ const normalized = {
+ existingOnly: [...membership.existingOnly].sort(),
+ migrationOnly: [...membership.migrationOnly].sort(),
+ general: [...membership.general].sort()
+ }
+ const all = [
+ ...normalized.existingOnly,
+ ...normalized.migrationOnly,
+ ...normalized.general
+ ]
+ if (
+ all.length !== configuredCellIds.size ||
+ new Set(all).size !== all.length ||
+ all.some((cellId) => !configuredCellIds.has(cellId))
+ ) {
+ throw new Error('selector membership must contain every configured cell exactly once')
+ }
+ return normalized
+}
+
+export function selectorCellState(
+ selector: AdmissionSelector,
+ cellId: string
+): AdmissionState {
+ if (selector.membership.existingOnly.includes(cellId)) return 'existing-only'
+ if (selector.membership.migrationOnly.includes(cellId)) return 'migration-only'
+ if (selector.membership.general.includes(cellId)) return 'general'
+ throw new Error(`selector does not contain ${cellId}`)
+}
+
+export function effectiveAdmissionState(
+ selector: AdmissionSelector,
+ legacyEnabled: boolean,
+ cellId: string
+): AdmissionState {
+ if (selector.generation === 0) return legacyEnabled ? 'general' : 'existing-only'
+ return selectorCellState(selector, cellId)
+}
+
+export function exactAdmissionSelector(
+ actual: AdmissionSelector,
+ expected: AdmissionSelector
+): boolean {
+ return (
+ actual.generation === expected.generation &&
+ JSON.stringify(actual.membership) === JSON.stringify(expected.membership)
+ )
+}
diff --git a/cloud/apps/relay-ops/src/index.ts b/cloud/apps/relay-ops/src/index.ts
new file mode 100644
index 00000000000..0544cfe7595
--- /dev/null
+++ b/cloud/apps/relay-ops/src/index.ts
@@ -0,0 +1,108 @@
+import { randomBytes, timingSafeEqual } from 'node:crypto'
+import { readFile } from 'node:fs/promises'
+import { resolve } from 'node:path'
+import { serve } from '@hono/node-server'
+import { Hono } from 'hono'
+import { z } from 'zod'
+import { DashboardSnapshotCache } from './dashboard-snapshot.js'
+import { createGcloudClient } from './gcloud-client.js'
+import {
+ dispatchStagingPowerWorkflow,
+ parseStagingPowerRequest
+} from './staging-workflow.js'
+
+const QuerySchema = z.object({
+ environment: z.enum(['production', 'staging']).default('production'),
+ window: z.coerce.number().int().min(30).max(1440).default(360)
+})
+const port = z.coerce.number().int().min(1024).max(65_535).parse(process.env.PORT ?? 2455)
+const controlsEnabled = process.env.RELAY_OPS_ENABLE_STAGING_CONTROLS === '1'
+const csrfToken = randomBytes(32).toString('base64url')
+const publicDirectory = resolve(import.meta.dirname, '../public')
+const cache = new DashboardSnapshotCache(createGcloudClient())
+const app = new Hono()
+
+function safeEqual(left: string, right: string): boolean {
+ const leftBuffer = Buffer.from(left)
+ const rightBuffer = Buffer.from(right)
+ return leftBuffer.length === rightBuffer.length && timingSafeEqual(leftBuffer, rightBuffer)
+}
+
+app.use('*', async (context, next) => {
+ await next()
+ context.header('Cache-Control', 'no-store')
+ context.header('Content-Security-Policy', [
+ "default-src 'self'",
+ "script-src 'self'",
+ "style-src 'self'",
+ "font-src 'self'",
+ "connect-src 'self'",
+ "img-src 'self' data:",
+ "object-src 'none'",
+ "base-uri 'none'",
+ "frame-ancestors 'none'",
+ "form-action 'self'"
+ ].join('; '))
+ context.header('Referrer-Policy', 'no-referrer')
+ context.header('X-Content-Type-Options', 'nosniff')
+ context.header('X-Frame-Options', 'DENY')
+})
+
+app.get('/health', (context) => context.json({ status: 'ok' }))
+app.get('/api/config', (context) => context.json({
+ stagingControlsEnabled: controlsEnabled,
+ csrfToken: controlsEnabled ? csrfToken : null
+}))
+app.get('/api/snapshot', async (context) => {
+ const query = QuerySchema.safeParse(context.req.query())
+ if (!query.success) return context.json({ error: 'Invalid dashboard query' }, 400)
+ try {
+ return context.json(await cache.read(query.data.environment, query.data.window))
+ } catch {
+ return context.json({
+ error: 'Relay operations data is unavailable. Check local gcloud and gh authentication.'
+ }, 503)
+ }
+})
+app.post('/api/staging/power', async (context) => {
+ if (!controlsEnabled) return context.json({ error: 'Staging controls are disabled' }, 403)
+ const origin = context.req.header('origin')
+ const expectedOrigin = `http://127.0.0.1:${port}`
+ if (origin !== expectedOrigin) return context.json({ error: 'Origin rejected' }, 403)
+ if (!safeEqual(context.req.header('x-csrf-token') ?? '', csrfToken)) {
+ return context.json({ error: 'Request token rejected' }, 403)
+ }
+ try {
+ const request = parseStagingPowerRequest(await context.req.json())
+ await dispatchStagingPowerWorkflow(request)
+ return context.json({ accepted: true })
+ } catch {
+ return context.json({ error: 'Invalid or failed staging workflow dispatch' }, 400)
+ }
+})
+
+const staticTypes: Record = {
+ '/app.js': 'text/javascript; charset=utf-8',
+ '/styles.css': 'text/css; charset=utf-8'
+}
+for (const [route, contentType] of Object.entries(staticTypes)) {
+ app.get(route, async (context) => {
+ const file = route.slice(1)
+ try {
+ const content = await readFile(resolve(publicDirectory, file))
+ return context.body(content, 200, { 'Content-Type': contentType })
+ } catch {
+ return context.notFound()
+ }
+ })
+}
+app.get('/', async (context) => {
+ const html = await readFile(resolve(publicDirectory, 'index.html'), 'utf8')
+ return context.html(html)
+})
+
+serve({ fetch: app.fetch, hostname: '127.0.0.1', port }, () => {
+ // Loopback is intentional; operators may add authenticated Tailscale Serve separately.
+ console.log(`Orca Relay Operations: http://127.0.0.1:${port}`)
+ console.log(`Staging controls: ${controlsEnabled ? 'enabled through GitHub workflow' : 'read-only'}`)
+})
diff --git a/cloud/apps/relay-ops/src/monitoring-snapshot.test.ts b/cloud/apps/relay-ops/src/monitoring-snapshot.test.ts
new file mode 100644
index 00000000000..3bcd40c3c4b
--- /dev/null
+++ b/cloud/apps/relay-ops/src/monitoring-snapshot.test.ts
@@ -0,0 +1,85 @@
+import { describe, expect, it } from 'vitest'
+import { RELAY_METRICS, readMonitoringSnapshot } from './monitoring-snapshot.js'
+import type { GcloudClient } from './gcloud-client.js'
+import { RELAY_OPS_ENVIRONMENTS } from './environment-config.js'
+
+const gcloud: GcloudClient = {
+ accessToken: async () => 'a'.repeat(40)
+}
+
+function distribution(at: string, mean: number | undefined, count = 1) {
+ return {
+ interval: { endTime: at },
+ value: { distributionValue: { count, ...(mean === undefined ? {} : { mean }) } }
+ }
+}
+
+describe('readMonitoringSnapshot', () => {
+ it('aggregates gauge series per minute and distribution deltas by count', async () => {
+ const fetchImpl: typeof fetch = async (input) => {
+ const url = new URL(String(input))
+ if (url.pathname.endsWith('/alertPolicies')) {
+ return Response.json({ alertPolicies: [] })
+ }
+ const filter = url.searchParams.get('filter') ?? ''
+ if (filter.includes('orca_relay_controls')) {
+ return Response.json({ timeSeries: [
+ {
+ metric: { labels: { cell_id: 'production-gce-c1' } },
+ resource: { type: 'gce_instance', labels: { instance_id: 'one' } },
+ points: [
+ distribution('2026-07-15T12:00:10Z', 1),
+ distribution('2026-07-15T12:00:50Z', 2)
+ ]
+ },
+ {
+ metric: { labels: { cell_id: 'production-gce-c1' } },
+ resource: { type: 'gce_instance', labels: { instance_id: 'two' } },
+ points: [distribution('2026-07-15T12:00:20Z', 3)]
+ },
+ {
+ metric: { labels: { cell_id: 'production-gce-c1' } },
+ resource: { type: 'gce_instance', labels: { instance_id: 'stale' } },
+ points: [distribution('2026-07-15T11:59:20Z', 100)]
+ }
+ ] })
+ }
+ if (filter.includes('orca_relay_forwarded_bytes')) {
+ return Response.json({ timeSeries: [{
+ metric: { labels: { cell_id: 'production-gce-c1' } },
+ resource: { type: 'gce_instance', labels: { instance_id: 'one' } },
+ points: [distribution('2026-07-15T12:00:20Z', 10, 4)]
+ }] })
+ }
+ return Response.json({ timeSeries: [] })
+ }
+
+ const result = await readMonitoringSnapshot(RELAY_OPS_ENVIRONMENTS.production, gcloud, {
+ now: new Date('2026-07-15T12:01:00Z'),
+ windowMinutes: 30,
+ fetchImpl
+ })
+
+ expect(result.warnings).toEqual([])
+ expect(result.metrics.controls.points).toEqual([
+ { at: '2026-07-15T11:59:00.000Z', value: 100 },
+ { at: '2026-07-15T12:00:00.000Z', value: 5 }
+ ])
+ expect(result.metrics.controls.latestByCell).toEqual({ 'production-gce-c1': 5 })
+ expect(result.metrics.forwarded_bytes.latest).toBe(40)
+ expect(result.metrics.postgres_retries.available).toBe(true)
+ expect(Object.keys(result.metrics)).toHaveLength(RELAY_METRICS.length)
+ })
+
+ it('degrades safely when credentials are unavailable', async () => {
+ const unavailable: GcloudClient = {
+ accessToken: async () => { throw new Error('sensitive context') }
+ }
+ const result = await readMonitoringSnapshot(RELAY_OPS_ENVIRONMENTS.production, unavailable)
+ expect(result.warnings).toEqual([
+ 'Cloud Monitoring credentials are unavailable. Run gcloud auth login.'
+ ])
+ expect(result.metrics.postgres_retries.available).toBe(false)
+ expect(JSON.stringify(result)).not.toContain('sensitive context')
+ })
+})
diff --git a/cloud/apps/relay-ops/src/monitoring-snapshot.ts b/cloud/apps/relay-ops/src/monitoring-snapshot.ts
new file mode 100644
index 00000000000..9b5fe44bd4a
--- /dev/null
+++ b/cloud/apps/relay-ops/src/monitoring-snapshot.ts
@@ -0,0 +1,379 @@
+import { z } from 'zod'
+import type { RelayOpsEnvironment } from './environment-config.js'
+import type { GcloudClient } from './gcloud-client.js'
+
+type MetricMode = 'gauge-sum' | 'delta-sum' | 'maximum'
+
+export type RelayMetricName =
+ | 'total_connections'
+ | 'controls'
+ | 'splices'
+ | 'pending_splices'
+ | 'queued_bytes'
+ | 'http_latency_ms'
+ | 'sql_latency_ms'
+ | 'heap_used_bytes'
+ | 'event_loop_ms_p99'
+ | 'forwarded_bytes'
+ | 'auth_successes'
+ | 'auth_failures'
+ | 'reconnects'
+ | 'sql_queries'
+ | 'sql_failures'
+ | 'assignment_5xx'
+ | 'postgres_retries'
+ | 'postgres_retry_exhausted'
+ | 'db_pool_total'
+ | 'db_pool_idle'
+ | 'db_pool_waiting'
+ | 'db_waiters_max'
+ | 'db_oldest_wait_ms'
+ | 'db_wait_ms_max'
+
+type MetricDefinition = {
+ name: RelayMetricName
+ label: string
+ unit: 'count' | 'bytes' | 'milliseconds'
+ mode: MetricMode
+}
+
+export const RELAY_METRICS: MetricDefinition[] = [
+ { name: 'total_connections', label: 'Connections', unit: 'count', mode: 'gauge-sum' },
+ { name: 'controls', label: 'Desktop controls', unit: 'count', mode: 'gauge-sum' },
+ { name: 'splices', label: 'Phone splices', unit: 'count', mode: 'gauge-sum' },
+ { name: 'pending_splices', label: 'Pending splices', unit: 'count', mode: 'gauge-sum' },
+ { name: 'queued_bytes', label: 'Queued bytes', unit: 'bytes', mode: 'maximum' },
+ { name: 'http_latency_ms', label: 'HTTP latency', unit: 'milliseconds', mode: 'maximum' },
+ { name: 'sql_latency_ms', label: 'SQL latency', unit: 'milliseconds', mode: 'maximum' },
+ { name: 'heap_used_bytes', label: 'Heap used', unit: 'bytes', mode: 'maximum' },
+ {
+ name: 'event_loop_ms_p99',
+ label: 'Event-loop p99',
+ unit: 'milliseconds',
+ mode: 'maximum'
+ },
+ { name: 'forwarded_bytes', label: 'Forwarded bytes', unit: 'bytes', mode: 'delta-sum' },
+ { name: 'auth_successes', label: 'Auth successes', unit: 'count', mode: 'delta-sum' },
+ { name: 'auth_failures', label: 'Auth failures', unit: 'count', mode: 'delta-sum' },
+ { name: 'reconnects', label: 'Reconnects', unit: 'count', mode: 'delta-sum' },
+ { name: 'sql_queries', label: 'SQL queries', unit: 'count', mode: 'delta-sum' },
+ { name: 'sql_failures', label: 'SQL failures', unit: 'count', mode: 'delta-sum' },
+ { name: 'assignment_5xx', label: 'Assignment 5xx', unit: 'count', mode: 'delta-sum' },
+ {
+ name: 'postgres_retries',
+ label: 'PostgreSQL retries',
+ unit: 'count',
+ mode: 'delta-sum'
+ },
+ {
+ name: 'postgres_retry_exhausted',
+ label: 'PostgreSQL retry exhausted',
+ unit: 'count',
+ mode: 'delta-sum'
+ },
+ { name: 'db_pool_total', label: 'Database pool total', unit: 'count', mode: 'gauge-sum' },
+ { name: 'db_pool_idle', label: 'Database pool idle', unit: 'count', mode: 'gauge-sum' },
+ {
+ name: 'db_pool_waiting',
+ label: 'Database pool waiting',
+ unit: 'count',
+ mode: 'gauge-sum'
+ },
+ {
+ name: 'db_waiters_max',
+ label: 'Database waiters max',
+ unit: 'count',
+ mode: 'maximum'
+ },
+ {
+ name: 'db_oldest_wait_ms',
+ label: 'Database oldest wait',
+ unit: 'milliseconds',
+ mode: 'maximum'
+ },
+ {
+ name: 'db_wait_ms_max',
+ label: 'Database wait max',
+ unit: 'milliseconds',
+ mode: 'maximum'
+ }
+]
+
+const NumericSchema = z.union([z.number(), z.string()]).transform((value) => Number(value))
+const DistributionSchema = z.object({
+ count: NumericSchema.default(0),
+ mean: NumericSchema.optional()
+})
+const PointSchema = z.object({
+ interval: z.object({ endTime: z.string() }),
+ value: z.object({
+ doubleValue: NumericSchema.optional(),
+ int64Value: NumericSchema.optional(),
+ distributionValue: DistributionSchema.optional()
+ })
+})
+const TimeSeriesSchema = z.object({
+ metric: z.object({ labels: z.record(z.string()).default({}) }),
+ resource: z.object({ type: z.string(), labels: z.record(z.string()).default({}) }),
+ points: z.array(PointSchema).default([])
+})
+const TimeSeriesResponseSchema = z.object({
+ timeSeries: z.array(TimeSeriesSchema).default([])
+})
+
+export type MetricPoint = { at: string; value: number }
+
+export type RelayMetricSnapshot = MetricDefinition & {
+ available: boolean
+ points: MetricPoint[]
+ latest: number | null
+ latestAt: string | null
+ latestByCell: Record
+}
+
+export type AlertPolicySnapshot = {
+ id: string
+ displayName: string
+ enabled: boolean
+ documentation: string | null
+}
+
+export type MonitoringSnapshot = {
+ startAt: string
+ endAt: string
+ resolutionSeconds: number
+ metrics: Record
+ alertPolicies: AlertPolicySnapshot[]
+ warnings: string[]
+}
+
+type ParsedPoint = {
+ atMs: number
+ bucketMs: number
+ value: number
+ sampleTotal: number
+ seriesKey: string
+ cellId: string
+}
+
+function pointValue(point: z.infer): { value: number; sampleTotal: number } {
+ if (point.value.distributionValue) {
+ const count = point.value.distributionValue.count
+ const value = point.value.distributionValue.mean ?? 0
+ return { value, sampleTotal: value * count }
+ }
+ const value = point.value.doubleValue ?? point.value.int64Value ?? 0
+ return { value, sampleTotal: value }
+}
+
+function parsePoints(series: z.infer[]): ParsedPoint[] {
+ return series.flatMap((entry) => {
+ const cellId = entry.metric.labels.cell_id ?? 'unknown'
+ const resourceId =
+ entry.resource.labels.instance_id ?? entry.resource.labels.revision_name ?? entry.resource.type
+ const seriesKey = `${cellId}:${resourceId}`
+ return entry.points.flatMap((point) => {
+ const atMs = Date.parse(point.interval.endTime)
+ if (!Number.isFinite(atMs)) return []
+ const values = pointValue(point)
+ return [{
+ atMs,
+ bucketMs: Math.floor(atMs / 60_000) * 60_000,
+ value: values.value,
+ sampleTotal: values.sampleTotal,
+ seriesKey,
+ cellId
+ }]
+ })
+ })
+}
+
+function aggregatePoints(points: ParsedPoint[], mode: MetricMode): MetricPoint[] {
+ if (mode === 'gauge-sum') {
+ const buckets = new Map>()
+ for (const point of points) {
+ const bySeries = buckets.get(point.bucketMs) ?? new Map()
+ const previous = bySeries.get(point.seriesKey)
+ if (!previous || previous.atMs < point.atMs) bySeries.set(point.seriesKey, point)
+ buckets.set(point.bucketMs, bySeries)
+ }
+ return [...buckets.entries()]
+ .sort(([left], [right]) => left - right)
+ .map(([at, bySeries]) => ({
+ at: new Date(at).toISOString(),
+ value: [...bySeries.values()].reduce((total, point) => total + point.value, 0)
+ }))
+ }
+ const buckets = new Map()
+ for (const point of points) {
+ const value = mode === 'delta-sum' ? point.sampleTotal : point.value
+ const previous = buckets.get(point.bucketMs)
+ buckets.set(
+ point.bucketMs,
+ mode === 'maximum' ? Math.max(previous ?? 0, value) : (previous ?? 0) + value
+ )
+ }
+ return [...buckets.entries()]
+ .sort(([left], [right]) => left - right)
+ .map(([at, value]) => ({ at: new Date(at).toISOString(), value }))
+}
+
+function latestByCell(points: ParsedPoint[], mode: MetricMode): Record {
+ const newestBucketByCell = new Map()
+ for (const point of points) {
+ newestBucketByCell.set(
+ point.cellId,
+ Math.max(newestBucketByCell.get(point.cellId) ?? 0, point.bucketMs)
+ )
+ }
+ const newestBySeries = new Map()
+ for (const point of points) {
+ if (point.bucketMs !== newestBucketByCell.get(point.cellId)) continue
+ const previous = newestBySeries.get(point.seriesKey)
+ if (!previous || previous.atMs < point.atMs) newestBySeries.set(point.seriesKey, point)
+ }
+ const totals = new Map()
+ for (const point of newestBySeries.values()) {
+ const value = mode === 'delta-sum' ? point.sampleTotal : point.value
+ const previous = totals.get(point.cellId)
+ totals.set(
+ point.cellId,
+ mode === 'maximum' ? Math.max(previous ?? 0, value) : (previous ?? 0) + value
+ )
+ }
+ return Object.fromEntries(totals)
+}
+
+async function monitoringRequest(
+ fetchImpl: typeof fetch,
+ token: string,
+ url: URL
+): Promise {
+ const response = await fetchImpl(url, {
+ headers: { authorization: `Bearer ${token}` },
+ signal: AbortSignal.timeout(30_000)
+ })
+ if (!response.ok) throw new Error(`Cloud Monitoring returned ${response.status}`)
+ return await response.json()
+}
+
+async function readMetric(
+ environment: RelayOpsEnvironment,
+ definition: MetricDefinition,
+ token: string,
+ startAt: string,
+ endAt: string,
+ fetchImpl: typeof fetch
+): Promise {
+ const url = new URL(
+ `https://monitoring.googleapis.com/v3/projects/${environment.project}/timeSeries`
+ )
+ url.searchParams.set(
+ 'filter',
+ `metric.type="logging.googleapis.com/user/orca_relay_${definition.name}"`
+ )
+ url.searchParams.set('interval.startTime', startAt)
+ url.searchParams.set('interval.endTime', endAt)
+ url.searchParams.set('view', 'FULL')
+ url.searchParams.set('pageSize', '1000')
+ const body = TimeSeriesResponseSchema.parse(
+ await monitoringRequest(fetchImpl, token, url)
+ )
+ const parsed = parsePoints(body.timeSeries)
+ const points = aggregatePoints(parsed, definition.mode)
+ const latest = points.at(-1) ?? null
+ return {
+ ...definition,
+ available: true,
+ points,
+ latest: latest?.value ?? null,
+ latestAt: latest?.at ?? null,
+ latestByCell: latestByCell(parsed, definition.mode)
+ }
+}
+
+async function readAlertPolicies(
+ environment: RelayOpsEnvironment,
+ token: string,
+ fetchImpl: typeof fetch
+): Promise {
+ const url = new URL(
+ `https://monitoring.googleapis.com/v3/projects/${environment.project}/alertPolicies`
+ )
+ url.searchParams.set('pageSize', '100')
+ const body = (await monitoringRequest(fetchImpl, token, url)) as {
+ alertPolicies?: Array<{
+ name?: string
+ displayName?: string
+ enabled?: boolean
+ documentation?: { content?: string }
+ }>
+ }
+ return (body.alertPolicies ?? [])
+ .filter((policy) => policy.displayName?.startsWith('Orca Relay:'))
+ .map((policy) => ({
+ id: policy.name ?? '',
+ displayName: policy.displayName ?? 'Orca Relay alert',
+ enabled: policy.enabled === true,
+ documentation: policy.documentation?.content ?? null
+ }))
+ .sort((left, right) => left.displayName.localeCompare(right.displayName))
+}
+
+function emptyMetric(definition: MetricDefinition): RelayMetricSnapshot {
+ return {
+ ...definition,
+ available: false,
+ points: [],
+ latest: null,
+ latestAt: null,
+ latestByCell: {}
+ }
+}
+
+export async function readMonitoringSnapshot(
+ environment: RelayOpsEnvironment,
+ gcloud: GcloudClient,
+ options: { now?: Date; windowMinutes?: number; fetchImpl?: typeof fetch } = {}
+): Promise {
+ const now = options.now ?? new Date()
+ const windowMinutes = Math.min(24 * 60, Math.max(30, options.windowMinutes ?? 360))
+ const endAt = now.toISOString()
+ const startAt = new Date(now.getTime() - windowMinutes * 60_000).toISOString()
+ const fetchImpl = options.fetchImpl ?? fetch
+ const warnings: string[] = []
+ let token: string
+ try {
+ token = await gcloud.accessToken()
+ } catch {
+ return {
+ startAt,
+ endAt,
+ resolutionSeconds: 60,
+ metrics: Object.fromEntries(
+ RELAY_METRICS.map((definition) => [definition.name, emptyMetric(definition)])
+ ) as Record,
+ alertPolicies: [],
+ warnings: ['Cloud Monitoring credentials are unavailable. Run gcloud auth login.']
+ }
+ }
+ const settled = await Promise.allSettled(
+ RELAY_METRICS.map((definition) =>
+ readMetric(environment, definition, token, startAt, endAt, fetchImpl)
+ )
+ )
+ const metrics = {} as Record
+ settled.forEach((result, index) => {
+ const definition = RELAY_METRICS[index]!
+ if (result.status === 'fulfilled') metrics[definition.name] = result.value
+ else {
+ metrics[definition.name] = emptyMetric(definition)
+ warnings.push(`${definition.label} metric is unavailable.`)
+ }
+ })
+ const alertPolicies = await readAlertPolicies(environment, token, fetchImpl).catch(() => {
+ warnings.push('Cloud Monitoring alert policies are unavailable.')
+ return []
+ })
+ return { startAt, endAt, resolutionSeconds: 60, metrics, alertPolicies, warnings }
+}
diff --git a/cloud/apps/relay-ops/src/relay-repository.test.ts b/cloud/apps/relay-ops/src/relay-repository.test.ts
new file mode 100644
index 00000000000..e6a87c35e82
--- /dev/null
+++ b/cloud/apps/relay-ops/src/relay-repository.test.ts
@@ -0,0 +1,42 @@
+import { readdirSync, readFileSync } from 'node:fs'
+import { fileURLToPath } from 'node:url'
+import { describe, expect, it } from 'vitest'
+import {
+ RELAY_GITHUB_REPOSITORY,
+ RELAY_WORKFLOW_FILE_PREFIX,
+ relayRepositoryApiPath,
+ relayWorkflowFile
+} from './relay-repository.js'
+
+const sourceDir = fileURLToPath(new URL('.', import.meta.url))
+const sources = readdirSync(sourceDir)
+ .filter((name) => name.endsWith('.ts') && name !== 'relay-repository.ts')
+ .map((name) => ({ name, text: readFileSync(`${sourceDir}${name}`, 'utf8') }))
+
+describe('relay repository identity', () => {
+ it('builds API paths and workflow filenames from the one repository name', () => {
+ expect(relayRepositoryApiPath('actions/runs')).toBe(`repos/${RELAY_GITHUB_REPOSITORY}/actions/runs`)
+ expect(relayWorkflowFile('power-relay-staging.yml')).toBe(
+ `${RELAY_WORKFLOW_FILE_PREFIX}power-relay-staging.yml`
+ )
+ })
+
+ // Why: the public-repo copy renames the repository and prefixes every workflow file. Both have to
+ // be one edit, so no other module may restate either.
+ it('is the only module naming a GitHub repository', () => {
+ for (const { name, text } of sources) {
+ expect(text, `${name} restates a GitHub repository`).not.toMatch(/stablyai\//)
+ }
+ })
+
+ it('is the only module naming a workflow file', () => {
+ for (const { name, text } of sources) {
+ for (const match of text.matchAll(/'([^']*\.yml)'/g)) {
+ const file = match[1] ?? ''
+ expect(text, `${name} names ${file} outside relayWorkflowFile`).toMatch(
+ new RegExp(`relayWorkflowFile\\('${file.replaceAll('.', '\\.')}'\\)`)
+ )
+ }
+ }
+ })
+})
diff --git a/cloud/apps/relay-ops/src/relay-repository.ts b/cloud/apps/relay-ops/src/relay-repository.ts
new file mode 100644
index 00000000000..85158670adb
--- /dev/null
+++ b/cloud/apps/relay-ops/src/relay-repository.ts
@@ -0,0 +1,14 @@
+// Single place naming the GitHub repository that holds the Relay workflows. When the Relay tree is
+// copied to its public repository, only this file changes: the repository moves and every workflow
+// file gains a prefix, while the workflow display names stay as they are.
+export const RELAY_GITHUB_REPOSITORY = 'stablyai/orca-cloud'
+
+export const RELAY_WORKFLOW_FILE_PREFIX = ''
+
+export function relayWorkflowFile(name: string): string {
+ return `${RELAY_WORKFLOW_FILE_PREFIX}${name}`
+}
+
+export function relayRepositoryApiPath(resource: string): string {
+ return `repos/${RELAY_GITHUB_REPOSITORY}/${resource}`
+}
diff --git a/cloud/apps/relay-ops/src/resource-inventory.test.ts b/cloud/apps/relay-ops/src/resource-inventory.test.ts
new file mode 100644
index 00000000000..e2cfa13dccb
--- /dev/null
+++ b/cloud/apps/relay-ops/src/resource-inventory.test.ts
@@ -0,0 +1,370 @@
+import { describe, expect, it } from 'vitest'
+import { RELAY_OPS_ENVIRONMENTS } from './environment-config.js'
+import type { GcloudClient } from './gcloud-client.js'
+import { probeEndpointHealth, readResourceInventory } from './resource-inventory.js'
+
+const digest = `sha256:${'a'.repeat(64)}`
+const runService = {
+ template: {
+ scaling: { minInstanceCount: 0, maxInstanceCount: 2 },
+ containers: [{ image: 'registry/image:tag' }]
+ },
+ conditions: [{ state: 'CONDITION_SUCCEEDED' }],
+ latestReadyRevision: 'projects/project/revisions/revision-one'
+}
+
+const sleepingStagingGcloud: GcloudClient = { accessToken: async () => 'a'.repeat(40) }
+
+// Staging's Cloud SQL is stopped, so this inventory reads REST only and probes no endpoint.
+type MigOutcome = 'ok' | 'throw' | 'missing'
+const sleepingStagingFetch = (migOutcome: (migName: string) => MigOutcome): typeof fetch =>
+ async (input) => {
+ const url = new URL(String(input))
+ if (url.hostname === 'run.googleapis.com') return Response.json(runService)
+ if (url.hostname === 'sqladmin.googleapis.com') return Response.json({
+ state: 'STOPPED',
+ databaseVersion: 'POSTGRES_17',
+ settings: { activationPolicy: 'NEVER', availabilityType: 'ZONAL', tier: 'db-custom-1-3840' }
+ })
+ if (url.hostname === 'certificatemanager.googleapis.com') return Response.json({
+ managed: { domains: ['*.relay-staging.onorca.dev'], state: 'ACTIVE' }
+ })
+ if (url.pathname.includes('/instanceGroupManagers/')) {
+ const name = url.pathname.split('/').at(-1)!
+ const outcome = migOutcome(name)
+ if (outcome === 'throw') throw new TypeError('fetch failed')
+ if (outcome === 'missing') return new Response(null, { status: 404 })
+ return Response.json({
+ name,
+ targetSize: 0,
+ size: '0',
+ instanceGroup: `projects/project/zones/zone/instanceGroups/${name}`,
+ instanceTemplate: `projects/project/global/instanceTemplates/template-${name}`,
+ status: { isStable: true }
+ })
+ }
+ if (url.pathname.includes('/instanceTemplates/')) return Response.json({ properties: {} })
+ if (url.pathname.endsWith('/getHealth')) return Response.json([])
+ throw new Error(`Unexpected request to ${url.hostname}${url.pathname}`)
+ }
+
+describe('readResourceInventory', () => {
+ it('does not delay a healthy endpoint sample', async () => {
+ let calls = 0
+ let waits = 0
+ const result = await probeEndpointHealth(
+ 'https://c9.relay.onorca.dev',
+ async () => {
+ calls += 1
+ return new Response(null, { status: 200 })
+ },
+ {
+ wait: async () => {
+ waits += 1
+ }
+ }
+ )
+
+ expect(result.health).toBe(true)
+ expect(result.ready).toBe(true)
+ expect(calls).toBe(2)
+ expect(waits).toBe(0)
+ })
+
+ it('retries one transient endpoint failure within the same sample', async () => {
+ const calls = new Map()
+ const waits: number[] = []
+ const result = await probeEndpointHealth(
+ 'https://c9.relay.onorca.dev',
+ async (input) => {
+ const path = new URL(String(input)).pathname
+ const call = (calls.get(path) ?? 0) + 1
+ calls.set(path, call)
+ return new Response(null, { status: path === '/ready' && call === 1 ? 503 : 200 })
+ },
+ {
+ wait: async (ms) => {
+ waits.push(ms)
+ }
+ }
+ )
+
+ expect(result.health).toBe(true)
+ expect(result.ready).toBe(true)
+ expect(calls).toEqual(new Map([['/health', 2], ['/ready', 2]]))
+ expect(waits).toEqual([11_000])
+ })
+
+ it('fails closed when the endpoint retry is also unhealthy', async () => {
+ let calls = 0
+ const waits: number[] = []
+ const result = await probeEndpointHealth(
+ 'https://c9.relay.onorca.dev',
+ async () => {
+ calls += 1
+ return new Response(null, { status: 503 })
+ },
+ {
+ wait: async (ms) => {
+ waits.push(ms)
+ }
+ }
+ )
+
+ expect(result.health).toBe(false)
+ expect(result.ready).toBe(false)
+ expect(calls).toBe(4)
+ // A refusing endpoint is a reading, so only the independent retry runs.
+ expect(waits).toEqual([11_000])
+ })
+
+ it('treats a thrown fetch as no reading and re-asks that path once', async () => {
+ const calls: string[] = []
+ const waits: number[] = []
+ const result = await probeEndpointHealth(
+ 'https://c9.relay.onorca.dev',
+ async (input) => {
+ const path = new URL(String(input)).pathname
+ calls.push(path)
+ if (path === '/health' && calls.filter((call) => call === '/health').length === 1) {
+ throw new TypeError('fetch failed')
+ }
+ return new Response(null, { status: 200 })
+ },
+ {
+ wait: async (ms) => {
+ waits.push(ms)
+ }
+ }
+ )
+
+ expect(result.health).toBe(true)
+ expect(result.ready).toBe(true)
+ expect(calls.filter((call) => call === '/health')).toEqual(['/health', '/health'])
+ expect(waits).toEqual([1_000])
+ })
+
+ it('fails closed when both attempts of a path throw', async () => {
+ const calls: string[] = []
+ const waits: number[] = []
+ const result = await probeEndpointHealth(
+ 'https://c9.relay.onorca.dev',
+ async (input) => {
+ const path = new URL(String(input)).pathname
+ calls.push(path)
+ if (path === '/health') throw new TypeError('fetch failed')
+ return new Response(null, { status: 200 })
+ },
+ {
+ wait: async (ms) => {
+ waits.push(ms)
+ }
+ }
+ )
+
+ expect(result.health).toBe(false)
+ expect(calls.filter((call) => call === '/health')).toHaveLength(4)
+ expect(waits).toEqual([1_000, 11_000, 1_000])
+ })
+
+ it('accepts an auth-shaped endpoint that serves no readiness path', async () => {
+ const calls: string[] = []
+ const waits: number[] = []
+ const result = await probeEndpointHealth(
+ 'https://login.onorca.dev',
+ async (input) => {
+ const path = new URL(String(input)).pathname
+ calls.push(path)
+ return new Response(null, { status: path === '/ready' ? 404 : 200 })
+ },
+ {
+ requiresReady: false,
+ wait: async (ms) => {
+ waits.push(ms)
+ }
+ }
+ )
+
+ expect(result.health).toBe(true)
+ expect(result.ready).toBeNull()
+ expect(calls).toEqual(['/health'])
+ expect(waits).toEqual([])
+ })
+
+ it('still requires readiness for the director and cells', async () => {
+ const waits: number[] = []
+ const result = await probeEndpointHealth(
+ 'https://relay.onorca.dev',
+ async (input) => new Response(null, {
+ status: new URL(String(input)).pathname === '/ready' ? 503 : 200
+ }),
+ {
+ wait: async (ms) => {
+ waits.push(ms)
+ }
+ }
+ )
+
+ expect(result.health).toBe(true)
+ expect(result.ready).toBe(false)
+ expect(waits).toEqual([11_000])
+ })
+
+ it('measures latency as the answering round trip, not the retry delay', async () => {
+ let healthCalls = 0
+ const result = await probeEndpointHealth(
+ 'https://c9.relay.onorca.dev',
+ async (input) => {
+ if (new URL(String(input)).pathname !== '/health') return new Response(null, { status: 200 })
+ healthCalls += 1
+ if (healthCalls === 1) throw new TypeError('fetch failed')
+ return new Response(null, { status: 200 })
+ },
+ { wait: async (ms) => await new Promise((resolve) => setTimeout(resolve, Math.min(ms, 60))) }
+ )
+
+ expect(result.health).toBe(true)
+ expect(result.latencyMs).not.toBeNull()
+ expect(result.latencyMs!).toBeLessThan(60)
+ })
+
+ it('uses aggregate REST inventory without probing sleeping staging endpoints', async () => {
+ const gcloud: GcloudClient = { accessToken: async () => 'a'.repeat(40) }
+ let publicProbeCalls = 0
+ const fetchImpl: typeof fetch = async (input) => {
+ const url = new URL(String(input))
+ if (url.hostname.endsWith('onorca.dev')) {
+ publicProbeCalls += 1
+ return Response.json({ status: 'ok' })
+ }
+ if (url.hostname === 'run.googleapis.com') return Response.json(runService)
+ if (url.hostname === 'sqladmin.googleapis.com') return Response.json({
+ state: 'STOPPED',
+ databaseVersion: 'POSTGRES_17',
+ settings: {
+ activationPolicy: 'NEVER',
+ availabilityType: 'ZONAL',
+ tier: 'db-custom-1-3840'
+ }
+ })
+ if (url.hostname === 'certificatemanager.googleapis.com') return Response.json({
+ managed: { domains: ['*.relay-staging.onorca.dev'], state: 'ACTIVE' }
+ })
+ if (url.pathname.includes('/instanceGroupManagers/')) {
+ const name = url.pathname.split('/').at(-1)!
+ return Response.json({
+ name,
+ targetSize: 0,
+ size: '0',
+ instanceGroup: `projects/project/zones/zone/instanceGroups/${name}`,
+ instanceTemplate: `projects/project/global/instanceTemplates/template-${name}`,
+ status: { isStable: true }
+ })
+ }
+ if (url.pathname.includes('/instanceTemplates/')) return Response.json({
+ properties: { metadata: { items: [{
+ key: 'startup-script',
+ value: `SECRET_TEXT\nORCA_RELAY_IMAGE_DIGEST=%s\\n' '${digest}'`
+ }] } }
+ })
+ if (url.pathname.endsWith('/getHealth')) return Response.json([])
+ throw new Error(`Unexpected request to ${url.hostname}${url.pathname}`)
+ }
+
+ const result = await readResourceInventory(
+ RELAY_OPS_ENVIRONMENTS.staging,
+ gcloud,
+ fetchImpl
+ )
+
+ expect(publicProbeCalls).toBe(0)
+ expect(result.cells.every((cell) => cell.targetSize === 0)).toBe(true)
+ expect(result.cells.every((cell) => cell.endpoint.health === null)).toBe(true)
+ expect(result.cells.every((cell) => cell.imageDigest === digest)).toBe(true)
+ expect(JSON.stringify(result)).not.toContain('SECRET_TEXT')
+ })
+
+ it('re-asks a MIG read that failed once before calling a cell powered-unknown', async () => {
+ const parkedCell = RELAY_OPS_ENVIRONMENTS.staging.cells[0]!
+ const waits: number[] = []
+ let parkedMigCalls = 0
+ const result = await readResourceInventory(
+ RELAY_OPS_ENVIRONMENTS.staging,
+ sleepingStagingGcloud,
+ sleepingStagingFetch((migName) => {
+ if (!migName.endsWith(parkedCell.hostname)) return 'ok'
+ parkedMigCalls += 1
+ return parkedMigCalls === 1 ? 'throw' : 'ok'
+ }),
+ { wait: async (ms) => { waits.push(ms) } }
+ )
+
+ const parked = result.cells.find((cell) => cell.cellId === parkedCell.cellId)!
+ // The MIG was fine and parked at zero; one transient read must not erase that reading.
+ expect(parked.targetSize).toBe(0)
+ expect(parkedMigCalls).toBe(2)
+ expect(waits).toEqual([1_000])
+ expect(result.warnings).toEqual([])
+ })
+
+ it('reports a MIG unavailable only when the retry fails too', async () => {
+ const parkedCell = RELAY_OPS_ENVIRONMENTS.staging.cells[0]!
+ const waits: number[] = []
+ let parkedMigCalls = 0
+ const result = await readResourceInventory(
+ RELAY_OPS_ENVIRONMENTS.staging,
+ sleepingStagingGcloud,
+ sleepingStagingFetch((migName) => {
+ if (!migName.endsWith(parkedCell.hostname)) return 'ok'
+ parkedMigCalls += 1
+ return 'throw'
+ }),
+ { wait: async (ms) => { waits.push(ms) } }
+ )
+
+ const parked = result.cells.find((cell) => cell.cellId === parkedCell.cellId)!
+ expect(parked.targetSize).toBeNull()
+ expect(parked.backendHealth).toBe('unknown')
+ expect(parkedMigCalls).toBe(2)
+ expect(waits).toEqual([1_000])
+ expect(result.warnings).toEqual([
+ `${parkedCell.hostname.toUpperCase()} MIG inventory is unavailable.`
+ ])
+ })
+
+ it('does not re-ask a MIG read the API answered with 404', async () => {
+ const missingCell = RELAY_OPS_ENVIRONMENTS.staging.cells[0]!
+ const waits: number[] = []
+ let missingMigCalls = 0
+ const result = await readResourceInventory(
+ RELAY_OPS_ENVIRONMENTS.staging,
+ sleepingStagingGcloud,
+ sleepingStagingFetch((migName) => {
+ if (!migName.endsWith(missingCell.hostname)) return 'ok'
+ missingMigCalls += 1
+ return 'missing'
+ }),
+ { wait: async (ms) => { waits.push(ms) } }
+ )
+
+ expect(result.cells.find((cell) => cell.cellId === missingCell.cellId)!.targetSize).toBeNull()
+ expect(missingMigCalls).toBe(1)
+ expect(waits).toEqual([])
+ })
+
+ it('represents missing credentials as unknown inventory, never sleeping', async () => {
+ const gcloud: GcloudClient = {
+ accessToken: async () => { throw new Error('sensitive context') }
+ }
+ let fetchCalls = 0
+ const result = await readResourceInventory(
+ RELAY_OPS_ENVIRONMENTS.production,
+ gcloud,
+ async () => { fetchCalls += 1; return Response.json({}) }
+ )
+ expect(fetchCalls).toBe(0)
+ expect(result.cells.every((cell) => cell.targetSize === null)).toBe(true)
+ expect(result.cells.every((cell) => cell.backendHealth === 'unknown')).toBe(true)
+ expect(JSON.stringify(result)).not.toContain('sensitive context')
+ })
+})
diff --git a/cloud/apps/relay-ops/src/resource-inventory.ts b/cloud/apps/relay-ops/src/resource-inventory.ts
new file mode 100644
index 00000000000..da490685198
--- /dev/null
+++ b/cloud/apps/relay-ops/src/resource-inventory.ts
@@ -0,0 +1,437 @@
+import { z } from 'zod'
+import type { RelayOpsEnvironment, RelayOpsCellConfig } from './environment-config.js'
+import type { GcloudClient } from './gcloud-client.js'
+import { INCIDENT_MONITOR_THRESHOLDS } from './incident-monitor.js'
+
+const RunServiceSchema = z.object({
+ template: z.object({
+ scaling: z.object({
+ minInstanceCount: z.number().optional(),
+ maxInstanceCount: z.number().optional()
+ }).optional(),
+ containers: z.array(z.object({ image: z.string() })).min(1)
+ }),
+ conditions: z.array(z.object({ state: z.string() })).default([]),
+ latestReadyRevision: z.string().optional()
+})
+
+const SqlInstanceSchema = z.object({
+ state: z.string(),
+ databaseVersion: z.string(),
+ settings: z.object({
+ activationPolicy: z.string(),
+ availabilityType: z.string().optional(),
+ tier: z.string()
+ })
+})
+
+const MigSchema = z.object({
+ name: z.string(),
+ targetSize: z.number(),
+ size: z.union([z.string(), z.number()]).transform(Number).optional(),
+ instanceGroup: z.string(),
+ instanceTemplate: z.string(),
+ status: z.object({ isStable: z.boolean().default(false) }).default({ isStable: false })
+})
+
+const TemplateSchema = z.object({
+ properties: z.object({
+ metadata: z.object({
+ items: z.array(z.object({ key: z.string(), value: z.string().optional() })).default([])
+ }).optional()
+ })
+})
+
+const BackendHealthGroupSchema = z.object({
+ healthStatus: z.array(z.object({ healthState: z.string() })).default([])
+})
+const BackendHealthSchema = z.union([
+ BackendHealthGroupSchema,
+ z.array(z.object({ status: BackendHealthGroupSchema }))
+])
+
+const CertificateSchema = z.object({
+ expireTime: z.string().optional(),
+ managed: z.object({
+ domains: z.array(z.string()).default([]),
+ state: z.string()
+ })
+})
+
+export type EndpointHealth = {
+ health: boolean | null
+ ready: boolean | null
+ latencyMs: number | null
+}
+
+export type ServiceInventory = {
+ ready: boolean
+ revision: string | null
+ image: string
+ minInstances: number
+ maxInstances: number
+}
+
+export type CellInventory = RelayOpsCellConfig & {
+ migName: string
+ targetSize: number | null
+ runningInstances: number | null
+ stable: boolean | null
+ template: string | null
+ imageDigest: string | null
+ backendHealth: 'healthy' | 'unhealthy' | 'empty' | 'unknown'
+ endpoint: EndpointHealth
+}
+
+export type ResourceInventory = {
+ director: ServiceInventory | null
+ auth: ServiceInventory | null
+ sql: {
+ state: string
+ activationPolicy: string
+ tier: string
+ availabilityType: string
+ databaseVersion: string
+ } | null
+ certificate: { state: string; domains: string[]; expireTime: string | null } | null
+ directorEndpoint: EndpointHealth
+ authEndpoint: EndpointHealth
+ cells: CellInventory[]
+ warnings: string[]
+}
+
+const unavailableEndpoint = (): EndpointHealth => ({ health: null, ready: null, latencyMs: null })
+const independentEndpointRetryDelayMs = 11_000
+const transientProbeRetryDelayMs = 1_000
+const sleep = async (ms: number): Promise =>
+ await new Promise((resolvePromise) => setTimeout(resolvePromise, ms))
+
+function finalSegment(value: string): string {
+ return value.split('/').at(-1) ?? value
+}
+
+function parseService(value: unknown): ServiceInventory {
+ const service = RunServiceSchema.parse(value)
+ return {
+ ready: service.conditions.length > 0 && service.conditions.every(
+ (condition) => condition.state === 'CONDITION_SUCCEEDED'
+ ),
+ revision: service.latestReadyRevision ? finalSegment(service.latestReadyRevision) : null,
+ image: service.template.containers[0]!.image,
+ minInstances: service.template.scaling?.minInstanceCount ?? 0,
+ maxInstances: service.template.scaling?.maxInstanceCount ?? 0
+ }
+}
+
+class GoogleApiError extends Error {
+ constructor(readonly status: number) {
+ super(`Google API returned ${status}`)
+ }
+}
+
+async function googleRequest(
+ fetchImpl: typeof fetch,
+ token: string,
+ url: string,
+ init: RequestInit = {}
+): Promise {
+ const response = await fetchImpl(url, {
+ ...init,
+ headers: {
+ authorization: `Bearer ${token}`,
+ ...(init.body ? { 'content-type': 'application/json' } : {})
+ },
+ signal: AbortSignal.timeout(30_000)
+ })
+ if (!response.ok) throw new GoogleApiError(response.status)
+ return await response.json()
+}
+
+// A 404 is the API's answer about the resource; anything else is the absence of a reading, so re-ask.
+async function readOnceMore(
+ read: () => Promise,
+ wait: (ms: number) => Promise
+): Promise {
+ try {
+ return await read()
+ } catch (error) {
+ if (error instanceof GoogleApiError && error.status === 404) throw error
+ await wait(transientProbeRetryDelayMs)
+ return await read()
+ }
+}
+
+// A reading the endpoint actually produced: ok is its answer, latencyMs is that answer's round trip.
+type PathReading = { ok: boolean; latencyMs: number | null }
+
+async function probePath(
+ origin: string,
+ path: '/health' | '/ready',
+ fetchImpl: typeof fetch,
+ wait: (ms: number) => Promise
+): Promise {
+ // null means the request never produced an answer (DNS/TCP/TLS failure or the 8s abort).
+ const attempt = async (): Promise => {
+ const startedAt = performance.now()
+ try {
+ const response = await fetchImpl(`${origin}${path}`, {
+ redirect: 'error',
+ signal: AbortSignal.timeout(8_000)
+ })
+ return { ok: response.ok, latencyMs: Math.round(performance.now() - startedAt) }
+ } catch {
+ return null
+ }
+ }
+ const first = await attempt()
+ if (first) return first
+ // A thrown fetch is the absence of a reading, not an unhealthy answer, so re-ask before concluding.
+ await wait(transientProbeRetryDelayMs)
+ return (await attempt()) ?? { ok: false, latencyMs: null }
+}
+
+async function endpointProbe(
+ origin: string,
+ fetchImpl: typeof fetch,
+ requiresReady: boolean,
+ wait: (ms: number) => Promise
+): Promise {
+ const [health, ready] = await Promise.all([
+ probePath(origin, '/health', fetchImpl, wait),
+ requiresReady ? probePath(origin, '/ready', fetchImpl, wait) : null
+ ])
+ // Latency is the slowest answering round trip in this probe; retry delays are not serving latency.
+ const latencies = [health.latencyMs, ready?.latencyMs ?? null].filter(
+ (value): value is number => value !== null
+ )
+ return {
+ health: health.ok,
+ ready: ready ? ready.ok : null,
+ latencyMs: latencies.length > 0 ? Math.max(...latencies) : null
+ }
+}
+
+export type EndpointProbeOptions = {
+ // Auth serves no /ready by design, so it is judged on /health and latency alone.
+ requiresReady?: boolean
+ wait?: (ms: number) => Promise
+}
+
+export async function probeEndpointHealth(
+ origin: string,
+ fetchImpl: typeof fetch,
+ options: EndpointProbeOptions = {}
+): Promise {
+ const requiresReady = options.requiresReady ?? true
+ const wait = options.wait ?? sleep
+ const accepted = (probe: EndpointHealth): boolean =>
+ probe.health === true &&
+ (!requiresReady || probe.ready === true) &&
+ probe.latencyMs !== null &&
+ probe.latencyMs <= INCIDENT_MONITOR_THRESHOLDS.endpointLatencyMs
+ const first = await endpointProbe(origin, fetchImpl, requiresReady, wait)
+ if (accepted(first)) return first
+ // Outwait Relay's ten-second readiness cache before treating the retry as independent.
+ await wait(independentEndpointRetryDelayMs)
+ return await endpointProbe(origin, fetchImpl, requiresReady, wait)
+}
+
+function imageDigest(template: z.infer): string | null {
+ const startupScript = template.properties.metadata?.items.find(
+ (item) => item.key === 'startup-script'
+ )?.value
+ // Return only the immutable digest; startup metadata contains secret names and operational detail.
+ return startupScript?.match(/ORCA_RELAY_IMAGE_DIGEST=%s\\n' '(sha256:[a-f0-9]{64})'/)?.[1] ?? null
+}
+
+function backendState(value: unknown): CellInventory['backendHealth'] {
+ const parsedHealth = BackendHealthSchema.parse(value)
+ const groups = Array.isArray(parsedHealth)
+ ? parsedHealth.map((group) => group.status)
+ : [parsedHealth]
+ const states = groups.flatMap((group) => group.healthStatus.map((status) => status.healthState))
+ if (states.length === 0) return 'empty'
+ if (states.every((state) => state === 'HEALTHY')) return 'healthy'
+ return 'unhealthy'
+}
+
+function unavailableCell(cell: RelayOpsCellConfig, migName: string): CellInventory {
+ return {
+ ...cell,
+ migName,
+ targetSize: null,
+ runningInstances: null,
+ stable: null,
+ template: null,
+ imageDigest: null,
+ backendHealth: 'unknown',
+ endpoint: unavailableEndpoint()
+ }
+}
+
+async function readCell(
+ environment: RelayOpsEnvironment,
+ cell: RelayOpsCellConfig,
+ mig: z.infer | null,
+ token: string,
+ fetchImpl: typeof fetch
+): Promise {
+ const migName = `${environment.migPrefix}${cell.hostname}`
+ if (!mig) return unavailableCell(cell, migName)
+ // An empty fixed-one MIG cannot serve and must never be woken by observation.
+ const endpoint = mig.targetSize > 0
+ ? await probeEndpointHealth(cell.origin, fetchImpl)
+ : unavailableEndpoint()
+ const templateName = finalSegment(mig.instanceTemplate)
+ const [templateResult, healthResult] = await Promise.allSettled([
+ googleRequest(
+ fetchImpl,
+ token,
+ `https://compute.googleapis.com/compute/v1/projects/${environment.project}/global/instanceTemplates/${templateName}`
+ ),
+ googleRequest(
+ fetchImpl,
+ token,
+ `https://compute.googleapis.com/compute/v1/projects/${environment.project}/global/backendServices/${migName}/getHealth`,
+ { method: 'POST', body: JSON.stringify({ group: mig.instanceGroup }) }
+ )
+ ])
+ return {
+ ...cell,
+ migName,
+ targetSize: mig.targetSize,
+ runningInstances: mig.size ?? (mig.status.isStable ? mig.targetSize : null),
+ stable: mig.status.isStable,
+ template: templateName,
+ imageDigest: templateResult.status === 'fulfilled'
+ ? imageDigest(TemplateSchema.parse(templateResult.value))
+ : null,
+ backendHealth: healthResult.status === 'fulfilled'
+ ? backendState(healthResult.value)
+ : 'unknown',
+ endpoint
+ }
+}
+
+function parsed(
+ result: PromiseSettledResult,
+ schema: S,
+ warning: string,
+ warnings: string[]
+): z.infer | null {
+ if (result.status === 'rejected') {
+ warnings.push(warning)
+ return null
+ }
+ const parsedValue = schema.safeParse(result.value)
+ if (!parsedValue.success) {
+ warnings.push(warning)
+ return null
+ }
+ return parsedValue.data
+}
+
+function unavailableInventory(environment: RelayOpsEnvironment, warning: string): ResourceInventory {
+ return {
+ director: null,
+ auth: null,
+ sql: null,
+ certificate: null,
+ directorEndpoint: unavailableEndpoint(),
+ authEndpoint: unavailableEndpoint(),
+ cells: environment.cells.map((cell) =>
+ unavailableCell(cell, `${environment.migPrefix}${cell.hostname}`)
+ ),
+ warnings: [warning]
+ }
+}
+
+export type ResourceInventoryOptions = {
+ wait?: (ms: number) => Promise
+}
+
+export async function readResourceInventory(
+ environment: RelayOpsEnvironment,
+ gcloud: GcloudClient,
+ fetchImpl: typeof fetch = fetch,
+ options: ResourceInventoryOptions = {}
+): Promise {
+ const wait = options.wait ?? sleep
+ let token: string
+ try {
+ token = await gcloud.accessToken()
+ } catch {
+ return unavailableInventory(
+ environment,
+ 'Google Cloud credentials are unavailable. Run gcloud auth login.'
+ )
+ }
+ const runUrl = (service: string) =>
+ `https://run.googleapis.com/v2/projects/${environment.project}/locations/${environment.region}/services/${service}`
+ const migUrl = (cell: RelayOpsCellConfig) =>
+ `https://compute.googleapis.com/compute/v1/projects/${environment.project}/zones/${cell.zone}/instanceGroupManagers/${environment.migPrefix}${cell.hostname}`
+ const settled = await Promise.allSettled([
+ googleRequest(fetchImpl, token, runUrl(environment.directorService)),
+ googleRequest(fetchImpl, token, runUrl(environment.authService)),
+ googleRequest(
+ fetchImpl,
+ token,
+ `https://sqladmin.googleapis.com/sql/v1beta4/projects/${environment.project}/instances/${environment.sqlInstance}`
+ ),
+ googleRequest(
+ fetchImpl,
+ token,
+ `https://certificatemanager.googleapis.com/v1/projects/${environment.project}/locations/global/certificates/${environment.certificateName}`
+ ),
+ // One transient Compute read must never become a verdict on a cell's power state.
+ ...environment.cells.map((cell) =>
+ readOnceMore(async () => await googleRequest(fetchImpl, token, migUrl(cell)), wait)
+ )
+ ])
+ const warnings: string[] = []
+ const directorValue = parsed(settled[0]!, RunServiceSchema, 'Director service inventory is unavailable.', warnings)
+ const authValue = parsed(settled[1]!, RunServiceSchema, 'Auth service inventory is unavailable.', warnings)
+ const sqlValue = parsed(settled[2]!, SqlInstanceSchema, 'Cloud SQL inventory is unavailable.', warnings)
+ const certificateValue = parsed(
+ settled[3]!, CertificateSchema, 'TLS certificate inventory is unavailable.', warnings
+ )
+ const migValues = environment.cells.map((cell, index) => parsed(
+ settled[index + 4]!,
+ MigSchema,
+ `${cell.hostname.toUpperCase()} MIG inventory is unavailable.`,
+ warnings
+ ))
+ const controlPlaneSleeping =
+ environment.id === 'staging' && sqlValue?.settings.activationPolicy === 'NEVER'
+ // Health probes would cold-start scale-to-zero Cloud Run services, so sleeping staging is inventory-only.
+ const [directorEndpoint, authEndpoint] = controlPlaneSleeping
+ ? [unavailableEndpoint(), unavailableEndpoint()]
+ : await Promise.all([
+ probeEndpointHealth(environment.directorOrigin, fetchImpl),
+ // The auth service exposes no /ready, so requiring it would fail every first probe.
+ probeEndpointHealth(environment.authOrigin, fetchImpl, { requiresReady: false })
+ ])
+ const cells = await Promise.all(environment.cells.map((cell, index) =>
+ readCell(environment, cell, migValues[index] ?? null, token, fetchImpl)
+ ))
+ return {
+ director: directorValue ? parseService(directorValue) : null,
+ auth: authValue ? parseService(authValue) : null,
+ sql: sqlValue ? {
+ state: sqlValue.state,
+ activationPolicy: sqlValue.settings.activationPolicy,
+ tier: sqlValue.settings.tier,
+ availabilityType: sqlValue.settings.availabilityType ?? 'unknown',
+ databaseVersion: sqlValue.databaseVersion
+ } : null,
+ certificate: certificateValue ? {
+ state: certificateValue.managed.state,
+ domains: certificateValue.managed.domains,
+ expireTime: certificateValue.expireTime ?? null
+ } : null,
+ directorEndpoint,
+ authEndpoint,
+ cells,
+ warnings
+ }
+}
diff --git a/cloud/apps/relay-ops/src/staging-workflow.test.ts b/cloud/apps/relay-ops/src/staging-workflow.test.ts
new file mode 100644
index 00000000000..881ffb2bf9a
--- /dev/null
+++ b/cloud/apps/relay-ops/src/staging-workflow.test.ts
@@ -0,0 +1,19 @@
+import { describe, expect, it } from 'vitest'
+import { parseStagingPowerRequest } from './staging-workflow.js'
+
+describe('parseStagingPowerRequest', () => {
+ it('accepts the exact reviewed confirmations', () => {
+ expect(parseStagingPowerRequest({ mode: 'status', confirmation: '' })).toEqual({
+ mode: 'status', confirmation: ''
+ })
+ expect(parseStagingPowerRequest({ mode: 'wake', confirmation: 'WAKE_STAGING' }).mode).toBe('wake')
+ expect(parseStagingPowerRequest({ mode: 'sleep', confirmation: 'SLEEP_STAGING' }).mode).toBe('sleep')
+ })
+
+ it('rejects missing, swapped, or additional fields', () => {
+ expect(() => parseStagingPowerRequest({ mode: 'wake', confirmation: '' })).toThrow()
+ expect(() => parseStagingPowerRequest({ mode: 'sleep', confirmation: 'WAKE_STAGING' })).toThrow()
+ expect(() => parseStagingPowerRequest({ mode: 'production', confirmation: '' })).toThrow()
+ expect(() => parseStagingPowerRequest({ mode: 'status', confirmation: '', project: 'other' })).toThrow()
+ })
+})
diff --git a/cloud/apps/relay-ops/src/staging-workflow.ts b/cloud/apps/relay-ops/src/staging-workflow.ts
new file mode 100644
index 00000000000..749c6e865bb
--- /dev/null
+++ b/cloud/apps/relay-ops/src/staging-workflow.ts
@@ -0,0 +1,36 @@
+import { execFile } from 'node:child_process'
+import { promisify } from 'node:util'
+import { z } from 'zod'
+import { RELAY_GITHUB_REPOSITORY, relayWorkflowFile } from './relay-repository.js'
+
+const execFileAsync = promisify(execFile)
+const DispatchSchema = z.discriminatedUnion('mode', [
+ z.object({ mode: z.literal('status'), confirmation: z.literal('') }).strict(),
+ z.object({ mode: z.literal('wake'), confirmation: z.literal('WAKE_STAGING') }).strict(),
+ z.object({ mode: z.literal('sleep'), confirmation: z.literal('SLEEP_STAGING') }).strict()
+])
+
+export type StagingPowerRequest = z.infer
+
+export function parseStagingPowerRequest(value: unknown): StagingPowerRequest {
+ return DispatchSchema.parse(value)
+}
+
+export async function dispatchStagingPowerWorkflow(request: StagingPowerRequest): Promise {
+ const args = [
+ 'workflow', 'run', relayWorkflowFile('power-relay-staging.yml'),
+ '--repo', RELAY_GITHUB_REPOSITORY,
+ '-f', `mode=${request.mode}`,
+ '-f', 'wake-cells=configured'
+ ]
+ if (request.confirmation) args.push('-f', `confirmation=${request.confirmation}`)
+ try {
+ await execFileAsync('gh', args, {
+ encoding: 'utf8',
+ timeout: 30_000,
+ maxBuffer: 1024 * 1024
+ })
+ } catch {
+ throw new Error('Staging power workflow dispatch failed')
+ }
+}
diff --git a/cloud/apps/relay-ops/tsconfig.build.json b/cloud/apps/relay-ops/tsconfig.build.json
new file mode 100644
index 00000000000..0a11719fef1
--- /dev/null
+++ b/cloud/apps/relay-ops/tsconfig.build.json
@@ -0,0 +1,4 @@
+{
+ "extends": "./tsconfig.json",
+ "exclude": ["src/**/*.test.ts"]
+}
diff --git a/cloud/apps/relay-ops/tsconfig.json b/cloud/apps/relay-ops/tsconfig.json
new file mode 100644
index 00000000000..d266baf8ad6
--- /dev/null
+++ b/cloud/apps/relay-ops/tsconfig.json
@@ -0,0 +1,11 @@
+{
+ "extends": "../../tsconfig.base.json",
+ "compilerOptions": {
+ "rootDir": "src",
+ "outDir": "dist",
+ "types": ["node", "vitest"],
+ "exactOptionalPropertyTypes": true,
+ "verbatimModuleSyntax": true
+ },
+ "include": ["src/**/*.ts"]
+}
diff --git a/cloud/apps/relay/Dockerfile b/cloud/apps/relay/Dockerfile
new file mode 100644
index 00000000000..12516cbf749
--- /dev/null
+++ b/cloud/apps/relay/Dockerfile
@@ -0,0 +1,25 @@
+FROM node:24-alpine AS build
+WORKDIR /app
+RUN corepack enable
+COPY package.json pnpm-lock.yaml pnpm-workspace.yaml tsconfig.base.json ./
+COPY packages/relay-contract/package.json packages/relay-contract/package.json
+COPY apps/relay/package.json apps/relay/package.json
+RUN pnpm install --frozen-lockfile
+COPY packages/relay-contract packages/relay-contract
+COPY apps/relay apps/relay
+RUN pnpm --filter @orca-cloud/relay-contract build && pnpm --filter @orca-cloud/relay build
+
+FROM node:24-alpine AS runtime
+ENV NODE_ENV=production
+ENV PORT=8080
+WORKDIR /app
+RUN corepack enable
+COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
+COPY packages/relay-contract/package.json packages/relay-contract/package.json
+COPY apps/relay/package.json apps/relay/package.json
+COPY --from=build /app/packages/relay-contract/dist packages/relay-contract/dist
+COPY --from=build /app/apps/relay/dist apps/relay/dist
+RUN pnpm install --prod --frozen-lockfile --filter @orca-cloud/relay...
+USER node
+EXPOSE 8080
+CMD ["node", "apps/relay/dist/index.js"]
diff --git a/cloud/apps/relay/package.json b/cloud/apps/relay/package.json
new file mode 100644
index 00000000000..4c2b2e4269c
--- /dev/null
+++ b/cloud/apps/relay/package.json
@@ -0,0 +1,35 @@
+{
+ "name": "@orca-cloud/relay",
+ "private": true,
+ "version": "0.0.0",
+ "type": "module",
+ "main": "dist/index.js",
+ "scripts": {
+ "build": "pnpm clean && tsc -p tsconfig.build.json",
+ "clean": "node -e \"require('fs').rmSync('dist', { recursive: true, force: true })\"",
+ "dev": "tsx watch src/index.ts",
+ "lint": "tsc -p tsconfig.json --noEmit",
+ "pretest": "pnpm --filter @orca-cloud/relay-contract build",
+ "start": "node dist/index.js",
+ "test": "vitest run",
+ "typecheck": "tsc -p tsconfig.json --noEmit"
+ },
+ "dependencies": {
+ "@hono/node-server": "^1.19.14",
+ "@orca-cloud/relay-contract": "workspace:*",
+ "hono": "^4.12.27",
+ "jose": "^6.1.3",
+ "pg": "^8.22.0",
+ "tweetnacl": "^1.0.3",
+ "ws": "^8.18.3",
+ "zod": "^3.25.76"
+ },
+ "devDependencies": {
+ "@types/node": "^24.10.0",
+ "@types/pg": "^8.20.0",
+ "@types/ws": "^8.18.1",
+ "tsx": "^4.21.0",
+ "typescript": "^5.9.3",
+ "vitest": "^4.0.8"
+ }
+}
diff --git a/cloud/apps/relay/src/admin-token-verifier.test.ts b/cloud/apps/relay/src/admin-token-verifier.test.ts
new file mode 100644
index 00000000000..f0870dbb09c
--- /dev/null
+++ b/cloud/apps/relay/src/admin-token-verifier.test.ts
@@ -0,0 +1,98 @@
+import { describe, expect, it } from 'vitest'
+import {
+ RELAY_ASIA_PROOF_ADMIN_ROUTES,
+ RELAY_CAPACITY_ADMIN_ROUTES,
+ RELAY_FENCE_BROKER_ADMIN_ROUTES,
+ RELAY_FENCE_ADMIN_ROUTES,
+ RELAY_MONITOR_ADMIN_ROUTES,
+ relayAdminIdentityMayAccess
+} from './admin-token-verifier.js'
+
+const mutationRoutes = [
+ '/v1/admin/drain',
+ '/v1/admin/evacuate',
+ '/v1/admin/migration-complete',
+ '/v1/admin/migration-supersede-cell',
+ '/v1/admin/rebalance-dormant',
+ '/v1/admin/admission-selector/apply',
+ '/v1/admin/admission-selector/add-migration-cells',
+ '/v1/admin/cell-state',
+ '/v1/admin/cell-fence-adopt-legacy',
+ '/v1/admin/cell-fence-commit-legacy-adoption',
+ '/v1/admin/cell-fence-attest',
+ '/v1/admin/cell-fence-attempt-prepare',
+ '/v1/admin/cell-fence-attempt-start',
+ '/v1/admin/cell-fence-attempt-operation',
+ '/v1/admin/cell-fence-attempt-abort',
+ '/v1/admin/drain-attempt-prepare',
+ '/v1/admin/drain-attempt-send',
+ '/v1/admin/drain-attempt-receipt',
+ '/v1/admin/drain-attempt-recover-forward',
+ '/v1/admin/cell-config',
+ '/v1/admin/evacuate-cell',
+ '/v1/admin/cell-heartbeat',
+ '/v1/admin/regional-rehome-control',
+ '/v1/admin/regional-rehome-trust-probe'
+] as const
+
+describe('Relay admin route authorization', () => {
+ it('allows the staging capacity identity only its transition routes', () => {
+ for (const route of RELAY_CAPACITY_ADMIN_ROUTES) {
+ expect(relayAdminIdentityMayAccess('capacity', route)).toBe(true)
+ }
+ for (const route of mutationRoutes) {
+ if ((RELAY_CAPACITY_ADMIN_ROUTES as readonly string[]).includes(route)) continue
+ expect(relayAdminIdentityMayAccess('capacity', route)).toBe(false)
+ }
+ expect(RELAY_CAPACITY_ADMIN_ROUTES).toContain('/v1/admin/cell-state')
+ expect(relayAdminIdentityMayAccess('capacity', '/v1/admin/evacuation-status')).toBe(false)
+ })
+
+ it('allows the Asia proof identity only its selector and status routes', () => {
+ for (const route of RELAY_ASIA_PROOF_ADMIN_ROUTES) {
+ expect(relayAdminIdentityMayAccess('asia-proof', route)).toBe(true)
+ }
+ expect(relayAdminIdentityMayAccess('asia-proof', '/v1/admin/drain')).toBe(false)
+ expect(relayAdminIdentityMayAccess('asia-proof', '/v1/admin/cell-state')).toBe(false)
+ expect(relayAdminIdentityMayAccess('asia-proof', '/v1/admin/admission-selector/apply')).toBe(false)
+ expect(relayAdminIdentityMayAccess('asia-proof', '/v1/admin/add-migration-cells')).toBe(false)
+ })
+
+ it('keeps the monitor identity on exact aggregate read routes', () => {
+ for (const route of RELAY_MONITOR_ADMIN_ROUTES) {
+ expect(relayAdminIdentityMayAccess('monitor', route)).toBe(true)
+ }
+ for (const route of [...mutationRoutes, ...RELAY_FENCE_ADMIN_ROUTES]) {
+ if ((RELAY_MONITOR_ADMIN_ROUTES as readonly string[]).includes(route)) continue
+ expect(relayAdminIdentityMayAccess('monitor', route)).toBe(false)
+ }
+ expect(RELAY_MONITOR_ADMIN_ROUTES).toContain('/v1/admin/evacuation-status')
+ })
+
+ it('allows only reviewed fence evidence mutations beyond aggregate reads', () => {
+ for (const route of RELAY_FENCE_ADMIN_ROUTES) {
+ expect(relayAdminIdentityMayAccess('fence', route)).toBe(true)
+ }
+ for (const route of mutationRoutes) {
+ if ((RELAY_FENCE_ADMIN_ROUTES as readonly string[]).includes(route)) continue
+ expect(relayAdminIdentityMayAccess('fence', route)).toBe(false)
+ }
+ })
+
+ it('allows the broker only exact fence inspection and mutation routes', () => {
+ for (const route of RELAY_FENCE_BROKER_ADMIN_ROUTES) {
+ expect(relayAdminIdentityMayAccess('fence-broker', route)).toBe(true)
+ }
+ for (const route of [...mutationRoutes, ...RELAY_FENCE_ADMIN_ROUTES]) {
+ if ((RELAY_FENCE_BROKER_ADMIN_ROUTES as readonly string[]).includes(route)) continue
+ expect(relayAdminIdentityMayAccess('fence-broker', route)).toBe(false)
+ }
+ })
+
+ it('rejects unknown routes for dedicated identities', () => {
+ for (const identity of ['capacity', 'asia-proof', 'monitor', 'fence', 'fence-broker'] as const) {
+ expect(relayAdminIdentityMayAccess(identity, '/v1/admin/future-mutation')).toBe(false)
+ expect(relayAdminIdentityMayAccess(identity, '/health')).toBe(false)
+ }
+ })
+})
diff --git a/cloud/apps/relay/src/admin-token-verifier.ts b/cloud/apps/relay/src/admin-token-verifier.ts
new file mode 100644
index 00000000000..4b8ad26e695
--- /dev/null
+++ b/cloud/apps/relay/src/admin-token-verifier.ts
@@ -0,0 +1,202 @@
+import { createRemoteJWKSet, jwtVerify } from 'jose'
+import type { RelayConfig } from './config.js'
+
+export const RELAY_MONITOR_ADMIN_ROUTES = [
+ '/v1/admin/admission-selector/status',
+ '/v1/admin/cell-status',
+ '/v1/admin/evacuation-status',
+ '/v1/admin/regional-rehome-control',
+ '/v1/admin/runtime-status'
+] as const
+
+export const RELAY_FENCE_ADMIN_ROUTES = [
+ ...RELAY_MONITOR_ADMIN_ROUTES,
+ '/v1/admin/evacuation-capacity',
+ '/v1/admin/cell-fence-attempt-status'
+] as const
+
+export const RELAY_CAPACITY_ADMIN_ROUTES = [
+ '/v1/admin/admission-selector/status',
+ '/v1/admin/admission-selector/apply',
+ '/v1/admin/cell-state',
+ '/v1/admin/cell-status',
+ '/v1/admin/runtime-status',
+ '/v1/admin/drain'
+] as const
+
+export const RELAY_ASIA_PROOF_ADMIN_ROUTES = [
+ '/v1/admin/admission-selector/status',
+ '/v1/admin/admission-selector/apply-staging-asia-proof',
+ '/v1/admin/cell-status',
+ '/v1/admin/runtime-status'
+] as const
+
+export const RELAY_FENCE_BROKER_ADMIN_ROUTES = [
+ ...RELAY_FENCE_ADMIN_ROUTES,
+ '/v1/admin/cell-fence-adopt-legacy',
+ '/v1/admin/cell-fence-commit-legacy-adoption',
+ '/v1/admin/cell-fence-attest',
+ '/v1/admin/cell-fence-attempt-prepare',
+ '/v1/admin/cell-fence-attempt-start',
+ '/v1/admin/cell-fence-attempt-plan',
+ '/v1/admin/cell-fence-attempt-operation',
+ '/v1/admin/cell-fence-attempt-abort',
+ '/v1/admin/migration-supersede-cell'
+] as const
+
+type RelayAdminIdentity =
+ | 'deploy'
+ | 'capacity'
+ | 'asia-proof'
+ | 'monitor'
+ | 'fence'
+ | 'fence-broker'
+
+function createGoogleServiceTokenVerifier(input: {
+ jwksUrl: string
+ audience: string
+ serviceAccount: string
+}): (token: string) => Promise {
+ const jwks = createRemoteJWKSet(new URL(input.jwksUrl))
+ return async (token) => {
+ try {
+ const { payload } = await jwtVerify(token, jwks, {
+ issuer: ['https://accounts.google.com', 'accounts.google.com'],
+ audience: input.audience,
+ algorithms: ['RS256']
+ })
+ return payload.email === input.serviceAccount && payload.email_verified === true
+ } catch {
+ return false
+ }
+ }
+}
+
+function createGoogleServiceTokenIdentityVerifier(input: {
+ jwksUrl: string
+ audience: string
+ serviceAccounts: ReadonlyMap
+}): (token: string) => Promise {
+ const jwks = createRemoteJWKSet(new URL(input.jwksUrl))
+ return async (token) => {
+ try {
+ const { payload } = await jwtVerify(token, jwks, {
+ issuer: ['https://accounts.google.com', 'accounts.google.com'],
+ audience: input.audience,
+ algorithms: ['RS256']
+ })
+ if (payload.email_verified !== true || typeof payload.email !== 'string') return null
+ return input.serviceAccounts.get(payload.email) ?? null
+ } catch {
+ return null
+ }
+ }
+}
+
+export function relayAdminIdentityMayAccess(
+ identity: RelayAdminIdentity,
+ route: string
+): boolean {
+ if (identity === 'deploy') return route.startsWith('/v1/admin/')
+ if (identity === 'capacity') {
+ return (RELAY_CAPACITY_ADMIN_ROUTES as readonly string[]).includes(route)
+ }
+ if (identity === 'asia-proof') {
+ return (RELAY_ASIA_PROOF_ADMIN_ROUTES as readonly string[]).includes(route)
+ }
+ if (identity === 'monitor') {
+ return (RELAY_MONITOR_ADMIN_ROUTES as readonly string[]).includes(route)
+ }
+ if (identity === 'fence') {
+ return (RELAY_FENCE_ADMIN_ROUTES as readonly string[]).includes(route)
+ }
+ return (RELAY_FENCE_BROKER_ADMIN_ROUTES as readonly string[]).includes(route)
+}
+
+export function createReadOnlyAdminTokenVerifier(
+ config: RelayConfig
+): (token: string) => Promise {
+ const serviceAccounts = new Map()
+ if (config.monitorServiceAccount) serviceAccounts.set(config.monitorServiceAccount, 'monitor')
+ if (config.fenceServiceAccount) serviceAccounts.set(config.fenceServiceAccount, 'fence')
+ if (serviceAccounts.size === 0) return async () => false
+ const verifyIdentity = createGoogleServiceTokenIdentityVerifier({
+ jwksUrl: config.adminJwksUrl,
+ audience: config.adminAudience,
+ serviceAccounts
+ })
+ return async (token) => (await verifyIdentity(token)) !== null
+}
+
+export function createAdminTokenVerifier(
+ config: RelayConfig
+): (token: string, route?: string) => Promise {
+ const serviceAccounts = new Map([
+ [config.deployServiceAccount, 'deploy']
+ ])
+ if (config.capacityServiceAccount) {
+ serviceAccounts.set(config.capacityServiceAccount, 'capacity')
+ }
+ if (config.asiaProofServiceAccount) {
+ serviceAccounts.set(config.asiaProofServiceAccount, 'asia-proof')
+ }
+ if (config.monitorServiceAccount) serviceAccounts.set(config.monitorServiceAccount, 'monitor')
+ if (config.fenceServiceAccount) serviceAccounts.set(config.fenceServiceAccount, 'fence')
+ if (config.fenceBrokerServiceAccount) {
+ serviceAccounts.set(config.fenceBrokerServiceAccount, 'fence-broker')
+ }
+ const verifyIdentity = createGoogleServiceTokenIdentityVerifier({
+ jwksUrl: config.adminJwksUrl,
+ audience: config.adminAudience,
+ serviceAccounts
+ })
+ return async (token, route) => {
+ const identity = await verifyIdentity(token)
+ return identity !== null && (!route || relayAdminIdentityMayAccess(identity, route))
+ }
+}
+
+export function createRegionalRehomeControlApplyTokenVerifier(
+ config: RelayConfig
+): (token: string) => Promise {
+ return createGoogleServiceTokenVerifier({
+ jwksUrl: config.adminJwksUrl,
+ audience: config.adminAudience,
+ serviceAccount: config.deployServiceAccount
+ })
+}
+
+export function createRuntimeTokenVerifier(
+ config: RelayConfig
+): (token: string) => Promise {
+ if (!config.heartbeatAudience) return async () => false
+ return createGoogleServiceTokenVerifier({
+ jwksUrl: config.adminJwksUrl,
+ audience: config.heartbeatAudience,
+ serviceAccount: config.runtimeServiceAccount
+ })
+}
+
+export function createRegionalRehomeTokenVerifier(
+ config: RelayConfig
+): (token: string) => Promise {
+ if (!config.rehomeAudience || !config.rehomeDirectorServiceAccount) {
+ return async () => false
+ }
+ return createGoogleServiceTokenVerifier({
+ jwksUrl: config.adminJwksUrl,
+ audience: config.rehomeAudience,
+ serviceAccount: config.rehomeDirectorServiceAccount
+ })
+}
+
+export function createRegionalRehomeRuntimeTokenVerifier(
+ config: RelayConfig
+): (token: string) => Promise {
+ if (!config.rehomeAudience) return async () => false
+ return createGoogleServiceTokenVerifier({
+ jwksUrl: config.adminJwksUrl,
+ audience: config.rehomeAudience,
+ serviceAccount: config.runtimeServiceAccount
+ })
+}
diff --git a/cloud/apps/relay/src/app.ts b/cloud/apps/relay/src/app.ts
new file mode 100644
index 00000000000..3df01d9e9ce
--- /dev/null
+++ b/cloud/apps/relay/src/app.ts
@@ -0,0 +1,1850 @@
+import {
+ AssignmentRequestSchema,
+ isRelayCellConnectionHardCap,
+ RELAY_ADMISSION_BUDGETS,
+ RELAY_DEFAULT_REGION,
+ RELAY_MAX_CELL_CONNECTION_UNOBSERVED_BOUND,
+ RELAY_PROTOCOL_LIMITS,
+ RelayRegionSchema,
+ ResolveRequestSchema,
+ cellPlacementCeiling,
+ relayCellAdmissionBounds,
+ type RelayCellConnectionHardCap,
+ type RelayRegion
+} from '@orca-cloud/relay-contract'
+import { Hono, type Context } from 'hono'
+import { SignJWT } from 'jose'
+import { z } from 'zod'
+import {
+ createAdminTokenVerifier,
+ createReadOnlyAdminTokenVerifier,
+ createRegionalRehomeControlApplyTokenVerifier,
+ createRegionalRehomeRuntimeTokenVerifier,
+ createRegionalRehomeTokenVerifier,
+ createRuntimeTokenVerifier
+} from './admin-token-verifier.js'
+import type {
+ CellFenceAttemptEvidence,
+ RelayAssignment,
+ RelayAssignmentStore
+} from './assignment-store.js'
+import { AssignmentRejectionLogWindow } from './assignment-rejection-log-window.js'
+import { CELL_ADMISSION_STATES } from './cell-admission-selector.js'
+import { RELAY_MAX_CELL_CAPACITY_REQUESTS, type RelayConfig } from './config.js'
+import type { RelayCredentialStore } from './credential-store.js'
+import { isRelayDatabaseTransientError } from './database.js'
+import { googleMetadataIdentityToken } from './google-metadata-identity-token.js'
+import {
+ RelayPublicAssignmentAdmission,
+ type AssignmentAdmissionRejection
+} from './public-assignment-admission.js'
+import { relayHostLogDigest } from './relay-host-log-digest.js'
+import type { RelayRuntimeCounts } from './relay-observability.js'
+import {
+ isRegionalRehomeTrustProbe,
+ probeRegionalRehomeTrust
+} from './regional-rehome-trust-probe.js'
+import { createRelayTokenVerifier, readBearer } from './relay-token-verifier.js'
+import { stagingAsiaProofMembership } from './staging-asia-proof-admission.js'
+
+const RelayCellConnectionHardCapSchema = z.custom(
+ isRelayCellConnectionHardCap
+)
+
+const ASSIGNMENT_REJECTION_LOG_WINDOW_MS = 10_000
+const REGION_CATALOG_CACHE_MS = 30_000
+
+type AdmissionRejectionLogEntry = {
+ route: 'assign' | 'resolve'
+ lane: 'sticky' | 'placement'
+ hinted: boolean
+ relayHostId: string
+ reason: AssignmentAdmissionRejection
+}
+
+export function createRelayApp(
+ config: RelayConfig,
+ operations: {
+ store: RelayCredentialStore
+ assignments: RelayAssignmentStore
+ drain: (graceMs: number) => void
+ drainHost?: (input: {
+ attemptId: string
+ userId: string
+ relayHostId: string
+ sourceAssignmentEpoch: number
+ graceMs: number
+ }) => 'accepted' | 'already-accepted' | 'host-not-connected'
+ regionalRehomeIdentityToken?: (audience: string) => Promise
+ regionalRehomeFetch?: typeof fetch
+ regionalRehomeTrustProbeHostExists?: (input: {
+ userId: string
+ relayHostId: string
+ }) => boolean
+ cellIncarnation?: string
+ isDraining?: () => boolean
+ runtimeCounts?: () => RelayRuntimeCounts
+ ready: () => Promise
+ recordAssignmentAdmission?: (
+ outcome: 'sticky' | 'sticky-rejected' | 'placement' | 'placement-rejected'
+ ) => void
+ recordAssignmentRejectionReason?: (
+ lane: 'sticky' | 'placement',
+ reason: AssignmentAdmissionRejection
+ ) => void
+ recordRegionRequest?: (region: RelayRegion | undefined) => void
+ recordRegionSelection?: (input: {
+ targetRegion: RelayRegion
+ selectedRegion?: RelayRegion
+ fallback: boolean
+ }) => void
+ }
+): Hono {
+ const app = new Hono()
+ let regionCatalogCache:
+ | { expiresAt: number; value: Awaited> }
+ | undefined
+ let regionCatalogRefresh:
+ | Promise>>
+ | undefined
+ const regionCatalog = async () => {
+ const now = Date.now()
+ if (regionCatalogCache && regionCatalogCache.expiresAt > now) {
+ return regionCatalogCache.value
+ }
+ regionCatalogRefresh ??= operations.assignments.regionCatalog().then((value) => {
+ regionCatalogCache = { expiresAt: Date.now() + REGION_CATALOG_CACHE_MS, value }
+ return value
+ })
+ try {
+ return await regionCatalogRefresh
+ } finally {
+ regionCatalogRefresh = undefined
+ }
+ }
+ const verifyRelayToken = createRelayTokenVerifier(config)
+ const verifyAdminToken = createAdminTokenVerifier(config)
+ const verifyReadOnlyAdminToken = createReadOnlyAdminTokenVerifier(config)
+ const verifyRegionalRehomeControlApplyToken =
+ createRegionalRehomeControlApplyTokenVerifier(config)
+ const verifyRuntimeToken = createRuntimeTokenVerifier(config)
+ const verifyRegionalRehomeToken = createRegionalRehomeTokenVerifier(config)
+ const verifyRegionalRehomeRuntimeToken =
+ createRegionalRehomeRuntimeTokenVerifier(config)
+ const regionalRehomeFetch = operations.regionalRehomeFetch ?? fetch
+ const regionalRehomeIdentityToken =
+ operations.regionalRehomeIdentityToken ??
+ ((audience: string) => googleMetadataIdentityToken(audience, regionalRehomeFetch))
+ const publicAssignmentAdmission = new RelayPublicAssignmentAdmission({
+ maxConcurrent: config.publicAssignmentConcurrency,
+ maxQueued: config.publicAssignmentQueueMax,
+ waitMs: config.publicAssignmentWaitMs,
+ maxReservedConcurrent: config.publicResolveConcurrency,
+ reservedWaitMs: config.publicResolveWaitMs,
+ minIntervalMs: config.publicAssignmentRetryAfterSeconds * 1_000,
+ onRejected: (reason) => operations.recordAssignmentRejectionReason?.('placement', reason)
+ })
+ const rejectPublicAssignment = (context: Context): Response => {
+ context.header('Retry-After', String(config.publicAssignmentRetryAfterSeconds))
+ return context.json({ error: 'assignments_temporarily_unavailable' }, 503)
+ }
+ // Reconnecting hosts declare themselves and are verified against the durable
+ // assignment inside this bounded lane, so a placement backlog can never
+ // starve session recovery. Unhinted traffic never touches this lane.
+ const stickyRetryAfterSeconds = config.publicStickyRetryAfterSeconds ?? 2
+ const stickyAssignmentAdmission = new RelayPublicAssignmentAdmission({
+ maxConcurrent: config.publicStickyConcurrency ?? 1,
+ maxQueued: config.publicStickyQueueMax ?? 64,
+ waitMs: config.publicStickyWaitMs ?? 2_000,
+ minIntervalMs: stickyRetryAfterSeconds * 1_000,
+ onRejected: (reason) => operations.recordAssignmentRejectionReason?.('sticky', reason)
+ })
+ const rejectStickyAssignment = (context: Context): Response => {
+ context.header('Retry-After', String(stickyRetryAfterSeconds))
+ return context.json({ error: 'assignments_temporarily_unavailable' }, 503)
+ }
+ // Aggregate counters cannot separate a handful of pathological hosts from a broad
+ // population, so every admission rejection names its host and reason. Keyed on
+ // route:lane:reason rather than host, the log stays bounded under load.
+ const rejectionLogWindow = new AssignmentRejectionLogWindow({
+ windowMs: ASSIGNMENT_REJECTION_LOG_WINDOW_MS,
+ onWindowClosed: ({ suppressed, sample }) => logAssignmentRejection({ ...sample, suppressed })
+ })
+ const logAdmissionRejection = (input: {
+ route: 'assign' | 'resolve'
+ lane: 'sticky' | 'placement'
+ hinted: boolean
+ relayHostId: string
+ reason: AssignmentAdmissionRejection | undefined
+ }): void => {
+ if (!input.reason) return
+ const entry: AdmissionRejectionLogEntry = { ...input, reason: input.reason }
+ if (!rejectionLogWindow.admit(`${entry.route}:${entry.lane}:${entry.reason}`, entry)) return
+ logAssignmentRejection(entry)
+ }
+ app.use('/v1/admin/*', async (context, next) => {
+ if (
+ context.req.path === '/v1/admin/cell-heartbeat' ||
+ context.req.path === '/v1/admin/cell-rehome-status'
+ ) {
+ return await next()
+ }
+ const bearer = readBearer(context.req.header('authorization'))
+ if (!bearer || !(await verifyAdminToken(bearer))) return await next()
+ if (!(await verifyAdminToken(bearer, context.req.path))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ return await next()
+ })
+
+ // Not /healthz: Google Front End reserves that path before the container.
+ app.get('/health', (context) =>
+ context.json({ ok: true, connectionCapacityProtocol: 2 })
+ )
+ app.get('/ready', async (context) =>
+ (await operations.ready())
+ ? context.json({ ok: true })
+ : context.json({ error: 'dependency_unavailable' }, 503)
+ )
+ app.get('/v1/regions', async (context) => {
+ if (config.role === 'cell') return context.json({ error: 'director_only' }, 404)
+ return context.json({ v: 1, regions: await regionCatalog() })
+ })
+ app.post('/v1/assign', async (context) => {
+ if (config.role === 'cell') return context.json({ error: 'director_only' }, 404)
+ if (!config.publicAssignmentsEnabled) return rejectPublicAssignment(context)
+ const bearer = readBearer(context.req.header('authorization'))
+ if (!bearer) return context.json({ error: 'invalid_token' }, 401)
+ const claims = await verifyRelayToken(bearer)
+ if (!claims) return context.json({ error: 'invalid_token' }, 401)
+ if (Number(context.req.header('content-length') ?? 0) > 4 * 1024) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = AssignmentRequestSchema.safeParse(await context.req.json().catch(() => null))
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ if (body.data.relayHostId !== claims.relayHostId) {
+ return context.json({ error: 'host_identity_mismatch' }, 403)
+ }
+ const identity = { userId: claims.sub, relayHostId: claims.relayHostId }
+ const requestedRegion = body.data.preferredRegion
+ const targetRegion =
+ config.regionalPlacementEnabled !== false && requestedRegion
+ ? requestedRegion
+ : RELAY_DEFAULT_REGION
+ operations.recordRegionRequest?.(requestedRegion)
+ let admission: { release(): void } | null = null
+ let lane: 'sticky' | 'placement' = 'placement'
+ if (body.data.reconnect) {
+ let rejection: AssignmentAdmissionRejection | undefined
+ const fastLane = await stickyAssignmentAdmission.acquire(claims.relayHostId, (reason) => {
+ rejection = reason
+ })
+ if (!fastLane) {
+ operations.recordAssignmentAdmission?.('sticky-rejected')
+ logAdmissionRejection({
+ route: 'assign',
+ lane: 'sticky',
+ hinted: true,
+ relayHostId: claims.relayHostId,
+ reason: rejection
+ })
+ return rejectStickyAssignment(context)
+ }
+ let verified = false
+ try {
+ verified = (await operations.assignments.resolve(identity)) !== null
+ } catch (error) {
+ fastLane.release()
+ operations.recordAssignmentAdmission?.('sticky-rejected')
+ if (isRelayDatabaseTransientError(error)) {
+ logAssignmentRejection({
+ route: 'assign-verify',
+ lane: 'none',
+ hinted: true,
+ relayHostId: claims.relayHostId,
+ reason: operationError(error)
+ })
+ return rejectStickyAssignment(context)
+ }
+ throw error
+ }
+ if (verified) {
+ lane = 'sticky'
+ admission = fastLane
+ operations.recordAssignmentAdmission?.('sticky')
+ } else {
+ // An unverified hint joins the placement lane with today's semantics.
+ fastLane.release()
+ }
+ }
+ if (!admission) {
+ let rejection: AssignmentAdmissionRejection | undefined
+ admission = await publicAssignmentAdmission.acquire(claims.relayHostId, (reason) => {
+ rejection = reason
+ })
+ operations.recordAssignmentAdmission?.(admission ? 'placement' : 'placement-rejected')
+ if (!admission) {
+ logAdmissionRejection({
+ route: 'assign',
+ lane: 'placement',
+ hinted: Boolean(body.data.reconnect),
+ relayHostId: claims.relayHostId,
+ reason: rejection
+ })
+ return rejectPublicAssignment(context)
+ }
+ }
+ let assignment: RelayAssignment
+ try {
+ assignment = requestedRegion
+ ? await operations.assignments.assign(identity, requestedRegion, targetRegion)
+ : await operations.assignments.assign(identity)
+ } catch (error) {
+ if (isRelayAssignmentCapacityError(error) || isRelayDatabaseTransientError(error)) {
+ logAssignmentRejection({
+ route: 'assign',
+ lane,
+ hinted: Boolean(body.data.reconnect),
+ relayHostId: claims.relayHostId,
+ reason: operationError(error)
+ })
+ }
+ if (isRelayAssignmentCapacityError(error)) {
+ if (lane === 'placement') {
+ operations.recordRegionSelection?.({ targetRegion, fallback: false })
+ }
+ return context.json({ error: operationError(error) }, 503)
+ }
+ if (isRelayDatabaseTransientError(error)) {
+ return lane === 'sticky' ? rejectStickyAssignment(context) : rejectPublicAssignment(context)
+ }
+ throw error
+ } finally {
+ admission.release()
+ }
+ operations.recordRegionSelection?.({
+ targetRegion,
+ selectedRegion: assignment.region,
+ fallback: lane === 'placement' && assignment.region !== targetRegion
+ })
+ // Grant-side counterpart of the rejection log: reconnect grants are rare
+ // enough to log and make "which cell is this host on" answerable.
+ if (lane === 'sticky') {
+ console.warn(
+ `[orca-relay] assignment granted lane=sticky host=${relayHostLogDigest(claims.relayHostId)}` +
+ ` cell=${assignment.cellId}`
+ )
+ }
+ const lease = await new SignJWT({
+ purpose: 'cell-assignment',
+ cellId: assignment.cellId,
+ cellUrl: assignment.cellUrl,
+ assignmentEpoch: assignment.assignmentEpoch,
+ relayHostId: claims.relayHostId
+ })
+ .setProtectedHeader({ alg: 'HS256' })
+ .setIssuer(config.publicUrl)
+ .setAudience('orca-relay-cell')
+ .setSubject(claims.sub)
+ .setIssuedAt()
+ .setExpirationTime('5m')
+ .sign(config.assignmentSigningKey)
+ return context.json({
+ v: 1,
+ cellUrl: assignment.cellUrl,
+ assignmentEpoch: assignment.assignmentEpoch,
+ lease
+ })
+ })
+ app.post('/v1/resolve', async (context) => {
+ if (config.role === 'cell') return context.json({ error: 'director_only' }, 404)
+ if (!config.publicAssignmentsEnabled) return rejectPublicAssignment(context)
+ if (Number(context.req.header('content-length') ?? 0) > RELAY_PROTOCOL_LIMITS.maxHttpBodyBytes) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = ResolveRequestSchema.safeParse(await context.req.json().catch(() => null))
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ let rejection: AssignmentAdmissionRejection | undefined
+ const admission = await publicAssignmentAdmission.acquireReserved(
+ body.data.relayHostId,
+ (reason) => {
+ rejection = reason
+ }
+ )
+ if (!admission) {
+ logAdmissionRejection({
+ route: 'resolve',
+ lane: 'placement',
+ hinted: false,
+ relayHostId: body.data.relayHostId,
+ reason: rejection
+ })
+ return rejectPublicAssignment(context)
+ }
+ try {
+ const resolved = await operations.store.resolveResume(
+ body.data.relayHostId,
+ body.data.resumeToken
+ )
+ if (!resolved) return context.json({ error: 'invalid_credential' }, 401)
+ const identity = {
+ userId: resolved.userId,
+ relayHostId: body.data.relayHostId
+ }
+ // This also migrates credentials created by the staging-only combined service.
+ const assignment =
+ (await operations.assignments.resolve(identity)) ??
+ (await operations.assignments.assign(identity))
+ return context.json({
+ v: 1,
+ cellUrl: assignment.cellUrl,
+ assignmentEpoch: assignment.assignmentEpoch,
+ leaseExpiresAt: assignment.leaseExpiresAt
+ })
+ } catch (error) {
+ if (isRelayAssignmentCapacityError(error) || isRelayDatabaseTransientError(error)) {
+ logAssignmentRejection({
+ route: 'resolve',
+ lane: 'none',
+ hinted: false,
+ relayHostId: body.data.relayHostId,
+ reason: operationError(error)
+ })
+ }
+ if (isRelayAssignmentCapacityError(error)) {
+ return context.json({ error: operationError(error) }, 503)
+ }
+ if (isRelayDatabaseTransientError(error)) return rejectPublicAssignment(context)
+ throw error
+ } finally {
+ admission.release()
+ }
+ })
+ app.post('/v1/admin/drain', async (context) => {
+ const bearer = readBearer(context.req.header('authorization'))
+ if (!bearer || !(await verifyAdminToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ const body = z
+ .object({ v: z.literal(1), graceMs: z.number().int().nonnegative().max(60 * 60 * 1000) })
+ .strict()
+ .safeParse(await context.req.json().catch(() => null))
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ operations.drain(body.data.graceMs)
+ return context.json({ ok: true })
+ })
+ app.post('/v1/admin/host-drain', async (context) => {
+ if (config.role !== 'cell' || !operations.drainHost) {
+ return context.json({ error: 'cell_only' }, 404)
+ }
+ const bearer = readBearer(context.req.header('authorization'))
+ if (!bearer || !(await verifyRegionalRehomeToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = RegionalHostDrainSchema.safeParse(
+ await context.req.json().catch(() => null)
+ )
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ if (
+ body.data.sourceCellId !== config.cellId ||
+ !operations.cellIncarnation ||
+ body.data.sourceCellIncarnation !== operations.cellIncarnation
+ ) {
+ return context.json({ error: 'regional_rehome_source_generation_mismatch' }, 409)
+ }
+ try {
+ const trustProbe = isRegionalRehomeTrustProbe(body.data)
+ let sharedRuntimeIdentityRejected: true | undefined
+ if (trustProbe) {
+ const runtimeToken = await regionalRehomeIdentityToken(config.rehomeAudience!)
+ if (
+ !(await verifyRegionalRehomeRuntimeToken(runtimeToken)) ||
+ (await verifyRegionalRehomeToken(runtimeToken))
+ ) {
+ throw new Error('regional_rehome_shared_runtime_rejection_not_proven')
+ }
+ if (
+ !operations.regionalRehomeTrustProbeHostExists ||
+ operations.regionalRehomeTrustProbeHostExists(body.data)
+ ) {
+ throw new Error('regional_rehome_trust_probe_host_not_absent')
+ }
+ sharedRuntimeIdentityRejected = true
+ }
+ const outcome = operations.drainHost(body.data)
+ return context.json({
+ v: 1,
+ outcome,
+ ...(sharedRuntimeIdentityRejected ? { sharedRuntimeIdentityRejected } : {})
+ })
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 409)
+ }
+ })
+ app.post('/v1/admin/runtime-status', async (context) => {
+ const bearer = readBearer(context.req.header('authorization'))
+ if (!bearer || !(await verifyAdminToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = RuntimeStatusSchema.safeParse(await context.req.json().catch(() => null))
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ return context.json({
+ v: 1,
+ role: config.role,
+ cellId: config.cellId,
+ cellUrl: config.cellUrl,
+ region: config.region ?? RELAY_DEFAULT_REGION,
+ imageDigest: config.imageDigest ?? null,
+ draining: operations.isDraining?.() ?? false,
+ regionalRehomeProtocol:
+ config.rehomeAudience && config.rehomeDirectorServiceAccount ? 1 : 0,
+ connectionCapacity:
+ config.connectionHardCap === undefined
+ ? null
+ : {
+ hardCap: config.connectionHardCap,
+ controlRebindReserve: RELAY_ADMISSION_BUDGETS.reservedHostControls,
+ ordinaryConnectionLimit: relayCellAdmissionBounds(config.connectionHardCap)
+ .socketAdmissionCeiling,
+ unobservedBound: config.connectionUnobservedBound!,
+ normalAdmissionPause: cellPlacementCeiling(
+ config.connectionHardCap,
+ config.connectionUnobservedBound!
+ )
+ },
+ runtime: operations.runtimeCounts?.() ?? null
+ })
+ })
+ app.post('/v1/admin/cell-heartbeat', async (context) => {
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ const bearer = readBearer(context.req.header('authorization'))
+ if (!bearer || !(await verifyRuntimeToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = CellHeartbeatSchema.safeParse(await context.req.json().catch(() => null))
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ try {
+ await operations.assignments.recordCellHeartbeat(body.data)
+ return context.json({ ok: true })
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 409)
+ }
+ })
+ app.post('/v1/admin/cell-rehome-status', async (context) => {
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ const bearer = readBearer(context.req.header('authorization'))
+ if (!bearer || !(await verifyRuntimeToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = CellRegionalRehomeStatusSchema.safeParse(
+ await context.req.json().catch(() => null)
+ )
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ try {
+ await operations.assignments.recordCellRegionalRehomeStatus(body.data)
+ return context.json({ ok: true })
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 409)
+ }
+ })
+ app.post('/v1/admin/regional-rehome-control', async (context) => {
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ const bearer = readBearer(context.req.header('authorization'))
+ if (!bearer || !(await verifyAdminToken(bearer, context.req.path))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = RegionalRehomeControlSchema.safeParse(
+ await context.req.json().catch(() => null)
+ )
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ if (body.data.action === 'inspect') {
+ const control = await operations.assignments.inspectRegionalRehomeControl()
+ return context.json({ v: 1, control })
+ }
+ if (!(await verifyRegionalRehomeControlApplyToken(bearer))) {
+ return context.json({ error: 'insufficient_permission' }, 403)
+ }
+ try {
+ const control = await operations.assignments.applyRegionalRehomeControl(body.data)
+ return context.json({ v: 1, control })
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 409)
+ }
+ })
+ app.post('/v1/admin/regional-rehome-trust-probe', async (context) => {
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ const bearer = readBearer(context.req.header('authorization'))
+ if (!bearer || !(await verifyAdminToken(bearer, context.req.path))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = RegionalRehomeTrustProbeSchema.safeParse(
+ await context.req.json().catch(() => null)
+ )
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ if (!config.rehomeAudience || !config.rehomeDirectorServiceAccount) {
+ return context.json({ error: 'regional_rehome_trust_not_configured' }, 409)
+ }
+ try {
+ const source = await operations.assignments.cellDeploymentStatus(
+ body.data.sourceCellId
+ )
+ if (
+ source.region !== RELAY_DEFAULT_REGION ||
+ !source.runtime ||
+ source.runtime.cellIncarnation !== body.data.sourceCellIncarnation ||
+ !source.runtime.ready ||
+ !source.runtime.heartbeatFresh ||
+ source.runtime.regionalRehomeProtocol < 1
+ ) {
+ throw new Error('regional_rehome_trust_probe_source_unavailable')
+ }
+ const result = await probeRegionalRehomeTrust({
+ sourceCellUrl: source.cellUrl,
+ sourceCellId: body.data.sourceCellId,
+ sourceCellIncarnation: body.data.sourceCellIncarnation,
+ audience: config.rehomeAudience,
+ identityToken: regionalRehomeIdentityToken,
+ fetch: regionalRehomeFetch
+ })
+ return context.json(result)
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 409)
+ }
+ })
+ app.post('/v1/admin/evacuate', async (context) => {
+ const bearer = readBearer(context.req.header('authorization'))
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ if (!bearer || !(await verifyAdminToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = AdminAssignmentMoveSchema.safeParse(await context.req.json().catch(() => null))
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ try {
+ const migration = await operations.assignments.startEvacuation(
+ { userId: body.data.userId, relayHostId: body.data.relayHostId },
+ body.data.targetCellId
+ )
+ return context.json({ v: 1, migration })
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 409)
+ }
+ })
+ app.post('/v1/admin/migration-complete', async (context) => {
+ const bearer = readBearer(context.req.header('authorization'))
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ if (!bearer || !(await verifyAdminToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = AdminMigrationCompleteSchema.safeParse(await context.req.json().catch(() => null))
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ try {
+ await operations.assignments.completeEvacuation(
+ { userId: body.data.userId, relayHostId: body.data.relayHostId },
+ body.data.assignmentEpoch
+ )
+ return context.json({ ok: true })
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 409)
+ }
+ })
+ app.post('/v1/admin/migration-supersede-cell', async (context) => {
+ const bearer = readBearer(context.req.header('authorization'))
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ if (!bearer || !(await verifyAdminToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = AdminRegisteredCellMigrationSupersedeSchema.safeParse(
+ await context.req.json().catch(() => null)
+ )
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ try {
+ const superseded = await operations.assignments.supersedeRegisteredCellEvacuations(
+ body.data.sourceCellId,
+ body.data.currentTargetCellId,
+ body.data.replacementTargetCellId,
+ body.data.limit
+ )
+ return context.json({ v: 1, superseded })
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 409)
+ }
+ })
+ app.post('/v1/admin/rebalance-dormant', async (context) => {
+ const bearer = readBearer(context.req.header('authorization'))
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ if (!bearer || !(await verifyAdminToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = AdminAssignmentMoveSchema.safeParse(await context.req.json().catch(() => null))
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ try {
+ const assignment = await operations.assignments.rebalanceDormant(
+ { userId: body.data.userId, relayHostId: body.data.relayHostId },
+ body.data.targetCellId
+ )
+ return context.json({ v: 1, assignment })
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 409)
+ }
+ })
+ app.post('/v1/admin/admission-selector/apply', async (context) => {
+ const bearer = readBearer(context.req.header('authorization'))
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ if (!bearer || !(await verifyAdminToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = AdminAdmissionSelectorApplySchema.safeParse(
+ await context.req.json().catch(() => null)
+ )
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ try {
+ const result = await operations.assignments.applyCellAdmissionSelector(body.data)
+ return context.json({ v: 1, ...result })
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 409)
+ }
+ })
+ app.post('/v1/admin/admission-selector/apply-staging-asia-proof', async (context) => {
+ const bearer = readBearer(context.req.header('authorization'))
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ if (!bearer || !(await verifyAdminToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = AdminStagingAsiaProofAdmissionApplySchema.safeParse(
+ await context.req.json().catch(() => null)
+ )
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ try {
+ const current = await operations.assignments.inspectCellAdmissionSelector()
+ const result = await operations.assignments.applyCellAdmissionSelector({
+ attemptId: body.data.attemptId,
+ expectedGeneration: body.data.expectedGeneration,
+ membership: stagingAsiaProofMembership(current.selector.membership, body.data.state)
+ })
+ return context.json({ v: 1, ...result })
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 409)
+ }
+ })
+ app.post('/v1/admin/admission-selector/status', async (context) => {
+ const bearer = readBearer(context.req.header('authorization'))
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ if (!bearer || !(await verifyAdminToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = AdminAdmissionSelectorStatusSchema.safeParse(
+ await context.req.json().catch(() => null)
+ )
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ try {
+ const result = await operations.assignments.inspectCellAdmissionSelector(
+ body.data.attemptId
+ )
+ return context.json({ v: 1, ...result })
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 409)
+ }
+ })
+ app.post('/v1/admin/admission-selector/add-migration-cells', async (context) => {
+ const bearer = readBearer(context.req.header('authorization'))
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ if (!bearer || !(await verifyAdminToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = AdminAdmissionSelectorAddMigrationCellsSchema.safeParse(
+ await context.req.json().catch(() => null)
+ )
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ try {
+ const result = await operations.assignments.addMigrationCells({
+ attemptId: body.data.attemptId,
+ expectedGeneration: body.data.expectedGeneration,
+ cells: body.data.cells.map((cell) => ({
+ id: cell.cellId,
+ url: cell.cellUrl,
+ capacityRequests: cell.capacityRequests,
+ region: cell.region,
+ connectionHardCap: cell.connectionHardCap,
+ connectionUnobservedBound: cell.connectionUnobservedBound
+ }))
+ })
+ return context.json({ v: 1, ...result })
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 409)
+ }
+ })
+ app.post('/v1/admin/cell-state', async (context) => {
+ const bearer = readBearer(context.req.header('authorization'))
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ if (!bearer || !(await verifyAdminToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = AdminCellStateSchema.safeParse(await context.req.json().catch(() => null))
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ try {
+ await operations.assignments.setCellAdmissionState(
+ body.data.cellId,
+ 'state' in body.data
+ ? body.data.state
+ : body.data.enabled
+ ? 'general'
+ : 'existing-only'
+ )
+ return context.json({ ok: true })
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 409)
+ }
+ })
+ app.post('/v1/admin/cell-fence-adopt-legacy', async (context) => {
+ const bearer = readBearer(context.req.header('authorization'))
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ if (!bearer || !(await verifyAdminToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = AdminCellFenceLegacyAdoptionSchema.safeParse(
+ await context.req.json().catch(() => null)
+ )
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ try {
+ const expiresAt = await operations.assignments.adoptLegacyCellFence(
+ body.data.cellId,
+ body.data.cellIncarnation
+ )
+ return context.json({ v: 1, cellId: body.data.cellId, expiresAt })
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 409)
+ }
+ })
+ app.post('/v1/admin/cell-fence-commit-legacy-adoption', async (context) => {
+ const bearer = readBearer(context.req.header('authorization'))
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ if (!bearer || !(await verifyAdminToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = AdminCellFenceLegacyAdoptionCommitSchema.safeParse(
+ await context.req.json().catch(() => null)
+ )
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ try {
+ await operations.assignments.commitLegacyCellFenceAdoption(
+ body.data.cellId,
+ body.data.cellIncarnation
+ )
+ return context.json({ v: 1, cellId: body.data.cellId, committed: true })
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 409)
+ }
+ })
+ app.post('/v1/admin/cell-fence-attest', async (context) => {
+ const bearer = readBearer(context.req.header('authorization'))
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ if (!bearer || !(await verifyAdminToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = AdminCellFenceAttestSchema.safeParse(
+ await context.req.json().catch(() => null)
+ )
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ try {
+ const evidence = cellFenceAttemptEvidence(body.data)
+ const result = await operations.assignments.attestCellFenceAttempt(
+ evidence,
+ body.data.gceOperation
+ )
+ return context.json({
+ v: 1,
+ cellId: body.data.cellId,
+ expiresAt: result.expiresAt,
+ attempt: result.attempt
+ })
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 409)
+ }
+ })
+ app.post('/v1/admin/cell-fence-attempt-prepare', async (context) => {
+ const bearer = readBearer(context.req.header('authorization'))
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ if (!bearer || !(await verifyAdminToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = AdminCellFenceAttemptPrepareSchema.safeParse(
+ await context.req.json().catch(() => null)
+ )
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ try {
+ const evidence = cellFenceAttemptEvidence(body.data)
+ const attempt = await operations.assignments.prepareCellFenceAttempt(evidence)
+ return context.json({ v: 1, attempt })
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 409)
+ }
+ })
+ app.post('/v1/admin/cell-fence-attempt-start', async (context) => {
+ const bearer = readBearer(context.req.header('authorization'))
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ if (!bearer || !(await verifyAdminToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = AdminCellFenceAttemptUpdateSchema.safeParse(
+ await context.req.json().catch(() => null)
+ )
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ try {
+ const evidence = cellFenceAttemptEvidence(body.data)
+ const result = await operations.assignments.startCellFenceApply(
+ evidence,
+ body.data.invocationId,
+ body.data.invocationRequestReason
+ )
+ return context.json({ v: 1, ...result })
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 409)
+ }
+ })
+ app.post('/v1/admin/cell-fence-attempt-plan', async (context) => {
+ const bearer = readBearer(context.req.header('authorization'))
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ if (!bearer || !(await verifyAdminToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = AdminCellFencePlanSchema.safeParse(
+ await context.req.json().catch(() => null)
+ )
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ try {
+ const evidence = cellFenceAttemptEvidence(body.data)
+ const attempt = await operations.assignments.bindCellFencePlanGeneration(
+ evidence,
+ body.data.planObjectGeneration
+ )
+ return context.json({ v: 1, attempt })
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 409)
+ }
+ })
+ app.post('/v1/admin/cell-fence-attempt-operation', async (context) => {
+ const bearer = readBearer(context.req.header('authorization'))
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ if (!bearer || !(await verifyAdminToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = AdminCellFenceOperationSchema.safeParse(
+ await context.req.json().catch(() => null)
+ )
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ try {
+ const evidence = cellFenceAttemptEvidence(body.data)
+ const result = await operations.assignments.recordCellFenceOperation(
+ evidence,
+ body.data.invocationId,
+ body.data.invocationRequestReason,
+ body.data.gceOperation
+ )
+ return context.json({ v: 1, ...result })
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 409)
+ }
+ })
+ app.post('/v1/admin/cell-fence-attempt-status', async (context) => {
+ const bearer = readBearer(context.req.header('authorization'))
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ if (!bearer || !(await verifyAdminToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = AdminCellFenceAttemptStatusSchema.safeParse(
+ await context.req.json().catch(() => null)
+ )
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ try {
+ const attempt = await operations.assignments.cellFenceAttempt(body.data.cellId)
+ return context.json({ v: 1, attempt })
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 409)
+ }
+ })
+ app.post('/v1/admin/cell-fence-attempt-abort', async (context) => {
+ const bearer = readBearer(context.req.header('authorization'))
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ if (!bearer || !(await verifyAdminToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = AdminCellFenceAttemptAbortSchema.safeParse(
+ await context.req.json().catch(() => null)
+ )
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ try {
+ const evidence = cellFenceAttemptEvidence(body.data)
+ const attempt = await operations.assignments.abortCellFenceAttempt(evidence)
+ return context.json({ v: 1, attempt })
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 409)
+ }
+ })
+ app.post('/v1/admin/drain-attempt-prepare', async (context) => {
+ const bearer = readBearer(context.req.header('authorization'))
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ if (!bearer || !(await verifyAdminToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = AdminDrainAttemptPrepareSchema.safeParse(
+ await context.req.json().catch(() => null)
+ )
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ try {
+ const result = await operations.assignments.prepareCellDrainAttempt({
+ attemptId: body.data.attemptId,
+ cellId: body.data.cellId,
+ cellIncarnation: body.data.cellIncarnation,
+ traceValue: body.data.traceValue,
+ plannedGraceMs: body.data.graceMs
+ })
+ return context.json({ v: 1, ...result })
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 409)
+ }
+ })
+ app.post('/v1/admin/drain-attempt-send', async (context) => {
+ const bearer = readBearer(context.req.header('authorization'))
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ if (!bearer || !(await verifyAdminToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = AdminDrainAttemptMutationSchema.safeParse(
+ await context.req.json().catch(() => null)
+ )
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ try {
+ const attempt = await operations.assignments.beginCellDrainSend(body.data)
+ return context.json({ v: 1, attempt })
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 409)
+ }
+ })
+ app.post('/v1/admin/drain-attempt-receipt', async (context) => {
+ const bearer = readBearer(context.req.header('authorization'))
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ if (!bearer || !(await verifyAdminToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = AdminDrainAttemptReceiptSchema.safeParse(
+ await context.req.json().catch(() => null)
+ )
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ try {
+ const attempt = await operations.assignments.recordCellDrainApplicationReceipt(
+ body.data
+ )
+ return context.json({ v: 1, attempt })
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 409)
+ }
+ })
+ app.post('/v1/admin/drain-attempt-recover-forward', async (context) => {
+ const bearer = readBearer(context.req.header('authorization'))
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ if (!bearer || !(await verifyAdminToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = AdminDrainAttemptRecoverSchema.safeParse(
+ await context.req.json().catch(() => null)
+ )
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ try {
+ const result = await operations.assignments.prepareCellDrainRecovery(body.data)
+ return context.json({ v: 1, ...result })
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 409)
+ }
+ })
+ app.post('/v1/admin/cell-config', async (context) => {
+ const bearer = readBearer(context.req.header('authorization'))
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ if (!bearer || !(await verifyAdminToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = AdminCellConfigSchema.safeParse(await context.req.json().catch(() => null))
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ try {
+ await operations.assignments.configureCell(
+ {
+ id: body.data.cellId,
+ url: body.data.cellUrl,
+ capacityRequests: body.data.capacityRequests,
+ connectionHardCap: body.data.connectionHardCap,
+ connectionUnobservedBound: body.data.connectionUnobservedBound
+ },
+ 'state' in body.data ? body.data.state : body.data.enabled
+ )
+ return context.json({ ok: true })
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 409)
+ }
+ })
+ app.post('/v1/admin/evacuate-cell', async (context) => {
+ const bearer = readBearer(context.req.header('authorization'))
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ if (!bearer || !(await verifyAdminToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = AdminCellEvacuateSchema.safeParse(await context.req.json().catch(() => null))
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ try {
+ const started = await operations.assignments.startActiveCellEvacuations(
+ body.data.sourceCellId,
+ body.data.targetCellId,
+ body.data.limit
+ )
+ return context.json({ v: 1, started })
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 409)
+ }
+ })
+ app.post('/v1/admin/evacuation-capacity', async (context) => {
+ const bearer = readBearer(context.req.header('authorization'))
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ if (!bearer || !(await verifyAdminToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = AdminCellEvacuationCapacitySchema.safeParse(
+ await context.req.json().catch(() => null)
+ )
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ try {
+ const capacity = await operations.assignments.cellEvacuationCapacity(
+ body.data.sourceCellId,
+ body.data.targetCellId
+ )
+ return context.json({ v: 1, ...capacity })
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 409)
+ }
+ })
+ app.post('/v1/admin/evacuation-status', async (context) => {
+ const bearer = readBearer(context.req.header('authorization'))
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ if (!bearer || !(await verifyAdminToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = AdminCellEvacuationStatusSchema.safeParse(
+ await context.req.json().catch(() => null)
+ )
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ if (body.data.completeReady && (await verifyReadOnlyAdminToken(bearer))) {
+ return context.json({ error: 'insufficient_permission' }, 403)
+ }
+ const status = await operations.assignments.cellEvacuationStatus(
+ body.data.sourceCellId,
+ body.data.targetCellId,
+ body.data.completeReady
+ )
+ return context.json({ v: 1, ...status })
+ })
+ app.post('/v1/admin/cell-status', async (context) => {
+ const bearer = readBearer(context.req.header('authorization'))
+ if (config.role !== 'director') return context.json({ error: 'director_only' }, 404)
+ if (!bearer || !(await verifyAdminToken(bearer))) {
+ return context.json({ error: 'invalid_token' }, 401)
+ }
+ if (requestTooLarge(context.req.header('content-length'))) {
+ return context.json({ error: 'request_too_large' }, 413)
+ }
+ const body = AdminCellStatusSchema.safeParse(await context.req.json().catch(() => null))
+ if (!body.success) return context.json({ error: 'invalid_request' }, 400)
+ try {
+ const status = await operations.assignments.cellDeploymentStatus(body.data.cellId)
+ return context.json({ v: 1, status })
+ } catch (error) {
+ return context.json({ error: operationError(error) }, 404)
+ }
+ })
+ return app
+}
+
+const AdminAssignmentMoveSchema = z
+ .object({
+ v: z.literal(1),
+ userId: z.string().min(1).max(256),
+ relayHostId: z.string().regex(/^[A-Za-z0-9_-]{16}$/),
+ targetCellId: z.string().min(1).max(128)
+ })
+ .strict()
+
+const RuntimeStatusSchema = z.object({ v: z.literal(1) }).strict()
+
+const RegionalRehomeSafetySchema = z
+ .object({
+ observedAt: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
+ sqlFailures: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
+ reconnects: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
+ controlActivityRecoveryFailures: z
+ .number()
+ .int()
+ .nonnegative()
+ .max(Number.MAX_SAFE_INTEGER),
+ databasePoolWaiting: z.number().int().nonnegative().max(100),
+ databasePoolWaitersMax: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
+ databasePoolWaitMsMax: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
+ // Cells spread the full pool-pressure counts into the payload; rejecting
+ // the extra gauges 400'd every rehome-status heartbeat since 2026-08-15,
+ // so no cell ever recorded rehome protocol 1.
+ databasePoolTotal: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER).optional(),
+ databasePoolIdle: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER).optional(),
+ databasePoolOldestWaitMs: z
+ .number()
+ .int()
+ .nonnegative()
+ .max(Number.MAX_SAFE_INTEGER)
+ .optional()
+ })
+ .strict()
+
+const CellHeartbeatSchema = z
+ .object({
+ v: z.literal(1),
+ cellId: z.string().min(1).max(128),
+ cellUrl: z.string().url().max(2_048).refine(isCanonicalRelayOrigin),
+ region: RelayRegionSchema.default(RELAY_DEFAULT_REGION),
+ cellIncarnation: z.string().uuid(),
+ startedAt: z.number().int().positive().max(Number.MAX_SAFE_INTEGER),
+ ready: z.boolean(),
+ observedRequests: z.number().int().nonnegative().max(RELAY_MAX_CELL_CAPACITY_REQUESTS),
+ totalConnections: z
+ .number()
+ .int()
+ .nonnegative()
+ .max(RELAY_MAX_CELL_CAPACITY_REQUESTS)
+ .optional(),
+ inFlightConnections: z
+ .number()
+ .int()
+ .nonnegative()
+ .max(RELAY_MAX_CELL_CAPACITY_REQUESTS)
+ .optional(),
+ reservedConnectionUnits: z
+ .number()
+ .int()
+ .nonnegative()
+ .max(RELAY_MAX_CELL_CAPACITY_REQUESTS)
+ .optional(),
+ enforcedConnectionUnits: z
+ .number()
+ .int()
+ .nonnegative()
+ .max(RELAY_MAX_CELL_CAPACITY_REQUESTS)
+ .optional(),
+ connectionInclusionWatermark: z
+ .number()
+ .int()
+ .nonnegative()
+ .max(Number.MAX_SAFE_INTEGER)
+ .optional(),
+ connectionHardCap: RelayCellConnectionHardCapSchema.optional(),
+ connectionUnobservedBound: z
+ .number()
+ .int()
+ .nonnegative()
+ .max(RELAY_MAX_CELL_CONNECTION_UNOBSERVED_BOUND)
+ .optional()
+ })
+ .strict()
+ .superRefine((value, context) => {
+ const values = [
+ value.totalConnections,
+ value.inFlightConnections,
+ value.reservedConnectionUnits,
+ value.enforcedConnectionUnits,
+ value.connectionHardCap,
+ value.connectionUnobservedBound
+ ]
+ if (
+ values.some((candidate) => candidate !== undefined) &&
+ values.some((candidate) => candidate === undefined)
+ ) {
+ context.addIssue({
+ code: 'custom',
+ message: 'connection telemetry must be complete'
+ })
+ }
+ if (
+ value.enforcedConnectionUnits !== undefined &&
+ value.totalConnections !== undefined &&
+ value.inFlightConnections !== undefined &&
+ value.reservedConnectionUnits !== undefined &&
+ value.enforcedConnectionUnits !==
+ value.totalConnections +
+ value.inFlightConnections +
+ value.reservedConnectionUnits
+ ) {
+ context.addIssue({
+ code: 'custom',
+ message: 'connection telemetry sum does not match'
+ })
+ }
+ if (
+ value.connectionHardCap !== undefined &&
+ value.connectionUnobservedBound! >
+ relayCellAdmissionBounds(value.connectionHardCap).maxUnobservedBound
+ ) {
+ context.addIssue({
+ code: 'custom',
+ message: 'connection unobserved bound must leave ordinary admission capacity'
+ })
+ }
+ })
+
+const CellRegionalRehomeStatusSchema = z
+ .object({
+ v: z.literal(1),
+ cellId: z.string().min(1).max(128),
+ cellIncarnation: z.string().uuid(),
+ regionalRehomeProtocol: z.number().int().min(0).max(1),
+ safety: RegionalRehomeSafetySchema
+ })
+ .strict()
+
+const RegionalRehomeControlSchema = z.discriminatedUnion('action', [
+ z.object({ v: z.literal(1), action: z.literal('inspect') }).strict(),
+ z.object({
+ v: z.literal(1),
+ action: z.literal('apply'),
+ expectedGeneration: z.number().int().nonnegative(),
+ enabled: z.boolean(),
+ notBefore: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
+ ratePerMinute: z.number().int().min(1).max(120),
+ preferenceMaxAgeMs: z
+ .number()
+ .int()
+ .min(60_000)
+ .max(30 * 24 * 60 * 60_000),
+ drainGraceMs: z.number().int().min(60_000).max(60 * 60_000),
+ confirmation: z.enum([
+ 'ENABLE_REGIONAL_REHOMING',
+ 'DISABLE_REGIONAL_REHOMING'
+ ])
+ }).strict()
+]).superRefine((value, context) => {
+ if (value.action !== 'apply') return
+ const expected = value.enabled
+ ? 'ENABLE_REGIONAL_REHOMING'
+ : 'DISABLE_REGIONAL_REHOMING'
+ if (value.confirmation !== expected) {
+ context.addIssue({ code: 'custom', message: 'confirmation does not match state' })
+ }
+})
+
+const RegionalRehomeTrustProbeSchema = z
+ .object({
+ v: z.literal(1),
+ sourceCellId: z.string().min(1).max(128),
+ sourceCellIncarnation: z.string().uuid()
+ })
+ .strict()
+
+const AdminMigrationCompleteSchema = z
+ .object({
+ v: z.literal(1),
+ userId: z.string().min(1).max(256),
+ relayHostId: z.string().regex(/^[A-Za-z0-9_-]{16}$/),
+ assignmentEpoch: z.number().int().positive().max(Number.MAX_SAFE_INTEGER)
+ })
+ .strict()
+
+const AdminRegisteredCellMigrationSupersedeSchema = z
+ .object({
+ v: z.literal(1),
+ sourceCellId: z.string().min(1).max(128),
+ currentTargetCellId: z.string().min(1).max(128),
+ replacementTargetCellId: z.string().min(1).max(128),
+ limit: z.number().int().min(1).max(100),
+ confirmation: z.literal('SUPERSEDE_REGISTERED_CELL_MIGRATIONS')
+ })
+ .strict()
+ .refine(
+ (value) =>
+ new Set([
+ value.sourceCellId,
+ value.currentTargetCellId,
+ value.replacementTargetCellId
+ ]).size === 3
+ )
+
+const CellIdSchema = z.string().min(1).max(128)
+const CellAdmissionStateSchema = z.enum(CELL_ADMISSION_STATES)
+const AdmissionSelectorAttemptIdSchema = z.string().regex(/^[A-Za-z0-9_-]{8,128}$/)
+const AdmissionSelectorMembershipSchema = z
+ .object({
+ existingOnly: z.array(CellIdSchema).max(256),
+ migrationOnly: z.array(CellIdSchema).max(256),
+ general: z.array(CellIdSchema).max(256)
+ })
+ .strict()
+
+const AdminAdmissionSelectorApplySchema = z
+ .object({
+ v: z.literal(1),
+ attemptId: AdmissionSelectorAttemptIdSchema,
+ expectedGeneration: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
+ expectedMembershipSha256: z.string().regex(/^[a-f0-9]{64}$/).optional(),
+ membership: AdmissionSelectorMembershipSchema
+ })
+ .strict()
+ .refine(
+ (value) => value.expectedGeneration > 0 || value.expectedMembershipSha256 !== undefined
+ )
+
+const AdminStagingAsiaProofAdmissionApplySchema = z
+ .object({
+ v: z.literal(1),
+ attemptId: AdmissionSelectorAttemptIdSchema,
+ expectedGeneration: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
+ state: z.enum(['general', 'migration-only'])
+ })
+ .strict()
+
+const AdminAdmissionSelectorStatusSchema = z
+ .object({ v: z.literal(1), attemptId: AdmissionSelectorAttemptIdSchema.optional() })
+ .strict()
+
+const AdminAdmissionSelectorAddMigrationCellsSchema = z
+ .object({
+ v: z.literal(1),
+ attemptId: AdmissionSelectorAttemptIdSchema,
+ expectedGeneration: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
+ cells: z
+ .array(
+ z
+ .object({
+ cellId: CellIdSchema,
+ cellUrl: z.string().url().max(2_048).refine(isCanonicalRelayOrigin),
+ capacityRequests: z
+ .number()
+ .int()
+ .positive()
+ .max(RELAY_MAX_CELL_CAPACITY_REQUESTS),
+ region: RelayRegionSchema.default(RELAY_DEFAULT_REGION),
+ connectionHardCap: RelayCellConnectionHardCapSchema,
+ connectionUnobservedBound: z
+ .number()
+ .int()
+ .nonnegative()
+ .max(RELAY_MAX_CELL_CONNECTION_UNOBSERVED_BOUND)
+ })
+ .strict()
+ .superRefine((value, context) => {
+ if (
+ value.connectionUnobservedBound >
+ relayCellAdmissionBounds(value.connectionHardCap).maxUnobservedBound
+ ) {
+ context.addIssue({
+ code: 'custom',
+ path: ['connectionUnobservedBound'],
+ message: 'connection unobserved bound must leave ordinary admission capacity'
+ })
+ }
+ })
+ )
+ .min(1)
+ .max(128)
+ })
+ .strict()
+ .refine(
+ ({ cells }) =>
+ new Set(cells.map(({ cellId }) => cellId)).size === cells.length &&
+ new Set(cells.map(({ cellUrl }) => cellUrl)).size === cells.length
+ )
+
+const AdminCellStateSchema = z.union([
+ z.object({ v: z.literal(1), cellId: CellIdSchema, enabled: z.boolean() }).strict(),
+ z.object({ v: z.literal(1), cellId: CellIdSchema, state: CellAdmissionStateSchema }).strict()
+])
+
+const CellConfigShape = {
+ v: z.literal(1),
+ cellId: CellIdSchema,
+ cellUrl: z.string().url().max(2_048).refine(isCanonicalRelayOrigin),
+ capacityRequests: z.number().int().positive().max(RELAY_MAX_CELL_CAPACITY_REQUESTS),
+ connectionHardCap: RelayCellConnectionHardCapSchema.optional(),
+ connectionUnobservedBound: z
+ .number()
+ .int()
+ .nonnegative()
+ .max(RELAY_MAX_CELL_CONNECTION_UNOBSERVED_BOUND)
+ .optional()
+} as const
+
+const AdminCellConfigSchema = z
+ .union([
+ z
+ .object({
+ ...CellConfigShape,
+ enabled: z.boolean()
+ })
+ .strict(),
+ z
+ .object({
+ ...CellConfigShape,
+ state: CellAdmissionStateSchema
+ })
+ .strict()
+ ])
+ .refine(
+ (value) =>
+ (value.connectionHardCap === undefined) ===
+ (value.connectionUnobservedBound === undefined),
+ { message: 'connection hard cap and unobserved bound must be configured together' }
+ )
+ .refine(
+ (value) =>
+ value.connectionHardCap === undefined ||
+ value.connectionUnobservedBound! <=
+ relayCellAdmissionBounds(value.connectionHardCap).maxUnobservedBound,
+ { message: 'connection unobserved bound must leave ordinary admission capacity' }
+ )
+
+const CellPairShape = {
+ v: z.literal(1),
+ sourceCellId: z.string().min(1).max(128),
+ targetCellId: z.string().min(1).max(128)
+} as const
+
+const AdminCellEvacuateSchema = z
+ .object({ ...CellPairShape, limit: z.number().int().min(1).max(100) })
+ .strict()
+ .refine((value) => value.sourceCellId !== value.targetCellId)
+
+const AdminCellEvacuationCapacitySchema = z
+ .object(CellPairShape)
+ .strict()
+ .refine((value) => value.sourceCellId !== value.targetCellId)
+
+const AdminCellEvacuationStatusSchema = z
+ .object({ ...CellPairShape, completeReady: z.boolean().default(false) })
+ .strict()
+ .refine((value) => value.sourceCellId !== value.targetCellId)
+
+const TerraformStateLineageSchema = z
+ .string()
+ .regex(/^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/i)
+
+const CellFenceAttemptBaseShape = {
+ attemptId: z.string().uuid(),
+ environment: z.enum(['staging', 'production']),
+ cellId: z.string().min(1).max(128),
+ cellIncarnation: z.string().uuid(),
+ migName: z.string().min(1).max(128),
+ instanceGroup: z.string().url().max(2048),
+ generationIdentity: z.string().url().max(2048),
+ fenceCommit: z.string().regex(/^[a-f0-9]{40}$/),
+ planSha256: z.string().regex(/^[a-f0-9]{64}$/),
+ planObjectName: z
+ .string()
+ .regex(/^terraform\/state\/relay-fence-plans\/(?:staging|production)\/[0-9a-f-]{36}\.tfplan$/),
+ varFileSha256: z.string().regex(/^[a-f0-9]{64}$/),
+ terraformStateLineage: TerraformStateLineageSchema,
+ terraformStateSerial: z.number().int().nonnegative().safe(),
+ terraformStateObjectGeneration: z.string().regex(/^[1-9][0-9]{0,30}$/),
+ terraformStateObjectSha256: z.string().regex(/^[a-f0-9]{64}$/),
+ requestReason: z
+ .string()
+ .regex(/^orca-relay-fence\/[0-9a-f]{8}-[0-9a-f-]{27}$/)
+} as const
+const CellFenceAttemptEvidenceShape = {
+ ...CellFenceAttemptBaseShape,
+ planObjectGeneration: z.string().regex(/^[1-9][0-9]{0,30}$/)
+} as const
+
+const AdminCellFenceAttemptPrepareSchema = z
+ .object({
+ v: z.literal(1),
+ ...CellFenceAttemptBaseShape,
+ confirmation: z.literal('PREPARE_TERRAFORM_CELL_FENCE')
+ })
+ .strict()
+
+const AdminCellFencePlanSchema = z
+ .object({
+ v: z.literal(1),
+ ...CellFenceAttemptEvidenceShape,
+ confirmation: z.literal('BIND_TERRAFORM_CELL_FENCE_PLAN')
+ })
+ .strict()
+
+const AdminCellFenceAttemptUpdateSchema = z
+ .object({
+ v: z.literal(1),
+ ...CellFenceAttemptEvidenceShape,
+ invocationId: z.string().uuid(),
+ invocationRequestReason: z.string().min(1).max(256),
+ confirmation: z.literal('START_TERRAFORM_CELL_FENCE')
+ })
+ .strict()
+
+const AdminCellFenceOperationSchema = z
+ .object({
+ v: z.literal(1),
+ ...CellFenceAttemptEvidenceShape,
+ invocationId: z.string().uuid(),
+ invocationRequestReason: z.string().min(1).max(256),
+ gceOperation: z.string().min(1).max(256),
+ confirmation: z.literal('RECORD_TERRAFORM_CELL_FENCE_OPERATION')
+ })
+ .strict()
+
+const AdminCellFenceAttemptStatusSchema = z
+ .object({ v: z.literal(1), cellId: z.string().min(1).max(128) })
+ .strict()
+
+const AdminCellFenceLegacyAdoptionSchema = z
+ .object({
+ v: z.literal(1),
+ cellId: z.string().min(1).max(128),
+ cellIncarnation: z.string().uuid(),
+ confirmation: z.literal('ADOPT_LEGACY_TERRAFORM_CELL_FENCE')
+ })
+ .strict()
+
+const AdminCellFenceLegacyAdoptionCommitSchema = z
+ .object({
+ v: z.literal(1),
+ cellId: z.string().min(1).max(128),
+ cellIncarnation: z.string().uuid(),
+ confirmation: z.literal('COMMIT_LEGACY_TERRAFORM_CELL_FENCE_ADOPTION')
+ })
+ .strict()
+
+const AdminCellFenceAttemptAbortSchema = z
+ .object({
+ v: z.literal(1),
+ ...CellFenceAttemptBaseShape,
+ confirmation: z.literal('ABORT_UNSTARTED_TERRAFORM_CELL_FENCE')
+ })
+ .strict()
+
+const AdminCellFenceAttestSchema = z
+ .object({
+ v: z.literal(1),
+ ...CellFenceAttemptEvidenceShape,
+ gceOperation: z.string().min(1).max(256),
+ confirmation: z.literal('ATTEST_TERRAFORM_FENCED_CELL')
+ })
+ .strict()
+
+const AdminDrainAttemptPrepareSchema = z
+ .object({
+ v: z.literal(1),
+ attemptId: z.string().uuid(),
+ cellId: z.string().min(1).max(128),
+ cellIncarnation: z.string().uuid(),
+ traceValue: z.string().uuid(),
+ graceMs: z.literal(120_000),
+ confirmation: z.literal('PREPARE_LEGACY_DRAIN')
+ })
+ .strict()
+
+const AdminDrainAttemptMutationSchema = z
+ .object({
+ v: z.literal(1),
+ attemptId: z.string().uuid(),
+ cellId: z.string().min(1).max(128),
+ cellIncarnation: z.string().uuid()
+ })
+ .strict()
+
+const AdminDrainAttemptReceiptSchema = AdminDrainAttemptMutationSchema.extend({
+ traceValue: z.string().uuid(),
+ backendStatus: z.number().int().min(200).max(299),
+ backendInstance: z.string().min(1).max(256).optional()
+}).strict()
+
+const AdminDrainAttemptRecoverSchema = z
+ .object({
+ v: z.literal(1),
+ cellId: z.string().min(1).max(128),
+ cellIncarnation: z.string().uuid(),
+ confirmation: z.literal('RECOVER_LEGACY_DRAIN')
+ })
+ .strict()
+
+const RegionalHostDrainSchema = z
+ .object({
+ v: z.literal(1),
+ attemptId: z.string().uuid(),
+ userId: z.string().min(1).max(256),
+ relayHostId: z.string().regex(/^[A-Za-z0-9_-]{16}$/),
+ sourceCellId: z.string().min(1).max(128),
+ sourceCellIncarnation: z.string().uuid(),
+ sourceAssignmentEpoch: z.number().int().positive(),
+ graceMs: z.number().int().nonnegative().max(60 * 60 * 1000)
+ })
+ .strict()
+
+const AdminCellStatusSchema = z
+ .object({ v: z.literal(1), cellId: z.string().min(1).max(128) })
+ .strict()
+
+function cellFenceAttemptEvidence(
+ value: CellFenceAttemptEvidence
+): CellFenceAttemptEvidence {
+ return {
+ attemptId: value.attemptId,
+ environment: value.environment,
+ cellId: value.cellId,
+ cellIncarnation: value.cellIncarnation,
+ migName: value.migName,
+ instanceGroup: value.instanceGroup,
+ generationIdentity: value.generationIdentity,
+ fenceCommit: value.fenceCommit,
+ planSha256: value.planSha256,
+ planObjectName: value.planObjectName,
+ planObjectGeneration: value.planObjectGeneration,
+ varFileSha256: value.varFileSha256,
+ terraformStateLineage: value.terraformStateLineage,
+ terraformStateSerial: value.terraformStateSerial,
+ terraformStateObjectGeneration: value.terraformStateObjectGeneration,
+ terraformStateObjectSha256: value.terraformStateObjectSha256,
+ requestReason: value.requestReason
+ }
+}
+
+function operationError(error: unknown): string {
+ return error instanceof Error ? error.message : 'operation_failed'
+}
+
+export { relayHostLogDigest }
+
+// `suppressed` is present only on a window-closing line, and counts the rejections
+// that line stands for beyond the one already logged when the window opened.
+function logAssignmentRejection(input: {
+ route: 'assign' | 'assign-verify' | 'resolve'
+ lane: 'sticky' | 'placement' | 'none'
+ hinted: boolean
+ relayHostId: string
+ reason: string
+ suppressed?: number
+}): void {
+ console.warn(
+ `[orca-relay] assignment rejected route=${input.route} lane=${input.lane}` +
+ ` hinted=${input.hinted} reason=${input.reason}` +
+ ` host=${relayHostLogDigest(input.relayHostId)}` +
+ (input.suppressed === undefined ? '' : ` suppressed=${input.suppressed}`)
+ )
+}
+
+function isRelayAssignmentCapacityError(error: unknown): boolean {
+ return (
+ error instanceof Error &&
+ ['relay_capacity_exhausted', 'relay_connection_headroom_exhausted'].includes(
+ error.message
+ )
+ )
+}
+
+function isCanonicalRelayOrigin(value: string): boolean {
+ const url = new URL(value)
+ const loopback = ['127.0.0.1', 'localhost', '::1', '[::1]'].includes(url.hostname)
+ return (
+ url.origin === value &&
+ url.pathname === '/' &&
+ (url.protocol === 'https:' || (loopback && url.protocol === 'http:'))
+ )
+}
+
+function requestTooLarge(contentLength: string | undefined): boolean {
+ return Number(contentLength ?? 0) > RELAY_PROTOCOL_LIMITS.maxHttpBodyBytes
+}
diff --git a/cloud/apps/relay/src/assignment-cleanup-steps.test.ts b/cloud/apps/relay/src/assignment-cleanup-steps.test.ts
new file mode 100644
index 00000000000..02c21f76ada
--- /dev/null
+++ b/cloud/apps/relay/src/assignment-cleanup-steps.test.ts
@@ -0,0 +1,67 @@
+import { describe, expect, it, vi } from 'vitest'
+import {
+ assignmentCleanupSteps,
+ runAssignmentCleanup,
+ type AssignmentCleanupStore
+} from './assignment-cleanup-steps.js'
+
+function stubStore(overrides: Partial = {}) {
+ const calls: string[] = []
+ const method = (name: string) =>
+ vi.fn(async () => {
+ calls.push(name)
+ })
+ const store: AssignmentCleanupStore = {
+ refreshRegionalRehomeLeases: method('refreshRegionalRehomeLeases'),
+ completeReadyEvacuations: method('completeReadyEvacuations'),
+ completeReadyRegionalRehomes: method('completeReadyRegionalRehomes'),
+ abortExpiredEvacuations: method('abortExpiredEvacuations'),
+ abortExpiredRegionalRehomes: method('abortExpiredRegionalRehomes'),
+ reapRegionalRehomeAttempts: method('reapRegionalRehomeAttempts'),
+ releaseExpiredActivityLeases: method('releaseExpiredActivityLeases'),
+ releaseExpiredActivity: method('releaseExpiredActivity'),
+ releaseExpiredRegionPreferences: method('releaseExpiredRegionPreferences'),
+ evacuateDeadCells: method('evacuateDeadCells'),
+ ...overrides
+ }
+ return { store, calls }
+}
+
+describe('assignment cleanup steps', () => {
+ it('runs every later sweep when an early one fails, naming the step', async () => {
+ const { store, calls } = stubStore({
+ completeReadyRegionalRehomes: vi.fn(async () => {
+ throw new Error('regional_rehome_assignment_mismatch')
+ })
+ })
+ const warn = vi.fn()
+
+ await runAssignmentCleanup(store, warn)
+
+ expect(calls).toEqual([
+ 'refreshRegionalRehomeLeases',
+ 'completeReadyEvacuations',
+ 'abortExpiredEvacuations',
+ 'abortExpiredRegionalRehomes',
+ 'reapRegionalRehomeAttempts',
+ 'releaseExpiredActivityLeases',
+ 'releaseExpiredActivity',
+ 'releaseExpiredRegionPreferences',
+ 'evacuateDeadCells'
+ ])
+ expect(warn).toHaveBeenCalledTimes(1)
+ expect(String(warn.mock.calls[0]![0])).toContain(
+ '[orca-relay] assignment cleanup failed: complete-ready-regional-rehomes'
+ )
+ })
+
+ it('covers all ten sweeps exactly once per run', async () => {
+ const { store, calls } = stubStore()
+
+ await runAssignmentCleanup(store)
+
+ expect(calls).toHaveLength(10)
+ expect(new Set(calls).size).toBe(10)
+ expect(assignmentCleanupSteps(store)).toHaveLength(10)
+ })
+})
diff --git a/cloud/apps/relay/src/assignment-cleanup-steps.ts b/cloud/apps/relay/src/assignment-cleanup-steps.ts
new file mode 100644
index 00000000000..0b6547d2ae4
--- /dev/null
+++ b/cloud/apps/relay/src/assignment-cleanup-steps.ts
@@ -0,0 +1,52 @@
+import { runRelayBackgroundOperation } from './relay-background-operation.js'
+
+// The ten periodic assignment sweeps the director runs every 30s. Each step
+// re-derives its state from the database and is idempotent, so they carry no
+// intra-tick ordering dependency — which is what makes per-step isolation
+// sound: one failing sweep costs one tick of itself, never the other nine.
+// (A single poisoned rehome row once silenced the whole chained form
+// fleet-wide.) Sweep failures are logged, never fed into the rehome worker's
+// dispatch-failure budget: a sweep exception is not a dispatch failure and
+// must not durably disable regional rehoming.
+export type AssignmentCleanupStore = {
+ refreshRegionalRehomeLeases(): Promise
+ completeReadyEvacuations(): Promise
+ completeReadyRegionalRehomes(): Promise
+ abortExpiredEvacuations(): Promise
+ abortExpiredRegionalRehomes(): Promise
+ reapRegionalRehomeAttempts(): Promise
+ releaseExpiredActivityLeases(): Promise
+ releaseExpiredActivity(): Promise
+ releaseExpiredRegionPreferences(): Promise
+ evacuateDeadCells(): Promise
+}
+
+export function assignmentCleanupSteps(
+ assignments: AssignmentCleanupStore
+): ReadonlyArray Promise]> {
+ return [
+ ['refresh-regional-rehome-leases', () => assignments.refreshRegionalRehomeLeases()],
+ ['complete-ready-evacuations', () => assignments.completeReadyEvacuations()],
+ ['complete-ready-regional-rehomes', () => assignments.completeReadyRegionalRehomes()],
+ ['abort-expired-evacuations', () => assignments.abortExpiredEvacuations()],
+ ['abort-expired-regional-rehomes', () => assignments.abortExpiredRegionalRehomes()],
+ ['reap-regional-rehome-attempts', () => assignments.reapRegionalRehomeAttempts()],
+ ['release-expired-activity-leases', () => assignments.releaseExpiredActivityLeases()],
+ ['release-expired-activity', () => assignments.releaseExpiredActivity()],
+ ['release-expired-region-preferences', () => assignments.releaseExpiredRegionPreferences()],
+ ['evacuate-dead-cells', () => assignments.evacuateDeadCells()]
+ ]
+}
+
+export async function runAssignmentCleanup(
+ assignments: AssignmentCleanupStore,
+ warn?: (message: string) => void
+): Promise {
+ for (const [step, operation] of assignmentCleanupSteps(assignments)) {
+ await runRelayBackgroundOperation(
+ operation,
+ `[orca-relay] assignment cleanup failed: ${step}`,
+ warn
+ )
+ }
+}
diff --git a/cloud/apps/relay/src/assignment-connection-headroom-postgres.test.ts b/cloud/apps/relay/src/assignment-connection-headroom-postgres.test.ts
new file mode 100644
index 00000000000..80a74a47eeb
--- /dev/null
+++ b/cloud/apps/relay/src/assignment-connection-headroom-postgres.test.ts
@@ -0,0 +1,348 @@
+import pg from 'pg'
+import { afterAll, beforeAll, describe, expect, it } from 'vitest'
+import { ASSIGNMENT_CONNECTION_HEADROOM_QUERY } from './assignment-connection-headroom-query.js'
+import { RelayAssignmentStore } from './assignment-store.js'
+import {
+ openRelayDatabase,
+ type RelayDatabase
+} from './database.js'
+
+const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
+const describePostgres = databaseUrl ? describe : describe.skip
+const headroomIndexName = 'relay_control_connection_reservation_headroom'
+const cell = {
+ id: 'connection-headroom-postgres',
+ url: 'https://connection-headroom-postgres.example.com',
+ capacityRequests: 1_000,
+ connectionHardCap: 600 as const,
+ connectionUnobservedBound: 50
+}
+
+describePostgres('PostgreSQL assignment connection headroom', () => {
+ const databases: RelayDatabase[] = []
+
+ beforeAll(async () => {
+ databases.push(
+ await openRelayDatabase({ databaseUrl, dataDir: '' }),
+ await openRelayDatabase({ databaseUrl, dataDir: '' }),
+ await openRelayDatabase({ databaseUrl, dataDir: '' })
+ )
+ })
+
+ afterAll(async () => {
+ const database = databases[0]
+ if (database) {
+ await database.query(
+ `DELETE FROM relay_control_connection_reservations
+ WHERE user_id LIKE 'connection-headroom-postgres-%'`
+ )
+ await database.query(
+ `DELETE FROM relay_assignment_activity_leases
+ WHERE user_id LIKE 'connection-headroom-postgres-%'`
+ )
+ await database.query(
+ `DELETE FROM relay_assignments
+ WHERE user_id LIKE 'connection-headroom-postgres-%'`
+ )
+ // A snapshot left by an aborted run rejects the replayed watermark
+ // with stale_connection_snapshot.
+ await database.query(
+ `DELETE FROM relay_cell_connection_snapshots WHERE cell_id = ?`,
+ [cell.id]
+ )
+ await database.query(
+ `DELETE FROM relay_cell_connection_runtime WHERE cell_id = ?`,
+ [cell.id]
+ )
+ await database.query(
+ `DELETE FROM relay_cell_connection_limits WHERE cell_id = ?`,
+ [cell.id]
+ )
+ await database.query(`DELETE FROM relay_cell_runtime WHERE cell_id = ?`, [cell.id])
+ await database.query(`DELETE FROM relay_cells WHERE cell_id = ?`, [cell.id])
+ }
+ for (const connection of databases) await connection.close()
+ })
+
+ it('commits only one parallel assignment at the admission boundary', async () => {
+ const stores = databases.map(
+ (database) =>
+ new RelayAssignmentStore(database, () => 100, {
+ requireLiveCells: true,
+ heartbeatTtlMs: 45_000
+ })
+ )
+ await databases[0]!.query(
+ `DELETE FROM relay_control_connection_reservations
+ WHERE user_id LIKE 'connection-headroom-postgres-%'`
+ )
+ await databases[0]!.query(
+ `DELETE FROM relay_assignment_activity_leases
+ WHERE user_id LIKE 'connection-headroom-postgres-%'`
+ )
+ await databases[0]!.query(
+ `DELETE FROM relay_assignments
+ WHERE user_id LIKE 'connection-headroom-postgres-%'`
+ )
+ await stores[0]!.reconcileCells([cell], false)
+ await stores[0]!.recordCellHeartbeat({
+ cellId: cell.id,
+ cellUrl: cell.url,
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ startedAt: 50,
+ ready: true,
+ observedRequests: 0,
+ totalConnections: 449,
+ inFlightConnections: 0,
+ reservedConnectionUnits: 0,
+ enforcedConnectionUnits: 449,
+ connectionHardCap: 600,
+ connectionUnobservedBound: 50
+ })
+ for (const suffix of ['1', '2']) {
+ await databases[0]!.query(
+ `INSERT INTO relay_assignments
+ (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at,
+ last_activity_at, reserved_controls, reserved_splices, reserved_invites,
+ pending_installs, pending_confirmations, migration_leases)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+ [
+ `connection-headroom-postgres-${suffix}`,
+ `headroomhost000${suffix}`,
+ cell.id,
+ 1,
+ 90_100,
+ 100,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ]
+ )
+ }
+
+ const results = await Promise.allSettled([
+ stores[1]!.assign({
+ userId: 'connection-headroom-postgres-1',
+ relayHostId: 'headroomhost0001'
+ }),
+ stores[2]!.assign({
+ userId: 'connection-headroom-postgres-2',
+ relayHostId: 'headroomhost0002'
+ })
+ ])
+ expect(results.filter(({ status }) => status === 'fulfilled')).toHaveLength(1)
+ expect(results.filter(({ status }) => status === 'rejected')).toHaveLength(1)
+
+ const assignments = await databases[0]!.query(
+ `SELECT COALESCE(SUM(reserved_controls), 0) AS count FROM relay_assignments
+ WHERE user_id LIKE 'connection-headroom-postgres-%'`
+ )
+ const pending = await databases[0]!.query(
+ `SELECT COUNT(*) AS count FROM relay_assignment_activity_leases
+ WHERE user_id LIKE 'connection-headroom-postgres-%'
+ AND activity_kind = 'control'
+ AND activity_id LIKE 'control-pending:%'`
+ )
+ const reservations = await databases[0]!.query(
+ `SELECT COUNT(*) AS count FROM relay_control_connection_reservations
+ WHERE user_id LIKE 'connection-headroom-postgres-%'
+ AND state <> 'released'`
+ )
+ expect(Number(assignments[0]!.count)).toBe(1)
+ expect(Number(pending[0]!.count)).toBe(1)
+ expect(Number(reservations[0]!.count)).toBe(1)
+ }, 15_000)
+
+ it('uses the composite headroom index when released history dominates', async () => {
+ await databases[0]!.query(
+ `INSERT INTO relay_control_connection_reservations
+ (reservation_id, idempotency_key, user_id, relay_host_id,
+ assignment_epoch, cell_id, state, created_at, timeout_at, updated_at)
+ SELECT 'headroom-plan-' || value, 'headroom-plan-' || value,
+ 'connection-headroom-postgres-plan', 'plan-host-' || value,
+ 1, ?, 'released', 1, 2, 1
+ FROM generate_series(1, 50000) AS value`,
+ [cell.id]
+ )
+ await databases[0]!.query(`ANALYZE relay_control_connection_reservations`)
+
+ const client = new pg.Client({ connectionString: databaseUrl })
+ await client.connect()
+ let reservationIndexPlan: Record | undefined
+ try {
+ const plan = await client.query(
+ `EXPLAIN (FORMAT JSON) ${ASSIGNMENT_CONNECTION_HEADROOM_QUERY}`
+ )
+ reservationIndexPlan = findReservationHeadroomIndexPlan(
+ plan.rows[0]?.['QUERY PLAN']
+ )
+ } finally {
+ await client.end()
+ }
+
+ expect(reservationIndexPlan).toBeDefined()
+ expect(String(reservationIndexPlan?.['Index Cond'])).toContain('cell_id')
+ expect(String(reservationIndexPlan?.['Index Cond'])).toContain('state = ANY')
+ expect(reservationIndexPlan?.['Filter']).toBeUndefined()
+ })
+
+ it('deduplicates expired debt after a fresh PostgreSQL snapshot', async () => {
+ const now = 200
+ const store = new RelayAssignmentStore(databases[0]!, () => now, {
+ requireLiveCells: true,
+ heartbeatTtlMs: 45_000
+ })
+ const identity = {
+ userId: 'connection-headroom-postgres-debt',
+ relayHostId: 'headroomdebt0001'
+ }
+ await databases[0]!.query(
+ `INSERT INTO relay_control_connection_reservations
+ (reservation_id, idempotency_key, user_id, relay_host_id,
+ assignment_epoch, cell_id, state, inclusion_watermark,
+ claim_activity_id, created_at, timeout_at, claimed_at, released_at,
+ updated_at)
+ VALUES
+ (?, ?, ?, ?, 1, ?, 'late-arrival-debt', NULL, NULL, 100, 150, NULL, NULL, 150),
+ (?, ?, ?, ?, 1, ?, 'late-arrival-debt', NULL, NULL, 101, 150, NULL, NULL, 150)`,
+ [
+ 'postgres-debt-1',
+ 'postgres-debt-1',
+ identity.userId,
+ identity.relayHostId,
+ cell.id,
+ 'postgres-debt-2',
+ 'postgres-debt-2',
+ identity.userId,
+ identity.relayHostId,
+ cell.id
+ ]
+ )
+
+ await store.recordCellHeartbeat({
+ cellId: cell.id,
+ cellUrl: cell.url,
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ startedAt: 50,
+ ready: true,
+ observedRequests: 0,
+ totalConnections: 449,
+ inFlightConnections: 0,
+ reservedConnectionUnits: 0,
+ enforcedConnectionUnits: 449,
+ connectionInclusionWatermark: 10,
+ connectionHardCap: 600,
+ connectionUnobservedBound: 50
+ })
+
+ expect(
+ await databases[0]!.query(
+ `SELECT state
+ FROM relay_control_connection_reservations
+ WHERE user_id = ?
+ ORDER BY created_at`,
+ [identity.userId]
+ )
+ ).toEqual([
+ { state: 'late-arrival-debt' },
+ { state: 'released' }
+ ])
+ })
+
+ it('releases an aborted older epoch after a fresh PostgreSQL snapshot', async () => {
+ const now = 300
+ const store = new RelayAssignmentStore(databases[0]!, () => now, {
+ requireLiveCells: true,
+ heartbeatTtlMs: 45_000
+ })
+ const identity = {
+ userId: 'connection-headroom-postgres-aborted',
+ relayHostId: 'headroomabort001'
+ }
+ await databases[0]!.query(
+ `INSERT INTO relay_assignments
+ (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at,
+ last_activity_at, reserved_controls, reserved_splices, reserved_invites,
+ pending_installs, pending_confirmations, migration_leases)
+ VALUES (?, ?, ?, 3, ?, ?, 0, 0, 0, 0, 0, 0)`,
+ [identity.userId, identity.relayHostId, cell.id, now, now]
+ )
+ await databases[0]!.query(
+ `INSERT INTO relay_assignment_migrations
+ (user_id, relay_host_id, source_cell_id, target_cell_id,
+ previous_epoch, assignment_epoch, source_request_units,
+ target_reserved_units, expires_at, target_registered_at,
+ completed_at, aborted_at, created_at, updated_at)
+ VALUES (?, ?, ?, ?, 1, 2, 0, 1, ?, ?, NULL, ?, ?, ?)`,
+ [
+ identity.userId,
+ identity.relayHostId,
+ 'source',
+ cell.id,
+ now - 2,
+ now - 3,
+ now - 1,
+ now - 4,
+ now - 1
+ ]
+ )
+ await databases[0]!.query(
+ `INSERT INTO relay_control_connection_reservations
+ (reservation_id, idempotency_key, user_id, relay_host_id,
+ assignment_epoch, cell_id, state, inclusion_watermark,
+ claim_activity_id, created_at, timeout_at, claimed_at, released_at,
+ updated_at)
+ VALUES (?, ?, ?, ?, 2, ?, 'late-arrival-debt', NULL, NULL, ?, ?, NULL, NULL, ?)`,
+ [
+ 'postgres-aborted-reservation',
+ 'postgres-aborted-reservation',
+ identity.userId,
+ identity.relayHostId,
+ cell.id,
+ now - 4,
+ now - 2,
+ now - 1
+ ]
+ )
+
+ await store.recordCellHeartbeat({
+ cellId: cell.id,
+ cellUrl: cell.url,
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ startedAt: 50,
+ ready: true,
+ observedRequests: 0,
+ totalConnections: 449,
+ inFlightConnections: 0,
+ reservedConnectionUnits: 0,
+ enforcedConnectionUnits: 449,
+ connectionInclusionWatermark: 20,
+ connectionHardCap: 600,
+ connectionUnobservedBound: 50
+ })
+
+ expect(
+ await databases[0]!.query(
+ `SELECT state FROM relay_control_connection_reservations
+ WHERE reservation_id = ?`,
+ ['postgres-aborted-reservation']
+ )
+ ).toEqual([{ state: 'released' }])
+ })
+})
+
+function findReservationHeadroomIndexPlan(
+ value: unknown
+): Record | undefined {
+ if (value === null || typeof value !== 'object') return undefined
+ const record = value as Record
+ if (!Array.isArray(value) && record['Index Name'] === headroomIndexName) return record
+ for (const child of Object.values(value)) {
+ const match = findReservationHeadroomIndexPlan(child)
+ if (match) return match
+ }
+ return undefined
+}
diff --git a/cloud/apps/relay/src/assignment-connection-headroom-query.ts b/cloud/apps/relay/src/assignment-connection-headroom-query.ts
new file mode 100644
index 00000000000..c0237aecf26
--- /dev/null
+++ b/cloud/apps/relay/src/assignment-connection-headroom-query.ts
@@ -0,0 +1,15 @@
+export const ASSIGNMENT_CONNECTION_HEADROOM_QUERY =
+ `SELECT limits.cell_id, limits.hard_cap, limits.unobserved_bound,
+ connection_snapshot.enforced_connection_units,
+ connection_snapshot.snapshot_at AS last_heartbeat_at,
+ connection_snapshot.cell_incarnation AS connection_incarnation,
+ current_runtime.cell_incarnation AS current_incarnation,
+ (SELECT COUNT(*) FROM relay_control_connection_reservations reservation
+ WHERE reservation.cell_id = limits.cell_id
+ AND reservation.state IN
+ ('reserved', 'late-arrival-debt', 'claimed')) AS outstanding_reservations
+ FROM relay_cell_connection_limits limits
+ LEFT JOIN relay_cell_connection_snapshots connection_snapshot
+ ON connection_snapshot.cell_id = limits.cell_id
+ LEFT JOIN relay_cell_runtime current_runtime
+ ON current_runtime.cell_id = limits.cell_id`
diff --git a/cloud/apps/relay/src/assignment-connection-headroom.test.ts b/cloud/apps/relay/src/assignment-connection-headroom.test.ts
new file mode 100644
index 00000000000..19f5d46aa33
--- /dev/null
+++ b/cloud/apps/relay/src/assignment-connection-headroom.test.ts
@@ -0,0 +1,1007 @@
+import { ASSIGNMENT_LIMITS } from '@orca-cloud/relay-contract'
+import { afterEach, describe, expect, it } from 'vitest'
+import { RelayAssignmentStore } from './assignment-store.js'
+import type { RelayCellConfig } from './config.js'
+import {
+ openInMemoryRelayDatabase,
+ type RelayDatabase
+} from './database.js'
+
+const LIMITED_CELL: RelayCellConfig = {
+ id: 'limited',
+ url: 'https://limited.example.com',
+ capacityRequests: 1_000,
+ connectionHardCap: 600,
+ connectionUnobservedBound: 50
+}
+
+const databases: RelayDatabase[] = []
+
+afterEach(async () => {
+ for (const database of databases.splice(0)) await database.close()
+})
+
+async function setup(
+ connectionUnits: number,
+ now: () => number = () => 100
+): Promise<{ database: RelayDatabase; store: RelayAssignmentStore }> {
+ const database = await openInMemoryRelayDatabase()
+ databases.push(database)
+ const store = new RelayAssignmentStore(database, now, {
+ requireLiveCells: true,
+ heartbeatTtlMs: 45_000
+ })
+ await store.reconcileCells([LIMITED_CELL], true)
+ await store.recordCellHeartbeat({
+ cellId: LIMITED_CELL.id,
+ cellUrl: LIMITED_CELL.url,
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ startedAt: 50,
+ ready: true,
+ observedRequests: 0,
+ totalConnections: connectionUnits,
+ inFlightConnections: 0,
+ reservedConnectionUnits: 0,
+ enforcedConnectionUnits: connectionUnits,
+ connectionHardCap: 600,
+ connectionUnobservedBound: 50
+ })
+ return { database, store }
+}
+
+async function setupHeadroomReassignment(): Promise<{
+ database: RelayDatabase
+ store: RelayAssignmentStore
+ source: RelayCellConfig
+ target: RelayCellConfig
+}> {
+ const database = await openInMemoryRelayDatabase()
+ databases.push(database)
+ const store = new RelayAssignmentStore(database, () => 100, {
+ requireLiveCells: true,
+ heartbeatTtlMs: 45_000
+ })
+ const source = {
+ ...LIMITED_CELL,
+ id: 'saturated-source',
+ url: 'https://saturated-source.example.com'
+ }
+ const target = {
+ ...LIMITED_CELL,
+ id: 'available-target',
+ url: 'https://available-target.example.com'
+ }
+ await store.reconcileCells([source, target], true)
+ await store.recordCellHeartbeat({
+ cellId: source.id,
+ cellUrl: source.url,
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ startedAt: 50,
+ ready: true,
+ observedRequests: 0,
+ totalConnections: 450,
+ inFlightConnections: 0,
+ reservedConnectionUnits: 0,
+ enforcedConnectionUnits: 450,
+ connectionHardCap: 600,
+ connectionUnobservedBound: 50
+ })
+ await store.recordCellHeartbeat({
+ cellId: target.id,
+ cellUrl: target.url,
+ cellIncarnation: '22222222-2222-4222-8222-222222222222',
+ startedAt: 50,
+ ready: true,
+ observedRequests: 0,
+ totalConnections: 0,
+ inFlightConnections: 0,
+ reservedConnectionUnits: 0,
+ enforcedConnectionUnits: 0,
+ connectionHardCap: 600,
+ connectionUnobservedBound: 50
+ })
+ return { database, store, source, target }
+}
+
+describe('relay assignment connection headroom', () => {
+ it('requires exact, internally consistent telemetry from limited cells', async () => {
+ const database = await openInMemoryRelayDatabase()
+ databases.push(database)
+ const store = new RelayAssignmentStore(database, () => 100)
+ await store.reconcileCells([LIMITED_CELL], true)
+ const heartbeat = {
+ cellId: LIMITED_CELL.id,
+ cellUrl: LIMITED_CELL.url,
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ startedAt: 50,
+ ready: true,
+ observedRequests: 0
+ }
+
+ await expect(store.recordCellHeartbeat(heartbeat)).rejects.toThrow(
+ 'cell_connection_telemetry_mismatch'
+ )
+ await expect(
+ store.recordCellHeartbeat({
+ ...heartbeat,
+ totalConnections: 550,
+ inFlightConnections: 2,
+ reservedConnectionUnits: 3,
+ enforcedConnectionUnits: 554,
+ connectionHardCap: 600,
+ connectionUnobservedBound: 50
+ })
+ ).rejects.toThrow('cell_connection_telemetry_mismatch')
+ await expect(
+ store.recordCellHeartbeat({
+ ...heartbeat,
+ totalConnections: 601,
+ inFlightConnections: 0,
+ reservedConnectionUnits: 0,
+ enforcedConnectionUnits: 601,
+ connectionHardCap: 600,
+ connectionUnobservedBound: 50
+ })
+ ).resolves.toBeUndefined()
+ })
+
+ it('reports the limited-cell capacity contract and telemetry components', async () => {
+ const { store } = await setup(0)
+ await store.recordCellHeartbeat({
+ cellId: LIMITED_CELL.id,
+ cellUrl: LIMITED_CELL.url,
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ startedAt: 50,
+ ready: true,
+ observedRequests: 0,
+ totalConnections: 120,
+ inFlightConnections: 2,
+ reservedConnectionUnits: 3,
+ enforcedConnectionUnits: 125,
+ connectionHardCap: 600,
+ connectionUnobservedBound: 50
+ })
+
+ expect((await store.cellDeploymentStatus(LIMITED_CELL.id)).connectionCapacity).toEqual({
+ hardCap: 600,
+ controlRebindReserve: 100,
+ ordinaryConnectionLimit: 500,
+ unobservedBound: 50,
+ normalAdmissionPause: 450,
+ observedConnections: 120,
+ inFlightConnections: 2,
+ reservedConnectionUnits: 3,
+ enforcedConnectionUnits: 125,
+ pendingControlReservations: 0,
+ heartbeatFresh: true
+ })
+ })
+
+ it('fails placement closed while a cell changes connection limits', async () => {
+ const { store } = await setup(449)
+ const expanded = {
+ ...LIMITED_CELL,
+ connectionHardCap: 1_000 as const
+ }
+
+ await store.reconcileCells([expanded], true)
+ await expect(
+ store.assign({ userId: 'transition-user', relayHostId: 'host000000000099' })
+ ).rejects.toThrow('relay_capacity_exhausted')
+ await expect(store.cellDeploymentStatus(expanded.id)).resolves.toMatchObject({
+ connectionCapacity: { hardCap: 1_000, heartbeatFresh: false }
+ })
+
+ await store.recordCellHeartbeat({
+ cellId: expanded.id,
+ cellUrl: expanded.url,
+ cellIncarnation: '22222222-2222-4222-8222-222222222222',
+ startedAt: 200,
+ ready: true,
+ observedRequests: 0,
+ totalConnections: 449,
+ inFlightConnections: 0,
+ reservedConnectionUnits: 0,
+ enforcedConnectionUnits: 449,
+ connectionHardCap: 1_000,
+ connectionUnobservedBound: 50
+ })
+ await expect(store.cellDeploymentStatus(expanded.id)).resolves.toMatchObject({
+ connectionCapacity: {
+ hardCap: 1_000,
+ ordinaryConnectionLimit: 900,
+ normalAdmissionPause: 850,
+ heartbeatFresh: true
+ }
+ })
+ })
+
+ it('fails placement closed while the admin path changes connection limits', async () => {
+ const { store } = await setup(449)
+ const expanded = {
+ ...LIMITED_CELL,
+ connectionHardCap: 1_000 as const
+ }
+
+ await store.configureCell(expanded, 'general')
+ await expect(
+ store.assign({ userId: 'admin-transition', relayHostId: 'host000000000098' })
+ ).rejects.toThrow('relay_capacity_exhausted')
+ await expect(store.cellDeploymentStatus(expanded.id)).resolves.toMatchObject({
+ connectionCapacity: { hardCap: 1_000, heartbeatFresh: false }
+ })
+
+ await store.recordCellHeartbeat({
+ cellId: expanded.id,
+ cellUrl: expanded.url,
+ cellIncarnation: '22222222-2222-4222-8222-222222222222',
+ startedAt: 200,
+ ready: true,
+ observedRequests: 0,
+ totalConnections: 449,
+ inFlightConnections: 0,
+ reservedConnectionUnits: 0,
+ enforcedConnectionUnits: 449,
+ connectionHardCap: 1_000,
+ connectionUnobservedBound: 50
+ })
+ await expect(
+ store.assign({ userId: 'admin-restored', relayHostId: 'host000000000097' })
+ ).resolves.toMatchObject({ cellId: expanded.id })
+ })
+
+ it('admits exactly one reservation at the admission boundary', async () => {
+ const { database, store } = await setup(449)
+ const results = await Promise.allSettled([
+ store.assign({ userId: 'user-1', relayHostId: 'host000000000001' }),
+ store.assign({ userId: 'user-2', relayHostId: 'host000000000002' })
+ ])
+
+ expect(results.filter(({ status }) => status === 'fulfilled')).toHaveLength(1)
+ expect(
+ results.filter(({ status }) => status === 'rejected').map((result) =>
+ result.status === 'rejected' ? result.reason : null
+ )
+ ).toEqual([expect.objectContaining({ message: 'relay_capacity_exhausted' })])
+ expect(
+ await database.query(
+ `SELECT activity_id FROM relay_assignment_activity_leases
+ WHERE activity_kind = 'control' AND activity_id LIKE 'control-pending:%'`
+ )
+ ).toHaveLength(1)
+ })
+
+ it('rejects at the boundary before mutating durable assignment state', async () => {
+ const { database, store } = await setup(450)
+
+ await expect(
+ store.assign({ userId: 'user-1', relayHostId: 'host000000000001' })
+ ).rejects.toThrow('relay_capacity_exhausted')
+ expect(await database.query(`SELECT * FROM relay_assignments`)).toEqual([])
+ expect(await database.query(`SELECT * FROM relay_assignment_activity_leases`)).toEqual([])
+ })
+
+ it('charges every active reservation state at the admission boundary', async () => {
+ const { database, store } = await setup(447)
+ await database.query(
+ `INSERT INTO relay_control_connection_reservations
+ (reservation_id, idempotency_key, user_id, relay_host_id,
+ assignment_epoch, cell_id, state, created_at, timeout_at, updated_at)
+ VALUES
+ ('active-reserved', 'active-reserved', 'state-user', 'state-host-1',
+ 1, ?, 'reserved', 1, 2, 1),
+ ('active-debt', 'active-debt', 'state-user', 'state-host-2',
+ 1, ?, 'late-arrival-debt', 1, 2, 1),
+ ('active-claimed', 'active-claimed', 'state-user', 'state-host-3',
+ 1, ?, 'claimed', 1, 2, 1)`,
+ [LIMITED_CELL.id, LIMITED_CELL.id, LIMITED_CELL.id]
+ )
+
+ await expect(
+ store.assign({ userId: 'blocked-user', relayHostId: 'blockedhost00001' })
+ ).rejects.toThrow('relay_capacity_exhausted')
+ expect(await database.query(`SELECT * FROM relay_assignment_activity_leases`)).toEqual([])
+ })
+
+ it('does not charge released reservation history', async () => {
+ const { database, store } = await setup(449)
+ await database.query(
+ `INSERT INTO relay_control_connection_reservations
+ (reservation_id, idempotency_key, user_id, relay_host_id,
+ assignment_epoch, cell_id, state, created_at, timeout_at, updated_at)
+ VALUES ('released-history', 'released-history', 'state-user', 'state-host',
+ 1, ?, 'released', 1, 2, 1)`,
+ [LIMITED_CELL.id]
+ )
+
+ await expect(
+ store.assign({ userId: 'admitted-user', relayHostId: 'admittedhost0001' })
+ ).resolves.toMatchObject({ cellId: LIMITED_CELL.id })
+ })
+
+ it('rejects sticky control restoration before mutating assignment state', async () => {
+ const { database, store } = await setup(450)
+ const identity = { userId: 'sticky-user', relayHostId: 'stickyhost000001' }
+ await database.query(
+ `INSERT INTO relay_assignments
+ (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at,
+ last_activity_at, reserved_controls, reserved_splices, reserved_invites,
+ pending_installs, pending_confirmations, migration_leases)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+ [
+ identity.userId,
+ identity.relayHostId,
+ LIMITED_CELL.id,
+ 1,
+ 10_000,
+ 100,
+ 0,
+ 0,
+ 1,
+ 0,
+ 0,
+ 0
+ ]
+ )
+
+ await expect(store.assign(identity)).rejects.toThrow(
+ 'relay_connection_headroom_exhausted'
+ )
+ expect(
+ await database.query(
+ `SELECT cell_id, assignment_epoch, reserved_controls, reserved_invites
+ FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([
+ {
+ cell_id: LIMITED_CELL.id,
+ assignment_epoch: 1,
+ reserved_controls: 0,
+ reserved_invites: 1
+ }
+ ])
+ expect(await database.query(`SELECT * FROM relay_assignment_activity_leases`)).toEqual([])
+ })
+
+ it('moves a zero-activity assignment off a general cell without connection headroom', async () => {
+ const { database, store, source, target } = await setupHeadroomReassignment()
+ const identity = { userId: 'movable-user', relayHostId: 'movablehost000001' }
+ await database.query(
+ `INSERT INTO relay_assignments
+ (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at,
+ last_activity_at, reserved_controls, reserved_splices, reserved_invites,
+ pending_installs, pending_confirmations, migration_leases)
+ VALUES (?, ?, ?, ?, ?, ?, 0, 0, 0, 0, 0, 0)`,
+ [identity.userId, identity.relayHostId, source.id, 7, 10_000, 100]
+ )
+ await database.query(
+ `INSERT INTO relay_control_connection_reservations
+ (reservation_id, idempotency_key, user_id, relay_host_id,
+ assignment_epoch, cell_id, state, inclusion_watermark,
+ claim_activity_id, created_at, timeout_at, claimed_at, released_at,
+ updated_at)
+ VALUES (?, ?, ?, ?, ?, ?, 'late-arrival-debt', NULL, NULL, ?, ?, NULL, NULL, ?)`,
+ [
+ 'superseded-reservation',
+ 'superseded-reservation',
+ identity.userId,
+ identity.relayHostId,
+ 7,
+ source.id,
+ 50,
+ 99,
+ 100
+ ]
+ )
+
+ await expect(store.assign(identity)).resolves.toMatchObject({
+ cellId: target.id,
+ assignmentEpoch: 8
+ })
+ expect(
+ await database.query(
+ `SELECT cell_id, assignment_epoch, reserved_controls
+ FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([{ cell_id: target.id, assignment_epoch: 8, reserved_controls: 1 }])
+ expect(
+ await database.query(
+ `SELECT cell_id, activity_kind FROM relay_assignment_activity_leases
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([{ cell_id: target.id, activity_kind: 'control' }])
+ expect(
+ await database.query(
+ `SELECT cell_id, assignment_epoch, state
+ FROM relay_control_connection_reservations
+ WHERE user_id = ? AND relay_host_id = ?
+ ORDER BY assignment_epoch`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([
+ { cell_id: source.id, assignment_epoch: 7, state: 'released' },
+ { cell_id: target.id, assignment_epoch: 8, state: 'reserved' }
+ ])
+ })
+
+ it('keeps non-control activity pinned when its cell has no connection headroom', async () => {
+ const { database, store, source } = await setupHeadroomReassignment()
+ const identity = { userId: 'pinned-user', relayHostId: 'pinnedhost0000001' }
+ await database.query(
+ `INSERT INTO relay_assignments
+ (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at,
+ last_activity_at, reserved_controls, reserved_splices, reserved_invites,
+ pending_installs, pending_confirmations, migration_leases)
+ VALUES (?, ?, ?, ?, ?, ?, 0, 0, 1, 0, 0, 0)`,
+ [identity.userId, identity.relayHostId, source.id, 7, 10_000, 100]
+ )
+
+ await expect(store.assign(identity)).rejects.toThrow(
+ 'relay_connection_headroom_exhausted'
+ )
+ expect(await store.resolve(identity)).toMatchObject({
+ cellId: source.id,
+ assignmentEpoch: 7
+ })
+ })
+
+ it.each(['existing-only', 'migration-only'] as const)(
+ 'keeps a zero-activity assignment pinned on a %s cell without connection headroom',
+ async (admission) => {
+ const { database, store, source } = await setupHeadroomReassignment()
+ const identity = {
+ userId: `pinned-${admission}-user`,
+ relayHostId: `pinned${admission.replace('-', '')}`
+ }
+ await store.setCellAdmissionState(source.id, admission)
+ await database.query(
+ `INSERT INTO relay_assignments
+ (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at,
+ last_activity_at, reserved_controls, reserved_splices, reserved_invites,
+ pending_installs, pending_confirmations, migration_leases)
+ VALUES (?, ?, ?, ?, ?, ?, 0, 0, 0, 0, 0, 0)`,
+ [identity.userId, identity.relayHostId, source.id, 7, 10_000, 100]
+ )
+
+ await expect(store.assign(identity)).rejects.toThrow(
+ 'relay_connection_headroom_exhausted'
+ )
+ expect(await store.resolve(identity)).toMatchObject({
+ cellId: source.id,
+ assignmentEpoch: 7
+ })
+ expect(await database.query(`SELECT * FROM relay_assignment_activity_leases`)).toEqual([])
+ }
+ )
+
+ it('renames a pending reservation exactly once on control activation', async () => {
+ const { database, store } = await setup(449)
+ const identity = { userId: 'user-1', relayHostId: 'host000000000001' }
+ const assignment = await store.assign(identity)
+
+ await store.activateControl(identity, {
+ cellId: LIMITED_CELL.id,
+ assignmentEpoch: assignment.assignmentEpoch,
+ generation: 1
+ })
+ await store.activateControl(identity, {
+ cellId: LIMITED_CELL.id,
+ assignmentEpoch: assignment.assignmentEpoch,
+ generation: 1
+ })
+
+ expect(
+ await database.query(
+ `SELECT activity_id, request_units FROM relay_assignment_activity_leases
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([{ activity_id: 'control:limited:1', request_units: 1 }])
+ })
+
+ it('keeps expired pending reservations charged as late-arrival debt', async () => {
+ let now = 100
+ const { database, store } = await setup(449, () => now)
+ await store.assign({ userId: 'user-1', relayHostId: 'host000000000001' })
+ now += ASSIGNMENT_LIMITS.activityLeaseMs + 1
+ await store.releaseExpiredActivityLeases()
+ await store.recordCellHeartbeat({
+ cellId: LIMITED_CELL.id,
+ cellUrl: LIMITED_CELL.url,
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ startedAt: 50,
+ ready: true,
+ observedRequests: 0,
+ totalConnections: 449,
+ inFlightConnections: 0,
+ reservedConnectionUnits: 0,
+ enforcedConnectionUnits: 449,
+ connectionHardCap: 600,
+ connectionUnobservedBound: 50
+ })
+
+ await expect(
+ store.assign({ userId: 'user-2', relayHostId: 'host000000000002' })
+ ).rejects.toThrow('relay_capacity_exhausted')
+ expect(
+ await database.query(
+ `SELECT activity_id FROM relay_assignment_activity_leases
+ WHERE activity_kind = 'control' AND activity_id LIKE 'control-pending:%'`
+ )
+ ).toHaveLength(0)
+ expect(
+ await database.query(
+ `SELECT state FROM relay_control_connection_reservations`
+ )
+ ).toEqual([{ state: 'late-arrival-debt' }])
+ })
+
+ it('reconciles late-arrival debt only after a covering absolute snapshot', async () => {
+ let now = 100
+ const { database, store } = await setup(449, () => now)
+ const identity = { userId: 'late-user', relayHostId: 'latehost00000001' }
+ const assignment = await store.assign(identity)
+ now += ASSIGNMENT_LIMITS.activityLeaseMs + 1
+ await store.releaseExpiredActivityLeases()
+ await store.recordCellHeartbeat({
+ cellId: LIMITED_CELL.id,
+ cellUrl: LIMITED_CELL.url,
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ startedAt: 50,
+ ready: true,
+ observedRequests: 0,
+ totalConnections: 449,
+ inFlightConnections: 0,
+ reservedConnectionUnits: 0,
+ enforcedConnectionUnits: 449,
+ connectionInclusionWatermark: 4,
+ connectionHardCap: 600,
+ connectionUnobservedBound: 50
+ })
+
+ await store.activateControl(identity, {
+ cellId: LIMITED_CELL.id,
+ assignmentEpoch: assignment.assignmentEpoch,
+ generation: 1,
+ connectionInclusionWatermark: 5
+ })
+ expect(
+ await database.query(
+ `SELECT state, inclusion_watermark FROM relay_control_connection_reservations`
+ )
+ ).toEqual([{ state: 'claimed', inclusion_watermark: 5 }])
+
+ await store.recordCellHeartbeat({
+ cellId: LIMITED_CELL.id,
+ cellUrl: LIMITED_CELL.url,
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ startedAt: 50,
+ ready: true,
+ observedRequests: 0,
+ totalConnections: 450,
+ inFlightConnections: 0,
+ reservedConnectionUnits: 0,
+ enforcedConnectionUnits: 450,
+ connectionInclusionWatermark: 5,
+ connectionHardCap: 600,
+ connectionUnobservedBound: 50
+ })
+ expect(
+ await database.query(
+ `SELECT state, released_at FROM relay_control_connection_reservations`
+ )
+ ).toEqual([{ state: 'released', released_at: now }])
+ await expect(
+ store.assign({ userId: 'blocked-user', relayHostId: 'blockedhost00001' })
+ ).rejects.toThrow('relay_capacity_exhausted')
+ })
+
+ it('rejects duplicate and out-of-order absolute snapshots', async () => {
+ const { database, store } = await setup(0)
+ const heartbeat = {
+ cellId: LIMITED_CELL.id,
+ cellUrl: LIMITED_CELL.url,
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ startedAt: 50,
+ ready: true,
+ observedRequests: 0,
+ totalConnections: 10,
+ inFlightConnections: 0,
+ reservedConnectionUnits: 0,
+ enforcedConnectionUnits: 10,
+ connectionInclusionWatermark: 10,
+ connectionHardCap: 600 as const,
+ connectionUnobservedBound: 50
+ }
+ await store.recordCellHeartbeat(heartbeat)
+ await expect(store.recordCellHeartbeat(heartbeat)).rejects.toThrow(
+ 'stale_connection_snapshot'
+ )
+ await expect(
+ store.recordCellHeartbeat({
+ ...heartbeat,
+ totalConnections: 9,
+ enforcedConnectionUnits: 9,
+ connectionInclusionWatermark: 9
+ })
+ ).rejects.toThrow('stale_connection_snapshot')
+ expect(
+ await database.query(
+ `SELECT inclusion_watermark, enforced_connection_units
+ FROM relay_cell_connection_snapshots`
+ )
+ ).toEqual([{ inclusion_watermark: 10, enforced_connection_units: 10 }])
+ })
+
+ it('keeps crash retries bound to one reservation claim', async () => {
+ let now = 100
+ const { database, store } = await setup(448, () => now)
+ const identity = { userId: 'retry-user', relayHostId: 'retryhost0000001' }
+ const assignment = await store.assign(identity)
+ await new RelayAssignmentStore(database, () => now, {
+ requireLiveCells: true,
+ heartbeatTtlMs: 45_000
+ }).assign(identity)
+ expect(
+ await database.query(
+ `SELECT state FROM relay_control_connection_reservations`
+ )
+ ).toEqual([{ state: 'reserved' }])
+ now += ASSIGNMENT_LIMITS.activityLeaseMs + 1
+ await store.releaseExpiredActivityLeases()
+ await store.recordCellHeartbeat({
+ cellId: LIMITED_CELL.id,
+ cellUrl: LIMITED_CELL.url,
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ startedAt: 50,
+ ready: true,
+ observedRequests: 0,
+ totalConnections: 448,
+ inFlightConnections: 0,
+ reservedConnectionUnits: 0,
+ enforcedConnectionUnits: 448,
+ connectionHardCap: 600,
+ connectionUnobservedBound: 50
+ })
+ await store.assign(identity)
+ const restarted = new RelayAssignmentStore(database, () => now, {
+ requireLiveCells: true,
+ heartbeatTtlMs: 45_000
+ })
+
+ const activation = {
+ cellId: LIMITED_CELL.id,
+ assignmentEpoch: assignment.assignmentEpoch,
+ generation: 1,
+ connectionInclusionWatermark: 5
+ }
+ await restarted.activateControl(identity, activation)
+ await restarted.activateControl(identity, activation)
+
+ expect(
+ await database.query(
+ `SELECT state, claim_activity_id
+ FROM relay_control_connection_reservations
+ ORDER BY created_at ASC, reservation_id ASC`
+ )
+ ).toEqual([{ state: 'claimed', claim_activity_id: 'control:limited:1' }])
+ })
+
+ it('releases only redundant expired debt after a fresh absolute snapshot', async () => {
+ let now = 100
+ const { database, store } = await setup(449, () => now)
+ const identity = { userId: 'debt-user', relayHostId: 'debthost000000001' }
+ const assignment = await store.assign(identity)
+ now += ASSIGNMENT_LIMITS.activityLeaseMs + 1
+ await store.releaseExpiredActivityLeases()
+ await database.query(
+ `INSERT INTO relay_control_connection_reservations
+ (reservation_id, idempotency_key, user_id, relay_host_id,
+ assignment_epoch, cell_id, state, inclusion_watermark,
+ claim_activity_id, created_at, timeout_at, claimed_at, released_at,
+ updated_at)
+ VALUES (?, ?, ?, ?, ?, ?, 'late-arrival-debt', NULL, NULL, ?, ?, NULL, NULL, ?)`,
+ [
+ 'duplicate-reservation',
+ 'duplicate-reservation',
+ identity.userId,
+ identity.relayHostId,
+ assignment.assignmentEpoch,
+ LIMITED_CELL.id,
+ 101,
+ now - 1,
+ now
+ ]
+ )
+
+ await store.recordCellHeartbeat({
+ cellId: LIMITED_CELL.id,
+ cellUrl: LIMITED_CELL.url,
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ startedAt: 50,
+ ready: true,
+ observedRequests: 0,
+ totalConnections: 449,
+ inFlightConnections: 0,
+ reservedConnectionUnits: 0,
+ enforcedConnectionUnits: 449,
+ connectionInclusionWatermark: 5,
+ connectionHardCap: 600,
+ connectionUnobservedBound: 50
+ })
+
+ expect(
+ await database.query(
+ `SELECT state, COUNT(*) AS count
+ FROM relay_control_connection_reservations
+ GROUP BY state ORDER BY state`
+ )
+ ).toEqual([
+ { state: 'late-arrival-debt', count: 1 },
+ { state: 'released', count: 1 }
+ ])
+ })
+
+ it('releases expired debt for a snapshot-proven aborted older epoch', async () => {
+ const now = 1_000
+ const { database, store } = await setup(0, () => now)
+ const identity = { userId: 'aborted-user', relayHostId: 'abortedhost00001' }
+ await database.query(
+ `INSERT INTO relay_assignments
+ (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at,
+ last_activity_at, reserved_controls, reserved_splices, reserved_invites,
+ pending_installs, pending_confirmations, migration_leases)
+ VALUES (?, ?, ?, ?, ?, ?, 0, 0, 0, 0, 0, 0)`,
+ [identity.userId, identity.relayHostId, LIMITED_CELL.id, 3, now, now]
+ )
+ await database.query(
+ `INSERT INTO relay_assignment_migrations
+ (user_id, relay_host_id, source_cell_id, target_cell_id,
+ previous_epoch, assignment_epoch, source_request_units,
+ target_reserved_units, expires_at, target_registered_at,
+ completed_at, aborted_at, created_at, updated_at)
+ VALUES (?, ?, ?, ?, 1, 2, 0, 1, ?, ?, NULL, ?, ?, ?)`,
+ [
+ identity.userId,
+ identity.relayHostId,
+ 'source',
+ LIMITED_CELL.id,
+ now - 2,
+ now - 3,
+ now - 1,
+ now - 4,
+ now - 1
+ ]
+ )
+ await database.query(
+ `INSERT INTO relay_control_connection_reservations
+ (reservation_id, idempotency_key, user_id, relay_host_id,
+ assignment_epoch, cell_id, state, inclusion_watermark,
+ claim_activity_id, created_at, timeout_at, claimed_at, released_at,
+ updated_at)
+ VALUES (?, ?, ?, ?, 2, ?, 'late-arrival-debt', NULL, NULL, ?, ?, NULL, NULL, ?)`,
+ [
+ 'aborted-reservation',
+ 'aborted-reservation',
+ identity.userId,
+ identity.relayHostId,
+ LIMITED_CELL.id,
+ now - 4,
+ now - 2,
+ now - 1
+ ]
+ )
+
+ await store.recordCellHeartbeat({
+ cellId: LIMITED_CELL.id,
+ cellUrl: LIMITED_CELL.url,
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ startedAt: 50,
+ ready: true,
+ observedRequests: 0,
+ totalConnections: 0,
+ inFlightConnections: 0,
+ reservedConnectionUnits: 0,
+ enforcedConnectionUnits: 0,
+ connectionInclusionWatermark: 5,
+ connectionHardCap: 600,
+ connectionUnobservedBound: 50
+ })
+
+ expect(
+ await database.query(
+ `SELECT state FROM relay_control_connection_reservations
+ WHERE reservation_id = ?`,
+ ['aborted-reservation']
+ )
+ ).toEqual([{ state: 'released' }])
+ })
+
+ it('fails a limited cell closed on stale telemetry while leaving legacy cells unchanged', async () => {
+ let now = 100
+ const { store } = await setup(0, () => now)
+ now += 45_001
+ await expect(
+ store.assign({ userId: 'user-1', relayHostId: 'host000000000001' })
+ ).rejects.toThrow('relay_capacity_exhausted')
+
+ const legacyDatabase = await openInMemoryRelayDatabase()
+ databases.push(legacyDatabase)
+ const legacyStore = new RelayAssignmentStore(legacyDatabase, () => now, {
+ requireLiveCells: true,
+ heartbeatTtlMs: 45_000
+ })
+ const legacy = {
+ id: 'legacy',
+ url: 'https://legacy.example.com',
+ capacityRequests: 10
+ }
+ await legacyStore.reconcileCells([legacy], true)
+ await legacyStore.recordCellHeartbeat({
+ cellId: legacy.id,
+ cellUrl: legacy.url,
+ cellIncarnation: '22222222-2222-4222-8222-222222222222',
+ startedAt: now,
+ ready: true,
+ observedRequests: 0
+ })
+ await expect(
+ legacyStore.assign({ userId: 'user-2', relayHostId: 'host000000000002' })
+ ).resolves.toMatchObject({ cellId: 'legacy' })
+ })
+
+ it('does not create connection debt for mixed-version uncapped cells', async () => {
+ let now = 100
+ const legacy = {
+ id: 'legacy',
+ url: 'https://legacy.example.com',
+ capacityRequests: 10
+ }
+ const { database, store } = await setup(0, () => now)
+ await store.reconcileCells([legacy], true)
+ await store.recordCellHeartbeat({
+ cellId: legacy.id,
+ cellUrl: legacy.url,
+ cellIncarnation: '22222222-2222-4222-8222-222222222222',
+ startedAt: 50,
+ ready: true,
+ observedRequests: 0
+ })
+
+ await store.assign({ userId: 'legacy-user', relayHostId: 'legacyhost000001' })
+ expect(
+ await database.query(`SELECT * FROM relay_control_connection_reservations`)
+ ).toEqual([])
+ expect(await store.releaseExpiredActivityLeases()).toBe(0)
+ now += ASSIGNMENT_LIMITS.activityLeaseMs + 1
+ expect(await store.releaseExpiredActivityLeases()).toBe(1)
+ expect(
+ await database.query(`SELECT * FROM relay_control_connection_reservations`)
+ ).toEqual([])
+ })
+
+ it('rejects dormant rebalance before changing the assignment epoch', async () => {
+ const now = ASSIGNMENT_LIMITS.dormantTtlMs + 100
+ const database = await openInMemoryRelayDatabase()
+ databases.push(database)
+ const store = new RelayAssignmentStore(database, () => now, {
+ requireLiveCells: true,
+ heartbeatTtlMs: 45_000
+ })
+ const source = {
+ id: 'source',
+ url: 'https://source.example.com',
+ capacityRequests: 10
+ }
+ await store.reconcileCells([source, LIMITED_CELL], true)
+ await store.recordCellHeartbeat({
+ cellId: source.id,
+ cellUrl: source.url,
+ cellIncarnation: '22222222-2222-4222-8222-222222222222',
+ startedAt: now - 50,
+ ready: true,
+ observedRequests: 0
+ })
+ await store.recordCellHeartbeat({
+ cellId: LIMITED_CELL.id,
+ cellUrl: LIMITED_CELL.url,
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ startedAt: now - 50,
+ ready: true,
+ observedRequests: 0,
+ totalConnections: 450,
+ inFlightConnections: 0,
+ reservedConnectionUnits: 0,
+ enforcedConnectionUnits: 450,
+ connectionHardCap: 600,
+ connectionUnobservedBound: 50
+ })
+ const identity = { userId: 'dormant-user', relayHostId: 'dormanthost00001' }
+ await database.query(
+ `INSERT INTO relay_assignments
+ (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at,
+ last_activity_at, reserved_controls, reserved_splices, reserved_invites,
+ pending_installs, pending_confirmations, migration_leases)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+ [
+ identity.userId,
+ identity.relayHostId,
+ source.id,
+ 7,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ]
+ )
+
+ await expect(store.rebalanceDormant(identity, LIMITED_CELL.id)).rejects.toThrow(
+ 'relay_connection_headroom_exhausted'
+ )
+ expect(await store.resolve(identity)).toMatchObject({
+ cellId: source.id,
+ assignmentEpoch: 7
+ })
+ expect(await database.query(`SELECT * FROM relay_assignment_activity_leases`)).toEqual([])
+ })
+
+ it('rejects an evacuation target at its pause before migration mutation', async () => {
+ const database = await openInMemoryRelayDatabase()
+ databases.push(database)
+ const store = new RelayAssignmentStore(database, () => 100, {
+ requireLiveCells: true,
+ heartbeatTtlMs: 45_000
+ })
+ const source = {
+ id: 'source',
+ url: 'https://source.example.com',
+ capacityRequests: 10
+ }
+ await store.reconcileCells(
+ [source, { ...LIMITED_CELL, initiallyEnabled: false }],
+ true
+ )
+ await store.recordCellHeartbeat({
+ cellId: source.id,
+ cellUrl: source.url,
+ cellIncarnation: '22222222-2222-4222-8222-222222222222',
+ startedAt: 50,
+ ready: true,
+ observedRequests: 0
+ })
+ await store.recordCellHeartbeat({
+ cellId: LIMITED_CELL.id,
+ cellUrl: LIMITED_CELL.url,
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ startedAt: 50,
+ ready: true,
+ observedRequests: 0,
+ totalConnections: 450,
+ inFlightConnections: 0,
+ reservedConnectionUnits: 0,
+ enforcedConnectionUnits: 450,
+ connectionHardCap: 600,
+ connectionUnobservedBound: 50
+ })
+ const identity = { userId: 'user-1', relayHostId: 'host000000000001' }
+ const assignment = await store.assign(identity)
+ expect(assignment.cellId).toBe(source.id)
+ await store.setCellEnabled(LIMITED_CELL.id, true)
+
+ await expect(
+ store.startEvacuation(identity, LIMITED_CELL.id)
+ ).rejects.toThrow('relay_connection_headroom_exhausted')
+ expect(await database.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([])
+ expect(await store.resolve(identity)).toMatchObject({
+ cellId: source.id,
+ assignmentEpoch: assignment.assignmentEpoch
+ })
+ })
+})
diff --git a/cloud/apps/relay/src/assignment-control-supersession-postgres.test.ts b/cloud/apps/relay/src/assignment-control-supersession-postgres.test.ts
new file mode 100644
index 00000000000..cf8819686b5
--- /dev/null
+++ b/cloud/apps/relay/src/assignment-control-supersession-postgres.test.ts
@@ -0,0 +1,134 @@
+import { afterAll, beforeAll, describe, expect, it } from 'vitest'
+import { RelayAssignmentStore } from './assignment-store.js'
+import { openRelayDatabase, type RelayDatabase } from './database.js'
+
+const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
+const describePostgres = databaseUrl ? describe : describe.skip
+const cell = {
+ id: 'control-supersession-postgres',
+ url: 'https://control-supersession-postgres.example.com',
+ capacityRequests: 1_000,
+ connectionHardCap: 600 as const,
+ connectionUnobservedBound: 50
+}
+const identity = {
+ userId: 'control-supersession-postgres-user',
+ relayHostId: 'supersedehost001'
+}
+
+describePostgres('PostgreSQL control supersession', () => {
+ const databases: RelayDatabase[] = []
+
+ beforeAll(async () => {
+ databases.push(
+ await openRelayDatabase({ databaseUrl, dataDir: '' }),
+ await openRelayDatabase({ databaseUrl, dataDir: '' })
+ )
+ })
+
+ afterAll(async () => {
+ const database = databases[0]
+ if (database) {
+ await database.query(
+ `DELETE FROM relay_control_connection_reservations WHERE user_id = ?`,
+ [identity.userId]
+ )
+ await database.query(
+ `DELETE FROM relay_assignment_activity_leases WHERE user_id = ?`,
+ [identity.userId]
+ )
+ await database.query(`DELETE FROM relay_assignments WHERE user_id = ?`, [identity.userId])
+ // A snapshot left by an aborted run rejects the replayed watermark with stale_connection_snapshot.
+ await database.query(`DELETE FROM relay_cell_connection_snapshots WHERE cell_id = ?`, [
+ cell.id
+ ])
+ await database.query(`DELETE FROM relay_cell_connection_runtime WHERE cell_id = ?`, [cell.id])
+ await database.query(`DELETE FROM relay_cell_connection_limits WHERE cell_id = ?`, [cell.id])
+ await database.query(`DELETE FROM relay_cell_runtime WHERE cell_id = ?`, [cell.id])
+ await database.query(`DELETE FROM relay_cells WHERE cell_id = ?`, [cell.id])
+ }
+ for (const connection of databases) await connection.close()
+ })
+
+ it('serializes parallel generations into one durable control', async () => {
+ const stores = databases.map((database) => new RelayAssignmentStore(database, () => 100))
+ await stores[0]!.reconcileCells([cell])
+ await stores[0]!.recordCellHeartbeat({
+ cellId: cell.id,
+ cellUrl: cell.url,
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ startedAt: 50,
+ ready: true,
+ observedRequests: 0,
+ totalConnections: 0,
+ inFlightConnections: 0,
+ reservedConnectionUnits: 0,
+ enforcedConnectionUnits: 0,
+ connectionInclusionWatermark: 1,
+ connectionHardCap: 600,
+ connectionUnobservedBound: 50
+ })
+ const assignment = await stores[0]!.assign(identity)
+
+ await Promise.all([
+ stores[0]!.activateControl(identity, {
+ cellId: cell.id,
+ assignmentEpoch: assignment.assignmentEpoch,
+ generation: 1,
+ connectionInclusionWatermark: 10
+ }),
+ stores[1]!.activateControl(identity, {
+ cellId: cell.id,
+ assignmentEpoch: assignment.assignmentEpoch,
+ generation: 2,
+ connectionInclusionWatermark: 11
+ })
+ ])
+ await databases[0]!.query(
+ `INSERT INTO relay_assignment_activity_leases
+ (user_id, relay_host_id, activity_id, activity_kind, cell_id,
+ request_units, expires_at, updated_at)
+ VALUES (?, ?, ?, 'control', ?, 1, 90100, 100),
+ (?, ?, ?, 'control', ?, 1, 90100, 100)`,
+ [
+ identity.userId,
+ identity.relayHostId,
+ `control:${cell.id}:100`,
+ cell.id,
+ identity.userId,
+ identity.relayHostId,
+ `control:${cell.id}:101`,
+ cell.id
+ ]
+ )
+ await stores[0]!.activateControl(identity, {
+ cellId: cell.id,
+ assignmentEpoch: assignment.assignmentEpoch,
+ generation: 102,
+ connectionInclusionWatermark: 12
+ })
+
+ const controls = await databases[0]!.query(
+ `SELECT COUNT(*) AS count FROM relay_assignment_activity_leases
+ WHERE user_id = ? AND activity_kind = 'control'`,
+ [identity.userId]
+ )
+ expect(Number(controls[0]!.count)).toBe(1)
+ const assignments = await databases[0]!.query(
+ `SELECT reserved_controls FROM relay_assignments WHERE user_id = ?`,
+ [identity.userId]
+ )
+ expect(Number(assignments[0]!.reserved_controls)).toBe(1)
+ const cells = await databases[0]!.query(
+ `SELECT reserved_requests FROM relay_cells WHERE cell_id = ?`,
+ [cell.id]
+ )
+ expect(Number(cells[0]!.reserved_requests)).toBe(1)
+ const claims = await databases[0]!.query(
+ `SELECT claim_activity_id FROM relay_control_connection_reservations
+ WHERE user_id = ? AND state = 'claimed'`,
+ [identity.userId]
+ )
+ expect(claims).toEqual([{ claim_activity_id: `control:${cell.id}:102` }])
+ }, 15_000)
+})
diff --git a/cloud/apps/relay/src/assignment-deployment-status-postgres.test.ts b/cloud/apps/relay/src/assignment-deployment-status-postgres.test.ts
new file mode 100644
index 00000000000..9e8e3b85bd9
--- /dev/null
+++ b/cloud/apps/relay/src/assignment-deployment-status-postgres.test.ts
@@ -0,0 +1,87 @@
+import pg from 'pg'
+import { afterAll, beforeAll, describe, expect, it } from 'vitest'
+import { RelayAssignmentStore } from './assignment-store.js'
+import { openRelayDatabase, type RelayDatabase } from './database.js'
+
+const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
+const describePostgres = databaseUrl ? describe : describe.skip
+const schema = 'relay_deployment_status_test'
+const cell = {
+ id: 'deployment-status-postgres',
+ url: 'https://deployment-status-postgres.example.com',
+ capacityRequests: 20
+}
+
+describePostgres('PostgreSQL deployment status', () => {
+ let database: RelayDatabase | undefined
+
+ beforeAll(async () => {
+ const client = new pg.Client({ connectionString: databaseUrl })
+ await client.connect()
+ try {
+ await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
+ await client.query(`CREATE SCHEMA ${schema}`)
+ } finally {
+ await client.end()
+ }
+ const url = new URL(databaseUrl!)
+ url.searchParams.set('options', `-c search_path=${schema}`)
+ database = await openRelayDatabase({ databaseUrl: url.toString(), dataDir: '' })
+ })
+
+ afterAll(async () => {
+ await database?.close()
+ const client = new pg.Client({ connectionString: databaseUrl })
+ await client.connect()
+ try {
+ await client.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`)
+ } finally {
+ await client.end()
+ }
+ })
+
+ it('classifies pending controls and restart blockers in one PostgreSQL query', async () => {
+ const store = new RelayAssignmentStore(database!, () => 100)
+ await store.reconcileCells([cell])
+ await store.assign({ userId: 'status-user', relayHostId: 'a1b2c3d4e5f6' })
+
+ expect(await store.cellDeploymentStatus(cell.id)).toMatchObject({
+ activityLeases: 1,
+ activityRequestUnits: 1,
+ reservedRequests: 1,
+ restartBlockingActivityLeases: 0,
+ restartBlockingActivityRequestUnits: 0,
+ restartBlockingReservedRequests: 0
+ })
+
+ await database!.query(
+ `INSERT INTO relay_assignment_activity_leases
+ (user_id, relay_host_id, activity_id, activity_kind, cell_id,
+ request_units, expires_at, updated_at)
+ VALUES ('blocking-user', 'b1c2d3e4f5a6', 'invite:test', 'invite', ?, 1, 90100, 100),
+ ('malformed-user', 'c1d2e3f4a5b6', 'control-pending:bad', 'control', ?, 2, 90100, 100)`,
+ [cell.id, cell.id]
+ )
+ await database!.query(
+ `UPDATE relay_cells SET reserved_requests = reserved_requests + 3 WHERE cell_id = ?`,
+ [cell.id]
+ )
+
+ expect(await store.cellDeploymentStatus(cell.id)).toMatchObject({
+ activityLeases: 3,
+ activityRequestUnits: 4,
+ reservedRequests: 4,
+ restartBlockingActivityLeases: 2,
+ restartBlockingActivityRequestUnits: 3,
+ restartBlockingReservedRequests: 3
+ })
+
+ await database!.query(
+ `UPDATE relay_cells SET reserved_requests = 0 WHERE cell_id = ?`,
+ [cell.id]
+ )
+ expect(await store.cellDeploymentStatus(cell.id)).toMatchObject({
+ restartBlockingReservedRequests: -1
+ })
+ })
+})
diff --git a/cloud/apps/relay/src/assignment-identity-queue.test.ts b/cloud/apps/relay/src/assignment-identity-queue.test.ts
new file mode 100644
index 00000000000..930f0b692df
--- /dev/null
+++ b/cloud/apps/relay/src/assignment-identity-queue.test.ts
@@ -0,0 +1,70 @@
+import { describe, expect, it } from 'vitest'
+import { AssignmentIdentityQueue } from './assignment-identity-queue.js'
+
+function deferred(): {
+ promise: Promise
+ resolve: () => void
+} {
+ let resolve!: () => void
+ const promise = new Promise((complete) => {
+ resolve = complete
+ })
+ return { promise, resolve }
+}
+
+describe('AssignmentIdentityQueue', () => {
+ it('serializes operations for the same assignment', async () => {
+ const queue = new AssignmentIdentityQueue()
+ const firstStarted = deferred()
+ const firstRelease = deferred()
+ const started: string[] = []
+ const identity = { userId: 'user-1', relayHostId: 'host-1' }
+
+ const first = queue.run(identity, async () => {
+ started.push('first')
+ firstStarted.resolve()
+ await firstRelease.promise
+ })
+ const second = queue.run(identity, async () => {
+ started.push('second')
+ })
+
+ await firstStarted.promise
+ expect(started).toEqual(['first'])
+ firstRelease.resolve()
+ await Promise.all([first, second])
+ expect(started).toEqual(['first', 'second'])
+ })
+
+ it('allows different assignments to run concurrently', async () => {
+ const queue = new AssignmentIdentityQueue()
+ const release = deferred()
+ const started: string[] = []
+
+ const first = queue.run({ userId: 'user-1', relayHostId: 'host-1' }, async () => {
+ started.push('first')
+ await release.promise
+ })
+ const second = queue.run({ userId: 'user-2', relayHostId: 'host-1' }, async () => {
+ started.push('second')
+ })
+
+ await second
+ expect(started).toEqual(['first', 'second'])
+ release.resolve()
+ await first
+ })
+
+ it('continues after a rejected operation', async () => {
+ const queue = new AssignmentIdentityQueue()
+ const identity = { userId: 'user-1', relayHostId: 'host-1' }
+
+ const failed = queue.run(identity, async () => {
+ throw new Error('failed')
+ })
+ const recovered = queue.run(identity, async () => 'recovered')
+
+ await expect(failed).rejects.toThrow('failed')
+ await expect(recovered).resolves.toBe('recovered')
+ })
+})
diff --git a/cloud/apps/relay/src/assignment-identity-queue.ts b/cloud/apps/relay/src/assignment-identity-queue.ts
new file mode 100644
index 00000000000..f39d9c0a03c
--- /dev/null
+++ b/cloud/apps/relay/src/assignment-identity-queue.ts
@@ -0,0 +1,24 @@
+export type AssignmentIdentity = {
+ userId: string
+ relayHostId: string
+}
+
+export class AssignmentIdentityQueue {
+ private readonly tails = new Map>()
+
+ async run(identity: AssignmentIdentity, operation: () => Promise): Promise {
+ const key = JSON.stringify([identity.userId, identity.relayHostId])
+ const previous = this.tails.get(key) ?? Promise.resolve()
+ const result = previous.catch(() => undefined).then(operation)
+ const tail = result.then(
+ () => undefined,
+ () => undefined
+ )
+ this.tails.set(key, tail)
+ try {
+ return await result
+ } finally {
+ if (this.tails.get(key) === tail) this.tails.delete(key)
+ }
+ }
+}
diff --git a/cloud/apps/relay/src/assignment-inventory-snapshot.test.ts b/cloud/apps/relay/src/assignment-inventory-snapshot.test.ts
new file mode 100644
index 00000000000..afc35f7e601
--- /dev/null
+++ b/cloud/apps/relay/src/assignment-inventory-snapshot.test.ts
@@ -0,0 +1,107 @@
+import { describe, expect, it } from 'vitest'
+import {
+ formatAssignmentInventorySnapshot,
+ readAssignmentInventorySnapshot
+} from './assignment-inventory-snapshot.js'
+import { openInMemoryRelayDatabase } from './database.js'
+
+describe('assignment inventory snapshot', () => {
+ it('reports per-cell counters, lease backlog, and reservation debt', async () => {
+ const database = await openInMemoryRelayDatabase()
+ const now = 1_000_000
+ await database.query(
+ `INSERT INTO relay_cells
+ (cell_id, cell_url, enabled, capacity_requests, reserved_requests,
+ observed_requests, last_heartbeat_at, updated_at)
+ VALUES ('cell-a', 'https://a.example.test', 1, 4000, 3999, 5, ?, ?)`,
+ [now, now]
+ )
+ await database.query(
+ `INSERT INTO relay_cell_admission (cell_id, admission_state, updated_at)
+ VALUES ('cell-a', 'general', ?)`,
+ [now]
+ )
+ await database.query(
+ `INSERT INTO relay_cell_runtime
+ (cell_id, cell_url, cell_incarnation, started_at, ready, observed_requests,
+ last_heartbeat_at, updated_at)
+ VALUES ('cell-a', 'https://a.example.test', 'inc-1', ?, 1, 5, ?, ?)`,
+ [now - 60_000, now - 10_000, now]
+ )
+ await database.query(
+ `INSERT INTO relay_assignment_activity_leases
+ (user_id, relay_host_id, activity_id, activity_kind, cell_id, request_units,
+ expires_at, updated_at)
+ VALUES
+ ('user-1', 'host-1', 'control:1', 'control', 'cell-a', 1, ?, ?),
+ ('user-1', 'host-2', 'control:1', 'control', 'cell-a', 3, ?, ?)`,
+ [now - 1, now, now + 90_000, now]
+ )
+ await database.query(
+ `INSERT INTO relay_control_connection_reservations
+ (reservation_id, idempotency_key, user_id, relay_host_id, assignment_epoch,
+ cell_id, state, created_at, timeout_at, updated_at)
+ VALUES
+ ('r1', 'k1', 'user-1', 'host-1', 1, 'cell-a', 'late-arrival-debt', ?, ?, ?),
+ ('r2', 'k2', 'user-1', 'host-2', 1, 'cell-a', 'reserved', ?, ?, ?),
+ ('r3', 'k3', 'user-1', 'host-3', 1, 'cell-a', 'claimed', ?, ?, ?),
+ ('r4', 'k4', 'user-1', 'host-4', 1, 'cell-a', 'released', ?, ?, ?)`,
+ [now, now, now, now, now, now, now, now, now, now, now, now]
+ )
+
+ const snapshot = await readAssignmentInventorySnapshot(database, now)
+
+ expect(snapshot.cells).toEqual([
+ {
+ cellId: 'cell-a',
+ region: 'us-central1',
+ admissionState: 'general',
+ enabled: true,
+ capacityRequests: 4000,
+ reservedRequests: 3999,
+ runtimeReady: true,
+ heartbeatAgeMs: 10_000
+ }
+ ])
+ expect(snapshot.activityLeases).toEqual({ total: 2, expired: 1, requestUnits: 4 })
+ expect(snapshot.connectionReservations).toEqual({ outstanding: 3, lateArrivalDebt: 1 })
+ expect(snapshot.regionalRehomes).toEqual({
+ active: 0,
+ awaitingReceipt: 0,
+ targetRegistered: 0,
+ completedLast24Hours: 0,
+ abortedLast24Hours: 0,
+ oldestActiveAgeMs: null
+ })
+
+ const lines = formatAssignmentInventorySnapshot(snapshot)
+ expect(lines).toHaveLength(3)
+ expect(lines[0]).toContain('cellId=cell-a')
+ expect(lines[0]).toContain('reserved=3999')
+ expect(lines[1]).toContain('expiredLeases=1')
+ expect(lines[1]).toContain('lateArrivalDebt=1')
+ expect(lines[2]).toContain('active=0')
+ })
+
+ it('reports cells missing runtime and admission rows without failing', async () => {
+ const database = await openInMemoryRelayDatabase()
+ await database.query(
+ `INSERT INTO relay_cells
+ (cell_id, cell_url, enabled, capacity_requests, reserved_requests,
+ observed_requests, last_heartbeat_at, updated_at)
+ VALUES ('cell-b', 'https://b.example.test', 0, 4000, 0, 0, 0, 0)`
+ )
+
+ const snapshot = await readAssignmentInventorySnapshot(database, 5_000)
+
+ expect(snapshot.cells[0]).toMatchObject({
+ cellId: 'cell-b',
+ admissionState: 'unset',
+ enabled: false,
+ runtimeReady: null,
+ heartbeatAgeMs: null
+ })
+ expect(snapshot.activityLeases).toEqual({ total: 0, expired: 0, requestUnits: 0 })
+ expect(snapshot.regionalRehomes.active).toBe(0)
+ })
+})
diff --git a/cloud/apps/relay/src/assignment-inventory-snapshot.ts b/cloud/apps/relay/src/assignment-inventory-snapshot.ts
new file mode 100644
index 00000000000..0675bd49b94
--- /dev/null
+++ b/cloud/apps/relay/src/assignment-inventory-snapshot.ts
@@ -0,0 +1,170 @@
+import type { RelayDatabase, SqlRow } from './database.js'
+
+export type CellInventorySnapshotRow = {
+ cellId: string
+ region: string
+ admissionState: string
+ enabled: boolean
+ capacityRequests: number
+ reservedRequests: number
+ runtimeReady: boolean | null
+ heartbeatAgeMs: number | null
+}
+
+export type AssignmentInventorySnapshot = {
+ cells: CellInventorySnapshotRow[]
+ activityLeases: { total: number; expired: number; requestUnits: number }
+ connectionReservations: { outstanding: number; lateArrivalDebt: number }
+ regionalRehomes: {
+ active: number
+ awaitingReceipt: number
+ targetRegistered: number
+ completedLast24Hours: number
+ abortedLast24Hours: number
+ oldestActiveAgeMs: number | null
+ }
+}
+
+// Plain SELECTs only: this snapshot must never take the cell-inventory lock,
+// or observability itself would add to the assign-path lock contention.
+export async function readAssignmentInventorySnapshot(
+ database: RelayDatabase,
+ now: number
+): Promise {
+ const cellRows = await database.query(
+ `SELECT cell.cell_id, cell.enabled, cell.capacity_requests, cell.reserved_requests,
+ admission.admission_state, region.region,
+ runtime.ready AS runtime_ready, runtime.last_heartbeat_at AS runtime_heartbeat_at
+ FROM relay_cells cell
+ LEFT JOIN relay_cell_admission admission ON admission.cell_id = cell.cell_id
+ LEFT JOIN relay_cell_regions region ON region.cell_id = cell.cell_id
+ LEFT JOIN relay_cell_runtime runtime ON runtime.cell_id = cell.cell_id
+ ORDER BY cell.cell_id ASC`
+ )
+ const leaseRow = (
+ await database.query(
+ `SELECT COUNT(*) AS total,
+ COALESCE(SUM(CASE WHEN expires_at <= ? THEN 1 ELSE 0 END), 0) AS expired,
+ COALESCE(SUM(request_units), 0) AS request_units
+ FROM relay_assignment_activity_leases`,
+ [now]
+ )
+ )[0]
+ const reservationRow = (
+ await database.query(
+ // relay_cells is append-only in production; joining it lets the composite
+ // index skip released history without hiding any production cell's debt.
+ `SELECT COUNT(reservation.reservation_id) AS outstanding,
+ COALESCE(SUM(CASE WHEN reservation.state = 'late-arrival-debt'
+ THEN 1 ELSE 0 END), 0) AS late_arrival_debt
+ FROM relay_cells cell
+ LEFT JOIN relay_control_connection_reservations reservation
+ ON reservation.cell_id = cell.cell_id
+ AND reservation.state IN ('reserved', 'late-arrival-debt', 'claimed')`
+ )
+ )[0]
+ const regionalRehomeRow = (
+ await database.query(
+ `SELECT
+ COALESCE(SUM(CASE WHEN attempt.completed_at IS NULL AND attempt.aborted_at IS NULL
+ THEN 1 ELSE 0 END), 0) AS active,
+ COALESCE(SUM(CASE WHEN attempt.completed_at IS NULL AND attempt.aborted_at IS NULL
+ AND attempt.drain_receipt_at IS NULL
+ THEN 1 ELSE 0 END), 0) AS awaiting_receipt,
+ COALESCE(SUM(CASE WHEN attempt.completed_at IS NULL AND attempt.aborted_at IS NULL
+ AND migration.target_registered_at IS NOT NULL
+ THEN 1 ELSE 0 END), 0) AS target_registered,
+ COALESCE(SUM(CASE WHEN attempt.completed_at >= ? THEN 1 ELSE 0 END), 0)
+ AS completed_last_24_hours,
+ COALESCE(SUM(CASE WHEN attempt.aborted_at >= ? THEN 1 ELSE 0 END), 0)
+ AS aborted_last_24_hours,
+ MIN(CASE WHEN attempt.completed_at IS NULL AND attempt.aborted_at IS NULL
+ THEN attempt.created_at END) AS oldest_active_at
+ FROM relay_region_rehome_attempts attempt
+ LEFT JOIN relay_assignment_migrations migration
+ ON migration.user_id = attempt.user_id
+ AND migration.relay_host_id = attempt.relay_host_id
+ AND migration.assignment_epoch = attempt.assignment_epoch`,
+ [now - 24 * 60 * 60_000, now - 24 * 60 * 60_000]
+ )
+ )[0]
+ const oldestActiveAt = optionalInteger(regionalRehomeRow, 'oldest_active_at')
+ return {
+ cells: cellRows.map((row) => ({
+ cellId: asText(row, 'cell_id'),
+ region: optionalText(row, 'region') ?? 'us-central1',
+ admissionState: optionalText(row, 'admission_state') ?? 'unset',
+ enabled: asInteger(row, 'enabled') === 1,
+ capacityRequests: asInteger(row, 'capacity_requests'),
+ reservedRequests: asInteger(row, 'reserved_requests'),
+ runtimeReady: row['runtime_ready'] == null ? null : asInteger(row, 'runtime_ready') === 1,
+ heartbeatAgeMs:
+ row['runtime_heartbeat_at'] == null ? null : now - asInteger(row, 'runtime_heartbeat_at')
+ })),
+ activityLeases: {
+ total: asInteger(leaseRow, 'total'),
+ expired: asInteger(leaseRow, 'expired'),
+ requestUnits: asInteger(leaseRow, 'request_units')
+ },
+ connectionReservations: {
+ outstanding: asInteger(reservationRow, 'outstanding'),
+ lateArrivalDebt: asInteger(reservationRow, 'late_arrival_debt')
+ },
+ regionalRehomes: {
+ active: asInteger(regionalRehomeRow, 'active'),
+ awaitingReceipt: asInteger(regionalRehomeRow, 'awaiting_receipt'),
+ targetRegistered: asInteger(regionalRehomeRow, 'target_registered'),
+ completedLast24Hours: asInteger(regionalRehomeRow, 'completed_last_24_hours'),
+ abortedLast24Hours: asInteger(regionalRehomeRow, 'aborted_last_24_hours'),
+ oldestActiveAgeMs: oldestActiveAt === null ? null : now - oldestActiveAt
+ }
+ }
+}
+
+export function formatAssignmentInventorySnapshot(
+ snapshot: AssignmentInventorySnapshot
+): string[] {
+ const lines = snapshot.cells.map(
+ (cell) =>
+ `[orca-relay] cell inventory cellId=${cell.cellId}` +
+ ` region=${cell.region}` +
+ ` admission=${cell.admissionState} enabled=${cell.enabled}` +
+ ` capacity=${cell.capacityRequests} reserved=${cell.reservedRequests}` +
+ ` ready=${cell.runtimeReady ?? 'none'}` +
+ ` heartbeatAgeMs=${cell.heartbeatAgeMs ?? 'none'}`
+ )
+ lines.push(
+ `[orca-relay] lease inventory leases=${snapshot.activityLeases.total}` +
+ ` expiredLeases=${snapshot.activityLeases.expired}` +
+ ` leaseRequestUnits=${snapshot.activityLeases.requestUnits}` +
+ ` outstandingReservations=${snapshot.connectionReservations.outstanding}` +
+ ` lateArrivalDebt=${snapshot.connectionReservations.lateArrivalDebt}`
+ )
+ lines.push(
+ `[orca-relay] regional rehome inventory active=${snapshot.regionalRehomes.active}` +
+ ` awaitingReceipt=${snapshot.regionalRehomes.awaitingReceipt}` +
+ ` targetRegistered=${snapshot.regionalRehomes.targetRegistered}` +
+ ` completedLast24Hours=${snapshot.regionalRehomes.completedLast24Hours}` +
+ ` abortedLast24Hours=${snapshot.regionalRehomes.abortedLast24Hours}` +
+ ` oldestActiveAgeMs=${snapshot.regionalRehomes.oldestActiveAgeMs ?? 'none'}`
+ )
+ return lines
+}
+
+function asText(row: SqlRow | undefined, column: string): string {
+ return String(row?.[column] ?? '')
+}
+
+function optionalText(row: SqlRow | undefined, column: string): string | null {
+ const value = row?.[column]
+ return value == null ? null : String(value)
+}
+
+function asInteger(row: SqlRow | undefined, column: string): number {
+ return Number(row?.[column] ?? 0)
+}
+
+function optionalInteger(row: SqlRow | undefined, column: string): number | null {
+ const value = row?.[column]
+ return value == null ? null : Number(value)
+}
diff --git a/cloud/apps/relay/src/assignment-rejection-log-window.test.ts b/cloud/apps/relay/src/assignment-rejection-log-window.test.ts
new file mode 100644
index 00000000000..8ef078eff86
--- /dev/null
+++ b/cloud/apps/relay/src/assignment-rejection-log-window.test.ts
@@ -0,0 +1,100 @@
+import { describe, expect, it } from 'vitest'
+import { AssignmentRejectionLogWindow } from './assignment-rejection-log-window.js'
+
+describe('assignment rejection log window', () => {
+ it('emits once per key per window and closes it with its own suppressed count', () => {
+ const timers = manualTimers()
+ const closed: { key: string; suppressed: number; sample: string }[] = []
+ const window = new AssignmentRejectionLogWindow({
+ windowMs: 10_000,
+ schedule: timers.schedule,
+ onWindowClosed: (input) => closed.push(input)
+ })
+ const key = 'assign:placement:host-rate-limited'
+
+ expect(window.admit(key, 'host-a')).toBe(true)
+ expect(window.admit(key, 'host-b')).toBe(false)
+ expect(window.admit(key, 'host-c')).toBe(false)
+ expect(closed).toEqual([])
+
+ timers.runPending()
+ expect(closed).toEqual([{ key, suppressed: 2, sample: 'host-c' }])
+
+ // The next window starts clean instead of inheriting the closed window's count.
+ expect(window.admit(key, 'host-d')).toBe(true)
+ timers.runPending()
+ expect(closed).toHaveLength(1)
+ })
+
+ it('reports the final count for a key that goes quiet', () => {
+ const timers = manualTimers()
+ const closed: { key: string; suppressed: number }[] = []
+ const window = new AssignmentRejectionLogWindow({
+ windowMs: 10_000,
+ schedule: timers.schedule,
+ onWindowClosed: ({ key, suppressed }) => closed.push({ key, suppressed })
+ })
+
+ window.admit('assign:sticky:host-rate-limited', 'host-a')
+ window.admit('assign:sticky:host-rate-limited', 'host-a')
+ window.admit('assign:sticky:host-rate-limited', 'host-a')
+ // No further rejection ever arrives for this key.
+ timers.runPending()
+
+ expect(closed).toEqual([{ key: 'assign:sticky:host-rate-limited', suppressed: 2 }])
+ })
+
+ it('keeps distinct keys on independent windows', () => {
+ const timers = manualTimers()
+ const window = new AssignmentRejectionLogWindow({
+ windowMs: 10_000,
+ schedule: timers.schedule,
+ onWindowClosed: () => undefined
+ })
+
+ expect(window.admit('assign:placement:host-rate-limited', 'host-a')).toBe(true)
+ expect(window.admit('assign:placement:queue-full', 'host-a')).toBe(true)
+ expect(window.admit('assign:sticky:host-rate-limited', 'host-a')).toBe(true)
+ expect(window.admit('assign:placement:queue-full', 'host-a')).toBe(false)
+ })
+
+ it('bounds the tracked keys and reports what an evicted window suppressed', () => {
+ const timers = manualTimers()
+ const closed: { key: string; suppressed: number }[] = []
+ const window = new AssignmentRejectionLogWindow({
+ windowMs: 10_000,
+ schedule: timers.schedule,
+ onWindowClosed: ({ key, suppressed }) => closed.push({ key, suppressed })
+ })
+
+ window.admit('key-0', 'host-a')
+ window.admit('key-0', 'host-b')
+ for (let index = 1; index < 200; index++) window.admit(`key-${index}`, 'host-a')
+
+ expect(closed[0]).toEqual({ key: 'key-0', suppressed: 1 })
+ // Evicted keys emit again rather than staying silently suppressed.
+ expect(window.admit('key-0', 'host-a')).toBe(true)
+ expect(window.admit('key-199', 'host-a')).toBe(false)
+ })
+})
+
+function manualTimers(): {
+ schedule: (callback: () => void) => () => void
+ runPending: () => void
+} {
+ const pending = new Map void>()
+ let nextId = 0
+ return {
+ schedule: (callback) => {
+ const id = nextId++
+ pending.set(id, callback)
+ return () => pending.delete(id)
+ },
+ runPending: () => {
+ for (const [id, callback] of [...pending]) {
+ pending.delete(id)
+ callback()
+ }
+ }
+ }
+}
diff --git a/cloud/apps/relay/src/assignment-rejection-log-window.ts b/cloud/apps/relay/src/assignment-rejection-log-window.ts
new file mode 100644
index 00000000000..21ad4e72af9
--- /dev/null
+++ b/cloud/apps/relay/src/assignment-rejection-log-window.ts
@@ -0,0 +1,60 @@
+type CancelWindow = () => void
+
+// Admission rejections run at ~17/s per director instance, so the log summarizes
+// instead of streaming: one line when a key's window opens, then a closing line
+// carrying whatever that same window suppressed. The closing line is driven by the
+// window's own timer, so a key that falls quiet still reports its final count
+// instead of waiting for a rejection that may never arrive.
+const MAX_TRACKED_KEYS = 64
+
+type RejectionWindow = {
+ suppressed: number
+ sample: TSample
+ cancel: CancelWindow
+}
+
+export class AssignmentRejectionLogWindow {
+ private readonly windows = new Map>()
+
+ constructor(
+ private readonly options: {
+ windowMs: number
+ onWindowClosed: (input: { key: string; suppressed: number; sample: TSample }) => void
+ schedule?: (callback: () => void, delayMs: number) => CancelWindow
+ }
+ ) {}
+
+ // Returns true when the caller should log this rejection immediately.
+ admit(key: string, sample: TSample): boolean {
+ const open = this.windows.get(key)
+ if (open) {
+ open.suppressed++
+ // Keep the most recent host/reason so the closing line names a real rejection.
+ open.sample = sample
+ return false
+ }
+ const schedule = this.options.schedule ?? defaultSchedule
+ const window: RejectionWindow = { suppressed: 0, sample, cancel: () => undefined }
+ this.windows.set(key, window)
+ if (this.windows.size > MAX_TRACKED_KEYS) {
+ this.close(this.windows.keys().next().value!)
+ }
+ window.cancel = schedule(() => this.close(key), this.options.windowMs)
+ return true
+ }
+
+ private close(key: string): void {
+ const window = this.windows.get(key)
+ if (!window) return
+ this.windows.delete(key)
+ window.cancel()
+ if (window.suppressed === 0) return
+ this.options.onWindowClosed({ key, suppressed: window.suppressed, sample: window.sample })
+ }
+}
+
+function defaultSchedule(callback: () => void, delayMs: number): CancelWindow {
+ const timer = setTimeout(callback, delayMs)
+ timer.unref?.()
+ return () => clearTimeout(timer)
+}
diff --git a/cloud/apps/relay/src/assignment-rejection-logging.test.ts b/cloud/apps/relay/src/assignment-rejection-logging.test.ts
new file mode 100644
index 00000000000..b00d791fce9
--- /dev/null
+++ b/cloud/apps/relay/src/assignment-rejection-logging.test.ts
@@ -0,0 +1,244 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import type { RelayAssignment } from './assignment-store.js'
+import type { RelayConfig } from './config.js'
+
+const fakes = vi.hoisted(() => ({
+ verifyRelayToken: vi.fn(async (token: string) => ({
+ sub: 'user-1',
+ relayHostId: token
+ }))
+}))
+
+vi.mock('./relay-token-verifier.js', () => ({
+ createRelayTokenVerifier: () => fakes.verifyRelayToken,
+ readBearer: (value: string | undefined) => value?.replace(/^Bearer /, '') ?? null
+}))
+
+import { createRelayApp, relayHostLogDigest } from './app.js'
+
+describe('assignment rejection logging', () => {
+ afterEach(() => {
+ vi.useRealTimers()
+ vi.restoreAllMocks()
+ })
+
+ it('logs the store reason and host digest when assign() rejects on capacity', async () => {
+ const host = 'hhhhhhhhhhhhhhhh'
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+ const app = createRelayApp(config(), {
+ store: {} as never,
+ assignments: {
+ assign: vi.fn(async () => {
+ throw new Error('relay_capacity_exhausted')
+ }),
+ resolve: vi.fn(async () => assignment('cell-r', host))
+ } as never,
+ drain: vi.fn(),
+ ready: vi.fn(async () => true)
+ })
+
+ const hinted = await app.request('/v1/assign', assignmentRequest(host, { reconnect: true }))
+
+ expect(hinted.status).toBe(503)
+ expect(await hinted.json()).toEqual({ error: 'relay_capacity_exhausted' })
+ const line = warn.mock.calls.map((call) => String(call[0])).find((entry) =>
+ entry.includes('assignment rejected')
+ )
+ expect(line).toContain('route=assign')
+ expect(line).toContain('lane=sticky')
+ expect(line).toContain('hinted=true')
+ expect(line).toContain('reason=relay_capacity_exhausted')
+ expect(line).toContain(`host=${relayHostLogDigest(host)}`)
+ expect(line).not.toContain(host)
+ })
+
+ it('logs an unhinted placement rejection without the raw host id', async () => {
+ const host = 'gggggggggggggggg'
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+ const app = createRelayApp(config(), {
+ store: {} as never,
+ assignments: {
+ assign: vi.fn(async () => {
+ throw new Error('relay_connection_headroom_exhausted')
+ }),
+ resolve: vi.fn(async () => null)
+ } as never,
+ drain: vi.fn(),
+ ready: vi.fn(async () => true)
+ })
+
+ const response = await app.request('/v1/assign', assignmentRequest(host))
+
+ expect(response.status).toBe(503)
+ const line = warn.mock.calls.map((call) => String(call[0])).find((entry) =>
+ entry.includes('assignment rejected')
+ )
+ expect(line).toContain('route=assign')
+ expect(line).toContain('lane=placement')
+ expect(line).toContain('hinted=false')
+ expect(line).toContain('reason=relay_connection_headroom_exhausted')
+ expect(line).not.toContain(host)
+ })
+
+ it('names the throttled host once per window and closes it with the suppressed count', async () => {
+ vi.useFakeTimers()
+ const host = 'mmmmmmmmmmmmmmmm'
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+ const app = createRelayApp(config(), {
+ store: {} as never,
+ assignments: {
+ assign: vi.fn(async () => assignment('cell-r', host)),
+ resolve: vi.fn(async () => null)
+ } as never,
+ drain: vi.fn(),
+ ready: vi.fn(async () => true)
+ })
+
+ expect((await app.request('/v1/assign', assignmentRequest(host))).status).toBe(200)
+ expect((await app.request('/v1/assign', assignmentRequest(host))).status).toBe(503)
+ expect((await app.request('/v1/assign', assignmentRequest(host))).status).toBe(503)
+
+ const throttled = (): string[] =>
+ warn.mock.calls
+ .map((call) => String(call[0]))
+ .filter((entry) => entry.includes('reason=host-rate-limited'))
+ expect(throttled()).toHaveLength(1)
+ expect(throttled()[0]).toContain('route=assign')
+ expect(throttled()[0]).toContain('lane=placement')
+ expect(throttled()[0]).toContain(`host=${relayHostLogDigest(host)}`)
+ expect(throttled()[0]).not.toContain('suppressed=')
+ expect(throttled()[0]).not.toContain(host)
+
+ await vi.advanceTimersByTimeAsync(10_000)
+ expect(throttled()).toHaveLength(2)
+ expect(throttled()[1]).toContain('suppressed=1')
+ expect(throttled()[1]).toContain(`host=${relayHostLogDigest(host)}`)
+ })
+
+ it('stays silent for successful assignments', async () => {
+ const host = 'ssssssssssssssss'
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+ const app = createRelayApp(config(), {
+ store: {} as never,
+ assignments: {
+ assign: vi.fn(async () => assignment('cell-r', host)),
+ resolve: vi.fn(async () => null)
+ } as never,
+ drain: vi.fn(),
+ ready: vi.fn(async () => true)
+ })
+
+ expect((await app.request('/v1/assign', assignmentRequest(host))).status).toBe(200)
+ expect(
+ warn.mock.calls.map((call) => String(call[0])).filter((entry) =>
+ entry.includes('assignment rejected')
+ )
+ ).toEqual([])
+ })
+})
+
+function assignmentRequest(
+ relayHostId: string,
+ extra: Record = {}
+): RequestInit & { headers: Record } {
+ return {
+ method: 'POST',
+ headers: {
+ authorization: `Bearer ${relayHostId}`,
+ 'content-type': 'application/json'
+ },
+ body: JSON.stringify({ v: 1, relayHostId, ...extra })
+ }
+}
+
+function assignment(cellId: string, relayHostId: string): RelayAssignment {
+ return {
+ userId: 'user-1',
+ relayHostId,
+ cellId,
+ cellUrl: `https://${cellId}.relay.example.test`,
+ assignmentEpoch: 1,
+ leaseExpiresAt: Date.now() + 300_000
+ }
+}
+
+function config(overrides: Partial = {}): RelayConfig {
+ return {
+ port: 8080,
+ publicUrl: 'https://relay.example.test',
+ cellUrl: 'https://relay.example.test',
+ authIssuer: 'https://auth.example.test',
+ authAudience: 'orca-relay',
+ jwksUrl: 'https://auth.example.test/jwks',
+ assignmentSigningKey: new TextEncoder().encode('assignment-key-with-at-least-32-bytes'),
+ role: 'director',
+ cellId: 'director',
+ cells: [],
+ adminAudience: 'https://relay.example.test/v1/admin/drain',
+ deployServiceAccount: 'deploy@example.test',
+ runtimeServiceAccount: 'runtime@example.test',
+ adminJwksUrl: 'https://auth.example.test/jwks',
+ databasePoolMax: 3,
+ publicAssignmentsEnabled: true,
+ publicAssignmentConcurrency: 2,
+ publicAssignmentQueueMax: 128,
+ publicAssignmentWaitMs: 4_000,
+ publicResolveConcurrency: 1,
+ publicResolveWaitMs: 5_000,
+ publicAssignmentRetryAfterSeconds: 5,
+ dataDir: './data',
+ ...overrides
+ }
+}
+
+describe('assignment grant logging', () => {
+ afterEach(() => {
+ vi.restoreAllMocks()
+ })
+
+ it('logs sticky grants with cell id and host digest only', async () => {
+ const host = 'gggggggggggggggg'
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+ const app = createRelayApp(config(), {
+ store: {} as never,
+ assignments: {
+ assign: vi.fn(async () => assignment('cell-r', host)),
+ resolve: vi.fn(async () => assignment('cell-r', host))
+ } as never,
+ drain: vi.fn(),
+ ready: vi.fn(async () => true)
+ })
+
+ const response = await app.request('/v1/assign', assignmentRequest(host, { reconnect: true }))
+
+ expect(response.status).toBe(200)
+ const line = warn.mock.calls.map((call) => String(call[0])).find((entry) =>
+ entry.includes('assignment granted')
+ )
+ expect(line).toContain('lane=sticky')
+ expect(line).toContain('cell=cell-r')
+ expect(line).toContain(`host=${relayHostLogDigest(host)}`)
+ expect(line).not.toContain(host)
+ })
+
+ it('does not log unhinted placement grants', async () => {
+ const host = 'pppppppppppppppp'
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+ const app = createRelayApp(config(), {
+ store: {} as never,
+ assignments: {
+ assign: vi.fn(async () => assignment('cell-r', host)),
+ resolve: vi.fn(async () => null)
+ } as never,
+ drain: vi.fn(),
+ ready: vi.fn(async () => true)
+ })
+
+ expect((await app.request('/v1/assign', assignmentRequest(host))).status).toBe(200)
+ expect(
+ warn.mock.calls.map((call) => String(call[0])).filter((entry) =>
+ entry.includes('assignment granted')
+ )
+ ).toEqual([])
+ })
+})
diff --git a/cloud/apps/relay/src/assignment-store-lock-order.test.ts b/cloud/apps/relay/src/assignment-store-lock-order.test.ts
new file mode 100644
index 00000000000..61baedfb1e2
--- /dev/null
+++ b/cloud/apps/relay/src/assignment-store-lock-order.test.ts
@@ -0,0 +1,486 @@
+import { describe, expect, it } from 'vitest'
+import { RelayAssignmentStore } from './assignment-store.js'
+import type { RelayDatabase, RelayLockOptions, SqlRow } from './database.js'
+
+const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+const activityId = 'splice:connection-1'
+
+class LockOrderDatabase implements RelayDatabase {
+ readonly lockedTables: string[] = []
+
+ constructor(
+ private readonly cleanupCandidate: boolean,
+ private readonly currentLeaseExpiresAt: number,
+ private readonly activityLeasePresent = true
+ ) {}
+
+ async query(sql: string): Promise {
+ if (sql.includes('SELECT user_id, relay_host_id, activity_id')) {
+ return this.cleanupCandidate
+ ? [
+ {
+ user_id: identity.userId,
+ relay_host_id: identity.relayHostId,
+ activity_id: activityId
+ }
+ ]
+ : []
+ }
+ if (
+ sql.includes('UPDATE relay_cells SET reserved_requests') &&
+ sql.includes('RETURNING cell_id')
+ ) {
+ this.lockedTables.push('cell')
+ return [{ cell_id: 'cell-a' }]
+ }
+ return [{ changes: 1 }]
+ }
+
+ async queryLocked(sql: string): Promise {
+ if (sql.includes('FROM relay_assignments ')) {
+ this.lockedTables.push('assignment')
+ return [{ cell_id: 'cell-a', assignment_epoch: 1 }]
+ }
+ if (sql.includes('FROM relay_assignment_activity_leases')) {
+ this.lockedTables.push('activity')
+ if (!this.activityLeasePresent) return []
+ return [
+ {
+ user_id: identity.userId,
+ relay_host_id: identity.relayHostId,
+ activity_id: activityId,
+ activity_kind: 'splice',
+ cell_id: 'cell-a',
+ request_units: 2,
+ expires_at: this.currentLeaseExpiresAt
+ }
+ ]
+ }
+ if (sql.trim() === 'SELECT * FROM relay_cells ORDER BY cell_id ASC') {
+ this.lockedTables.push('cell-inventory')
+ return [{ cell_id: 'cell-a', reserved_requests: 3, capacity_requests: 10 }]
+ }
+ if (sql.includes('FROM relay_cells')) {
+ this.lockedTables.push('cell')
+ return [{ reserved_requests: 3, capacity_requests: 10 }]
+ }
+ return []
+ }
+
+ async transaction(operation: (transaction: RelayDatabase) => Promise): Promise {
+ return await operation(this)
+ }
+
+ async close(): Promise {}
+}
+
+class ReassignmentLockOrderDatabase implements RelayDatabase {
+ readonly locks: string[] = []
+
+ async query(sql: string): Promise {
+ if (sql.includes('SELECT cell_id, region FROM relay_cell_regions')) {
+ return ['cell-a', 'cell-b'].map((cell_id) => ({ cell_id, region: 'us-central1' }))
+ }
+ if (sql.includes('SELECT region FROM relay_cell_regions')) {
+ return [{ region: 'us-central1' }]
+ }
+ if (sql.includes('SELECT * FROM relay_cells WHERE cell_id')) {
+ return [cellRow('cell-b', 1)]
+ }
+ if (sql.includes('JOIN relay_cell_runtime') && sql.includes('cell.cell_id = ?')) {
+ return []
+ }
+ if (sql.includes('SELECT cell_id, observed_requests FROM relay_cell_runtime')) {
+ return [{ cell_id: 'cell-a', observed_requests: 0 }]
+ }
+ if (sql.includes('LEFT JOIN relay_cell_admission')) {
+ return [
+ { cell_id: 'cell-a', admission_state: 'general' },
+ { cell_id: 'cell-b', admission_state: 'general' }
+ ]
+ }
+ if (sql.includes('FROM relay_cell_connection_limits')) return []
+ return [{ changes: 1 }]
+ }
+
+ async queryLocked(sql: string, params: unknown[] = []): Promise {
+ if (sql.includes('FROM relay_assignments')) {
+ this.locks.push('assignment')
+ return [
+ {
+ user_id: identity.userId,
+ relay_host_id: identity.relayHostId,
+ cell_id: 'cell-b',
+ assignment_epoch: 1,
+ lease_expires_at: 100,
+ last_activity_at: 100,
+ reserved_controls: 0,
+ reserved_splices: 0,
+ reserved_invites: 0,
+ pending_installs: 0,
+ pending_confirmations: 0,
+ migration_leases: 0
+ }
+ ]
+ }
+ if (sql.trim() === 'SELECT * FROM relay_cells ORDER BY cell_id ASC') {
+ this.locks.push('cell-inventory')
+ return [cellRow('cell-a', 0), cellRow('cell-b', 1)]
+ }
+ if (sql.includes('SELECT * FROM relay_cells WHERE cell_id')) {
+ const cellId = String(params[0])
+ this.locks.push(cellId)
+ return [cellRow(cellId, cellId === 'cell-b' ? 1 : 0)]
+ }
+ return []
+ }
+
+ async transaction(operation: (transaction: RelayDatabase) => Promise): Promise {
+ return await operation(this)
+ }
+
+ async close(): Promise {}
+}
+
+class AggregateCleanupDatabase implements RelayDatabase {
+ failIfUnavailable: boolean | null = null
+
+ async query(): Promise {
+ return [{ changes: 1 }]
+ }
+
+ async queryLocked(
+ sql: string,
+ _params: unknown[] = [],
+ options: RelayLockOptions = {}
+ ): Promise {
+ if (sql.includes('FROM relay_assignments WHERE lease_expires_at')) {
+ this.failIfUnavailable = options.failIfUnavailable ?? false
+ }
+ return []
+ }
+
+ async transaction(operation: (transaction: RelayDatabase) => Promise): Promise {
+ return await operation(this)
+ }
+
+ async close(): Promise {}
+}
+
+class HeartbeatLockDatabase implements RelayDatabase {
+ readonly locks: string[] = []
+ readonly reservationCleanupTransactions: number[] = []
+ legacyHeartbeatWritten = false
+ private transactionNumber = 0
+ private activeTransaction = 0
+
+ constructor(
+ private readonly cleanupIncarnation = '11111111-1111-4111-8111-111111111111'
+ ) {}
+
+ async query(sql: string, params: unknown[] = []): Promise {
+ if (sql.includes('SELECT region FROM relay_cell_regions')) {
+ return [{ cell_id: String(params[0]), region: 'us-central1' }]
+ }
+ if (sql.includes('SELECT * FROM relay_cells WHERE cell_id')) {
+ return [cellRow('cell-a', 0)]
+ }
+ if (sql.includes('UPDATE relay_cells SET observed_requests')) {
+ this.legacyHeartbeatWritten = true
+ }
+ if (sql.includes('UPDATE relay_control_connection_reservations')) {
+ this.reservationCleanupTransactions.push(this.activeTransaction)
+ }
+ return [{ changes: 1 }]
+ }
+
+ async queryLocked(sql: string): Promise {
+ if (sql.includes('FROM relay_cells')) {
+ this.locks.push('cell')
+ return [cellRow('cell-a', 0)]
+ }
+ if (sql.includes('FROM relay_cell_runtime')) {
+ this.locks.push('runtime')
+ return this.activeTransaction === 2
+ ? [{ cell_incarnation: this.cleanupIncarnation }]
+ : []
+ }
+ if (sql.includes('FROM relay_cell_connection_limits')) {
+ this.locks.push('connection-limit')
+ return [{ hard_cap: 600, unobserved_bound: 99 }]
+ }
+ if (sql.includes('FROM relay_cell_connection_snapshots')) {
+ this.locks.push('snapshot')
+ return this.activeTransaction === 2
+ ? [{ cell_incarnation: this.cleanupIncarnation, inclusion_watermark: 0 }]
+ : []
+ }
+ return []
+ }
+
+ async transaction(operation: (transaction: RelayDatabase) => Promise): Promise {
+ this.activeTransaction = ++this.transactionNumber
+ const result = await operation(this)
+ this.activeTransaction = 0
+ return result
+ }
+
+ async close(): Promise {}
+}
+
+class NewAssignmentLockDatabase implements RelayDatabase {
+ readonly inventoryLocks: string[] = []
+ private generalLockFailed = false
+
+ constructor(
+ private failGeneralOnce = false,
+ private readonly assignmentAppearsAfterFailure = false
+ ) {}
+
+ async query(sql: string, params: unknown[] = []): Promise {
+ if (sql.includes('SELECT cell_id, region FROM relay_cell_regions')) {
+ return [
+ { cell_id: 'cell-existing', region: 'us-central1' },
+ { cell_id: 'cell-general', region: 'us-central1' }
+ ]
+ }
+ if (sql.includes('SELECT region FROM relay_cell_regions')) {
+ return [{ cell_id: String(params[0]), region: 'us-central1' }]
+ }
+ if (sql.includes('SELECT cell_id, observed_requests FROM relay_cell_runtime')) {
+ return [{ cell_id: 'cell-general', observed_requests: 0 }]
+ }
+ if (sql.includes('LEFT JOIN relay_cell_admission')) {
+ return [{ cell_id: 'cell-general', admission_state: 'general' }]
+ }
+ if (sql.includes('JOIN relay_cell_runtime') && sql.includes('cell.cell_id = ?')) {
+ return [{ cell_id: 'cell-existing' }]
+ }
+ if (sql.includes('FROM relay_cell_connection_limits')) {
+ return [
+ {
+ cell_id: 'cell-general',
+ hard_cap: 600,
+ unobserved_bound: 99,
+ enforced_connection_units: 0,
+ outstanding_reservations: 0,
+ last_heartbeat_at: 100,
+ connection_incarnation: 'incarnation-a',
+ current_incarnation: 'incarnation-a'
+ }
+ ]
+ }
+ return [{ changes: 1 }]
+ }
+
+ async queryLocked(sql: string): Promise {
+ if (
+ sql.includes('FROM relay_assignments') &&
+ this.assignmentAppearsAfterFailure &&
+ this.generalLockFailed
+ ) {
+ return [
+ {
+ user_id: identity.userId,
+ relay_host_id: identity.relayHostId,
+ cell_id: 'cell-existing',
+ assignment_epoch: 1,
+ lease_expires_at: 100,
+ last_activity_at: 100,
+ reserved_controls: 1,
+ reserved_splices: 0,
+ reserved_invites: 0,
+ pending_installs: 0,
+ pending_confirmations: 0,
+ migration_leases: 0
+ }
+ ]
+ }
+ if (sql.includes('SELECT cell_id FROM relay_cell_admission')) {
+ this.inventoryLocks.push('general')
+ if (this.failGeneralOnce) {
+ this.failGeneralOnce = false
+ this.generalLockFailed = true
+ throw new Error('database_lock_unavailable')
+ }
+ return [cellRow('cell-general', 0)]
+ }
+ if (sql.trim() === 'SELECT * FROM relay_cells ORDER BY cell_id ASC') {
+ this.inventoryLocks.push('all')
+ return [cellRow('cell-existing', 0), cellRow('cell-general', 0)]
+ }
+ if (sql.includes('SELECT * FROM relay_cells WHERE cell_id')) {
+ return [cellRow('cell-general', 0)]
+ }
+ return []
+ }
+
+ async transaction(operation: (transaction: RelayDatabase) => Promise): Promise {
+ return await operation(this)
+ }
+
+ async close(): Promise {}
+}
+
+function cellRow(cellId: string, reservedRequests: number): SqlRow {
+ return {
+ cell_id: cellId,
+ cell_url: `https://${cellId}.example.com`,
+ enabled: 1,
+ capacity_requests: 10,
+ reserved_requests: reservedRequests,
+ observed_requests: 0
+ }
+}
+
+describe('RelayAssignmentStore activity lock order', () => {
+ it('updates the cell only after assignment activity during release', async () => {
+ const database = new LockOrderDatabase(false, 100)
+ const store = new RelayAssignmentStore(database, () => 100)
+
+ await expect(store.releaseActivity(identity, activityId)).resolves.toBe(true)
+ expect(database.lockedTables).toEqual(['assignment', 'activity', 'cell'])
+ })
+
+ it('updates the cell only after inserting new activity', async () => {
+ const database = new LockOrderDatabase(false, 100, false)
+ const store = new RelayAssignmentStore(database, () => 100)
+
+ await expect(
+ store.acquireActivity(identity, { activityId, kind: 'splice', cellId: 'cell-a' })
+ ).resolves.toBeUndefined()
+ expect(database.lockedTables).toEqual(['assignment', 'activity', 'cell'])
+ })
+
+ it('rechecks an expired candidate after taking the assignment lock', async () => {
+ const database = new LockOrderDatabase(true, 101)
+ const store = new RelayAssignmentStore(database, () => 100)
+
+ await expect(store.releaseExpiredActivityLeases()).resolves.toBe(0)
+ expect(database.lockedTables).toEqual(['assignment', 'activity'])
+ })
+
+ it('fails fast before aggregate cleanup waits on mixed-version assignment rows', async () => {
+ const database = new AggregateCleanupDatabase()
+ const store = new RelayAssignmentStore(database, () => 100)
+
+ await expect(store.releaseExpiredActivity()).resolves.toBe(0)
+ expect(database.failIfUnavailable).toBe(true)
+ })
+
+ it('releases the placement capacity lock before heartbeat reservation cleanup', async () => {
+ const database = new HeartbeatLockDatabase()
+ const store = new RelayAssignmentStore(database, () => 100, { requireLiveCells: true })
+
+ await store.recordCellHeartbeat({
+ cellId: 'cell-a',
+ cellUrl: 'https://cell-a.example.com',
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ startedAt: 50,
+ ready: true,
+ observedRequests: 0,
+ totalConnections: 0,
+ inFlightConnections: 0,
+ reservedConnectionUnits: 0,
+ enforcedConnectionUnits: 0,
+ connectionHardCap: 600,
+ connectionUnobservedBound: 99
+ })
+
+ expect(database.locks).toEqual([
+ 'cell',
+ 'runtime',
+ 'connection-limit',
+ 'snapshot',
+ 'runtime',
+ 'snapshot'
+ ])
+ expect(database.legacyHeartbeatWritten).toBe(false)
+ expect(database.reservationCleanupTransactions).toEqual([2, 2, 2])
+ })
+
+ it('does not let an old heartbeat clean replacement-incarnation reservations', async () => {
+ const database = new HeartbeatLockDatabase('22222222-2222-4222-8222-222222222222')
+ const store = new RelayAssignmentStore(database, () => 100, { requireLiveCells: true })
+
+ await store.recordCellHeartbeat({
+ cellId: 'cell-a',
+ cellUrl: 'https://cell-a.example.com',
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ startedAt: 50,
+ ready: true,
+ observedRequests: 0,
+ totalConnections: 0,
+ inFlightConnections: 0,
+ reservedConnectionUnits: 0,
+ enforcedConnectionUnits: 0,
+ connectionHardCap: 600,
+ connectionUnobservedBound: 99
+ })
+
+ expect(database.reservationCleanupTransactions).toEqual([])
+ })
+
+ it('locks only general-admission inventory for a brand-new assignment', async () => {
+ const database = new NewAssignmentLockDatabase()
+ const store = new RelayAssignmentStore(database, () => 100, {
+ requireLiveCells: true,
+ heartbeatTtlMs: 45
+ })
+
+ await expect(store.assign(identity)).resolves.toMatchObject({
+ cellId: 'cell-general',
+ assignmentEpoch: 1
+ })
+ expect(database.inventoryLocks).toEqual(['general'])
+ })
+
+ it('keeps a brand-new assignment retry scoped to general admission', async () => {
+ const database = new NewAssignmentLockDatabase(true)
+ const store = new RelayAssignmentStore(database, () => 100, {
+ requireLiveCells: true,
+ heartbeatTtlMs: 45
+ })
+
+ await expect(store.assign(identity)).resolves.toMatchObject({
+ cellId: 'cell-general',
+ assignmentEpoch: 1
+ })
+ expect(database.inventoryLocks).toEqual(['general', 'general'])
+ })
+
+ it('restarts with full inventory if an assignment appears during a general retry', async () => {
+ const database = new NewAssignmentLockDatabase(true, true)
+ const store = new RelayAssignmentStore(database, () => 100, {
+ requireLiveCells: true,
+ heartbeatTtlMs: 45
+ })
+
+ await expect(store.assign(identity)).resolves.toMatchObject({
+ cellId: 'cell-existing',
+ assignmentEpoch: 1
+ })
+ expect(database.inventoryLocks).toEqual(['general', 'general', 'all'])
+ })
+
+ it('probes the sticky cell before locking full inventory for dead-cell reassignment', async () => {
+ const database = new ReassignmentLockOrderDatabase()
+ const store = new RelayAssignmentStore(database, () => 100, {
+ requireLiveCells: true,
+ heartbeatTtlMs: 45
+ })
+
+ await expect(store.assign(identity)).resolves.toMatchObject({
+ cellId: 'cell-a',
+ assignmentEpoch: 2
+ })
+ expect(database.locks).toEqual([
+ 'assignment',
+ 'cell-b',
+ 'assignment',
+ 'cell-inventory',
+ 'cell-b',
+ 'cell-a'
+ ])
+ })
+})
diff --git a/cloud/apps/relay/src/assignment-store.test.ts b/cloud/apps/relay/src/assignment-store.test.ts
new file mode 100644
index 00000000000..e9d27a83db4
--- /dev/null
+++ b/cloud/apps/relay/src/assignment-store.test.ts
@@ -0,0 +1,4471 @@
+import { ASSIGNMENT_LIMITS } from '@orca-cloud/relay-contract'
+import { createHash } from 'node:crypto'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import type { RelayCellConfig } from './config.js'
+import { RelayAssignmentStore, STRANDED_MIGRATION_ABANDON_MS } from './assignment-store.js'
+import {
+ encodeMembership,
+ type CellAdmissionMembership
+} from './cell-admission-selector.js'
+import {
+ openInMemoryRelayDatabase,
+ type RelayDatabase,
+ type RelayLockOptions,
+ type RelayTransactionOptions,
+ type SqlRow
+} from './database.js'
+
+const CELLS: RelayCellConfig[] = [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 2 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 2 }
+]
+const NO_EXPIRED_EVACUATION_DIAGNOSTICS = {
+ expiredUnregistered: 0,
+ repairableExpiredUnregistered: 0,
+ abortableExpiredUnregistered: 0,
+ blockedExpiredUnregistered: 0,
+ blockedExpiredOnNewerTargetAssignment: 0
+}
+const NO_REGISTERED_EVACUATION_DIAGNOSTICS = {
+ registeredSourceActive: 0,
+ registeredCompletable: 0,
+ registeredTargetInactive: 0
+}
+const NO_ACTIVE_MIGRATION_LEASE = {
+ oldestExpiresAt: null,
+ oldestRemainingMs: null
+}
+const FENCE_PLAN_BINDING = {
+ planObjectName:
+ 'terraform/state/relay-fence-plans/production/22222222-2222-4222-8222-222222222222.tfplan',
+ planObjectGeneration: '123456789',
+ varFileSha256: 'c'.repeat(64),
+ terraformStateLineage: '33333333-3333-4333-8333-333333333333',
+ terraformStateSerial: 7,
+ terraformStateObjectGeneration: '987654321',
+ terraformStateObjectSha256: 'd'.repeat(64),
+ requestReason:
+ 'orca-relay-fence/22222222-2222-4222-8222-222222222222'
+}
+const FENCE_INVOCATION_ID = '55555555-5555-4555-8555-555555555555'
+const FENCE_INVOCATION_REASON =
+ `${FENCE_PLAN_BINDING.requestReason}/${FENCE_INVOCATION_ID}`
+
+async function applyGenerationZeroSelector(
+ store: RelayAssignmentStore,
+ input: {
+ attemptId: string
+ membership: CellAdmissionMembership
+ }
+) {
+ const current = (await store.inspectCellAdmissionSelector()).selector.membership
+ return await store.applyCellAdmissionSelector({
+ ...input,
+ expectedGeneration: 0,
+ expectedMembershipSha256: createHash('sha256')
+ .update(encodeMembership(current))
+ .digest('hex')
+ })
+}
+
+function cellFenceEvidence(attemptId = '22222222-2222-4222-8222-222222222222') {
+ return {
+ attemptId,
+ environment: 'production' as const,
+ cellId: 'cell-a',
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ migName: 'orca-relay-c1',
+ instanceGroup: 'https://compute.example/instanceGroups/orca-relay-c1',
+ generationIdentity: 'https://compute.example/instanceTemplates/orca-relay-c1-abc',
+ fenceCommit: 'a'.repeat(40),
+ planSha256: 'b'.repeat(64),
+ ...FENCE_PLAN_BINDING
+ }
+}
+
+class OneShotInventoryFailureDatabase implements RelayDatabase {
+ readonly retryLocks: string[] = []
+ private armed = false
+ private failed = false
+
+ constructor(private readonly delegate: RelayDatabase) {}
+
+ arm(): void {
+ this.armed = true
+ }
+
+ async query(sql: string, params: unknown[] = []): Promise {
+ return await this.delegate.query(sql, params)
+ }
+
+ async queryLocked(
+ sql: string,
+ params: unknown[] = [],
+ options: RelayLockOptions = {}
+ ): Promise {
+ return await this.lockedQuery(this.delegate, sql, params, options)
+ }
+
+ async transaction(operation: (transaction: RelayDatabase) => Promise): Promise {
+ return await this.delegate.transaction(
+ async (transaction) =>
+ await operation({
+ query: async (sql, params = []) => await transaction.query(sql, params),
+ queryLocked: async (sql, params = [], options = {}) =>
+ await this.lockedQuery(transaction, sql, params, options),
+ transaction: async (nested) => await transaction.transaction(nested),
+ close: async () => {}
+ })
+ )
+ }
+
+ async close(): Promise {
+ await this.delegate.close()
+ }
+
+ private async lockedQuery(
+ database: RelayDatabase,
+ sql: string,
+ params: unknown[],
+ options: RelayLockOptions
+ ): Promise {
+ const inventory = sql.trim() === 'SELECT * FROM relay_cells ORDER BY cell_id ASC'
+ if (this.armed && inventory && options.failIfUnavailable) {
+ this.armed = false
+ this.failed = true
+ this.retryLocks.push('inventory-nowait-failed')
+ throw new Error('database_lock_unavailable')
+ }
+ if (this.failed && inventory) this.retryLocks.push('inventory-first')
+ if (this.failed && sql.includes('FROM relay_assignments')) {
+ this.retryLocks.push(options.failIfUnavailable ? 'assignment-nowait' : 'assignment')
+ }
+ return await database.queryLocked(sql, params, options)
+ }
+}
+
+class RepeatedDrainAccountingFailureDatabase implements RelayDatabase {
+ refreshAttempts = 0
+ private armed = false
+ private failuresRemaining = 0
+
+ constructor(private readonly delegate: RelayDatabase) {}
+
+ arm(failures: number): void {
+ this.armed = true
+ this.failuresRemaining = failures
+ this.refreshAttempts = 0
+ }
+
+ async query(sql: string, params: unknown[] = []): Promise {
+ return await this.delegate.query(sql, params)
+ }
+
+ async queryLocked(
+ sql: string,
+ params: unknown[] = [],
+ options: RelayLockOptions = {}
+ ): Promise {
+ return await this.lockedQuery(this.delegate, sql, params, options)
+ }
+
+ async transaction(operation: (transaction: RelayDatabase) => Promise): Promise {
+ return await this.delegate.transaction(
+ async (transaction) =>
+ await operation({
+ query: async (sql, params = []) => await transaction.query(sql, params),
+ queryLocked: async (sql, params = [], options = {}) =>
+ await this.lockedQuery(transaction, sql, params, options),
+ transaction: async (nested) => await transaction.transaction(nested),
+ close: async () => {}
+ })
+ )
+ }
+
+ async close(): Promise {
+ await this.delegate.close()
+ }
+
+ private async lockedQuery(
+ database: RelayDatabase,
+ sql: string,
+ params: unknown[],
+ options: RelayLockOptions
+ ): Promise {
+ const drainRefresh =
+ sql.includes('SELECT migration.*') &&
+ sql.includes('WHERE migration.source_cell_id = ?') &&
+ !sql.includes('source_cell_incarnation')
+ if (this.armed && drainRefresh) {
+ this.refreshAttempts++
+ if (this.failuresRemaining > 0) {
+ this.failuresRemaining--
+ throw new Error('migration_activity_accounting_mismatch')
+ }
+ }
+ return await database.queryLocked(sql, params, options)
+ }
+}
+
+// Runs a side effect between the sticky lane and the placement lane, the window
+// in which a host can acquire control after the sticky read called it dormant.
+class SeedBetweenAssignmentLanesDatabase implements RelayDatabase {
+ private transactions = 0
+ private armed = false
+
+ constructor(
+ private readonly delegate: RelayDatabase,
+ private readonly seed: () => Promise
+ ) {}
+
+ arm(): void {
+ this.armed = true
+ this.transactions = 0
+ }
+
+ async query(sql: string, params: unknown[] = []): Promise {
+ return await this.delegate.query(sql, params)
+ }
+
+ async queryLocked(
+ sql: string,
+ params: unknown[] = [],
+ options: RelayLockOptions = {}
+ ): Promise {
+ return await this.delegate.queryLocked(sql, params, options)
+ }
+
+ async transaction(
+ operation: (transaction: RelayDatabase) => Promise,
+ options?: RelayTransactionOptions
+ ): Promise {
+ if (this.armed && ++this.transactions === 2) await this.seed()
+ return await this.delegate.transaction(operation, options)
+ }
+
+ async close(): Promise {
+ await this.delegate.close()
+ }
+}
+
+describe('RelayAssignmentStore', () => {
+ let database: RelayDatabase | undefined
+
+ afterEach(async () => await database?.close())
+
+ async function setup(now: () => number, cells = CELLS): Promise {
+ database = await openInMemoryRelayDatabase()
+ const store = new RelayAssignmentStore(database, now)
+ await store.reconcileCells(cells)
+ return store
+ }
+
+ async function setupWithHeartbeats(
+ now: () => number,
+ cells = CELLS
+ ): Promise {
+ database = await openInMemoryRelayDatabase()
+ const store = new RelayAssignmentStore(database, now, {
+ requireLiveCells: true,
+ heartbeatTtlMs: 45_000
+ })
+ await store.reconcileCells(cells)
+ return store
+ }
+
+ async function heartbeat(
+ store: RelayAssignmentStore,
+ cell = CELLS[0]!,
+ input: {
+ incarnation?: string
+ startedAt?: number
+ ready?: boolean
+ observedRequests?: number
+ } = {}
+ ): Promise {
+ await store.recordCellHeartbeat({
+ cellId: cell.id,
+ cellUrl: cell.url,
+ cellIncarnation: input.incarnation ?? '11111111-1111-4111-8111-111111111111',
+ startedAt: input.startedAt ?? 50,
+ ready: input.ready ?? true,
+ observedRequests: input.observedRequests ?? 0,
+ ...(cell.connectionHardCap === undefined
+ ? {}
+ : {
+ totalConnections: 0,
+ inFlightConnections: 0,
+ reservedConnectionUnits: 0,
+ enforcedConnectionUnits: 0,
+ connectionHardCap: cell.connectionHardCap,
+ connectionUnobservedBound: cell.connectionUnobservedBound
+ })
+ })
+ }
+
+ it('admits only cells with a fresh ready heartbeat', async () => {
+ let now = 100
+ const store = await setupWithHeartbeats(() => now, [CELLS[0]!])
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+
+ await expect(store.assign(identity)).rejects.toThrow('relay_capacity_exhausted')
+ await heartbeat(store, CELLS[0]!, { ready: false })
+ await expect(store.assign(identity)).rejects.toThrow('relay_capacity_exhausted')
+ await heartbeat(store)
+ expect((await store.assign(identity)).cellId).toBe('cell-a')
+ now += 45_001
+ expect(await store.resolve(identity)).toBeNull()
+ })
+
+ it('fences stale incarnations and origin mismatches', async () => {
+ const store = await setupWithHeartbeats(() => 100, [CELLS[0]!])
+ await heartbeat(store, CELLS[0]!, { startedAt: 50 })
+ await expect(
+ heartbeat(store, { ...CELLS[0]!, url: 'https://other.example.com' })
+ ).rejects.toThrow('cell_origin_mismatch')
+ await expect(
+ heartbeat(store, CELLS[0]!, {
+ incarnation: '22222222-2222-4222-8222-222222222222',
+ startedAt: 50
+ })
+ ).rejects.toThrow('stale_cell_incarnation')
+ await heartbeat(store, CELLS[0]!, {
+ incarnation: '22222222-2222-4222-8222-222222222222',
+ startedAt: 51
+ })
+ await expect(heartbeat(store, CELLS[0]!, { startedAt: 50 })).rejects.toThrow(
+ 'stale_cell_incarnation'
+ )
+ })
+
+ it('records receipt-relative legacy drain states and pins post-send migrations', async () => {
+ let now = 100
+ const store = await setupWithHeartbeats(() => now)
+ await heartbeat(store, CELLS[0]!)
+ await heartbeat(store, CELLS[1]!, {
+ incarnation: '22222222-2222-4222-8222-222222222222'
+ })
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ await store.assign(identity)
+ await store.setCellEnabled('cell-a', false)
+ const migration = await store.startEvacuation(identity, 'cell-b')
+ await database!.query(
+ `UPDATE relay_assignments SET migration_leases = migration_leases + 1
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ const incarnation = '11111111-1111-4111-8111-111111111111'
+ const attemptId = '33333333-3333-4333-8333-333333333333'
+ const traceValue = '44444444-4444-4444-8444-444444444444'
+
+ await expect(
+ store.prepareCellDrainAttempt({
+ attemptId,
+ cellId: 'cell-a',
+ cellIncarnation: incarnation,
+ traceValue,
+ plannedGraceMs: 120_000
+ })
+ ).resolves.toMatchObject({ state: 'prepared', shouldSend: false })
+ expect(
+ await database!.query(`SELECT * FROM relay_post_drain_migration_pins`)
+ ).toEqual([])
+ await expect(
+ store.prepareCellDrainRecovery({
+ attemptId,
+ cellId: 'cell-a',
+ cellIncarnation: incarnation
+ })
+ ).resolves.toMatchObject({
+ shouldSend: false,
+ preparedAttempt: {
+ attemptId,
+ state: 'prepared',
+ traceValue,
+ plannedGraceMs: 120_000
+ }
+ })
+
+ await expect(
+ store.beginCellDrainSend({
+ attemptId,
+ cellId: 'cell-a',
+ cellIncarnation: incarnation
+ })
+ ).resolves.toMatchObject({
+ state: 'send-may-have-started',
+ shouldSend: true,
+ sendPermitExpiresAt: 30_100
+ })
+ await expect(
+ database!.query(
+ `SELECT migration_leases FROM relay_assignments
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).resolves.toEqual([{ migration_leases: 1 }])
+ await expect(
+ store.beginCellDrainSend({
+ attemptId,
+ cellId: 'cell-a',
+ cellIncarnation: incarnation
+ })
+ ).resolves.toMatchObject({
+ state: 'send-may-have-started',
+ shouldSend: false
+ })
+ await expect(
+ store.prepareCellDrainRecovery({
+ attemptId,
+ cellId: 'cell-a',
+ cellIncarnation: incarnation
+ })
+ ).rejects.toThrow('drain_application_receipt_missing')
+ expect(
+ await database!.query(
+ `SELECT drain_attempt_id, source_cell_id, source_cell_incarnation,
+ target_cell_id, assignment_epoch
+ FROM relay_post_drain_migration_pins`
+ )
+ ).toEqual([
+ {
+ drain_attempt_id: attemptId,
+ source_cell_id: 'cell-a',
+ source_cell_incarnation: incarnation,
+ target_cell_id: 'cell-b',
+ assignment_epoch: migration.assignmentEpoch
+ }
+ ])
+
+ now = migration.expiresAt + 1
+ expect(await store.abortExpiredEvacuations()).toBe(0)
+ expect(await store.releaseExpiredActivityLeases()).toBe(1)
+ expect(
+ await database!.query(
+ `SELECT activity_kind, cell_id FROM relay_assignment_activity_leases
+ WHERE user_id = ? ORDER BY activity_kind`,
+ [identity.userId]
+ )
+ ).toEqual([
+ { activity_kind: 'control', cell_id: 'cell-b' },
+ { activity_kind: 'migration', cell_id: 'cell-b' }
+ ])
+
+ now = 200_000
+ await expect(
+ store.recordCellDrainApplicationReceipt({
+ attemptId,
+ cellId: 'cell-a',
+ cellIncarnation: incarnation,
+ traceValue,
+ backendStatus: 200
+ })
+ ).resolves.toMatchObject({
+ state: 'application-receipt',
+ applicationReceiptAt: 200_000,
+ retryAfter: 350_000
+ })
+ await expect(
+ store.prepareCellDrainRecovery({ attemptId, cellId: 'cell-a', cellIncarnation: incarnation })
+ ).rejects.toThrow('drain_recovery_too_early')
+ now = 350_000
+ await expect(
+ store.prepareCellDrainRecovery({ attemptId, cellId: 'cell-a', cellIncarnation: incarnation })
+ ).resolves.toEqual({
+ shouldSend: true,
+ retryAfter: 350_000
+ })
+ await expect(
+ store.prepareCellDrainRecovery({ attemptId, cellId: 'cell-a', cellIncarnation: incarnation })
+ ).resolves.toEqual({
+ shouldSend: false,
+ retryAfter: 350_000
+ })
+ })
+
+ it('recovers a proven drain once after the source cell is replaced', async () => {
+ let now = 100
+ const store = await setupWithHeartbeats(() => now)
+ const oldIncarnation = '11111111-1111-4111-8111-111111111111'
+ const newIncarnation = '22222222-2222-4222-8222-222222222222'
+ const newerIncarnation = '55555555-5555-4555-8555-555555555555'
+ const attempt = {
+ attemptId: '33333333-3333-4333-8333-333333333333',
+ cellId: 'cell-a',
+ cellIncarnation: oldIncarnation,
+ traceValue: '44444444-4444-4444-8444-444444444444',
+ plannedGraceMs: 120_000
+ }
+ await heartbeat(store)
+ await store.setCellEnabled('cell-a', false)
+ await store.prepareCellDrainAttempt(attempt)
+ await store.beginCellDrainSend(attempt)
+ now = 200
+ await store.recordCellDrainApplicationReceipt({
+ ...attempt,
+ backendStatus: 200
+ })
+
+ now = 150_200
+ await heartbeat(store, CELLS[0]!, {
+ incarnation: newIncarnation,
+ startedAt: 51
+ })
+ await expect(
+ store.prepareCellDrainRecovery({
+ cellId: 'cell-a',
+ cellIncarnation: newIncarnation
+ })
+ ).resolves.toEqual({ shouldSend: true, retryAfter: 150_200 })
+ await expect(
+ store.prepareCellDrainRecovery({
+ cellId: 'cell-a',
+ cellIncarnation: newIncarnation
+ })
+ ).resolves.toEqual({ shouldSend: false, retryAfter: 150_200 })
+
+ now = 150_300
+ await heartbeat(store, CELLS[0]!, {
+ incarnation: newerIncarnation,
+ startedAt: 52
+ })
+ const concurrentRecoveries = await Promise.all([
+ store.prepareCellDrainRecovery({
+ cellId: 'cell-a',
+ cellIncarnation: newerIncarnation
+ }),
+ store.prepareCellDrainRecovery({
+ cellId: 'cell-a',
+ cellIncarnation: newerIncarnation
+ })
+ ])
+ expect(concurrentRecoveries.map((recovery) => recovery.shouldSend).sort()).toEqual([
+ false,
+ true
+ ])
+ await expect(
+ store.prepareCellDrainRecovery({
+ cellId: 'cell-a',
+ cellIncarnation: newerIncarnation
+ })
+ ).resolves.toEqual({ shouldSend: false, retryAfter: 150_200 })
+ await expect(
+ database!.query(
+ `SELECT cell_incarnation FROM relay_cell_drain_recovery_attempts
+ WHERE drain_attempt_id = ? ORDER BY cell_incarnation`,
+ [attempt.attemptId]
+ )
+ ).resolves.toEqual([
+ { cell_incarnation: newIncarnation },
+ { cell_incarnation: newerIncarnation }
+ ])
+ })
+
+ it('rejects an invalid receipt and a prepared drain from an old incarnation', async () => {
+ let now = 100
+ const store = await setupWithHeartbeats(() => now)
+ const oldIncarnation = '11111111-1111-4111-8111-111111111111'
+ const newIncarnation = '22222222-2222-4222-8222-222222222222'
+ const attempt = {
+ attemptId: '33333333-3333-4333-8333-333333333333',
+ cellId: 'cell-a',
+ cellIncarnation: oldIncarnation,
+ traceValue: '44444444-4444-4444-8444-444444444444',
+ plannedGraceMs: 120_000
+ }
+ await heartbeat(store)
+ await store.setCellEnabled('cell-a', false)
+ await store.prepareCellDrainAttempt(attempt)
+ now = 150_200
+ await heartbeat(store, CELLS[0]!, {
+ incarnation: newIncarnation,
+ startedAt: 51
+ })
+ await expect(
+ store.prepareCellDrainRecovery({
+ cellId: 'cell-a',
+ cellIncarnation: newIncarnation
+ })
+ ).rejects.toThrow('drain_application_receipt_missing')
+
+ await database!.query(
+ `UPDATE relay_cell_drain_attempt_states
+ SET state = 'application-receipt', application_receipt_at = ?,
+ receipt_cell_incarnation = ?, retry_after = ?
+ WHERE attempt_id = ?`,
+ [200, oldIncarnation, 150_200, attempt.attemptId]
+ )
+ await expect(
+ store.prepareCellDrainRecovery({
+ cellId: 'cell-a',
+ cellIncarnation: newIncarnation
+ })
+ ).rejects.toThrow('drain_application_receipt_missing')
+ })
+
+ it('restores an expired registered migration lease before a prepared drain send', async () => {
+ let now = 100
+ const store = await setupWithHeartbeats(() => now)
+ await heartbeat(store, CELLS[0]!)
+ await heartbeat(store, CELLS[1]!, {
+ incarnation: '22222222-2222-4222-8222-222222222222'
+ })
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ await store.assign(identity)
+ await store.setCellEnabled('cell-a', false)
+ const migration = await store.startEvacuation(identity, 'cell-b')
+ await store.markMigrationTargetRegistered(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch
+ })
+ const attempt = {
+ attemptId: '55555555-5555-4555-8555-555555555555',
+ cellId: 'cell-a',
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ traceValue: '66666666-6666-4666-8666-666666666666',
+ plannedGraceMs: 120_000
+ }
+ await store.prepareCellDrainAttempt(attempt)
+
+ now = migration.expiresAt + 1
+ expect(await store.releaseExpiredActivityLeases()).toBeGreaterThan(0)
+ await heartbeat(store, CELLS[0]!)
+ await heartbeat(store, CELLS[1]!, {
+ incarnation: '22222222-2222-4222-8222-222222222222'
+ })
+ await expect(
+ store.beginCellDrainSend({
+ attemptId: attempt.attemptId,
+ cellId: attempt.cellId,
+ cellIncarnation: attempt.cellIncarnation
+ })
+ ).resolves.toMatchObject({
+ state: 'send-may-have-started',
+ shouldSend: true
+ })
+ await expect(
+ database!.query(
+ `SELECT activity_id, activity_kind, cell_id, request_units, expires_at
+ FROM relay_assignment_activity_leases
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).resolves.toContainEqual({
+ activity_id: `migration:${migration.assignmentEpoch}`,
+ activity_kind: 'migration',
+ cell_id: 'cell-b',
+ request_units: 1,
+ expires_at: now + ASSIGNMENT_LIMITS.migrationLeaseMs
+ })
+ })
+
+ it('refreshes registered migration leases before prepared drain recovery', async () => {
+ let now = 100
+ const cappedCells = CELLS.map((cell) => ({
+ ...cell,
+ connectionHardCap: 600 as const,
+ connectionUnobservedBound: 50
+ }))
+ const store = await setupWithHeartbeats(() => now, cappedCells)
+ await heartbeat(store, cappedCells[0]!)
+ await heartbeat(store, cappedCells[1]!, {
+ incarnation: '22222222-2222-4222-8222-222222222222'
+ })
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ await store.assign(identity)
+ await store.setCellEnabled('cell-a', false)
+ const migration = await store.startEvacuation(identity, 'cell-b')
+ await store.markMigrationTargetRegistered(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch
+ })
+ const attempt = {
+ attemptId: '99999999-9999-4999-8999-999999999999',
+ cellId: 'cell-a',
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ traceValue: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa',
+ plannedGraceMs: 120_000
+ }
+ await store.prepareCellDrainAttempt(attempt)
+
+ now = migration.expiresAt - 500_000
+ await heartbeat(store, cappedCells[0]!)
+ await heartbeat(store, cappedCells[1]!, {
+ incarnation: '22222222-2222-4222-8222-222222222222'
+ })
+ await expect(store.prepareCellDrainRecovery(attempt)).resolves.toMatchObject({
+ shouldSend: false,
+ preparedAttempt: { attemptId: attempt.attemptId }
+ })
+ const refreshedExpiresAt = now + ASSIGNMENT_LIMITS.migrationLeaseMs
+ await expect(
+ database!.query(
+ `SELECT expires_at FROM relay_assignment_migrations
+ WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
+ [identity.userId, identity.relayHostId, migration.assignmentEpoch]
+ )
+ ).resolves.toEqual([{ expires_at: refreshedExpiresAt }])
+ await expect(
+ database!.query(
+ `SELECT state, timeout_at FROM relay_control_connection_reservations
+ WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
+ [identity.userId, identity.relayHostId, migration.assignmentEpoch]
+ )
+ ).resolves.toEqual([{ state: 'reserved', timeout_at: refreshedExpiresAt }])
+ })
+
+ it('bounds repeated accounting repair before prepared drain recovery', async () => {
+ const delegate = await openInMemoryRelayDatabase()
+ const retryDatabase = new RepeatedDrainAccountingFailureDatabase(delegate)
+ database = retryDatabase
+ const store = new RelayAssignmentStore(retryDatabase, () => 100, {
+ requireLiveCells: true,
+ heartbeatTtlMs: 45_000
+ })
+ await store.reconcileCells(CELLS)
+ await heartbeat(store, CELLS[0]!)
+ await heartbeat(store, CELLS[1]!, {
+ incarnation: '22222222-2222-4222-8222-222222222222'
+ })
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ await store.assign(identity)
+ await store.setCellEnabled('cell-a', false)
+ const migration = await store.startEvacuation(identity, 'cell-b')
+ await store.markMigrationTargetRegistered(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch
+ })
+ const attempt = {
+ attemptId: '99999999-9999-4999-8999-999999999999',
+ cellId: 'cell-a',
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ traceValue: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa',
+ plannedGraceMs: 120_000
+ }
+ await store.prepareCellDrainAttempt(attempt)
+ retryDatabase.arm(2)
+
+ await expect(store.prepareCellDrainRecovery(attempt)).resolves.toMatchObject({
+ shouldSend: false,
+ preparedAttempt: { attemptId: attempt.attemptId }
+ })
+ expect(retryDatabase.refreshAttempts).toBe(3)
+
+ retryDatabase.arm(4)
+ await expect(store.prepareCellDrainRecovery(attempt)).rejects.toThrow(
+ 'migration_activity_accounting_mismatch'
+ )
+ expect(retryDatabase.refreshAttempts).toBe(4)
+ })
+
+ it('retires a pinned migration superseded by a newer authoritative assignment', async () => {
+ let now = 100
+ const cells = [
+ ...CELLS,
+ {
+ id: 'cell-c',
+ url: 'https://relay-c.example.com',
+ capacityRequests: 2
+ }
+ ]
+ const store = await setupWithHeartbeats(() => now, cells)
+ await heartbeat(store, cells[0]!)
+ await heartbeat(store, cells[1]!, {
+ incarnation: '22222222-2222-4222-8222-222222222222'
+ })
+ await heartbeat(store, cells[2]!, {
+ incarnation: '33333333-3333-4333-8333-333333333333'
+ })
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ await store.assign(identity)
+ await store.setCellEnabled('cell-a', false)
+ const migration = await store.startEvacuation(identity, 'cell-b')
+ await store.markMigrationTargetRegistered(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch
+ })
+ const attempt = {
+ attemptId: '99999999-9999-4999-8999-999999999999',
+ cellId: 'cell-a',
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ traceValue: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa',
+ plannedGraceMs: 120_000
+ }
+ await store.prepareCellDrainAttempt(attempt)
+ await store.beginCellDrainSend(attempt)
+ const receipt = await store.recordCellDrainApplicationReceipt({
+ ...attempt,
+ backendStatus: 200
+ })
+
+ await database!.query(
+ `DELETE FROM relay_assignment_activity_leases
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ await database!.query(
+ `UPDATE relay_assignments SET reserved_controls = 0, reserved_splices = 0,
+ reserved_invites = 0, pending_installs = 0, pending_confirmations = 0,
+ migration_leases = 0, lease_expires_at = 0, last_activity_at = 0
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ await database!.query(
+ `UPDATE relay_cells SET reserved_requests = 0 WHERE cell_id IN (?, ?)`,
+ ['cell-a', 'cell-b']
+ )
+ now = ASSIGNMENT_LIMITS.dormantTtlMs + 1
+ await heartbeat(store, cells[2]!, {
+ incarnation: '33333333-3333-4333-8333-333333333333'
+ })
+ const current = await store.rebalanceDormant(identity, 'cell-c')
+ await store.activateControl(identity, {
+ cellId: 'cell-c',
+ assignmentEpoch: current.assignmentEpoch,
+ generation: 1
+ })
+
+ expect(receipt.retryAfter).toBeLessThanOrEqual(now)
+ await heartbeat(store, cells[0]!)
+ await expect(store.prepareCellDrainRecovery(attempt)).resolves.toMatchObject({
+ shouldSend: true
+ })
+ await expect(
+ database!.query(
+ `SELECT aborted_at FROM relay_assignment_migrations
+ WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
+ [identity.userId, identity.relayHostId, migration.assignmentEpoch]
+ )
+ ).resolves.toEqual([{ aborted_at: now }])
+ await expect(
+ database!.query(
+ `SELECT cell_id, assignment_epoch FROM relay_assignments
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).resolves.toEqual([
+ { cell_id: 'cell-c', assignment_epoch: current.assignmentEpoch }
+ ])
+ await expect(
+ database!.query(
+ `SELECT activity_id, cell_id FROM relay_assignment_activity_leases
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).resolves.toEqual([{ activity_id: 'control:cell-c:1', cell_id: 'cell-c' }])
+ })
+
+ it('refuses to restore a missing migration lease without durable target registration', async () => {
+ let now = 100
+ const store = await setupWithHeartbeats(() => now)
+ await heartbeat(store, CELLS[0]!)
+ await heartbeat(store, CELLS[1]!, {
+ incarnation: '22222222-2222-4222-8222-222222222222'
+ })
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ await store.assign(identity)
+ await store.setCellEnabled('cell-a', false)
+ const migration = await store.startEvacuation(identity, 'cell-b')
+ const attempt = {
+ attemptId: '77777777-7777-4777-8777-777777777777',
+ cellId: 'cell-a',
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ traceValue: '88888888-8888-4888-8888-888888888888',
+ plannedGraceMs: 120_000
+ }
+ await store.prepareCellDrainAttempt(attempt)
+
+ now = migration.expiresAt + 1
+ expect(await store.releaseExpiredActivityLeases()).toBeGreaterThan(0)
+ await heartbeat(store, CELLS[0]!)
+ await heartbeat(store, CELLS[1]!, {
+ incarnation: '22222222-2222-4222-8222-222222222222'
+ })
+ await expect(
+ store.beginCellDrainSend({
+ attemptId: attempt.attemptId,
+ cellId: attempt.cellId,
+ cellIncarnation: attempt.cellIncarnation
+ })
+ ).rejects.toThrow('migration_activity_lease_shape_mismatch')
+ })
+
+ it('moves a dead assignment only after a completed exact-incarnation fence attempt', async () => {
+ let now = 100
+ const cappedCells = CELLS.map((cell) => ({
+ ...cell,
+ connectionHardCap: 600 as const,
+ connectionUnobservedBound: 50
+ }))
+ const store = await setupWithHeartbeats(() => now, cappedCells)
+ await heartbeat(store, cappedCells[0]!)
+ await heartbeat(store, cappedCells[1]!, {
+ incarnation: '22222222-2222-4222-8222-222222222222'
+ })
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ await store.assign(identity)
+ await store.setCellEnabled('cell-a', false)
+ now += 45_001
+ await heartbeat(store, cappedCells[1]!, {
+ incarnation: '22222222-2222-4222-8222-222222222222'
+ })
+
+ await store.attestCellFence('cell-a', '11111111-1111-4111-8111-111111111111')
+ expect(
+ await database!.query(
+ `SELECT cell_id FROM relay_cell_legacy_fence_adoptions WHERE cell_id = ?`,
+ ['cell-a']
+ )
+ ).toEqual([])
+ expect(await store.evacuateDeadCells()).toBe(0)
+ expect(
+ await database!.query(
+ `SELECT cell_id FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([{ cell_id: 'cell-a' }])
+
+ now += 300_001
+ await heartbeat(store, cappedCells[1]!, {
+ incarnation: '22222222-2222-4222-8222-222222222222'
+ })
+ const evidence = {
+ attemptId: '22222222-2222-4222-8222-222222222222',
+ environment: 'production' as const,
+ cellId: 'cell-a',
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ migName: 'orca-relay-c1',
+ instanceGroup: 'https://compute.example/instanceGroups/orca-relay-c1',
+ generationIdentity: 'https://compute.example/instanceTemplates/orca-relay-c1-abc',
+ fenceCommit: 'a'.repeat(40),
+ planSha256: 'b'.repeat(64),
+ ...FENCE_PLAN_BINDING
+ }
+ await store.prepareCellFenceAttempt(evidence)
+ await store.bindCellFencePlanGeneration(evidence, evidence.planObjectGeneration)
+ await store.startCellFenceApply(
+ evidence,
+ FENCE_INVOCATION_ID,
+ FENCE_INVOCATION_REASON
+ )
+ await store.recordCellFenceOperation(
+ evidence,
+ FENCE_INVOCATION_ID,
+ FENCE_INVOCATION_REASON,
+ 'operation-1'
+ )
+ await store.attestCellFenceAttempt(evidence, 'operation-1')
+
+ expect(await store.evacuateDeadCells()).toBe(1)
+ expect((await store.resolve(identity))?.cellId).toBe('cell-b')
+ })
+
+ it('preserves proven non-delivery and permits one fresh full-grace attempt', async () => {
+ const store = await setupWithHeartbeats(() => 100, [CELLS[0]!])
+ await heartbeat(store)
+ await store.setCellEnabled('cell-a', false)
+ const first = {
+ attemptId: '33333333-3333-4333-8333-333333333333',
+ cellId: 'cell-a',
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ traceValue: '44444444-4444-4444-8444-444444444444',
+ plannedGraceMs: 120_000
+ }
+ await store.prepareCellDrainAttempt(first)
+ await store.beginCellDrainSend(first)
+ await expect(store.proveCellDrainNotDelivered(first)).resolves.toMatchObject({
+ state: 'proven-not-delivered',
+ provenNotDeliveredAt: 100
+ })
+ await expect(
+ store.prepareCellDrainAttempt({
+ ...first,
+ attemptId: '55555555-5555-4555-8555-555555555555',
+ traceValue: '66666666-6666-4666-8666-666666666666'
+ })
+ ).resolves.toMatchObject({ state: 'prepared' })
+ })
+
+ it('binds fence attestation to one durable Terraform attempt', async () => {
+ let now = 100
+ const store = await setupWithHeartbeats(() => now, [CELLS[0]!])
+ await heartbeat(store)
+ await store.setCellEnabled('cell-a', false)
+ const evidence = {
+ attemptId: '22222222-2222-4222-8222-222222222222',
+ environment: 'production' as const,
+ cellId: 'cell-a',
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ migName: 'orca-relay-c1',
+ instanceGroup: 'https://compute.example/instanceGroups/orca-relay-c1',
+ generationIdentity: 'https://compute.example/instanceTemplates/orca-relay-c1-abc',
+ fenceCommit: 'a'.repeat(40),
+ planSha256: 'b'.repeat(64),
+ ...FENCE_PLAN_BINDING
+ }
+ const prepared = await store.prepareCellFenceAttempt(evidence)
+ expect(prepared).toMatchObject({ createdAt: 100, expiresAt: 3_600_100 })
+ expect(prepared.planObjectGeneration).toBeUndefined()
+ await expect(
+ store.bindCellFencePlanGeneration(
+ evidence,
+ evidence.planObjectGeneration
+ )
+ ).resolves.toMatchObject({
+ planObjectGeneration: evidence.planObjectGeneration
+ })
+ await expect(
+ store.bindCellFencePlanGeneration(
+ evidence,
+ evidence.planObjectGeneration
+ )
+ ).resolves.toMatchObject({
+ planObjectGeneration: evidence.planObjectGeneration
+ })
+ await expect(
+ store.bindCellFencePlanGeneration(
+ evidence,
+ '987654321'
+ )
+ ).rejects.toThrow('cell_fence_plan_generation_mismatch')
+ for (const changed of [
+ { attemptId: '33333333-3333-4333-8333-333333333333' },
+ { environment: 'staging' as const },
+ { cellId: 'cell-b' },
+ { cellIncarnation: '33333333-3333-4333-8333-333333333333' },
+ { migName: 'orca-relay-other' },
+ { instanceGroup: `${evidence.instanceGroup}-other` },
+ { generationIdentity: `${evidence.generationIdentity}-other` },
+ { fenceCommit: 'c'.repeat(40) },
+ { planSha256: 'c'.repeat(64) }
+ ]) {
+ await expect(
+ store.startCellFenceApply(
+ { ...evidence, ...changed },
+ FENCE_INVOCATION_ID,
+ FENCE_INVOCATION_REASON
+ )
+ ).rejects.toThrow()
+ }
+ await store.startCellFenceApply(
+ evidence,
+ FENCE_INVOCATION_ID,
+ FENCE_INVOCATION_REASON
+ )
+ await store.recordCellFenceOperation(
+ evidence,
+ FENCE_INVOCATION_ID,
+ FENCE_INVOCATION_REASON,
+ 'operation-1'
+ )
+ now += 45_001
+ await expect(
+ store.attestCellFenceAttempt(evidence, 'operation-other')
+ ).rejects.toThrow('cell_fence_operation_not_attested')
+ const attested = await store.attestCellFenceAttempt(evidence, 'operation-1')
+ expect(attested).toMatchObject({
+ expiresAt: now + 300_000,
+ attempt: { ...evidence, gceOperation: 'operation-1', completedAt: now }
+ })
+ await expect(store.attestCellFenceAttempt(evidence, 'operation-1')).resolves.toEqual(
+ attested
+ )
+ now += 300_001
+ await expect(
+ store.attestCellFenceAttempt(evidence, 'operation-1')
+ ).resolves.toMatchObject({
+ expiresAt: now + 300_000,
+ attempt: { completedAt: attested.attempt.completedAt }
+ })
+ })
+
+ it('aborts only a Terraform fence whose apply never started', async () => {
+ const store = await setupWithHeartbeats(() => 100, [CELLS[0]!])
+ await heartbeat(store)
+ await store.setCellEnabled('cell-a', false)
+ const evidence = {
+ attemptId: '22222222-2222-4222-8222-222222222222',
+ environment: 'production' as const,
+ cellId: 'cell-a',
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ migName: 'orca-relay-c1',
+ instanceGroup: 'https://compute.example/instanceGroups/orca-relay-c1',
+ generationIdentity: 'https://compute.example/instanceTemplates/orca-relay-c1-abc',
+ fenceCommit: 'a'.repeat(40),
+ planSha256: 'b'.repeat(64),
+ ...FENCE_PLAN_BINDING
+ }
+ await store.prepareCellFenceAttempt(evidence)
+ await store.bindCellFencePlanGeneration(
+ evidence,
+ evidence.planObjectGeneration
+ )
+ await expect(store.abortCellFenceAttempt(evidence)).resolves.toMatchObject({
+ abortedAt: 100
+ })
+ await expect(
+ store.startCellFenceApply(
+ evidence,
+ FENCE_INVOCATION_ID,
+ FENCE_INVOCATION_REASON
+ )
+ ).rejects.toThrow(
+ 'cell_fence_attempt_aborted'
+ )
+ })
+
+ it('adopts a stale disabled legacy fence only when no durable attempt exists', async () => {
+ let now = 100
+ const store = await setupWithHeartbeats(() => now, [CELLS[0]!])
+ await heartbeat(store)
+ await store.setCellEnabled('cell-a', false)
+ now += 45_001
+
+ await expect(
+ store.adoptLegacyCellFence('cell-a', '11111111-1111-4111-8111-111111111111')
+ ).resolves.toBe(now + 300_000)
+ await expect(
+ database!.query(
+ `SELECT cell_id, cell_incarnation FROM relay_cell_fences WHERE cell_id = ?`,
+ ['cell-a']
+ )
+ ).resolves.toEqual([
+ {
+ cell_id: 'cell-a',
+ cell_incarnation: '11111111-1111-4111-8111-111111111111'
+ }
+ ])
+ await expect(
+ database!.query(
+ `SELECT cell_id, cell_incarnation
+ FROM relay_cell_legacy_fence_adoptions WHERE cell_id = ?`,
+ ['cell-a']
+ )
+ ).resolves.toEqual([])
+ await store.commitLegacyCellFenceAdoption(
+ 'cell-a',
+ '11111111-1111-4111-8111-111111111111'
+ )
+ await expect(
+ database!.query(
+ `SELECT cell_id, cell_incarnation
+ FROM relay_cell_legacy_fence_adoptions WHERE cell_id = ?`,
+ ['cell-a']
+ )
+ ).resolves.toEqual([
+ {
+ cell_id: 'cell-a',
+ cell_incarnation: '11111111-1111-4111-8111-111111111111'
+ }
+ ])
+
+ now += 1
+ await expect(
+ store.adoptLegacyCellFence('cell-a', '11111111-1111-4111-8111-111111111111')
+ ).resolves.toBe(now + 300_000)
+ await store.commitLegacyCellFenceAdoption(
+ 'cell-a',
+ '11111111-1111-4111-8111-111111111111'
+ )
+ await expect(
+ database!.query(
+ `SELECT attested_at, expires_at
+ FROM relay_cell_legacy_fence_adoptions WHERE cell_id = ?`,
+ ['cell-a']
+ )
+ ).resolves.toEqual([{ attested_at: now, expires_at: now + 300_000 }])
+
+ await heartbeat(store)
+ await expect(
+ database!.query(
+ `SELECT cell_id FROM relay_cell_legacy_fence_adoptions WHERE cell_id = ?`,
+ ['cell-a']
+ )
+ ).resolves.toEqual([])
+ })
+
+ it.each(['active', 'completed', 'aborted'] as const)(
+ 'rejects legacy adoption after a %s durable attempt',
+ async (status) => {
+ let now = 100
+ const store = await setupWithHeartbeats(() => now, [CELLS[0]!])
+ await heartbeat(store)
+ await store.setCellEnabled('cell-a', false)
+ const evidence = cellFenceEvidence()
+ await store.prepareCellFenceAttempt(evidence)
+ if (status === 'aborted') await store.abortCellFenceAttempt(evidence)
+ if (status === 'completed') {
+ await store.bindCellFencePlanGeneration(evidence, evidence.planObjectGeneration)
+ await store.startCellFenceApply(
+ evidence,
+ FENCE_INVOCATION_ID,
+ FENCE_INVOCATION_REASON
+ )
+ await store.recordCellFenceOperation(
+ evidence,
+ FENCE_INVOCATION_ID,
+ FENCE_INVOCATION_REASON,
+ 'operation-1'
+ )
+ }
+ now += 45_001
+ if (status === 'completed') {
+ await store.attestCellFenceAttempt(evidence, 'operation-1')
+ }
+
+ await expect(
+ store.adoptLegacyCellFence('cell-a', '11111111-1111-4111-8111-111111111111')
+ ).rejects.toThrow('legacy_cell_fence_attempt_exists')
+ }
+ )
+
+ it('serializes legacy adoption against durable attempt preparation', async () => {
+ let now = 100
+ const store = await setupWithHeartbeats(() => now, [CELLS[0]!])
+ await heartbeat(store)
+ await store.setCellEnabled('cell-a', false)
+ now += 45_001
+
+ const results = await Promise.allSettled([
+ store.adoptLegacyCellFence('cell-a', '11111111-1111-4111-8111-111111111111'),
+ store.prepareCellFenceAttempt(cellFenceEvidence())
+ ])
+ expect(results.filter(({ status }) => status === 'fulfilled')).toHaveLength(1)
+ const attempts = await database!.query(
+ `SELECT COUNT(*) AS count FROM relay_cell_fence_attempts WHERE cell_id = ?`,
+ ['cell-a']
+ )
+ const fences = await database!.query(
+ `SELECT COUNT(*) AS count FROM relay_cell_fences WHERE cell_id = ?`,
+ ['cell-a']
+ )
+ const adoptions = await database!.query(
+ `SELECT COUNT(*) AS count FROM relay_cell_legacy_fence_adoptions
+ WHERE cell_id = ?`,
+ ['cell-a']
+ )
+ expect(Number(attempts[0]!.count) + Number(fences[0]!.count)).toBe(1)
+ expect(Number(adoptions[0]!.count)).toBe(0)
+ })
+
+ it('rejects an expired durable Terraform fence attempt', async () => {
+ let now = 100
+ const store = await setupWithHeartbeats(() => now, [CELLS[0]!])
+ await heartbeat(store)
+ await store.setCellEnabled('cell-a', false)
+ const evidence = {
+ attemptId: '22222222-2222-4222-8222-222222222222',
+ environment: 'production' as const,
+ cellId: 'cell-a',
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ migName: 'orca-relay-c1',
+ instanceGroup: 'https://compute.example/instanceGroups/orca-relay-c1',
+ generationIdentity: 'https://compute.example/instanceTemplates/orca-relay-c1-abc',
+ fenceCommit: 'a'.repeat(40),
+ planSha256: 'b'.repeat(64),
+ ...FENCE_PLAN_BINDING
+ }
+ await store.prepareCellFenceAttempt(evidence)
+ await store.bindCellFencePlanGeneration(
+ evidence,
+ evidence.planObjectGeneration
+ )
+ now += 3_600_001
+ await expect(
+ store.startCellFenceApply(
+ evidence,
+ FENCE_INVOCATION_ID,
+ FENCE_INVOCATION_REASON
+ )
+ ).rejects.toThrow(
+ 'cell_fence_attempt_expired'
+ )
+ })
+
+ it('keeps a started Terraform fence attempt recoverable after its preparation TTL', async () => {
+ let now = 100
+ const store = await setupWithHeartbeats(() => now, [CELLS[0]!])
+ await heartbeat(store)
+ await store.setCellEnabled('cell-a', false)
+ const evidence = {
+ attemptId: '22222222-2222-4222-8222-222222222222',
+ environment: 'production' as const,
+ cellId: 'cell-a',
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ migName: 'orca-relay-c1',
+ instanceGroup: 'https://compute.example/instanceGroups/orca-relay-c1',
+ generationIdentity: 'https://compute.example/instanceTemplates/orca-relay-c1-abc',
+ fenceCommit: 'a'.repeat(40),
+ planSha256: 'b'.repeat(64),
+ ...FENCE_PLAN_BINDING
+ }
+ await store.prepareCellFenceAttempt(evidence)
+ await store.bindCellFencePlanGeneration(
+ evidence,
+ evidence.planObjectGeneration
+ )
+ await store.startCellFenceApply(
+ evidence,
+ FENCE_INVOCATION_ID,
+ FENCE_INVOCATION_REASON
+ )
+ now += 3_600_001
+ await expect(
+ store.recordCellFenceOperation(
+ evidence,
+ FENCE_INVOCATION_ID,
+ FENCE_INVOCATION_REASON,
+ 'operation-1'
+ )
+ ).resolves.toMatchObject({
+ attempt: { gceOperation: 'operation-1' },
+ invocation: { gceOperation: 'operation-1' }
+ })
+ await expect(
+ store.prepareCellFenceAttempt({
+ ...evidence,
+ attemptId: '44444444-4444-4444-8444-444444444444',
+ planObjectName:
+ 'terraform/state/relay-fence-plans/production/44444444-4444-4444-8444-444444444444.tfplan',
+ requestReason:
+ 'orca-relay-fence/44444444-4444-4444-8444-444444444444'
+ })
+ ).rejects.toThrow('cell_fence_attempt_evidence_mismatch')
+ })
+
+ it('reports aggregate deployment state and heartbeat freshness without identities', async () => {
+ let now = 100
+ const store = await setupWithHeartbeats(() => now, [CELLS[0]!])
+ await heartbeat(store)
+ const identity = { userId: 'private-user', relayHostId: 'host000000000001' }
+ const assignment = await store.assign(identity)
+ expect(await store.cellDeploymentStatus('cell-a')).toMatchObject({
+ activityLeases: 1,
+ activityRequestUnits: 1,
+ restartBlockingActivityLeases: 0,
+ restartBlockingActivityRequestUnits: 0,
+ restartBlockingReservedRequests: 0
+ })
+ await store.activateControl(identity, {
+ cellId: 'cell-a',
+ assignmentEpoch: assignment.assignmentEpoch,
+ generation: 1
+ })
+
+ expect(await store.cellDeploymentStatus('cell-a')).toEqual({
+ cellId: 'cell-a',
+ cellUrl: 'https://relay-a.example.com',
+ region: 'us-central1',
+ enabled: true,
+ admissionState: 'general',
+ capacityRequests: 2,
+ reservedRequests: 1,
+ assignments: 1,
+ activityLeases: 1,
+ activityRequestUnits: 1,
+ restartBlockingActivityLeases: 1,
+ restartBlockingActivityRequestUnits: 1,
+ restartBlockingReservedRequests: 1,
+ outgoingMigrations: 0,
+ incomingMigrations: 0,
+ connectionCapacity: null,
+ runtime: {
+ cellUrl: 'https://relay-a.example.com',
+ cellIncarnation: '11111111-1111-4111-8111-111111111111',
+ startedAt: 50,
+ ready: true,
+ observedRequests: 0,
+ lastHeartbeatAt: 100,
+ heartbeatFresh: true,
+ regionalRehomeProtocol: 0
+ }
+ })
+ now += 45_001
+ expect((await store.cellDeploymentStatus('cell-a')).runtime?.heartbeatFresh).toBe(false)
+ await expect(store.cellDeploymentStatus('missing')).rejects.toThrow('cell_not_found')
+ })
+
+ it('keeps concurrent sticky assignment grants restart-safe', async () => {
+ const store = await setup(() => 100, [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 }
+ ])
+ const identity = { userId: 'private-user', relayHostId: 'host000000000001' }
+
+ await Promise.all(Array.from({ length: 20 }, async () => await store.assign(identity)))
+
+ expect(await store.cellDeploymentStatus('cell-a')).toMatchObject({
+ activityLeases: 1,
+ activityRequestUnits: 1,
+ reservedRequests: 1,
+ restartBlockingActivityLeases: 0,
+ restartBlockingActivityRequestUnits: 0,
+ restartBlockingReservedRequests: 0
+ })
+ })
+
+ it('classifies activated and non-control activity as restart-blocking', async () => {
+ const store = await setup(() => 100, [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 }
+ ])
+ const identity = { userId: 'private-user', relayHostId: 'host000000000001' }
+ const assignment = await store.assign(identity)
+ await store.activateControl(identity, {
+ cellId: assignment.cellId,
+ assignmentEpoch: assignment.assignmentEpoch,
+ generation: 1
+ })
+ for (const kind of ['splice', 'invite', 'install', 'confirmation', 'migration'] as const) {
+ await store.acquireActivity(identity, {
+ activityId: `${kind}:test`,
+ kind,
+ cellId: assignment.cellId
+ })
+ }
+
+ expect(await store.cellDeploymentStatus('cell-a')).toMatchObject({
+ activityLeases: 6,
+ activityRequestUnits: 7,
+ reservedRequests: 7,
+ restartBlockingActivityLeases: 6,
+ restartBlockingActivityRequestUnits: 7,
+ restartBlockingReservedRequests: 7
+ })
+ })
+
+ it('fails closed for malformed pending controls and unexplained reservations', async () => {
+ const store = await setup(() => 100, [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 }
+ ])
+ await store.assign({ userId: 'private-user', relayHostId: 'host000000000001' })
+ await database!.query(
+ `UPDATE relay_assignment_activity_leases SET request_units = 2
+ WHERE cell_id = ?`,
+ ['cell-a']
+ )
+ expect(await store.cellDeploymentStatus('cell-a')).toMatchObject({
+ restartBlockingActivityLeases: 1,
+ restartBlockingActivityRequestUnits: 2,
+ restartBlockingReservedRequests: 1
+ })
+
+ await database!.query(
+ `UPDATE relay_assignment_activity_leases SET request_units = 1 WHERE cell_id = ?`,
+ ['cell-a']
+ )
+ await database!.query(
+ `UPDATE relay_cells SET reserved_requests = 0 WHERE cell_id = ?`,
+ ['cell-a']
+ )
+ expect(await store.cellDeploymentStatus('cell-a')).toMatchObject({
+ restartBlockingActivityLeases: 0,
+ restartBlockingActivityRequestUnits: 0,
+ restartBlockingReservedRequests: -1
+ })
+
+ await database!.query(
+ `UPDATE relay_cells SET reserved_requests = 2 WHERE cell_id = ?`,
+ ['cell-a']
+ )
+ expect(await store.cellDeploymentStatus('cell-a')).toMatchObject({
+ restartBlockingActivityLeases: 0,
+ restartBlockingActivityRequestUnits: 0,
+ restartBlockingReservedRequests: 1
+ })
+ })
+
+ it('keeps a migration blocking when its target also has a pending control', async () => {
+ const store = await setup(() => 100, [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 20 }
+ ])
+ const identity = { userId: 'private-user', relayHostId: 'host000000000001' }
+ await store.assign(identity)
+ await store.startEvacuation(identity, 'cell-b')
+
+ expect(await store.cellDeploymentStatus('cell-b')).toMatchObject({
+ activityLeases: 2,
+ restartBlockingActivityLeases: 1,
+ restartBlockingActivityRequestUnits: 1,
+ restartBlockingReservedRequests: 1,
+ incomingMigrations: 1
+ })
+ })
+
+ it('starts declared candidates disabled without overwriting later operator state', async () => {
+ const candidate: RelayCellConfig = {
+ id: 'candidate',
+ url: 'https://candidate.example.com',
+ capacityRequests: 20,
+ initiallyEnabled: false
+ }
+ const store = await setup(() => 100, [candidate])
+ expect((await store.cellDeploymentStatus('candidate')).enabled).toBe(false)
+ await store.setCellEnabled('candidate', true)
+ await store.reconcileCells([candidate])
+ expect((await store.cellDeploymentStatus('candidate')).enabled).toBe(true)
+ })
+
+ it('moves a stranded host off an existing-only cell that stopped serving it', async () => {
+ // Why: C3's decommission created an existing-only cell that rejects
+ // attaches; the #194 pin then loops returning hosts forever while each
+ // grant refreshes their own activity (issue #225). C3 has no
+ // connection-limits row and expired leases are cleaned within a
+ // maintenance cycle, so the only durable evidence is the assignment row:
+ // a recent grant with no live real activity behind it.
+ let now = 100
+ const store = await setup(() => now)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ await store.setCellEnabled(first.cellId, false)
+
+ // The pin holds while the last grant is young enough to still attach.
+ now += 10_000
+ const pinned = await store.assign(identity)
+ expect(pinned.cellId).toBe(first.cellId)
+
+ // The grant had ample time to attach and produced nothing live.
+ now += 61_000
+ const moved = await store.assign(identity)
+ expect(moved.cellId).not.toBe(first.cellId)
+ expect(moved.assignmentEpoch).toBe(first.assignmentEpoch + 1)
+
+ // The move is durable: no bounce back to the closed cell.
+ now += 1_000
+ const settled = await store.assign(identity)
+ expect(settled.cellId).toBe(moved.cellId)
+ expect(settled.assignmentEpoch).toBe(moved.assignmentEpoch)
+ })
+
+ it('keeps a serving existing-only cell pinned for hosts with live activity', async () => {
+ let now = 100
+ const store = await setup(() => now)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ await store.setCellEnabled(first.cellId, false)
+ // A live claimed control proves the cell still serves this host.
+ await database!.query(
+ `INSERT INTO relay_assignment_activity_leases
+ (user_id, relay_host_id, activity_id, activity_kind, cell_id,
+ request_units, expires_at, updated_at)
+ VALUES (?, ?, ?, 'control', ?, 1, ?, ?)`,
+ [identity.userId, identity.relayHostId, 'control:live-1', first.cellId, now + 600_000, now]
+ )
+ now += 61_000
+ const pinned = await store.assign(identity)
+ expect(pinned.cellId).toBe(first.cellId)
+ })
+
+ it('keeps migration-only cells pinned inside the stranded window', async () => {
+ let now = 100
+ const store = await setup(() => now)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ await store.setCellAdmissionState(first.cellId, 'migration-only')
+ now += 61_000
+ const pinned = await store.assign(identity)
+ expect(pinned.cellId).toBe(first.cellId)
+ })
+
+ it('leaves quiet existing-only assignments to the normal dormancy rule', async () => {
+ // Why: outside the 15-minute stranded window there is no active retry
+ // loop to break; ordinary returns stay pinned until 24h dormancy.
+ let now = 100
+ const store = await setup(() => now)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ await store.setCellEnabled(first.cellId, false)
+ now += 20 * 60_000
+ const pinned = await store.assign(identity)
+ expect(pinned.cellId).toBe(first.cellId)
+ })
+
+ it('reassigns an active assignment from an uncapped stale cell without a fence', async () => {
+ let now = 100
+ const store = await setupWithHeartbeats(() => now)
+ await heartbeat(store, CELLS[0]!)
+ await heartbeat(store, CELLS[1]!, {
+ incarnation: '22222222-2222-4222-8222-222222222222'
+ })
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ await store.changeActivity(identity, 'invite', 1)
+ await database!.query(
+ `INSERT INTO relay_assignment_migrations
+ (user_id, relay_host_id, source_cell_id, target_cell_id,
+ previous_epoch, assignment_epoch, source_request_units,
+ target_reserved_units, expires_at, target_registered_at,
+ completed_at, aborted_at, created_at, updated_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL, ?, ?)`,
+ [
+ identity.userId,
+ identity.relayHostId,
+ 'cell-b',
+ 'cell-a',
+ -1,
+ 0,
+ 0,
+ 1,
+ 90_100,
+ now,
+ now
+ ]
+ )
+ await database!.query(
+ `INSERT INTO relay_post_drain_migration_pins
+ (user_id, relay_host_id, assignment_epoch, drain_attempt_id,
+ source_cell_id, source_cell_incarnation, target_cell_id,
+ target_cell_incarnation, source_request_units, target_reserved_units,
+ pinned_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+ [
+ identity.userId,
+ identity.relayHostId,
+ 0,
+ '33333333-3333-4333-8333-333333333333',
+ 'cell-b',
+ '22222222-2222-4222-8222-222222222222',
+ 'cell-a',
+ '11111111-1111-4111-8111-111111111111',
+ 0,
+ 1,
+ now
+ ]
+ )
+
+ now += 45_001
+ await heartbeat(store, CELLS[1]!, {
+ incarnation: '22222222-2222-4222-8222-222222222222'
+ })
+ expect(await store.evacuateDeadCells()).toBe(1)
+ const replacement = await store.resolve(identity)
+ expect(replacement).toMatchObject({
+ cellId: 'cell-b',
+ assignmentEpoch: first.assignmentEpoch + 1
+ })
+ expect(
+ await database!.query(
+ `SELECT activity_kind, cell_id FROM relay_assignment_activity_leases
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([{ activity_kind: 'control', cell_id: 'cell-b' }])
+ })
+
+ it('does not let an unfenced capped cell starve eligible legacy recovery', async () => {
+ let now = 100
+ const cells: RelayCellConfig[] = [
+ {
+ id: 'cell-a',
+ url: 'https://relay-a.example.com',
+ capacityRequests: 10,
+ connectionHardCap: 600,
+ connectionUnobservedBound: 50
+ },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 },
+ { id: 'cell-c', url: 'https://relay-c.example.com', capacityRequests: 10 }
+ ]
+ const store = await setupWithHeartbeats(() => now, cells)
+ for (const cell of cells) await heartbeat(store, cell)
+ await store.setCellEnabled('cell-b', false)
+ await store.setCellEnabled('cell-c', false)
+ const cappedIdentity = { userId: 'a-capped', relayHostId: 'host000000000001' }
+ expect(await store.assign(cappedIdentity)).toMatchObject({ cellId: 'cell-a' })
+
+ await store.setCellEnabled('cell-a', false)
+ await store.setCellEnabled('cell-b', true)
+ const legacyIdentity = { userId: 'z-legacy', relayHostId: 'host000000000002' }
+ expect(await store.assign(legacyIdentity)).toMatchObject({ cellId: 'cell-b' })
+ await store.setCellEnabled('cell-c', true)
+
+ now += 45_001
+ await heartbeat(store, cells[2]!)
+ expect(await store.evacuateDeadCells(1)).toBe(1)
+ expect(await store.resolve(legacyIdentity)).toMatchObject({ cellId: 'cell-c' })
+ expect(
+ await database!.query(
+ `SELECT cell_id FROM relay_assignments WHERE user_id = ?`,
+ [cappedIdentity.userId]
+ )
+ ).toEqual([{ cell_id: 'cell-a' }])
+ })
+
+ it('refuses evacuation into a cell without a fresh ready heartbeat', async () => {
+ const store = await setupWithHeartbeats(() => 100)
+ await heartbeat(store, CELLS[0]!)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ await store.assign(identity)
+
+ await expect(store.startEvacuation(identity, 'cell-b')).rejects.toThrow(
+ 'target_cell_unavailable'
+ )
+ })
+
+ it('keeps an active assignment sticky without increasing its epoch or reservation', async () => {
+ const store = await setup(() => 100)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ const second = await store.assign(identity)
+
+ expect(second).toEqual(first)
+ expect(await cellReservations(database!)).toEqual({ 'cell-a': 1, 'cell-b': 0 })
+ })
+
+ it('selects the least-loaded cell with a stable cell-id tie break', async () => {
+ const store = await setup(() => 100)
+ const first = await store.assign({ userId: 'user-a', relayHostId: 'host000000000001' })
+ const second = await store.assign({ userId: 'user-b', relayHostId: 'host000000000002' })
+
+ expect([first.cellId, second.cellId]).toEqual(['cell-a', 'cell-b'])
+ })
+
+ it('admits exactly through the configured capacity boundary', async () => {
+ const store = await setup(() => 100, [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 2 }
+ ])
+ await store.assign({ userId: 'user-a', relayHostId: 'host000000000001' })
+ await store.assign({ userId: 'user-b', relayHostId: 'host000000000002' })
+
+ await expect(
+ store.assign({ userId: 'user-c', relayHostId: 'host000000000003' })
+ ).rejects.toThrow('relay_capacity_exhausted')
+ expect(await cellReservations(database!)).toEqual({ 'cell-a': 2 })
+ })
+
+ it('does not oversubscribe when assignments race', async () => {
+ const store = await setup(() => 100, [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 2 }
+ ])
+ const results = await Promise.allSettled(
+ Array.from({ length: 8 }, (_, index) =>
+ store.assign({ userId: `user-${index}`, relayHostId: `host${String(index).padStart(12, '0')}` })
+ )
+ )
+
+ expect(results.filter(({ status }) => status === 'fulfilled')).toHaveLength(2)
+ expect(await cellReservations(database!)).toEqual({ 'cell-a': 2 })
+ })
+
+ it('reassigns only after all activity expires and the dormant TTL elapses', async () => {
+ let now = 100
+ const store = await setup(() => now)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ await store.changeActivity(identity, 'invite', 1)
+ await store.changeActivity(identity, 'control', -1)
+
+ now += ASSIGNMENT_LIMITS.activityLeaseMs + 1
+ await store.releaseExpiredActivityLeases()
+ await store.releaseExpiredActivity()
+ await database!.query(`UPDATE relay_cells SET observed_requests = 2 WHERE cell_id = ?`, [
+ first.cellId
+ ])
+ now += ASSIGNMENT_LIMITS.dormantTtlMs
+ const reassigned = await store.assign(identity)
+
+ expect(reassigned.cellId).not.toBe(first.cellId)
+ expect(reassigned.assignmentEpoch).toBe(first.assignmentEpoch + 1)
+ })
+
+ it('will not normally move an assignment while any durable activity remains', async () => {
+ let now = 100
+ const store = await setup(() => now)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ await store.changeActivity(identity, 'migration', 1)
+ await store.changeActivity(identity, 'control', -1)
+ await database!.query(`UPDATE relay_cells SET observed_requests = 2 WHERE cell_id = ?`, [
+ first.cellId
+ ])
+ now += ASSIGNMENT_LIMITS.dormantTtlMs + 1
+
+ expect(await store.assign(identity)).toMatchObject({
+ cellId: first.cellId,
+ assignmentEpoch: first.assignmentEpoch
+ })
+ })
+
+ it('releases every expired activity reservation without going negative', async () => {
+ let now = 100
+ const store = await setup(() => now, [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }
+ ])
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ await store.assign(identity)
+ for (const kind of ['splice', 'invite', 'install', 'confirmation', 'migration'] as const) {
+ await store.changeActivity(identity, kind, 1)
+ }
+ expect(await cellReservations(database!)).toEqual({ 'cell-a': 7 })
+
+ now += ASSIGNMENT_LIMITS.activityLeaseMs + 1
+ expect(await store.releaseExpiredActivityLeases()).toBe(1)
+ expect(await store.releaseExpiredActivity()).toBe(1)
+ expect(await store.releaseExpiredActivity()).toBe(0)
+ expect(await cellReservations(database!)).toEqual({ 'cell-a': 0 })
+ })
+
+ it('isolates assignments by host and verifies both cell and epoch', async () => {
+ const store = await setup(() => 100)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const assignment = await store.assign(identity)
+
+ await expect(
+ store.verifyCellAssignment({ ...identity, cellId: assignment.cellId, assignmentEpoch: 1 })
+ ).resolves.toBe(true)
+ await expect(
+ store.verifyCellAssignment({ ...identity, cellId: 'cell-b', assignmentEpoch: 1 })
+ ).resolves.toBe(false)
+ await expect(
+ store.verifyCellAssignment({ ...identity, cellId: assignment.cellId, assignmentEpoch: 2 })
+ ).resolves.toBe(false)
+ await expect(
+ store.resolve({ userId: 'other', relayHostId: identity.relayHostId })
+ ).resolves.toBeNull()
+ })
+
+ it('converts the director reservation into an idempotent active-control lease', async () => {
+ const store = await setup(() => 100)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const assignment = await store.assign(identity)
+
+ const activityId = await store.activateControl(identity, {
+ cellId: assignment.cellId,
+ assignmentEpoch: assignment.assignmentEpoch,
+ generation: 1
+ })
+ await store.activateControl(identity, {
+ cellId: assignment.cellId,
+ assignmentEpoch: assignment.assignmentEpoch,
+ generation: 1
+ })
+
+ expect(activityId).toBe(`control:${assignment.cellId}:1`)
+ expect(await cellReservations(database!)).toEqual({ 'cell-a': 1, 'cell-b': 0 })
+ const leases = await database!.query(
+ `SELECT activity_id FROM relay_assignment_activity_leases`
+ )
+ expect(leases).toEqual([{ activity_id: activityId }])
+ })
+
+ it('transactionally supersedes older controls on only the same cell', async () => {
+ const cells = [
+ {
+ id: 'cell-a',
+ url: 'https://relay-a.example.com',
+ capacityRequests: 10
+ },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
+ ]
+ const store = await setup(() => 100, cells)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ await store.setCellEnabled('cell-b', false)
+ const assignment = await store.assign(identity)
+ await store.activateControl(identity, {
+ cellId: assignment.cellId,
+ assignmentEpoch: assignment.assignmentEpoch,
+ generation: 1
+ })
+ await store.setCellEnabled('cell-b', true)
+ const migration = await store.startEvacuation(identity, 'cell-b')
+ await store.activateControl(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch,
+ generation: 1
+ })
+ await database!.query(
+ `INSERT INTO relay_assignment_activity_leases
+ (user_id, relay_host_id, activity_id, activity_kind, cell_id,
+ request_units, expires_at, updated_at)
+ VALUES (?, ?, ?, 'control', 'cell-b', 1, 90100, 100),
+ (?, ?, ?, 'control', 'cell-b', 1, 90100, 100)`,
+ [
+ identity.userId,
+ identity.relayHostId,
+ 'control:cell-b:2',
+ identity.userId,
+ identity.relayHostId,
+ 'control:cell-b:3'
+ ]
+ )
+ const latest = await store.activateControl(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch,
+ generation: 4
+ })
+
+ expect(
+ await database!.query(
+ `SELECT activity_id, cell_id FROM relay_assignment_activity_leases
+ WHERE activity_kind = 'control' ORDER BY cell_id`
+ )
+ ).toEqual([
+ { activity_id: 'control:cell-a:1', cell_id: 'cell-a' },
+ { activity_id: latest, cell_id: 'cell-b' }
+ ])
+ expect(await cellReservations(database!)).toEqual({ 'cell-a': 1, 'cell-b': 2 })
+ expect(
+ await database!.query(
+ `SELECT reserved_controls FROM relay_assignments
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([{ reserved_controls: 2 }])
+ })
+
+ it('re-reserves a sticky grant for a host holding no control lease', async () => {
+ const store = await setup(() => 100)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const assignment = await store.assign(identity)
+ const control = await store.activateControl(identity, {
+ cellId: assignment.cellId,
+ assignmentEpoch: assignment.assignmentEpoch,
+ generation: 1
+ })
+ await store.releaseActivity(identity, control)
+ expect(await cellReservations(database!)).toEqual({ 'cell-a': 0, 'cell-b': 0 })
+
+ await expect(store.assign(identity)).resolves.toMatchObject({
+ cellId: assignment.cellId,
+ assignmentEpoch: assignment.assignmentEpoch
+ })
+ expect(await cellReservations(database!)).toEqual({ 'cell-a': 1, 'cell-b': 0 })
+ expect(
+ await database!.query(
+ `SELECT activity_id FROM relay_assignment_activity_leases`
+ )
+ ).toEqual([{ activity_id: 'control-pending:1' }])
+ expect(
+ await database!.query(
+ `SELECT reserved_controls FROM relay_assignments
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([{ reserved_controls: 1 }])
+ })
+
+ it('re-pins a host that took control after the sticky lane read it dormant', async () => {
+ const now = 100
+ const delegate = await openInMemoryRelayDatabase()
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const seeded = new SeedBetweenAssignmentLanesDatabase(delegate, async () => {
+ await delegate.query(
+ `INSERT INTO relay_assignment_activity_leases
+ (user_id, relay_host_id, activity_id, activity_kind, cell_id,
+ request_units, expires_at, updated_at)
+ VALUES (?, ?, 'control:cell-a:1', 'control', 'cell-a', 1, ?, ?),
+ (?, ?, 'control-pending:1', 'control', 'cell-a', 1, ?, ?)`,
+ [
+ identity.userId,
+ identity.relayHostId,
+ now + ASSIGNMENT_LIMITS.activityLeaseMs,
+ now,
+ identity.userId,
+ identity.relayHostId,
+ now + ASSIGNMENT_LIMITS.activityLeaseMs,
+ now
+ ]
+ )
+ await delegate.query(
+ `UPDATE relay_assignments SET reserved_controls = 2
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ await delegate.query(
+ `UPDATE relay_cells SET reserved_requests = 2 WHERE cell_id = 'cell-a'`
+ )
+ })
+ database = seeded
+ const store = new RelayAssignmentStore(seeded, () => now)
+ await store.reconcileCells(CELLS)
+ await delegate.query(
+ `INSERT INTO relay_assignments
+ (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at,
+ last_activity_at, reserved_controls, reserved_splices, reserved_invites,
+ pending_installs, pending_confirmations, migration_leases)
+ VALUES (?, ?, 'cell-a', 1, ?, ?, 0, 0, 0, 0, 0, 0)`,
+ [identity.userId, identity.relayHostId, now, now - ASSIGNMENT_LIMITS.dormantTtlMs]
+ )
+ seeded.arm()
+
+ await expect(store.assign(identity)).resolves.toMatchObject({
+ cellId: 'cell-a',
+ assignmentEpoch: 1
+ })
+ expect(
+ await delegate.query(
+ `SELECT reserved_controls FROM relay_assignments
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([{ reserved_controls: 2 }])
+ expect(await cellReservations(delegate)).toEqual({ 'cell-a': 2, 'cell-b': 0 })
+ expect(
+ await delegate.query(
+ `SELECT lease_expires_at, last_activity_at FROM relay_assignments
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([
+ { lease_expires_at: now + ASSIGNMENT_LIMITS.activityLeaseMs, last_activity_at: now }
+ ])
+ })
+
+ it('reserves control on the pinned cell when the only lease sits on another', async () => {
+ const now = 100
+ const store = await setup(() => now)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ await database!.query(
+ `INSERT INTO relay_assignments
+ (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at,
+ last_activity_at, reserved_controls, reserved_splices, reserved_invites,
+ pending_installs, pending_confirmations, migration_leases)
+ VALUES (?, ?, 'cell-b', 2, ?, ?, 1, 0, 0, 0, 0, 0)`,
+ [
+ identity.userId,
+ identity.relayHostId,
+ now + ASSIGNMENT_LIMITS.activityLeaseMs,
+ now
+ ]
+ )
+ await database!.query(
+ `INSERT INTO relay_assignment_activity_leases
+ (user_id, relay_host_id, activity_id, activity_kind, cell_id,
+ request_units, expires_at, updated_at)
+ VALUES (?, ?, 'control:cell-a:1', 'control', 'cell-a', 1, ?, ?)`,
+ [
+ identity.userId,
+ identity.relayHostId,
+ now + ASSIGNMENT_LIMITS.activityLeaseMs,
+ now
+ ]
+ )
+ await database!.query(
+ `UPDATE relay_cells SET reserved_requests = 1 WHERE cell_id = 'cell-a'`
+ )
+
+ await expect(store.assign(identity)).resolves.toMatchObject({
+ cellId: 'cell-b',
+ assignmentEpoch: 2
+ })
+ expect(
+ await database!.query(
+ `SELECT activity_id, cell_id FROM relay_assignment_activity_leases
+ ORDER BY activity_id ASC`
+ )
+ ).toEqual([
+ { activity_id: 'control-pending:2', cell_id: 'cell-b' },
+ { activity_id: 'control:cell-a:1', cell_id: 'cell-a' }
+ ])
+ expect(await cellReservations(database!)).toEqual({ 'cell-a': 1, 'cell-b': 1 })
+ expect(
+ await database!.query(
+ `SELECT reserved_controls FROM relay_assignments
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([{ reserved_controls: 2 }])
+ })
+
+ it('mints a pending control when the only lease is an older epoch pending', async () => {
+ const now = 100
+ const store = await setup(() => now)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ await database!.query(
+ `INSERT INTO relay_assignments
+ (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at,
+ last_activity_at, reserved_controls, reserved_splices, reserved_invites,
+ pending_installs, pending_confirmations, migration_leases)
+ VALUES (?, ?, 'cell-a', 3, ?, ?, 1, 0, 0, 0, 0, 0)`,
+ [
+ identity.userId,
+ identity.relayHostId,
+ now + ASSIGNMENT_LIMITS.activityLeaseMs,
+ now
+ ]
+ )
+ await database!.query(
+ `INSERT INTO relay_assignment_activity_leases
+ (user_id, relay_host_id, activity_id, activity_kind, cell_id,
+ request_units, expires_at, updated_at)
+ VALUES (?, ?, 'control-pending:1', 'control', 'cell-a', 1, ?, ?)`,
+ [
+ identity.userId,
+ identity.relayHostId,
+ now + ASSIGNMENT_LIMITS.activityLeaseMs,
+ now
+ ]
+ )
+ await database!.query(
+ `UPDATE relay_cells SET reserved_requests = 1 WHERE cell_id = 'cell-a'`
+ )
+
+ await expect(store.assign(identity)).resolves.toMatchObject({
+ cellId: 'cell-a',
+ assignmentEpoch: 3
+ })
+ expect(
+ await database!.query(
+ `SELECT activity_id FROM relay_assignment_activity_leases
+ ORDER BY activity_id ASC`
+ )
+ ).toEqual([{ activity_id: 'control-pending:1' }, { activity_id: 'control-pending:3' }])
+ expect(await cellReservations(database!)).toEqual({ 'cell-a': 2, 'cell-b': 0 })
+ expect(
+ await database!.query(
+ `SELECT reserved_controls FROM relay_assignments
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([{ reserved_controls: 2 }])
+ })
+
+ it('re-reserves a re-pinned grant for a host holding no control lease', async () => {
+ const now = 100
+ const delegate = await openInMemoryRelayDatabase()
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const seeded = new SeedBetweenAssignmentLanesDatabase(delegate, async () => {
+ await delegate.query(
+ `INSERT INTO relay_assignment_activity_leases
+ (user_id, relay_host_id, activity_id, activity_kind, cell_id,
+ request_units, expires_at, updated_at)
+ VALUES (?, ?, 'splice:connection-1', 'splice', 'cell-a', 1, ?, ?)`,
+ [
+ identity.userId,
+ identity.relayHostId,
+ now + ASSIGNMENT_LIMITS.activityLeaseMs,
+ now
+ ]
+ )
+ await delegate.query(
+ `UPDATE relay_assignments SET reserved_splices = 1
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ await delegate.query(
+ `UPDATE relay_cells SET reserved_requests = 1 WHERE cell_id = 'cell-a'`
+ )
+ })
+ database = seeded
+ const store = new RelayAssignmentStore(seeded, () => now)
+ await store.reconcileCells(CELLS)
+ await delegate.query(
+ `INSERT INTO relay_assignments
+ (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at,
+ last_activity_at, reserved_controls, reserved_splices, reserved_invites,
+ pending_installs, pending_confirmations, migration_leases)
+ VALUES (?, ?, 'cell-a', 1, ?, ?, 0, 0, 0, 0, 0, 0)`,
+ [identity.userId, identity.relayHostId, now, now - ASSIGNMENT_LIMITS.dormantTtlMs]
+ )
+ seeded.arm()
+
+ await expect(store.assign(identity)).resolves.toMatchObject({
+ cellId: 'cell-a',
+ assignmentEpoch: 1
+ })
+ expect(
+ await delegate.query(
+ `SELECT reserved_controls, reserved_splices FROM relay_assignments
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([{ reserved_controls: 1, reserved_splices: 1 }])
+ expect(
+ await delegate.query(
+ `SELECT activity_id FROM relay_assignment_activity_leases
+ ORDER BY activity_id ASC`
+ )
+ ).toEqual([{ activity_id: 'control-pending:1' }, { activity_id: 'splice:connection-1' }])
+ expect(await cellReservations(delegate)).toEqual({ 'cell-a': 2, 'cell-b': 0 })
+ })
+
+ it('extends the lease of a control-holding host on a sticky grant', async () => {
+ let now = 100
+ const store = await setup(() => now)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const assignment = await store.assign(identity)
+ await store.activateControl(identity, {
+ cellId: assignment.cellId,
+ assignmentEpoch: assignment.assignmentEpoch,
+ generation: 1
+ })
+
+ now = 40_000
+ await expect(store.assign(identity)).resolves.toMatchObject({
+ cellId: assignment.cellId
+ })
+ expect(
+ await database!.query(
+ `SELECT lease_expires_at, last_activity_at FROM relay_assignments
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([
+ { lease_expires_at: now + ASSIGNMENT_LIMITS.activityLeaseMs, last_activity_at: now }
+ ])
+ })
+
+ // The old absolute write shortened a 15-minute migration lease to the 90s
+ // grant lease; only the touch's monotonic CASE keeps the longer one now.
+ it('never shortens a migration lease to the grant lease', async () => {
+ let now = 100
+ const store = await setup(() => now)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const assignment = await store.assign(identity)
+ await store.activateControl(identity, {
+ cellId: assignment.cellId,
+ assignmentEpoch: assignment.assignmentEpoch,
+ generation: 1
+ })
+ const migration = await store.startEvacuation(identity, 'cell-b')
+ await store.activateControl(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch,
+ generation: 1
+ })
+ const before = await database!.query(
+ `SELECT lease_expires_at FROM relay_assignments
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ expect(before).toEqual([{ lease_expires_at: 100 + ASSIGNMENT_LIMITS.migrationLeaseMs }])
+
+ now = 1000
+ await expect(store.assign(identity)).resolves.toMatchObject({ cellId: 'cell-b' })
+ expect(
+ await database!.query(
+ `SELECT lease_expires_at, last_activity_at FROM relay_assignments
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([
+ { lease_expires_at: 100 + ASSIGNMENT_LIMITS.migrationLeaseMs, last_activity_at: now }
+ ])
+ })
+
+ it('moves an origin-scoped activity reservation without double counting', async () => {
+ const store = await setup(() => 100, [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
+ ])
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ await store.assign(identity)
+ await store.acquireActivity(identity, {
+ activityId: 'splice:connection-1',
+ kind: 'splice',
+ cellId: 'cell-a'
+ })
+ await store.startEvacuation(identity, 'cell-b')
+ await store.acquireActivity(identity, {
+ activityId: 'splice:connection-1',
+ kind: 'splice',
+ cellId: 'cell-b'
+ })
+
+ expect(await cellReservations(database!)).toEqual({ 'cell-a': 1, 'cell-b': 6 })
+ await expect(store.releaseActivity(identity, 'splice:connection-1')).resolves.toBe(true)
+ await expect(store.releaseActivity(identity, 'splice:connection-1')).resolves.toBe(false)
+ expect(await cellReservations(database!)).toEqual({ 'cell-a': 1, 'cell-b': 4 })
+ })
+
+ it('rejects a durable activity before it can exceed cell capacity', async () => {
+ const store = await setup(() => 100, [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 2 }
+ ])
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ await store.assign(identity)
+
+ await expect(
+ store.acquireActivity(identity, {
+ activityId: 'splice:connection-1',
+ kind: 'splice',
+ cellId: 'cell-a'
+ })
+ ).rejects.toThrow('relay_capacity_exhausted')
+ expect(await cellReservations(database!)).toEqual({ 'cell-a': 1 })
+ })
+
+ it('moves active assignments target-first and completes only after source drain', async () => {
+ const store = await setup(() => 100, [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
+ ])
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ const sourceControl = await store.activateControl(identity, {
+ cellId: first.cellId,
+ assignmentEpoch: first.assignmentEpoch,
+ generation: 1
+ })
+
+ const migration = await store.startEvacuation(identity, 'cell-b')
+ expect(await store.startEvacuation(identity, 'cell-b')).toEqual(migration)
+ expect(migration).toMatchObject({
+ sourceCellId: 'cell-a',
+ targetCellId: 'cell-b',
+ previousEpoch: 1,
+ assignmentEpoch: 2
+ })
+ expect(await cellReservations(database!)).toEqual({ 'cell-a': 1, 'cell-b': 2 })
+
+ const targetControl = await store.activateControl(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: 2,
+ generation: 1
+ })
+ await store.markMigrationTargetRegistered(identity, { cellId: 'cell-b', assignmentEpoch: 2 })
+ await expect(store.completeEvacuation(identity, 2)).rejects.toThrow(
+ 'migration_source_still_active'
+ )
+ await store.releaseActivity(identity, sourceControl)
+ await store.completeEvacuation(identity, 2)
+
+ expect(await cellReservations(database!)).toEqual({ 'cell-a': 0, 'cell-b': 1 })
+ expect(await store.resolve(identity)).toMatchObject({ cellId: 'cell-b', assignmentEpoch: 2 })
+ expect(
+ await database!.query(
+ `SELECT activity_id FROM relay_assignment_activity_leases ORDER BY activity_id`
+ )
+ ).toEqual([{ activity_id: targetControl }])
+ await expect(
+ store.acquireActivity(identity, {
+ activityId: 'splice:late-source',
+ kind: 'splice',
+ cellId: 'cell-a'
+ })
+ ).rejects.toThrow('activity_cell_not_authoritative')
+ })
+
+ it('automatically completes only after the source runtime is freshly quiescent', async () => {
+ let now = 100
+ const cells = [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
+ ]
+ const store = await setupWithHeartbeats(() => now, cells)
+ await heartbeat(store, cells[0]!)
+ await heartbeat(store, cells[1]!)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ const sourceControl = await store.activateControl(identity, {
+ cellId: first.cellId,
+ assignmentEpoch: first.assignmentEpoch,
+ generation: 1
+ })
+ const migration = await store.startEvacuation(identity, 'cell-b')
+ await store.activateControl(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch,
+ generation: 1
+ })
+ await store.markMigrationTargetRegistered(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch
+ })
+ await store.releaseActivity(identity, sourceControl)
+ await store.setCellEnabled('cell-a', false)
+ await heartbeat(store, cells[0]!, { observedRequests: 1 })
+
+ expect(await store.completeReadyEvacuations()).toBe(0)
+ await heartbeat(store, cells[0]!, { observedRequests: 0 })
+ now = 150
+ await heartbeat(store, cells[1]!, {
+ incarnation: '22222222-2222-4222-8222-222222222222',
+ startedAt: 125
+ })
+ expect(await store.completeReadyEvacuations()).toBe(0)
+ await store.activateControl(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch,
+ generation: 2
+ })
+ expect(await store.completeReadyEvacuations()).toBe(1)
+ })
+
+ it('completes a fenced migration only after the source heartbeat is stale', async () => {
+ let now = 100
+ const cells = [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
+ ]
+ const store = await setupWithHeartbeats(() => now, cells)
+ await heartbeat(store, cells[0]!, { observedRequests: 1 })
+ await heartbeat(store, cells[1]!)
+ await store.setCellEnabled('cell-b', false)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ const sourceControl = await store.activateControl(identity, {
+ cellId: first.cellId,
+ assignmentEpoch: first.assignmentEpoch,
+ generation: 1
+ })
+ await store.setCellEnabled('cell-b', true)
+ const migration = await store.startEvacuation(identity, 'cell-b')
+ await store.activateControl(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch,
+ generation: 1
+ })
+ await store.markMigrationTargetRegistered(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch
+ })
+ await store.releaseActivity(identity, sourceControl)
+ await store.setCellEnabled('cell-a', false)
+
+ expect(await store.cellEvacuationStatus('cell-a', 'cell-b', true)).toMatchObject({
+ inProgress: 1,
+ blocked: 1
+ })
+ now += 45_001
+ await heartbeat(store, cells[1]!)
+ await store.attestCellFence(
+ 'cell-a',
+ '11111111-1111-4111-8111-111111111111'
+ )
+ expect(await store.cellEvacuationStatus('cell-a', 'cell-b', true)).toMatchObject({
+ inProgress: 0,
+ completed: 1
+ })
+ })
+
+ it('blocks aggregate fenced completion on assignment accounting mismatch', async () => {
+ let now = 100
+ const cells = [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
+ ]
+ const store = await setupWithHeartbeats(() => now, cells)
+ await heartbeat(store, cells[0]!)
+ await heartbeat(store, cells[1]!)
+ await store.setCellEnabled('cell-b', false)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ const sourceControl = await store.activateControl(identity, {
+ cellId: first.cellId,
+ assignmentEpoch: first.assignmentEpoch,
+ generation: 1
+ })
+ await store.setCellEnabled('cell-b', true)
+ const migration = await store.startEvacuation(identity, 'cell-b')
+ await store.activateControl(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch,
+ generation: 1
+ })
+ await store.markMigrationTargetRegistered(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch
+ })
+ await store.releaseActivity(identity, sourceControl)
+ await store.setCellEnabled('cell-a', false)
+ await database!.query(
+ `UPDATE relay_assignments SET reserved_splices = reserved_splices + 1
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ now += 45_001
+ await heartbeat(store, cells[1]!)
+ await store.attestCellFence(
+ 'cell-a',
+ '11111111-1111-4111-8111-111111111111'
+ )
+
+ expect(await store.cellEvacuationStatus('cell-a', 'cell-b', true)).toMatchObject({
+ inProgress: 1,
+ blocked: 1
+ })
+ })
+
+ it('blocks aggregate fenced completion on an unexpected third-cell lease', async () => {
+ let now = 100
+ const cells = [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 },
+ { id: 'cell-c', url: 'https://relay-c.example.com', capacityRequests: 10 }
+ ]
+ const store = await setupWithHeartbeats(() => now, cells)
+ for (const cell of cells) await heartbeat(store, cell)
+ await store.setCellEnabled('cell-b', false)
+ await store.setCellEnabled('cell-c', false)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ const sourceControl = await store.activateControl(identity, {
+ cellId: first.cellId,
+ assignmentEpoch: first.assignmentEpoch,
+ generation: 1
+ })
+ await store.setCellEnabled('cell-b', true)
+ const migration = await store.startEvacuation(identity, 'cell-b')
+ await store.activateControl(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch,
+ generation: 1
+ })
+ await store.markMigrationTargetRegistered(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch
+ })
+ await store.releaseActivity(identity, sourceControl)
+ await store.setCellEnabled('cell-a', false)
+ await database!.query(
+ `INSERT INTO relay_assignment_activity_leases
+ (user_id, relay_host_id, activity_id, activity_kind, cell_id,
+ request_units, expires_at, updated_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
+ [
+ identity.userId,
+ identity.relayHostId,
+ 'splice:unexpected-cell',
+ 'splice',
+ 'cell-c',
+ 2,
+ now + ASSIGNMENT_LIMITS.activityLeaseMs,
+ now
+ ]
+ )
+ await database!.query(
+ `UPDATE relay_assignments SET reserved_splices = reserved_splices + 1
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ await database!.query(
+ `UPDATE relay_cells SET reserved_requests = reserved_requests + 2 WHERE cell_id = ?`,
+ ['cell-c']
+ )
+ now += 45_001
+ await heartbeat(store, cells[1]!)
+ await store.attestCellFence(
+ 'cell-a',
+ '11111111-1111-4111-8111-111111111111'
+ )
+
+ expect(await store.cellEvacuationStatus('cell-a', 'cell-b', true)).toMatchObject({
+ inProgress: 1,
+ blocked: 1
+ })
+ })
+
+ it('rolls back an unregistered expired evacuation with a strictly newer epoch', async () => {
+ let now = 100
+ const store = await setup(() => now, [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
+ ])
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ await store.activateControl(identity, {
+ cellId: first.cellId,
+ assignmentEpoch: first.assignmentEpoch,
+ generation: 1
+ })
+ await store.startEvacuation(identity, 'cell-b')
+
+ now += ASSIGNMENT_LIMITS.migrationLeaseMs + 1
+ expect(await store.abortExpiredEvacuations()).toBe(1)
+ expect(await store.resolve(identity)).toMatchObject({ cellId: 'cell-a', assignmentEpoch: 3 })
+ expect(await cellReservations(database!)).toEqual({ 'cell-a': 1, 'cell-b': 0 })
+ })
+
+ // Why: completion needs an enabled target and expiry rollback needs an unregistered one, so a
+ // target disabled after it registered satisfies neither and the migration never leaves the table.
+ async function wedgeRegisteredMigration(
+ now: () => number,
+ cells: RelayCellConfig[],
+ disableTarget = true,
+ disableSource = true,
+ releaseSourceControl = true
+ ): Promise<{
+ store: RelayAssignmentStore
+ identity: { userId: string; relayHostId: string }
+ }> {
+ const store = await setupWithHeartbeats(now, cells)
+ await heartbeat(store, cells[0]!)
+ await heartbeat(store, cells[1]!)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ const sourceControl = await store.activateControl(identity, {
+ cellId: first.cellId,
+ assignmentEpoch: first.assignmentEpoch,
+ generation: 1
+ })
+ const migration = await store.startEvacuation(identity, 'cell-b')
+ const targetControl = await store.activateControl(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch,
+ generation: 1
+ })
+ await store.markMigrationTargetRegistered(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch
+ })
+ if (releaseSourceControl) await store.releaseActivity(identity, sourceControl)
+ if (disableSource) await store.setCellEnabled('cell-a', false)
+ // The desktop leaves the half-migrated target, then an operator disables that cell.
+ await store.releaseActivity(identity, targetControl)
+ if (disableTarget) await store.setCellEnabled('cell-b', false)
+ return { store, identity }
+ }
+
+ async function insertReservedControlConnection(
+ identity: { userId: string; relayHostId: string },
+ cellId: string,
+ assignmentEpoch: number,
+ now: number
+ ): Promise {
+ await database!.query(
+ `INSERT INTO relay_control_connection_reservations
+ (reservation_id, idempotency_key, user_id, relay_host_id, assignment_epoch,
+ cell_id, state, created_at, timeout_at, updated_at)
+ VALUES ('abandoned-reservation', 'abandoned-key', ?, ?, ?, ?, 'reserved', ?, ?, ?)`,
+ [identity.userId, identity.relayHostId, assignmentEpoch, cellId, now, now + 60_000, now]
+ )
+ }
+
+ it('reaps an expired migration whose registered target cell was disabled', async () => {
+ let now = 100
+ const cells = [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
+ ]
+ const { store, identity } = await wedgeRegisteredMigration(() => now, cells)
+
+ now += ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1
+ // A disabled target can never satisfy completion, so waiting cannot help.
+ expect(await store.completeReadyEvacuations()).toBe(0)
+ expect(await store.abortExpiredEvacuations()).toBe(1)
+ // Rollback lands on the disabled source, so the host resolves to nothing and is placed afresh
+ // by evacuateDeadCells; the point is that the row is retired rather than reaped every tick.
+ expect(await store.resolve(identity)).toBeNull()
+ expect(await store.abortExpiredEvacuations()).toBe(0)
+ })
+
+ it('leaves a freshly disabled target for supersede-target to recover', async () => {
+ let now = 100
+ const cells = [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
+ ]
+ const { store } = await wedgeRegisteredMigration(() => now, cells)
+
+ // Expired, but a disabled target is what supersede-target itself creates, so the operator
+ // window must stay open; aborting here would fail their run with migration_already_superseded.
+ now += ASSIGNMENT_LIMITS.migrationLeaseMs + 1
+ expect(await store.abortExpiredEvacuations()).toBe(0)
+
+ now += STRANDED_MIGRATION_ABANDON_MS
+ expect(await store.abortExpiredEvacuations()).toBe(1)
+ })
+
+ it('rolls an abandoned disabled target back while its source remains active', async () => {
+ let now = 100
+ const cells = [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
+ ]
+ const { store, identity } = await wedgeRegisteredMigration(
+ () => now,
+ cells,
+ true,
+ false,
+ false
+ )
+
+ now += ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1
+ expect(await store.abortExpiredEvacuations()).toBe(1)
+ expect(
+ await database!.query(
+ `SELECT cell_id, assignment_epoch FROM relay_assignments
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([{ cell_id: 'cell-a', assignment_epoch: 3 }])
+ expect(
+ await database!.query(
+ `SELECT completed_at, aborted_at FROM relay_assignment_migrations
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([{ completed_at: null, aborted_at: now }])
+ })
+
+ it('starts the abandon window when an old migration target is disabled', async () => {
+ let now = 100
+ const cells = [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
+ ]
+ const { store } = await wedgeRegisteredMigration(() => now, cells, false, false)
+ now += ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1
+
+ await store.setCellEnabled('cell-b', false)
+ expect(await store.abortExpiredEvacuations()).toBe(0)
+
+ now += STRANDED_MIGRATION_ABANDON_MS + 1
+ expect(await store.abortExpiredEvacuations()).toBe(1)
+ })
+
+ it('reaps an expired migration from a retired source when its target control is gone', async () => {
+ let now = 100
+ const cells = [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
+ ]
+ const { store, identity } = await wedgeRegisteredMigration(() => now, cells, false)
+ await applyGenerationZeroSelector(store, {
+ attemptId: 'retired_source_migration_target',
+ membership: {
+ existingOnly: ['cell-a'],
+ migrationOnly: ['cell-b'],
+ general: []
+ }
+ })
+ await insertReservedControlConnection(identity, 'cell-b', 2, now)
+
+ now += ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1
+ expect(await store.abortExpiredEvacuations()).toBe(1)
+ expect(
+ await database!.query(
+ `SELECT assignment.cell_id, assignment.assignment_epoch,
+ migration.completed_at, migration.aborted_at
+ FROM relay_assignments assignment
+ JOIN relay_assignment_migrations migration
+ ON migration.user_id = assignment.user_id
+ AND migration.relay_host_id = assignment.relay_host_id
+ WHERE assignment.user_id = ? AND assignment.relay_host_id = ?
+ AND migration.assignment_epoch = ?`,
+ [identity.userId, identity.relayHostId, 2]
+ )
+ ).toEqual([
+ {
+ cell_id: 'cell-b',
+ assignment_epoch: 2,
+ completed_at: now,
+ aborted_at: null
+ }
+ ])
+ expect(
+ await database!.query(
+ `SELECT reserved_controls, migration_leases FROM relay_assignments
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([{ reserved_controls: 0, migration_leases: 0 }])
+ expect(
+ await database!.query(
+ `SELECT activity_id FROM relay_assignment_activity_leases
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([])
+ expect(await cellReservations(database!)).toEqual({ 'cell-a': 0, 'cell-b': 0 })
+ expect(
+ await database!.query(
+ `SELECT state FROM relay_control_connection_reservations
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([{ state: 'released' }])
+ })
+
+ it('reaps a retired-side migration with only a pending target control', async () => {
+ let now = 100
+ const cells = [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
+ ]
+ const store = await setupWithHeartbeats(() => now, cells)
+ await heartbeat(store, cells[0]!)
+ await heartbeat(store, cells[1]!)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ const sourceControl = await store.activateControl(identity, {
+ cellId: first.cellId,
+ assignmentEpoch: first.assignmentEpoch,
+ generation: 1
+ })
+ const migration = await store.startEvacuation(identity, 'cell-b')
+ await store.markMigrationTargetRegistered(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch
+ })
+ await store.releaseActivity(identity, sourceControl)
+ await applyGenerationZeroSelector(store, {
+ attemptId: 'retired_source_pending_target',
+ membership: {
+ existingOnly: ['cell-a'],
+ migrationOnly: ['cell-b'],
+ general: []
+ }
+ })
+ await insertReservedControlConnection(identity, 'cell-b', migration.assignmentEpoch, now)
+
+ now += ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1
+ await database!.query(
+ `UPDATE relay_assignment_activity_leases SET expires_at = ?
+ WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`,
+ [now, identity.userId, identity.relayHostId, `control-pending:${migration.assignmentEpoch}`]
+ )
+ expect(await store.abortExpiredEvacuations()).toBe(1)
+ expect(
+ await database!.query(
+ `SELECT cell_id, assignment_epoch FROM relay_assignments
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([{ cell_id: 'cell-b', assignment_epoch: migration.assignmentEpoch }])
+ expect(
+ await database!.query(
+ `SELECT reserved_controls, migration_leases FROM relay_assignments
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([{ reserved_controls: 0, migration_leases: 0 }])
+ expect(
+ await database!.query(
+ `SELECT activity_id FROM relay_assignment_activity_leases
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([])
+ expect(await cellReservations(database!)).toEqual({ 'cell-a': 0, 'cell-b': 0 })
+ expect(
+ await database!.query(
+ `SELECT state FROM relay_control_connection_reservations
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([{ state: 'released' }])
+ })
+
+ it('preserves a fresh target grant created after abandoned migration selection', async () => {
+ let now = 100
+ const cells: RelayCellConfig[] = [
+ {
+ id: 'cell-a',
+ url: 'https://relay-a.example.com',
+ capacityRequests: 10,
+ connectionHardCap: 600,
+ connectionUnobservedBound: 50
+ },
+ {
+ id: 'cell-b',
+ url: 'https://relay-b.example.com',
+ capacityRequests: 10,
+ connectionHardCap: 600,
+ connectionUnobservedBound: 50
+ }
+ ]
+ const { store, identity } = await wedgeRegisteredMigration(() => now, cells, false)
+ await applyGenerationZeroSelector(store, {
+ attemptId: 'retired_source_fresh_target_grant',
+ membership: {
+ existingOnly: ['cell-a'],
+ migrationOnly: ['cell-b'],
+ general: []
+ }
+ })
+
+ now += 45_001
+ await heartbeat(store, cells[1]!)
+ await store.adoptLegacyCellFence(
+ 'cell-a',
+ '11111111-1111-4111-8111-111111111111'
+ )
+ await store.commitLegacyCellFenceAdoption(
+ 'cell-a',
+ '11111111-1111-4111-8111-111111111111'
+ )
+ now += ASSIGNMENT_LIMITS.migrationLeaseMs + 1
+ await heartbeat(store, cells[1]!)
+ const query = database!.query.bind(database)
+ let grant: Awaited> | undefined
+ vi.spyOn(database!, 'query').mockImplementationOnce(async (sql, params) => {
+ const rows = await query(sql, params)
+ grant = await store.assign(identity)
+ return rows
+ })
+
+ expect(await store.abortExpiredEvacuations()).toBe(0)
+ expect(grant).toMatchObject({ cellId: 'cell-b', assignmentEpoch: 2 })
+ expect(
+ await database!.query(
+ `SELECT activity_id, expires_at FROM relay_assignment_activity_leases
+ WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`,
+ [identity.userId, identity.relayHostId, 'control-pending:2']
+ )
+ ).toEqual([{ activity_id: 'control-pending:2', expires_at: grant!.leaseExpiresAt }])
+ expect(
+ await database!.query(
+ `SELECT state FROM relay_control_connection_reservations
+ WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?
+ AND cell_id = ? AND state = 'reserved'`,
+ [identity.userId, identity.relayHostId, 2, 'cell-b']
+ )
+ ).toEqual([{ state: 'reserved' }])
+
+ await store.activateControl(identity, {
+ cellId: grant!.cellId,
+ assignmentEpoch: grant!.assignmentEpoch,
+ generation: 2
+ })
+ expect(
+ await database!.query(
+ `SELECT activity_id FROM relay_assignment_activity_leases
+ WHERE user_id = ? AND relay_host_id = ? AND activity_kind = 'control'`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([{ activity_id: 'control:cell-b:2' }])
+ expect(
+ await database!.query(
+ `SELECT completed_at FROM relay_assignment_migrations
+ WHERE user_id = ? AND relay_host_id = ? AND assignment_epoch = ?`,
+ [identity.userId, identity.relayHostId, 2]
+ )
+ ).toEqual([{ completed_at: null }])
+ })
+
+ it('keeps an abandoned target migration open while its source is still active', async () => {
+ let now = 100
+ const cells = [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
+ ]
+ const { store, identity } = await wedgeRegisteredMigration(
+ () => now,
+ cells,
+ false,
+ true,
+ false
+ )
+ await applyGenerationZeroSelector(store, {
+ attemptId: 'retired_source_active',
+ membership: {
+ existingOnly: ['cell-a'],
+ migrationOnly: ['cell-b'],
+ general: []
+ }
+ })
+
+ now += ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1
+ expect(await store.abortExpiredEvacuations()).toBe(0)
+ expect(
+ await database!.query(
+ `SELECT completed_at, aborted_at FROM relay_assignment_migrations
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([{ completed_at: null, aborted_at: null }])
+ })
+
+ it('starts the abandon window when an old migration source is retired', async () => {
+ let now = 100
+ const cells = [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
+ ]
+ const { store } = await wedgeRegisteredMigration(() => now, cells, false, false)
+ now += ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1
+
+ await store.setCellEnabled('cell-a', false)
+ expect(await store.abortExpiredEvacuations()).toBe(0)
+
+ now += STRANDED_MIGRATION_ABANDON_MS + 1
+ expect(await store.abortExpiredEvacuations()).toBe(1)
+ })
+
+ it('reaps after an old source migration lease is refreshed and expires', async () => {
+ let now = 100
+ const cells = [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
+ ]
+ const { store } = await wedgeRegisteredMigration(() => now, cells, false)
+ await applyGenerationZeroSelector(store, {
+ attemptId: 'retired_source_refreshed_lease',
+ membership: {
+ existingOnly: ['cell-a'],
+ migrationOnly: ['cell-b'],
+ general: []
+ }
+ })
+ now += STRANDED_MIGRATION_ABANDON_MS + ASSIGNMENT_LIMITS.migrationLeaseMs + 1
+ await database!.query(
+ `UPDATE relay_assignment_migrations SET expires_at = ?`,
+ [now - 1]
+ )
+
+ expect(await store.abortExpiredEvacuations()).toBe(1)
+ })
+
+ it('keeps a freshly retired target open despite an old retired source', async () => {
+ let now = 100
+ const cells = [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
+ ]
+ const { store } = await wedgeRegisteredMigration(() => now, cells, false)
+ now += ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1
+
+ await store.setCellEnabled('cell-b', false)
+ expect(await store.abortExpiredEvacuations()).toBe(0)
+
+ now += STRANDED_MIGRATION_ABANDON_MS + 1
+ expect(await store.abortExpiredEvacuations()).toBe(1)
+ })
+
+ it('fails supersession closed when reaping wins after selection', async () => {
+ let now = 100
+ const cells = [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 },
+ { id: 'cell-c', url: 'https://relay-c.example.com', capacityRequests: 10 }
+ ]
+ const { store } = await wedgeRegisteredMigration(() => now, cells)
+ now += ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1
+ const query = database!.query.bind(database)
+ vi.spyOn(database!, 'query').mockImplementationOnce(async (sql, params) => {
+ const rows = await query(sql, params)
+ expect(await store.abortExpiredEvacuations()).toBe(1)
+ return rows
+ })
+
+ await expect(
+ store.supersedeRegisteredCellEvacuations('cell-a', 'cell-b', 'cell-c', 100)
+ ).rejects.toThrow('migration_already_superseded')
+ })
+
+ it('fails supersession closed when reaping wins after preparation', async () => {
+ let now = 100
+ const cells = [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 },
+ { id: 'cell-c', url: 'https://relay-c.example.com', capacityRequests: 10 }
+ ]
+ const { store } = await wedgeRegisteredMigration(() => now, cells)
+ now += ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1
+ await store.attestCellFence('cell-b', '11111111-1111-4111-8111-111111111111')
+ const supersede = store.supersedeRegisteredEvacuation.bind(store)
+ vi.spyOn(store, 'supersedeRegisteredEvacuation').mockImplementationOnce(async (...args) => {
+ expect(await store.abortExpiredEvacuations()).toBe(1)
+ return await supersede(...args)
+ })
+
+ await expect(
+ store.supersedeRegisteredCellEvacuations('cell-a', 'cell-b', 'cell-c', 100)
+ ).rejects.toThrow('migration_already_superseded')
+ })
+
+ it('fails supersession closed when reaping wins before an accounting retry', async () => {
+ let now = 100
+ const cells = [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 },
+ { id: 'cell-c', url: 'https://relay-c.example.com', capacityRequests: 10 }
+ ]
+ const { store } = await wedgeRegisteredMigration(() => now, cells)
+ now += ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1
+ await heartbeat(store, cells[0]!)
+ await heartbeat(store, cells[2]!)
+ await store.attestCellFence('cell-b', '11111111-1111-4111-8111-111111111111')
+ await database!.query(`UPDATE relay_cells SET reserved_requests = 1 WHERE cell_id = ?`, [
+ 'cell-c'
+ ])
+ const supersede = store.supersedeRegisteredEvacuation.bind(store)
+ let attempts = 0
+ vi.spyOn(store, 'supersedeRegisteredEvacuation').mockImplementation(async (...args) => {
+ attempts++
+ if (attempts === 2) expect(await store.abortExpiredEvacuations()).toBe(1)
+ return await supersede(...args)
+ })
+
+ await expect(
+ store.supersedeRegisteredCellEvacuations('cell-a', 'cell-b', 'cell-c', 100)
+ ).rejects.toThrow('migration_already_superseded')
+ expect(attempts).toBe(2)
+ })
+
+ it('reaps a pinned expired migration once its target cell is disabled', async () => {
+ let now = 100
+ const cells = [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
+ ]
+ const { store, identity } = await wedgeRegisteredMigration(() => now, cells)
+ // Drains pin their migrations and nothing ever deletes a pin, so the pin outlives the drain.
+ await database!.query(
+ `INSERT INTO relay_post_drain_migration_pins
+ (user_id, relay_host_id, assignment_epoch, drain_attempt_id, source_cell_id,
+ source_cell_incarnation, target_cell_id, target_cell_incarnation,
+ source_request_units, target_reserved_units, pinned_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+ [
+ identity.userId,
+ identity.relayHostId,
+ 2,
+ 'attempt-0000',
+ 'cell-a',
+ '11111111-1111-4111-8111-111111111111',
+ 'cell-b',
+ '11111111-1111-4111-8111-111111111111',
+ 1,
+ 1,
+ now
+ ]
+ )
+
+ now += ASSIGNMENT_LIMITS.migrationLeaseMs + STRANDED_MIGRATION_ABANDON_MS + 1
+ expect(await store.completeReadyEvacuations()).toBe(0)
+ expect(await store.abortExpiredEvacuations()).toBe(1)
+ expect(await store.abortExpiredEvacuations()).toBe(0)
+ })
+
+ it('repairs an expired migration marker from its exact active target control', async () => {
+ let now = 100
+ const store = await setup(() => now, [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
+ ])
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ const sourceControl = await store.activateControl(identity, {
+ cellId: first.cellId,
+ assignmentEpoch: first.assignmentEpoch,
+ generation: 1
+ })
+ await store.startEvacuation(identity, 'cell-b')
+ await store.activateControl(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: 2,
+ generation: 1
+ })
+
+ now += ASSIGNMENT_LIMITS.migrationLeaseMs + 1
+ await store.activateControl(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: 2,
+ generation: 1
+ })
+ expect(await store.abortExpiredEvacuations()).toBe(0)
+ expect(await store.cellEvacuationStatus('cell-a', 'cell-b', false)).toEqual({
+ inProgress: 1,
+ oldestExpiresAt: 100 + ASSIGNMENT_LIMITS.migrationLeaseMs,
+ oldestRemainingMs: -1,
+ targetRegistered: 1,
+ registeredSourceActive: 1,
+ registeredCompletable: 0,
+ registeredTargetInactive: 0,
+ completed: 0,
+ blocked: 0,
+ ...NO_EXPIRED_EVACUATION_DIAGNOSTICS
+ })
+ await store.releaseActivity(identity, sourceControl)
+ expect(await store.cellEvacuationStatus('cell-a', 'cell-b', true)).toEqual({
+ inProgress: 0,
+ ...NO_ACTIVE_MIGRATION_LEASE,
+ targetRegistered: 0,
+ ...NO_REGISTERED_EVACUATION_DIAGNOSTICS,
+ completed: 1,
+ blocked: 0,
+ ...NO_EXPIRED_EVACUATION_DIAGNOSTICS
+ })
+ expect(await store.resolve(identity)).toMatchObject({ cellId: 'cell-b', assignmentEpoch: 2 })
+ })
+
+ it('keeps a registered migration pending while its proven target control is offline', async () => {
+ let now = 100
+ const store = await setup(() => now, [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
+ ])
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ const sourceControl = await store.activateControl(identity, {
+ cellId: first.cellId,
+ assignmentEpoch: first.assignmentEpoch,
+ generation: 1
+ })
+ const migration = await store.startEvacuation(identity, 'cell-b')
+ const targetControl = await store.activateControl(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch,
+ generation: 2
+ })
+ expect(
+ await store.markMigrationTargetRegistered(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch
+ })
+ ).toBe(true)
+ expect(await store.completeReadyEvacuations()).toBe(0)
+ await store.releaseActivity(identity, sourceControl)
+ now += ASSIGNMENT_LIMITS.activityLeaseMs + 1
+ expect(await store.completeReadyEvacuations()).toBe(0)
+ await expect(
+ store.completeEvacuation(identity, migration.assignmentEpoch)
+ ).rejects.toThrow('migration_target_not_active')
+ await store.releaseActivity(identity, targetControl)
+ expect(await store.completeReadyEvacuations()).toBe(0)
+
+ expect(await store.cellEvacuationStatus('cell-a', 'cell-b', true)).toEqual({
+ inProgress: 1,
+ oldestExpiresAt: 100 + ASSIGNMENT_LIMITS.migrationLeaseMs,
+ oldestRemainingMs:
+ ASSIGNMENT_LIMITS.migrationLeaseMs - ASSIGNMENT_LIMITS.activityLeaseMs - 1,
+ targetRegistered: 1,
+ registeredSourceActive: 0,
+ registeredCompletable: 0,
+ registeredTargetInactive: 1,
+ completed: 0,
+ blocked: 1,
+ ...NO_EXPIRED_EVACUATION_DIAGNOSTICS
+ })
+
+ await store.activateControl(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch,
+ generation: 3
+ })
+ expect(await store.completeReadyEvacuations()).toBe(1)
+ expect(await store.cellEvacuationStatus('cell-a', 'cell-b', false)).toEqual({
+ inProgress: 0,
+ ...NO_ACTIVE_MIGRATION_LEASE,
+ targetRegistered: 0,
+ ...NO_REGISTERED_EVACUATION_DIAGNOSTICS,
+ completed: 0,
+ blocked: 0,
+ ...NO_EXPIRED_EVACUATION_DIAGNOSTICS
+ })
+ expect(await cellReservations(database!)).toEqual({ 'cell-a': 0, 'cell-b': 1 })
+ })
+
+ it('completes a registered migration after its disabled source is proven dead', async () => {
+ let now = 100
+ const store = await setupWithHeartbeats(() => now, [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
+ ])
+ await heartbeat(store, { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 })
+ await heartbeat(store, { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 })
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ const sourceControl = await store.activateControl(identity, {
+ cellId: first.cellId,
+ assignmentEpoch: first.assignmentEpoch,
+ generation: 1
+ })
+ await store.setCellEnabled('cell-a', false)
+ const migration = await store.startEvacuation(identity, 'cell-b')
+ await store.activateControl(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch,
+ generation: 1
+ })
+ await store.markMigrationTargetRegistered(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch
+ })
+ await store.releaseActivity(identity, sourceControl)
+ now += 45_001
+ await heartbeat(
+ store,
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 },
+ { startedAt: 50 }
+ )
+ await store.attestCellFence(
+ 'cell-a',
+ '11111111-1111-4111-8111-111111111111'
+ )
+
+ const input = {
+ assignmentEpoch: migration.assignmentEpoch,
+ sourceCellId: 'cell-a',
+ targetCellId: 'cell-b'
+ }
+ await expect(store.completeEvacuationFromDeadSource(identity, input)).resolves.toEqual({
+ changed: true,
+ assignmentEpoch: 2,
+ sourceCellId: 'cell-a',
+ targetCellId: 'cell-b'
+ })
+ await expect(store.completeEvacuationFromDeadSource(identity, input)).resolves.toEqual({
+ changed: false,
+ assignmentEpoch: 2,
+ sourceCellId: 'cell-a',
+ targetCellId: 'cell-b'
+ })
+ expect(await cellReservations(database!)).toEqual({ 'cell-a': 0, 'cell-b': 1 })
+ expect(await store.cellEvacuationStatus('cell-a', 'cell-b', false)).toMatchObject({
+ inProgress: 0
+ })
+ })
+
+ it('retires an inactive registered migration after its source is proven dead', async () => {
+ let now = 100
+ const cells = [
+ {
+ id: 'cell-a',
+ url: 'https://relay-a.example.com',
+ capacityRequests: 10,
+ connectionHardCap: 600 as const,
+ connectionUnobservedBound: 50
+ },
+ {
+ id: 'cell-b',
+ url: 'https://relay-b.example.com',
+ capacityRequests: 10,
+ connectionHardCap: 600 as const,
+ connectionUnobservedBound: 50
+ }
+ ]
+ const store = await setupWithHeartbeats(() => now, cells)
+ await heartbeat(store, cells[0]!)
+ await heartbeat(store, cells[1]!)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ const sourceControl = await store.activateControl(identity, {
+ cellId: first.cellId,
+ assignmentEpoch: first.assignmentEpoch,
+ generation: 1
+ })
+ await store.setCellEnabled('cell-a', false)
+ const migration = await store.startEvacuation(identity, 'cell-b')
+ await store.markMigrationTargetRegistered(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch
+ })
+ await store.releaseActivity(identity, sourceControl)
+ now += 45_001
+ await heartbeat(store, cells[1]!, { startedAt: 50 })
+ await store.attestCellFence('cell-a', '11111111-1111-4111-8111-111111111111')
+
+ await expect(
+ store.cellEvacuationStatus('cell-a', 'cell-b', true)
+ ).resolves.toMatchObject({ inProgress: 0, completed: 1, blocked: 0 })
+ expect(await cellReservations(database!)).toEqual({ 'cell-a': 0, 'cell-b': 0 })
+ expect(
+ await database!.query(
+ `SELECT activity_id FROM relay_assignment_activity_leases
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([])
+ expect(
+ await database!.query(
+ `SELECT state FROM relay_control_connection_reservations
+ WHERE user_id = ? AND relay_host_id = ? AND cell_id = ? AND assignment_epoch = ?`,
+ [identity.userId, identity.relayHostId, 'cell-b', migration.assignmentEpoch]
+ )
+ ).toEqual([{ state: 'released' }])
+ expect(
+ await database!.query(
+ `SELECT cell_id, assignment_epoch, reserved_controls, migration_leases
+ FROM relay_assignments
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([
+ {
+ cell_id: 'cell-b',
+ assignment_epoch: 2,
+ reserved_controls: 0,
+ migration_leases: 0
+ }
+ ])
+ })
+
+ it.each(['expired', 'previous-incarnation'] as const)(
+ 'retires a registered migration and its %s target control after the source dies',
+ async (targetControlState) => {
+ let now = 100
+ const cells = [
+ {
+ id: 'cell-a',
+ url: 'https://relay-a.example.com',
+ capacityRequests: 10,
+ connectionHardCap: 600 as const,
+ connectionUnobservedBound: 50
+ },
+ {
+ id: 'cell-b',
+ url: 'https://relay-b.example.com',
+ capacityRequests: 10,
+ connectionHardCap: 600 as const,
+ connectionUnobservedBound: 50
+ }
+ ]
+ const store = await setupWithHeartbeats(() => now, cells)
+ await heartbeat(store, cells[0]!)
+ await heartbeat(store, cells[1]!)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ const sourceControl = await store.activateControl(identity, {
+ cellId: first.cellId,
+ assignmentEpoch: first.assignmentEpoch,
+ generation: 1
+ })
+ await store.setCellEnabled('cell-a', false)
+ const migration = await store.startEvacuation(identity, 'cell-b')
+ await store.activateControl(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch,
+ generation: 1
+ })
+ await store.markMigrationTargetRegistered(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch
+ })
+ await store.releaseActivity(identity, sourceControl)
+ now +=
+ targetControlState === 'expired'
+ ? ASSIGNMENT_LIMITS.activityLeaseMs + 1
+ : 45_001
+ await heartbeat(store, cells[1]!, {
+ startedAt: targetControlState === 'previous-incarnation' ? now - 1 : 50,
+ incarnation:
+ targetControlState === 'previous-incarnation'
+ ? '22222222-2222-4222-8222-222222222222'
+ : '11111111-1111-4111-8111-111111111111'
+ })
+ await store.attestCellFence(
+ 'cell-a',
+ '11111111-1111-4111-8111-111111111111'
+ )
+
+ await expect(
+ store.cellEvacuationStatus('cell-a', 'cell-b', false)
+ ).resolves.toMatchObject({
+ inProgress: 1,
+ registeredCompletable: 0,
+ registeredTargetInactive: 1
+ })
+ await expect(
+ store.cellEvacuationStatus('cell-a', 'cell-b', true)
+ ).resolves.toMatchObject({ inProgress: 0, completed: 1, blocked: 0 })
+ expect(await cellReservations(database!)).toEqual({ 'cell-a': 0, 'cell-b': 0 })
+ expect(
+ await database!.query(
+ `SELECT activity_id FROM relay_assignment_activity_leases
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([])
+ expect(
+ await database!.query(
+ `SELECT cell_id, assignment_epoch, reserved_controls, migration_leases
+ FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([
+ {
+ cell_id: 'cell-b',
+ assignment_epoch: migration.assignmentEpoch,
+ reserved_controls: 0,
+ migration_leases: 0
+ }
+ ])
+ }
+ )
+
+ it('refuses dead-source completion while the source heartbeat is fresh', async () => {
+ const store = await setupWithHeartbeats(() => 100, [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
+ ])
+ await heartbeat(store, { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 })
+ await heartbeat(store, { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 })
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ const sourceControl = await store.activateControl(identity, {
+ cellId: first.cellId,
+ assignmentEpoch: first.assignmentEpoch,
+ generation: 1
+ })
+ await store.setCellEnabled('cell-a', false)
+ const migration = await store.startEvacuation(identity, 'cell-b')
+ await store.activateControl(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch,
+ generation: 1
+ })
+ await store.markMigrationTargetRegistered(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch
+ })
+ await store.releaseActivity(identity, sourceControl)
+
+ await expect(
+ store.attestCellFence('cell-a', '11111111-1111-4111-8111-111111111111')
+ ).rejects.toThrow('cell_fence_runtime_not_stale')
+ await expect(
+ store.completeEvacuationFromDeadSource(identity, {
+ assignmentEpoch: migration.assignmentEpoch,
+ sourceCellId: 'cell-a',
+ targetCellId: 'cell-b'
+ })
+ ).rejects.toThrow('cell_fence_attestation_missing')
+ expect(await cellReservations(database!)).toEqual({ 'cell-a': 0, 'cell-b': 2 })
+ })
+
+ it('supersedes a registered migration after its disabled target is proven unavailable', async () => {
+ let now = 100
+ const cells = [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 },
+ { id: 'cell-c', url: 'https://relay-c.example.com', capacityRequests: 10 }
+ ]
+ const store = await setupWithHeartbeats(() => now, cells)
+ for (const cell of cells) await heartbeat(store, cell)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ await store.activateControl(identity, {
+ cellId: first.cellId,
+ assignmentEpoch: first.assignmentEpoch,
+ generation: 1
+ })
+ await store.setCellEnabled('cell-a', false)
+ const migration = await store.startEvacuation(identity, 'cell-b')
+ await store.activateControl(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch,
+ generation: 1
+ })
+ await store.markMigrationTargetRegistered(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch
+ })
+ await database!.query(
+ `INSERT INTO relay_control_connection_reservations
+ (reservation_id, idempotency_key, user_id, relay_host_id,
+ assignment_epoch, cell_id, state, inclusion_watermark,
+ claim_activity_id, created_at, timeout_at, claimed_at, released_at,
+ updated_at)
+ VALUES (?, ?, ?, ?, ?, ?, 'late-arrival-debt', NULL, NULL, ?, ?, NULL, NULL, ?)`,
+ [
+ 'superseded-reservation',
+ 'superseded-reservation',
+ identity.userId,
+ identity.relayHostId,
+ migration.assignmentEpoch,
+ 'cell-b',
+ 100,
+ 100,
+ 100
+ ]
+ )
+ await store.setCellEnabled('cell-b', false)
+ now += 45_001
+ await heartbeat(store, cells[0]!, { startedAt: 50 })
+ await heartbeat(store, cells[2]!, { startedAt: 50 })
+ await store.attestCellFence(
+ 'cell-b',
+ '11111111-1111-4111-8111-111111111111'
+ )
+ const input = {
+ assignmentEpoch: migration.assignmentEpoch,
+ sourceCellId: 'cell-a',
+ currentTargetCellId: 'cell-b',
+ replacementTargetCellId: 'cell-c'
+ }
+
+ const replacement = await store.supersedeRegisteredEvacuation(identity, input)
+ expect(replacement).toMatchObject({
+ sourceCellId: 'cell-a',
+ targetCellId: 'cell-c',
+ previousEpoch: 2,
+ assignmentEpoch: 3
+ })
+ await expect(store.supersedeRegisteredEvacuation(identity, input)).resolves.toEqual(replacement)
+ expect(await store.resolve(identity)).toMatchObject({ cellId: 'cell-c', assignmentEpoch: 3 })
+ expect(
+ await database!.query(
+ `SELECT state FROM relay_control_connection_reservations
+ WHERE reservation_id = ?`,
+ ['superseded-reservation']
+ )
+ ).toEqual([{ state: 'released' }])
+ expect(await cellReservations(database!)).toEqual({
+ 'cell-a': 1,
+ 'cell-b': 0,
+ 'cell-c': 2
+ })
+ expect(
+ await database!.query(
+ `SELECT activity_kind, cell_id FROM relay_assignment_activity_leases
+ WHERE user_id = ? AND relay_host_id = ? ORDER BY cell_id, activity_id`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([
+ { activity_kind: 'control', cell_id: 'cell-a' },
+ { activity_kind: 'control', cell_id: 'cell-c' },
+ { activity_kind: 'migration', cell_id: 'cell-c' }
+ ])
+ expect(await store.cellEvacuationStatus('cell-a', 'cell-b', false)).toMatchObject({
+ inProgress: 0
+ })
+ expect(await store.cellEvacuationStatus('cell-a', 'cell-c', false)).toMatchObject({
+ inProgress: 1
+ })
+ })
+
+ it('retries registered migration supersession with inventory locked first', async () => {
+ let now = 100
+ const cells = [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 },
+ { id: 'cell-c', url: 'https://relay-c.example.com', capacityRequests: 10 }
+ ]
+ const delegate = await openInMemoryRelayDatabase()
+ const retryDatabase = new OneShotInventoryFailureDatabase(delegate)
+ database = retryDatabase
+ const store = new RelayAssignmentStore(retryDatabase, () => now, {
+ requireLiveCells: true,
+ heartbeatTtlMs: 45_000
+ })
+ await store.reconcileCells(cells)
+ for (const cell of cells) await heartbeat(store, cell)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ await store.activateControl(identity, {
+ cellId: first.cellId,
+ assignmentEpoch: first.assignmentEpoch,
+ generation: 1
+ })
+ await store.setCellEnabled('cell-a', false)
+ const migration = await store.startEvacuation(identity, 'cell-b')
+ await store.activateControl(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch,
+ generation: 1
+ })
+ await store.markMigrationTargetRegistered(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch
+ })
+ await store.setCellEnabled('cell-b', false)
+ now += 45_001
+ await heartbeat(store, cells[0]!, { startedAt: 50 })
+ await heartbeat(store, cells[2]!, { startedAt: 50 })
+ await store.attestCellFence(
+ 'cell-b',
+ '11111111-1111-4111-8111-111111111111'
+ )
+ retryDatabase.arm()
+
+ await expect(
+ store.supersedeRegisteredEvacuation(identity, {
+ assignmentEpoch: migration.assignmentEpoch,
+ sourceCellId: 'cell-a',
+ currentTargetCellId: 'cell-b',
+ replacementTargetCellId: 'cell-c'
+ })
+ ).resolves.toMatchObject({ targetCellId: 'cell-c', assignmentEpoch: 3 })
+ expect(retryDatabase.retryLocks).toEqual([
+ 'inventory-nowait-failed',
+ 'inventory-first',
+ 'assignment-nowait'
+ ])
+ })
+
+ it('reconciles durable cell accounting once before aggregate supersession', async () => {
+ let now = 100
+ const cells = [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 },
+ { id: 'cell-c', url: 'https://relay-c.example.com', capacityRequests: 10 }
+ ]
+ const store = await setupWithHeartbeats(() => now, cells)
+ for (const cell of cells) await heartbeat(store, cell)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ await store.activateControl(identity, {
+ cellId: first.cellId,
+ assignmentEpoch: first.assignmentEpoch,
+ generation: 1
+ })
+ await store.setCellEnabled('cell-a', false)
+ const migration = await store.startEvacuation(identity, 'cell-b')
+ await store.activateControl(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch,
+ generation: 1
+ })
+ await store.markMigrationTargetRegistered(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch
+ })
+ await store.setCellEnabled('cell-b', false)
+ now += 45_001
+ await heartbeat(store, cells[0]!, { startedAt: 50 })
+ await heartbeat(store, cells[2]!, { startedAt: 50 })
+ await store.attestCellFence(
+ 'cell-b',
+ '11111111-1111-4111-8111-111111111111'
+ )
+ await database!.query(
+ `UPDATE relay_cells
+ SET reserved_requests = CASE WHEN cell_id = ? THEN 1 ELSE 0 END
+ WHERE cell_id IN (?, ?)`,
+ ['cell-c', 'cell-a', 'cell-c']
+ )
+
+ await expect(
+ store.supersedeRegisteredCellEvacuations('cell-a', 'cell-b', 'cell-c', 100)
+ ).resolves.toBe(1)
+ expect(await cellReservations(database!)).toEqual({
+ 'cell-a': 1,
+ 'cell-b': 0,
+ 'cell-c': 2
+ })
+ expect(
+ await database!.query(
+ `SELECT assignment_epoch, target_cell_id, aborted_at
+ FROM relay_assignment_migrations
+ WHERE user_id = ? ORDER BY assignment_epoch`,
+ [identity.userId]
+ )
+ ).toEqual([
+ { assignment_epoch: 2, target_cell_id: 'cell-b', aborted_at: 45_101 },
+ { assignment_epoch: 3, target_cell_id: 'cell-c', aborted_at: null }
+ ])
+ })
+
+ it('preserves healthy third-cell activity during aggregate supersession', async () => {
+ let now = 100
+ const cells = [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 },
+ { id: 'cell-c', url: 'https://relay-c.example.com', capacityRequests: 10 },
+ { id: 'cell-d', url: 'https://relay-d.example.com', capacityRequests: 10 }
+ ]
+ const store = await setupWithHeartbeats(() => now, cells)
+ for (const cell of cells) await heartbeat(store, cell)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ const sourceControl = await store.activateControl(identity, {
+ cellId: first.cellId,
+ assignmentEpoch: first.assignmentEpoch,
+ generation: 1
+ })
+ await store.setCellEnabled('cell-a', false)
+ const migration = await store.startEvacuation(identity, 'cell-b')
+ await store.activateControl(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch,
+ generation: 1
+ })
+ await store.markMigrationTargetRegistered(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch
+ })
+ await database!.query(
+ `UPDATE relay_assignment_activity_leases SET cell_id = ?
+ WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?`,
+ ['cell-d', identity.userId, identity.relayHostId, sourceControl]
+ )
+ await database!.query(
+ `UPDATE relay_cells SET reserved_requests = ? WHERE cell_id = ?`,
+ [5, 'cell-c']
+ )
+ await store.setCellEnabled('cell-b', false)
+ now += 45_001
+ await heartbeat(store, cells[0]!, { startedAt: 50 })
+ await heartbeat(store, cells[2]!, { startedAt: 50 })
+ await heartbeat(store, cells[3]!, { startedAt: 50 })
+ await store.attestCellFence(
+ 'cell-b',
+ '11111111-1111-4111-8111-111111111111'
+ )
+
+ await expect(
+ store.supersedeRegisteredCellEvacuations('cell-a', 'cell-b', 'cell-c', 100)
+ ).resolves.toBe(1)
+ expect(await cellReservations(database!)).toEqual({
+ 'cell-a': 0,
+ 'cell-b': 0,
+ 'cell-c': 1,
+ 'cell-d': 1
+ })
+ expect(
+ await database!.query(
+ `SELECT activity_kind, cell_id FROM relay_assignment_activity_leases
+ WHERE user_id = ? AND relay_host_id = ? ORDER BY cell_id, activity_id`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([
+ { activity_kind: 'control', cell_id: 'cell-c' },
+ { activity_kind: 'migration', cell_id: 'cell-c' },
+ { activity_kind: 'control', cell_id: 'cell-d' }
+ ])
+ })
+
+ it('refuses supersession while the registered target remains available', async () => {
+ const cells = [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 },
+ { id: 'cell-c', url: 'https://relay-c.example.com', capacityRequests: 10 }
+ ]
+ const store = await setupWithHeartbeats(() => 100, cells)
+ for (const cell of cells) await heartbeat(store, cell)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ await store.activateControl(identity, {
+ cellId: first.cellId,
+ assignmentEpoch: first.assignmentEpoch,
+ generation: 1
+ })
+ await store.setCellEnabled('cell-a', false)
+ const migration = await store.startEvacuation(identity, 'cell-b')
+ await store.activateControl(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch,
+ generation: 1
+ })
+ await store.markMigrationTargetRegistered(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch
+ })
+ await store.setCellEnabled('cell-b', false)
+
+ await expect(
+ store.attestCellFence('cell-b', '11111111-1111-4111-8111-111111111111')
+ ).rejects.toThrow('cell_fence_runtime_not_stale')
+ await expect(
+ store.supersedeRegisteredEvacuation(identity, {
+ assignmentEpoch: migration.assignmentEpoch,
+ sourceCellId: 'cell-a',
+ currentTargetCellId: 'cell-b',
+ replacementTargetCellId: 'cell-c'
+ })
+ ).rejects.toThrow('cell_fence_attestation_missing')
+ expect(
+ await database!.query(
+ `SELECT cell_id, assignment_epoch FROM relay_assignments
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([{ cell_id: 'cell-b', assignment_epoch: 2 }])
+ })
+
+ it('serializes concurrent retries of registered migration supersession', async () => {
+ let now = 100
+ const cells = [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 },
+ { id: 'cell-c', url: 'https://relay-c.example.com', capacityRequests: 10 }
+ ]
+ const store = await setupWithHeartbeats(() => now, cells)
+ for (const cell of cells) await heartbeat(store, cell)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ await store.activateControl(identity, {
+ cellId: first.cellId,
+ assignmentEpoch: first.assignmentEpoch,
+ generation: 1
+ })
+ await store.setCellEnabled('cell-a', false)
+ const migration = await store.startEvacuation(identity, 'cell-b')
+ await store.activateControl(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch,
+ generation: 1
+ })
+ await store.markMigrationTargetRegistered(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch
+ })
+ await store.setCellEnabled('cell-b', false)
+ now += 45_001
+ await heartbeat(store, cells[0]!, { startedAt: 50 })
+ await heartbeat(store, cells[2]!, { startedAt: 50 })
+ await store.attestCellFence(
+ 'cell-b',
+ '11111111-1111-4111-8111-111111111111'
+ )
+ const input = {
+ assignmentEpoch: migration.assignmentEpoch,
+ sourceCellId: 'cell-a',
+ currentTargetCellId: 'cell-b',
+ replacementTargetCellId: 'cell-c'
+ }
+
+ const results = await Promise.all([
+ store.supersedeRegisteredEvacuation(identity, input),
+ store.supersedeRegisteredEvacuation(identity, input)
+ ])
+ expect(results[0]).toEqual(results[1])
+ expect(
+ await database!.query(
+ `SELECT assignment_epoch FROM relay_assignment_migrations
+ WHERE user_id = ? AND relay_host_id = ? ORDER BY assignment_epoch`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([{ assignment_epoch: 2 }, { assignment_epoch: 3 }])
+ expect(await cellReservations(database!)).toEqual({
+ 'cell-a': 1,
+ 'cell-b': 0,
+ 'cell-c': 2
+ })
+ })
+
+ it('classifies an expired migration blocked by a newer target assignment', async () => {
+ let now = 100
+ const store = await setup(() => now, [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 20 }
+ ])
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ await store.activateControl(identity, {
+ cellId: first.cellId,
+ assignmentEpoch: first.assignmentEpoch,
+ generation: 1
+ })
+ await store.startEvacuation(identity, 'cell-b')
+ await store.activateControl(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: 2,
+ generation: 1
+ })
+ await database!.query(
+ `UPDATE relay_assignments SET assignment_epoch = ?
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [3, identity.userId, identity.relayHostId]
+ )
+
+ now += ASSIGNMENT_LIMITS.migrationLeaseMs + 1
+ expect(await store.cellEvacuationStatus('cell-a', 'cell-b', false)).toEqual({
+ inProgress: 1,
+ oldestExpiresAt: 100 + ASSIGNMENT_LIMITS.migrationLeaseMs,
+ oldestRemainingMs: -1,
+ targetRegistered: 0,
+ ...NO_REGISTERED_EVACUATION_DIAGNOSTICS,
+ completed: 0,
+ blocked: 0,
+ expiredUnregistered: 1,
+ repairableExpiredUnregistered: 0,
+ abortableExpiredUnregistered: 0,
+ blockedExpiredUnregistered: 1,
+ blockedExpiredOnNewerTargetAssignment: 1
+ })
+ })
+
+ it('does not complete a registered migration after the assignment epoch advances', async () => {
+ const store = await setup(() => 100, [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 20 }
+ ])
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ const sourceControl = await store.activateControl(identity, {
+ cellId: first.cellId,
+ assignmentEpoch: first.assignmentEpoch,
+ generation: 1
+ })
+ const migration = await store.startEvacuation(identity, 'cell-b')
+ await store.activateControl(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch,
+ generation: 1
+ })
+ await store.markMigrationTargetRegistered(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch
+ })
+ await store.releaseActivity(identity, sourceControl)
+ await database!.query(
+ `UPDATE relay_assignments SET assignment_epoch = ?
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [migration.assignmentEpoch + 1, identity.userId, identity.relayHostId]
+ )
+
+ expect(await store.completeReadyEvacuations()).toBe(0)
+ await expect(
+ store.completeEvacuation(identity, migration.assignmentEpoch)
+ ).rejects.toThrow('migration_assignment_mismatch')
+ expect(await store.cellEvacuationStatus('cell-a', 'cell-b', false)).toMatchObject({
+ inProgress: 1,
+ targetRegistered: 1
+ })
+ })
+
+ it('retires an expired migration without rewriting its newer target assignment', async () => {
+ let now = 100
+ const store = await setup(() => now, [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 20 }
+ ])
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ await store.activateControl(identity, {
+ cellId: first.cellId,
+ assignmentEpoch: first.assignmentEpoch,
+ generation: 1
+ })
+ await store.startEvacuation(identity, 'cell-b')
+ await store.activateControl(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: 2,
+ generation: 1
+ })
+ await database!.query(
+ `UPDATE relay_assignments SET assignment_epoch = ?
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [3, identity.userId, identity.relayHostId]
+ )
+
+ now += ASSIGNMENT_LIMITS.migrationLeaseMs + 1
+ expect(await store.abortExpiredEvacuations()).toBe(1)
+ expect(await store.resolve(identity)).toMatchObject({
+ cellId: 'cell-b',
+ assignmentEpoch: 3
+ })
+ expect(
+ await database!.query(
+ `SELECT activity_id FROM relay_assignment_activity_leases
+ WHERE user_id = ? AND relay_host_id = ? ORDER BY activity_id`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([
+ { activity_id: 'control:cell-a:1' },
+ { activity_id: 'control:cell-b:1' }
+ ])
+ expect(await store.cellEvacuationStatus('cell-a', 'cell-b', false)).toMatchObject({
+ inProgress: 0
+ })
+ })
+
+ it.each([
+ { assignmentEpoch: 1, removeAssignment: false, expected: 'migration_assignment_mismatch' },
+ { assignmentEpoch: 2, removeAssignment: true, expected: 'migration_assignment_missing' }
+ ])(
+ 'fails closed when expired migration assignment state is not superseding: $expected',
+ async ({ assignmentEpoch, removeAssignment, expected }) => {
+ let now = 100
+ const store = await setup(() => now, [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 20 }
+ ])
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ await store.assign(identity)
+ await store.startEvacuation(identity, 'cell-b')
+ if (removeAssignment) {
+ await database!.query(
+ `DELETE FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ } else {
+ await database!.query(
+ `UPDATE relay_assignments SET assignment_epoch = ?
+ WHERE user_id = ? AND relay_host_id = ?`,
+ [assignmentEpoch, identity.userId, identity.relayHostId]
+ )
+ }
+
+ now += ASSIGNMENT_LIMITS.migrationLeaseMs + 1
+ await expect(store.abortExpiredEvacuations()).rejects.toThrow(expected)
+ expect(await store.cellEvacuationStatus('cell-a', 'cell-b', false)).toMatchObject({
+ inProgress: 1,
+ targetRegistered: 0
+ })
+ }
+ )
+
+ it('reconciles duplicated assignment counters and cell reservations after evacuation', async () => {
+ const store = await setup(() => 100, [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 20 }
+ ])
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ const sourceControl = await store.activateControl(identity, {
+ cellId: first.cellId,
+ assignmentEpoch: first.assignmentEpoch,
+ generation: 1
+ })
+ await store.startEvacuation(identity, 'cell-b')
+ await store.activateControl(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: 2,
+ generation: 1
+ })
+ await store.markMigrationTargetRegistered(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: 2
+ })
+ await store.releaseActivity(identity, sourceControl)
+ await database!.query(
+ `UPDATE relay_assignments SET reserved_controls = 9, reserved_splices = 4,
+ reserved_invites = 3, pending_installs = 2, pending_confirmations = 2,
+ migration_leases = 7 WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ await database!.query(
+ `UPDATE relay_cells SET reserved_requests =
+ CASE WHEN cell_id = ? THEN 11 ELSE 3 END`,
+ ['cell-a']
+ )
+
+ expect(await store.cellEvacuationStatus('cell-a', 'cell-b', true)).toEqual({
+ inProgress: 0,
+ ...NO_ACTIVE_MIGRATION_LEASE,
+ targetRegistered: 0,
+ ...NO_REGISTERED_EVACUATION_DIAGNOSTICS,
+ completed: 1,
+ blocked: 0,
+ ...NO_EXPIRED_EVACUATION_DIAGNOSTICS
+ })
+ expect(await cellReservations(database!)).toEqual({ 'cell-a': 0, 'cell-b': 1 })
+ expect(
+ await database!.query(
+ `SELECT reserved_controls, reserved_splices, reserved_invites,
+ pending_installs, pending_confirmations, migration_leases
+ FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`,
+ [identity.userId, identity.relayHostId]
+ )
+ ).toEqual([
+ {
+ reserved_controls: 1,
+ reserved_splices: 0,
+ reserved_invites: 0,
+ pending_installs: 0,
+ pending_confirmations: 0,
+ migration_leases: 0
+ }
+ ])
+ })
+
+ it('refuses reservation reconciliation when an activity lease has no assignment', async () => {
+ const store = await setup(() => 100, [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 20 }
+ ])
+ await database!.query(
+ `INSERT INTO relay_assignment_activity_leases
+ (user_id, relay_host_id, activity_id, activity_kind, cell_id,
+ request_units, expires_at, updated_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
+ ['orphan-user', 'orphanhost000001', 'control:orphan', 'control', 'cell-a', 1, 200, 100]
+ )
+
+ await expect(store.cellEvacuationStatus('cell-a', 'cell-b', true)).rejects.toThrow(
+ 'activity_lease_assignment_missing'
+ )
+ expect(await cellReservations(database!)).toEqual({ 'cell-a': 0, 'cell-b': 0 })
+ })
+
+ it('refuses reservation reconciliation when an activity lease names no cell', async () => {
+ const store = await setup(() => 100, [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 20 }
+ ])
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ await store.assign(identity)
+ await database!.query(
+ `INSERT INTO relay_assignment_activity_leases
+ (user_id, relay_host_id, activity_id, activity_kind, cell_id,
+ request_units, expires_at, updated_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
+ [
+ identity.userId,
+ identity.relayHostId,
+ 'splice:missing-cell',
+ 'splice',
+ 'missing-cell',
+ 2,
+ 200,
+ 100
+ ]
+ )
+
+ await expect(store.cellEvacuationStatus('cell-a', 'cell-b', true)).rejects.toThrow(
+ 'activity_lease_cell_missing'
+ )
+ expect(await cellReservations(database!)).toEqual({ 'cell-a': 1, 'cell-b': 0 })
+ })
+
+ it('leaves accounting for cells outside the selected evacuation pair unchanged', async () => {
+ const store = await setup(() => 100, [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 20 },
+ { id: 'cell-c', url: 'https://relay-c.example.com', capacityRequests: 20 }
+ ])
+ await database!.query(
+ `INSERT INTO relay_assignments
+ (user_id, relay_host_id, cell_id, assignment_epoch, lease_expires_at,
+ last_activity_at, reserved_controls, reserved_splices, reserved_invites,
+ pending_installs, pending_confirmations, migration_leases)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+ ['unrelated-user', 'unrelatedhost001', 'cell-c', 1, 200, 100, 9, 0, 0, 0, 0, 0]
+ )
+ await database!.query(
+ `INSERT INTO relay_assignment_activity_leases
+ (user_id, relay_host_id, activity_id, activity_kind, cell_id,
+ request_units, expires_at, updated_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
+ [
+ 'unrelated-user',
+ 'unrelatedhost001',
+ 'control:unrelated',
+ 'control',
+ 'cell-c',
+ 1,
+ 200,
+ 100
+ ]
+ )
+ await database!.query(
+ `UPDATE relay_cells SET reserved_requests = 9 WHERE cell_id = ?`,
+ ['cell-c']
+ )
+
+ expect(await store.cellEvacuationStatus('cell-a', 'cell-b', true)).toMatchObject({
+ inProgress: 0
+ })
+ expect(await cellReservations(database!)).toEqual({
+ 'cell-a': 0,
+ 'cell-b': 0,
+ 'cell-c': 9
+ })
+ expect(
+ await database!.query(
+ `SELECT reserved_controls FROM relay_assignments
+ WHERE user_id = ? AND relay_host_id = ?`,
+ ['unrelated-user', 'unrelatedhost001']
+ )
+ ).toEqual([{ reserved_controls: 9 }])
+ })
+
+ it('rebalances only a fully inactive assignment after the dormant TTL', async () => {
+ let now = 100
+ const store = await setup(() => now, [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
+ ])
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ const control = await store.activateControl(identity, {
+ cellId: first.cellId,
+ assignmentEpoch: first.assignmentEpoch,
+ generation: 1
+ })
+ await expect(store.rebalanceDormant(identity, 'cell-b')).rejects.toThrow('assignment_active')
+ await store.releaseActivity(identity, control)
+ now += ASSIGNMENT_LIMITS.dormantTtlMs + 1
+
+ await expect(store.rebalanceDormant(identity, 'cell-b')).resolves.toMatchObject({
+ cellId: 'cell-b',
+ assignmentEpoch: 2
+ })
+ expect(await cellReservations(database!)).toEqual({ 'cell-a': 0, 'cell-b': 1 })
+ })
+
+ it('durably removes an unhealthy cell from new assignment without moving existing hosts', async () => {
+ const store = await setup(() => 100, [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
+ ])
+ const existing = { userId: 'user-a', relayHostId: 'host000000000001' }
+ expect(await store.assign(existing)).toMatchObject({ cellId: 'cell-a' })
+ await store.setCellEnabled('cell-a', false)
+ await store.reconcileCells([
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 10 }
+ ])
+ expect(await store.resolve(existing)).toMatchObject({ cellId: 'cell-a' })
+ await expect(
+ store.assign({ userId: 'user-b', relayHostId: 'host000000000002' })
+ ).resolves.toMatchObject({ cellId: 'cell-b' })
+ await store.setCellEnabled('cell-a', true)
+ await expect(store.setCellEnabled('missing', false)).rejects.toThrow('cell_not_found')
+ })
+
+ it('bulk-migrates active controls without exposing assignment identities', async () => {
+ const store = await setup(() => 100, [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 20 }
+ ])
+ await store.setCellEnabled('cell-b', false)
+ const identities = [1, 2, 3].map((index) => ({
+ userId: `user-${index}`,
+ relayHostId: `host${String(index).padStart(12, '0')}`
+ }))
+ const sourceControls: string[] = []
+ for (const identity of identities) {
+ const assignment = await store.assign(identity)
+ sourceControls.push(
+ await store.activateControl(identity, {
+ cellId: 'cell-a',
+ assignmentEpoch: assignment.assignmentEpoch,
+ generation: 1
+ })
+ )
+ }
+ await store.setCellEnabled('cell-b', true)
+ expect(await store.cellEvacuationCapacity('cell-a', 'cell-b')).toEqual({
+ sourceAssignments: 3,
+ requiredTargetUnits: 6,
+ availableTargetUnits: 20
+ })
+ expect(await store.startActiveCellEvacuations('cell-a', 'cell-b', 2)).toBe(2)
+ expect(await store.startActiveCellEvacuations('cell-a', 'cell-b', 2)).toBe(1)
+ expect(await store.startActiveCellEvacuations('cell-a', 'cell-b', 2)).toBe(0)
+ expect(await store.cellEvacuationStatus('cell-a', 'cell-b', false)).toEqual({
+ inProgress: 3,
+ oldestExpiresAt: 100 + ASSIGNMENT_LIMITS.migrationLeaseMs,
+ oldestRemainingMs: ASSIGNMENT_LIMITS.migrationLeaseMs,
+ targetRegistered: 0,
+ ...NO_REGISTERED_EVACUATION_DIAGNOSTICS,
+ completed: 0,
+ blocked: 0,
+ ...NO_EXPIRED_EVACUATION_DIAGNOSTICS
+ })
+ for (const identity of identities) {
+ const assignment = await store.resolve(identity)
+ await store.activateControl(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: assignment!.assignmentEpoch,
+ generation: 2
+ })
+ await store.markMigrationTargetRegistered(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: assignment!.assignmentEpoch
+ })
+ }
+ expect(await store.cellEvacuationStatus('cell-a', 'cell-b', true)).toEqual({
+ inProgress: 3,
+ oldestExpiresAt: 100 + ASSIGNMENT_LIMITS.migrationLeaseMs,
+ oldestRemainingMs: ASSIGNMENT_LIMITS.migrationLeaseMs,
+ targetRegistered: 3,
+ registeredSourceActive: 3,
+ registeredCompletable: 0,
+ registeredTargetInactive: 0,
+ completed: 0,
+ blocked: 3,
+ ...NO_EXPIRED_EVACUATION_DIAGNOSTICS
+ })
+ for (let index = 0; index < identities.length; index++) {
+ await store.releaseActivity(identities[index]!, sourceControls[index]!)
+ }
+ expect(await store.cellEvacuationStatus('cell-a', 'cell-b', true)).toEqual({
+ inProgress: 0,
+ ...NO_ACTIVE_MIGRATION_LEASE,
+ targetRegistered: 0,
+ ...NO_REGISTERED_EVACUATION_DIAGNOSTICS,
+ completed: 3,
+ blocked: 0,
+ ...NO_EXPIRED_EVACUATION_DIAGNOSTICS
+ })
+ })
+
+ it.each(['cell-b', 'cell-c'])(
+ 'skips a batch row that concurrently moved from the selected source to %s',
+ async (movedCellId) => {
+ const store = await setup(() => 100, [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 20 },
+ { id: 'cell-c', url: 'https://relay-c.example.com', capacityRequests: 20 }
+ ])
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const assignment = await store.assign(identity)
+ await store.activateControl(identity, {
+ cellId: 'cell-a',
+ assignmentEpoch: assignment.assignmentEpoch,
+ generation: 1
+ })
+ const startEvacuation = store.startEvacuation.bind(store)
+ vi.spyOn(store, 'startEvacuation').mockImplementationOnce(async (...args) => {
+ await database!.query(
+ `UPDATE relay_assignments SET cell_id = ? WHERE user_id = ? AND relay_host_id = ?`,
+ [movedCellId, identity.userId, identity.relayHostId]
+ )
+ return await startEvacuation(...args)
+ })
+
+ expect(await store.startActiveCellEvacuations('cell-a', 'cell-b', 10)).toBe(0)
+ expect(await store.resolve(identity)).toMatchObject({ cellId: movedCellId })
+ expect(await database!.query(`SELECT * FROM relay_assignment_migrations`)).toEqual([])
+ }
+ )
+
+ it('does not count an existing migration after a stale batch row moves to its target', async () => {
+ const store = await setup(() => 100, [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 20 }
+ ])
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const assignment = await store.assign(identity)
+ await store.activateControl(identity, {
+ cellId: 'cell-a',
+ assignmentEpoch: assignment.assignmentEpoch,
+ generation: 1
+ })
+ const startEvacuation = store.startEvacuation.bind(store)
+ vi.spyOn(store, 'startEvacuation').mockImplementationOnce(async (...args) => {
+ await startEvacuation(identity, 'cell-b')
+ return await startEvacuation(...args)
+ })
+
+ expect(await store.startActiveCellEvacuations('cell-a', 'cell-b', 10)).toBe(0)
+ expect(await store.resolve(identity)).toMatchObject({ cellId: 'cell-b' })
+ expect(await store.cellEvacuationStatus('cell-a', 'cell-b', false)).toMatchObject({
+ inProgress: 1
+ })
+ })
+
+ it('preserves operator-owned tagged URLs and admission state across reconciliation', async () => {
+ const store = await setup(() => 100, [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 }
+ ])
+ await store.configureCell(
+ { id: 'cell-a', url: 'https://old---relay-a.example.com', capacityRequests: 20 },
+ false
+ )
+ await store.reconcileCells([
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 }
+ ])
+ const rows = await database!.query(
+ `SELECT cell_url, enabled, capacity_requests FROM relay_cells WHERE cell_id = ?`,
+ ['cell-a']
+ )
+ expect(rows).toEqual([
+ {
+ cell_url: 'https://old---relay-a.example.com',
+ enabled: 0,
+ capacity_requests: 20
+ }
+ ])
+ })
+
+ it('bulk-migrates activity even when its source control is temporarily absent', async () => {
+ const store = await setup(() => 100, [
+ { id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 20 },
+ { id: 'cell-b', url: 'https://relay-b.example.com', capacityRequests: 20 }
+ ])
+ const identity = { userId: 'user-1', relayHostId: 'host000000000001' }
+ const assignment = await store.assign(identity)
+ const control = await store.activateControl(identity, {
+ cellId: 'cell-a',
+ assignmentEpoch: assignment.assignmentEpoch,
+ generation: 1
+ })
+ await store.acquireActivity(identity, {
+ activityId: 'splice-without-control',
+ kind: 'splice',
+ cellId: 'cell-a'
+ })
+ await store.releaseActivity(identity, control)
+ expect(await store.cellEvacuationCapacity('cell-a', 'cell-b')).toEqual({
+ sourceAssignments: 1,
+ requiredTargetUnits: 3,
+ availableTargetUnits: 20
+ })
+ expect(await store.startActiveCellEvacuations('cell-a', 'cell-b', 10)).toBe(1)
+ expect(await store.resolve(identity)).toMatchObject({ cellId: 'cell-b', assignmentEpoch: 2 })
+ })
+})
+
+async function cellReservations(database: RelayDatabase): Promise> {
+ const rows = await database.query(
+ `SELECT cell_id, reserved_requests FROM relay_cells ORDER BY cell_id ASC`
+ )
+ return Object.fromEntries(rows.map((row) => [String(row.cell_id), Number(row.reserved_requests)]))
+}
diff --git a/cloud/apps/relay/src/assignment-store.ts b/cloud/apps/relay/src/assignment-store.ts
new file mode 100644
index 00000000000..226df9b3984
--- /dev/null
+++ b/cloud/apps/relay/src/assignment-store.ts
@@ -0,0 +1,8505 @@
+import { randomUUID } from 'node:crypto'
+import { performance } from 'node:perf_hooks'
+import {
+ ASSIGNMENT_LIMITS,
+ mayNormallyReassign,
+ RELAY_ADMISSION_BUDGETS,
+ RELAY_DEFAULT_REGION,
+ RELAY_REGIONS,
+ RELAY_PROTOCOL_LIMITS,
+ type RelayRegion
+} from '@orca-cloud/relay-contract'
+import {
+ cellAdmissionState,
+ cellAdmissionStates,
+ ensureCellAdmission,
+ parseCellAdmissionState,
+ RelayCellAdmissionSelector,
+ setCellAdmissionBeforeBoundary,
+ stateFromEnabled,
+ synchronizeCellAdmissionBoundary,
+ type CellAdmissionMembership,
+ type CellAdmissionSelectorInspection,
+ type CellAdmissionState
+} from './cell-admission-selector.js'
+import {
+ RelayMigrationCellRegistrar,
+ type MigrationCellRegistration
+} from './cell-admission-migration-registration.js'
+import {
+ ASSIGNMENT_CONNECTION_HEADROOM_QUERY
+} from './assignment-connection-headroom-query.js'
+import { AssignmentIdentityQueue } from './assignment-identity-queue.js'
+import type { RelayCellConfig } from './config.js'
+import type {
+ RelayDatabase,
+ RelayLockOptions,
+ RelayTransactionOptions,
+ SqlRow
+} from './database.js'
+import type { RegionalRehomeSafetySnapshot } from './relay-observability.js'
+import {
+ combineRegionalRehomeSafety,
+ REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT,
+ REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT,
+ regionalRehomePoolPressure,
+ regionalRehomeSafetyFailure
+} from './regional-rehome-safety.js'
+import {
+ ABANDONED_REGISTERED_MIGRATION,
+ DURABLY_FENCED_MIGRATION_SOURCE,
+ REGISTERED_MIGRATION_ABANDON_MS
+} from './registered-migration-abandonment.js'
+
+type AssignmentIdentity = { userId: string; relayHostId: string }
+type CellHeartbeat = {
+ cellId: string
+ cellUrl: string
+ cellIncarnation: string
+ startedAt: number
+ ready: boolean
+ observedRequests: number
+ region?: RelayRegion
+ totalConnections?: number
+ inFlightConnections?: number
+ reservedConnectionUnits?: number
+ enforcedConnectionUnits?: number
+ connectionInclusionWatermark?: number
+ connectionHardCap?: number
+ connectionUnobservedBound?: number
+}
+
+type CellRegionalRehomeStatus = {
+ cellId: string
+ cellIncarnation: string
+ regionalRehomeProtocol: number
+ safety: RegionalRehomeSafetySnapshot
+}
+
+type RelayAssignmentStoreOptions = {
+ requireLiveCells?: boolean
+ heartbeatTtlMs?: number
+ recordControlRenewal?: (durationMs: number, outcome: ControlRenewalOutcome) => void
+}
+
+export type ControlRenewalOutcome =
+ | 'renewed'
+ | 'assignment_not_found'
+ | 'activity_cell_not_authoritative'
+ | 'control_activity_not_found'
+ | 'control_activity_moved'
+ | 'database_error'
+
+const CONTROL_RENEWAL_OUTCOMES = new Set([
+ 'renewed',
+ 'assignment_not_found',
+ 'activity_cell_not_authoritative',
+ 'control_activity_not_found',
+ 'control_activity_moved'
+])
+export type RelayAssignment = AssignmentIdentity & {
+ cellId: string
+ cellUrl: string
+ assignmentEpoch: number
+ leaseExpiresAt: number
+ region?: RelayRegion
+}
+
+export type RelayRegionCatalogEntry = {
+ region: RelayRegion
+ probeOrigins: string[]
+}
+
+export type RelayAssignmentMigration = AssignmentIdentity & {
+ sourceCellId: string
+ targetCellId: string
+ previousEpoch: number
+ assignmentEpoch: number
+ expiresAt: number
+ targetRegisteredAt?: number
+}
+
+export type RegionalRehomeAttempt = AssignmentIdentity & {
+ attemptId: string
+ preferredRegion: 'asia-east2'
+ sourceCellId: string
+ sourceCellUrl: string
+ sourceCellIncarnation: string
+ targetCellId: string
+ targetCellIncarnation: string
+ previousEpoch: number
+ assignmentEpoch: number
+ drainGraceMs: number
+ sendAttempts: number
+}
+
+export type RegionalHostDrainOutcome =
+ | 'accepted'
+ | 'already-accepted'
+ | 'host-not-connected'
+
+export type RegionalRehomeFleetSafety = RegionalRehomeSafetySnapshot & {
+ requiredCells: number
+ missingCells: number
+ maxReconnects: number
+}
+
+export type RegionalRehomeControl = {
+ generation: number
+ enabled: boolean
+ observationStartedAt: number
+ notBefore: number
+ ratePerMinute: number
+ preferenceMaxAgeMs: number
+ drainGraceMs: number
+}
+
+type RegisteredEvacuationSupersessionInput = {
+ assignmentEpoch: number
+ sourceCellId: string
+ currentTargetCellId: string
+ replacementTargetCellId: string
+}
+
+export type DeadSourceCompletionResult = {
+ changed: boolean
+ assignmentEpoch: number
+ sourceCellId: string
+ targetCellId: string
+}
+
+export type CellEvacuationStatus = {
+ inProgress: number
+ oldestExpiresAt: number | null
+ oldestRemainingMs: number | null
+ targetRegistered: number
+ registeredSourceActive: number
+ registeredCompletable: number
+ registeredTargetInactive: number
+ completed: number
+ blocked: number
+ expiredUnregistered: number
+ repairableExpiredUnregistered: number
+ abortableExpiredUnregistered: number
+ blockedExpiredUnregistered: number
+ blockedExpiredOnNewerTargetAssignment: number
+}
+
+export type CellEvacuationCapacity = {
+ sourceAssignments: number
+ requiredTargetUnits: number
+ availableTargetUnits: number
+}
+
+export type CellDeploymentStatus = {
+ cellId: string
+ cellUrl: string
+ region: RelayRegion
+ enabled: boolean
+ admissionState: CellAdmissionState
+ capacityRequests: number
+ reservedRequests: number
+ assignments: number
+ activityLeases: number
+ activityRequestUnits: number
+ restartBlockingActivityLeases: number
+ restartBlockingActivityRequestUnits: number
+ restartBlockingReservedRequests: number
+ outgoingMigrations: number
+ incomingMigrations: number
+ connectionCapacity: null | {
+ hardCap: number
+ controlRebindReserve: number
+ ordinaryConnectionLimit: number
+ unobservedBound: number
+ normalAdmissionPause: number
+ observedConnections: number
+ inFlightConnections: number
+ reservedConnectionUnits: number
+ enforcedConnectionUnits: number
+ pendingControlReservations: number
+ heartbeatFresh: boolean
+ }
+ runtime: null | {
+ cellUrl: string
+ cellIncarnation: string
+ startedAt: number
+ ready: boolean
+ observedRequests: number
+ lastHeartbeatAt: number
+ heartbeatFresh: boolean
+ regionalRehomeProtocol: number
+ }
+}
+
+export type CellFenceAttemptEvidence = {
+ attemptId: string
+ environment: 'staging' | 'production'
+ cellId: string
+ cellIncarnation: string
+ migName: string
+ instanceGroup: string
+ generationIdentity: string
+ fenceCommit: string
+ planSha256: string
+ planObjectName: string
+ planObjectGeneration?: string
+ varFileSha256: string
+ terraformStateLineage: string
+ terraformStateSerial: number
+ terraformStateObjectGeneration: string
+ terraformStateObjectSha256: string
+ requestReason: string
+}
+
+export type CellFenceApplyInvocation = {
+ invocationId: string
+ requestReason: string
+ startedAt: number
+ gceOperation?: string
+}
+
+export type CellFenceAttempt = CellFenceAttemptEvidence & {
+ applyInvocations?: CellFenceApplyInvocation[]
+ gceOperation?: string
+ createdAt: number
+ expiresAt: number
+ applyStartedAt?: number
+ completedAt?: number
+ abortedAt?: number
+}
+
+export type CellDrainAttemptState =
+ | 'prepared'
+ | 'send-may-have-started'
+ | 'application-receipt'
+ | 'proven-not-delivered'
+
+export type CellDrainAttempt = {
+ attemptId: string
+ cellId: string
+ cellIncarnation: string
+ traceValue: string
+ plannedGraceMs: number
+ state: CellDrainAttemptState
+ preparedAt: number
+ sendMayHaveStartedAt?: number
+ sendPermitExpiresAt?: number
+ applicationReceiptAt?: number
+ backendSuccessStatus?: number
+ backendInstance?: string
+ receiptCellIncarnation?: string
+ retryAfter?: number
+ recoverForwardAttemptedAt?: number
+ provenNotDeliveredAt?: number
+}
+
+export type AssignmentActivityKind =
+ | 'control'
+ | 'splice'
+ | 'invite'
+ | 'install'
+ | 'confirmation'
+ | 'migration'
+
+const ACTIVITY_COLUMN: Record = {
+ control: 'reserved_controls',
+ splice: 'reserved_splices',
+ invite: 'reserved_invites',
+ install: 'pending_installs',
+ confirmation: 'pending_confirmations',
+ migration: 'migration_leases'
+}
+
+const ACTIVITY_REQUEST_UNITS: Record = {
+ control: 1,
+ splice: 2,
+ invite: 1,
+ install: 1,
+ confirmation: 1,
+ migration: 1
+}
+
+const ASSIGNMENT_LOCK_RETRY_DEADLINE_MS = 15_000
+// Why: one global FOR UPDATE over a 23-row table serialises every director and
+// cell. At the 1s pool lock_timeout each blocked waiter also holds a pooled
+// client for a full second, so the queue converts contention into pool
+// exhaustion. The lock is held to COMMIT and the assignment path runs many
+// statements after taking it, and no hold-time telemetry existed before this
+// change, so 500ms is a first value to tune once cellInventoryHoldMsMax lands.
+export const CELL_INVENTORY_LOCK_TIMEOUT_MS = 500
+
+// The same inventory lock is taken by live requests and by background sweeps,
+// and the right failure mode differs per caller.
+export type CellInventoryLockMode =
+ // Bound the wait so a blocked request stops occupying a pooled client.
+ | 'request'
+ // Never queue: the caller handles database_lock_unavailable and moves on.
+ | 'nowait'
+ // A sweep can enter here, so keep the pool default. Failing sooner would turn
+ // ordinary contention into a 55P03 the retry wrapper reports as terminal, which
+ // spends the incident gate's bounded exhausted-retry budget (300 per 5 min).
+ | 'pool-default'
+// Why: stranded detection (issue #225) needs a grant old enough that a real
+// attach would have registered (the 90s activity lease covers dial +
+// activation), yet recent enough to prove an active retry loop rather than
+// ordinary dormancy — which stays governed by the 24h rule.
+const STRANDED_MIN_GRANT_AGE_MS = 60_000
+const STRANDED_RECENT_ACTIVITY_MS = 15 * 60_000
+const REGION_PREFERENCE_RETENTION_MS = 30 * 24 * 60 * 60_000
+const REGIONAL_REHOME_UNREGISTERED_REFRESH_MS = 5 * 60_000
+const REGIONAL_REHOME_MAX_REFRESH_MS = 24 * 60 * 60_000
+// Drain grace is enforced by session-scoped cell state that any control
+// reconnect sheds, so receipted attempts can stall dual-homed past grace;
+// redrains re-dispatch them with the elapsed (zero) grace until they detach.
+export const REGIONAL_REHOME_REDRAIN_INTERVAL_MS = 60_000
+export const REGIONAL_REHOME_REDRAIN_SEND_LIMIT = 20
+export const REGIONAL_REHOME_QUARANTINE_FAILURES = 3
+export const REGIONAL_REHOME_QUARANTINE_MS = 15 * 60_000
+const REGIONAL_REHOME_QUARANTINE_EXCLUSION_LIMIT = 50
+const REGIONAL_REHOME_QUARANTINE_MEMORY_LIMIT = 1_000
+const REGIONAL_REHOME_OBSERVATION_MS = 24 * 60 * 60_000
+const ASSIGNMENT_LOCK_RETRY_MAX_DELAY_MS = 50
+type AssignmentInventoryScope = 'none' | 'general' | 'all'
+type RetriedAssignmentInventoryScope = Exclude
+
+class AssignmentInventoryLockUnavailable extends Error {
+ constructor(readonly inventoryScope: RetriedAssignmentInventoryScope) {
+ super('database_lock_unavailable')
+ }
+}
+
+class AssignmentInventoryScopeChanged extends Error {
+ constructor() {
+ super('assignment_inventory_scope_changed')
+ }
+}
+
+// Debt holds connection headroom for a control that may still arrive shortly
+// after its director-side timeout. Nothing legitimately arrives minutes late
+// (attach deadline 10s, orphan grace 30s); unretired debt from hosts that
+// never return otherwise starves connection headroom fleet-wide and turns
+// every placement into relay_capacity_exhausted.
+const LATE_ARRIVAL_DEBT_RETENTION_MS = 10 * 60 * 1_000
+const CELL_FENCE_TTL_MS = 5 * 60 * 1_000
+const CELL_FENCE_ATTEMPT_TTL_MS = 60 * 60 * 1_000
+const CELL_DRAIN_SEND_PERMIT_MS = 30_000
+const MAX_DRAIN_ACCOUNTING_REPAIR_ATTEMPTS = 3
+const CELL_FENCE_ATTEMPT_SELECT = `
+ SELECT attempts.*, bindings.plan_object_name, bindings.plan_object_generation,
+ bindings.var_file_sha256, bindings.terraform_state_lineage,
+ bindings.terraform_state_serial, bindings.terraform_state_object_generation,
+ bindings.terraform_state_object_sha256, bindings.request_reason
+ FROM relay_cell_fence_attempts attempts
+ JOIN relay_cell_fence_plan_bindings bindings
+ ON bindings.attempt_id = attempts.attempt_id`
+export const STRANDED_MIGRATION_ABANDON_MS = REGISTERED_MIGRATION_ABANDON_MS
+const ABORTABLE_EXPIRED_MIGRATION = `(
+ (
+ migration.target_registered_at IS NULL
+ AND NOT EXISTS (
+ SELECT 1 FROM relay_post_drain_migration_pins pin
+ WHERE pin.user_id = migration.user_id
+ AND pin.relay_host_id = migration.relay_host_id
+ AND pin.assignment_epoch = migration.assignment_epoch
+ )
+ )
+ OR ${ABANDONED_REGISTERED_MIGRATION}
+)`
+
+export class RelayAssignmentStore {
+ private readonly requireLiveCells: boolean
+ private readonly heartbeatTtlMs: number
+ // Poisoned attempts never complete or abort and stay the oldest rows, so
+ // unquarantined they eventually fill the sweeps' LIMIT page and starve
+ // every healthy candidate. Process-local on purpose: it resets on deploy
+ // and each director relearns within a few ticks.
+ private readonly regionalRehomeCandidateQuarantine = new Map<
+ string,
+ { failures: number; until: number }
+ >()
+ private readonly recordControlRenewal?: RelayAssignmentStoreOptions['recordControlRenewal']
+ private readonly admissionSelector: RelayCellAdmissionSelector
+ private readonly migrationCellRegistrar: RelayMigrationCellRegistrar
+ private readonly activityQueue = new AssignmentIdentityQueue()
+ private assignmentTail: Promise = Promise.resolve()
+ private pendingRegionalRehomeDisableLog: Record | null = null
+
+ constructor(
+ private readonly database: RelayDatabase,
+ private readonly now: () => number = Date.now,
+ options: RelayAssignmentStoreOptions = {}
+ ) {
+ this.requireLiveCells = options.requireLiveCells ?? false
+ this.heartbeatTtlMs = options.heartbeatTtlMs ?? 45_000
+ this.recordControlRenewal = options.recordControlRenewal
+ this.admissionSelector = new RelayCellAdmissionSelector(database, now)
+ this.migrationCellRegistrar = new RelayMigrationCellRegistrar(database, now)
+ }
+
+ async reconcileCells(cells: RelayCellConfig[], disableMissing = true): Promise