chore(relay): cut the cell LB connection drain to 60 s and allow ten-cell same-cap batches (#21848)

* perf(relay): cut the cell LB drain to 60s and widen the same-cap batch to ten cells

Two independent sources of relay roll wall clock, neither of which protects a
host:

1. `connection_draining_timeout_sec` on the per-cell backend services was 300s.
   The same-cap job drains every host off the cell to a restart-safe condition
   before Terraform runs, so the LB drain only ever covers a host still
   mid-handshake. Measured 2026-09-16 over ten same-cap cell jobs, it sat as
   ~5m55s of dead time between `Apply complete` and the old VM powering off,
   inside an 8.5-minute `wait-until --stable` step. Now 60s, and pinned in the
   topology `check` block beside the other fixed-one invariants.

2. The same-cap wave capped a batch at four cells, so a 22-cell roll needed six
   batches, six single-use monitor gates, and a human handoff per batch. The
   wave workflow now declares cell_1..cell_10 with the identical serial shape
   and chaining, and the validator accepts two to ten.

The shared wave-index rule (`relay-monitor-evidence.mjs` and the relay-ops
preflight CLI) widens from 0-3 to 0-9 so the later cells can present the same
evidence; each job workflow keeps its own narrower range, so the capacity wave
stays at four. Cells remain strictly serial, one at a time behind the rollout
lease, each with its own live preflight.

Claude-Session: https://claude.ai/session/relay-roll-drain-timeout-and-batch-cap

* fix(relay): align the Asia topology plan validator with the 60s cell drain

`validate-relay-asia-topology-plan.mjs` rejected any Asia backend whose
`connection_draining_timeout_sec` was not 300, and
`cloud-deploy-relay-asia-topology.yml` targets
`google_compute_backend_service.relay_gce_cell["<cell>"]` per cell. With the
Terraform local at 60 that workflow would have failed its own plan review.

The validator's two restated topology values are now named exports, and a new
census test reads `relay-gce-cells.tf` and equates three statements of each:
the `relay_gce_topology` local, the topology `check` assert that pins it, and
the validator constant. Terraform cannot export a local to JS, so reading the
source is the only way to stop them drifting; the test was confirmed to fail
when the local alone is moved back to 300.

Repo-wide grep finds no other pin of the drain value.

Claude-Session: https://claude.ai/session/relay-roll-drain-timeout-and-batch-cap

cloud/dev/scripts/relay-monitor-evidence.mjs

@@ -10,7 +10,9 @@ const JWT = /^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/
 const EVIDENCE_MAX_AGE_MS = 5 * 60_000
 // Matches the same-cap cell job timeout-minutes; bounds each predecessor wave.
 const WAVE_PREDECESSOR_TIMEOUT_MS = 75 * 60_000
-const WAVE_INDEX = /^[0-3]$/
+// Widest any wave chain declares (same-cap's cell_1..cell_10); each job workflow
+// pins its own narrower range.
+const WAVE_INDEX = /^[0-9]$/
 const EVIDENCE_SAMPLE_INTERVAL_MS = 60_000
 const EVIDENCE_MAX_LINEAGE_MS = 25 * 60_000
 const MIGRATION_POLICIES = new Set([

cloud/dev/scripts/relay-monitor-evidence.test.mjs

@@ -242,7 +242,7 @@ test('later same-cap waves accept evidence aged by predecessor cell rolls', asyn
     await ageState(17 * 60_000)
     await assert.rejects(authorityAt('0'), /authority is incomplete or stale/)
     await assert.doesNotReject(authorityAt('1'))
-    await assert.rejects(authorityAt('4'), /wave index is invalid/)
+    await assert.rejects(authorityAt('10'), /wave index is invalid/)
     await assert.rejects(authorityAt('x'), /wave index is invalid/)
     // Both edges of one predecessor job timeout: 5min + 75min exactly.
     await ageState(80 * 60_000)
@@ -259,6 +259,11 @@ test('later same-cap waves accept evidence aged by predecessor cell rolls', asyn
     await assert.doesNotReject(authorityAt('3'))
     await ageState(230 * 60_000 + 1)
     await assert.rejects(authorityAt('3'), /authority is incomplete or stale/)
+    // The last cell of a ten-cell same-cap batch: 5min + 9 * 75min exactly.
+    await ageState(680 * 60_000)
+    await assert.doesNotReject(authorityAt('9'))
+    await ageState(680 * 60_000 + 1)
+    await assert.rejects(authorityAt('9'), /authority is incomplete or stale/)
   } finally {
     await rm(directory, { recursive: true, force: true })
   }

cloud/dev/scripts/relay-production-same-cap-wave.mjs

@@ -34,7 +34,7 @@ function cells(value) {
   const parsed = value.split(',').map((cell) => cell.trim()).filter(Boolean)
   if (
     parsed.length < 1 ||
-    parsed.length > 4 ||
+    parsed.length > 10 ||
     new Set(parsed).size !== parsed.length ||
     parsed.some((cell) => !SAME_CAP_CELLS.includes(cell))
   ) throw new Error('same-cap wave cells are invalid')
@@ -76,8 +76,9 @@ export function validateSameCapWave(input) {
   if (input.mode === 'canary-apply' && selected.length !== 1) {
     throw new Error('canary mode requires exactly one cell')
   }
-  if (input.mode === 'batch-apply' && (selected.length < 2 || selected.length > 4)) {
-    throw new Error('batch mode requires two to four cells')
+  // Ten is the wave workflow's statically declared serial cell-job chain, cell_1..cell_10.
+  if (input.mode === 'batch-apply' && (selected.length < 2 || selected.length > 10)) {
+    throw new Error('batch mode requires two to ten cells')
   }
   // Later waves expect the selector to advance by exactly 2 per predecessor,
   // which a resumed rollback cell (isolate skipped, +1) violates.

cloud/dev/scripts/relay-production-same-cap-wave.test.mjs

@@ -57,6 +57,48 @@ test('requires one canary or a bounded reviewed batch', () => {
   }), /cells/)
 })
 
+// The bound is the wave workflow's static cell_1..cell_10 chain: a batch longer than the
+// chain would silently drop its tail cells, so it is refused before any mutation.
+test('a batch fills the serial cell chain and never overflows it', () => {
+  const general = SAME_CAP_CELLS.filter((cell) => entryAdmission(cell) === 'general')
+  const batch = (count) => {
+    const cellIds = general.slice(0, count).join(',')
+    return validateSameCapWave({
+      mode: 'batch-apply',
+      cellIds,
+      targetDigest,
+      rollbackDigest,
+      confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${cellIds}`,
+      canaryRunId: '42'
+    })
+  }
+  assert.equal(batch(10).cells.length, 10)
+  assert.throws(() => batch(11), /same-cap wave cells are invalid/)
+  assert.throws(() => batch(1), /batch mode requires two to ten cells/)
+})
+
+// The validator's ten-cell bound is only true if the workflow really declares ten strictly
+// serial cell jobs and frees the lease after all of them.
+test('the wave workflow chains exactly ten serial cell jobs', () => {
+  const dispatch = readRelayWorkflow('deploy-relay-production-same-cap.yml')
+  for (let index = 0; index < 10; index += 1) {
+    const job = index + 1
+    assert.match(dispatch, new RegExp(`\n  cell_${job}:\n`), `cell_${job} is missing`)
+    assert.match(dispatch, new RegExp(`fromJSON\\(needs\\.gate\\.outputs\\.cells\\)\\[${index}\\]`))
+    assert.match(dispatch, new RegExp(`wave-index: '${index}'`))
+    if (index > 0) {
+      assert.match(dispatch, new RegExp(`needs: \\[gate, cell_${index}\\]`))
+      assert.match(
+        dispatch,
+        new RegExp(`if: \\$\\{\\{ needs\\.cell_${index}\\.result == 'success' && ` +
+          `fromJSON\\(needs\\.gate\\.outputs\\.cells\\)\\[${index}\\] != null \\}\\}`)
+      )
+    }
+    assert.match(dispatch, new RegExp(`\n      - cell_${job}\n`), `release_lease must need cell_${job}`)
+  }
+  assert.doesNotMatch(dispatch, /\n  cell_11:/)
+})
+
 test('rolls the migration-only cells but never mixes the two classes in one wave', () => {
   for (const cellId of SAME_CAP_MIGRATION_ONLY_CELLS) {
     assert.equal(SAME_CAP_CELLS.includes(cellId), true, cellId)

cloud/dev/scripts/relay-regional-rehome-workflow.test.mjs

@@ -139,9 +139,10 @@ test('same-cap wrapper is reusable, canary-bound, and sequential', () => {
   // The wrapper validates the override before anything runs, passes it to every
   // cell, seals it into the canary artifact, and prints it in the run summary.
   assert.match(wrapper, /--gate-override-reason "\$\{GATE_OVERRIDE_REASON\}" \\\n {12}--gate-override-confirmation "\$\{GATE_OVERRIDE_CONFIRMATION\}"\)/)
+  // One per cell job in the serial cell_1..cell_10 chain.
   assert.equal(
     wrapper.match(/gate-override-confirmation: \$\{\{ inputs\.gate-override-confirmation \}\}/g).length,
-    4
+    10
   )
   assert.match(wrapper, /Aggregate monitor gate overridden \(break-glass\)/)
   assert.match(wrapper, /ACTOR: \$\{\{ github\.actor \}\}/)
@@ -163,12 +164,12 @@ test('same-cap wrapper is reusable, canary-bound, and sequential', () => {
   ]) {
     const body = readFileSync(fileURLToPath(new URL(source, import.meta.url)), 'utf8')
     assert.match(body, /WAVE_PREDECESSOR_TIMEOUT_MS = 75 \* 60_000/)
-    assert.match(body, /\^\[0-3\]\$/)
+    assert.match(body, /\^\[0-9\]\$/)
   }
   // Aged-evidence replay via job re-runs is fenced: mutations are
   // single-dispatch, so a failed cell needs a fresh gate and monitor run.
   assert.match(job, /test "\$\{GITHUB_RUN_ATTEMPT\}" = 1/)
-  for (const index of [0, 1, 2, 3]) {
+  for (const index of [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) {
     assert.match(wrapper, new RegExp(`wave-index: '${index}'`))
   }
   assert.doesNotMatch(job, /EFFECTIVE_SELECTOR_GENERATION \+ 1\)/)

cloud/dev/scripts/validate-relay-asia-topology-plan.mjs

@@ -2,6 +2,11 @@ import { readFileSync } from 'node:fs'
 import { fileURLToPath } from 'node:url'
 
 const REGION = 'asia-east2'
+// Mirrors local.relay_gce_topology in infra/terraform/relay-gce-cells.tf, which Terraform
+// cannot export to JS; the census test below the validator equates the two by reading the
+// .tf source, so this pair and the topology `check` assert cannot drift apart.
+export const RELAY_CELL_BACKEND_TIMEOUT_SECONDS = 86_400
+export const RELAY_CELL_CONNECTION_DRAIN_SECONDS = 60
 const CELL_SHAPES = {
   production: {
     domain: 'relay.onorca.dev',
@@ -117,8 +122,8 @@ function requireCellBackend(change, config, cellId) {
   const hostname = cellId.split('-').at(-1)
   const name = `${relayGceName(config.environment)}-${hostname}`
   if (
-    after?.timeout_sec !== 86_400 ||
-    after?.connection_draining_timeout_sec !== 300 ||
+    after?.timeout_sec !== RELAY_CELL_BACKEND_TIMEOUT_SECONDS ||
+    after?.connection_draining_timeout_sec !== RELAY_CELL_CONNECTION_DRAIN_SECONDS ||
     after?.load_balancing_scheme !== 'EXTERNAL_MANAGED' ||
     after?.protocol !== 'HTTP' ||
     after?.port_name !== 'relay' ||

cloud/dev/scripts/validate-relay-asia-topology-plan.test.mjs

@@ -1,6 +1,11 @@
 import assert from 'node:assert/strict'
+import { readFileSync } from 'node:fs'
 import { test } from 'node:test'
-import { validateRelayAsiaTopologyPlan } from './validate-relay-asia-topology-plan.mjs'
+import {
+  RELAY_CELL_BACKEND_TIMEOUT_SECONDS,
+  RELAY_CELL_CONNECTION_DRAIN_SECONDS,
+  validateRelayAsiaTopologyPlan
+} from './validate-relay-asia-topology-plan.mjs'
 
 const image = `us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:${'a'.repeat(64)}`
 const config = { environment: 'staging', cells: ['staging-gce-c4'], image }
@@ -49,7 +54,8 @@ const resources = [
     update_policy: [{ replacement_method: 'RECREATE', max_surge_fixed: 0, max_unavailable_fixed: 1 }]
   }),
   create('google_compute_backend_service.relay_gce_cell["staging-gce-c4"]', {
-    timeout_sec: 86_400, connection_draining_timeout_sec: 300,
+    timeout_sec: RELAY_CELL_BACKEND_TIMEOUT_SECONDS,
+    connection_draining_timeout_sec: RELAY_CELL_CONNECTION_DRAIN_SECONDS,
     load_balancing_scheme: 'EXTERNAL_MANAGED', protocol: 'HTTP', port_name: 'relay',
     session_affinity: 'NONE',
     health_checks: ['projects/p/global/healthChecks/orca-cloud-staging-relay-gce-ready'],
@@ -221,3 +227,31 @@ test('rejects incomplete NAT, backend, and URL routing shapes', () => {
     /no exact backend route/
   )
 })
+
+// Terraform cannot export a local to JS, so this validator restates two topology values that
+// the Asia workflow applies. Read the .tf source and equate all three statements of each: the
+// local, the topology `check` assert that pins it, and the constant above.
+test('the reviewed backend constants match the Terraform topology they validate', () => {
+  const terraform = readFileSync(
+    new URL('../../infra/terraform/relay-gce-cells.tf', import.meta.url),
+    'utf8'
+  )
+  const local = (name) => {
+    const found = new RegExp(`\\n\\s*${name}\\s*=\\s*(\\d+)\\n`).exec(terraform)
+    assert.notEqual(found, null, `relay_gce_topology has no ${name}`)
+    return Number(found[1])
+  }
+  const asserted = (name) => {
+    const found =
+      new RegExp(`local\\.relay_gce_topology\\.${name}\\s*==\\s*(\\d+)`).exec(terraform)
+    assert.notEqual(found, null, `the topology check does not pin ${name}`)
+    return Number(found[1])
+  }
+  for (const [name, constant] of [
+    ['backend_timeout_seconds', RELAY_CELL_BACKEND_TIMEOUT_SECONDS],
+    ['connection_drain_seconds', RELAY_CELL_CONNECTION_DRAIN_SECONDS]
+  ]) {
+    assert.equal(local(name), constant, `${name} local differs from the validator constant`)
+    assert.equal(asserted(name), constant, `${name} check assert differs from the validator`)
+  }
+})