fix(relay): move director traffic in two steps so a deploy does not land on one instance (#31051)

* fix(relay): move director traffic in two steps so a deploy does not land on one instance

Cloud Run routes a new director revision's floor instances their first request
roughly 60-110 s after they start, while a traffic move reaches clients in
15-30 s. The one-step 100% move therefore put every assignment on the single
instance started with the revision, which sits at its own admission limit at
peak; the 2026-10-20 deploy returned ~550 assign 503s in two minutes that way.

With a floor above one, the deploy now moves 20% first, waits until Cloud Run's
request log shows every floor instance serving (six-minute deadline), then
moves the rest. A timeout or failure at that stage returns all traffic to the
previous serving revision, which still holds its warm floor. The staging deploy
identity gains log read access for the same check.

* fix(relay): bound the director spread log read, log the instances seen at timeout, and document split recovery

A hung gcloud log read could hold the 20/80 split past the six-minute deadline;
each read now times out after 30 s and counts as an unread poll. A timeout logs
the distinct instances seen, so a partial share packed onto fewer instances than
the floor shows up. The runbook covers restoring a split left by a cancelled job
and the missing selector-rollback tag after a timed-out deploy.

cloud/dev/fixtures/terraform-root-partition/families.json

@@ -206,6 +206,7 @@
     "google_project_iam_member.github_staging_relay_capacity_mutation",
     "google_project_iam_member.github_staging_relay_capacity_viewer",
     "google_project_iam_member.github_staging_relay_deploy_compute_viewer",
+    "google_project_iam_member.github_staging_relay_deploy_logging_viewer",
     "google_project_iam_member.github_staging_relay_power",
     "google_project_iam_member.push_runtime_cloudsql_client",
     "google_project_iam_member.push_runtime_fcm_admin",

cloud/dev/scripts/deploy-relay-blue-green.mjs

@@ -30,6 +30,14 @@ export const DIRECTOR_ADMISSION_ENVIRONMENT = Object.freeze({
 })
 const DIRECTOR_STARTUP_PROBE =
   'tcpSocket.port=8080,timeoutSeconds=120,periodSeconds=120,failureThreshold=1'
+// Why: Cloud Run routes a new revision's floor instances their first request up to ~110 s after
+// they start, so a one-step move lands every assignment on the one pre-started instance.
+export const DIRECTOR_STAGED_TRAFFIC_PERCENT = 20
+// Floor instances have started up to ~220 s after the revision and taken ~110 s more to route.
+const DIRECTOR_SPREAD_TIMEOUT_MS = 6 * 60 * 1000
+const DIRECTOR_SPREAD_POLL_INTERVAL_MS = 15_000
+// A hung read must not hold the 20/80 split past the deadline.
+const DIRECTOR_SPREAD_READ_TIMEOUT_MS = 30_000
 
 export function taggedRevisionOrigin(serviceOrigin, tag) {
   const url = new URL(serviceOrigin)
@@ -461,8 +469,13 @@ export function parseArguments(argv) {
   return values
 }
 
-function commandJson(args) {
-  const result = spawnSync('gcloud', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] })
+function commandJson(args, { timeoutMs } = {}) {
+  const result = spawnSync('gcloud', args, {
+    encoding: 'utf8',
+    stdio: ['ignore', 'pipe', 'pipe'],
+    timeout: timeoutMs
+  })
+  if (result.error) throw new Error(`gcloud ${args.slice(0, 4).join(' ')} failed: ${result.error.message}`)
   if (result.status !== 0) {
     throw new Error(`gcloud ${args.slice(0, 4).join(' ')} failed: ${result.stderr.trim()}`)
   }
@@ -727,6 +740,59 @@ function cellIdentifier(sourceCellId, releaseId) {
   return `${sourceCellId.slice(0, available)}--${suffix}`
 }
 
+// Instances that served a request since `since`, read from Cloud Run's own request log.
+function servingInstanceIds(config, revision, since) {
+  const filter = [
+    'resource.type="cloud_run_revision"',
+    `resource.labels.service_name="${config.service}"`,
+    `resource.labels.revision_name="${revision}"`,
+    `logName="projects/${config.project}/logs/run.googleapis.com%2Frequests"`,
+    `timestamp>="${since}"`
+  ].join(' AND ')
+  const entries = commandJson([
+    'logging',
+    'read',
+    filter,
+    '--project',
+    config.project,
+    '--limit',
+    '1000',
+    '--format=json(labels.instanceId)'
+  ], { timeoutMs: DIRECTOR_SPREAD_READ_TIMEOUT_MS })
+  return new Set(entries.map((entry) => entry.labels?.instanceId).filter(Boolean))
+}
+
+export async function waitForServingInstances(
+  config,
+  revision,
+  required,
+  since,
+  {
+    readInstances = servingInstanceIds,
+    pollIntervalMs = DIRECTOR_SPREAD_POLL_INTERVAL_MS,
+    timeoutMs = DIRECTOR_SPREAD_TIMEOUT_MS
+  } = {}
+) {
+  const deadline = Date.now() + timeoutMs
+  let instances = 0
+  for (;;) {
+    try {
+      instances = readInstances(config, revision, since).size
+      console.warn(JSON.stringify({ event: 'director_candidate_spread', revision, instances, required }))
+      if (instances >= required) return
+    } catch (error) {
+      // A throttled log read is not evidence either way; the deadline still bounds the wait.
+      console.warn(JSON.stringify({ event: 'director_candidate_spread_unread', revision, error: String(error) }))
+    }
+    if (Date.now() >= deadline) {
+      // Fewer than the floor here can mean Cloud Run packs the partial share onto fewer instances.
+      console.warn(JSON.stringify({ event: 'director_candidate_spread_timeout', revision, instances, required }))
+      throw new Error(`candidate ${revision} served from ${instances} of ${required} instances before the deadline`)
+    }
+    await new Promise((resolve) => setTimeout(resolve, pollIntervalMs))
+  }
+}
+
 async function waitForHealth(origin, connectionCapacityProtocol) {
   const response = await fetch(`${origin}/health`, { signal: AbortSignal.timeout(15_000) })
   const body = await response.json()
@@ -952,6 +1018,7 @@ export async function deployDirector(config, tag, overrides = {}) {
     deleteRevision,
     updateTraffic,
     waitForHealth,
+    waitForServingInstances,
     assertRegionalRehomeDisabled,
     readReserveCells,
     readDirectorPlacement,
@@ -962,7 +1029,8 @@ export async function deployDirector(config, tag, overrides = {}) {
   // 100% of traffic. Inheriting the serving floor keeps one owner for the number instead of
   // restating it here; public admission is per-instance, so losing it shrinks fleet capacity.
   const initialService = operations.describeService(config)
-  const servingRevision = operations.describeRevision(config, activeRevision(initialService))
+  const servingRevisionName = activeRevision(initialService)
+  const servingRevision = operations.describeRevision(config, servingRevisionName)
   const shapeBaseline = directorShapeBaseline(
     config,
     initialService,
@@ -1058,6 +1126,7 @@ export async function deployDirector(config, tag, overrides = {}) {
   }
   removeDirectorTrafficTags(config, operations)
   let deployed
+  let staged = false
   let promoted = false
   try {
     operations.deployCandidate(
@@ -1178,6 +1247,18 @@ export async function deployDirector(config, tag, overrides = {}) {
         })
       )
     }
+    // A floor of one is the pre-started instance itself; nothing else needs to warm.
+    if (requiredMinimumInstances > 1) {
+      const since = new Date().toISOString()
+      staged = true
+      operations.updateTraffic(config, [`--to-tags=${tag}=${DIRECTOR_STAGED_TRAFFIC_PERCENT}`])
+      await operations.waitForServingInstances(
+        config,
+        candidate.revision,
+        requiredMinimumInstances,
+        since
+      )
+    }
     operations.updateTraffic(config, [`--to-tags=${tag}=100`])
     promoted = true
     removeDirectorTrafficTags(config, operations, new Set([SELECTOR_ROLLBACK_TAG]))
@@ -1195,6 +1276,13 @@ export async function deployDirector(config, tag, overrides = {}) {
       } catch (rollbackError) {
         recoveryErrors.push(rollbackError)
       }
+    } else if (staged) {
+      // The serving revision still holds most traffic and its warm floor; the cold rollback has neither.
+      try {
+        operations.updateTraffic(config, [`--to-revisions=${servingRevisionName}=100`])
+      } catch (rollbackError) {
+        recoveryErrors.push(rollbackError)
+      }
     }
     try {
       removeDirectorTrafficTags(

cloud/dev/scripts/deploy-relay-blue-green.test.mjs

@@ -32,7 +32,8 @@ import {
   taggedRevisionOrigin,
   taggedTraffic,
   trafficTags,
-  waitForEvacuationCapacity
+  waitForEvacuationCapacity,
+  waitForServingInstances
 } from './deploy-relay-blue-green.mjs'
 
 // Terraform declares these values; audited blue/green stamps the same values without targeting
@@ -443,7 +444,8 @@ function directorHarness({
   placements = {},
   callerEmail,
   // The revision template's Cloud SQL transport; a Terraform apply changes it under the serving one.
-  templateTransport
+  templateTransport,
+  spreadFailure
 } = {}) {
   const state = {
     activeRevision: 'relay-00001-old',
@@ -476,6 +478,8 @@ function directorHarness({
   }
   const removed = []
   const healthProtocols = []
+  const trafficMoves = []
+  const spreadWaits = []
   let pendingCleanupFailure = cleanupFailsBeforeRemoval
   const operations = {
     describeService: () => ({
@@ -563,19 +567,25 @@ function directorHarness({
         if (cleanupReportsFailure) throw new Error('gcloud reported failed latest revision')
         return
       }
-      const promote = args.find((argument) => argument.startsWith('--to-tags='))
-      assert.ok(promote)
-      const tag = promote.slice('--to-tags='.length).split('=')[0]
-      state.activeRevision = state.tags.get(tag)
+      const move = args.find((argument) => /^--to-(tags|revisions)=/.test(argument))
+      assert.ok(move)
+      trafficMoves.push(move)
+      const [target, percent] = move.slice(move.indexOf('=') + 1).split('=')
+      if (percent !== '100') return
+      state.activeRevision = move.startsWith('--to-tags=') ? state.tags.get(target) : target
     },
     waitForHealth: async (_origin, connectionCapacityProtocol) => {
       healthProtocols.push(connectionCapacityProtocol)
     },
+    waitForServingInstances: async (_config, revision, required) => {
+      spreadWaits.push({ revision, required })
+      if (spreadFailure) throw spreadFailure
+    },
     readReserveCells: async (_config, origin) => reserveCells[origin],
     readDirectorPlacement: async (_config, origin) => placements[origin] ?? 'unreported',
     adminIdentityToken: () => adminToken(callerEmail)
   }
-  return { state, removed, healthProtocols, operations }
+  return { state, removed, healthProtocols, trafficMoves, spreadWaits, operations }
 }
 
 test('director deploy removes stale and promoted Cloud Run tags', async () => {
@@ -701,6 +711,73 @@ test('director candidate inherits the serving warm-instance floor', async () =>
   assert.equal(rollback.minimum, 0)
 })
 
+test('a multi-instance floor moves part of the traffic until every floor instance serves', async () => {
+  const harness = directorHarness({ servingMinimum: 5 })
+  await deployDirector({}, 'candidate-new', harness.operations)
+  assert.deepEqual(harness.trafficMoves, ['--to-tags=candidate-new=20', '--to-tags=candidate-new=100'])
+  assert.deepEqual(harness.spreadWaits, [{ revision: 'relay-00003-new', required: 5 }])
+  assert.equal(harness.state.activeRevision, 'relay-00003-new')
+})
+
+test('a floor of one moves all traffic at once', async () => {
+  const harness = directorHarness()
+  await deployDirector({}, 'candidate-new', harness.operations)
+  assert.deepEqual(harness.trafficMoves, ['--to-tags=candidate-new=100'])
+  assert.deepEqual(harness.spreadWaits, [])
+})
+
+test('a candidate that never spreads returns traffic to the warm serving revision', async () => {
+  const failure = new Error('candidate served from 1 of 5 instances before the deadline')
+  const harness = directorHarness({ servingMinimum: 5, spreadFailure: failure })
+  await assert.rejects(deployDirector({}, 'candidate-new', harness.operations), failure)
+  assert.deepEqual(harness.trafficMoves, [
+    '--to-tags=candidate-new=20',
+    '--to-revisions=relay-00001-old=100'
+  ])
+  assert.equal(harness.state.activeRevision, 'relay-00001-old')
+  assert.equal(harness.state.tags.size, 0)
+})
+
+test('the spread wait counts distinct instances that served since the partial move', async () => {
+  const reads = [new Set(['a']), new Set(['a', 'b']), new Set(['a', 'b', 'c'])]
+  const seen = []
+  await waitForServingInstances({}, 'relay-00003-new', 3, '2026-10-20T05:25:00.000Z', {
+    readInstances: (_config, revision, since) => {
+      seen.push([revision, since])
+      return reads.shift()
+    },
+    pollIntervalMs: 0,
+    timeoutMs: 60_000
+  })
+  assert.equal(seen.length, 3)
+  assert.deepEqual(seen[0], ['relay-00003-new', '2026-10-20T05:25:00.000Z'])
+})
+
+test('the spread wait retries a failed log read', async () => {
+  let reads = 0
+  await waitForServingInstances({}, 'relay-00003-new', 2, '2026-10-20T05:25:00.000Z', {
+    readInstances: () => {
+      reads += 1
+      if (reads === 1) throw new Error('RESOURCE_EXHAUSTED')
+      return new Set(['a', 'b'])
+    },
+    pollIntervalMs: 0,
+    timeoutMs: 60_000
+  })
+  assert.equal(reads, 2)
+})
+
+test('the spread wait fails at its deadline', async () => {
+  await assert.rejects(
+    waitForServingInstances({}, 'relay-00003-new', 5, '2026-10-20T05:25:00.000Z', {
+      readInstances: () => new Set(['a']),
+      pollIntervalMs: 0,
+      timeoutMs: 0
+    }),
+    /served from 1 of 5 instances before the deadline/
+  )
+})
+
 test('director deploy rejects a serving maximum above the checked budget', async () => {
   const harness = directorHarness({ servingMaximum: 6 })
   await assert.rejects(

cloud/dev/scripts/relay-staging-deploy-identity.test.mjs

@@ -150,6 +150,7 @@ test('the staging deploy identity declares exactly its enumerated grants', () =>
     'google_cloud_run_v2_service_iam_member.github_staging_relay_deploy_director_developer',
     'google_iam_workload_identity_pool_provider.github_staging_relay_deploy',
     'google_project_iam_member.github_staging_relay_deploy_compute_viewer',
+    'google_project_iam_member.github_staging_relay_deploy_logging_viewer',
     'google_service_account.github_staging_relay_deploy',
     'google_service_account_iam_member.github_staging_relay_deploy_auth_runtime_user',
     'google_service_account_iam_member.github_staging_relay_deploy_workload_identity_user',
@@ -189,7 +190,7 @@ test('the staging deploy identity declares exactly its enumerated grants', () =>
   const projectRoles = declaredFamilies()
     .filter((family) => family.startsWith('google_project_iam_member.'))
     .map((family) => block('google_project_iam_member', family.split('.')[1]).match(/role\s*=\s*"([^"]+)"/)[1])
-  assert.deepEqual(projectRoles, ['roles/compute.viewer'])
+  assert.deepEqual(projectRoles, ['roles/compute.viewer', 'roles/logging.viewer'])
   assert.doesNotMatch(identity, /roles\/artifactregistry\.writer/)
   assert.doesNotMatch(identity, /"roles\/(?:owner|editor|viewer)"/)
 })

cloud/docs/orca-relay-operations.md

@@ -241,7 +241,19 @@ Every later change must use the selector CAS.
 Deploy the selector-compatible director before cutover. Blue/green deployment
 creates a separate `selector-rollback` revision at minimum instances zero,
 promotes a second compatible revision, and retains older revisions through the
-stabilization window. Cutover removes every older revision and refuses to
+stabilization window. With a floor above one, promotion first moves 20% of
+traffic and waits (up to six minutes) until every floor instance has served a
+request, because Cloud Run routes new floor instances only ~100 s after they
+start; a timeout returns all traffic to the previous serving revision.
+
+If the deploy job is cancelled or loses its runner during that wait, the 20/80
+split stays in place, and both the next deploy and the rehome workflow refuse a
+director without one 100% revision. Restore it with
+`gcloud run services update-traffic orca-cloud-relay --project onorca-cloud --region us-central1 --to-revisions=<previous serving revision>=100`
+(the revision that held 100% before the deploy). A timed-out deploy restores
+that itself but removes every tag, including `selector-rollback`, so the rehome
+workflow's rollback-digest check refuses (`rollback tag missing`) until the next
+successful director deploy. Cutover removes every older revision and refuses to
 proceed unless only the compatible active and rollback revisions remain.
 
 First inspect the current generation and exact membership:

cloud/infra/terraform/relay-staging-deploy-iam.tf

@@ -170,3 +170,13 @@ resource "google_project_iam_member" "github_staging_relay_deploy_compute_viewer
   role    = "roles/compute.viewer"
   member  = google_service_account.github_staging_relay_deploy[0].member
 }
+
+# Deploy Relay Staging step "Deploy director blue/green" with a director floor above one reads
+# the candidate's Cloud Run request log to confirm every floor instance serves before the full move.
+resource "google_project_iam_member" "github_staging_relay_deploy_logging_viewer" {
+  count = local.create_staging_relay_deploy_identity ? 1 : 0
+
+  project = var.project_id
+  role    = "roles/logging.viewer"
+  member  = google_service_account.github_staging_relay_deploy[0].member
+}