feat(relay): one-cell flag write workflow with dry run, compare-and-swap and read-back (#26837)

* feat(relay): one-cell flag write workflow with dry run, compare-and-swap and read-back

Operate Relay Production Cell Flags writes cells/<cellId>.json for one
cell: it checks the origin answers as that cell with a flag channel,
refuses a stale expected generation, writes with ifGenerationMatch,
stamps the actor and run on the object, and waits up to 15 s for the
cell's runtime status to report the new generation applied. The deploy
Workload Identity condition admits the new workflow.

* fix(relay): flag workflow retries a transient 5xx, shows the plan before the generation check, and names the failure

Runtime-status reads go through the shared one-retry admin helper, and a
failed read-back poll no longer ends the wait early. The plan line is
logged before the generation check, a failure is logged as one JSON line
the job summary carries, and the object and summary name the triggering
actor.

.github/workflows/cloud-operate-relay-production-cell-flags.yml

@@ -0,0 +1,111 @@
+name: Operate Relay Production Cell Flags
+
+# Writes one cell's runtime switch object and waits for the cell to report it applied.
+# One cell per run; never fleet-wide. dry-run writes nothing.
+on:
+  workflow_dispatch:
+    inputs:
+      mode:
+        description: dry-run prints the change; write applies it
+        required: true
+        default: dry-run
+        type: choice
+        options: [dry-run, write]
+      cell-id:
+        description: Exact cell id, e.g. production-gce-c26
+        required: true
+        type: string
+      readiness-local:
+        description: /ready answers from the cell alone (local readiness)
+        required: true
+        default: 'false'
+        type: choice
+        options: ['false', 'true']
+      ticket-check:
+        description: Shadow-check the director's assignment lease on host hellos
+        required: true
+        default: 'off'
+        type: choice
+        options: ['off', 'shadow']
+      expected-generation:
+        description: Exact current object generation from a dry run, or 0 if none exists
+        required: true
+        type: string
+      confirmation:
+        description: Required for write, exactly "WRITE_RELAY_CELL_FLAGS <cell-id>"
+        required: false
+        type: string
+
+permissions:
+  contents: read
+  id-token: write
+
+# One write at a time across the fleet keeps the audit trail in order.
+concurrency:
+  group: relay-production-cell-flags
+  cancel-in-progress: false
+
+defaults:
+  run:
+    # Explicit bash runs with pipefail, so the script's exit code survives the tee.
+    shell: bash
+    working-directory: cloud
+
+jobs:
+  operate:
+    if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    environment: production
+    env:
+      MODE: ${{ inputs.mode }}
+      CELL_ID: ${{ inputs.cell-id }}
+      READINESS_LOCAL: ${{ inputs.readiness-local }}
+      TICKET_CHECK: ${{ inputs.ticket-check }}
+      EXPECTED_GENERATION: ${{ inputs.expected-generation }}
+      CONFIRMATION: ${{ inputs.confirmation }}
+      GCP_PROJECT_ID: onorca-cloud
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 24
+
+      - id: google-auth
+        uses: google-github-actions/auth@v2
+        with:
+          workload_identity_provider: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ vars.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT }}
+          token_format: id_token
+          id_token_audience: https://relay.onorca.dev/v1/admin/drain
+          id_token_include_email: true
+
+      - uses: google-github-actions/setup-gcloud@v2
+
+      - name: Write or preview the cell's switch object
+        env:
+          ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.google-auth.outputs.id_token }}
+        run: |
+          [[ "${CELL_ID}" =~ ^production-gce-c[0-9]{1,3}$ ]]
+          ORCA_RELAY_CONTROL_ACCESS_TOKEN="$(gcloud auth print-access-token)"
+          echo "::add-mask::${ORCA_RELAY_CONTROL_ACCESS_TOKEN}"
+          export ORCA_RELAY_CONTROL_ACCESS_TOKEN
+          node dev/scripts/operate-relay-cell-flags.mjs \
+            --mode "${MODE}" --cell-id "${CELL_ID}" \
+            --cell-origin "https://${CELL_ID#production-gce-}.relay.onorca.dev" \
+            --project-id "${GCP_PROJECT_ID}" \
+            --readiness-local "${READINESS_LOCAL}" --ticket-check "${TICKET_CHECK}" \
+            --expected-generation "${EXPECTED_GENERATION}" --confirmation "${CONFIRMATION}" \
+            | tee "${RUNNER_TEMP}/relay-cell-flags.jsonl"
+
+      - name: Audit summary
+        if: ${{ always() }}
+        run: |
+          {
+            echo "### Relay cell flags: ${MODE} ${CELL_ID}"
+            echo "Triggered by: ${GITHUB_TRIGGERING_ACTOR}, run ${GITHUB_RUN_ID} attempt ${GITHUB_RUN_ATTEMPT}"
+            echo '```json'
+            cat "${RUNNER_TEMP}/relay-cell-flags.jsonl" 2>/dev/null || echo '(no output)'
+            echo '```'
+          } >> "${GITHUB_STEP_SUMMARY}"

cloud/dev/scripts/operate-relay-cell-flags.mjs

@@ -0,0 +1,240 @@
+#!/usr/bin/env node
+// Writes one cell's runtime switch object (cells/<cellId>.json in <project>-relay-control) and
+// reads back what the cell applied. One cell per run by construction: the object is per cell.
+//
+// dry-run: checks the cell and the current generation, prints the change, writes nothing.
+// write:   the same checks, then a compare-and-swap write (ifGenerationMatch), then waits for
+//          the cell's runtime status to report the new generation applied.
+import { parseArgs } from 'node:util'
+import { pathToFileURL } from 'node:url'
+import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs'
+
+export const CELL_ID_PATTERN = /^(production|staging)-gce-c[0-9]{1,3}$/
+export const WRITE_CONFIRMATION = 'WRITE_RELAY_CELL_FLAGS'
+// The cell polls every 5 s (+20% jitter); 15 s leaves two polls of slack.
+export const READ_BACK_TIMEOUT_MS = 15_000
+const READ_BACK_INTERVAL_MS = 1_000
+const STORAGE_ROOT = 'https://storage.googleapis.com'
+
+export function parseCellFlagsRequest(values) {
+  const mode = values.mode
+  if (mode !== 'dry-run' && mode !== 'write') throw new Error('mode must be dry-run or write')
+  const cellId = values['cell-id'] ?? ''
+  if (!CELL_ID_PATTERN.test(cellId)) throw new Error('cell id is not a relay GCE cell id')
+  if (values['readiness-local'] !== 'true' && values['readiness-local'] !== 'false') {
+    throw new Error('readiness-local must be true or false')
+  }
+  const ticketCheck = values['ticket-check']
+  if (ticketCheck !== 'off' && ticketCheck !== 'shadow') {
+    throw new Error('ticket-check must be off or shadow')
+  }
+  // 0 means "no object yet": the write then creates it and refuses if one appeared.
+  const expectedGeneration = values['expected-generation'] ?? ''
+  if (!/^(0|[1-9][0-9]{0,18})$/.test(expectedGeneration)) {
+    throw new Error('expected-generation must be 0 or the exact current object generation')
+  }
+  if (mode === 'write' && values.confirmation !== `${WRITE_CONFIRMATION} ${cellId}`) {
+    throw new Error(`write requires the confirmation "${WRITE_CONFIRMATION} ${cellId}"`)
+  }
+  const projectId = values['project-id'] ?? ''
+  if (!/^[a-z][a-z0-9-]{4,62}$/.test(projectId)) throw new Error('project id is invalid')
+  const cellOrigin = values['cell-origin'] ?? ''
+  if (!/^https:\/\/c[0-9]{1,3}\.relay(-staging)?\.onorca\.dev$/.test(cellOrigin)) {
+    throw new Error('cell origin is not a relay cell origin')
+  }
+  if (new URL(cellOrigin).hostname.split('.')[0] !== cellId.replace(/^[a-z]+-gce-/, '')) {
+    throw new Error('cell origin names another cell')
+  }
+  return {
+    mode,
+    cellId,
+    cellOrigin,
+    projectId,
+    expectedGeneration,
+    object: {
+      v: 1,
+      cellId,
+      flags: { readinessLocal: values['readiness-local'] === 'true', ticketCheck }
+    }
+  }
+}
+
+function objectUrl(request, suffix = '') {
+  const bucket = `${request.projectId}-relay-control`
+  return `${STORAGE_ROOT}/storage/v1/b/${bucket}/o/${encodeURIComponent(
+    `cells/${request.cellId}.json`
+  )}${suffix}`
+}
+
+// One transient 5xx or network failure is retried once, as every relay admin step does.
+async function readRuntime(fetchImpl, request, idToken, wait) {
+  const response = await fetchAdminOnceMore(
+    fetchImpl,
+    `${request.cellOrigin}/v1/admin/runtime-status`,
+    {
+      method: 'POST',
+      headers: { authorization: `Bearer ${idToken}`, 'content-type': 'application/json' },
+      body: JSON.stringify({ v: 1 })
+    },
+    { wait, timeoutMs: 10_000, retryDelayMs: 1_000 }
+  )
+  if (!response.ok) {
+    await response.body?.cancel().catch(() => undefined)
+    throw new Error(`cell runtime status returned ${response.status}`)
+  }
+  return await response.json()
+}
+
+async function readCurrentObject(fetchImpl, request, accessToken) {
+  const headers = { authorization: `Bearer ${accessToken}` }
+  const metadata = await fetchImpl(objectUrl(request), { headers })
+  if (metadata.status === 404) {
+    await metadata.body?.cancel().catch(() => undefined)
+    return { generation: '0', object: null }
+  }
+  if (!metadata.ok) throw new Error(`object metadata read returned ${metadata.status}`)
+  const { generation } = await metadata.json()
+  const media = await fetchImpl(objectUrl(request, `?alt=media&ifGenerationMatch=${generation}`), {
+    headers
+  })
+  if (!media.ok) throw new Error(`object read returned ${media.status}`)
+  return { generation: String(generation), object: await media.json().catch(() => null) }
+}
+
+async function writeObject(fetchImpl, request, accessToken, audit) {
+  const boundary = `relay-cell-flags-${Date.now()}`
+  const metadata = {
+    name: `cells/${request.cellId}.json`,
+    contentType: 'application/json',
+    cacheControl: 'no-store',
+    // Who and which run, beside the generation the bucket's versioning already keeps.
+    metadata: audit
+  }
+  const body = [
+    `--${boundary}`,
+    'Content-Type: application/json; charset=UTF-8',
+    '',
+    JSON.stringify(metadata),
+    `--${boundary}`,
+    'Content-Type: application/json',
+    '',
+    JSON.stringify(request.object),
+    `--${boundary}--`,
+    ''
+  ].join('\r\n')
+  const bucket = `${request.projectId}-relay-control`
+  const response = await fetchImpl(
+    `${STORAGE_ROOT}/upload/storage/v1/b/${bucket}/o?uploadType=multipart&ifGenerationMatch=${request.expectedGeneration}`,
+    {
+      method: 'POST',
+      headers: {
+        authorization: `Bearer ${accessToken}`,
+        'content-type': `multipart/related; boundary=${boundary}`
+      },
+      body
+    }
+  )
+  if (response.status === 412) {
+    await response.body?.cancel().catch(() => undefined)
+    throw new Error('the object changed since expected-generation; re-read and retry')
+  }
+  if (!response.ok) throw new Error(`object write returned ${response.status}`)
+  return String((await response.json()).generation)
+}
+
+const sameFlags = (left, right) =>
+  left?.readinessLocal === right.readinessLocal && left?.ticketCheck === right.ticketCheck
+
+export async function operateCellFlags(request, dependencies) {
+  const { fetchImpl = fetch, accessToken, idToken, audit, log = console.log } = dependencies
+  const now = dependencies.now ?? Date.now
+  const sleep = dependencies.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)))
+  const runtime = await readRuntime(fetchImpl, request, idToken, sleep)
+  // Proves the origin is this cell, and that its image reads switch files at all.
+  if (runtime.role !== 'cell' || runtime.cellId !== request.cellId) {
+    throw new Error(`origin answers as ${runtime.role}/${runtime.cellId}, not ${request.cellId}`)
+  }
+  if (!runtime.flagsApplied) throw new Error('cell image has no flag channel (no flagsApplied)')
+  const current = await readCurrentObject(fetchImpl, request, accessToken)
+  const plan = {
+    event: 'orca_relay_cell_flags_plan',
+    mode: request.mode,
+    cellId: request.cellId,
+    currentGeneration: current.generation,
+    currentObject: current.object,
+    appliedBefore: runtime.flagsApplied,
+    desired: request.object
+  }
+  // Before the generation check, so a dry run always shows the generation to pass next.
+  log(JSON.stringify(plan))
+  if (current.generation !== request.expectedGeneration) {
+    throw new Error(
+      `current generation is ${current.generation}, not expected ${request.expectedGeneration}`
+    )
+  }
+  if (request.mode === 'dry-run') return { ...plan, written: false }
+  const generation = await writeObject(fetchImpl, request, accessToken, audit)
+  log(JSON.stringify({ event: 'orca_relay_cell_flags_written', cellId: request.cellId, generation }))
+  const deadline = now() + READ_BACK_TIMEOUT_MS
+  let last = runtime.flagsApplied
+  while (now() < deadline) {
+    await sleep(READ_BACK_INTERVAL_MS)
+    try {
+      last = (await readRuntime(fetchImpl, request, idToken, sleep)).flagsApplied
+    } catch {
+      // The object is written; a failed poll only delays the read-back until the deadline.
+      continue
+    }
+    if (String(last?.generation) === generation && sameFlags(last.flags, request.object.flags)) {
+      const result = { ...plan, written: true, generation, applied: last }
+      log(JSON.stringify({ ...result, event: 'orca_relay_cell_flags_applied_read_back' }))
+      return result
+    }
+  }
+  throw new Error(
+    `cell did not apply generation ${generation} within ${READ_BACK_TIMEOUT_MS} ms ` +
+      `(last applied ${JSON.stringify(last)}); the object is written`
+  )
+}
+
+if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) {
+  const { values } = parseArgs({
+    options: Object.fromEntries(
+      [
+        'mode',
+        'cell-id',
+        'cell-origin',
+        'project-id',
+        'readiness-local',
+        'ticket-check',
+        'expected-generation',
+        'confirmation'
+      ].map((name) => [name, { type: 'string' }])
+    )
+  })
+  try {
+    const request = parseCellFlagsRequest(values)
+    const accessToken = process.env.ORCA_RELAY_CONTROL_ACCESS_TOKEN
+    const idToken = process.env.ORCA_RELAY_ADMIN_ID_TOKEN
+    if (!accessToken || !idToken) throw new Error('both tokens are required')
+    await operateCellFlags(request, {
+      accessToken,
+      idToken,
+      audit: {
+        // The person who started this attempt, which differs from GITHUB_ACTOR on a re-run.
+        writtenBy: process.env.GITHUB_TRIGGERING_ACTOR ?? process.env.GITHUB_ACTOR ?? 'unknown',
+        runId: process.env.GITHUB_RUN_ID ?? 'local',
+        runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? '0'
+      }
+    })
+  } catch (error) {
+    // On stdout, so the job summary's copy of the run carries the reason.
+    console.log(
+      JSON.stringify({
+        event: 'orca_relay_cell_flags_failed',
+        message: error instanceof Error ? error.message : String(error)
+      })
+    )
+    process.exitCode = 1
+  }
+}

cloud/dev/scripts/operate-relay-cell-flags.test.mjs

@@ -0,0 +1,184 @@
+import assert from 'node:assert/strict'
+import { test } from 'node:test'
+import {
+  operateCellFlags,
+  parseCellFlagsRequest,
+  READ_BACK_TIMEOUT_MS,
+  WRITE_CONFIRMATION
+} from './operate-relay-cell-flags.mjs'
+
+const CELL = 'production-gce-c26'
+const values = (overrides = {}) => ({
+  mode: 'write',
+  'cell-id': CELL,
+  'cell-origin': 'https://c26.relay.onorca.dev',
+  'project-id': 'onorca-cloud',
+  'readiness-local': 'true',
+  'ticket-check': 'off',
+  'expected-generation': '0',
+  confirmation: `${WRITE_CONFIRMATION} ${CELL}`,
+  ...overrides
+})
+
+// A cell whose runtime status applies whatever was written, after `applyAfterPolls` polls.
+function fakeGoogleAndCell({ stored = null, applyAfterPolls = 2, role = 'cell', cellId = CELL } = {}) {
+  const state = { stored, writes: [], polls: 0, applied: { generation: 0, flags: { readinessLocal: false, ticketCheck: 'off' } } }
+  const fetchImpl = async (url, init = {}) => {
+    const target = new URL(url)
+    if (target.pathname === '/v1/admin/runtime-status') {
+      state.polls += 1
+      if (state.stored && state.polls > applyAfterPolls) {
+        state.applied = { generation: Number(state.stored.generation), flags: state.stored.object.flags }
+      }
+      return Response.json({ v: 1, role, cellId, flagsApplied: state.applied })
+    }
+    if (target.pathname.startsWith('/upload/')) {
+      const expected = target.searchParams.get('ifGenerationMatch')
+      if (expected !== (state.stored?.generation ?? '0')) return new Response(null, { status: 412 })
+      const parts = String(init.body).split('\r\n')
+      const object = JSON.parse(parts[parts.indexOf('Content-Type: application/json') + 2])
+      const metadata = JSON.parse(parts[3])
+      const generation = String(Number(state.stored?.generation ?? 1000) + 1)
+      state.stored = { generation, object }
+      state.writes.push({ generation, object, metadata })
+      return Response.json({ generation })
+    }
+    if (!state.stored) return new Response(null, { status: 404 })
+    if (target.searchParams.get('alt') === 'media') return Response.json(state.stored.object)
+    return Response.json({ generation: state.stored.generation })
+  }
+  return { state, fetchImpl }
+}
+
+const run = (request, fake, extra = {}) => {
+  let clock = 0
+  const lines = []
+  return {
+    lines,
+    result: operateCellFlags(request, {
+      fetchImpl: fake.fetchImpl,
+      accessToken: 'access',
+      idToken: 'id',
+      audit: { writtenBy: 'octocat', runId: '42', runAttempt: '1' },
+      log: (line) => lines.push(JSON.parse(line)),
+      now: () => clock,
+      sleep: async (ms) => {
+        clock += ms
+      },
+      ...extra
+    })
+  }
+}
+
+test('refuses malformed requests before touching anything', () => {
+  assert.throws(() => parseCellFlagsRequest(values({ mode: 'apply' })), /mode/)
+  assert.throws(() => parseCellFlagsRequest(values({ 'cell-id': 'all' })), /cell id/)
+  assert.throws(() => parseCellFlagsRequest(values({ 'ticket-check': 'enforce' })), /ticket-check/)
+  assert.throws(() => parseCellFlagsRequest(values({ 'expected-generation': '' })), /expected/)
+  assert.throws(() => parseCellFlagsRequest(values({ confirmation: WRITE_CONFIRMATION })), /confirmation/)
+  assert.throws(
+    () => parseCellFlagsRequest(values({ 'cell-origin': 'https://c27.relay.onorca.dev' })),
+    /another cell/
+  )
+  assert.throws(() => parseCellFlagsRequest(values({ 'cell-origin': 'https://evil.example' })), /origin/)
+  // A dry run needs no confirmation.
+  assert.equal(parseCellFlagsRequest(values({ mode: 'dry-run', confirmation: '' })).mode, 'dry-run')
+})
+
+test('a dry run reports the change and writes nothing', async () => {
+  const fake = fakeGoogleAndCell()
+  const { result, lines } = run(parseCellFlagsRequest(values({ mode: 'dry-run' })), fake)
+  const outcome = await result
+  assert.equal(outcome.written, false)
+  assert.equal(fake.state.writes.length, 0)
+  assert.equal(lines[0].event, 'orca_relay_cell_flags_plan')
+  assert.deepEqual(lines[0].desired.flags, { readinessLocal: true, ticketCheck: 'off' })
+})
+
+test('writes with compare-and-swap, records who wrote it, and waits for the cell to apply it', async () => {
+  const fake = fakeGoogleAndCell()
+  const { result, lines } = run(parseCellFlagsRequest(values()), fake)
+  const outcome = await result
+  assert.equal(outcome.written, true)
+  assert.equal(outcome.generation, '1001')
+  assert.deepEqual(outcome.applied, { generation: 1001, flags: { readinessLocal: true, ticketCheck: 'off' } })
+  assert.deepEqual(fake.state.writes[0].metadata.metadata, { writtenBy: 'octocat', runId: '42', runAttempt: '1' })
+  assert.deepEqual(fake.state.writes[0].object, {
+    v: 1,
+    cellId: CELL,
+    flags: { readinessLocal: true, ticketCheck: 'off' }
+  })
+  assert.deepEqual(lines.map((line) => line.event), [
+    'orca_relay_cell_flags_plan',
+    'orca_relay_cell_flags_written',
+    'orca_relay_cell_flags_applied_read_back'
+  ])
+  // Flip back: the next write must name the generation just written.
+  const back = run(
+    parseCellFlagsRequest(values({ 'readiness-local': 'false', 'expected-generation': '1001' })),
+    fake
+  )
+  assert.equal((await back.result).generation, '1002')
+})
+
+test('refuses a stale expected generation, and a concurrent writer between read and write', async () => {
+  const fake = fakeGoogleAndCell({
+    stored: { generation: '7', object: { v: 1, cellId: CELL, flags: {} } }
+  })
+  await assert.rejects(run(parseCellFlagsRequest(values()), fake).result, /current generation is 7/)
+  assert.equal(fake.state.writes.length, 0)
+  const racing = fakeGoogleAndCell()
+  const write = racing.fetchImpl
+  racing.fetchImpl = async (url, init) => {
+    if (new URL(url).pathname.startsWith('/upload/')) racing.state.stored = { generation: '9', object: {} }
+    return await write(url, init)
+  }
+  await assert.rejects(run(parseCellFlagsRequest(values()), racing).result, /changed since/)
+})
+
+test('a dry run against a stale generation still shows the plan, then refuses', async () => {
+  const fake = fakeGoogleAndCell({
+    stored: { generation: '7', object: { v: 1, cellId: CELL, flags: {} } }
+  })
+  const { result, lines } = run(parseCellFlagsRequest(values({ mode: 'dry-run' })), fake)
+  await assert.rejects(result, /current generation is 7/)
+  assert.equal(lines[0].event, 'orca_relay_cell_flags_plan')
+  assert.equal(lines[0].currentGeneration, '7')
+})
+
+test('rides out a transient 5xx on the first read and during the read-back', async () => {
+  const fake = fakeGoogleAndCell()
+  const inner = fake.fetchImpl
+  let runtimeCalls = 0
+  fake.fetchImpl = async (url, init) => {
+    if (new URL(url).pathname === '/v1/admin/runtime-status') {
+      runtimeCalls += 1
+      // The first call, and the first read-back poll after the write.
+      if (runtimeCalls === 1 || runtimeCalls === 3) return new Response(null, { status: 503 })
+    }
+    return await inner(url, init)
+  }
+  const outcome = await run(parseCellFlagsRequest(values()), fake).result
+  assert.equal(outcome.written, true)
+  assert.equal(outcome.generation, '1001')
+})
+
+test('refuses an origin that is not the named cell, or an image without the flag channel', async () => {
+  await assert.rejects(
+    run(parseCellFlagsRequest(values()), fakeGoogleAndCell({ cellId: 'production-gce-c25' })).result,
+    /not production-gce-c26/
+  )
+  const old = fakeGoogleAndCell()
+  const fetchImpl = old.fetchImpl
+  old.fetchImpl = async (url, init) =>
+    new URL(url).pathname === '/v1/admin/runtime-status'
+      ? Response.json({ v: 1, role: 'cell', cellId: CELL })
+      : await fetchImpl(url, init)
+  await assert.rejects(run(parseCellFlagsRequest(values()), old).result, /no flag channel/)
+})
+
+test('fails loudly when the cell does not apply the write within the read-back window', async () => {
+  const fake = fakeGoogleAndCell({ applyAfterPolls: READ_BACK_TIMEOUT_MS })
+  await assert.rejects(run(parseCellFlagsRequest(values()), fake).result, /did not apply generation 1001/)
+  assert.equal(fake.state.writes.length, 1)
+})

cloud/dev/scripts/relay-production-identity-boundaries.test.mjs

@@ -31,6 +31,7 @@ test('no workflow names the retired generic production deploy identity', async (
     'deploy-relay-production-same-cap.yml',
     'deploy-relay-production.yml',
     'operate-relay-asia-admission.yml',
+    'operate-relay-production-cell-flags.yml',
     'operate-relay-production-rehome-job.yml',
     'publish-relay-production.yml'
   ].map((name) => relayWorkflowFile(name)).sort())

cloud/dev/scripts/relay-public-workflow-contract.test.mjs

@@ -20,7 +20,7 @@ const UNGATED = relayWorkflowFile('verify.yml')
 const relayWorkflows = () => workflowFiles().filter((file) => file !== UNGATED)
 
 test('the copy carries every relay workflow', () => {
-  assert.equal(relayWorkflows().length, 25)
+  assert.equal(relayWorkflows().length, 26)
 })
 
 // Why: workflow_run chains match by display name, not filename. Renaming a file is safe; renaming

cloud/dev/scripts/workload-identity-attribute-conditions.test.mjs

@@ -34,7 +34,7 @@ const EXPECTED_CONDITIONS = {
         "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && assertion.event_name == 'workflow_dispatch' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-push-deploy.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-push-deploy.yml@refs/heads/main'",
 
       github:
-        "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && ((assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-fence-broker.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-director.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-multi-target.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-asia-admission.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-publish-relay-production.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-rehome.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-rehome-job.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main' && (assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml@refs/heads/main' || assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main')))",
+        "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && ((assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-fence-broker.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-director.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-multi-target.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-asia-admission.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-cell-flags.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-publish-relay-production.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-rehome.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-rehome-job.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main' && (assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml@refs/heads/main' || assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main')))",
       github_monitor:
         "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-monitor-relay-production.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-monitor-relay-production-job.yml@refs/heads/main'",
       github_fence:

cloud/infra/terraform/relay-github-actions.tf

@@ -19,6 +19,7 @@ locals {
     "deploy-relay-production-multi-target.yml",
     "deploy-relay-production.yml",
     "operate-relay-asia-admission.yml",
+    "operate-relay-production-cell-flags.yml",
     "publish-relay-production.yml"
 
   ]

cloud/package.json

@@ -22,7 +22,7 @@
     "load:relay:recovery-gate": "node dev/scripts/run-relay-recovery-wave-gate.mjs",
     "ops:relay": "pnpm --filter @orca-cloud/relay-ops dev",
     "pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-lock-contention-alerts.test.mjs dev/scripts/relay-reconnect-herd-alerts.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs",
-    "test": "pnpm -r test && node --test dev/scripts/check-relay-same-cap-headroom.test.mjs dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/drive-relay-director-deploy.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/measure-relay-drain-disconnect-gap.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/push-gateway-workflow.test.mjs dev/scripts/push-gateway-recovery.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-job-mode-conditions.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-same-cap-shadow-gate.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs",
+    "test": "pnpm -r test && node --test dev/scripts/check-relay-same-cap-headroom.test.mjs dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/drive-relay-director-deploy.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/measure-relay-drain-disconnect-gap.test.mjs dev/scripts/operate-relay-cell-flags.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/push-gateway-workflow.test.mjs dev/scripts/push-gateway-recovery.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-job-mode-conditions.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-same-cap-shadow-gate.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs",
     "typecheck": "pnpm -r typecheck"
   },
   "devDependencies": {