feat(cloud): native push gateway and dedicated infrastructure (1/3) (#19912)

* refactor(cloud): share PostgreSQL schema startup between services

* feat(cloud): add durable native push notification gateway

* infra(push): define dedicated gateway resources and operational checks

* fix(push): bound cross-host admission and simplify gateway configuration

* fix(push): validate deploy configuration and preserve topic-error registrations

cloud/dev/scripts/push-cloud-run-model.mjs

@@ -0,0 +1,140 @@
+import { readFileSync, writeFileSync } from 'node:fs'
+
+// Executable fake gcloud: revision deletion obeys the platform's latest/traffic constraints.
+const path = process.env.MODEL_STATE
+const state = JSON.parse(readFileSync(path, 'utf8'))
+const args = process.argv.slice(2)
+const option = (name) => args[args.indexOf(name) + 1]
+const has = (name) => args.includes(name)
+const fail = (message) => {
+  throw new Error(message)
+}
+const persist = () => writeFileSync(path, JSON.stringify(state))
+const output = (value) => console.log(typeof value === 'string' ? value : JSON.stringify(value))
+const revision = (name) => state.revisions[name] ?? fail(`missing revision ${name}`)
+const traffic = () => [
+  { revisionName: state.serving, percent: 100 },
+  ...Object.entries(state.tags).map(([tag, name]) => ({
+    tag,
+    revisionName: name,
+    url: `https://${tag}.test`
+  }))
+]
+state.trace.push(args.join(' '))
+try {
+  if (args[0] === 'curl') {
+    if (state.failure === 'public' && args.some((arg) => arg.includes('https://public.test'))) {
+      fail('public check failed')
+    }
+    const url = args.find((arg) => arg.startsWith('https://'))
+    const tag = new URL(url).hostname.split('.')[0]
+    const name = state.tags[tag] ?? state.serving
+    if (has('-w')) {
+      output('200')
+    } else {
+      output({
+        ok: true,
+        deliveryProtocol: 2,
+        mode: revision(name).spec.containers[0].env.some((entry) => entry.value === 'validation')
+          ? 'validation'
+          : 'active'
+      })
+    }
+  } else if (args.slice(0, 2).join(' ') === 'run deploy') {
+    const name = `${option('deploy')}-${option('--revision-suffix')}`
+    if (state.failure === 'deploy-before') {
+      fail('deploy failed before create')
+    }
+    const item = structuredClone(revision(state.latest))
+    item.metadata.name = name
+    item.spec.containers[0].image = option('--image')
+    item.status.imageDigest = option('--image')
+    item.spec.containers[0].env = item.spec.containers[0].env.filter(
+      (entry) => entry.name !== 'ORCA_PUSH_MODE'
+    )
+    if (has('--update-env-vars')) {
+      item.spec.containers[0].env.push({ name: 'ORCA_PUSH_MODE', value: 'validation' })
+    }
+    state.revisions[name] = item
+    state.latest = name
+    if (has('--tag')) {
+      state.tags[option('--tag')] = name
+    }
+    state.peak = Math.max(state.peak, Object.keys(state.revisions).length)
+    if (state.peak > 3) {
+      fail('three-revision budget exceeded')
+    }
+    if (state.failure === 'deploy-after') {
+      fail('deploy failed after create')
+    }
+  } else if (args.slice(0, 3).join(' ') === 'run services describe') {
+    if (state.failure === 'describe') {
+      fail('describe failed')
+    }
+    if (args.some((arg) => arg.includes('value(status.latestCreatedRevisionName)'))) {
+      output(state.latest)
+    } else {
+      const template = structuredClone(revision(state.latest))
+      delete template.spec.containers[0].name
+      output({
+        spec: { template },
+        status: { latestCreatedRevisionName: state.latest, traffic: traffic() }
+      })
+    }
+  } else if (args.slice(0, 3).join(' ') === 'run services update-traffic') {
+    if (has('--to-revisions')) {
+      const name = option('--to-revisions').split('=')[0]
+      revision(name)
+      state.serving = name
+    }
+    if (has('--remove-tags')) {
+      for (const tag of option('--remove-tags').split(',')) {
+        delete state.tags[tag]
+      }
+    }
+    if (has('--clear-tags')) {
+      state.tags = {}
+    }
+    if (state.failure === 'traffic-after') {
+      fail('traffic changed but response failed')
+    }
+  } else if (args.slice(0, 3).join(' ') === 'run revisions list') {
+    const names = Object.keys(state.revisions)
+    output(
+      (has('--filter')
+        ? names.filter((name) => name === option('--filter').split('=')[1])
+        : names
+      ).join('\n')
+    )
+  } else if (args.slice(0, 3).join(' ') === 'run revisions describe') {
+    const item = revision(args[3])
+    const format = args.find((arg) => arg.startsWith('--format=')) ?? option('--format')
+    if (format.includes('minScale')) {
+      output(item.metadata.annotations['autoscaling.knative.dev/minScale'])
+    } else if (format.includes('maxScale')) {
+      output(item.metadata.annotations['autoscaling.knative.dev/maxScale'])
+    } else {
+      output(item)
+    }
+  } else if (args.slice(0, 3).join(' ') === 'run revisions delete') {
+    const name = args[3]
+    if (name === state.latest) {
+      fail('FAILED_PRECONDITION: latest created Revision cannot be directly deleted')
+    }
+    if (name === state.serving || Object.values(state.tags).includes(name)) {
+      fail('revision has traffic or tags')
+    }
+    if (state.failure === 'delete') {
+      fail('delete failed')
+    }
+    revision(name)
+    delete state.revisions[name]
+  } else {
+    fail(`unmodeled gcloud call ${args.join(' ')}`)
+  }
+} catch (error) {
+  console.error(error.message)
+  process.exitCode = 1
+} finally {
+  persist()
+}

cloud/dev/scripts/push-gateway-recovery.test.mjs

@@ -0,0 +1,373 @@
+import assert from 'node:assert/strict'
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { spawnSync } from 'node:child_process'
+import test from 'node:test'
+import { readRelayWorkflow } from './relay-repository.mjs'
+
+const workflow = readRelayWorkflow('push-deploy.yml')
+function step(name) {
+  const start = workflow.indexOf(`      - name: ${name}\n`)
+  assert.notEqual(start, -1)
+  const end = workflow.indexOf('\n      - ', start + 1)
+  const block = workflow.slice(start, end === -1 ? undefined : end)
+  return block
+    .slice(block.indexOf('        run: |\n') + '        run: |\n'.length)
+    .split('\n')
+    .filter((line) => line.startsWith('          '))
+    .map((line) => line.slice(10))
+    .join('\n')
+}
+const names = {
+  preflight: 'Record the serving revision and require its Terraform-owned scaling',
+  candidate: 'Deploy the candidate revision with no traffic',
+  activate: 'Retire inert validation and activate the verified image',
+  shift: 'Shift all traffic to the verified candidate',
+  public: 'Verify the public origin after the shift',
+  rollback: 'Roll traffic back to the previous revision',
+  restore: 'Restore the known-good service template',
+  promoteRecovery: 'Promote and verify the known-good recovery revision',
+  cleanup: 'Delete the rejected candidate revision',
+  retire: 'Retire previous consumers after public checks'
+}
+const image = `registry/push@sha256:${'a'.repeat(64)}`
+const spec = {
+  serviceAccountName: 'runtime@test',
+  containerConcurrency: 40,
+  containers: [
+    {
+      image,
+      name: 'push-test-1',
+      env: [
+        {
+          name: 'ORCA_PUSH_DATABASE_URL',
+          valueFrom: { secretKeyRef: { name: 'database', key: '7' } }
+        },
+        { name: 'ORCA_PUSH_DATABASE_POOL_MAX', value: '2' }
+      ]
+    }
+  ]
+}
+const prior = {
+  metadata: {
+    name: 'push-test-old',
+    annotations: {
+      'autoscaling.knative.dev/minScale': '1',
+      'autoscaling.knative.dev/maxScale': '2'
+    }
+  },
+  spec,
+  status: { imageDigest: image }
+}
+const model = fileURLToPath(new URL('./push-cloud-run-model.mjs', import.meta.url))
+const options = { skip: process.platform === 'win32' }
+function exercise(callback) {
+  const dir = mkdtempSync(join(tmpdir(), 'push-workflow-'))
+  const statePath = join(dir, 'state.json')
+  writeFileSync(
+    statePath,
+    JSON.stringify({
+      revisions: { 'push-test-old': prior },
+      latest: 'push-test-old',
+      serving: 'push-test-old',
+      tags: {},
+      peak: 1,
+      trace: []
+    })
+  )
+  writeFileSync(join(dir, 'env'), '')
+  const env = {
+    ...process.env,
+    SERVICE_NAME: 'push-test',
+    GCP_PROJECT_ID: 'test',
+    GCP_REGION: 'test',
+    GITHUB_RUN_ID: '123',
+    GITHUB_RUN_ATTEMPT: '1',
+    IMAGE: `registry/push@sha256:${'b'.repeat(64)}`,
+    PUSH_MIN_INSTANCES: '1',
+    PUSH_MAX_INSTANCES: '2',
+    PUSH_RUNTIME_SERVICE_ACCOUNT: 'runtime@test',
+    PUSH_ORIGIN: 'https://public.test',
+    MODEL_STATE: statePath,
+    MODEL_SCRIPT: model,
+    RUNNER_TEMP: dir,
+    GITHUB_ENV: join(dir, 'env'),
+    GITHUB_STEP_SUMMARY: join(dir, 'summary')
+  }
+  const state = () => JSON.parse(readFileSync(statePath, 'utf8'))
+  const change = (edit) => {
+    const value = state()
+    edit(value)
+    writeFileSync(statePath, JSON.stringify(value))
+  }
+  const run = (key, ok = true, extra = '') => {
+    const result = spawnSync(
+      'bash',
+      [
+        '-c',
+        `
+      set -a
+      source "$GITHUB_ENV"
+      gcloud() { node "$MODEL_SCRIPT" "$@"; }
+      curl() { node "$MODEL_SCRIPT" curl "$@"; }
+      sleep() { :; }
+      ${extra}
+      ${names[key] ? step(names[key]) : key}
+    `
+      ],
+      { cwd: dir, env, encoding: 'utf8', timeout: 30000 }
+    )
+    assert.equal(result.status === 0, ok, `${key}: ${result.stderr}\n${result.stdout}`)
+    return result
+  }
+  try {
+    callback({ run, state, change, dir })
+  } finally {
+    rmSync(dir, { recursive: true, force: true })
+  }
+}
+function recover(h) {
+  h.change((state) => {
+    delete state.failure
+  })
+  h.run('restore')
+  h.run('promoteRecovery')
+  h.run('cleanup')
+  h.run('retire')
+  const state = h.state()
+  assert.equal(state.serving, 'push-test-r123-1')
+  assert.equal(state.latest, state.serving)
+  assert.deepEqual(Object.keys(state.revisions), [state.serving])
+  assert.equal(state.revisions[state.serving].spec.containers[0].image, image)
+  assert.ok(state.peak <= 3)
+}
+
+test(
+  'the executable Cloud Run model rejects deleting latest even without tags or traffic',
+  options,
+  () =>
+    exercise((h) => {
+      h.run('preflight')
+      h.run('candidate')
+      h.run('gcloud run services update-traffic "$SERVICE_NAME" --clear-tags')
+      const result = h.run('gcloud run revisions delete "$CANDIDATE_REVISION"', false)
+      assert.match(result.stderr, /FAILED_PRECONDITION: latest created Revision/)
+    })
+)
+
+test(
+  'success creates successor before retirement and repeated rollouts retain one consumer',
+  options,
+  () =>
+    exercise((h) => {
+      for (const attempt of ['1', '2']) {
+        if (attempt === '2') {
+          writeFileSync(join(h.dir, 'env'), 'GITHUB_RUN_ATTEMPT=2\n')
+        }
+        for (const key of ['preflight', 'candidate', 'activate', 'shift', 'public', 'retire']) {
+          h.run(key)
+        }
+        const state = h.state()
+        assert.equal(state.serving, `push-test-a123-${attempt}`)
+        assert.deepEqual(Object.keys(state.revisions), [state.serving])
+        assert.equal(state.peak, 3)
+      }
+    })
+)
+
+for (const failure of ['deploy-before', 'deploy-after', 'describe']) {
+  test(`validation ${failure} recovers without deleting latest`, options, () =>
+    exercise((h) => {
+      h.run('preflight')
+      h.change((state) => {
+        state.failure = failure
+      })
+      h.run('candidate', false)
+      recover(h)
+    })
+  )
+}
+for (const failure of ['deploy-before', 'deploy-after', 'delete', 'describe']) {
+  test(
+    `activation ${failure} frees validation slot before recovery and stays within three`,
+    options,
+    () =>
+      exercise((h) => {
+        h.run('preflight')
+        h.run('candidate')
+        h.change((state) => {
+          state.failure = failure
+        })
+        h.run('activate', false)
+        recover(h)
+      })
+  )
+}
+
+test(
+  'ambiguous traffic shift records intent before mutation, rolls back and recovers',
+  options,
+  () =>
+    exercise((h) => {
+      h.run('preflight')
+      h.run('candidate')
+      h.run('activate')
+      h.change((state) => {
+        state.failure = 'traffic-after'
+      })
+      h.run('shift', false)
+      assert.match(readFileSync(join(h.dir, 'env'), 'utf8'), /TRAFFIC_SHIFT_ATTEMPTED=true/)
+      h.change((state) => {
+        delete state.failure
+      })
+      h.run('rollback')
+      recover(h)
+    })
+)
+
+test('failed public check rolls back and recovers', options, () =>
+  exercise((h) => {
+    for (const key of ['preflight', 'candidate', 'activate', 'shift']) {
+      h.run(key)
+    }
+    h.change((state) => {
+      state.failure = 'public'
+    })
+    h.run('public', false)
+    h.change((state) => {
+      delete state.failure
+    })
+    h.run('rollback')
+    recover(h)
+  })
+)
+
+for (const defect of [
+  'runtime',
+  'secret',
+  'mode',
+  'image',
+  'traffic',
+  'scaling',
+  'deploy-before',
+  'deploy-after'
+]) {
+  test(`recovery rejects ${defect} and preserves partial-create state`, options, () =>
+    exercise((h) => {
+      h.run('preflight')
+      h.run('candidate')
+      h.change((state) => {
+        const revision = state.revisions[state.latest]
+        if (defect === 'runtime') {
+          revision.spec.serviceAccountName = 'wrong@test'
+        }
+        if (defect === 'secret') {
+          revision.spec.containers[0].env[0].valueFrom.secretKeyRef.key = '8'
+        }
+        if (defect === 'scaling') {
+          revision.metadata.annotations['autoscaling.knative.dev/maxScale'] = '3'
+        }
+        if (defect === 'traffic') {
+          state.serving = state.latest
+        }
+        if (defect.startsWith('deploy-')) {
+          state.failure = defect
+        }
+      })
+      // Corrupt the recovery response after the modeled deploy while keeping real jq assertions.
+      const extra = ['mode', 'image'].includes(defect)
+        ? `
+      gcloud() {
+        node "$MODEL_SCRIPT" "$@" > "$RUNNER_TEMP/out" || return $?
+        if [[ "$*" == 'run services describe '* && "$*" == *'--format=json'* ]]; then
+          jq '${defect === 'mode' ? '.spec.template.spec.containers[0].env += [{name:"ORCA_PUSH_MODE",value:"validation"}]' : '.spec.template.spec.containers[0].image = "wrong"'}' "$RUNNER_TEMP/out"
+        else cat "$RUNNER_TEMP/out"; fi
+      }`
+        : ''
+      h.run('restore', false, extra)
+      const recorded = readFileSync(join(h.dir, 'env'), 'utf8')
+      assert.match(recorded, /TEMPLATE_RECOVERY_REVISION=push-test-r123-1/)
+      assert.doesNotMatch(recorded, /TEMPLATE_RESTORED=true/)
+      assert.ok(h.state().peak <= 3)
+    })
+  )
+}
+
+test('failed validation retirement blocks a fourth revision during recovery', options, () =>
+  exercise((h) => {
+    h.run('preflight')
+    h.run('candidate')
+    h.change((state) => {
+      state.failure = 'delete'
+    })
+    h.run('activate', false)
+    h.run('restore', false)
+    assert.equal(h.state().peak, 3)
+    assert.equal(h.state().revisions['push-test-r123-1'], undefined)
+  })
+)
+
+test(
+  'failed retirement after public checks leaves verified serving and blocks the next run',
+  options,
+  () =>
+    exercise((h) => {
+      for (const key of ['preflight', 'candidate', 'activate', 'shift', 'public']) {
+        h.run(key)
+      }
+      h.change((state) => {
+        state.failure = 'delete'
+      })
+      h.run('retire', false)
+      assert.equal(h.state().serving, 'push-test-a123-1')
+      h.run('preflight', false)
+      assert.match(workflow, /env.ROLLOUT_VERIFIED != 'true'/)
+    })
+)
+
+test(
+  'recovery promotion failure keeps consumers for operator diagnosis and blocks new rollout',
+  options,
+  () =>
+    exercise((h) => {
+      h.run('preflight')
+      h.run('candidate')
+      h.run('restore')
+      h.change((state) => {
+        state.failure = 'public'
+      })
+      h.run('promoteRecovery', false)
+      assert.doesNotMatch(readFileSync(join(h.dir, 'env'), 'utf8'), /RECOVERY_VERIFIED=true/)
+      h.run('preflight', false)
+    })
+)
+
+const capability = step('Require image support for inert validation')
+for (const [label, source, ok] of [
+  ['old image', 'export function loadPushConfig() { return {}; }', false],
+  [
+    'invalid mode accepted',
+    'export function loadPushConfig(env) { return { mode: env.ORCA_PUSH_MODE }; }',
+    false
+  ],
+  [
+    'validation supported',
+    `export function loadPushConfig(env) {
+    if (env.ORCA_PUSH_MODE !== 'validation') throw new Error('invalid mode');
+    return { mode: 'validation' };
+  }`,
+    true
+  ]
+]) {
+  test(`pre-production image smoke: ${label}`, options, () =>
+    exercise((h) => {
+      const dist = join(h.dir, 'apps', 'push', 'dist')
+      mkdirSync(dist, { recursive: true })
+      writeFileSync(join(h.dir, 'package.json'), '{"type":"module"}')
+      writeFileSync(join(dist, 'config.js'), source)
+      h.run(capability, ok, 'docker() { node "${@: -3}"; }')
+    })
+  )
+}

cloud/dev/scripts/push-gateway-workflow.test.mjs

@@ -0,0 +1,334 @@
+import assert from 'node:assert/strict'
+import { readFileSync } from 'node:fs'
+import test from 'node:test'
+import {
+  concurrencyBlocks,
+  jobIf,
+  jobs,
+  LEASE_ACTION,
+  leaseSteps
+} from './cloud-sql-rollout-lock-census.mjs'
+import { readRelayWorkflow, relayWorkflowFile } from './relay-repository.mjs'
+
+// Why: the push gateway holds the APNs key and is the only thing standing between a paired
+// phone and a silent notification pipeline. Its deploy is a blue/green rollout against the
+// dedicated Cloud SQL instance, and each of the guarantees below is one careless edit from gone.
+const WORKFLOW = 'push-deploy.yml'
+const workflow = readRelayWorkflow(WORKFLOW)
+const deploy = () => {
+  const job = jobs(workflow).find((entry) => entry.id === 'deploy')
+  assert.ok(job, 'the workflow no longer declares a deploy job')
+  return job
+}
+
+function terraform(file) {
+  return readFileSync(new URL(`../../infra/terraform/${file}`, import.meta.url), 'utf8')
+}
+
+// The ordered step names; every assertion below reads positions out of this list rather than
+// restating them, so a reordering that breaks the no-traffic guarantee fails here.
+const stepNames = () => [...workflow.matchAll(/^ {6}- name: (.+)$/gm)].map((match) => match[1])
+
+const indexOfStep = (name) => {
+  const index = stepNames().indexOf(name)
+  assert.notEqual(index, -1, `the workflow no longer has a "${name}" step`)
+  return index
+}
+
+test('the whole surface stays inert until the owner enables cloud operations', () => {
+  const guard = jobIf(deploy().text)
+  assert.ok(guard.includes("vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true'"), guard)
+  assert.ok(guard.includes("github.ref == 'refs/heads/main'"), guard)
+  assert.equal(jobs(workflow).length, 1, 'a second job would need its own gate')
+})
+
+test('it authenticates through Workload Identity and holds no repository secret', () => {
+  assert.match(workflow, /uses: google-github-actions\/auth@v2/)
+  assert.match(workflow, /workload_identity_provider: \$\{\{ vars\.PRODUCTION_GCP_PUSH_DEPLOY_WORKLOAD_IDENTITY_PROVIDER \}\}/)
+  assert.match(workflow, /service_account: \$\{\{ vars\.PRODUCTION_GCP_PUSH_DEPLOY_SERVICE_ACCOUNT \}\}/)
+  assert.match(workflow, /environment: production/)
+  for (const [, name] of workflow.matchAll(/secrets\.([A-Za-z_][A-Za-z0-9_]*)/g)) {
+    assert.equal(name, 'GITHUB_TOKEN', `the workflow reads secrets.${name}`)
+  }
+})
+
+// Why: Terraform trusts exact workflow filenames, not a prefix. A rename here without the
+// matching tfvars-independent list entry would fail authentication at dispatch time only.
+test('Terraform trusts this exact workflow file on the production deploy provider', () => {
+  assert.match(terraform('push-deploy-identity.tf'), /push-deploy\.yml@refs\/heads\/main/)
+  assert.doesNotMatch(terraform('relay-github-actions.tf'), /push-deploy\.yml/)
+  assert.equal(relayWorkflowFile(WORKFLOW), 'cloud-push-deploy.yml')
+})
+
+test('the rollout is serialized and leases its dedicated push rollout lock', () => {
+  const blocks = concurrencyBlocks(workflow)
+  assert.equal(blocks.length, 1)
+  assert.equal(blocks[0].group, 'production-push-rollout')
+  assert.equal(blocks[0].cancelInProgress, 'false')
+  const steps = leaseSteps(workflow)
+  assert.equal(steps.length, 1, 'exactly one lease step, held for the whole run')
+  assert.equal(steps[0].bucket, 'onorca-cloud-terraform-state')
+  assert.equal(steps[0].object, 'terraform/state/push-rollout/production.lock')
+  assert.equal(steps[0].release, undefined, 'release stays at its default for a single-job run')
+})
+
+// Why: the ops guardrail is that a piped command only fails the step when pipefail is set, and
+// pipefail only applies under an explicit bash shell. Every multi-line body here opts in.
+test('every multi-line command runs under bash with pipefail', () => {
+  const bodies = [...workflow.matchAll(/^ {8}(shell: bash\n {8})?run: \|\n((?: {10}.*\n|\n)+)/gm)]
+  assert.ok(bodies.length >= 8, `only ${bodies.length} multi-line commands were found`)
+  for (const match of bodies) {
+    assert.ok(match[1], `a multi-line command does not declare shell: bash:\n${match[2].slice(0, 120)}`)
+    assert.match(match[2], /^ {10}set -euo pipefail$/m)
+  }
+})
+
+test('the candidate revision takes no traffic and is addressed by its own tag', () => {
+  assert.match(workflow, /gcloud run deploy "\$\{SERVICE_NAME\}"/)
+  assert.match(workflow, /^ {12}--no-traffic \\$/m)
+  assert.match(workflow, /--tag "\$\{tag\}"/)
+  assert.match(workflow, /test "\$\{CANDIDATE_REVISION\}" != "\$\{ROLLBACK_REVISION\}"/)
+  assert.ok(
+    indexOfStep('Record the serving revision and require its Terraform-owned scaling') <
+      indexOfStep('Deploy the candidate revision with no traffic'),
+    'the rollback target must be captured before the candidate exists'
+  )
+})
+
+// Why: scaling is a Terraform-owned field that `lifecycle.ignore_changes` does not cover, so a
+// deploy that passed --max-instances would revert a later push_max_instances raise on every run.
+// The workflow asserts the shape instead of writing it, on the serving revision before the
+// candidate exists and on the candidate that inherits it.
+test('the deploy asserts the Terraform-owned scaling instead of mutating it', () => {
+  assert.doesNotMatch(workflow, /--max-instances/, 'the deploy must not write a scaling field')
+  assert.doesNotMatch(workflow, /--min-instances "/, 'the deploy must not write a scaling field')
+  // The floor is the variables.tf default; production.tfvars overrides only the ceiling, down to
+  // the two instances the Cloud SQL connection budget leaves room for.
+  assert.match(workflow, /PUSH_MIN_INSTANCES: 1$/m)
+  assert.match(workflow, /PUSH_MAX_INSTANCES: 2$/m)
+  assert.match(terraform('variables.tf'), /variable "push_min_instances"[\s\S]*?default {5}= 1/)
+  assert.match(terraform('environments/production.tfvars'), /^push_max_instances {9}= 2$/m)
+  const gate = indexOfStep('Record the serving revision and require its Terraform-owned scaling')
+  assert.ok(gate < indexOfStep('Deploy the candidate revision with no traffic'))
+  assert.match(workflow, /autoscaling\.knative\.dev\/minScale/)
+  assert.match(workflow, /\[\[ "\$\{floor:-0\}" -lt "\$\{PUSH_MIN_INSTANCES\}" \]\]/)
+  assert.match(workflow, /test "\$\{ceiling\}" = "\$\{PUSH_MAX_INSTANCES\}"/)
+  assert.match(workflow, /test "\$\{candidate_ceiling\}" = "\$\{PUSH_MAX_INSTANCES\}"/)
+})
+
+// Why: the image build is not a Cloud SQL operation, and the lease is a global serialization
+// point. A build inside it blocks every relay deploy and rehome for its duration.
+test('the image is built before the rollout lease is taken', () => {
+  const lease = workflow.indexOf(`- uses: ${LEASE_ACTION}`)
+  assert.notEqual(lease, -1)
+  const build = workflow.indexOf('- name: Build and publish the immutable gateway image')
+  const deployCandidate = workflow.indexOf('- name: Deploy the candidate revision with no traffic')
+  assert.ok(build < lease, 'the build must finish before the run takes the lease')
+  assert.ok(lease < deployCandidate, 'the lease must still cover the deploy, probe, and shift')
+})
+
+// Why: the gateway's Cloud SQL draw is instances x pool, and the root that takes the rollout
+// lease can only account for a pool it declares. Leaving it at the application default hid it.
+test('the database pool size is Terraform-owned and bounded at plan time', () => {
+  const source = terraform('push-gateway.tf')
+  assert.match(source, /name {2}= "ORCA_PUSH_DATABASE_POOL_MAX"/)
+  assert.match(source, /value = tostring\(var\.push_database_pool_max\)/)
+  assert.match(terraform('variables.tf'), /variable "push_database_pool_max"[\s\S]*?default {5}= 2/)
+  const block = /resource "google_cloud_run_v2_service" "push"[\s\S]*?\n  lifecycle \{([\s\S]*?)\n  \}/.exec(source)
+  assert.ok(block, 'the push service no longer declares a lifecycle block')
+  assert.match(
+    block[1],
+    /var\.push_max_instances \* var\.push_database_pool_max \* 3 <= 64/,
+    'instances x pool must be bounded at plan time'
+  )
+  assert.match(
+    readFileSync(new URL('../../apps/push/src/config.ts', import.meta.url), 'utf8'),
+    /ORCA_PUSH_DATABASE_POOL_MAX/,
+    'the gateway must read the variable Terraform sets'
+  )
+})
+
+test('the candidate is probed on its own URL before any traffic moves', () => {
+  const probe = indexOfStep('Probe the candidate readiness endpoint')
+  assert.ok(probe > indexOfStep('Deploy the candidate revision with no traffic'))
+  assert.ok(probe < indexOfStep('Shift all traffic to the verified candidate'))
+  assert.match(workflow, /"\$\{CANDIDATE_URL\}\/ready"/)
+  assert.match(workflow, /test "\$\{code\}" = 200/)
+  assert.ok(workflow.indexOf('${CANDIDATE_URL}/ready') < workflow.indexOf('${CANDIDATE_URL}/health'))
+  assert.match(workflow, /\.deliveryProtocol == 2/, 'verify the durable gateway after readiness')
+})
+
+// Why: a gateway that answers /ready can still hold no usable FCM credential. The probe must be
+// validate-only, must use a token that cannot exist, and must treat a denied credential as the
+// failure. Accepting PERMISSION_DENIED would make the whole step decorative.
+test('the FCM probe is validate-only and separates a bad token from a bad credential', () => {
+  const fcm = indexOfStep('Prove the runtime identity can reach FCM')
+  assert.ok(fcm > indexOfStep('Probe the candidate readiness endpoint'))
+  assert.ok(fcm < indexOfStep('Shift all traffic to the verified candidate'))
+  assert.match(workflow, /"validate_only":true/)
+  assert.match(workflow, /https:\/\/fcm\.googleapis\.com\/v1\/projects\/\$\{GCP_PROJECT_ID\}\/messages:send/)
+  assert.match(workflow, /GCP_PROJECT_ID: onorca-cloud$/m)
+  assert.match(workflow, /orca-push-deploy-probe-invalid-token/)
+  assert.match(workflow, /test "\$\{status\}" = INVALID_ARGUMENT/)
+  assert.match(workflow, /test "\$\{status\}" = PERMISSION_DENIED/)
+  // Only those four answers are conclusive; a 429 or a 5xx says nothing about the credential, so
+  // it is retried rather than read as either verdict. A denied credential still fails at once.
+  assert.match(workflow, /for attempt in \$\(seq 1 5\); do/)
+  const probe = workflow.slice(
+    workflow.indexOf('- name: Prove the runtime identity can reach FCM'),
+    workflow.indexOf('- name: Shift all traffic to the verified candidate')
+  )
+  assert.match(probe, /for attempt in \$\(seq 1 5\); do/)
+  assert.match(probe, /test "\$\{code\}" = 401 \|\| test "\$\{code\}" = 403; then\n {14}break/)
+  assert.match(
+    workflow,
+    /--impersonate-service-account "\$\{PUSH_RUNTIME_SERVICE_ACCOUNT\}"/,
+    'the probe must exercise the runtime credential, not the deploy identity'
+  )
+  // Why: that token reads the Apple signing key. Masking it means a later `set -x` or a
+  // debug re-run cannot print it into a public log.
+  assert.match(
+    probe,
+    /test -n "\$\{token\}"\n {10}echo "::add-mask::\$\{token\}"/,
+    'the impersonated token must be masked before anything else runs'
+  )
+  assert.match(workflow, /PUSH_RUNTIME_SERVICE_ACCOUNT: orca-cloud-push@onorca-cloud\.iam\.gserviceaccount\.com/)
+})
+
+// Why: a deploy ends with traffic pinned to an exact revision, and a rollback pins it to the
+// previous one. Terraform reverting the service to 100% LATEST would undo either silently.
+test('Terraform does not own the image or the traffic split', () => {
+  const source = terraform('push-gateway.tf')
+  const block = /resource "google_cloud_run_v2_service" "push"[\s\S]*?\n  lifecycle \{([\s\S]*?)\n  \}/.exec(source)
+  assert.ok(block, 'the push service no longer declares a lifecycle block')
+  assert.match(block[1], /template\[0\]\.containers\[0\]\.image/)
+  assert.match(block[1], /^\s*traffic$/m)
+})
+
+test('impersonating the runtime identity is a Terraform-declared grant', () => {
+  const source = terraform('push-gateway.tf')
+  assert.match(source, /resource "google_service_account_iam_member" "github_production_push_runtime_token_creator"/)
+  assert.match(source, /role\s+= "roles\/iam\.serviceAccountTokenCreator"/)
+  assert.match(source, /resource "google_cloud_run_v2_service_iam_member" "github_production_push_developer"/)
+})
+
+test('the traffic shift is all-or-nothing and is verified after the fact', () => {
+  const shift = indexOfStep('Shift all traffic to the verified candidate')
+  assert.match(workflow, /gcloud run services update-traffic "\$\{SERVICE_NAME\}"/)
+  assert.match(workflow, /--to-revisions "\$\{CANDIDATE_REVISION\}=100"/)
+  assert.match(workflow, /test "\$\{serving\}" = "\$\{CANDIDATE_REVISION\}"/)
+  assert.ok(shift < indexOfStep('Verify the public origin after the shift'))
+  assert.match(workflow, /PUSH_ORIGIN: https:\/\/push\.onorca\.dev/)
+  assert.match(workflow, /"\$\{PUSH_ORIGIN\}\/ready"/)
+})
+
+// Why: the origin can lag the traffic move by seconds, and a single unlucky curl would otherwise
+// roll a healthy deploy back. It retries on the same schedule as the candidate probe.
+test('the post-shift origin check retries like the candidate probe', () => {
+  const check = workflow.slice(
+    workflow.indexOf('- name: Verify the public origin after the shift'),
+    workflow.indexOf('- name: Roll traffic back to the previous revision')
+  )
+  assert.match(check, /for attempt in \$\(seq 1 30\); do/)
+  assert.match(check, /sleep 5/)
+  assert.match(check, /test "\$\{code\}" = 200/)
+})
+
+// Why: the summary carries the rollback target. Writing it after the origin check meant the one
+// run that needed it, the run whose check failed, was the one run that never got it.
+test('the summary is written before anything that can fail after the shift', () => {
+  const summary = indexOfStep('Publish the rollout summary')
+  assert.ok(summary > indexOfStep('Shift all traffic to the verified candidate'))
+  assert.ok(summary < indexOfStep('Verify the public origin after the shift'))
+  assert.match(workflow, /Known-good image:/)
+  assert.match(workflow, /GITHUB_STEP_SUMMARY/)
+})
+
+// Why: everything after the shift runs with production on the candidate, so a failure there is a
+// live gateway that has to go back. The marker is what separates that case from a failure before
+// the shift, where production never moved and the candidate is the thing to clean up.
+test('a failure after the shift rolls production back automatically', () => {
+  const rollback = indexOfStep('Roll traffic back to the previous revision')
+  assert.ok(rollback > indexOfStep('Verify the public origin after the shift'))
+  assert.match(workflow, /echo "TRAFFIC_SHIFTED=true" >> "\$\{GITHUB_ENV\}"/)
+  const shift = workflow.indexOf('- name: Shift all traffic to the verified candidate')
+  assert.ok(
+    workflow.indexOf('echo "TRAFFIC_SHIFTED=true"') > shift,
+    'the success marker follows the shift step'
+  )
+  const body = workflow.slice(
+    workflow.indexOf('- name: Roll traffic back to the previous revision'),
+    workflow.indexOf('- name: Delete the rejected candidate revision')
+  )
+  assert.match(
+    body,
+    /if: \$\{\{ \(failure\(\) \|\| cancelled\(\)\) && env\.TRAFFIC_SHIFT_ATTEMPTED == 'true' && env\.ROLLOUT_VERIFIED != 'true' \}\}/,
+    'the rollback must be conditioned on both failure and the shift marker'
+  )
+  assert.match(body, /test -n "\$\{ROLLBACK_REVISION:-\}"/)
+  assert.match(body, /--to-revisions "\$\{ROLLBACK_REVISION\}=100"/)
+  assert.match(body, /test "\$\{serving\}" = "\$\{ROLLBACK_REVISION\}"/)
+  assert.match(body, /GITHUB_STEP_SUMMARY/, 'the rollback must be reported in the summary')
+})
+
+// Why: a candidate that never took traffic still holds a warm instance and a Cloud SQL pool. Its
+// tag comes off first, because Cloud Run refuses to delete a revision a traffic target names.
+test('verified recovery authorizes rejected candidate deletion', () => {
+  const body = workflow.slice(
+    workflow.indexOf('- name: Delete the rejected candidate revision'),
+    workflow.indexOf('- name: Drop the candidate traffic tag')
+  )
+  assert.match(
+    body,
+    /env\.RECOVERY_VERIFIED == 'true'/,
+    'cleanup must wait for verified recovery traffic and public checks'
+  )
+  assert.match(body, /if test -z "\$\{CANDIDATE_REVISION:-\}"; then/)
+  assert.ok(
+    body.indexOf('--remove-tags') < body.indexOf('gcloud run revisions delete'),
+    'the tag must come off before the revision is deleted'
+  )
+  assert.match(body, /echo "CANDIDATE_TAG=" >> "\$\{GITHUB_ENV\}"/)
+})
+
+test('the run always drops its traffic tag', () => {
+  const cleanup = indexOfStep('Drop the candidate traffic tag')
+  assert.equal(cleanup, stepNames().length - 1, 'tag cleanup must be the last step')
+  assert.match(workflow, /--remove-tags "\$\{CANDIDATE_TAG\}"/)
+  const body = workflow.slice(workflow.indexOf('- name: Drop the candidate traffic tag'))
+  assert.match(body, /if: always\(\)/)
+  assert.match(body, /test -n "\$\{CANDIDATE_TAG:-\}" \|\| exit 0/)
+})
+
+test('push credentials cannot assume the shared Relay deploy identity', () => {
+  const source = terraform('push-deploy-identity.tf')
+  assert.match(source, /"attribute.push_deploy"\s*=\s*"'production'"/)
+  assert.doesNotMatch(source, /"attribute.repository"\s*=/)
+  assert.match(source, /attribute\.push_deploy\/production/)
+  assert.doesNotMatch(workflow, /PRODUCTION_GCP_RELAY_DEPLOY_/)
+  assert.doesNotMatch(terraform('push-gateway.tf'), /member\s*=\s*local\.relay_github_deploy_service_account_member/)
+})
+
+// A latest revision needs a successor even when validation is inert.
+test('dedicated database admits three simultaneous revision pools', () => {
+  assert.match(terraform('push-gateway.tf'), /var\.push_max_instances \* var\.push_database_pool_max \* 3 <= 64/)
+})
+
+test('push has only a dedicated database attachment and a narrowly scoped deployment lease', () => {
+  const service = terraform('push-gateway.tf')
+  const database = terraform('push-dedicated-database.tf')
+  assert.match(service, /instances = \[google_sql_database_instance\.push_dedicated\[0\]\.connection_name\]/)
+  assert.match(service, /secret\s*= google_secret_manager_secret\.push_dedicated_database_url\[0\]\.secret_id/)
+  assert.match(service, /version = google_secret_manager_secret_version\.push_dedicated_database_url\[0\]\.version/)
+  assert.doesNotMatch(service + database, /push_dedicated_database_(?:active|enabled)|local\.relay_database_connection_name|resource "google_sql_database" "push"/)
+  assert.match(database, /tier\s*= "db-custom-2-7680"/)
+  assert.match(database, /availability_type = "REGIONAL"/)
+  assert.match(database, /deletion_protection\s*= true/)
+  assert.match(database, /deletion_protection_enabled = true/)
+  const identity = terraform('push-deploy-identity.tf')
+  const lease = identity.match(/resource "google_storage_bucket_iam_member" "github_push_rollout_lease" \{([\s\S]*?)\n\}/)?.[1]
+  assert.ok(lease)
+  assert.match(lease, /member = local\.push_deploy_member/)
+  assert.match(lease, /role\s*= "roles\/storage.objectAdmin"/)
+  assert.match(lease, /resource.name == 'projects\/_\/buckets\/\$\{var.project_id\}-terraform-state\/objects\/terraform\/state\/push-rollout\/production.lock'/)
+})

cloud/dev/scripts/push-validation-workflow.test.mjs

@@ -0,0 +1,48 @@
+import assert from 'node:assert/strict'
+import { readFileSync } from 'node:fs'
+import test from 'node:test'
+import { readRelayWorkflow } from './relay-repository.mjs'
+
+const workflow = readRelayWorkflow('push-deploy.yml')
+const position = (name) => {
+  const index = workflow.indexOf(`- name: ${name}`)
+  assert.notEqual(index, -1)
+  return index
+}
+const capability = position('Require image support for inert validation')
+const deploy = position('Deploy the candidate revision with no traffic')
+const activation = position('Retire inert validation and activate the verified image')
+const shift = position('Shift all traffic to the verified candidate')
+
+test('the exact build digest must support validation before production boot', () => {
+  assert.match(workflow, /docker buildx build --push --platform linux\/amd64 --provenance=false --metadata-file/)
+  assert.match(workflow, /containerimage\.digest/)
+  assert.doesNotMatch(workflow, /gcloud artifacts docker images describe/)
+  assert.ok(capability < deploy)
+  const preflight = workflow.slice(capability, deploy)
+  assert.match(preflight, /docker run --rm --network none --entrypoint node "\$\{IMAGE\}"/)
+  assert.match(preflight, /loadPushConfig\(env\)\.mode !== "validation"/)
+  assert.match(preflight, /validation_mode_not_fail_closed/)
+})
+
+test('inert validation and credential checks precede deliberate activation of the same digest', () => {
+  assert.match(workflow.slice(deploy, activation), /--update-env-vars ORCA_PUSH_MODE=validation/)
+  assert.match(workflow.slice(deploy, activation), /\.mode == "validation"/)
+  assert.ok(position('Prove the runtime identity can reach FCM') < activation)
+  const active = workflow.slice(activation, shift)
+  assert.ok(active.indexOf('gcloud run deploy') < active.indexOf('gcloud run revisions delete'))
+  assert.match(active, /--image "\$\{IMAGE\}"/)
+  assert.match(active, /--remove-env-vars ORCA_PUSH_MODE/)
+  assert.match(active, /\.spec\.containers\[0\]\.image == \$image/)
+  assert.match(active, /\.spec\.serviceAccountName == \$account/)
+  assert.match(active, /\.mode == "active"/)
+  assert.ok(active.indexOf('ACTIVATION_ATTEMPTED=true') < active.indexOf('gcloud run deploy'))
+  assert.match(workflow, /deletion below must stop its workers/)
+})
+
+test('production startup connects read-only and gates all background work in validation', () => {
+  const entry = readFileSync(new URL('../../apps/push/src/index.ts', import.meta.url), 'utf8')
+  assert.match(entry, /readOnly: config\.mode === 'validation'/)
+  assert.match(entry, /startPushBackground\(config,/)
+  assert.doesNotMatch(entry, /worker\.start\(/)
+})

cloud/dev/scripts/relay-cloud-sql-connection-budget.test.mjs

@@ -6,7 +6,7 @@ import {
   readRelayCloudSqlConnectionBudget
 } from './relay-cloud-sql-connection-budget.mjs'
 
-test('production plus three Asia pools preserves allowance and reserve below the ceiling', () => {
+test('production shared consumers keep allowance and reserve below the ceiling', () => {
   const report = readRelayCloudSqlConnectionBudget()
 
   assert.deepEqual(report.consumers, { cells: 230, directors: 15, auth: 20, api: 50 })
@@ -63,7 +63,11 @@ test('excludes fenced cell pools and reads per-cell pool overrides', () => {
           }
         }
       `,
-      terraformVariables: 'variable "relay_director_database_pool_max" { default = 3 }',
+      terraformVariables: [
+        'variable "relay_director_database_pool_max" { default = 3 }',
+        'variable "push_max_instances" { default = 1 }',
+        'variable "push_database_pool_max" { default = 2 }'
+      ].join('\n'),
       relayConfig: 'export const RELAY_DATABASE_POOL_MAX = 10'
     },
     maxConnections: 100,
@@ -76,6 +80,37 @@ test('excludes fenced cell pools and reads per-cell pool overrides', () => {
   assert.equal(report.budgetedTotal, 47)
 })
 
+test('dedicated push scaling does not consume shared capacity', () => {
+  const report = readRelayCloudSqlConnectionBudget({
+    proposedAsiaCellCount: 1,
+    appConsumers: { authInstances: 1, authPoolMax: 10, apiInstances: 1, apiPoolMax: 5, maxConnections: 100 },
+    sources: {
+      productionTfvars: `
+        relay_max_instances = 1
+        push_max_instances  = 3
+        relay_gce_fenced_cells = []
+        relay_gce_cells = {
+          "production-gce-c2" = { database_pool_max = 4
+          }
+        }
+      `,
+      terraformVariables: [
+        'variable "relay_director_database_pool_max" { default = 3 }',
+        'variable "push_max_instances" { default = 1 }',
+        'variable "push_database_pool_max" { default = 2 }'
+      ].join('\n'),
+      relayConfig: 'export const RELAY_DATABASE_POOL_MAX = 10'
+    },
+    maxConnections: 100,
+    maintenanceAdminAllowance: 1,
+    explicitReserve: 1
+  })
+
+  assert.equal(report.consumers.push, undefined)
+  assert.equal(report.rolloutOverlap.pushCandidate, undefined)
+  assert.equal(report.operatingMaximum, 46)
+})
+
 test('requires strict headroom below the physical ceiling', () => {
   const report = calculateRelayCloudSqlConnectionBudget({
     cellPoolTotal: 20,

cloud/dev/scripts/render-workload-identity-conditions.mjs

@@ -18,6 +18,7 @@ const TERRAFORM_ROOTS = {
       'infra/terraform/relay-shared.tf',
       'infra/terraform/relay-github-workflow-trust.tf',
       'infra/terraform/relay-github-actions.tf',
+      'infra/terraform/push-deploy-identity.tf',
       'infra/terraform/relay-staging-deploy-iam.tf',
       'infra/terraform/relay-asia-topology-iam.tf',
       'infra/terraform/relay-asia-proof-iam.tf'
@@ -475,7 +476,7 @@ function collectTfvars(source, variables) {
       offset += line.length + 1
       continue
     }
-    const structured = /^([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(?=[[{])/.exec(line)
+    const structured = /^([A-Za-z_][A-Za-z0-9_]*)\s*=\s*/.exec(line)
     if (structured) {
       try {
         variables[structured[1]] = parseValueAt(source, offset + structured[0].length)

cloud/dev/scripts/workload-identity-attribute-conditions.test.mjs

@@ -30,6 +30,9 @@ const EXPECTED_CONDITIONS = {
   },
   production: {
     relay: {
+      github_push:
+        "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && assertion.event_name == 'workflow_dispatch' && assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-push-deploy.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-push-deploy.yml@refs/heads/main'",
+
       github:
         "assertion.repository == 'stablyai/orca' && assertion.repository_id == '1183888342' && assertion.repository_owner_id == '127256420' && assertion.ref == 'refs/heads/main' && assertion.environment == 'production' && ((assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-fence-broker.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-capacity.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-director.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-multi-target.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-asia-admission.yml@refs/heads/main' || assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-publish-relay-production.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-rehome.yml@refs/heads/main' && assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-operate-relay-production-rehome-job.yml@refs/heads/main') || (assertion.workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main' && (assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml@refs/heads/main' || assertion.job_workflow_ref == 'stablyai/orca/.github/workflows/cloud-deploy-relay-production-same-cap.yml@refs/heads/main')))",
       github_monitor: