orca/cloud/dev/scripts/push-gateway-workflow.test.mjs

import assert from 'node:assert/strict'
import { readFileSync } from 'node:fs'
import test from 'node:test'
import {
  concurrencyBlocks,
  jobIf,
  jobs,
  LEASE_ACTION,
  leaseSteps
} from './cloud-sql-rollout-lock-census.mjs'
import { readRelayWorkflow, relayWorkflowFile } from './relay-repository.mjs'

// Why: the push gateway holds the APNs key and is the only thing standing between a paired
// phone and a silent notification pipeline. Its deploy is a blue/green rollout against the
// dedicated Cloud SQL instance, and each of the guarantees below is one careless edit from gone.
const WORKFLOW = 'push-deploy.yml'
const workflow = readRelayWorkflow(WORKFLOW)
const deploy = () => {
  const job = jobs(workflow).find((entry) => entry.id === 'deploy')
  assert.ok(job, 'the workflow no longer declares a deploy job')
  return job
}

function terraform(file) {
  return readFileSync(new URL(`../../infra/terraform/${file}`, import.meta.url), 'utf8')
}

// The ordered step names; every assertion below reads positions out of this list rather than
// restating them, so a reordering that breaks the no-traffic guarantee fails here.
const stepNames = () => [...workflow.matchAll(/^ {6}- name: (.+)$/gm)].map((match) => match[1])

const indexOfStep = (name) => {
  const index = stepNames().indexOf(name)
  assert.notEqual(index, -1, `the workflow no longer has a "${name}" step`)
  return index
}

test('the whole surface stays inert until the owner enables cloud operations', () => {
  const guard = jobIf(deploy().text)
  assert.ok(guard.includes("vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true'"), guard)
  assert.ok(guard.includes("github.ref == 'refs/heads/main'"), guard)
  assert.equal(jobs(workflow).length, 1, 'a second job would need its own gate')
})

test('it authenticates through Workload Identity and holds no repository secret', () => {
  assert.match(workflow, /uses: google-github-actions\/auth@v2/)
  assert.match(workflow, /workload_identity_provider: \$\{\{ vars\.PRODUCTION_GCP_PUSH_DEPLOY_WORKLOAD_IDENTITY_PROVIDER \}\}/)
  assert.match(workflow, /service_account: \$\{\{ vars\.PRODUCTION_GCP_PUSH_DEPLOY_SERVICE_ACCOUNT \}\}/)
  assert.match(workflow, /environment: production/)
  for (const [, name] of workflow.matchAll(/secrets\.([A-Za-z_][A-Za-z0-9_]*)/g)) {
    assert.equal(name, 'GITHUB_TOKEN', `the workflow reads secrets.${name}`)
  }
})

// Why: Terraform trusts exact workflow filenames, not a prefix. A rename here without the
// matching tfvars-independent list entry would fail authentication at dispatch time only.
test('Terraform trusts this exact workflow file on the production deploy provider', () => {
  assert.match(terraform('push-deploy-identity.tf'), /push-deploy\.yml@refs\/heads\/main/)
  assert.doesNotMatch(terraform('relay-github-actions.tf'), /push-deploy\.yml/)
  assert.equal(relayWorkflowFile(WORKFLOW), 'cloud-push-deploy.yml')
})

test('the rollout is serialized and leases its dedicated push rollout lock', () => {
  const blocks = concurrencyBlocks(workflow)
  assert.equal(blocks.length, 1)
  assert.equal(blocks[0].group, 'production-push-rollout')
  assert.equal(blocks[0].cancelInProgress, 'false')
  const steps = leaseSteps(workflow)
  assert.equal(steps.length, 1, 'exactly one lease step, held for the whole run')
  assert.equal(steps[0].bucket, 'onorca-cloud-terraform-state')
  assert.equal(steps[0].object, 'terraform/state/push-rollout/production.lock')
  assert.equal(steps[0].release, undefined, 'release stays at its default for a single-job run')
})

// Why: the ops guardrail is that a piped command only fails the step when pipefail is set, and
// pipefail only applies under an explicit bash shell. Every multi-line body here opts in.
test('every multi-line command runs under bash with pipefail', () => {
  const bodies = [...workflow.matchAll(/^ {8}(shell: bash\n {8})?run: \|\n((?: {10}.*\n|\n)+)/gm)]
  assert.ok(bodies.length >= 8, `only ${bodies.length} multi-line commands were found`)
  for (const match of bodies) {
    assert.ok(match[1], `a multi-line command does not declare shell: bash:\n${match[2].slice(0, 120)}`)
    assert.match(match[2], /^ {10}set -euo pipefail$/m)
  }
})

test('the candidate revision takes no traffic and is addressed by its own tag', () => {
  assert.match(workflow, /gcloud run deploy "\$\{SERVICE_NAME\}"/)
  assert.match(workflow, /^ {12}--no-traffic \\$/m)
  assert.match(workflow, /--tag "\$\{tag\}"/)
  assert.match(workflow, /test "\$\{CANDIDATE_REVISION\}" != "\$\{ROLLBACK_REVISION\}"/)
  assert.ok(
    indexOfStep('Record the serving revision and require its Terraform-owned scaling') <
      indexOfStep('Deploy the candidate revision with no traffic'),
    'the rollback target must be captured before the candidate exists'
  )
})

// Why: scaling is a Terraform-owned field that `lifecycle.ignore_changes` does not cover, so a
// deploy that passed --max-instances would revert a later push_max_instances raise on every run.
// The workflow asserts the shape instead of writing it, on the serving revision before the
// candidate exists and on the candidate that inherits it.
test('the deploy asserts the Terraform-owned scaling instead of mutating it', () => {
  assert.doesNotMatch(workflow, /--max-instances/, 'the deploy must not write a scaling field')
  assert.doesNotMatch(workflow, /--min-instances "/, 'the deploy must not write a scaling field')
  // The floor is the variables.tf default; production.tfvars overrides only the ceiling, down to
  // the two instances the Cloud SQL connection budget leaves room for.
  assert.match(workflow, /PUSH_MIN_INSTANCES: 1$/m)
  assert.match(workflow, /PUSH_MAX_INSTANCES: 2$/m)
  assert.match(terraform('variables.tf'), /variable "push_min_instances"[\s\S]*?default {5}= 1/)
  assert.match(terraform('environments/production.tfvars'), /^push_max_instances {9}= 2$/m)
  const gate = indexOfStep('Record the serving revision and require its Terraform-owned scaling')
  assert.ok(gate < indexOfStep('Deploy the candidate revision with no traffic'))
  assert.match(workflow, /autoscaling\.knative\.dev\/minScale/)
  assert.match(workflow, /\[\[ "\$\{floor:-0\}" -lt "\$\{PUSH_MIN_INSTANCES\}" \]\]/)
  assert.match(workflow, /test "\$\{ceiling\}" = "\$\{PUSH_MAX_INSTANCES\}"/)
  assert.match(workflow, /test "\$\{candidate_ceiling\}" = "\$\{PUSH_MAX_INSTANCES\}"/)
})

// Why: the image build is not a Cloud SQL operation, and the lease is a global serialization
// point. A build inside it blocks every relay deploy and rehome for its duration.
test('the image is built before the rollout lease is taken', () => {
  const lease = workflow.indexOf(`- uses: ${LEASE_ACTION}`)
  assert.notEqual(lease, -1)
  const build = workflow.indexOf('- name: Build and publish the immutable gateway image')
  const deployCandidate = workflow.indexOf('- name: Deploy the candidate revision with no traffic')
  assert.ok(build < lease, 'the build must finish before the run takes the lease')
  assert.ok(lease < deployCandidate, 'the lease must still cover the deploy, probe, and shift')
})

// Why: the gateway's Cloud SQL draw is instances x pool, and the root that takes the rollout
// lease can only account for a pool it declares. Leaving it at the application default hid it.
test('the database pool size is Terraform-owned and bounded at plan time', () => {
  const source = terraform('push-gateway.tf')
  assert.match(source, /name {2}= "ORCA_PUSH_DATABASE_POOL_MAX"/)
  assert.match(source, /value = tostring\(var\.push_database_pool_max\)/)
  assert.match(terraform('variables.tf'), /variable "push_database_pool_max"[\s\S]*?default {5}= 2/)
  const block = /resource "google_cloud_run_v2_service" "push"[\s\S]*?\n  lifecycle \{([\s\S]*?)\n  \}/.exec(source)
  assert.ok(block, 'the push service no longer declares a lifecycle block')
  assert.match(
    block[1],
    /var\.push_max_instances \* var\.push_database_pool_max \* 3 <= 64/,
    'instances x pool must be bounded at plan time'
  )
  assert.match(
    readFileSync(new URL('../../apps/push/src/config.ts', import.meta.url), 'utf8'),
    /ORCA_PUSH_DATABASE_POOL_MAX/,
    'the gateway must read the variable Terraform sets'
  )
})

test('the candidate is probed on its own URL before any traffic moves', () => {
  const probe = indexOfStep('Probe the candidate readiness endpoint')
  assert.ok(probe > indexOfStep('Deploy the candidate revision with no traffic'))
  assert.ok(probe < indexOfStep('Shift all traffic to the verified candidate'))
  assert.match(workflow, /"\$\{CANDIDATE_URL\}\/ready"/)
  assert.match(workflow, /test "\$\{code\}" = 200/)
  assert.ok(workflow.indexOf('${CANDIDATE_URL}/ready') < workflow.indexOf('${CANDIDATE_URL}/health'))
  assert.match(workflow, /\.deliveryProtocol == 2/, 'verify the durable gateway after readiness')
})

// Why: a gateway that answers /ready can still hold no usable FCM credential. The probe must be
// validate-only, must use a token that cannot exist, and must treat a denied credential as the
// failure. Accepting PERMISSION_DENIED would make the whole step decorative.
test('the FCM probe is validate-only and separates a bad token from a bad credential', () => {
  const fcm = indexOfStep('Prove the runtime identity can reach FCM')
  assert.ok(fcm > indexOfStep('Probe the candidate readiness endpoint'))
  assert.ok(fcm < indexOfStep('Shift all traffic to the verified candidate'))
  assert.match(workflow, /"validate_only":true/)
  assert.match(workflow, /https:\/\/fcm\.googleapis\.com\/v1\/projects\/\$\{GCP_PROJECT_ID\}\/messages:send/)
  assert.match(workflow, /GCP_PROJECT_ID: onorca-cloud$/m)
  assert.match(workflow, /orca-push-deploy-probe-invalid-token/)
  assert.match(workflow, /test "\$\{status\}" = INVALID_ARGUMENT/)
  assert.match(workflow, /test "\$\{status\}" = PERMISSION_DENIED/)
  // Only those four answers are conclusive; a 429 or a 5xx says nothing about the credential, so
  // it is retried rather than read as either verdict. A denied credential still fails at once.
  assert.match(workflow, /for attempt in \$\(seq 1 5\); do/)
  const probe = workflow.slice(
    workflow.indexOf('- name: Prove the runtime identity can reach FCM'),
    workflow.indexOf('- name: Shift all traffic to the verified candidate')
  )
  assert.match(probe, /for attempt in \$\(seq 1 5\); do/)
  assert.match(probe, /test "\$\{code\}" = 401 \|\| test "\$\{code\}" = 403; then\n {14}break/)
  assert.match(
    workflow,
    /--impersonate-service-account "\$\{PUSH_RUNTIME_SERVICE_ACCOUNT\}"/,
    'the probe must exercise the runtime credential, not the deploy identity'
  )
  // Why: that token reads the Apple signing key. Masking it means a later `set -x` or a
  // debug re-run cannot print it into a public log.
  assert.match(
    probe,
    /test -n "\$\{token\}"\n {10}echo "::add-mask::\$\{token\}"/,
    'the impersonated token must be masked before anything else runs'
  )
  assert.match(workflow, /PUSH_RUNTIME_SERVICE_ACCOUNT: orca-cloud-push@onorca-cloud\.iam\.gserviceaccount\.com/)
})

// Why: a deploy ends with traffic pinned to an exact revision, and a rollback pins it to the
// previous one. Terraform reverting the service to 100% LATEST would undo either silently.
test('Terraform does not own the image or the traffic split', () => {
  const source = terraform('push-gateway.tf')
  const block = /resource "google_cloud_run_v2_service" "push"[\s\S]*?\n  lifecycle \{([\s\S]*?)\n  \}/.exec(source)
  assert.ok(block, 'the push service no longer declares a lifecycle block')
  assert.match(block[1], /template\[0\]\.containers\[0\]\.image/)
  assert.match(block[1], /^\s*traffic$/m)
})

test('impersonating the runtime identity is a Terraform-declared grant', () => {
  const source = terraform('push-gateway.tf')
  assert.match(source, /resource "google_service_account_iam_member" "github_production_push_runtime_token_creator"/)
  assert.match(source, /role\s+= "roles\/iam\.serviceAccountTokenCreator"/)
  assert.match(source, /resource "google_cloud_run_v2_service_iam_member" "github_production_push_developer"/)
})

test('the traffic shift is all-or-nothing and is verified after the fact', () => {
  const shift = indexOfStep('Shift all traffic to the verified candidate')
  assert.match(workflow, /gcloud run services update-traffic "\$\{SERVICE_NAME\}"/)
  assert.match(workflow, /--to-revisions "\$\{CANDIDATE_REVISION\}=100"/)
  assert.match(workflow, /test "\$\{serving\}" = "\$\{CANDIDATE_REVISION\}"/)
  assert.ok(shift < indexOfStep('Verify the public origin after the shift'))
  assert.match(workflow, /PUSH_ORIGIN: https:\/\/push\.onorca\.dev/)
  assert.match(workflow, /"\$\{PUSH_ORIGIN\}\/ready"/)
})

// Why: the origin can lag the traffic move by seconds, and a single unlucky curl would otherwise
// roll a healthy deploy back. It retries on the same schedule as the candidate probe.
test('the post-shift origin check retries like the candidate probe', () => {
  const check = workflow.slice(
    workflow.indexOf('- name: Verify the public origin after the shift'),
    workflow.indexOf('- name: Roll traffic back to the previous revision')
  )
  assert.match(check, /for attempt in \$\(seq 1 30\); do/)
  assert.match(check, /sleep 5/)
  assert.match(check, /test "\$\{code\}" = 200/)
})

// Why: the summary carries the rollback target. Writing it after the origin check meant the one
// run that needed it, the run whose check failed, was the one run that never got it.
test('the summary is written before anything that can fail after the shift', () => {
  const summary = indexOfStep('Publish the rollout summary')
  assert.ok(summary > indexOfStep('Shift all traffic to the verified candidate'))
  assert.ok(summary < indexOfStep('Verify the public origin after the shift'))
  assert.match(workflow, /Known-good image:/)
  assert.match(workflow, /GITHUB_STEP_SUMMARY/)
})

// Why: everything after the shift runs with production on the candidate, so a failure there is a
// live gateway that has to go back. The marker is what separates that case from a failure before
// the shift, where production never moved and the candidate is the thing to clean up.
test('a failure after the shift rolls production back automatically', () => {
  const rollback = indexOfStep('Roll traffic back to the previous revision')
  assert.ok(rollback > indexOfStep('Verify the public origin after the shift'))
  assert.match(workflow, /echo "TRAFFIC_SHIFTED=true" >> "\$\{GITHUB_ENV\}"/)
  const shift = workflow.indexOf('- name: Shift all traffic to the verified candidate')
  assert.ok(
    workflow.indexOf('echo "TRAFFIC_SHIFTED=true"') > shift,
    'the success marker follows the shift step'
  )
  const body = workflow.slice(
    workflow.indexOf('- name: Roll traffic back to the previous revision'),
    workflow.indexOf('- name: Delete the rejected candidate revision')
  )
  assert.match(
    body,
    /if: \$\{\{ \(failure\(\) \|\| cancelled\(\)\) && env\.TRAFFIC_SHIFT_ATTEMPTED == 'true' && env\.ROLLOUT_VERIFIED != 'true' \}\}/,
    'the rollback must be conditioned on both failure and the shift marker'
  )
  assert.match(body, /test -n "\$\{ROLLBACK_REVISION:-\}"/)
  assert.match(body, /--to-revisions "\$\{ROLLBACK_REVISION\}=100"/)
  assert.match(body, /test "\$\{serving\}" = "\$\{ROLLBACK_REVISION\}"/)
  assert.match(body, /GITHUB_STEP_SUMMARY/, 'the rollback must be reported in the summary')
})

// Why: a candidate that never took traffic still holds a warm instance and a Cloud SQL pool. Its
// tag comes off first, because Cloud Run refuses to delete a revision a traffic target names.
test('verified recovery authorizes rejected candidate deletion', () => {
  const body = workflow.slice(
    workflow.indexOf('- name: Delete the rejected candidate revision'),
    workflow.indexOf('- name: Drop the candidate traffic tag')
  )
  assert.match(
    body,
    /env\.RECOVERY_VERIFIED == 'true'/,
    'cleanup must wait for verified recovery traffic and public checks'
  )
  assert.match(body, /if test -z "\$\{CANDIDATE_REVISION:-\}"; then/)
  assert.ok(
    body.indexOf('--remove-tags') < body.indexOf('gcloud run revisions delete'),
    'the tag must come off before the revision is deleted'
  )
  assert.match(body, /echo "CANDIDATE_TAG=" >> "\$\{GITHUB_ENV\}"/)
})

test('the run always drops its traffic tag', () => {
  const cleanup = indexOfStep('Drop the candidate traffic tag')
  assert.equal(cleanup, stepNames().length - 1, 'tag cleanup must be the last step')
  assert.match(workflow, /--remove-tags "\$\{CANDIDATE_TAG\}"/)
  const body = workflow.slice(workflow.indexOf('- name: Drop the candidate traffic tag'))
  assert.match(body, /if: always\(\)/)
  assert.match(body, /test -n "\$\{CANDIDATE_TAG:-\}" \|\| exit 0/)
})

test('push credentials cannot assume the shared Relay deploy identity', () => {
  const source = terraform('push-deploy-identity.tf')
  assert.match(source, /"attribute.push_deploy"\s*=\s*"'production'"/)
  assert.doesNotMatch(source, /"attribute.repository"\s*=/)
  assert.match(source, /attribute\.push_deploy\/production/)
  assert.doesNotMatch(workflow, /PRODUCTION_GCP_RELAY_DEPLOY_/)
  assert.doesNotMatch(terraform('push-gateway.tf'), /member\s*=\s*local\.relay_github_deploy_service_account_member/)
})

// A latest revision needs a successor even when validation is inert.
test('dedicated database admits three simultaneous revision pools', () => {
  assert.match(terraform('push-gateway.tf'), /var\.push_max_instances \* var\.push_database_pool_max \* 3 <= 64/)
})

test('push has only a dedicated database attachment and a narrowly scoped deployment lease', () => {
  const service = terraform('push-gateway.tf')
  const database = terraform('push-dedicated-database.tf')
  assert.match(service, /instances = \[google_sql_database_instance\.push_dedicated\[0\]\.connection_name\]/)
  assert.match(service, /secret\s*= google_secret_manager_secret\.push_dedicated_database_url\[0\]\.secret_id/)
  assert.match(service, /version = google_secret_manager_secret_version\.push_dedicated_database_url\[0\]\.version/)
  assert.doesNotMatch(service + database, /push_dedicated_database_(?:active|enabled)|local\.relay_database_connection_name|resource "google_sql_database" "push"/)
  assert.match(database, /tier\s*= "db-custom-2-7680"/)
  assert.match(database, /availability_type = "REGIONAL"/)
  assert.match(database, /deletion_protection\s*= true/)
  assert.match(database, /deletion_protection_enabled = true/)
  const identity = terraform('push-deploy-identity.tf')
  const lease = identity.match(/resource "google_storage_bucket_iam_member" "github_push_rollout_lease" \{([\s\S]*?)\n\}/)?.[1]
  assert.ok(lease)
  assert.match(lease, /member = local\.push_deploy_member/)
  assert.match(lease, /role\s*= "roles\/storage.objectAdmin"/)
  assert.match(lease, /resource.name == 'projects\/_\/buckets\/\$\{var.project_id\}-terraform-state\/objects\/terraform\/state\/push-rollout\/production.lock'/)
})