orca/cloud/dev/scripts/relay-regional-rehome-workflow.test.mjs

import assert from 'node:assert/strict'
import { readFileSync } from 'node:fs'
import { test } from 'node:test'
import { fileURLToPath } from 'node:url'
import { relayWorkflowUrl } from './relay-repository.mjs'

function workflow(name) {
  return readFileSync(
    fileURLToPath(relayWorkflowUrl(name)),
    'utf8'
  )
}

test('same-cap wrapper is reusable, canary-bound, and sequential', () => {
  const wrapper = workflow('deploy-relay-production-same-cap.yml')
  const job = workflow('deploy-relay-production-same-cap-job.yml')
  assert.match(wrapper, /options: \[verify, canary-apply, batch-apply, rollback\]/)
  assert.match(wrapper, /relay-same-cap-canary-\$\{\{ inputs\.canary-run-id \}\}/)
  assert.match(wrapper, /needs: \[gate, cell_1\]/)
  assert.match(wrapper, /needs: \[gate, cell_2\]/)
  assert.match(wrapper, /needs: \[gate, cell_3\]/)
  assert.match(job, /on:\n  workflow_call:/)
  assert.match(job, /c27\|c28\|c29/)
  assert.match(job, /EXPECTED_HARD_CAP=3000/)
  assert.match(job, /EXPECTED_REGION=asia-east2/)
  assert.match(job, /--hard-cap "\$\{EXPECTED_HARD_CAP\}"/)
  assert.match(job, /--regional-rehome-protocol "\$\{DESIRED_REHOME_PROTOCOL\}"/)
  assert.match(job, /--argjson protocol "\$\{PREDECESSOR_REHOME_PROTOCOL\}"/)
  assert.match(job, /runtime predecessor mismatch fields=/)
  // A rollback interrupted between apply and restore must be resumable.
  assert.match(job, /ROLLBACK_RESUME=true/)
  assert.match(job, /test "\$\{LIVE_IMAGE_DIGEST\}" = "\$\{DESIRED_IMAGE_DIGEST\}"/)
  // Resume must skip BOTH the drain (no restart will clear the flag) and the
  // apply (state already converged), and prove convergence instead.
  assert.match(
    job,
    /Reversibly isolate and drain only the selected cell\n        if: \$\{\{ inputs\.mode != 'verify' && env\.ROLLBACK_RESUME != 'true' \}\}/
  )
  assert.match(
    job,
    /Apply only the selected same-cap template and MIG\n        if: \$\{\{ inputs\.mode != 'verify' && env\.ROLLBACK_RESUME != 'true' \}\}/
  )
  assert.match(
    job,
    /Require converged Terraform state and a stable MIG on resume\n        if: \$\{\{ inputs\.mode != 'verify' && env\.ROLLBACK_RESUME == 'true' \}\}/
  )
  assert.match(job, /resume found unconverged resources/)
  // A canary or batch cell that failed before its template apply also
  // resumes here with template drift from repo changes since its last roll;
  // only a plan the reviewed validator approves for the image the cell
  // already serves may pass, and resume still applies nothing.
  assert.match(job, /requiring reviewed rollback-image drift/)
  assert.match(
    job,
    /--image "\$\{DESIRED_IMAGE\}" \\\n {16}--rollback-image "\$\{DESIRED_IMAGE\}"/
  )
  // The relaxation is only safe if the reviewed validator actually runs on
  // the NON-converged branch, in same-cap-cell mode, with the trust config
  // the validator requires, restricted to the template-and-MIG change pair or,
  // when only the reviewed backend attributes are left, to those alone — and
  // that last case then has to be applied, not waved through as converged.
  assert.match(
    job,
    /if ! terraform -chdir=infra\/terraform show -json[\s\S]{0,220}\| length == 0' >\/dev\/null\n          then\n/
  )
  assert.match(
    job,
    /requiring reviewed rollback-image drift'\n[\s\S]{0,400}?\n {16}--mode same-cap-cell --cell-id "\$\{TARGET_CELL_ID\}" \\\n/
  )
  assert.match(
    job,
    /Require converged Terraform state and a stable MIG on resume[\s\S]{0,300}CAPACITY_SERVICE_ACCOUNT: \$\{\{ vars\.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT \}\}\n {10}DIRECTOR_RUNTIME_SERVICE_ACCOUNT: \$\{\{ vars\.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT \}\}/
  )
  assert.match(
    job,
    /--rollback-image "\$\{DESIRED_IMAGE\}" \\\n {16}--capacity-service-account "\$\{CAPACITY_SERVICE_ACCOUNT\}" \\\n {16}--rehome-director-service-account "\$\{DIRECTOR_RUNTIME_SERVICE_ACCOUNT\}"/
  )
  assert.match(
    job,
    /host-drain \\\n {16}--regional-rehome-protocol "\$\{DESIRED_REHOME_PROTOCOL\}" \\\n {16}"\$\{POOL_ARGUMENTS\[@\]\}"\)"\n {12}echo "\$\{RESUME_REVIEW\}"\n {12}jq -e '\.changes == 2\n {16}or \(\.changes == 0 and \(\(\.backendUpdate \/\/ \[\]\) \| length\) > 0\)' \\\n {14}<<< "\$\{RESUME_REVIEW\}" >\/dev\/null/
  )
  // A resume whose only unapplied change is the reviewed backend update must apply it. Leaving
  // it is how a cell keeps the 300-second drain and no request logging behind a green resume.
  assert.match(
    job,
    /if test "\$\(jq -er '\.changes' <<< "\$\{RESUME_REVIEW\}"\)" = 0; then\n {14}terraform -chdir=infra\/terraform apply -auto-approve \\\n {16}"\$\{RUNNER_TEMP\}\/relay-same-cap-resume\.tfplan"\n {12}fi\n/
  )
  // Template-and-MIG drift still applies nothing on resume, which is what a resume means.
  const resumeStep = job.slice(
    job.indexOf('- name: Require converged Terraform state and a stable MIG on resume'),
    job.indexOf('- name: Apply only the selected same-cap template and MIG')
  )
  assert.equal(resumeStep.split('terraform -chdir=infra/terraform apply').length, 2)
  assert.match(job, /resume requires the isolated migration-only cell/)
  assert.match(job, /test "\$\{TARGET_INCARNATION\}" = "\$\{SOURCE_INCARNATION\}"/)
  assert.match(job, /\(.regionalRehomeProtocol \/\/ 0\) == \$protocol/)
  assert.match(job, /\(\.draining == false or \$drainingOk\)/)
  // Selector expectations must follow the mutations' returned generations,
  // not fixed offsets: isolate is a no-op on a cell a failed canary already
  // isolated, and the restore inspect must expect post-restore membership.
  assert.match(job, /SELECTOR_GENERATION_AFTER_ISOLATE=\$\{EFFECTIVE_SELECTOR_GENERATION\}/)
  assert.match(job, /SELECTOR_GENERATION_AFTER_ISOLATE=\$\{ISOLATE_GENERATION\}/)
  assert.match(job, /--expected-selector-generation "\$\{SELECTOR_GENERATION_AFTER_ISOLATE\}"/)
  assert.match(job, /--expected-selector-generation "\$\{SELECTOR_GENERATION_AFTER_RESTORE\}"/)
  assert.match(job, /--expected-migration-only-cells "\$\{RESTORED_MIGRATION_CELLS\}"/)
  assert.match(job, /--expected-general-cells "\$\{RESTORED_GENERAL_CELLS\}"/)
  assert.match(job, /FAILSAFE_GENERATION/)
  // Later batch waves start after ~16-min predecessor rolls, so BOTH evidence
  // age checks must scale by wave or cell_2+ can never pass; the bound's
  // per-wave step is the cell job timeout, so the two must move together.
  assert.match(job, /--required-migration-policy strict \\\n            --wave-index "\$\{WAVE_INDEX\}"/)
  // Wave 0 must retry freshness-only failures too: one Cloud Monitoring publish
  // lag at the sample instant is not health evidence, and single-shot wave 0
  // failed a whole batch on a series that was fresh again a minute later.
  assert.match(
    job,
    /dry-run\.state\.json" \\\n {14}--wave-index "\$\{WAVE_INDEX\}" \\\n {14}--selector-wave-delta "\$\{SELECTOR_WAVE_DELTA\}" --retry-freshness/
  )
  assert.doesNotMatch(job, /RETRY_ARGS/)
  // Break-glass: the override skips the aggregate 15-minute monitor evidence and
  // nothing else. The live per-wave recheck still runs on the override path, off
  // the dispatch inputs the rehome inspect below verifies against the director.
  assert.match(
    job,
    /if test -n "\$\{GATE_OVERRIDE_CONFIRMATION\}"; then[\s\S]{0,700}?--no-monitor-state \\\n {14}--expected-selector-generation "\$\{EXPECTED_SELECTOR_GENERATION\}" \\\n {14}--selector-membership-file[\s\S]{0,160}?--wave-index "\$\{WAVE_INDEX\}" \\\n {14}--selector-wave-delta "\$\{SELECTOR_WAVE_DELTA\}" --retry-freshness/
  )
  // The override is re-validated here, not trusted from the caller, and it is
  // bound to the digest this wave installs.
  assert.match(
    job,
    /test "\$\{GATE_OVERRIDE_CONFIRMATION\}" = \\\n {14}"SKIP_RELAY_MONITOR_GATE \$\{TARGET_IMAGE_DIGEST\}"/
  )
  assert.match(job, /\[\[ "\$\{GATE_OVERRIDE_REASON\}" =~ \^\[\[:print:\]\]\{12,500\}\$ \]\]/)
  // Exactly the aggregate-evidence steps are skipped, and only them: every step
  // that reads or spends the sealed monitor artifact carries the override guard.
  const overrideSkipped = [
    'Require fresh aggregate monitor evidence reference',
    'Download private aggregate monitor evidence',
    'Verify monitor evidence provenance',
    "Download this wave's single-use safety authority",
    'Require safety evidence consumed by this workflow'
  ]
  for (const name of overrideSkipped) {
    assert.match(
      job,
      new RegExp(`- name: ${name}\\n {8}if: \\$\\{\\{ inputs\\.mode != 'verify' && inputs\\.gate-override-confirmation == '' \\}\\}`)
    )
  }
  assert.equal(
    job.match(/inputs\.gate-override-confirmation == ''/g).length,
    overrideSkipped.length
  )
  // The wrapper validates the override before anything runs, passes it to every
  // cell, seals it into the canary artifact, and prints it in the run summary.
  assert.match(wrapper, /--gate-override-reason "\$\{GATE_OVERRIDE_REASON\}" \\\n {12}--gate-override-confirmation "\$\{GATE_OVERRIDE_CONFIRMATION\}"\)/)
  // One per cell job in the serial cell_1..cell_10 chain.
  assert.equal(
    wrapper.match(/gate-override-confirmation: \$\{\{ inputs\.gate-override-confirmation \}\}/g).length,
    10
  )
  assert.match(wrapper, /Aggregate monitor gate overridden \(break-glass\)/)
  assert.match(wrapper, /ACTOR: \$\{\{ github\.actor \}\}/)
  for (const name of [
    'Reject previously consumed aggregate safety evidence',
    'Consume aggregate safety evidence for this exact wave'
  ]) {
    assert.match(
      wrapper,
      new RegExp(`- name: ${name}\\n {8}if: \\$\\{\\{ inputs\\.mode != 'verify' && inputs\\.gate-override-confirmation == '' \\}\\}`)
    )
  }
  assert.match(job, /timeout-minutes: 75/)
  // Both age gates step by the cell job timeout above; the constant is
  // duplicated across the two languages, so pin each copy to it.
  for (const source of [
    '../../dev/scripts/relay-monitor-evidence.mjs',
    '../../apps/relay-ops/src/incident-live-preflight-cli.ts'
  ]) {
    const body = readFileSync(fileURLToPath(new URL(source, import.meta.url)), 'utf8')
    assert.match(body, /WAVE_PREDECESSOR_TIMEOUT_MS = 75 \* 60_000/)
    assert.match(body, /\^\[0-9\]\$/)
  }
  // Aged-evidence replay via job re-runs is fenced: mutations are
  // single-dispatch, so a failed cell needs a fresh gate and monitor run.
  assert.match(job, /test "\$\{GITHUB_RUN_ATTEMPT\}" = 1/)
  for (const index of [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) {
    assert.match(wrapper, new RegExp(`wave-index: '${index}'`))
  }
  assert.doesNotMatch(job, /EFFECTIVE_SELECTOR_GENERATION \+ 1\)/)
  assert.doesNotMatch(job, /EFFECTIVE_SELECTOR_GENERATION \+ 2\)/)
  assert.match(job, /\$region == "us-central1" and \$protocol == 0 and [.]region == null/)
  assert.match(job, /[.]regionalRehomeProtocol \/\/ 0/)
  assert.match(job, /runtime predecessor normalized legacy fields=/)
  assert.match(job, /probe-relay-rehome-trust[.]mjs/)
  assert.doesNotMatch(job, /service_account: \$\{\{ vars\.PRODUCTION_GCP_RELAY_(?:DIRECTOR_)?RUNTIME_SERVICE_ACCOUNT/)
  assert.doesNotMatch(job, /roles\/iam\.serviceAccountTokenCreator/)
})

// Why: the same-cap caller defines release_lease itself, and a caller-defined job presents the
// caller as job_workflow_ref, so the pair must admit the caller alongside its reusable job.
test('shared deploy WIF admits the exact same-cap reusable workflow pair and the caller itself', () => {
  const terraform = readFileSync(
    fileURLToPath(new URL('../../infra/terraform/relay-github-actions.tf', import.meta.url)),
    'utf8'
  )
  const providerStart = terraform.indexOf(
    'resource "google_iam_workload_identity_pool_provider" "github"'
  )
  const providerEnd = terraform.indexOf('\nresource "', providerStart + 1)
  const sharedProvider = terraform.slice(providerStart, providerEnd)
  assert.ok(providerStart >= 0 && providerEnd > providerStart)
  assert.match(sharedProvider, /local\.relay_github_workflow_conditions\["github"\]/)
  // The pairing itself now lives in the clause the provider renders, once per accepted repository.
  assert.match(
    terraform,
    /assertion\.workflow_ref == '\$\{prefix\}\$\{local\.github_production_relay_same_cap_workflow_file\}@refs\/heads\/main' && \(assertion\.job_workflow_ref == '\$\{prefix\}\$\{local\.github_production_relay_same_cap_job_workflow_file\}@refs\/heads\/main' \|\| assertion\.job_workflow_ref == '\$\{prefix\}\$\{local\.github_production_relay_same_cap_workflow_file\}@refs\/heads\/main'\)/
  )
})

test('pause and disable precede optional installation and cloud diagnostics', () => {
  const job = workflow('operate-relay-production-rehome-job.yml')
  const emergency = job.indexOf('Apply emergency durable pause or disable before diagnostics')
  const install = job.indexOf('pnpm install --frozen-lockfile')
  const revision = job.indexOf('Verify exact serving and rollback director identities')
  assert.ok(emergency > 0)
  assert.ok(emergency < install)
  assert.ok(emergency < revision)
  assert.match(job, /inputs\.mode == 'pause' \|\| inputs\.mode == 'disable'/)
  assert.match(job, /Seal 24-hour aggregate region observation evidence/)
  assert.match(job, /--freshness=25h --limit=30000/)
  assert.match(job, /relay-region-observation-\$\{\{ github\.run_id \}\}-\$\{\{ github\.run_attempt \}\}/)
  assert.match(job, /test "\$\{RATE_PER_MINUTE\}" = 10/)
})

test('a failed enable independently restores and verifies durable disabled state', () => {
  const job = workflow('operate-relay-production-rehome-job.yml')
  const enable = job.indexOf('Apply exact durable regional rehome enable')
  const evidence = job.indexOf('Read fresh aggregate completion and abort evidence')
  const summary = job.indexOf('Publish aggregate control evidence')
  const recovery = job.indexOf('Fail closed after an unsuccessful enable run')
  assert.ok(enable > 0 && enable < evidence && evidence < summary && summary < recovery)
  const recoveryStep = job.slice(recovery)
  assert.match(
    recoveryStep,
    /failure\(\) && inputs\.mode == 'enable' && steps\.google-auth\.outcome == 'success'/
  )
  assert.match(recoveryStep, /--mode recover-enable/)
  assert.match(recoveryStep, /--expected-control-generation "\$\{EXPECTED_CONTROL_GENERATION\}"/)
  assert.match(recoveryStep, /RECOVER_FAILED_REGIONAL_REHOME_ENABLE/)
  assert.match(recoveryStep, /\.control\.enabled == false/)
  assert.doesNotMatch(recoveryStep, /gcloud|pnpm/)
})

test('director rollout has a strict one-time identity bootstrap', () => {
  const workflowBody = workflow('deploy-relay-production-director.yml')
  const script = readFileSync(
    fileURLToPath(new URL('./deploy-relay-blue-green.mjs', import.meta.url)),
    'utf8'
  )
  assert.match(workflowBody, /BOOTSTRAP_RELAY_DIRECTOR_REHOME_IDENTITY/)
  assert.match(workflowBody, /--predecessor-runtime-service-account/)
  assert.match(workflowBody, /--expected-rehome-generation/)
  assert.match(script, /args\.push\('--service-account', config\['runtime-service-account'\]\)/)
  assert.match(script, /director predecessor runtime service account does not match/)
  const candidateProof = script.indexOf('await verifyRehomeDisabled(candidate.origin)')
  const trafficMove = script.indexOf('operations.updateTraffic(config, [`--to-tags=')
  assert.ok(candidateProof > 0 && candidateProof < trafficMove)
  assert.equal(script.indexOf('verifyRehomeDisabled', trafficMove), -1)
})

test('rehome job pipes every control result through tee under pipefail', () => {
  const job = workflow('operate-relay-production-rehome-job.yml')
  // Without `shell: bash` the step exit code is tee's, so a thrown inspect/apply passes green.
  assert.match(job, /defaults:\n  run:\n(?:    #.*\n)*    shell: bash\n/)
  assert.ok((job.match(/\| tee "\$\{RUNNER_TEMP\}/g) ?? []).length >= 5)
})