starred/orca
1
0
Fork 0
mirror of https://github.com/stablyai/orca.git synced 2026-10-02 08:02:02 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
e899809ff88a29af6b00a9dff38d54f1df1f120f
orca/cloud/dev/scripts/relay-regional-rehome-workflow.test.mjs
T
Jinwoo Hong bf3f95245c feat(relay): declare Asia cell c30 at the c27 shape (#22375) 
* feat(relay): declare Asia cell c30 at the c27 shape

Adds production-gce-c30 in asia-east2-a at the reviewed Asia shape (6,000
request units, 3,000/60 connection limits, 16-connection pool, disabled) and
the rehome trust the other Asia cells carry.

Every Asia enumeration now knows C30. The topology, admission, and director
tools treat it as its own reviewed wave so its plan and registration never
touch the live launch cells. C30 promotion requires C27 general and fresh
staging evidence. The topology and director validators now pin the committed
production pool of 16 instead of the stale 10, which had made the topology
workflow reject the committed launch cells.

* fix(relay): plan C30 at live images and prove it with its own canary

The shared URL map pulls every cell into the C30 topology plan, so the workflow
now plans each non-target cell at the image its live template serves, and the
validator names any change to a cell outside the wave. C30 promotion runs the
same five-minute production canary and automatic rollback C27 used, with the
load report proving the canary control was placed on C30, instead of relying
on staging evidence. C30 leaves the shadow gate's fleet pool list until it
serves, rollback rejects mixed partial sets, and a budget test pins the
mixed-Asia-pool refusal.

* fix(relay): pin C30 to the production director's live image digest

C30 promotion requires the director and C30 to report one digest, so C30
takes the director's sha256:4158d8a2 (read 2026-09-22). C27-C29 keep their
committed lines; every Asia check compares only the cells named in a run.

* fix(relay): read the committed cell map from a plan, not console

terraform console evaluates every output against state, and the Relay
deployments output indexes each cell's MIG, so it fails with Invalid index
while C30 is declared but not created. Read the map from a no-refresh,
unlocked plan over the same targets instead, and refuse empty overlay input.

* fix(relay): keep console readers working and C30 migration-only until promotion

relay_gce_cell_deployments indexed each cell's MIG, backend, and template,
so once C30 is declared but not applied every production terraform console
reader printed a warning to stdout and broke its jq parse. Wrap those six
lookups in try(..., null).

Same-cap listed C30 as general, so a rollback dispatch on a migration-only
C30 would restore it with activate and skip its canary. List it with the
migration-only cells until the promotion follow-up moves it.
2026-09-22 23:41:09 -04:00

270 lines
15 KiB
JavaScript
Raw Blame History

import assert from 'node:assert/strict'
import { readFileSync } from 'node:fs'
import { test } from 'node:test'
import { fileURLToPath } from 'node:url'
import { relayWorkflowUrl } from './relay-repository.mjs'
function workflow(name) {
return readFileSync(
fileURLToPath(relayWorkflowUrl(name)),
'utf8'
)
}
test('same-cap wrapper is reusable, canary-bound, and sequential', () => {
const wrapper = workflow('deploy-relay-production-same-cap.yml')
const job = workflow('deploy-relay-production-same-cap-job.yml')
assert.match(wrapper, /options: \[verify, canary-apply, batch-apply, rollback\]/)
assert.match(wrapper, /relay-same-cap-canary-\$\{\{ inputs\.canary-run-id \}\}/)
assert.match(wrapper, /needs: \[gate, cell_1\]/)
assert.match(wrapper, /needs: \[gate, cell_2\]/)
assert.match(wrapper, /needs: \[gate, cell_3\]/)
assert.match(job, /on:\n workflow_call:/)
assert.match(job, /c27\|c28\|c29\|c30\)/)
assert.match(job, /EXPECTED_HARD_CAP=3000/)
assert.match(job, /EXPECTED_REGION=asia-east2/)
assert.match(job, /--hard-cap "\$\{EXPECTED_HARD_CAP\}"/)
assert.match(job, /--regional-rehome-protocol "\$\{DESIRED_REHOME_PROTOCOL\}"/)
assert.match(job, /--argjson protocol "\$\{PREDECESSOR_REHOME_PROTOCOL\}"/)
assert.match(job, /runtime predecessor mismatch fields=/)
// A rollback interrupted between apply and restore must be resumable.
assert.match(job, /ROLLBACK_RESUME=true/)
assert.match(job, /test "\$\{LIVE_IMAGE_DIGEST\}" = "\$\{DESIRED_IMAGE_DIGEST\}"/)
// Resume must skip BOTH the drain (no restart will clear the flag) and the
// apply (state already converged), and prove convergence instead.
assert.match(
job,
/Reversibly isolate and drain only the selected cell\n if: \$\{\{ inputs\.mode != 'verify' && env\.ROLLBACK_RESUME != 'true' \}\}/
)
assert.match(
job,
/Apply only the selected same-cap template and MIG\n if: \$\{\{ inputs\.mode != 'verify' && env\.ROLLBACK_RESUME != 'true' \}\}/
)
assert.match(
job,
/Require converged Terraform state and a stable MIG on resume\n if: \$\{\{ inputs\.mode != 'verify' && env\.ROLLBACK_RESUME == 'true' \}\}/
)
assert.match(job, /resume found unconverged resources/)
// A canary or batch cell that failed before its template apply also
// resumes here with template drift from repo changes since its last roll;
// only a plan the reviewed validator approves for the image the cell
// already serves may pass, and resume still applies nothing.
assert.match(job, /requiring reviewed rollback-image drift/)
assert.match(
job,
/--image "\$\{DESIRED_IMAGE\}" \\\n {16}--rollback-image "\$\{DESIRED_IMAGE\}"/
)
// The relaxation is only safe if the reviewed validator actually runs on
// the NON-converged branch, in same-cap-cell mode, with the trust config
// the validator requires, restricted to the template-and-MIG change pair.
assert.match(
job,
/if ! terraform -chdir=infra\/terraform show -json[\s\S]{0,220}\| length == 0' >\/dev\/null\n then\n/
)
assert.match(
job,
/requiring reviewed rollback-image drift'\n[\s\S]{0,400}?\n {16}--mode same-cap-cell --cell-id "\$\{TARGET_CELL_ID\}" \\\n/
)
assert.match(
job,
/Require converged Terraform state and a stable MIG on resume[\s\S]{0,300}CAPACITY_SERVICE_ACCOUNT: \$\{\{ vars\.PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT \}\}\n {10}DIRECTOR_RUNTIME_SERVICE_ACCOUNT: \$\{\{ vars\.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT \}\}/
)
assert.match(
job,
/--rollback-image "\$\{DESIRED_IMAGE\}" \\\n {16}--capacity-service-account "\$\{CAPACITY_SERVICE_ACCOUNT\}" \\\n {16}--rehome-director-service-account "\$\{DIRECTOR_RUNTIME_SERVICE_ACCOUNT\}"/
)
assert.match(
job,
/host-drain \\\n {16}--regional-rehome-protocol "\$\{DESIRED_REHOME_PROTOCOL\}" \\\n {16}"\$\{POOL_ARGUMENTS\[@\]\}"\)"\n {12}echo "\$\{RESUME_REVIEW\}"\n {12}jq -e '\.changes == 2' <<< "\$\{RESUME_REVIEW\}" >\/dev\/null/
)
// A resume applies nothing at all, which is what a resume means: the only accepted
// unconverged plan is the template-and-MIG rollback-image drift, and it is left pending.
const resumeStep = job.slice(
job.indexOf('- name: Require converged Terraform state and a stable MIG on resume'),
job.indexOf('- name: Apply only the selected same-cap template and MIG')
)
assert.equal(resumeStep.split('terraform -chdir=infra/terraform apply').length, 1)
assert.match(job, /resume requires the isolated migration-only cell/)
assert.match(job, /test "\$\{TARGET_INCARNATION\}" = "\$\{SOURCE_INCARNATION\}"/)
assert.match(job, /\(.regionalRehomeProtocol \/\/ 0\) == \$protocol/)
assert.match(job, /\(\.draining == false or \$drainingOk\)/)
// Selector expectations must follow the mutations' returned generations,
// not fixed offsets: isolate is a no-op on a cell a failed canary already
// isolated, and the restore inspect must expect post-restore membership.
assert.match(job, /SELECTOR_GENERATION_AFTER_ISOLATE=\$\{EFFECTIVE_SELECTOR_GENERATION\}/)
assert.match(job, /SELECTOR_GENERATION_AFTER_ISOLATE=\$\{ISOLATE_GENERATION\}/)
assert.match(job, /--expected-selector-generation "\$\{SELECTOR_GENERATION_AFTER_ISOLATE\}"/)
assert.match(job, /--expected-selector-generation "\$\{SELECTOR_GENERATION_AFTER_RESTORE\}"/)
assert.match(job, /--expected-migration-only-cells "\$\{RESTORED_MIGRATION_CELLS\}"/)
assert.match(job, /--expected-general-cells "\$\{RESTORED_GENERAL_CELLS\}"/)
assert.match(job, /FAILSAFE_GENERATION/)
// Later batch waves start after ~16-min predecessor rolls, so BOTH evidence
// age checks must scale by wave or cell_2+ can never pass; the bound's
// per-wave step is the cell job timeout, so the two must move together.
assert.match(job, /--required-migration-policy strict \\\n --wave-index "\$\{WAVE_INDEX\}"/)
// Wave 0 must retry freshness-only failures too: one Cloud Monitoring publish
// lag at the sample instant is not health evidence, and single-shot wave 0
// failed a whole batch on a series that was fresh again a minute later.
assert.match(
job,
/dry-run\.state\.json" \\\n {14}--wave-index "\$\{WAVE_INDEX\}" \\\n {14}--selector-wave-delta "\$\{SELECTOR_WAVE_DELTA\}" --retry-freshness/
)
assert.doesNotMatch(job, /RETRY_ARGS/)
// Break-glass: the override skips the aggregate 15-minute monitor evidence and
// nothing else. The live per-wave recheck still runs on the override path, off
// the dispatch inputs the rehome inspect below verifies against the director.
assert.match(
job,
/if test -n "\$\{GATE_OVERRIDE_CONFIRMATION\}"; then[\s\S]{0,700}?--no-monitor-state \\\n {14}--expected-selector-generation "\$\{EXPECTED_SELECTOR_GENERATION\}" \\\n {14}--selector-membership-file[\s\S]{0,160}?--wave-index "\$\{WAVE_INDEX\}" \\\n {14}--selector-wave-delta "\$\{SELECTOR_WAVE_DELTA\}" --retry-freshness/
)
// The override is re-validated here, not trusted from the caller, and it is
// bound to the digest this wave installs.
assert.match(
job,
/test "\$\{GATE_OVERRIDE_CONFIRMATION\}" = \\\n {14}"SKIP_RELAY_MONITOR_GATE \$\{TARGET_IMAGE_DIGEST\}"/
)
assert.match(job, /\[\[ "\$\{GATE_OVERRIDE_REASON\}" =~ \^\[\[:print:\]\]\{12,500\}\$ \]\]/)
// Exactly the aggregate-evidence steps are skipped, and only them: every step
// that reads or spends the sealed monitor artifact carries the override guard.
const overrideSkipped = [
'Require fresh aggregate monitor evidence reference',
'Download private aggregate monitor evidence',
'Verify monitor evidence provenance',
"Download this wave's single-use safety authority",
'Require safety evidence consumed by this workflow'
]
for (const name of overrideSkipped) {
assert.match(
job,
new RegExp(`- name: ${name}\\n {8}if: \\$\\{\\{ inputs\\.mode != 'verify' && inputs\\.gate-override-confirmation == '' \\}\\}`)
)
}
assert.equal(
job.match(/inputs\.gate-override-confirmation == ''/g).length,
overrideSkipped.length
)
// The wrapper validates the override before anything runs, passes it to every
// cell, seals it into the canary artifact, and prints it in the run summary.
assert.match(wrapper, /--gate-override-reason "\$\{GATE_OVERRIDE_REASON\}" \\\n {12}--gate-override-confirmation "\$\{GATE_OVERRIDE_CONFIRMATION\}"\)/)
// One per cell job in the serial cell_1..cell_10 chain.
assert.equal(
wrapper.match(/gate-override-confirmation: \$\{\{ inputs\.gate-override-confirmation \}\}/g).length,
10
)
assert.match(wrapper, /Aggregate monitor gate overridden \(break-glass\)/)
assert.match(wrapper, /ACTOR: \$\{\{ github\.actor \}\}/)
for (const name of [
'Reject previously consumed aggregate safety evidence',
'Consume aggregate safety evidence for this exact wave'
]) {
assert.match(
wrapper,
new RegExp(`- name: ${name}\\n {8}if: \\$\\{\\{ inputs\\.mode != 'verify' && inputs\\.gate-override-confirmation == '' \\}\\}`)
)
}
assert.match(job, /timeout-minutes: 75/)
// Both age gates step by the cell job timeout above; the constant is
// duplicated across the two languages, so pin each copy to it.
for (const source of [
'../../dev/scripts/relay-monitor-evidence.mjs',
'../../apps/relay-ops/src/incident-live-preflight-cli.ts'
]) {
const body = readFileSync(fileURLToPath(new URL(source, import.meta.url)), 'utf8')
assert.match(body, /WAVE_PREDECESSOR_TIMEOUT_MS = 75 \* 60_000/)
assert.match(body, /\^\[0-9\]\$/)
}
// Aged-evidence replay via job re-runs is fenced: mutations are
// single-dispatch, so a failed cell needs a fresh gate and monitor run.
assert.match(job, /test "\$\{GITHUB_RUN_ATTEMPT\}" = 1/)
for (const index of [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) {
assert.match(wrapper, new RegExp(`wave-index: '${index}'`))
}
assert.doesNotMatch(job, /EFFECTIVE_SELECTOR_GENERATION \+ 1\)/)
assert.doesNotMatch(job, /EFFECTIVE_SELECTOR_GENERATION \+ 2\)/)
assert.match(job, /\$region == "us-central1" and \$protocol == 0 and [.]region == null/)
assert.match(job, /[.]regionalRehomeProtocol \/\/ 0/)
assert.match(job, /runtime predecessor normalized legacy fields=/)
assert.match(job, /probe-relay-rehome-trust[.]mjs/)
assert.doesNotMatch(job, /service_account: \$\{\{ vars\.PRODUCTION_GCP_RELAY_(?:DIRECTOR_)?RUNTIME_SERVICE_ACCOUNT/)
assert.doesNotMatch(job, /roles\/iam\.serviceAccountTokenCreator/)
})
// Why: the same-cap caller defines release_lease itself, and a caller-defined job presents the
// caller as job_workflow_ref, so the pair must admit the caller alongside its reusable job.
test('shared deploy WIF admits the exact same-cap reusable workflow pair and the caller itself', () => {
const terraform = readFileSync(
fileURLToPath(new URL('../../infra/terraform/relay-github-actions.tf', import.meta.url)),
'utf8'
)
const providerStart = terraform.indexOf(
'resource "google_iam_workload_identity_pool_provider" "github"'
)
const providerEnd = terraform.indexOf('\nresource "', providerStart + 1)
const sharedProvider = terraform.slice(providerStart, providerEnd)
assert.ok(providerStart >= 0 && providerEnd > providerStart)
assert.match(sharedProvider, /local\.relay_github_workflow_conditions\["github"\]/)
// The pairing itself now lives in the clause the provider renders, once per accepted repository.
assert.match(
terraform,
/assertion\.workflow_ref == '\$\{prefix\}\$\{local\.github_production_relay_same_cap_workflow_file\}@refs\/heads\/main' && \(assertion\.job_workflow_ref == '\$\{prefix\}\$\{local\.github_production_relay_same_cap_job_workflow_file\}@refs\/heads\/main' \|\| assertion\.job_workflow_ref == '\$\{prefix\}\$\{local\.github_production_relay_same_cap_workflow_file\}@refs\/heads\/main'\)/
)
})
test('pause and disable precede optional installation and cloud diagnostics', () => {
const job = workflow('operate-relay-production-rehome-job.yml')
const emergency = job.indexOf('Apply emergency durable pause or disable before diagnostics')
const install = job.indexOf('pnpm install --frozen-lockfile')
const revision = job.indexOf('Verify exact serving and rollback director identities')
assert.ok(emergency > 0)
assert.ok(emergency < install)
assert.ok(emergency < revision)
assert.match(job, /inputs\.mode == 'pause' \|\| inputs\.mode == 'disable'/)
assert.match(job, /Seal 24-hour aggregate region observation evidence/)
assert.match(job, /--freshness=25h --limit=30000/)
assert.match(job, /relay-region-observation-\$\{\{ github\.run_id \}\}-\$\{\{ github\.run_attempt \}\}/)
assert.match(job, /test "\$\{RATE_PER_MINUTE\}" = 10/)
})
test('a failed enable independently restores and verifies durable disabled state', () => {
const job = workflow('operate-relay-production-rehome-job.yml')
const enable = job.indexOf('Apply exact durable regional rehome enable')
const evidence = job.indexOf('Read fresh aggregate completion and abort evidence')
const summary = job.indexOf('Publish aggregate control evidence')
const recovery = job.indexOf('Fail closed after an unsuccessful enable run')
assert.ok(enable > 0 && enable < evidence && evidence < summary && summary < recovery)
const recoveryStep = job.slice(recovery)
assert.match(
recoveryStep,
/failure\(\) && inputs\.mode == 'enable' && steps\.google-auth\.outcome == 'success'/
)
assert.match(recoveryStep, /--mode recover-enable/)
assert.match(recoveryStep, /--expected-control-generation "\$\{EXPECTED_CONTROL_GENERATION\}"/)
assert.match(recoveryStep, /RECOVER_FAILED_REGIONAL_REHOME_ENABLE/)
assert.match(recoveryStep, /\.control\.enabled == false/)
assert.doesNotMatch(recoveryStep, /gcloud|pnpm/)
})
test('director rollout has a strict one-time identity bootstrap', () => {
const workflowBody = workflow('deploy-relay-production-director.yml')
const script = readFileSync(
fileURLToPath(new URL('./deploy-relay-blue-green.mjs', import.meta.url)),
'utf8'
)
assert.match(workflowBody, /BOOTSTRAP_RELAY_DIRECTOR_REHOME_IDENTITY/)
assert.match(workflowBody, /--predecessor-runtime-service-account/)
assert.match(workflowBody, /--expected-rehome-generation/)
assert.match(script, /args\.push\('--service-account', config\['runtime-service-account'\]\)/)
assert.match(script, /director predecessor runtime service account does not match/)
const candidateProof = script.indexOf('await verifyRehomeDisabled(candidate.origin)')
const trafficMove = script.indexOf('operations.updateTraffic(config, [`--to-tags=')
assert.ok(candidateProof > 0 && candidateProof < trafficMove)
assert.equal(script.indexOf('verifyRehomeDisabled', trafficMove), -1)
})
test('rehome job pipes every control result through tee under pipefail', () => {
const job = workflow('operate-relay-production-rehome-job.yml')
// Without `shell: bash` the step exit code is tee's, so a thrown inspect/apply passes green.
assert.match(job, /defaults:\n run:\n(?: #.*\n)* shell: bash\n/)
assert.ok((job.match(/\| tee "\$\{RUNNER_TEMP\}/g) ?? []).length >= 5)
})
Reference in New Issue View Git Blame Copy Permalink