diff --git a/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml b/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml index 617c852d304..930da583aaa 100644 --- a/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml +++ b/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml @@ -78,7 +78,8 @@ jobs: [[ "${ROLLBACK_REHOME_PROTOCOL}" =~ ^(0|1|3)$ ]] [[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]] [[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]] - [[ "${WAVE_INDEX}" =~ ^[0-3]$ ]] + # cell_1..cell_10 in the calling wave; the chain is static, so this range is too. + [[ "${WAVE_INDEX}" =~ ^[0-9]$ ]] # The caller validated this too; re-check here so a malformed override # can never reach a mutation through this reusable workflow. if test -n "${GATE_OVERRIDE_REASON}${GATE_OVERRIDE_CONFIRMATION}"; then diff --git a/.github/workflows/cloud-deploy-relay-production-same-cap.yml b/.github/workflows/cloud-deploy-relay-production-same-cap.yml index 421fa080072..cd24792cc8c 100644 --- a/.github/workflows/cloud-deploy-relay-production-same-cap.yml +++ b/.github/workflows/cloud-deploy-relay-production-same-cap.yml @@ -10,7 +10,7 @@ on: type: choice options: [verify, canary-apply, batch-apply, rollback] cell-ids: - description: Ordered comma-separated serving cells; one canary or two to four batch cells + description: Ordered comma-separated serving cells; one canary or two to ten batch cells required: true type: string target-image-digest: @@ -313,6 +313,144 @@ jobs: wave-index: '3' secrets: inherit + cell_5: + if: ${{ needs.cell_4.result == 'success' && fromJSON(needs.gate.outputs.cells)[4] != null }} + needs: [gate, cell_4] + uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml + with: + mode: ${{ needs.gate.outputs.job-mode }} + target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[4] }} + target-image-digest: ${{ inputs.target-image-digest }} + rollback-image-digest: ${{ inputs.rollback-image-digest }} + target-rehome-protocol: ${{ inputs.target-rehome-protocol }} + rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }} + expected-selector-generation: ${{ inputs.expected-selector-generation }} + expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }} + expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }} + expected-general-cells: ${{ inputs.expected-general-cells }} + expected-rehome-generation: ${{ inputs.expected-rehome-generation }} + monitor-run-id: ${{ inputs.monitor-run-id }} + monitor-run-attempt: ${{ inputs.monitor-run-attempt }} + gate-override-reason: ${{ inputs.gate-override-reason }} + gate-override-confirmation: ${{ inputs.gate-override-confirmation }} + wave-index: '4' + secrets: inherit + + cell_6: + if: ${{ needs.cell_5.result == 'success' && fromJSON(needs.gate.outputs.cells)[5] != null }} + needs: [gate, cell_5] + uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml + with: + mode: ${{ needs.gate.outputs.job-mode }} + target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[5] }} + target-image-digest: ${{ inputs.target-image-digest }} + rollback-image-digest: ${{ inputs.rollback-image-digest }} + target-rehome-protocol: ${{ inputs.target-rehome-protocol }} + rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }} + expected-selector-generation: ${{ inputs.expected-selector-generation }} + expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }} + expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }} + expected-general-cells: ${{ inputs.expected-general-cells }} + expected-rehome-generation: ${{ inputs.expected-rehome-generation }} + monitor-run-id: ${{ inputs.monitor-run-id }} + monitor-run-attempt: ${{ inputs.monitor-run-attempt }} + gate-override-reason: ${{ inputs.gate-override-reason }} + gate-override-confirmation: ${{ inputs.gate-override-confirmation }} + wave-index: '5' + secrets: inherit + + cell_7: + if: ${{ needs.cell_6.result == 'success' && fromJSON(needs.gate.outputs.cells)[6] != null }} + needs: [gate, cell_6] + uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml + with: + mode: ${{ needs.gate.outputs.job-mode }} + target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[6] }} + target-image-digest: ${{ inputs.target-image-digest }} + rollback-image-digest: ${{ inputs.rollback-image-digest }} + target-rehome-protocol: ${{ inputs.target-rehome-protocol }} + rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }} + expected-selector-generation: ${{ inputs.expected-selector-generation }} + expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }} + expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }} + expected-general-cells: ${{ inputs.expected-general-cells }} + expected-rehome-generation: ${{ inputs.expected-rehome-generation }} + monitor-run-id: ${{ inputs.monitor-run-id }} + monitor-run-attempt: ${{ inputs.monitor-run-attempt }} + gate-override-reason: ${{ inputs.gate-override-reason }} + gate-override-confirmation: ${{ inputs.gate-override-confirmation }} + wave-index: '6' + secrets: inherit + + cell_8: + if: ${{ needs.cell_7.result == 'success' && fromJSON(needs.gate.outputs.cells)[7] != null }} + needs: [gate, cell_7] + uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml + with: + mode: ${{ needs.gate.outputs.job-mode }} + target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[7] }} + target-image-digest: ${{ inputs.target-image-digest }} + rollback-image-digest: ${{ inputs.rollback-image-digest }} + target-rehome-protocol: ${{ inputs.target-rehome-protocol }} + rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }} + expected-selector-generation: ${{ inputs.expected-selector-generation }} + expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }} + expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }} + expected-general-cells: ${{ inputs.expected-general-cells }} + expected-rehome-generation: ${{ inputs.expected-rehome-generation }} + monitor-run-id: ${{ inputs.monitor-run-id }} + monitor-run-attempt: ${{ inputs.monitor-run-attempt }} + gate-override-reason: ${{ inputs.gate-override-reason }} + gate-override-confirmation: ${{ inputs.gate-override-confirmation }} + wave-index: '7' + secrets: inherit + + cell_9: + if: ${{ needs.cell_8.result == 'success' && fromJSON(needs.gate.outputs.cells)[8] != null }} + needs: [gate, cell_8] + uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml + with: + mode: ${{ needs.gate.outputs.job-mode }} + target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[8] }} + target-image-digest: ${{ inputs.target-image-digest }} + rollback-image-digest: ${{ inputs.rollback-image-digest }} + target-rehome-protocol: ${{ inputs.target-rehome-protocol }} + rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }} + expected-selector-generation: ${{ inputs.expected-selector-generation }} + expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }} + expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }} + expected-general-cells: ${{ inputs.expected-general-cells }} + expected-rehome-generation: ${{ inputs.expected-rehome-generation }} + monitor-run-id: ${{ inputs.monitor-run-id }} + monitor-run-attempt: ${{ inputs.monitor-run-attempt }} + gate-override-reason: ${{ inputs.gate-override-reason }} + gate-override-confirmation: ${{ inputs.gate-override-confirmation }} + wave-index: '8' + secrets: inherit + + cell_10: + if: ${{ needs.cell_9.result == 'success' && fromJSON(needs.gate.outputs.cells)[9] != null }} + needs: [gate, cell_9] + uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml + with: + mode: ${{ needs.gate.outputs.job-mode }} + target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[9] }} + target-image-digest: ${{ inputs.target-image-digest }} + rollback-image-digest: ${{ inputs.rollback-image-digest }} + target-rehome-protocol: ${{ inputs.target-rehome-protocol }} + rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }} + expected-selector-generation: ${{ inputs.expected-selector-generation }} + expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }} + expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }} + expected-general-cells: ${{ inputs.expected-general-cells }} + expected-rehome-generation: ${{ inputs.expected-rehome-generation }} + monitor-run-id: ${{ inputs.monitor-run-id }} + monitor-run-attempt: ${{ inputs.monitor-run-attempt }} + gate-override-reason: ${{ inputs.gate-override-reason }} + gate-override-confirmation: ${{ inputs.gate-override-confirmation }} + wave-index: '9' + secrets: inherit + seal_canary: if: ${{ inputs.mode == 'canary-apply' }} needs: [gate, cell_1] @@ -360,6 +498,12 @@ jobs: - cell_2 - cell_3 - cell_4 + - cell_5 + - cell_6 + - cell_7 + - cell_8 + - cell_9 + - cell_10 - seal_canary runs-on: blacksmith-2vcpu-ubuntu-2204 timeout-minutes: 10 diff --git a/cloud/apps/relay-ops/src/incident-live-preflight-cli.test.ts b/cloud/apps/relay-ops/src/incident-live-preflight-cli.test.ts index 587143ed7e0..41f8ee49620 100644 --- a/cloud/apps/relay-ops/src/incident-live-preflight-cli.test.ts +++ b/cloud/apps/relay-ops/src/incident-live-preflight-cli.test.ts @@ -242,9 +242,16 @@ describe('relay incident live preflight', () => { await expect(runIncidentLivePreflight( ['--state-file', agedState(235 * 60_000 + 1), '--wave-index', '3'], deps )).rejects.toThrow('monitor evidence is incomplete or stale') - // The wave index is a strict single-use 0-3 argument. + // The last cell of a ten-cell same-cap batch: 10min + 9 * 75min exactly. await expect(runIncidentLivePreflight( - ['--state-file', stateFile(), '--wave-index', '4'], deps + ['--state-file', agedState(685 * 60_000), '--wave-index', '9'], deps + )).resolves.toBeUndefined() + await expect(runIncidentLivePreflight( + ['--state-file', agedState(685 * 60_000 + 1), '--wave-index', '9'], deps + )).rejects.toThrow('monitor evidence is incomplete or stale') + // The wave index is a strict single-use 0-9 argument. + await expect(runIncidentLivePreflight( + ['--state-file', stateFile(), '--wave-index', '10'], deps )).rejects.toThrow('usage:') await expect(runIncidentLivePreflight( ['--state-file', stateFile(), '--wave-index', ''], deps diff --git a/cloud/apps/relay-ops/src/incident-live-preflight-cli.ts b/cloud/apps/relay-ops/src/incident-live-preflight-cli.ts index 5183286e302..99ea42e318e 100644 --- a/cloud/apps/relay-ops/src/incident-live-preflight-cli.ts +++ b/cloud/apps/relay-ops/src/incident-live-preflight-cli.ts @@ -30,7 +30,9 @@ const FRESHNESS_RETRY_INTERVAL_MS = 15_000 const MONITOR_EVIDENCE_MAX_AGE_MS = 10 * 60_000 // Matches the same-cap cell job timeout-minutes; bounds each predecessor wave. const WAVE_PREDECESSOR_TIMEOUT_MS = 75 * 60_000 -const WAVE_INDEX_PATTERN = /^[0-3]$/ +// Widest any wave chain declares (same-cap's cell_1..cell_10); each job workflow +// pins its own narrower range. +const WAVE_INDEX_PATTERN = /^[0-9]$/ // 2 for a general cell's isolate-and-restore wave, 0 for a migration-only cell's no-op pair. const SELECTOR_WAVE_DELTA_PATTERN = /^[02]$/ diff --git a/cloud/dev/scripts/relay-monitor-evidence.mjs b/cloud/dev/scripts/relay-monitor-evidence.mjs index 26eb37d0d4d..072991ba8fe 100644 --- a/cloud/dev/scripts/relay-monitor-evidence.mjs +++ b/cloud/dev/scripts/relay-monitor-evidence.mjs @@ -10,7 +10,9 @@ const JWT = /^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/ const EVIDENCE_MAX_AGE_MS = 5 * 60_000 // Matches the same-cap cell job timeout-minutes; bounds each predecessor wave. const WAVE_PREDECESSOR_TIMEOUT_MS = 75 * 60_000 -const WAVE_INDEX = /^[0-3]$/ +// Widest any wave chain declares (same-cap's cell_1..cell_10); each job workflow +// pins its own narrower range. +const WAVE_INDEX = /^[0-9]$/ const EVIDENCE_SAMPLE_INTERVAL_MS = 60_000 const EVIDENCE_MAX_LINEAGE_MS = 25 * 60_000 const MIGRATION_POLICIES = new Set([ diff --git a/cloud/dev/scripts/relay-monitor-evidence.test.mjs b/cloud/dev/scripts/relay-monitor-evidence.test.mjs index 43d2ac02763..876b710d0c7 100644 --- a/cloud/dev/scripts/relay-monitor-evidence.test.mjs +++ b/cloud/dev/scripts/relay-monitor-evidence.test.mjs @@ -242,7 +242,7 @@ test('later same-cap waves accept evidence aged by predecessor cell rolls', asyn await ageState(17 * 60_000) await assert.rejects(authorityAt('0'), /authority is incomplete or stale/) await assert.doesNotReject(authorityAt('1')) - await assert.rejects(authorityAt('4'), /wave index is invalid/) + await assert.rejects(authorityAt('10'), /wave index is invalid/) await assert.rejects(authorityAt('x'), /wave index is invalid/) // Both edges of one predecessor job timeout: 5min + 75min exactly. await ageState(80 * 60_000) @@ -259,6 +259,11 @@ test('later same-cap waves accept evidence aged by predecessor cell rolls', asyn await assert.doesNotReject(authorityAt('3')) await ageState(230 * 60_000 + 1) await assert.rejects(authorityAt('3'), /authority is incomplete or stale/) + // The last cell of a ten-cell same-cap batch: 5min + 9 * 75min exactly. + await ageState(680 * 60_000) + await assert.doesNotReject(authorityAt('9')) + await ageState(680 * 60_000 + 1) + await assert.rejects(authorityAt('9'), /authority is incomplete or stale/) } finally { await rm(directory, { recursive: true, force: true }) } diff --git a/cloud/dev/scripts/relay-production-same-cap-wave.mjs b/cloud/dev/scripts/relay-production-same-cap-wave.mjs index 654d9fc55bd..f1ef2a9c4d5 100644 --- a/cloud/dev/scripts/relay-production-same-cap-wave.mjs +++ b/cloud/dev/scripts/relay-production-same-cap-wave.mjs @@ -34,7 +34,7 @@ function cells(value) { const parsed = value.split(',').map((cell) => cell.trim()).filter(Boolean) if ( parsed.length < 1 || - parsed.length > 4 || + parsed.length > 10 || new Set(parsed).size !== parsed.length || parsed.some((cell) => !SAME_CAP_CELLS.includes(cell)) ) throw new Error('same-cap wave cells are invalid') @@ -76,8 +76,9 @@ export function validateSameCapWave(input) { if (input.mode === 'canary-apply' && selected.length !== 1) { throw new Error('canary mode requires exactly one cell') } - if (input.mode === 'batch-apply' && (selected.length < 2 || selected.length > 4)) { - throw new Error('batch mode requires two to four cells') + // Ten is the wave workflow's statically declared serial cell-job chain, cell_1..cell_10. + if (input.mode === 'batch-apply' && (selected.length < 2 || selected.length > 10)) { + throw new Error('batch mode requires two to ten cells') } // Later waves expect the selector to advance by exactly 2 per predecessor, // which a resumed rollback cell (isolate skipped, +1) violates. diff --git a/cloud/dev/scripts/relay-production-same-cap-wave.test.mjs b/cloud/dev/scripts/relay-production-same-cap-wave.test.mjs index 0d2ac6f1a96..bed12988e0b 100644 --- a/cloud/dev/scripts/relay-production-same-cap-wave.test.mjs +++ b/cloud/dev/scripts/relay-production-same-cap-wave.test.mjs @@ -57,6 +57,48 @@ test('requires one canary or a bounded reviewed batch', () => { }), /cells/) }) +// The bound is the wave workflow's static cell_1..cell_10 chain: a batch longer than the +// chain would silently drop its tail cells, so it is refused before any mutation. +test('a batch fills the serial cell chain and never overflows it', () => { + const general = SAME_CAP_CELLS.filter((cell) => entryAdmission(cell) === 'general') + const batch = (count) => { + const cellIds = general.slice(0, count).join(',') + return validateSameCapWave({ + mode: 'batch-apply', + cellIds, + targetDigest, + rollbackDigest, + confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${cellIds}`, + canaryRunId: '42' + }) + } + assert.equal(batch(10).cells.length, 10) + assert.throws(() => batch(11), /same-cap wave cells are invalid/) + assert.throws(() => batch(1), /batch mode requires two to ten cells/) +}) + +// The validator's ten-cell bound is only true if the workflow really declares ten strictly +// serial cell jobs and frees the lease after all of them. +test('the wave workflow chains exactly ten serial cell jobs', () => { + const dispatch = readRelayWorkflow('deploy-relay-production-same-cap.yml') + for (let index = 0; index < 10; index += 1) { + const job = index + 1 + assert.match(dispatch, new RegExp(`\n cell_${job}:\n`), `cell_${job} is missing`) + assert.match(dispatch, new RegExp(`fromJSON\\(needs\\.gate\\.outputs\\.cells\\)\\[${index}\\]`)) + assert.match(dispatch, new RegExp(`wave-index: '${index}'`)) + if (index > 0) { + assert.match(dispatch, new RegExp(`needs: \\[gate, cell_${index}\\]`)) + assert.match( + dispatch, + new RegExp(`if: \\$\\{\\{ needs\\.cell_${index}\\.result == 'success' && ` + + `fromJSON\\(needs\\.gate\\.outputs\\.cells\\)\\[${index}\\] != null \\}\\}`) + ) + } + assert.match(dispatch, new RegExp(`\n - cell_${job}\n`), `release_lease must need cell_${job}`) + } + assert.doesNotMatch(dispatch, /\n cell_11:/) +}) + test('rolls the migration-only cells but never mixes the two classes in one wave', () => { for (const cellId of SAME_CAP_MIGRATION_ONLY_CELLS) { assert.equal(SAME_CAP_CELLS.includes(cellId), true, cellId) diff --git a/cloud/dev/scripts/relay-regional-rehome-workflow.test.mjs b/cloud/dev/scripts/relay-regional-rehome-workflow.test.mjs index e5294124466..a7457c7450a 100644 --- a/cloud/dev/scripts/relay-regional-rehome-workflow.test.mjs +++ b/cloud/dev/scripts/relay-regional-rehome-workflow.test.mjs @@ -139,9 +139,10 @@ test('same-cap wrapper is reusable, canary-bound, and sequential', () => { // The wrapper validates the override before anything runs, passes it to every // cell, seals it into the canary artifact, and prints it in the run summary. assert.match(wrapper, /--gate-override-reason "\$\{GATE_OVERRIDE_REASON\}" \\\n {12}--gate-override-confirmation "\$\{GATE_OVERRIDE_CONFIRMATION\}"\)/) + // One per cell job in the serial cell_1..cell_10 chain. assert.equal( wrapper.match(/gate-override-confirmation: \$\{\{ inputs\.gate-override-confirmation \}\}/g).length, - 4 + 10 ) assert.match(wrapper, /Aggregate monitor gate overridden \(break-glass\)/) assert.match(wrapper, /ACTOR: \$\{\{ github\.actor \}\}/) @@ -163,12 +164,12 @@ test('same-cap wrapper is reusable, canary-bound, and sequential', () => { ]) { const body = readFileSync(fileURLToPath(new URL(source, import.meta.url)), 'utf8') assert.match(body, /WAVE_PREDECESSOR_TIMEOUT_MS = 75 \* 60_000/) - assert.match(body, /\^\[0-3\]\$/) + assert.match(body, /\^\[0-9\]\$/) } // Aged-evidence replay via job re-runs is fenced: mutations are // single-dispatch, so a failed cell needs a fresh gate and monitor run. assert.match(job, /test "\$\{GITHUB_RUN_ATTEMPT\}" = 1/) - for (const index of [0, 1, 2, 3]) { + for (const index of [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) { assert.match(wrapper, new RegExp(`wave-index: '${index}'`)) } assert.doesNotMatch(job, /EFFECTIVE_SELECTOR_GENERATION \+ 1\)/) diff --git a/cloud/dev/scripts/validate-relay-asia-topology-plan.mjs b/cloud/dev/scripts/validate-relay-asia-topology-plan.mjs index 953a214156e..a3fe8f0e2a3 100644 --- a/cloud/dev/scripts/validate-relay-asia-topology-plan.mjs +++ b/cloud/dev/scripts/validate-relay-asia-topology-plan.mjs @@ -2,6 +2,11 @@ import { readFileSync } from 'node:fs' import { fileURLToPath } from 'node:url' const REGION = 'asia-east2' +// Mirrors local.relay_gce_topology in infra/terraform/relay-gce-cells.tf, which Terraform +// cannot export to JS; the census test below the validator equates the two by reading the +// .tf source, so this pair and the topology `check` assert cannot drift apart. +export const RELAY_CELL_BACKEND_TIMEOUT_SECONDS = 86_400 +export const RELAY_CELL_CONNECTION_DRAIN_SECONDS = 60 const CELL_SHAPES = { production: { domain: 'relay.onorca.dev', @@ -117,8 +122,8 @@ function requireCellBackend(change, config, cellId) { const hostname = cellId.split('-').at(-1) const name = `${relayGceName(config.environment)}-${hostname}` if ( - after?.timeout_sec !== 86_400 || - after?.connection_draining_timeout_sec !== 300 || + after?.timeout_sec !== RELAY_CELL_BACKEND_TIMEOUT_SECONDS || + after?.connection_draining_timeout_sec !== RELAY_CELL_CONNECTION_DRAIN_SECONDS || after?.load_balancing_scheme !== 'EXTERNAL_MANAGED' || after?.protocol !== 'HTTP' || after?.port_name !== 'relay' || diff --git a/cloud/dev/scripts/validate-relay-asia-topology-plan.test.mjs b/cloud/dev/scripts/validate-relay-asia-topology-plan.test.mjs index 154030489f5..5b269044b89 100644 --- a/cloud/dev/scripts/validate-relay-asia-topology-plan.test.mjs +++ b/cloud/dev/scripts/validate-relay-asia-topology-plan.test.mjs @@ -1,6 +1,11 @@ import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' import { test } from 'node:test' -import { validateRelayAsiaTopologyPlan } from './validate-relay-asia-topology-plan.mjs' +import { + RELAY_CELL_BACKEND_TIMEOUT_SECONDS, + RELAY_CELL_CONNECTION_DRAIN_SECONDS, + validateRelayAsiaTopologyPlan +} from './validate-relay-asia-topology-plan.mjs' const image = `us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:${'a'.repeat(64)}` const config = { environment: 'staging', cells: ['staging-gce-c4'], image } @@ -49,7 +54,8 @@ const resources = [ update_policy: [{ replacement_method: 'RECREATE', max_surge_fixed: 0, max_unavailable_fixed: 1 }] }), create('google_compute_backend_service.relay_gce_cell["staging-gce-c4"]', { - timeout_sec: 86_400, connection_draining_timeout_sec: 300, + timeout_sec: RELAY_CELL_BACKEND_TIMEOUT_SECONDS, + connection_draining_timeout_sec: RELAY_CELL_CONNECTION_DRAIN_SECONDS, load_balancing_scheme: 'EXTERNAL_MANAGED', protocol: 'HTTP', port_name: 'relay', session_affinity: 'NONE', health_checks: ['projects/p/global/healthChecks/orca-cloud-staging-relay-gce-ready'], @@ -221,3 +227,31 @@ test('rejects incomplete NAT, backend, and URL routing shapes', () => { /no exact backend route/ ) }) + +// Terraform cannot export a local to JS, so this validator restates two topology values that +// the Asia workflow applies. Read the .tf source and equate all three statements of each: the +// local, the topology `check` assert that pins it, and the constant above. +test('the reviewed backend constants match the Terraform topology they validate', () => { + const terraform = readFileSync( + new URL('../../infra/terraform/relay-gce-cells.tf', import.meta.url), + 'utf8' + ) + const local = (name) => { + const found = new RegExp(`\\n\\s*${name}\\s*=\\s*(\\d+)\\n`).exec(terraform) + assert.notEqual(found, null, `relay_gce_topology has no ${name}`) + return Number(found[1]) + } + const asserted = (name) => { + const found = + new RegExp(`local\\.relay_gce_topology\\.${name}\\s*==\\s*(\\d+)`).exec(terraform) + assert.notEqual(found, null, `the topology check does not pin ${name}`) + return Number(found[1]) + } + for (const [name, constant] of [ + ['backend_timeout_seconds', RELAY_CELL_BACKEND_TIMEOUT_SECONDS], + ['connection_drain_seconds', RELAY_CELL_CONNECTION_DRAIN_SECONDS] + ]) { + assert.equal(local(name), constant, `${name} local differs from the validator constant`) + assert.equal(asserted(name), constant, `${name} check assert differs from the validator`) + } +}) diff --git a/cloud/infra/terraform/README.md b/cloud/infra/terraform/README.md index 8aa7aa0a95c..eb5f1699621 100644 --- a/cloud/infra/terraform/README.md +++ b/cloud/infra/terraform/README.md @@ -311,8 +311,9 @@ Every `relay_gce_cells` entry is one durable cell generation and must pin both i image and its Artifact Registry relay image. Terraform creates one private COS instance template, one size-one zonal MIG, and one backend service for that exact host. The MIG uses `RECREATE`, zero surge, and one unavailable worker; `/health` alone drives autoheal while SQL/JWKS-backed `/ready` controls LB -admission. The backend timeout is 86,400 seconds with connection draining, and the URL map aborts -unknown wildcard hosts before they reach a worker. The startup script obtains short-lived metadata +admission. The backend timeout is 86,400 seconds and connection draining is 60 seconds, which +covers only a host still mid-handshake because the rollout drains a cell before Terraform runs. +The URL map aborts unknown wildcard hosts before they reach a worker. The startup script obtains short-lived metadata credentials, fetches the two relay secrets without logging them, and runs a digest-pinned Cloud SQL Auth Proxy beside the digest-pinned relay image. diff --git a/cloud/infra/terraform/relay-gce-cells.tf b/cloud/infra/terraform/relay-gce-cells.tf index 5023b7e1d50..8a505c8f769 100644 --- a/cloud/infra/terraform/relay-gce-cells.tf +++ b/cloud/infra/terraform/relay-gce-cells.tf @@ -16,7 +16,7 @@ locals { backend_group_count = 1 public_access_config_count = 0 backend_timeout_seconds = 86400 - connection_drain_seconds = 300 + connection_drain_seconds = 60 } relay_gce_cell_urls = { for cell_id, cell in var.relay_gce_cells : @@ -108,9 +108,12 @@ check "relay_gce_fixed_one_topology" { local.relay_gce_topology.max_unavailable == 1 && local.relay_gce_topology.backend_group_count == 1 && local.relay_gce_topology.public_access_config_count == 0 && - local.relay_gce_topology.backend_timeout_seconds == 86400 + local.relay_gce_topology.backend_timeout_seconds == 86400 && + # A rollout drains every host off the cell before Terraform runs, so this covers only a + # host still mid-handshake; pinned so a raise cannot re-add rollout wall clock unseen. + local.relay_gce_topology.connection_drain_seconds == 60 ) - error_message = "Relay cells require fixed-one RECREATE MIGs, one non-public backend, and the 86,400-second WebSocket timeout." + error_message = "Relay cells require fixed-one RECREATE MIGs, one non-public backend, the 86,400-second WebSocket timeout, and the 60-second connection drain." } }