diff --git a/.github/workflows/cloud-deploy-relay-asia-topology.yml b/.github/workflows/cloud-deploy-relay-asia-topology.yml index 5f594852eb5..29c0e43f737 100644 --- a/.github/workflows/cloud-deploy-relay-asia-topology.yml +++ b/.github/workflows/cloud-deploy-relay-asia-topology.yml @@ -76,6 +76,7 @@ jobs: staging:staging-gce-c4) ;; production:production-gce-c27,production-gce-c28,production-gce-c29) ;; production:production-gce-c30) ;; + production:production-gce-c31) ;; *) echo "cell-ids do not match the reviewed environment topology" >&2; exit 1 ;; esac [[ "${TARGET_IMAGE}" =~ ^us-central1-docker\.pkg\.dev/${GCP_PROJECT_ID}/orca-cloud/relay@sha256:[0-9a-f]{64}$ ]] diff --git a/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml b/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml index 65873a6214a..14dc9940c1c 100644 --- a/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml +++ b/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml @@ -278,7 +278,7 @@ jobs: EXPECTED_REGION=us-central1 EXPECTED_DATABASE_POOL_MAX= ;; - c27|c28|c29|c30) + c27|c28|c29|c30|c31) EXPECTED_HARD_CAP=3000 EXPECTED_REGION=asia-east2 EXPECTED_DATABASE_POOL_MAX=16 diff --git a/.github/workflows/cloud-operate-relay-asia-admission.yml b/.github/workflows/cloud-operate-relay-asia-admission.yml index 2c7c289da26..d8cea37a6d8 100644 --- a/.github/workflows/cloud-operate-relay-asia-admission.yml +++ b/.github/workflows/cloud-operate-relay-asia-admission.yml @@ -39,7 +39,7 @@ on: required: false type: string evidence-run-id: - description: Staging evidence run ID for C27, C27 canary run ID for C28/C29; C30 takes none and proves itself by its own canary + description: Staging evidence run ID for C27, C27 canary run ID for C28/C29; C30/C31 take none and each proves itself by its own canary required: false type: string evidence-run-attempt: @@ -155,9 +155,9 @@ jobs: evidence_kind=c27 artifact_name="relay-asia-c27-canary-${EVIDENCE_RUN_ID}-${EVIDENCE_RUN_ATTEMPT}" ;; - production-gce-c30) - # No earlier proof binds C30's generation; its own canary below rolls it back on failure. - canary_cell=production-gce-c30 + production-gce-c30|production-gce-c31) + # No earlier proof binds a later cell's generation; its own canary below rolls it back on failure. + canary_cell="${TARGET_CELL_IDS}" ;; *) echo "production promotion wave is not reviewed" >&2; exit 1 ;; esac @@ -317,15 +317,15 @@ jobs: shell: bash run: | set -euo pipefail - # C30's launch cells were checked general by the promotion; verify reads only C30's digest. + # A later cell's launch cells were checked general by the promotion; verify reads only its digest. case "${CANARY_CELL}" in production-gce-c27) verify_cells=production-gce-c27,production-gce-c28,production-gce-c29 expected_states='{"production-gce-c27":"general","production-gce-c28":"migration-only","production-gce-c29":"migration-only"}' ;; - production-gce-c30) - verify_cells=production-gce-c30 - expected_states='{"production-gce-c30":"general"}' + production-gce-c30|production-gce-c31) + verify_cells="${CANARY_CELL}" + expected_states="{\"${CANARY_CELL}\":\"general\"}" ;; *) exit 1 ;; esac diff --git a/cloud/dev/scripts/operate-relay-asia-admission.mjs b/cloud/dev/scripts/operate-relay-asia-admission.mjs index b02b7c04cda..fea4f073b70 100644 --- a/cloud/dev/scripts/operate-relay-asia-admission.mjs +++ b/cloud/dev/scripts/operate-relay-asia-admission.mjs @@ -19,16 +19,21 @@ const SHAPES = { production: { directorOrigin: 'https://relay.onorca.dev', domain: 'relay.onorca.dev', - allCells: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29', 'production-gce-c30'], + allCells: [ + 'production-gce-c27', 'production-gce-c28', 'production-gce-c29', 'production-gce-c30', + 'production-gce-c31' + ], // The launch set was registered together; each later cell registers alone beside it. registrationWaves: [ ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'], - ['production-gce-c30'] + ['production-gce-c30'], + ['production-gce-c31'] ], promotionWaves: [ ['production-gce-c27'], ['production-gce-c28', 'production-gce-c29'], - ['production-gce-c30'] + ['production-gce-c30'], + ['production-gce-c31'] ] } } diff --git a/cloud/dev/scripts/operate-relay-asia-admission.test.mjs b/cloud/dev/scripts/operate-relay-asia-admission.test.mjs index 0a84a3ca20d..2465f970cae 100644 --- a/cloud/dev/scripts/operate-relay-asia-admission.test.mjs +++ b/cloud/dev/scripts/operate-relay-asia-admission.test.mjs @@ -526,23 +526,30 @@ function admissionArguments(environment, mode, cellIds) { test('accepts only reviewed Asia admission waves', () => { const accepted = [ ['inspect', 'production-gce-c27,production-gce-c28,production-gce-c29'], - ['inspect', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30'], + ['inspect', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31'], ['inspect', 'production-gce-c30'], + ['inspect', 'production-gce-c31'], ['verify', 'production-gce-c27,production-gce-c28,production-gce-c29'], ['verify', 'production-gce-c30'], + ['verify', 'production-gce-c31'], ['initialize', 'production-gce-c27,production-gce-c28,production-gce-c29'], ['register', 'production-gce-c27,production-gce-c28,production-gce-c29'], ['register', 'production-gce-c30'], ['registered', 'production-gce-c30'], + ['register', 'production-gce-c31'], + ['registered', 'production-gce-c31'], ['promote', 'production-gce-c27'], ['promote', 'production-gce-c28,production-gce-c29'], ['promote', 'production-gce-c30'], ['recover-promotion', 'production-gce-c30'], + ['promote', 'production-gce-c31'], + ['recover-promotion', 'production-gce-c31'], ['rollback', 'production-gce-c27'], ['rollback', 'production-gce-c28,production-gce-c29'], ['rollback', 'production-gce-c30'], + ['rollback', 'production-gce-c31'], ['rollback', 'production-gce-c27,production-gce-c28,production-gce-c29'], - ['rollback', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30'] + ['rollback', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31'] ] for (const [mode, cellIds] of accepted) { assert.deepEqual( @@ -553,15 +560,21 @@ test('accepts only reviewed Asia admission waves', () => { } const rejected = [ ['initialize', 'production-gce-c30'], - ['initialize', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30'], - ['register', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30'], + ['initialize', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31'], + ['register', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31'], + ['register', 'production-gce-c30,production-gce-c31'], ['register', 'production-gce-c29,production-gce-c30'], - ['registered', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30'], + ['registered', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31'], + ['inspect', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30'], + ['verify', 'production-gce-c30,production-gce-c31'], ['verify', 'production-gce-c27,production-gce-c30'], ['promote', 'production-gce-c27,production-gce-c30'], ['promote', 'production-gce-c28,production-gce-c29,production-gce-c30'], - ['promote', 'production-gce-c31'], + ['promote', 'production-gce-c30,production-gce-c31'], + ['promote', 'production-gce-c32'], ['rollback', 'production-gce-c27,production-gce-c30'], + ['rollback', 'production-gce-c30,production-gce-c31'], + ['rollback', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30'], ['rollback', 'production-gce-c28,production-gce-c29,production-gce-c30'], ['rollback', 'production-gce-c29'], ['register', 'staging-gce-c4'] @@ -601,6 +614,25 @@ test('registers C30 alone beside the general launch cells', async () => { assert.deepEqual(subject.selector().membership.general, launchCells) }) +test('registers C31 alone beside the general C27-C30', async () => { + const general = [...launchCells, 'production-gce-c30'] + const subject = harness({ + generation: 11, + membership: { existingOnly: [], migrationOnly: [], general: [...general] } + }) + const result = await operateRelayAsiaAdmission({ + environment: 'production', mode: 'register', cells: ['production-gce-c31'], + expectedGeneration: 11, imageDigest: digest, attemptId: 'asia_register_c31', token: 'not-logged' + }, subject) + const request = subject.requests.find(({ path }) => path.endsWith('/add-migration-cells')) + assert.deepEqual(request.body.cells, [{ + cellId: 'production-gce-c31', cellUrl: 'https://c31.relay.onorca.dev', region: 'asia-east2', + capacityRequests: 6_000, connectionHardCap: 3_000, connectionUnobservedBound: 60 + }]) + assert.deepEqual(result.states, { 'production-gce-c31': 'migration-only' }) + assert.deepEqual(subject.selector().membership.general, general) +}) + test('requires the C27 canary to be general before promoting C30', async () => { const selector = (general) => ({ generation: 10, diff --git a/cloud/dev/scripts/prepare-relay-asia-topology-input.mjs b/cloud/dev/scripts/prepare-relay-asia-topology-input.mjs index 1199b3df377..19d14e4de6c 100644 --- a/cloud/dev/scripts/prepare-relay-asia-topology-input.mjs +++ b/cloud/dev/scripts/prepare-relay-asia-topology-input.mjs @@ -16,12 +16,14 @@ const SHAPES = { 'production-gce-c27': 'asia-east2-a', 'production-gce-c28': 'asia-east2-b', 'production-gce-c29': 'asia-east2-c', - 'production-gce-c30': 'asia-east2-a' + 'production-gce-c30': 'asia-east2-a', + 'production-gce-c31': 'asia-east2-b' }, // The launch set, then each later additive cell; a plan targets one wave, never live cells. waves: [ ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'], - ['production-gce-c30'] + ['production-gce-c30'], + ['production-gce-c31'] ] } } diff --git a/cloud/dev/scripts/prepare-relay-asia-topology-input.test.mjs b/cloud/dev/scripts/prepare-relay-asia-topology-input.test.mjs index 6cfd04eaf94..1e911ae3483 100644 --- a/cloud/dev/scripts/prepare-relay-asia-topology-input.test.mjs +++ b/cloud/dev/scripts/prepare-relay-asia-topology-input.test.mjs @@ -11,7 +11,8 @@ const productionCells = () => Object.fromEntries([ [27, 'asia-east2-a'], [28, 'asia-east2-b'], [29, 'asia-east2-c'], - [30, 'asia-east2-a'] + [30, 'asia-east2-a'], + [31, 'asia-east2-b'] ].map(([ordinal, zone]) => [`production-gce-c${ordinal}`, { hostname: `c${ordinal}`, region: 'asia-east2', zone, machine_type: 'e2-standard-4', boot_disk_gb: 30, @@ -46,6 +47,13 @@ test('accepts the additive C30 wave without re-planning the launch cells', () => assert.equal(result.relay_gce_cells['production-gce-c30'].zone, 'asia-east2-a') }) +test('accepts the additive C31 wave in the next zone of the rotation', () => { + const result = prepareRelayAsiaTopologyInput({ existingCells: productionCells(), + existingAdditionalRegions: additionalRegions, environment: 'production', + cellIds: 'production-gce-c31', image }) + assert.equal(result.relay_gce_cells['production-gce-c31'].zone, 'asia-east2-b') +}) + // Reads the committed file so a reviewed-shape constant cannot drift from what the plan reads. test('matches every committed production Asia cell entry', () => { const tfvars = readFileSync( @@ -64,10 +72,11 @@ test('matches every committed production Asia cell entry', () => { } committed[cellId] = cell } - // Each wave is pinned on its own: C30 launches on the director's digest, not C27's. + // Each wave is pinned on its own: C30 and C31 launch on the director's digest, not C27's. for (const wave of [ 'production-gce-c27,production-gce-c28,production-gce-c29', - 'production-gce-c30' + 'production-gce-c30', + 'production-gce-c31' ]) { const committedImage = committed[wave.split(',')[0]].image assert.doesNotThrow(() => prepareRelayAsiaTopologyInput({ @@ -109,7 +118,8 @@ test('rejects an uncommitted subnet or cell, partial wave, wrong image, and drif 'production-gce-c27,production-gce-c30', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30', 'production-gce-c30,production-gce-c30', - 'production-gce-c31' + 'production-gce-c30,production-gce-c31', + 'production-gce-c32' ]) { assert.throws(() => prepareRelayAsiaTopologyInput({ existingCells: productionCells(), existingAdditionalRegions: additionalRegions, diff --git a/cloud/dev/scripts/prepare-relay-production-capacity-canary.test.mjs b/cloud/dev/scripts/prepare-relay-production-capacity-canary.test.mjs index ee30b6340fa..4822a71223f 100644 --- a/cloud/dev/scripts/prepare-relay-production-capacity-canary.test.mjs +++ b/cloud/dev/scripts/prepare-relay-production-capacity-canary.test.mjs @@ -98,7 +98,7 @@ describe('production Relay capacity cell admission', () => { for (const cellId of [ 'production-gce-c27', 'production-gce-c28', 'production-gce-c29', 'production-gce-c30', // Migration-only canaries: the US-only capacity rollout never touches them either. - 'production-gce-c17', 'production-gce-c18' + 'production-gce-c17', 'production-gce-c18', 'production-gce-c31' ]) { const hostname = cellId.slice('production-gce-'.length) assert.deepEqual(parseProductionCapacityCellArguments([ @@ -115,7 +115,7 @@ describe('production Relay capacity cell admission', () => { paceWindowMs: 0 }) } - for (const cellId of ['production-gce-c12', 'production-gce-c31']) { + for (const cellId of ['production-gce-c12', 'production-gce-c32']) { const hostname = cellId.slice('production-gce-'.length) assert.throws(() => parseProductionCapacityCellArguments([ '--director-origin', 'https://relay.onorca.dev', diff --git a/cloud/dev/scripts/probe-relay-rehome-trust.mjs b/cloud/dev/scripts/probe-relay-rehome-trust.mjs index 7fef67cd237..a3498690289 100644 --- a/cloud/dev/scripts/probe-relay-rehome-trust.mjs +++ b/cloud/dev/scripts/probe-relay-rehome-trust.mjs @@ -2,8 +2,8 @@ import { pathToFileURL } from 'node:url' import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs' // Every cell that carries the rehome identity: the sixteen US cells and the -// four asia-east2 cells that drain mis-homed hosts back the other way. -const PRODUCTION_CELL = /^production-gce-c(?:7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26|27|28|29|30)$/ +// five asia-east2 cells that drain mis-homed hosts back the other way. +const PRODUCTION_CELL = /^production-gce-c(?:7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26|27|28|29|30|31)$/ const DIRECTOR_ORIGIN = 'https://relay.onorca.dev' export function parseRehomeTrustProbeArguments(argv, environment = process.env) { diff --git a/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs b/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs index f7c9696aeeb..63287a8405b 100644 --- a/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs +++ b/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs @@ -114,7 +114,8 @@ test('fails when both trust-probe attempts return a transient 503', async () => test('approves the asia-east2 rehome sources and still rejects unlisted cells', () => { for (const cellId of [ - 'production-gce-c27', 'production-gce-c28', 'production-gce-c29', 'production-gce-c30' + 'production-gce-c27', 'production-gce-c28', 'production-gce-c29', 'production-gce-c30', + 'production-gce-c31' ]) { const parsed = parseRehomeTrustProbeArguments( argv.map((value) => (value === 'production-gce-c7' ? cellId : value)), @@ -122,7 +123,7 @@ test('approves the asia-east2 rehome sources and still rejects unlisted cells', ) assert.equal(parsed.cellId, cellId) } - for (const cellId of ['production-gce-c1', 'production-gce-c17', 'production-gce-c31']) { + for (const cellId of ['production-gce-c1', 'production-gce-c17', 'production-gce-c32']) { assert.throws( () => parseRehomeTrustProbeArguments( diff --git a/cloud/dev/scripts/relay-asia-rollout-evidence.mjs b/cloud/dev/scripts/relay-asia-rollout-evidence.mjs index cf7d53c52bc..44bab9c19cf 100644 --- a/cloud/dev/scripts/relay-asia-rollout-evidence.mjs +++ b/cloud/dev/scripts/relay-asia-rollout-evidence.mjs @@ -11,7 +11,8 @@ const C27 = 'production-gce-c27' // Each canary proves its own cell under production load; C28/C29 promotion consumes only C27's. const PRODUCTION_CANARIES = { [C27]: { kind: 'production-c27-canary', origin: 'https://c27.relay.onorca.dev' }, - 'production-gce-c30': { kind: 'production-c30-canary', origin: 'https://c30.relay.onorca.dev' } + 'production-gce-c30': { kind: 'production-c30-canary', origin: 'https://c30.relay.onorca.dev' }, + 'production-gce-c31': { kind: 'production-c31-canary', origin: 'https://c31.relay.onorca.dev' } } const DIGEST_PATTERN = /^sha256:[a-f0-9]{64}$/ const SHA_PATTERN = /^[a-f0-9]{40}$/ diff --git a/cloud/dev/scripts/relay-asia-rollout-evidence.test.mjs b/cloud/dev/scripts/relay-asia-rollout-evidence.test.mjs index 1da4c962879..9b845e38d4c 100644 --- a/cloud/dev/scripts/relay-asia-rollout-evidence.test.mjs +++ b/cloud/dev/scripts/relay-asia-rollout-evidence.test.mjs @@ -389,6 +389,19 @@ test('builds and verifies a C30 canary from C30 runtime metrics and placement', ), /evidence kind is invalid/) }) +test('builds a C31 canary that only C31 placement satisfies', () => { + const c31 = 'production-gce-c31' + const evidence = buildProductionCanaryEvidence(canaryInput({}, c31)) + assert.equal(evidence.kind, 'production-c31-canary') + assert.equal(verifyRolloutEvidence( + evidence, workflowRun(evidence), + verifyExpected('production-c31-canary', { cellIds: [c31], selectorGeneration: 9 }) + ), evidence) + const onC30 = canaryInput({}, c31) + onC30.loadReport.assignedCellOrigins = ['https://c30.relay.onorca.dev'] + assert.throws(() => buildProductionCanaryEvidence(onC30), /C31 canary load was not placed only on C31/) +}) + test('rejects a C30 canary that C30 did not serve', () => { const onLaunchCell = canaryInput({}, c30) onLaunchCell.loadReport.assignedCellOrigins = ['https://c27.relay.onorca.dev'] diff --git a/cloud/dev/scripts/relay-asia-topology-workflow.test.mjs b/cloud/dev/scripts/relay-asia-topology-workflow.test.mjs index d8965f02869..2e594aae3e4 100644 --- a/cloud/dev/scripts/relay-asia-topology-workflow.test.mjs +++ b/cloud/dev/scripts/relay-asia-topology-workflow.test.mjs @@ -44,7 +44,8 @@ test('accepts only the reviewed Asia topology waves', () => { [ 'staging:staging-gce-c4', 'production:production-gce-c27,production-gce-c28,production-gce-c29', - 'production:production-gce-c30' + 'production:production-gce-c30', + 'production:production-gce-c31' ] ) }) diff --git a/cloud/dev/scripts/relay-cloud-sql-connection-budget.test.mjs b/cloud/dev/scripts/relay-cloud-sql-connection-budget.test.mjs index 333969e5684..3c6d1bed172 100644 --- a/cloud/dev/scripts/relay-cloud-sql-connection-budget.test.mjs +++ b/cloud/dev/scripts/relay-cloud-sql-connection-budget.test.mjs @@ -7,12 +7,12 @@ import { } from './relay-cloud-sql-connection-budget.mjs' test('production shared consumers keep allowance and reserve below the ceiling', () => { - // cells: 20 pools at 10 (200) + the four asia-east2 pools at 16 (64). + // cells: 20 pools at 10 (200) + the five asia-east2 pools at 16 (80). const report = readRelayCloudSqlConnectionBudget() - assert.deepEqual(report.consumers, { cells: 264, directors: 15, auth: 20, api: 50 }) - assert.deepEqual(report.asia, { cells: 4, poolMax: 16 }) - assert.equal(report.configuredMaximum, 349) + assert.deepEqual(report.consumers, { cells: 280, directors: 15, auth: 20, api: 50 }) + assert.deepEqual(report.asia, { cells: 5, poolMax: 16 }) + assert.equal(report.configuredMaximum, 365) assert.equal(report.rolloutOverlap.relayDirectorCandidate, 30) assert.equal(report.rolloutOverlap.apiCandidate, 65) assert.equal(report.rolloutOverlap.authCandidate, 35) @@ -22,10 +22,10 @@ test('production shared consumers keep allowance and reserve below the ceiling', assert.equal(report.maintenanceAdminAllowance, 5) assert.equal(report.explicitReserve, 10) assert.equal(report.usableCeiling, 490) - assert.equal(report.operatingMaximum, 419) - assert.equal(report.remainingWithinUsableCeiling, 71) - assert.equal(report.budgetedTotal, 429) - assert.equal(report.unallocated, 71) + assert.equal(report.operatingMaximum, 435) + assert.equal(report.remainingWithinUsableCeiling, 55) + assert.equal(report.budgetedTotal, 445) + assert.equal(report.unallocated, 55) assert.equal(report.withinBudget, true) }) diff --git a/cloud/dev/scripts/relay-production-same-cap-wave.mjs b/cloud/dev/scripts/relay-production-same-cap-wave.mjs index 212f3f1d16d..54b523fd75c 100644 --- a/cloud/dev/scripts/relay-production-same-cap-wave.mjs +++ b/cloud/dev/scripts/relay-production-same-cap-wave.mjs @@ -4,7 +4,10 @@ import { requireSameEvidenceCode } from './relay-evidence-code-provenance.mjs' // Migration-only by policy: zero hosts and no reservation, so a wave rolls one without // displacing anybody. It enters and must leave migration-only, never general. -export const SAME_CAP_MIGRATION_ONLY_CELLS = ['production-gce-c17', 'production-gce-c18'] +// C31 stays here until its Asia canary promotes it; that follow-up moves it to the general list. +export const SAME_CAP_MIGRATION_ONLY_CELLS = [ + 'production-gce-c17', 'production-gce-c18', 'production-gce-c31' +] export const SAME_CAP_CELLS = [ 'production-gce-c7', 'production-gce-c8', 'production-gce-c9', 'production-gce-c10', diff --git a/cloud/dev/scripts/relay-production-same-cap-wave.test.mjs b/cloud/dev/scripts/relay-production-same-cap-wave.test.mjs index 8a3cfc240ef..9494bb985da 100644 --- a/cloud/dev/scripts/relay-production-same-cap-wave.test.mjs +++ b/cloud/dev/scripts/relay-production-same-cap-wave.test.mjs @@ -56,12 +56,19 @@ test('requires one canary or a bounded reviewed batch', () => { rollbackDigest, confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c30` }).cells, ['production-gce-c30']) - assert.throws(() => validateSameCapWave({ + assert.deepEqual(validateSameCapWave({ mode: 'canary-apply', cellIds: 'production-gce-c31', targetDigest, rollbackDigest, confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c31` + }).cells, ['production-gce-c31']) + assert.throws(() => validateSameCapWave({ + mode: 'canary-apply', + cellIds: 'production-gce-c32', + targetDigest, + rollbackDigest, + confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c32` }), /cells/) }) @@ -107,8 +114,11 @@ test('the wave workflow chains exactly ten serial cell jobs', () => { assert.doesNotMatch(dispatch, /\n cell_11:/) }) -test('lists only C17 and C18 as migration-only now that C30 is promoted', () => { - assert.deepEqual(SAME_CAP_MIGRATION_ONLY_CELLS, ['production-gce-c17', 'production-gce-c18']) +test('lists C31 as migration-only beside C17 and C18 until its canary promotes it', () => { + assert.deepEqual( + SAME_CAP_MIGRATION_ONLY_CELLS, + ['production-gce-c17', 'production-gce-c18', 'production-gce-c31'] + ) assert.equal(SAME_CAP_CELLS.includes('production-gce-c30'), true) }) @@ -154,6 +164,17 @@ test('rolls the migration-only cells but never mixes the two classes in one wave confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${asiaMixed}`, canaryRunId: '42' }), /all general or all migration-only/) + // Until its canary promotes it, a same-cap restore must hand C31 back isolated, never activated. + assert.equal(entryAdmission('production-gce-c31'), 'migration-only') + const asiaUnpromoted = 'production-gce-c30,production-gce-c31' + assert.throws(() => validateSameCapWave({ + mode: 'batch-apply', + cellIds: asiaUnpromoted, + targetDigest, + rollbackDigest, + confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${asiaUnpromoted}`, + canaryRunId: '42' + }), /all general or all migration-only/) // A mixed wave has no single selector delta for its later cells to offset from. const mixed = 'production-gce-c7,production-gce-c17' assert.throws(() => validateSameCapWave({ diff --git a/cloud/dev/scripts/relay-same-cap-script-census.test.mjs b/cloud/dev/scripts/relay-same-cap-script-census.test.mjs index ff2c265ee72..3ae4888c6d1 100644 --- a/cloud/dev/scripts/relay-same-cap-script-census.test.mjs +++ b/cloud/dev/scripts/relay-same-cap-script-census.test.mjs @@ -291,7 +291,7 @@ describe('same-cap roll scripts accept every same-cap cell', () => { assert.equal(String(cellShape(cellId).cap), tfvarsHardCap(cellId), cellId) } assert.equal(resolveCellShape('production-gce-c12').status, 1) - assert.equal(resolveCellShape('production-gce-c31').status, 1) + assert.equal(resolveCellShape('production-gce-c32').status, 1) }) @@ -303,9 +303,10 @@ describe('same-cap roll scripts accept every same-cap cell', () => { const trusted = SAME_CAP_CELLS.filter((cell) => REHOME_SOURCE_CELLS.has(cell)) // Only a declared rehome source may roll at a trusted protocol at all; the job refuses // the rest before it plans, and the next test covers them at protocol 0. + // C31 is already a rehome source but stays migration-only until its Asia canary promotes it. assert.deepEqual( SAME_CAP_CELLS.filter((cell) => !REHOME_SOURCE_CELLS.has(cell)), - SAME_CAP_MIGRATION_ONLY_CELLS + SAME_CAP_MIGRATION_ONLY_CELLS.filter((cell) => cell !== 'production-gce-c31') ) for (const [cellId, protocol] of trusted.flatMap((cell) => [[cell, 1], [cell, 3]])) { const { cap, pool } = cellShape(cellId) diff --git a/cloud/dev/scripts/relay-staging-c4-refresh-workflow.test.mjs b/cloud/dev/scripts/relay-staging-c4-refresh-workflow.test.mjs index 67f30aa6d2b..67225f4d869 100644 --- a/cloud/dev/scripts/relay-staging-c4-refresh-workflow.test.mjs +++ b/cloud/dev/scripts/relay-staging-c4-refresh-workflow.test.mjs @@ -40,6 +40,8 @@ const launchDigest = '5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70ca const asiaCells = ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'] // C30 launches after the launch cells rolled, so it pins the director's digest instead. const c30Digest = '4158d8a2e18e9caec439d257f0c1e45d92ffea8c0262f057b2f08c76a134bcf0' +// C31 launches on the digest the director served when it was declared. +const c31Digest = 'f30b5cb1ec52b6b6145efecfa1b8be9e3d309403beffd8abcc64197a2087e269' function cellBlock(tfvars, cellId) { const start = tfvars.indexOf(`"${cellId}"`) @@ -50,13 +52,14 @@ function cellBlock(tfvars, cellId) { const productionCell = (cellId) => cellBlock(productionTfvars, cellId) // Scoped to C4 by name: staging C3 serves this digest too since its 2026-09-03 re-pin. -test('pins staging C4 and the launch Asia cells to one image, and C30 to the director image', () => { +test('pins staging C4 and the launch Asia cells to one image, and C30/C31 to director images', () => { assert.match(cellBlock(stagingTfvars, 'staging-gce-c4'), new RegExp(`relay@sha256:${launchDigest}"`)) for (const cellId of asiaCells) { assert.match(productionCell(cellId), new RegExp(`relay@sha256:${launchDigest}"`), cellId) } assert.match(recoveryWorkflow, new RegExp(`TARGET_IMAGE_DIGEST: sha256:${launchDigest}`)) assert.match(productionCell('production-gce-c30'), new RegExp(`relay@sha256:${c30Digest}"`)) + assert.match(productionCell('production-gce-c31'), new RegExp(`relay@sha256:${c31Digest}"`)) }) test('refreshes only empty staging C4 through the trusted capacity identity', () => { diff --git a/cloud/dev/scripts/validate-relay-asia-topology-plan.mjs b/cloud/dev/scripts/validate-relay-asia-topology-plan.mjs index 0a702771852..c41ce736a23 100644 --- a/cloud/dev/scripts/validate-relay-asia-topology-plan.mjs +++ b/cloud/dev/scripts/validate-relay-asia-topology-plan.mjs @@ -19,11 +19,13 @@ const CELL_SHAPES = { 'production-gce-c27': 'asia-east2-a', 'production-gce-c28': 'asia-east2-b', 'production-gce-c29': 'asia-east2-c', - 'production-gce-c30': 'asia-east2-a' + 'production-gce-c30': 'asia-east2-a', + 'production-gce-c31': 'asia-east2-b' }, waves: [ ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'], - ['production-gce-c30'] + ['production-gce-c30'], + ['production-gce-c31'] ] }, staging: { diff --git a/cloud/dev/scripts/validate-relay-asia-topology-plan.test.mjs b/cloud/dev/scripts/validate-relay-asia-topology-plan.test.mjs index 77071a94fce..82692b9d537 100644 --- a/cloud/dev/scripts/validate-relay-asia-topology-plan.test.mjs +++ b/cloud/dev/scripts/validate-relay-asia-topology-plan.test.mjs @@ -96,15 +96,16 @@ const productionConfig = { environment: 'production', cells: ['production-gce-c30'], image: productionImage } -// C30 joins a live Asia region: the network is a no-op and the existing C27 route is preserved. -function productionC30Plan() { +// A later cell joins a live Asia region: the network is a no-op and the existing C27 route is preserved. +function productionWavePlan(hostname = 'c30', zone = 'asia-east2-a') { const plan = JSON.parse(JSON.stringify(resources) .replaceAll('onorca-cloud-staging/', 'onorca-cloud/') .replaceAll('orca-cloud-staging-relay-gce', 'orca-cloud-relay-gce') - .replaceAll('staging-gce-c4', 'production-gce-c30') - .replaceAll('relay-gce-c4', 'relay-gce-c30') - .replaceAll('cell-c4', 'cell-c30') - .replaceAll('c4.relay-staging.onorca.dev', 'c30.relay.onorca.dev') + .replaceAll('staging-gce-c4', `production-gce-${hostname}`) + .replaceAll('relay-gce-c4', `relay-gce-${hostname}`) + .replaceAll('cell-c4', `cell-${hostname}`) + .replaceAll('c4.relay-staging.onorca.dev', `${hostname}.relay.onorca.dev`) + .replaceAll('asia-east2-a', zone) .replaceAll("'10'", "'16'")) for (const network of plan.slice(0, 3)) { network.change.actions = ['no-op'] @@ -124,23 +125,23 @@ function productionC30Plan() { test('accepts the additive production C30 wave at the 16-connection Asia pool', () => { assert.deepEqual( - validateRelayAsiaTopologyPlan({ resource_changes: productionC30Plan() }, productionConfig), + validateRelayAsiaTopologyPlan({ resource_changes: productionWavePlan() }, productionConfig), { environment: 'production', cells: ['production-gce-c30'], changes: 4 } ) - const staleShape = productionC30Plan() + const staleShape = productionWavePlan() staleShape[3].change.after.metadata_startup_script = staleShape[3].change.after.metadata_startup_script.replace("'16'", "'10'") assert.throws( () => validateRelayAsiaTopologyPlan({ resource_changes: staleShape }, productionConfig), /reviewed Asia cell shape/ ) - const wrongZone = productionC30Plan() + const wrongZone = productionWavePlan() wrongZone[4].change.after.zone = 'asia-east2-b' assert.throws( () => validateRelayAsiaTopologyPlan({ resource_changes: wrongZone }, productionConfig), /fixed-one Asia MIG shape/ ) - const liveCellTouched = productionC30Plan() + const liveCellTouched = productionWavePlan() liveCellTouched.push(create('google_compute_instance_template.relay_gce_cell["production-gce-c27"]')) assert.throws( () => validateRelayAsiaTopologyPlan({ resource_changes: liveCellTouched }, productionConfig), @@ -148,13 +149,25 @@ test('accepts the additive production C30 wave at the 16-connection Asia pool', ) }) +test('accepts the additive production C31 wave only in asia-east2-b', () => { + const c31Config = { ...productionConfig, cells: ['production-gce-c31'] } + assert.deepEqual( + validateRelayAsiaTopologyPlan({ resource_changes: productionWavePlan('c31', 'asia-east2-b') }, c31Config), + { environment: 'production', cells: ['production-gce-c31'], changes: 4 } + ) + assert.throws( + () => validateRelayAsiaTopologyPlan({ resource_changes: productionWavePlan('c31') }, c31Config), + /fixed-one Asia MIG shape/ + ) +}) + // The URL map pulls every cell's backend, MIG and template into a targeted plan. const liveCellResources = ['instance_template', 'instance_group_manager', 'backend_service'] .flatMap((kind) => ['production-gce-c1', 'production-gce-c27', 'production-gce-c28', 'production-gce-c29'] .map((cellId) => `google_compute_${kind}.relay_gce_cell["${cellId}"]`)) test('accepts live cells the URL map pulls in only while they stay unchanged', () => { - const plan = productionC30Plan() + const plan = productionWavePlan() for (const address of liveCellResources) plan.push({ address, change: { actions: ['no-op'] } }) assert.equal( validateRelayAsiaTopologyPlan({ resource_changes: plan }, productionConfig).changes, @@ -162,7 +175,7 @@ test('accepts live cells the URL map pulls in only while they stay unchanged', ( ) for (const address of liveCellResources) { for (const action of [['update'], ['delete'], ['create', 'delete'], ['delete', 'create']]) { - const drifted = productionC30Plan() + const drifted = productionWavePlan() drifted.push({ address, change: { actions: action } }) assert.throws( () => validateRelayAsiaTopologyPlan({ resource_changes: drifted }, productionConfig), @@ -180,7 +193,8 @@ test('accepts only a reviewed Asia topology wave', () => { for (const cellIds of [ 'production-gce-c27,production-gce-c28,production-gce-c29', 'production-gce-c29,production-gce-c27,production-gce-c28', - 'production-gce-c30' + 'production-gce-c30', + 'production-gce-c31' ]) { assert.doesNotThrow( () => parseRelayAsiaTopologyPlanArguments(argv('production', cellIds, productionImage)), @@ -192,7 +206,8 @@ test('accepts only a reviewed Asia topology wave', () => { 'production-gce-c27,production-gce-c30', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30', 'production-gce-c30,production-gce-c30', - 'production-gce-c31' + 'production-gce-c30,production-gce-c31', + 'production-gce-c32' ]) { assert.throws( () => parseRelayAsiaTopologyPlanArguments(argv('production', cellIds, productionImage)), diff --git a/cloud/docs/relay-workflows.md b/cloud/docs/relay-workflows.md index 31432bff8de..97450772903 100644 --- a/cloud/docs/relay-workflows.md +++ b/cloud/docs/relay-workflows.md @@ -166,11 +166,13 @@ atomically register the new cells as migration-only, binding every mutation to the exact live selector generation and a durable attempt ID. Deploy and verify the director configuration only after registration, then promote C27 alone before C28/C29. -The production Asia set is C27-C30. C27-C29 launched as one wave; C30 is an additive wave of its -own at the same shape. Its plan names C30's template, MIG, and backend plus the shared URL map, and -the URL map pulls every existing cell's backend, MIG, and template into the plan. Committed images -lag what same-cap rolls serve, so the workflow first reads each non-target cell's served image out -of its live template in state and plans that cell at it. It reads the committed cell map from a +The production Asia set is C27-C31. C27-C29 launched as one wave; C30 and C31 are each an additive +wave of their own at the same shape, and C31 takes `asia-east2-b` so the five cells spread 2/2/1 +across the zones. The C30 steps below apply to C31 unchanged, with C31 in place of C30. C30's plan +names its template, MIG, and backend plus the shared URL map, and the URL map pulls every existing +cell's backend, MIG, and template into the plan. Committed images lag what same-cap rolls serve, +so the workflow first reads each non-target cell's served image out of its live template in state +and plans that cell at it. It reads the committed cell map from a no-refresh, unlocked plan over the same targets, not `terraform console`. Console evaluates every output against state, so `relay_gce_cell_deployments` wraps each per-cell resource lookup in `try`: until C30's topology apply, console succeeds and that output shows C30 with null MIG, @@ -189,6 +191,8 @@ database-pool rules read C30's own metrics only. Director values are recorded un 2026-09-23, so the same-cap job now rolls it as a general cell and the shadow gate's fleet pool list reads it beside C27-C29. A later Asia cell stays in the same-cap migration-only list and out of the fleet pool list until its own promotion, then moves to both together, as its own reviewed wave. +C31 is in that state now: declared and listed as a same-cap migration-only cell, not yet in the +fleet pool list. Rollback returns Asia cells to migration-only; it does not destroy the network or use existing-only. The production topology dispatch remains unavailable until the diff --git a/cloud/infra/terraform/README.md b/cloud/infra/terraform/README.md index a41c609ddd5..f6fbe82eb46 100644 --- a/cloud/infra/terraform/README.md +++ b/cloud/infra/terraform/README.md @@ -332,7 +332,7 @@ cell templates/MIGs/backends, and exact shared URL-map host additions. It rejects deletes, replacements, loss of an existing host route, US-resource changes, and unrelated drift. Do not add production C27-C29 until the compatible image has been published and each entry can pin its immutable -digest. A later cell, such as C30, is its own reviewed wave. The shared URL map +digest. A later cell, such as C30 or C31, is its own reviewed wave. The shared URL map pulls every live cell into its plan, so the workflow plans each live cell at the image its state template already serves, and the validator rejects any change to a cell outside the wave. diff --git a/cloud/infra/terraform/environments/production.tfvars b/cloud/infra/terraform/environments/production.tfvars index e8f2f47485f..a2394260f01 100644 --- a/cloud/infra/terraform/environments/production.tfvars +++ b/cloud/infra/terraform/environments/production.tfvars @@ -407,6 +407,20 @@ relay_gce_cells = { connection_hard_cap = 3000 connection_unobserved_bound = 60 } + "production-gce-c31" = { + hostname = "c31" + region = "asia-east2" + zone = "asia-east2-b" + machine_type = "e2-standard-4" + boot_disk_gb = 30 + boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" + capacity_requests = 6000 + database_pool_max = 16 # 176 ms from us-central1 Postgres saturates 10 (94-156 waiters). + image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:f30b5cb1ec52b6b6145efecfa1b8be9e3d309403beffd8abcc64197a2087e269" + initially_enabled = false + connection_hard_cap = 3000 + connection_unobserved_bound = 60 + } } relay_region_rehome_source_cell_ids = [ @@ -430,7 +444,8 @@ relay_region_rehome_source_cell_ids = [ "production-gce-c27", "production-gce-c28", "production-gce-c29", - "production-gce-c30" + "production-gce-c30", + "production-gce-c31" ] # Slack #orca-relay-alerts, created out of band on 2026-08-05. Declared here because an apply