From 2365eea5edfdbd6ef5fe3e93c2b09a89cdb6b306 Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Mon, 12 Oct 2026 00:01:19 -0400 Subject: [PATCH] chore(relay): move Asia cell c35 to the general same-cap lists after promotion (#27761) --- ...-relay-production-capacity-canary.test.mjs | 4 ++-- .../relay-production-same-cap-wave.mjs | 6 ++--- .../relay-production-same-cap-wave.test.mjs | 24 +++++++++++++------ .../relay-same-cap-script-census.test.mjs | 3 +-- .../relay-same-cap-shadow-gate-verdict.mjs | 3 ++- .../relay-same-cap-shadow-gate.test.mjs | 6 +++-- cloud/docs/relay-workflows.md | 21 +++++++--------- .../terraform/environments/production.tfvars | 2 +- 8 files changed, 37 insertions(+), 32 deletions(-) diff --git a/cloud/dev/scripts/prepare-relay-production-capacity-canary.test.mjs b/cloud/dev/scripts/prepare-relay-production-capacity-canary.test.mjs index 9745c241544..80866fef433 100644 --- a/cloud/dev/scripts/prepare-relay-production-capacity-canary.test.mjs +++ b/cloud/dev/scripts/prepare-relay-production-capacity-canary.test.mjs @@ -97,9 +97,9 @@ describe('production Relay capacity cell admission', () => { it('admits the same-cap Asia and migration-only cells only under the same-cap allowlist', () => { for (const cellId of [ 'production-gce-c27', 'production-gce-c28', 'production-gce-c29', 'production-gce-c30', - 'production-gce-c31', 'production-gce-c34', + 'production-gce-c31', 'production-gce-c34', 'production-gce-c35', // Migration-only canaries: the US-only capacity rollout never touches them either. - 'production-gce-c17', 'production-gce-c18', 'production-gce-c35', + 'production-gce-c17', 'production-gce-c18', // US cells at the Asia shape: the 1,000-cap capacity rollout never touches them. 'production-gce-c32', 'production-gce-c33' ]) { diff --git a/cloud/dev/scripts/relay-production-same-cap-wave.mjs b/cloud/dev/scripts/relay-production-same-cap-wave.mjs index ce67ed0e62e..3389910ccfc 100644 --- a/cloud/dev/scripts/relay-production-same-cap-wave.mjs +++ b/cloud/dev/scripts/relay-production-same-cap-wave.mjs @@ -4,10 +4,7 @@ import { requireSameEvidenceCode } from './relay-evidence-code-provenance.mjs' // Migration-only by policy: zero hosts and no reservation, so a wave rolls one without // displacing anybody. It enters and must leave migration-only, never general. -// C35 is an Asia spare that stays migration-only until a reviewed change promotes it. -export const SAME_CAP_MIGRATION_ONLY_CELLS = [ - 'production-gce-c17', 'production-gce-c18', 'production-gce-c35' -] +export const SAME_CAP_MIGRATION_ONLY_CELLS = ['production-gce-c17', 'production-gce-c18'] export const SAME_CAP_CELLS = [ 'production-gce-c7', 'production-gce-c8', 'production-gce-c9', 'production-gce-c10', @@ -16,6 +13,7 @@ export const SAME_CAP_CELLS = [ 'production-gce-c23', 'production-gce-c24', 'production-gce-c25', 'production-gce-c26', 'production-gce-c27', 'production-gce-c28', 'production-gce-c29', 'production-gce-c30', 'production-gce-c31', 'production-gce-c32', 'production-gce-c33', 'production-gce-c34', + 'production-gce-c35', ...SAME_CAP_MIGRATION_ONLY_CELLS ] diff --git a/cloud/dev/scripts/relay-production-same-cap-wave.test.mjs b/cloud/dev/scripts/relay-production-same-cap-wave.test.mjs index 48630cc103e..d679ae3c7e4 100644 --- a/cloud/dev/scripts/relay-production-same-cap-wave.test.mjs +++ b/cloud/dev/scripts/relay-production-same-cap-wave.test.mjs @@ -136,14 +136,11 @@ test('the wave workflow chains exactly ten serial cell jobs', () => { assert.doesNotMatch(dispatch, /\n cell_11:/) }) -test('lists the C35 spare as migration-only beside C17 and C18, and C30-C34 as general', () => { - assert.deepEqual( - SAME_CAP_MIGRATION_ONLY_CELLS, - ['production-gce-c17', 'production-gce-c18', 'production-gce-c35'] - ) +test('lists only C17 and C18 as migration-only now that C35 is promoted, and C30-C35 as general', () => { + assert.deepEqual(SAME_CAP_MIGRATION_ONLY_CELLS, ['production-gce-c17', 'production-gce-c18']) for (const cellId of [ 'production-gce-c30', 'production-gce-c31', 'production-gce-c32', 'production-gce-c33', - 'production-gce-c34' + 'production-gce-c34', 'production-gce-c35' ]) { assert.equal(SAME_CAP_CELLS.includes(cellId), true, cellId) } @@ -221,6 +218,19 @@ test('rolls the migration-only cells but never mixes the two classes in one wave confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${c34Mixed}`, canaryRunId: '42' }), /all general or all migration-only/) + // C35 is general once its Asia canary promotes it, so a rollback restores it general. + assert.equal(entryAdmission('production-gce-c35'), 'general') + assert.equal(selectorWaveDelta('production-gce-c35'), 2) + const c35Mixed = 'production-gce-c35,production-gce-c17' + assert.throws(() => validateSameCapWave({ + mode: 'batch-apply', + cellIds: c35Mixed, + targetDigest, + rollbackDigest, + drainPaceWindowMs: '300000', + confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${c35Mixed}`, + canaryRunId: '42' + }), /all general or all migration-only/) const c31Mixed = 'production-gce-c31,production-gce-c18' assert.throws(() => validateSameCapWave({ mode: 'batch-apply', @@ -669,7 +679,7 @@ test('keeps Asia and migration-only cells on the default pace', () => { ) for (const cellId of SAME_CAP_CELLS) { const fast = SAME_CAP_FAST_DRAIN_PACE_CELLS.includes(cellId) - const asia = ['c27', 'c28', 'c29', 'c30', 'c31', 'c34'].some( + const asia = ['c27', 'c28', 'c29', 'c30', 'c31', 'c34', 'c35'].some( (cell) => cellId === `production-gce-${cell}` ) assert.equal(fast, entryAdmission(cellId) === 'general' && !asia, cellId) diff --git a/cloud/dev/scripts/relay-same-cap-script-census.test.mjs b/cloud/dev/scripts/relay-same-cap-script-census.test.mjs index 77f266bb7bd..2fea683b82e 100644 --- a/cloud/dev/scripts/relay-same-cap-script-census.test.mjs +++ b/cloud/dev/scripts/relay-same-cap-script-census.test.mjs @@ -279,10 +279,9 @@ describe('same-cap roll scripts accept every same-cap cell', () => { const trusted = SAME_CAP_CELLS.filter((cell) => REHOME_SOURCE_CELLS.has(cell)) // Only a declared rehome source may roll at a trusted protocol at all; the job refuses // the rest before it plans, and the next test covers them at protocol 0. - // The C35 spare is a rehome source that stays migration-only. assert.deepEqual( SAME_CAP_CELLS.filter((cell) => !REHOME_SOURCE_CELLS.has(cell)), - SAME_CAP_MIGRATION_ONLY_CELLS.filter((cell) => cell !== 'production-gce-c35') + SAME_CAP_MIGRATION_ONLY_CELLS ) for (const [cellId, protocol] of trusted.flatMap((cell) => [[cell, 1], [cell, 3]])) { const { cap, pool } = cellShape(cellId) diff --git a/cloud/dev/scripts/relay-same-cap-shadow-gate-verdict.mjs b/cloud/dev/scripts/relay-same-cap-shadow-gate-verdict.mjs index 019b31616fb..76523d435af 100644 --- a/cloud/dev/scripts/relay-same-cap-shadow-gate-verdict.mjs +++ b/cloud/dev/scripts/relay-same-cap-shadow-gate-verdict.mjs @@ -29,7 +29,8 @@ export const FLEET_POOL_CELL_IDS = [ 'production-gce-c29', 'production-gce-c30', 'production-gce-c31', - 'production-gce-c34' + 'production-gce-c34', + 'production-gce-c35' ] export const SHADOW_GATE_THRESHOLDS = { diff --git a/cloud/dev/scripts/relay-same-cap-shadow-gate.test.mjs b/cloud/dev/scripts/relay-same-cap-shadow-gate.test.mjs index 34ac47fec7f..681b06c3187 100644 --- a/cloud/dev/scripts/relay-same-cap-shadow-gate.test.mjs +++ b/cloud/dev/scripts/relay-same-cap-shadow-gate.test.mjs @@ -279,6 +279,7 @@ function productionLikeEntries() { ...metricSamples({ cellId: 'production-gce-c30', from: '2026-09-20T20:20:00Z', count: 20 }), ...metricSamples({ cellId: 'production-gce-c31', from: '2026-09-20T20:20:00Z', count: 20 }), ...metricSamples({ cellId: 'production-gce-c34', from: '2026-09-20T20:20:00Z', count: 20 }), + ...metricSamples({ cellId: 'production-gce-c35', from: '2026-09-20T20:20:00Z', count: 20 }), // One director instance's samples from ten minutes before the drain to the window's end. ...directorSamples({ from: '2026-09-20T19:50:00Z', count: 80, payload: {} }) ] @@ -340,10 +341,10 @@ function gcloudSeam(entries = productionLikeEntries()) { } } -test('reads every promoted asia-east2 cell as fleet pool, C34 included', () => { +test('reads every promoted asia-east2 cell as fleet pool, C35 included', () => { assert.deepEqual(FLEET_POOL_CELL_IDS, [ 'production-gce-c27', 'production-gce-c28', 'production-gce-c29', 'production-gce-c30', - 'production-gce-c31', 'production-gce-c34' + 'production-gce-c31', 'production-gce-c34', 'production-gce-c35' ]) }) @@ -365,6 +366,7 @@ test('a healthy roll reads as PASS and names the instance it proved serving', as 'fleetPool:production-gce-c30', 'fleetPool:production-gce-c31', 'fleetPool:production-gce-c34', + 'fleetPool:production-gce-c35', 'nonDrain503Budget' ]) assert.deepEqual(report.drain, { diff --git a/cloud/docs/relay-workflows.md b/cloud/docs/relay-workflows.md index aa844d9a79e..66b56b1e901 100644 --- a/cloud/docs/relay-workflows.md +++ b/cloud/docs/relay-workflows.md @@ -228,19 +228,14 @@ first rolled to the director's image as a migration-only same-cap wave. Once its it, it moved to the same-cap general list and the fleet pool list together, so the same-cap job now rolls it as a general cell and a rollback restores it general. It is a declared rehome source. -C35 is a seventh Asia cell at the C34 shape in `asia-east2-a`, starting a third zone round. It is -its own topology wave on image F (`sha256:41cba995…`), registers alone as migration-only, and then -the director is configured with `cell-ids` set to C35. It launches as a migration-only spare and has -a promotion wave of its own, with the same five-minute canary C30, C31 and C34 ran. The canary lands -on C35 because it is the emptiest general Asia cell once promoted. Promotion compares the director's -serving digest and C35's runtime digest with the one `image-digest` input, so C35 must first be -rolled to the director's image. That roll is a same-cap wave that enters and leaves migration-only, -so it moves nobody. C35 stays in the same-cap migration-only list until its promotion succeeds, -because a same-cap job reads a cell's class from that list, not from the selector. It then moves to -the general list and the fleet pool list together, as its own reviewed change, before any same-cap -wave names C35 again. Between promotion and that change, do not run a same-cap wave on C35; a -rollback there would demote it. Do not name it in the multi-target `promote-general-cell` or -`retire-migration-cell` modes, which accept any migration-only cell. It is a declared rehome source. +C35 is a seventh Asia cell at the C34 shape in `asia-east2-a`, starting a third zone round. It was +its own topology wave on image F (`sha256:41cba995…`), registered alone as migration-only, and then +the director was configured with `cell-ids` set to C35. It launched as a migration-only spare and has +a promotion wave of its own, with the same five-minute canary C30, C31 and C34 ran. Promotion +compares the director's serving digest and C35's runtime digest with the one `image-digest` input, +so C35 was first rolled to the director's image as a migration-only same-cap wave. Once its canary +promoted it, it moved to the same-cap general list and the fleet pool list together, so the same-cap +job now rolls it as a general cell and a rollback restores it general. It is a declared rehome source. Its pool of 16 leaves 3 connections under the usable Cloud SQL ceiling (`relay-cloud-sql-connection-budget.mjs`). Rollback returns diff --git a/cloud/infra/terraform/environments/production.tfvars b/cloud/infra/terraform/environments/production.tfvars index dedf797c6ee..d427b2c28a3 100644 --- a/cloud/infra/terraform/environments/production.tfvars +++ b/cloud/infra/terraform/environments/production.tfvars @@ -468,7 +468,7 @@ relay_gce_cells = { connection_hard_cap = 3000 connection_unobserved_bound = 60 } - # Asia spare: registered migration-only as a drain landing zone; a starts the third zone round. + # Seventh Asia cell: launched as a migration-only spare, now general; a starts the third zone round. "production-gce-c35" = { hostname = "c35" region = "asia-east2"