feat(relay): declare Asia spare cell c35 as migration-only (#27695)

* feat(relay): declare Asia spare cell c35 as migration-only

Adds a seventh asia-east2 cell at the C34 shape (cap 3000, 6000 request
units, pool 16, e2-standard-4) in asia-east2-a, pinned to image F. It gets
its own topology and registration wave but no promotion wave, so the
admission script and workflow refuse to promote it; it stays a
migration-only spare and out of the fleet pool list. Mirrors #25336.

* test(relay): accept the c35 spare under the same-cap capacity allowlist
This commit is contained in:
Jinwoo Hong
2026-10-11 22:29:58 -04:00
committed by GitHub
parent cbf129fd28
commit a4da555b4f
19 changed files with 135 additions and 39 deletions
@@ -79,6 +79,7 @@ jobs:
production:production-gce-c31) ;;
production:production-gce-c32,production-gce-c33) target_region=us-central1 ;;
production:production-gce-c34) ;;
production:production-gce-c35) ;;
*) echo "cell-ids do not match the reviewed environment topology" >&2; exit 1 ;;
esac
echo "TARGET_REGION=${target_region}" >> "${GITHUB_ENV}"
@@ -197,7 +197,7 @@ jobs:
EXPECTED_REGION=us-central1
EXPECTED_DATABASE_POOL_MAX=
;;
c27|c28|c29|c30|c31|c34)
c27|c28|c29|c30|c31|c34|c35)
EXPECTED_HARD_CAP=3000
EXPECTED_REGION=asia-east2
EXPECTED_DATABASE_POOL_MAX=16
@@ -21,7 +21,8 @@ const SHAPES = {
domain: 'relay.onorca.dev',
allCells: [
'production-gce-c27', 'production-gce-c28', 'production-gce-c29', 'production-gce-c30',
'production-gce-c31', 'production-gce-c32', 'production-gce-c33', 'production-gce-c34'
'production-gce-c31', 'production-gce-c32', 'production-gce-c33', 'production-gce-c34',
'production-gce-c35'
],
// The launch set was registered together; each later cell registers alone beside it.
registrationWaves: [
@@ -30,7 +31,8 @@ const SHAPES = {
['production-gce-c31'],
['production-gce-c32'],
['production-gce-c33'],
['production-gce-c34']
['production-gce-c34'],
['production-gce-c35']
],
promotionWaves: [
['production-gce-c27'],
@@ -554,13 +554,15 @@ test('accepts only reviewed Asia admission waves', () => {
['rollback', 'production-gce-c30'],
['rollback', 'production-gce-c31'],
['rollback', 'production-gce-c27,production-gce-c28,production-gce-c29'],
['rollback', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31,production-gce-c32,production-gce-c33,production-gce-c34'],
['rollback', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31,production-gce-c32,production-gce-c33,production-gce-c34,production-gce-c35'],
...['production-gce-c32', 'production-gce-c33'].flatMap((cellId) => [
'inspect', 'verify', 'register', 'registered', 'promote', 'recover-promotion', 'rollback'
].map((mode) => [mode, cellId])),
...['inspect', 'verify', 'register', 'registered', 'promote', 'recover-promotion', 'rollback']
.map((mode) => [mode, 'production-gce-c34']),
['inspect', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31,production-gce-c32,production-gce-c33,production-gce-c34']
...['inspect', 'verify', 'register', 'registered', 'rollback']
.map((mode) => [mode, 'production-gce-c35']),
['inspect', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31,production-gce-c32,production-gce-c33,production-gce-c34,production-gce-c35']
]
for (const [mode, cellIds] of accepted) {
assert.deepEqual(
@@ -570,6 +572,12 @@ test('accepts only reviewed Asia admission waves', () => {
)
}
const rejected = [
// The C35 spare stays migration-only: no reviewed promotion wave names it.
['promote', 'production-gce-c35'],
['recover-promotion', 'production-gce-c35'],
['register', 'production-gce-c34,production-gce-c35'],
['inspect', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31,production-gce-c32,production-gce-c33,production-gce-c34'],
['rollback', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31,production-gce-c32,production-gce-c33,production-gce-c34'],
['initialize', 'production-gce-c30'],
['initialize', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31'],
['register', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31'],
@@ -678,6 +686,29 @@ test('registers the C34 spare alone as migration-only in asia-east2', async () =
assert.equal(subject.selector().membership.general.includes('production-gce-c34'), false)
})
test('registers the C35 spare alone as migration-only in asia-east2', async () => {
const general = [...launchCells, 'production-gce-c30', 'production-gce-c31']
const subject = harness({
generation: 17,
membership: {
existingOnly: [],
migrationOnly: [],
general: [...general, 'production-gce-c32', 'production-gce-c33', 'production-gce-c34']
}
})
const result = await operateRelayAsiaAdmission({
environment: 'production', mode: 'register', cells: ['production-gce-c35'],
expectedGeneration: 17, imageDigest: digest, attemptId: 'asia_register_c35', token: 'not-logged'
}, subject)
const request = subject.requests.find(({ path }) => path.endsWith('/add-migration-cells'))
assert.deepEqual(request.body.cells, [{
cellId: 'production-gce-c35', cellUrl: 'https://c35.relay.onorca.dev', region: 'asia-east2',
capacityRequests: 6_000, connectionHardCap: 3_000, connectionUnobservedBound: 60
}])
assert.deepEqual(result.states, { 'production-gce-c35': 'migration-only' })
assert.equal(subject.selector().membership.general.includes('production-gce-c35'), false)
})
test('promotes the registered C34 spare to general and leaves every other cell alone', async () => {
const general = [
...launchCells, 'production-gce-c30', 'production-gce-c31', 'production-gce-c32', 'production-gce-c33'
@@ -21,7 +21,8 @@ const SHAPES = {
'production-gce-c31': 'asia-east2-b',
'production-gce-c32': 'us-central1-a',
'production-gce-c33': 'us-central1-b',
'production-gce-c34': 'asia-east2-c'
'production-gce-c34': 'asia-east2-c',
'production-gce-c35': 'asia-east2-a'
},
// The launch set, then each later additive cell; a plan targets one wave, never live cells.
waves: [
@@ -30,7 +31,8 @@ const SHAPES = {
['production-gce-c31'],
// Declared together, so they plan together: a lone C32 plan would hit C33's missing template.
['production-gce-c32', 'production-gce-c33'],
['production-gce-c34']
['production-gce-c34'],
['production-gce-c35']
]
}
}
@@ -15,7 +15,8 @@ const productionCells = () => Object.fromEntries([
[31, 'asia-east2-b'],
[32, 'us-central1-a'],
[33, 'us-central1-b'],
[34, 'asia-east2-c']
[34, 'asia-east2-c'],
[35, 'asia-east2-a']
].map(([ordinal, zone]) => [`production-gce-c${ordinal}`, {
hostname: `c${ordinal}`, region: zone.slice(0, -2), zone,
machine_type: 'e2-standard-4', boot_disk_gb: 30,
@@ -66,6 +67,14 @@ test('accepts the additive C34 spare wave in asia-east2-c at the Asia pool', ()
assert.equal(result.relay_gce_cells['production-gce-c34'].database_pool_max, 16)
})
test('accepts the additive C35 spare wave in asia-east2-a at the Asia pool', () => {
const result = prepareRelayAsiaTopologyInput({ existingCells: productionCells(),
existingAdditionalRegions: additionalRegions, environment: 'production',
cellIds: 'production-gce-c35', image })
assert.equal(result.relay_gce_cells['production-gce-c35'].zone, 'asia-east2-a')
assert.equal(result.relay_gce_cells['production-gce-c35'].database_pool_max, 16)
})
test('accepts the additive US C32+C33 wave at the default pool only', () => {
const cellIds = 'production-gce-c32,production-gce-c33'
for (const [cellId, zone] of [
@@ -113,7 +122,8 @@ test('matches every committed production Asia cell entry', () => {
'production-gce-c30',
'production-gce-c31',
'production-gce-c32,production-gce-c33',
'production-gce-c34'
'production-gce-c34',
'production-gce-c35'
]) {
const committedImage = committed[wave.split(',')[0]].image
assert.doesNotThrow(() => prepareRelayAsiaTopologyInput({
@@ -163,7 +173,8 @@ test('rejects an uncommitted subnet or cell, partial wave, wrong image, and drif
'production-gce-c32',
'production-gce-c33',
'production-gce-c33,production-gce-c34',
'production-gce-c35'
'production-gce-c34,production-gce-c35',
'production-gce-c36'
]) {
assert.throws(() => prepareRelayAsiaTopologyInput({
existingCells: productionCells(), existingAdditionalRegions: additionalRegions,
@@ -99,7 +99,7 @@ describe('production Relay capacity cell admission', () => {
'production-gce-c27', 'production-gce-c28', 'production-gce-c29', 'production-gce-c30',
'production-gce-c31', 'production-gce-c34',
// Migration-only canaries: the US-only capacity rollout never touches them either.
'production-gce-c17', 'production-gce-c18',
'production-gce-c17', 'production-gce-c18', 'production-gce-c35',
// US cells at the Asia shape: the 1,000-cap capacity rollout never touches them.
'production-gce-c32', 'production-gce-c33'
]) {
@@ -118,7 +118,7 @@ describe('production Relay capacity cell admission', () => {
paceWindowMs: 0
})
}
for (const cellId of ['production-gce-c12', 'production-gce-c35']) {
for (const cellId of ['production-gce-c12', 'production-gce-c36']) {
const hostname = cellId.slice('production-gce-'.length)
assert.throws(() => parseProductionCapacityCellArguments([
'--director-origin', 'https://relay.onorca.dev',
@@ -2,9 +2,9 @@ import { pathToFileURL } from 'node:url'
import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs'
// Every cell that carries the rehome identity: the eighteen US cells and the
// six asia-east2 cells that drain mis-homed hosts back the other way.
// seven asia-east2 cells that drain mis-homed hosts back the other way.
const PRODUCTION_CELL =
/^production-gce-c(?:7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26|27|28|29|30|31|32|33|34)$/
/^production-gce-c(?:7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26|27|28|29|30|31|32|33|34|35)$/
const DIRECTOR_ORIGIN = 'https://relay.onorca.dev'
export function parseRehomeTrustProbeArguments(argv, environment = process.env) {
@@ -116,7 +116,8 @@ test('fails when both trust-probe attempts return a transient 503', async () =>
test('approves the asia-east2 and US 3,000 rehome sources and still rejects unlisted cells', () => {
for (const cellId of [
'production-gce-c27', 'production-gce-c28', 'production-gce-c29', 'production-gce-c30',
'production-gce-c31', 'production-gce-c32', 'production-gce-c33', 'production-gce-c34'
'production-gce-c31', 'production-gce-c32', 'production-gce-c33', 'production-gce-c34',
'production-gce-c35'
]) {
const parsed = parseRehomeTrustProbeArguments(
argv.map((value) => (value === 'production-gce-c7' ? cellId : value)),
@@ -124,7 +125,7 @@ test('approves the asia-east2 and US 3,000 rehome sources and still rejects unli
)
assert.equal(parsed.cellId, cellId)
}
for (const cellId of ['production-gce-c1', 'production-gce-c17', 'production-gce-c35']) {
for (const cellId of ['production-gce-c1', 'production-gce-c17', 'production-gce-c36']) {
assert.throws(
() =>
parseRehomeTrustProbeArguments(
@@ -183,7 +184,7 @@ test('approves exactly the committed production rehome source cells', () => {
const sources = new Set(
[...tfvars.slice(start, tfvars.indexOf(']', start)).matchAll(/"([^"]+)"/g)].map(([, cell]) => cell)
)
assert.ok(sources.has('production-gce-c34'))
assert.ok(sources.has('production-gce-c35'))
for (let ordinal = 1; ordinal <= 40; ordinal++) {
const cellId = `production-gce-c${ordinal}`
const approved = (() => {
@@ -51,7 +51,8 @@ test('accepts only the reviewed Asia topology waves', () => {
'production:production-gce-c30',
'production:production-gce-c31',
'production:production-gce-c32,production-gce-c33',
'production:production-gce-c34'
'production:production-gce-c34',
'production:production-gce-c35'
]
)
})
@@ -7,12 +7,12 @@ import {
} from './relay-cloud-sql-connection-budget.mjs'
test('production shared consumers keep allowance and reserve below the ceiling', () => {
// cells: 22 pools at 10 (220, C32/C33 included) + the six asia-east2 pools at 16 (96).
// cells: 22 pools at 10 (220, C32/C33 included) + the seven asia-east2 pools at 16 (112).
const report = readRelayCloudSqlConnectionBudget()
assert.deepEqual(report.consumers, { cells: 316, directors: 15, auth: 20, api: 50 })
assert.deepEqual(report.asia, { cells: 6, poolMax: 16 })
assert.equal(report.configuredMaximum, 401)
assert.deepEqual(report.consumers, { cells: 332, directors: 15, auth: 20, api: 50 })
assert.deepEqual(report.asia, { cells: 7, poolMax: 16 })
assert.equal(report.configuredMaximum, 417)
assert.equal(report.rolloutOverlap.relayDirectorCandidate, 30)
assert.equal(report.rolloutOverlap.apiCandidate, 65)
assert.equal(report.rolloutOverlap.authCandidate, 35)
@@ -22,10 +22,10 @@ test('production shared consumers keep allowance and reserve below the ceiling',
assert.equal(report.maintenanceAdminAllowance, 5)
assert.equal(report.explicitReserve, 10)
assert.equal(report.usableCeiling, 490)
assert.equal(report.operatingMaximum, 471)
assert.equal(report.remainingWithinUsableCeiling, 19)
assert.equal(report.budgetedTotal, 481)
assert.equal(report.unallocated, 19)
assert.equal(report.operatingMaximum, 487)
assert.equal(report.remainingWithinUsableCeiling, 3)
assert.equal(report.budgetedTotal, 497)
assert.equal(report.unallocated, 3)
assert.equal(report.withinBudget, true)
})
@@ -4,7 +4,10 @@ import { requireSameEvidenceCode } from './relay-evidence-code-provenance.mjs'
// Migration-only by policy: zero hosts and no reservation, so a wave rolls one without
// displacing anybody. It enters and must leave migration-only, never general.
export const SAME_CAP_MIGRATION_ONLY_CELLS = ['production-gce-c17', 'production-gce-c18']
// C35 is an Asia spare that stays migration-only until a reviewed change promotes it.
export const SAME_CAP_MIGRATION_ONLY_CELLS = [
'production-gce-c17', 'production-gce-c18', 'production-gce-c35'
]
export const SAME_CAP_CELLS = [
'production-gce-c7', 'production-gce-c8', 'production-gce-c9', 'production-gce-c10',
@@ -85,11 +85,11 @@ test('requires one canary or a bounded reviewed batch', () => {
}
assert.throws(() => validateSameCapWave({
mode: 'canary-apply',
cellIds: 'production-gce-c35',
cellIds: 'production-gce-c36',
targetDigest,
rollbackDigest,
drainPaceWindowMs: '300000',
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c35`
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c36`
}), /cells/)
})
@@ -136,8 +136,11 @@ test('the wave workflow chains exactly ten serial cell jobs', () => {
assert.doesNotMatch(dispatch, /\n cell_11:/)
})
test('lists only C17 and C18 as migration-only now that C34 is promoted, and C30-C34 as general', () => {
assert.deepEqual(SAME_CAP_MIGRATION_ONLY_CELLS, ['production-gce-c17', 'production-gce-c18'])
test('lists the C35 spare as migration-only beside C17 and C18, and C30-C34 as general', () => {
assert.deepEqual(
SAME_CAP_MIGRATION_ONLY_CELLS,
['production-gce-c17', 'production-gce-c18', 'production-gce-c35']
)
for (const cellId of [
'production-gce-c30', 'production-gce-c31', 'production-gce-c32', 'production-gce-c33',
'production-gce-c34'
@@ -267,7 +267,7 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
assert.equal(String(cellShape(cellId).cap), tfvarsHardCap(cellId), cellId)
}
assert.equal(resolveCellShape('production-gce-c12').status, 1)
assert.equal(resolveCellShape('production-gce-c35').status, 1)
assert.equal(resolveCellShape('production-gce-c36').status, 1)
})
@@ -279,9 +279,10 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
const trusted = SAME_CAP_CELLS.filter((cell) => REHOME_SOURCE_CELLS.has(cell))
// Only a declared rehome source may roll at a trusted protocol at all; the job refuses
// the rest before it plans, and the next test covers them at protocol 0.
// The C35 spare is a rehome source that stays migration-only.
assert.deepEqual(
SAME_CAP_CELLS.filter((cell) => !REHOME_SOURCE_CELLS.has(cell)),
SAME_CAP_MIGRATION_ONLY_CELLS
SAME_CAP_MIGRATION_ONLY_CELLS.filter((cell) => cell !== 'production-gce-c35')
)
for (const [cellId, protocol] of trusted.flatMap((cell) => [[cell, 1], [cell, 3]])) {
const { cap, pool } = cellShape(cellId)
@@ -25,7 +25,8 @@ const CELL_SHAPES = {
'production-gce-c31': 'asia-east2-b',
'production-gce-c32': 'us-central1-a',
'production-gce-c33': 'us-central1-b',
'production-gce-c34': 'asia-east2-c'
'production-gce-c34': 'asia-east2-c',
'production-gce-c35': 'asia-east2-a'
},
waves: [
['production-gce-c27', 'production-gce-c28', 'production-gce-c29'],
@@ -33,7 +34,8 @@ const CELL_SHAPES = {
['production-gce-c31'],
// Declared together, so they plan together: a lone C32 plan would hit C33's missing template.
['production-gce-c32', 'production-gce-c33'],
['production-gce-c34']
['production-gce-c34'],
['production-gce-c35']
]
},
staging: {
@@ -173,6 +173,18 @@ test('accepts the additive production C34 spare wave only in asia-east2-c', () =
)
})
test('accepts the additive production C35 spare wave only in asia-east2-a', () => {
const c35Config = { ...productionConfig, cells: ['production-gce-c35'] }
assert.deepEqual(
validateRelayAsiaTopologyPlan({ resource_changes: productionWavePlan('c35', 'asia-east2-a') }, c35Config),
{ environment: 'production', cells: ['production-gce-c35'], changes: 4 }
)
assert.throws(
() => validateRelayAsiaTopologyPlan({ resource_changes: productionWavePlan('c35', 'asia-east2-c') }, c35Config),
/fixed-one Asia MIG shape/
)
})
// A US cell joins the root region: no additional-region network, no region label or line, and
// the default pool emits no line, exactly as the startup template renders a root-region cell.
function usCellPlan(hostname, zone) {
@@ -282,7 +294,8 @@ test('accepts only a reviewed Asia topology wave', () => {
'production-gce-c29,production-gce-c27,production-gce-c28',
'production-gce-c30',
'production-gce-c31',
'production-gce-c34'
'production-gce-c34',
'production-gce-c35'
]) {
assert.doesNotThrow(
() => parseRelayAsiaTopologyPlanArguments(argv('production', cellIds, productionImage)),
@@ -300,7 +313,8 @@ test('accepts only a reviewed Asia topology wave', () => {
'production-gce-c33',
'production-gce-c32,production-gce-c33,production-gce-c34',
'production-gce-c33,production-gce-c34',
'production-gce-c35'
'production-gce-c34,production-gce-c35',
'production-gce-c36'
]) {
assert.throws(
() => parseRelayAsiaTopologyPlanArguments(argv('production', cellIds, productionImage)),
+8
View File
@@ -227,6 +227,14 @@ director's serving digest and C34's runtime digest with the one `image-digest` i
first rolled to the director's image as a migration-only same-cap wave. Once its canary promoted
it, it moved to the same-cap general list and the fleet pool list together, so the same-cap job now
rolls it as a general cell and a rollback restores it general. It is a declared rehome source.
C35 is a seventh Asia cell at the C34 shape in `asia-east2-a`, starting a third zone round. It is
its own topology wave on image F (`sha256:41cba995…`), registers alone as migration-only, and then
the director is configured with `cell-ids` set to C35. It has no promotion wave yet: the Asia admission
script and workflow refuse `promote` for it. It is a declared rehome source, sits in the same-cap
migration-only list, and stays out of the fleet pool list. Its pool of 16 leaves 3 connections
under the usable Cloud SQL ceiling (`relay-cloud-sql-connection-budget.mjs`). Promoting it takes its
own reviewed change adding a promotion wave and canary entry, as C34 did.
Rollback returns
Asia cells to migration-only; it does not destroy the network or use
existing-only. The production topology dispatch remains unavailable until the
+1 -1
View File
@@ -332,7 +332,7 @@ cell templates/MIGs/backends, and exact shared URL-map host additions. It
rejects deletes, replacements, loss of an existing host route, US-resource
changes, and unrelated drift. Do not add production C27-C29 until the
compatible image has been published and each entry can pin its immutable
digest. A later cell, such as C30, C31 or the C34 spare, is its own reviewed wave. The shared URL map
digest. A later cell, such as C30, C31, C34 or the C35 spare, is its own reviewed wave. The shared URL map
pulls every live cell into its plan, so the workflow plans each live cell at the
image its state template already serves, and the validator rejects any change
to a cell outside the wave. US C32 and C33 use the same workflow at the same
@@ -468,6 +468,21 @@ relay_gce_cells = {
connection_hard_cap = 3000
connection_unobserved_bound = 60
}
# Asia spare: registered migration-only as a drain landing zone; a starts the third zone round.
"production-gce-c35" = {
hostname = "c35"
region = "asia-east2"
zone = "asia-east2-a"
machine_type = "e2-standard-4"
boot_disk_gb = 30
boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21"
capacity_requests = 6000
database_pool_max = 16 # 176 ms from us-central1 Postgres saturates 10 (94-156 waiters).
image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:41cba995b288cab70579882ebef23e74c5ee426b6f8924cea7c986aba23bcbfe"
initially_enabled = false
connection_hard_cap = 3000
connection_unobserved_bound = 60
}
}
relay_region_rehome_source_cell_ids = [
@@ -495,7 +510,8 @@ relay_region_rehome_source_cell_ids = [
"production-gce-c31",
"production-gce-c32",
"production-gce-c33",
"production-gce-c34"
"production-gce-c34",
"production-gce-c35"
]
# Slack #orca-relay-alerts, created out of band on 2026-08-05. Declared here because an apply