mirror of
https://github.com/stablyai/orca.git
synced 2026-10-02 16:02:15 +00:00
feat(relay): declare Asia cell c31 at the c30 shape (#24310)
Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010
This commit is contained in:
Jinwoo Hong
@@ -76,6 +76,7 @@ jobs:
|
||||
staging:staging-gce-c4) ;;
|
||||
production:production-gce-c27,production-gce-c28,production-gce-c29) ;;
|
||||
production:production-gce-c30) ;;
|
||||
production:production-gce-c31) ;;
|
||||
*) echo "cell-ids do not match the reviewed environment topology" >&2; exit 1 ;;
|
||||
esac
|
||||
[[ "${TARGET_IMAGE}" =~ ^us-central1-docker\.pkg\.dev/${GCP_PROJECT_ID}/orca-cloud/relay@sha256:[0-9a-f]{64}$ ]]
|
||||
|
||||
@@ -278,7 +278,7 @@ jobs:
|
||||
EXPECTED_REGION=us-central1
|
||||
EXPECTED_DATABASE_POOL_MAX=
|
||||
;;
|
||||
c27|c28|c29|c30)
|
||||
c27|c28|c29|c30|c31)
|
||||
EXPECTED_HARD_CAP=3000
|
||||
EXPECTED_REGION=asia-east2
|
||||
EXPECTED_DATABASE_POOL_MAX=16
|
||||
|
||||
@@ -39,7 +39,7 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
evidence-run-id:
|
||||
description: Staging evidence run ID for C27, C27 canary run ID for C28/C29; C30 takes none and proves itself by its own canary
|
||||
description: Staging evidence run ID for C27, C27 canary run ID for C28/C29; C30/C31 take none and each proves itself by its own canary
|
||||
required: false
|
||||
type: string
|
||||
evidence-run-attempt:
|
||||
@@ -155,9 +155,9 @@ jobs:
|
||||
evidence_kind=c27
|
||||
artifact_name="relay-asia-c27-canary-${EVIDENCE_RUN_ID}-${EVIDENCE_RUN_ATTEMPT}"
|
||||
;;
|
||||
production-gce-c30)
|
||||
# No earlier proof binds C30's generation; its own canary below rolls it back on failure.
|
||||
canary_cell=production-gce-c30
|
||||
production-gce-c30|production-gce-c31)
|
||||
# No earlier proof binds a later cell's generation; its own canary below rolls it back on failure.
|
||||
canary_cell="${TARGET_CELL_IDS}"
|
||||
;;
|
||||
*) echo "production promotion wave is not reviewed" >&2; exit 1 ;;
|
||||
esac
|
||||
@@ -317,15 +317,15 @@ jobs:
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# C30's launch cells were checked general by the promotion; verify reads only C30's digest.
|
||||
# A later cell's launch cells were checked general by the promotion; verify reads only its digest.
|
||||
case "${CANARY_CELL}" in
|
||||
production-gce-c27)
|
||||
verify_cells=production-gce-c27,production-gce-c28,production-gce-c29
|
||||
expected_states='{"production-gce-c27":"general","production-gce-c28":"migration-only","production-gce-c29":"migration-only"}'
|
||||
;;
|
||||
production-gce-c30)
|
||||
verify_cells=production-gce-c30
|
||||
expected_states='{"production-gce-c30":"general"}'
|
||||
production-gce-c30|production-gce-c31)
|
||||
verify_cells="${CANARY_CELL}"
|
||||
expected_states="{\"${CANARY_CELL}\":\"general\"}"
|
||||
;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
|
||||
@@ -19,16 +19,21 @@ const SHAPES = {
|
||||
production: {
|
||||
directorOrigin: 'https://relay.onorca.dev',
|
||||
domain: 'relay.onorca.dev',
|
||||
allCells: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29', 'production-gce-c30'],
|
||||
allCells: [
|
||||
'production-gce-c27', 'production-gce-c28', 'production-gce-c29', 'production-gce-c30',
|
||||
'production-gce-c31'
|
||||
],
|
||||
// The launch set was registered together; each later cell registers alone beside it.
|
||||
registrationWaves: [
|
||||
['production-gce-c27', 'production-gce-c28', 'production-gce-c29'],
|
||||
['production-gce-c30']
|
||||
['production-gce-c30'],
|
||||
['production-gce-c31']
|
||||
],
|
||||
promotionWaves: [
|
||||
['production-gce-c27'],
|
||||
['production-gce-c28', 'production-gce-c29'],
|
||||
['production-gce-c30']
|
||||
['production-gce-c30'],
|
||||
['production-gce-c31']
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -526,23 +526,30 @@ function admissionArguments(environment, mode, cellIds) {
|
||||
test('accepts only reviewed Asia admission waves', () => {
|
||||
const accepted = [
|
||||
['inspect', 'production-gce-c27,production-gce-c28,production-gce-c29'],
|
||||
['inspect', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30'],
|
||||
['inspect', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31'],
|
||||
['inspect', 'production-gce-c30'],
|
||||
['inspect', 'production-gce-c31'],
|
||||
['verify', 'production-gce-c27,production-gce-c28,production-gce-c29'],
|
||||
['verify', 'production-gce-c30'],
|
||||
['verify', 'production-gce-c31'],
|
||||
['initialize', 'production-gce-c27,production-gce-c28,production-gce-c29'],
|
||||
['register', 'production-gce-c27,production-gce-c28,production-gce-c29'],
|
||||
['register', 'production-gce-c30'],
|
||||
['registered', 'production-gce-c30'],
|
||||
['register', 'production-gce-c31'],
|
||||
['registered', 'production-gce-c31'],
|
||||
['promote', 'production-gce-c27'],
|
||||
['promote', 'production-gce-c28,production-gce-c29'],
|
||||
['promote', 'production-gce-c30'],
|
||||
['recover-promotion', 'production-gce-c30'],
|
||||
['promote', 'production-gce-c31'],
|
||||
['recover-promotion', 'production-gce-c31'],
|
||||
['rollback', 'production-gce-c27'],
|
||||
['rollback', 'production-gce-c28,production-gce-c29'],
|
||||
['rollback', 'production-gce-c30'],
|
||||
['rollback', 'production-gce-c31'],
|
||||
['rollback', 'production-gce-c27,production-gce-c28,production-gce-c29'],
|
||||
['rollback', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30']
|
||||
['rollback', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31']
|
||||
]
|
||||
for (const [mode, cellIds] of accepted) {
|
||||
assert.deepEqual(
|
||||
@@ -553,15 +560,21 @@ test('accepts only reviewed Asia admission waves', () => {
|
||||
}
|
||||
const rejected = [
|
||||
['initialize', 'production-gce-c30'],
|
||||
['initialize', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30'],
|
||||
['register', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30'],
|
||||
['initialize', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31'],
|
||||
['register', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31'],
|
||||
['register', 'production-gce-c30,production-gce-c31'],
|
||||
['register', 'production-gce-c29,production-gce-c30'],
|
||||
['registered', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30'],
|
||||
['registered', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30,production-gce-c31'],
|
||||
['inspect', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30'],
|
||||
['verify', 'production-gce-c30,production-gce-c31'],
|
||||
['verify', 'production-gce-c27,production-gce-c30'],
|
||||
['promote', 'production-gce-c27,production-gce-c30'],
|
||||
['promote', 'production-gce-c28,production-gce-c29,production-gce-c30'],
|
||||
['promote', 'production-gce-c31'],
|
||||
['promote', 'production-gce-c30,production-gce-c31'],
|
||||
['promote', 'production-gce-c32'],
|
||||
['rollback', 'production-gce-c27,production-gce-c30'],
|
||||
['rollback', 'production-gce-c30,production-gce-c31'],
|
||||
['rollback', 'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30'],
|
||||
['rollback', 'production-gce-c28,production-gce-c29,production-gce-c30'],
|
||||
['rollback', 'production-gce-c29'],
|
||||
['register', 'staging-gce-c4']
|
||||
@@ -601,6 +614,25 @@ test('registers C30 alone beside the general launch cells', async () => {
|
||||
assert.deepEqual(subject.selector().membership.general, launchCells)
|
||||
})
|
||||
|
||||
test('registers C31 alone beside the general C27-C30', async () => {
|
||||
const general = [...launchCells, 'production-gce-c30']
|
||||
const subject = harness({
|
||||
generation: 11,
|
||||
membership: { existingOnly: [], migrationOnly: [], general: [...general] }
|
||||
})
|
||||
const result = await operateRelayAsiaAdmission({
|
||||
environment: 'production', mode: 'register', cells: ['production-gce-c31'],
|
||||
expectedGeneration: 11, imageDigest: digest, attemptId: 'asia_register_c31', token: 'not-logged'
|
||||
}, subject)
|
||||
const request = subject.requests.find(({ path }) => path.endsWith('/add-migration-cells'))
|
||||
assert.deepEqual(request.body.cells, [{
|
||||
cellId: 'production-gce-c31', cellUrl: 'https://c31.relay.onorca.dev', region: 'asia-east2',
|
||||
capacityRequests: 6_000, connectionHardCap: 3_000, connectionUnobservedBound: 60
|
||||
}])
|
||||
assert.deepEqual(result.states, { 'production-gce-c31': 'migration-only' })
|
||||
assert.deepEqual(subject.selector().membership.general, general)
|
||||
})
|
||||
|
||||
test('requires the C27 canary to be general before promoting C30', async () => {
|
||||
const selector = (general) => ({
|
||||
generation: 10,
|
||||
|
||||
@@ -16,12 +16,14 @@ const SHAPES = {
|
||||
'production-gce-c27': 'asia-east2-a',
|
||||
'production-gce-c28': 'asia-east2-b',
|
||||
'production-gce-c29': 'asia-east2-c',
|
||||
'production-gce-c30': 'asia-east2-a'
|
||||
'production-gce-c30': 'asia-east2-a',
|
||||
'production-gce-c31': 'asia-east2-b'
|
||||
},
|
||||
// The launch set, then each later additive cell; a plan targets one wave, never live cells.
|
||||
waves: [
|
||||
['production-gce-c27', 'production-gce-c28', 'production-gce-c29'],
|
||||
['production-gce-c30']
|
||||
['production-gce-c30'],
|
||||
['production-gce-c31']
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,7 +11,8 @@ const productionCells = () => Object.fromEntries([
|
||||
[27, 'asia-east2-a'],
|
||||
[28, 'asia-east2-b'],
|
||||
[29, 'asia-east2-c'],
|
||||
[30, 'asia-east2-a']
|
||||
[30, 'asia-east2-a'],
|
||||
[31, 'asia-east2-b']
|
||||
].map(([ordinal, zone]) => [`production-gce-c${ordinal}`, {
|
||||
hostname: `c${ordinal}`, region: 'asia-east2', zone,
|
||||
machine_type: 'e2-standard-4', boot_disk_gb: 30,
|
||||
@@ -46,6 +47,13 @@ test('accepts the additive C30 wave without re-planning the launch cells', () =>
|
||||
assert.equal(result.relay_gce_cells['production-gce-c30'].zone, 'asia-east2-a')
|
||||
})
|
||||
|
||||
test('accepts the additive C31 wave in the next zone of the rotation', () => {
|
||||
const result = prepareRelayAsiaTopologyInput({ existingCells: productionCells(),
|
||||
existingAdditionalRegions: additionalRegions, environment: 'production',
|
||||
cellIds: 'production-gce-c31', image })
|
||||
assert.equal(result.relay_gce_cells['production-gce-c31'].zone, 'asia-east2-b')
|
||||
})
|
||||
|
||||
// Reads the committed file so a reviewed-shape constant cannot drift from what the plan reads.
|
||||
test('matches every committed production Asia cell entry', () => {
|
||||
const tfvars = readFileSync(
|
||||
@@ -64,10 +72,11 @@ test('matches every committed production Asia cell entry', () => {
|
||||
}
|
||||
committed[cellId] = cell
|
||||
}
|
||||
// Each wave is pinned on its own: C30 launches on the director's digest, not C27's.
|
||||
// Each wave is pinned on its own: C30 and C31 launch on the director's digest, not C27's.
|
||||
for (const wave of [
|
||||
'production-gce-c27,production-gce-c28,production-gce-c29',
|
||||
'production-gce-c30'
|
||||
'production-gce-c30',
|
||||
'production-gce-c31'
|
||||
]) {
|
||||
const committedImage = committed[wave.split(',')[0]].image
|
||||
assert.doesNotThrow(() => prepareRelayAsiaTopologyInput({
|
||||
@@ -109,7 +118,8 @@ test('rejects an uncommitted subnet or cell, partial wave, wrong image, and drif
|
||||
'production-gce-c27,production-gce-c30',
|
||||
'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30',
|
||||
'production-gce-c30,production-gce-c30',
|
||||
'production-gce-c31'
|
||||
'production-gce-c30,production-gce-c31',
|
||||
'production-gce-c32'
|
||||
]) {
|
||||
assert.throws(() => prepareRelayAsiaTopologyInput({
|
||||
existingCells: productionCells(), existingAdditionalRegions: additionalRegions,
|
||||
|
||||
@@ -98,7 +98,7 @@ describe('production Relay capacity cell admission', () => {
|
||||
for (const cellId of [
|
||||
'production-gce-c27', 'production-gce-c28', 'production-gce-c29', 'production-gce-c30',
|
||||
// Migration-only canaries: the US-only capacity rollout never touches them either.
|
||||
'production-gce-c17', 'production-gce-c18'
|
||||
'production-gce-c17', 'production-gce-c18', 'production-gce-c31'
|
||||
]) {
|
||||
const hostname = cellId.slice('production-gce-'.length)
|
||||
assert.deepEqual(parseProductionCapacityCellArguments([
|
||||
@@ -115,7 +115,7 @@ describe('production Relay capacity cell admission', () => {
|
||||
paceWindowMs: 0
|
||||
})
|
||||
}
|
||||
for (const cellId of ['production-gce-c12', 'production-gce-c31']) {
|
||||
for (const cellId of ['production-gce-c12', 'production-gce-c32']) {
|
||||
const hostname = cellId.slice('production-gce-'.length)
|
||||
assert.throws(() => parseProductionCapacityCellArguments([
|
||||
'--director-origin', 'https://relay.onorca.dev',
|
||||
|
||||
@@ -2,8 +2,8 @@ import { pathToFileURL } from 'node:url'
|
||||
import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs'
|
||||
|
||||
// Every cell that carries the rehome identity: the sixteen US cells and the
|
||||
// four asia-east2 cells that drain mis-homed hosts back the other way.
|
||||
const PRODUCTION_CELL = /^production-gce-c(?:7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26|27|28|29|30)$/
|
||||
// five asia-east2 cells that drain mis-homed hosts back the other way.
|
||||
const PRODUCTION_CELL = /^production-gce-c(?:7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26|27|28|29|30|31)$/
|
||||
const DIRECTOR_ORIGIN = 'https://relay.onorca.dev'
|
||||
|
||||
export function parseRehomeTrustProbeArguments(argv, environment = process.env) {
|
||||
|
||||
@@ -114,7 +114,8 @@ test('fails when both trust-probe attempts return a transient 503', async () =>
|
||||
|
||||
test('approves the asia-east2 rehome sources and still rejects unlisted cells', () => {
|
||||
for (const cellId of [
|
||||
'production-gce-c27', 'production-gce-c28', 'production-gce-c29', 'production-gce-c30'
|
||||
'production-gce-c27', 'production-gce-c28', 'production-gce-c29', 'production-gce-c30',
|
||||
'production-gce-c31'
|
||||
]) {
|
||||
const parsed = parseRehomeTrustProbeArguments(
|
||||
argv.map((value) => (value === 'production-gce-c7' ? cellId : value)),
|
||||
@@ -122,7 +123,7 @@ test('approves the asia-east2 rehome sources and still rejects unlisted cells',
|
||||
)
|
||||
assert.equal(parsed.cellId, cellId)
|
||||
}
|
||||
for (const cellId of ['production-gce-c1', 'production-gce-c17', 'production-gce-c31']) {
|
||||
for (const cellId of ['production-gce-c1', 'production-gce-c17', 'production-gce-c32']) {
|
||||
assert.throws(
|
||||
() =>
|
||||
parseRehomeTrustProbeArguments(
|
||||
|
||||
@@ -11,7 +11,8 @@ const C27 = 'production-gce-c27'
|
||||
// Each canary proves its own cell under production load; C28/C29 promotion consumes only C27's.
|
||||
const PRODUCTION_CANARIES = {
|
||||
[C27]: { kind: 'production-c27-canary', origin: 'https://c27.relay.onorca.dev' },
|
||||
'production-gce-c30': { kind: 'production-c30-canary', origin: 'https://c30.relay.onorca.dev' }
|
||||
'production-gce-c30': { kind: 'production-c30-canary', origin: 'https://c30.relay.onorca.dev' },
|
||||
'production-gce-c31': { kind: 'production-c31-canary', origin: 'https://c31.relay.onorca.dev' }
|
||||
}
|
||||
const DIGEST_PATTERN = /^sha256:[a-f0-9]{64}$/
|
||||
const SHA_PATTERN = /^[a-f0-9]{40}$/
|
||||
|
||||
@@ -389,6 +389,19 @@ test('builds and verifies a C30 canary from C30 runtime metrics and placement',
|
||||
), /evidence kind is invalid/)
|
||||
})
|
||||
|
||||
test('builds a C31 canary that only C31 placement satisfies', () => {
|
||||
const c31 = 'production-gce-c31'
|
||||
const evidence = buildProductionCanaryEvidence(canaryInput({}, c31))
|
||||
assert.equal(evidence.kind, 'production-c31-canary')
|
||||
assert.equal(verifyRolloutEvidence(
|
||||
evidence, workflowRun(evidence),
|
||||
verifyExpected('production-c31-canary', { cellIds: [c31], selectorGeneration: 9 })
|
||||
), evidence)
|
||||
const onC30 = canaryInput({}, c31)
|
||||
onC30.loadReport.assignedCellOrigins = ['https://c30.relay.onorca.dev']
|
||||
assert.throws(() => buildProductionCanaryEvidence(onC30), /C31 canary load was not placed only on C31/)
|
||||
})
|
||||
|
||||
test('rejects a C30 canary that C30 did not serve', () => {
|
||||
const onLaunchCell = canaryInput({}, c30)
|
||||
onLaunchCell.loadReport.assignedCellOrigins = ['https://c27.relay.onorca.dev']
|
||||
|
||||
@@ -44,7 +44,8 @@ test('accepts only the reviewed Asia topology waves', () => {
|
||||
[
|
||||
'staging:staging-gce-c4',
|
||||
'production:production-gce-c27,production-gce-c28,production-gce-c29',
|
||||
'production:production-gce-c30'
|
||||
'production:production-gce-c30',
|
||||
'production:production-gce-c31'
|
||||
]
|
||||
)
|
||||
})
|
||||
|
||||
@@ -7,12 +7,12 @@ import {
|
||||
} from './relay-cloud-sql-connection-budget.mjs'
|
||||
|
||||
test('production shared consumers keep allowance and reserve below the ceiling', () => {
|
||||
// cells: 20 pools at 10 (200) + the four asia-east2 pools at 16 (64).
|
||||
// cells: 20 pools at 10 (200) + the five asia-east2 pools at 16 (80).
|
||||
const report = readRelayCloudSqlConnectionBudget()
|
||||
|
||||
assert.deepEqual(report.consumers, { cells: 264, directors: 15, auth: 20, api: 50 })
|
||||
assert.deepEqual(report.asia, { cells: 4, poolMax: 16 })
|
||||
assert.equal(report.configuredMaximum, 349)
|
||||
assert.deepEqual(report.consumers, { cells: 280, directors: 15, auth: 20, api: 50 })
|
||||
assert.deepEqual(report.asia, { cells: 5, poolMax: 16 })
|
||||
assert.equal(report.configuredMaximum, 365)
|
||||
assert.equal(report.rolloutOverlap.relayDirectorCandidate, 30)
|
||||
assert.equal(report.rolloutOverlap.apiCandidate, 65)
|
||||
assert.equal(report.rolloutOverlap.authCandidate, 35)
|
||||
@@ -22,10 +22,10 @@ test('production shared consumers keep allowance and reserve below the ceiling',
|
||||
assert.equal(report.maintenanceAdminAllowance, 5)
|
||||
assert.equal(report.explicitReserve, 10)
|
||||
assert.equal(report.usableCeiling, 490)
|
||||
assert.equal(report.operatingMaximum, 419)
|
||||
assert.equal(report.remainingWithinUsableCeiling, 71)
|
||||
assert.equal(report.budgetedTotal, 429)
|
||||
assert.equal(report.unallocated, 71)
|
||||
assert.equal(report.operatingMaximum, 435)
|
||||
assert.equal(report.remainingWithinUsableCeiling, 55)
|
||||
assert.equal(report.budgetedTotal, 445)
|
||||
assert.equal(report.unallocated, 55)
|
||||
assert.equal(report.withinBudget, true)
|
||||
})
|
||||
|
||||
|
||||
@@ -4,7 +4,10 @@ import { requireSameEvidenceCode } from './relay-evidence-code-provenance.mjs'
|
||||
|
||||
// Migration-only by policy: zero hosts and no reservation, so a wave rolls one without
|
||||
// displacing anybody. It enters and must leave migration-only, never general.
|
||||
export const SAME_CAP_MIGRATION_ONLY_CELLS = ['production-gce-c17', 'production-gce-c18']
|
||||
// C31 stays here until its Asia canary promotes it; that follow-up moves it to the general list.
|
||||
export const SAME_CAP_MIGRATION_ONLY_CELLS = [
|
||||
'production-gce-c17', 'production-gce-c18', 'production-gce-c31'
|
||||
]
|
||||
|
||||
export const SAME_CAP_CELLS = [
|
||||
'production-gce-c7', 'production-gce-c8', 'production-gce-c9', 'production-gce-c10',
|
||||
|
||||
@@ -56,12 +56,19 @@ test('requires one canary or a bounded reviewed batch', () => {
|
||||
rollbackDigest,
|
||||
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c30`
|
||||
}).cells, ['production-gce-c30'])
|
||||
assert.throws(() => validateSameCapWave({
|
||||
assert.deepEqual(validateSameCapWave({
|
||||
mode: 'canary-apply',
|
||||
cellIds: 'production-gce-c31',
|
||||
targetDigest,
|
||||
rollbackDigest,
|
||||
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c31`
|
||||
}).cells, ['production-gce-c31'])
|
||||
assert.throws(() => validateSameCapWave({
|
||||
mode: 'canary-apply',
|
||||
cellIds: 'production-gce-c32',
|
||||
targetDigest,
|
||||
rollbackDigest,
|
||||
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c32`
|
||||
}), /cells/)
|
||||
})
|
||||
|
||||
@@ -107,8 +114,11 @@ test('the wave workflow chains exactly ten serial cell jobs', () => {
|
||||
assert.doesNotMatch(dispatch, /\n cell_11:/)
|
||||
})
|
||||
|
||||
test('lists only C17 and C18 as migration-only now that C30 is promoted', () => {
|
||||
assert.deepEqual(SAME_CAP_MIGRATION_ONLY_CELLS, ['production-gce-c17', 'production-gce-c18'])
|
||||
test('lists C31 as migration-only beside C17 and C18 until its canary promotes it', () => {
|
||||
assert.deepEqual(
|
||||
SAME_CAP_MIGRATION_ONLY_CELLS,
|
||||
['production-gce-c17', 'production-gce-c18', 'production-gce-c31']
|
||||
)
|
||||
assert.equal(SAME_CAP_CELLS.includes('production-gce-c30'), true)
|
||||
})
|
||||
|
||||
@@ -154,6 +164,17 @@ test('rolls the migration-only cells but never mixes the two classes in one wave
|
||||
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${asiaMixed}`,
|
||||
canaryRunId: '42'
|
||||
}), /all general or all migration-only/)
|
||||
// Until its canary promotes it, a same-cap restore must hand C31 back isolated, never activated.
|
||||
assert.equal(entryAdmission('production-gce-c31'), 'migration-only')
|
||||
const asiaUnpromoted = 'production-gce-c30,production-gce-c31'
|
||||
assert.throws(() => validateSameCapWave({
|
||||
mode: 'batch-apply',
|
||||
cellIds: asiaUnpromoted,
|
||||
targetDigest,
|
||||
rollbackDigest,
|
||||
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${asiaUnpromoted}`,
|
||||
canaryRunId: '42'
|
||||
}), /all general or all migration-only/)
|
||||
// A mixed wave has no single selector delta for its later cells to offset from.
|
||||
const mixed = 'production-gce-c7,production-gce-c17'
|
||||
assert.throws(() => validateSameCapWave({
|
||||
|
||||
@@ -291,7 +291,7 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
|
||||
assert.equal(String(cellShape(cellId).cap), tfvarsHardCap(cellId), cellId)
|
||||
}
|
||||
assert.equal(resolveCellShape('production-gce-c12').status, 1)
|
||||
assert.equal(resolveCellShape('production-gce-c31').status, 1)
|
||||
assert.equal(resolveCellShape('production-gce-c32').status, 1)
|
||||
})
|
||||
|
||||
|
||||
@@ -303,9 +303,10 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
|
||||
const trusted = SAME_CAP_CELLS.filter((cell) => REHOME_SOURCE_CELLS.has(cell))
|
||||
// Only a declared rehome source may roll at a trusted protocol at all; the job refuses
|
||||
// the rest before it plans, and the next test covers them at protocol 0.
|
||||
// C31 is already a rehome source but stays migration-only until its Asia canary promotes it.
|
||||
assert.deepEqual(
|
||||
SAME_CAP_CELLS.filter((cell) => !REHOME_SOURCE_CELLS.has(cell)),
|
||||
SAME_CAP_MIGRATION_ONLY_CELLS
|
||||
SAME_CAP_MIGRATION_ONLY_CELLS.filter((cell) => cell !== 'production-gce-c31')
|
||||
)
|
||||
for (const [cellId, protocol] of trusted.flatMap((cell) => [[cell, 1], [cell, 3]])) {
|
||||
const { cap, pool } = cellShape(cellId)
|
||||
|
||||
@@ -40,6 +40,8 @@ const launchDigest = '5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70ca
|
||||
const asiaCells = ['production-gce-c27', 'production-gce-c28', 'production-gce-c29']
|
||||
// C30 launches after the launch cells rolled, so it pins the director's digest instead.
|
||||
const c30Digest = '4158d8a2e18e9caec439d257f0c1e45d92ffea8c0262f057b2f08c76a134bcf0'
|
||||
// C31 launches on the digest the director served when it was declared.
|
||||
const c31Digest = 'f30b5cb1ec52b6b6145efecfa1b8be9e3d309403beffd8abcc64197a2087e269'
|
||||
|
||||
function cellBlock(tfvars, cellId) {
|
||||
const start = tfvars.indexOf(`"${cellId}"`)
|
||||
@@ -50,13 +52,14 @@ function cellBlock(tfvars, cellId) {
|
||||
const productionCell = (cellId) => cellBlock(productionTfvars, cellId)
|
||||
|
||||
// Scoped to C4 by name: staging C3 serves this digest too since its 2026-09-03 re-pin.
|
||||
test('pins staging C4 and the launch Asia cells to one image, and C30 to the director image', () => {
|
||||
test('pins staging C4 and the launch Asia cells to one image, and C30/C31 to director images', () => {
|
||||
assert.match(cellBlock(stagingTfvars, 'staging-gce-c4'), new RegExp(`relay@sha256:${launchDigest}"`))
|
||||
for (const cellId of asiaCells) {
|
||||
assert.match(productionCell(cellId), new RegExp(`relay@sha256:${launchDigest}"`), cellId)
|
||||
}
|
||||
assert.match(recoveryWorkflow, new RegExp(`TARGET_IMAGE_DIGEST: sha256:${launchDigest}`))
|
||||
assert.match(productionCell('production-gce-c30'), new RegExp(`relay@sha256:${c30Digest}"`))
|
||||
assert.match(productionCell('production-gce-c31'), new RegExp(`relay@sha256:${c31Digest}"`))
|
||||
})
|
||||
|
||||
test('refreshes only empty staging C4 through the trusted capacity identity', () => {
|
||||
|
||||
@@ -19,11 +19,13 @@ const CELL_SHAPES = {
|
||||
'production-gce-c27': 'asia-east2-a',
|
||||
'production-gce-c28': 'asia-east2-b',
|
||||
'production-gce-c29': 'asia-east2-c',
|
||||
'production-gce-c30': 'asia-east2-a'
|
||||
'production-gce-c30': 'asia-east2-a',
|
||||
'production-gce-c31': 'asia-east2-b'
|
||||
},
|
||||
waves: [
|
||||
['production-gce-c27', 'production-gce-c28', 'production-gce-c29'],
|
||||
['production-gce-c30']
|
||||
['production-gce-c30'],
|
||||
['production-gce-c31']
|
||||
]
|
||||
},
|
||||
staging: {
|
||||
|
||||
@@ -96,15 +96,16 @@ const productionConfig = {
|
||||
environment: 'production', cells: ['production-gce-c30'], image: productionImage
|
||||
}
|
||||
|
||||
// C30 joins a live Asia region: the network is a no-op and the existing C27 route is preserved.
|
||||
function productionC30Plan() {
|
||||
// A later cell joins a live Asia region: the network is a no-op and the existing C27 route is preserved.
|
||||
function productionWavePlan(hostname = 'c30', zone = 'asia-east2-a') {
|
||||
const plan = JSON.parse(JSON.stringify(resources)
|
||||
.replaceAll('onorca-cloud-staging/', 'onorca-cloud/')
|
||||
.replaceAll('orca-cloud-staging-relay-gce', 'orca-cloud-relay-gce')
|
||||
.replaceAll('staging-gce-c4', 'production-gce-c30')
|
||||
.replaceAll('relay-gce-c4', 'relay-gce-c30')
|
||||
.replaceAll('cell-c4', 'cell-c30')
|
||||
.replaceAll('c4.relay-staging.onorca.dev', 'c30.relay.onorca.dev')
|
||||
.replaceAll('staging-gce-c4', `production-gce-${hostname}`)
|
||||
.replaceAll('relay-gce-c4', `relay-gce-${hostname}`)
|
||||
.replaceAll('cell-c4', `cell-${hostname}`)
|
||||
.replaceAll('c4.relay-staging.onorca.dev', `${hostname}.relay.onorca.dev`)
|
||||
.replaceAll('asia-east2-a', zone)
|
||||
.replaceAll("'10'", "'16'"))
|
||||
for (const network of plan.slice(0, 3)) {
|
||||
network.change.actions = ['no-op']
|
||||
@@ -124,23 +125,23 @@ function productionC30Plan() {
|
||||
|
||||
test('accepts the additive production C30 wave at the 16-connection Asia pool', () => {
|
||||
assert.deepEqual(
|
||||
validateRelayAsiaTopologyPlan({ resource_changes: productionC30Plan() }, productionConfig),
|
||||
validateRelayAsiaTopologyPlan({ resource_changes: productionWavePlan() }, productionConfig),
|
||||
{ environment: 'production', cells: ['production-gce-c30'], changes: 4 }
|
||||
)
|
||||
const staleShape = productionC30Plan()
|
||||
const staleShape = productionWavePlan()
|
||||
staleShape[3].change.after.metadata_startup_script =
|
||||
staleShape[3].change.after.metadata_startup_script.replace("'16'", "'10'")
|
||||
assert.throws(
|
||||
() => validateRelayAsiaTopologyPlan({ resource_changes: staleShape }, productionConfig),
|
||||
/reviewed Asia cell shape/
|
||||
)
|
||||
const wrongZone = productionC30Plan()
|
||||
const wrongZone = productionWavePlan()
|
||||
wrongZone[4].change.after.zone = 'asia-east2-b'
|
||||
assert.throws(
|
||||
() => validateRelayAsiaTopologyPlan({ resource_changes: wrongZone }, productionConfig),
|
||||
/fixed-one Asia MIG shape/
|
||||
)
|
||||
const liveCellTouched = productionC30Plan()
|
||||
const liveCellTouched = productionWavePlan()
|
||||
liveCellTouched.push(create('google_compute_instance_template.relay_gce_cell["production-gce-c27"]'))
|
||||
assert.throws(
|
||||
() => validateRelayAsiaTopologyPlan({ resource_changes: liveCellTouched }, productionConfig),
|
||||
@@ -148,13 +149,25 @@ test('accepts the additive production C30 wave at the 16-connection Asia pool',
|
||||
)
|
||||
})
|
||||
|
||||
test('accepts the additive production C31 wave only in asia-east2-b', () => {
|
||||
const c31Config = { ...productionConfig, cells: ['production-gce-c31'] }
|
||||
assert.deepEqual(
|
||||
validateRelayAsiaTopologyPlan({ resource_changes: productionWavePlan('c31', 'asia-east2-b') }, c31Config),
|
||||
{ environment: 'production', cells: ['production-gce-c31'], changes: 4 }
|
||||
)
|
||||
assert.throws(
|
||||
() => validateRelayAsiaTopologyPlan({ resource_changes: productionWavePlan('c31') }, c31Config),
|
||||
/fixed-one Asia MIG shape/
|
||||
)
|
||||
})
|
||||
|
||||
// The URL map pulls every cell's backend, MIG and template into a targeted plan.
|
||||
const liveCellResources = ['instance_template', 'instance_group_manager', 'backend_service']
|
||||
.flatMap((kind) => ['production-gce-c1', 'production-gce-c27', 'production-gce-c28', 'production-gce-c29']
|
||||
.map((cellId) => `google_compute_${kind}.relay_gce_cell["${cellId}"]`))
|
||||
|
||||
test('accepts live cells the URL map pulls in only while they stay unchanged', () => {
|
||||
const plan = productionC30Plan()
|
||||
const plan = productionWavePlan()
|
||||
for (const address of liveCellResources) plan.push({ address, change: { actions: ['no-op'] } })
|
||||
assert.equal(
|
||||
validateRelayAsiaTopologyPlan({ resource_changes: plan }, productionConfig).changes,
|
||||
@@ -162,7 +175,7 @@ test('accepts live cells the URL map pulls in only while they stay unchanged', (
|
||||
)
|
||||
for (const address of liveCellResources) {
|
||||
for (const action of [['update'], ['delete'], ['create', 'delete'], ['delete', 'create']]) {
|
||||
const drifted = productionC30Plan()
|
||||
const drifted = productionWavePlan()
|
||||
drifted.push({ address, change: { actions: action } })
|
||||
assert.throws(
|
||||
() => validateRelayAsiaTopologyPlan({ resource_changes: drifted }, productionConfig),
|
||||
@@ -180,7 +193,8 @@ test('accepts only a reviewed Asia topology wave', () => {
|
||||
for (const cellIds of [
|
||||
'production-gce-c27,production-gce-c28,production-gce-c29',
|
||||
'production-gce-c29,production-gce-c27,production-gce-c28',
|
||||
'production-gce-c30'
|
||||
'production-gce-c30',
|
||||
'production-gce-c31'
|
||||
]) {
|
||||
assert.doesNotThrow(
|
||||
() => parseRelayAsiaTopologyPlanArguments(argv('production', cellIds, productionImage)),
|
||||
@@ -192,7 +206,8 @@ test('accepts only a reviewed Asia topology wave', () => {
|
||||
'production-gce-c27,production-gce-c30',
|
||||
'production-gce-c27,production-gce-c28,production-gce-c29,production-gce-c30',
|
||||
'production-gce-c30,production-gce-c30',
|
||||
'production-gce-c31'
|
||||
'production-gce-c30,production-gce-c31',
|
||||
'production-gce-c32'
|
||||
]) {
|
||||
assert.throws(
|
||||
() => parseRelayAsiaTopologyPlanArguments(argv('production', cellIds, productionImage)),
|
||||
|
||||
@@ -166,11 +166,13 @@ atomically register the new cells as migration-only, binding every mutation to
|
||||
the exact live selector generation and a durable attempt ID. Deploy and verify
|
||||
the director configuration only after registration, then promote C27 alone before C28/C29.
|
||||
|
||||
The production Asia set is C27-C30. C27-C29 launched as one wave; C30 is an additive wave of its
|
||||
own at the same shape. Its plan names C30's template, MIG, and backend plus the shared URL map, and
|
||||
the URL map pulls every existing cell's backend, MIG, and template into the plan. Committed images
|
||||
lag what same-cap rolls serve, so the workflow first reads each non-target cell's served image out
|
||||
of its live template in state and plans that cell at it. It reads the committed cell map from a
|
||||
The production Asia set is C27-C31. C27-C29 launched as one wave; C30 and C31 are each an additive
|
||||
wave of their own at the same shape, and C31 takes `asia-east2-b` so the five cells spread 2/2/1
|
||||
across the zones. The C30 steps below apply to C31 unchanged, with C31 in place of C30. C30's plan
|
||||
names its template, MIG, and backend plus the shared URL map, and the URL map pulls every existing
|
||||
cell's backend, MIG, and template into the plan. Committed images lag what same-cap rolls serve,
|
||||
so the workflow first reads each non-target cell's served image out of its live template in state
|
||||
and plans that cell at it. It reads the committed cell map from a
|
||||
no-refresh, unlocked plan over the same targets, not `terraform console`. Console evaluates every
|
||||
output against state, so `relay_gce_cell_deployments` wraps each per-cell resource lookup in
|
||||
`try`: until C30's topology apply, console succeeds and that output shows C30 with null MIG,
|
||||
@@ -189,6 +191,8 @@ database-pool rules read C30's own metrics only. Director values are recorded un
|
||||
2026-09-23, so the same-cap job now rolls it as a general cell and the shadow gate's fleet pool list
|
||||
reads it beside C27-C29. A later Asia cell stays in the same-cap migration-only list and out of the
|
||||
fleet pool list until its own promotion, then moves to both together, as its own reviewed wave.
|
||||
C31 is in that state now: declared and listed as a same-cap migration-only cell, not yet in the
|
||||
fleet pool list.
|
||||
Rollback returns
|
||||
Asia cells to migration-only; it does not destroy the network or use
|
||||
existing-only. The production topology dispatch remains unavailable until the
|
||||
|
||||
@@ -332,7 +332,7 @@ cell templates/MIGs/backends, and exact shared URL-map host additions. It
|
||||
rejects deletes, replacements, loss of an existing host route, US-resource
|
||||
changes, and unrelated drift. Do not add production C27-C29 until the
|
||||
compatible image has been published and each entry can pin its immutable
|
||||
digest. A later cell, such as C30, is its own reviewed wave. The shared URL map
|
||||
digest. A later cell, such as C30 or C31, is its own reviewed wave. The shared URL map
|
||||
pulls every live cell into its plan, so the workflow plans each live cell at the
|
||||
image its state template already serves, and the validator rejects any change
|
||||
to a cell outside the wave.
|
||||
|
||||
@@ -407,6 +407,20 @@ relay_gce_cells = {
|
||||
connection_hard_cap = 3000
|
||||
connection_unobserved_bound = 60
|
||||
}
|
||||
"production-gce-c31" = {
|
||||
hostname = "c31"
|
||||
region = "asia-east2"
|
||||
zone = "asia-east2-b"
|
||||
machine_type = "e2-standard-4"
|
||||
boot_disk_gb = 30
|
||||
boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21"
|
||||
capacity_requests = 6000
|
||||
database_pool_max = 16 # 176 ms from us-central1 Postgres saturates 10 (94-156 waiters).
|
||||
image = "us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:f30b5cb1ec52b6b6145efecfa1b8be9e3d309403beffd8abcc64197a2087e269"
|
||||
initially_enabled = false
|
||||
connection_hard_cap = 3000
|
||||
connection_unobserved_bound = 60
|
||||
}
|
||||
}
|
||||
|
||||
relay_region_rehome_source_cell_ids = [
|
||||
@@ -430,7 +444,8 @@ relay_region_rehome_source_cell_ids = [
|
||||
"production-gce-c27",
|
||||
"production-gce-c28",
|
||||
"production-gce-c29",
|
||||
"production-gce-c30"
|
||||
"production-gce-c30",
|
||||
"production-gce-c31"
|
||||
]
|
||||
|
||||
# Slack #orca-relay-alerts, created out of band on 2026-08-05. Declared here because an apply
|
||||
|
||||
Reference in New Issue
Block a user