import assert from 'node:assert/strict' import { execFileSync, spawnSync } from 'node:child_process' import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { dirname, join } from 'node:path' import { test } from 'node:test' import { CANARY_MIN_DRAINED_HOSTS, DEFAULT_SAME_CAP_DRAIN_PACE_WINDOW_MS, SAME_CAP_CELLS, SAME_CAP_DRAIN_PACE_WINDOWS_MS, SAME_CAP_FAST_DRAIN_PACE_CELLS, SAME_CAP_MIGRATION_ONLY_CELLS, canaryAuthority, entryAdmission, main, selectorWaveDelta, validateSameCapWave, verifyCanaryAuthority } from './relay-production-same-cap-wave.mjs' import { readRelayWorkflow } from './relay-repository.mjs' const targetDigest = `sha256:${'a'.repeat(64)}` const rollbackDigest = `sha256:${'b'.repeat(64)}` test('requires one canary or a bounded reviewed batch', () => { assert.deepEqual(validateSameCapWave({ mode: 'canary-apply', cellIds: 'production-gce-c7', targetDigest, rollbackDigest, drainPaceWindowMs: '300000', confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7` }).cells, ['production-gce-c7']) assert.throws(() => validateSameCapWave({ mode: 'canary-apply', cellIds: 'production-gce-c7,production-gce-c8', targetDigest, rollbackDigest, drainPaceWindowMs: '300000', confirmation: 'wrong' }), /canary/) assert.deepEqual(validateSameCapWave({ mode: 'batch-apply', cellIds: 'production-gce-c8,production-gce-c9', targetDigest, rollbackDigest, drainPaceWindowMs: '300000', confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c8,production-gce-c9`, canaryRunId: '42' }).cells, ['production-gce-c8', 'production-gce-c9']) assert.deepEqual(validateSameCapWave({ mode: 'canary-apply', cellIds: 'production-gce-c28', targetDigest, rollbackDigest, drainPaceWindowMs: '300000', confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c28` }).cells, ['production-gce-c28']) assert.deepEqual(validateSameCapWave({ mode: 'canary-apply', cellIds: 'production-gce-c30', targetDigest, rollbackDigest, drainPaceWindowMs: '300000', confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c30` }).cells, ['production-gce-c30']) assert.deepEqual(validateSameCapWave({ mode: 'canary-apply', cellIds: 'production-gce-c31', targetDigest, rollbackDigest, drainPaceWindowMs: '300000', confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c31` }).cells, ['production-gce-c31']) for (const cellId of ['production-gce-c32', 'production-gce-c33', 'production-gce-c34']) { assert.deepEqual(validateSameCapWave({ mode: 'canary-apply', cellIds: cellId, targetDigest, rollbackDigest, drainPaceWindowMs: '300000', confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${cellId}` }).cells, [cellId]) } assert.throws(() => validateSameCapWave({ mode: 'canary-apply', cellIds: 'production-gce-c35', targetDigest, rollbackDigest, drainPaceWindowMs: '300000', confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c35` }), /cells/) }) // The bound is the wave workflow's static cell_1..cell_10 chain: a batch longer than the // chain would silently drop its tail cells, so it is refused before any mutation. test('a batch fills the serial cell chain and never overflows it', () => { const general = SAME_CAP_CELLS.filter((cell) => entryAdmission(cell) === 'general') const batch = (count) => { const cellIds = general.slice(0, count).join(',') return validateSameCapWave({ mode: 'batch-apply', cellIds, targetDigest, rollbackDigest, drainPaceWindowMs: '300000', confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${cellIds}`, canaryRunId: '42' }) } assert.equal(batch(10).cells.length, 10) assert.throws(() => batch(11), /same-cap wave cells are invalid/) assert.throws(() => batch(1), /batch mode requires two to ten cells/) }) // The validator's ten-cell bound is only true if the workflow really declares ten strictly // serial cell jobs and frees the lease after all of them. test('the wave workflow chains exactly ten serial cell jobs', () => { const dispatch = readRelayWorkflow('deploy-relay-production-same-cap.yml') for (let index = 0; index < 10; index += 1) { const job = index + 1 assert.match(dispatch, new RegExp(`\n cell_${job}:\n`), `cell_${job} is missing`) assert.match(dispatch, new RegExp(`fromJSON\\(needs\\.gate\\.outputs\\.cells\\)\\[${index}\\]`)) assert.match(dispatch, new RegExp(`wave-index: '${index}'`)) if (index > 0) { assert.match(dispatch, new RegExp(`needs: \\[gate, cell_${index}\\]`)) assert.match( dispatch, new RegExp(`if: \\$\\{\\{ needs\\.cell_${index}\\.result == 'success' && ` + `fromJSON\\(needs\\.gate\\.outputs\\.cells\\)\\[${index}\\] != null \\}\\}`) ) } assert.match(dispatch, new RegExp(`\n - cell_${job}\n`), `release_lease must need cell_${job}`) } assert.doesNotMatch(dispatch, /\n cell_11:/) }) test('lists the C34 spare as migration-only beside C17 and C18, and C30-C33 as general', () => { assert.deepEqual( SAME_CAP_MIGRATION_ONLY_CELLS, ['production-gce-c17', 'production-gce-c18', 'production-gce-c34'] ) for (const cellId of [ 'production-gce-c30', 'production-gce-c31', 'production-gce-c32', 'production-gce-c33' ]) { assert.equal(SAME_CAP_CELLS.includes(cellId), true, cellId) } }) test('rolls the migration-only cells but never mixes the two classes in one wave', () => { for (const cellId of SAME_CAP_MIGRATION_ONLY_CELLS) { assert.equal(SAME_CAP_CELLS.includes(cellId), true, cellId) assert.equal(entryAdmission(cellId), 'migration-only', cellId) assert.deepEqual(validateSameCapWave({ mode: 'canary-apply', cellIds: cellId, targetDigest, rollbackDigest, drainPaceWindowMs: '300000', confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${cellId}` }).cells, [cellId]) } const cellIds = 'production-gce-c17,production-gce-c18' assert.deepEqual(validateSameCapWave({ mode: 'batch-apply', cellIds, targetDigest, rollbackDigest, drainPaceWindowMs: '300000', confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${cellIds}`, canaryRunId: '42' }).cells, ['production-gce-c17', 'production-gce-c18']) // C30 is general since its 2026-09-23 promotion, so it cannot share a wave with C17/C18. assert.equal(entryAdmission('production-gce-c30'), 'general') assert.equal(selectorWaveDelta('production-gce-c30'), 2) const asiaGeneral = 'production-gce-c29,production-gce-c30' assert.deepEqual(validateSameCapWave({ mode: 'batch-apply', cellIds: asiaGeneral, targetDigest, rollbackDigest, drainPaceWindowMs: '300000', confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${asiaGeneral}`, canaryRunId: '42' }).cells, ['production-gce-c29', 'production-gce-c30']) const asiaMixed = 'production-gce-c30,production-gce-c17' assert.throws(() => validateSameCapWave({ mode: 'batch-apply', cellIds: asiaMixed, targetDigest, rollbackDigest, drainPaceWindowMs: '300000', confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${asiaMixed}`, canaryRunId: '42' }), /all general or all migration-only/) // C31 is general since its 2026-10-01 promotion, so it rolls beside C30 but never C17/C18. assert.equal(entryAdmission('production-gce-c31'), 'general') assert.equal(selectorWaveDelta('production-gce-c31'), 2) const asiaPromoted = 'production-gce-c30,production-gce-c31' assert.deepEqual(validateSameCapWave({ mode: 'batch-apply', cellIds: asiaPromoted, targetDigest, rollbackDigest, drainPaceWindowMs: '300000', confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${asiaPromoted}`, canaryRunId: '42' }).cells, ['production-gce-c30', 'production-gce-c31']) const c31Mixed = 'production-gce-c31,production-gce-c18' assert.throws(() => validateSameCapWave({ mode: 'batch-apply', cellIds: c31Mixed, targetDigest, rollbackDigest, drainPaceWindowMs: '300000', confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${c31Mixed}`, canaryRunId: '42' }), /all general or all migration-only/) // C32 and C33 are general since their 2026-10-01 promotions: they roll beside US 1k cells, // isolate and restore (delta 2), and never share a wave with C17/C18. for (const cellId of ['production-gce-c32', 'production-gce-c33']) { assert.equal(entryAdmission(cellId), 'general', cellId) assert.equal(selectorWaveDelta(cellId), 2, cellId) } const usPromoted = 'production-gce-c26,production-gce-c32,production-gce-c33' assert.deepEqual(validateSameCapWave({ mode: 'batch-apply', cellIds: usPromoted, targetDigest, rollbackDigest, drainPaceWindowMs: '300000', confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${usPromoted}`, canaryRunId: '42' }).cells, ['production-gce-c26', 'production-gce-c32', 'production-gce-c33']) const usMixed = 'production-gce-c32,production-gce-c17' assert.throws(() => validateSameCapWave({ mode: 'batch-apply', cellIds: usMixed, targetDigest, rollbackDigest, drainPaceWindowMs: '300000', confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${usMixed}`, canaryRunId: '42' }), /all general or all migration-only/) // A mixed wave has no single selector delta for its later cells to offset from. const mixed = 'production-gce-c7,production-gce-c17' assert.throws(() => validateSameCapWave({ mode: 'batch-apply', cellIds: mixed, targetDigest, rollbackDigest, drainPaceWindowMs: '300000', confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${mixed}`, canaryRunId: '42' }), /all general or all migration-only/) }) test('seals a migration-only canary at the generation its wave leaves behind', () => { const seal = (cellId) => canaryAuthority({ cellIds: cellId, targetDigest, rollbackDigest, drainPaceWindowMs: '300000', confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${cellId}`, commitSha: 'c'.repeat(40), runId: '42', selectorGeneration: '11', rehomeGeneration: '4' }) // Isolate and restore are both no-ops on a migration-only cell, so nothing advances. assert.equal(seal('production-gce-c17').selectorGeneration, 11) assert.equal(seal('production-gce-c7').selectorGeneration, 13) // That canary still authorizes a later batch of its own class; it is evidence about the image. assert.equal(verifyCanaryAuthority(seal('production-gce-c17'), { commitSha: 'c'.repeat(40), runId: '42', cellIds: 'production-gce-c17,production-gce-c18', targetDigest, rollbackDigest, drainPaceWindowMs: '300000', selectorGeneration: '11', rehomeGeneration: '4' }).cellId, 'production-gce-c17') }) test('reports each approved cell\'s class and selector delta', () => { const printed = [] const write = process.stdout.write.bind(process.stdout) process.stdout.write = (chunk) => printed.push(String(chunk)) try { main(['cell-class', '--cell-id', 'production-gce-c17', '--drain-pace-window-ms', '300000']) main(['cell-class', '--cell-id', 'production-gce-c7', '--drain-pace-window-ms', '60000']) } finally { process.stdout.write = write } assert.deepEqual(printed.map((line) => JSON.parse(line)), [ { entryAdmission: 'migration-only', selectorWaveDelta: 0, drainPaceWindowMs: 300000 }, { entryAdmission: 'general', selectorWaveDelta: 2, drainPaceWindowMs: 60000 } ]) assert.throws(() => main([ 'cell-class', '--cell-id', 'production-gce-c12', '--drain-pace-window-ms', '300000' ]), /cells are invalid/) // The job re-checks the pace per cell, so a cell outside the fast list cannot drain fast. assert.throws(() => main([ 'cell-class', '--cell-id', 'production-gce-c28', '--drain-pace-window-ms', '30000' ]), /US general cells only, not production-gce-c28/) assert.throws(() => main(['cell-class', '--cell-id', 'production-gce-c7']), /drain pace window/) }) test('binds rollback confirmation to the exact digest and ordered cells', () => { assert.throws(() => validateSameCapWave({ mode: 'rollback', cellIds: 'production-gce-c7', targetDigest, rollbackDigest, drainPaceWindowMs: '300000', confirmation: `ROLL_BACK_RELAY_SAME_CAP ${targetDigest} production-gce-c7` }), /confirmation/) }) test('rollback rolls exactly one cell so later waves stay unreachable', () => { const cellIds = 'production-gce-c7,production-gce-c8' assert.throws(() => validateSameCapWave({ mode: 'rollback', cellIds, targetDigest, rollbackDigest, drainPaceWindowMs: '300000', confirmation: `ROLL_BACK_RELAY_SAME_CAP ${rollbackDigest} ${cellIds}` }), /rollback mode requires exactly one cell/) assert.deepEqual(validateSameCapWave({ mode: 'rollback', cellIds: 'production-gce-c7', targetDigest, rollbackDigest, drainPaceWindowMs: '300000', confirmation: `ROLL_BACK_RELAY_SAME_CAP ${rollbackDigest} production-gce-c7` }).cells, ['production-gce-c7']) }) test('seals and verifies canary authority for later batches', () => { const authority = canaryAuthority({ cellIds: 'production-gce-c7', targetDigest, rollbackDigest, drainPaceWindowMs: '300000', confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7`, commitSha: 'c'.repeat(40), runId: '42', selectorGeneration: '11', rehomeGeneration: '4' }) assert.equal(verifyCanaryAuthority(authority, { commitSha: 'c'.repeat(40), runId: '42', cellIds: 'production-gce-c8,production-gce-c9', targetDigest, rollbackDigest, drainPaceWindowMs: '300000', selectorGeneration: '13', rehomeGeneration: '4' }).cellId, 'production-gce-c7') assert.throws(() => verifyCanaryAuthority(authority, { commitSha: 'd'.repeat(40), runId: '42', cellIds: 'production-gce-c8,production-gce-c9', targetDigest, rollbackDigest, drainPaceWindowMs: '300000', selectorGeneration: '11', rehomeGeneration: '4' }), /does not match/) }) test('reuses a canary across selector advances only within the same control epoch', () => { const authority = canaryAuthority({ cellIds: 'production-gce-c7', targetDigest, rollbackDigest, drainPaceWindowMs: '300000', confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7`, commitSha: 'c'.repeat(40), runId: '42', selectorGeneration: '11', rehomeGeneration: '4' }) const expected = { commitSha: 'c'.repeat(40), runId: '42', cellIds: 'production-gce-c8,production-gce-c9', targetDigest, rollbackDigest, drainPaceWindowMs: '300000', selectorGeneration: '21', rehomeGeneration: '4' } for (const generation of ['13', '14', '21', '29']) { assert.equal(verifyCanaryAuthority(authority, { ...expected, selectorGeneration: generation }), authority) } for (const generation of ['12', '-1', 'NaN', 'Infinity', '13.5', '9007199254740992']) { assert.throws(() => verifyCanaryAuthority(authority, { ...expected, selectorGeneration: generation }), /does not match/) } for (const generation of [-1, NaN, Infinity, 13.5, '13', Number.MAX_SAFE_INTEGER + 1]) { assert.throws(() => verifyCanaryAuthority({ ...authority, selectorGeneration: generation }, expected), /does not match/) } for (const mismatch of [ { rehomeGeneration: '3' }, { rehomeGeneration: '5' }, { targetDigest: rollbackDigest }, { rollbackDigest: targetDigest }, { runId: '43' } ]) { assert.throws(() => verifyCanaryAuthority(authority, { ...expected, ...mismatch }), /does not match/) } }) function gitIn(root, ...args) { return execFileSync('git', ['-C', root, ...args], { encoding: 'utf8' }).trim() } async function canaryRepository() { const root = await mkdtemp(join(tmpdir(), 'relay-same-cap-canary-')) gitIn(root, 'init', '--quiet') gitIn(root, 'config', 'user.email', 'relay@example.test') gitIn(root, 'config', 'user.name', 'Relay Wave Test') gitIn(root, 'config', 'commit.gpgsign', 'false') const commit = async (path, body, message) => { await mkdir(dirname(join(root, path)), { recursive: true }) await writeFile(join(root, path), body) gitIn(root, 'add', '--all') gitIn(root, 'commit', '--quiet', '--no-verify', '--message', message) return gitIn(root, 'rev-parse', 'HEAD') } const sealed = await commit( 'cloud/dev/scripts/relay-production-same-cap-wave.mjs', 'export const v = 1\n', 'wave' ) const sameCode = await commit('README.md', 'an unrelated merge\n', 'unrelated') const changedCode = await commit( 'cloud/dev/scripts/relay-production-same-cap-wave.mjs', 'export const v = 2\n', 'wave change' ) return { root, sealed, sameCode, changedCode } } test('a batch trusts a canary sealed by identical code at an ancestor commit', async () => { const repository = await canaryRepository() try { const authority = canaryAuthority({ cellIds: 'production-gce-c7', targetDigest, rollbackDigest, drainPaceWindowMs: '300000', confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7`, commitSha: repository.sealed, runId: '42', selectorGeneration: '11', rehomeGeneration: '4' }) const verifyAt = (commitSha, repositoryRoot) => verifyCanaryAuthority(authority, { commitSha, runId: '42', cellIds: 'production-gce-c8,production-gce-c9', targetDigest, rollbackDigest, drainPaceWindowMs: '300000', selectorGeneration: '21', rehomeGeneration: '4' }, repositoryRoot) assert.equal(verifyAt(repository.sameCode, repository.root).cellId, 'production-gce-c7') assert.throws( () => verifyAt(repository.changedCode, repository.root), /code changed after it was sealed/ ) assert.throws(() => verifyAt('f'.repeat(40), repository.root), /unknown to this checkout/) } finally { await rm(repository.root, { recursive: true, force: true }) } }) function sealedCanary(cellId) { return canaryAuthority({ cellIds: cellId, targetDigest, rollbackDigest, drainPaceWindowMs: '300000', confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${cellId}`, commitSha: 'c'.repeat(40), runId: '42', selectorGeneration: '11', rehomeGeneration: '4' }) } // Why: a migration-only cell holds zero hosts at a different cap and its wave advances no // selector, so rolling one is no evidence for a general batch, and the reverse is no evidence // either. Nothing but the sealed cell id says which class a canary actually proved. test('refuses a canary sealed on a cell of the other admission class', () => { const expected = { commitSha: 'c'.repeat(40), runId: '42', targetDigest, rollbackDigest, drainPaceWindowMs: '300000', selectorGeneration: '99', rehomeGeneration: '4' } const general = 'production-gce-c8,production-gce-c9' const migrationOnly = SAME_CAP_MIGRATION_ONLY_CELLS.join(',') assert.throws( () => verifyCanaryAuthority(sealedCanary('production-gce-c17'), { ...expected, cellIds: general }), /canary authority cell production-gce-c17 is migration-only, but this batch is general/ ) assert.throws( () => verifyCanaryAuthority(sealedCanary('production-gce-c7'), { ...expected, cellIds: migrationOnly }), /canary authority cell production-gce-c7 is general, but this batch is migration-only/ ) assert.equal( verifyCanaryAuthority(sealedCanary('production-gce-c7'), { ...expected, cellIds: general }).cellId, 'production-gce-c7' ) assert.equal( verifyCanaryAuthority(sealedCanary('production-gce-c17'), { ...expected, cellIds: migrationOnly }).cellId, 'production-gce-c17' ) // A caller that names no batch at all gets no verdict, rather than an unchecked class. assert.throws( () => verifyCanaryAuthority(sealedCanary('production-gce-c7'), expected), /same-cap wave cells are invalid/ ) }) // The dispatch workflow is the only caller, so the class check only binds anything if that // step actually hands the batch over; run the step's own shell exactly as written. function verifyCanaryStepScript() { const dispatch = readRelayWorkflow('deploy-relay-production-same-cap.yml') const first = ' node dev/scripts/relay-production-same-cap-wave.mjs verify-canary \\\n' const start = dispatch.indexOf(first) assert.notEqual(start, -1, 'the dispatch workflow has no verify-canary step') const last = ' --rehome-generation "${REHOME_GENERATION}"\n' const end = dispatch.indexOf(last, start) assert.notEqual(end, -1, 'the verify-canary step does not end at the rehome generation') return dispatch.slice(start, end + last.length).replace(/^ {10}/gm, '') } async function runVerifyCanaryStep(authority, cellIds, drainPaceWindowMs = '300000') { const temporary = await mkdtemp(join(tmpdir(), 'relay-same-cap-verify-')) try { await mkdir(join(temporary, 'relay-same-cap-canary'), { recursive: true }) await writeFile( join(temporary, 'relay-same-cap-canary', 'authority.json'), JSON.stringify(authority) ) return spawnSync('bash', ['-euo', 'pipefail', '-c', verifyCanaryStepScript()], { cwd: new URL('../..', import.meta.url), env: { ...process.env, RUNNER_TEMP: temporary, GITHUB_SHA: authority.commitSha, CANARY_RUN_ID: authority.runId, CELL_IDS: cellIds, TARGET_DIGEST: targetDigest, ROLLBACK_DIGEST: rollbackDigest, SELECTOR_GENERATION: '99', REHOME_GENERATION: '4', DRAIN_PACE_WINDOW_MS: drainPaceWindowMs }, encoding: 'utf8' }) } finally { await rm(temporary, { recursive: true, force: true }) } } test('the batch gate hands its own cells to the canary check', async () => { const accepted = await runVerifyCanaryStep( sealedCanary('production-gce-c7'), 'production-gce-c8,production-gce-c9' ) assert.equal(accepted.status, 0, accepted.stderr) const crossed = await runVerifyCanaryStep( sealedCanary('production-gce-c17'), 'production-gce-c8,production-gce-c9' ) assert.equal(crossed.status, 1, crossed.stdout) assert.match( crossed.stderr, /canary authority cell production-gce-c17 is migration-only, but this batch is general/ ) const migrationOnly = await runVerifyCanaryStep( sealedCanary('production-gce-c17'), SAME_CAP_MIGRATION_ONLY_CELLS.join(',') ) assert.equal(migrationOnly.status, 0, migrationOnly.stderr) // The batch's own pace reaches the check, so a 300 s canary cannot authorize a faster batch. const faster = await runVerifyCanaryStep( sealedCanary('production-gce-c7'), 'production-gce-c8,production-gce-c9', '60000' ) assert.equal(faster.status, 1, faster.stdout) assert.match(faster.stderr, /drained over 300000 ms, so it cannot authorize a batch draining over 60000 ms/) }) const usGeneral = 'production-gce-c8,production-gce-c32' function fastWave(overrides = {}) { return { mode: 'batch-apply', cellIds: usGeneral, targetDigest, rollbackDigest, drainPaceWindowMs: '60000', confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${usGeneral} drain-pace-window-ms=60000`, canaryRunId: '42', ...overrides } } test('admits only the reviewed drain pace windows', () => { assert.deepEqual(SAME_CAP_DRAIN_PACE_WINDOWS_MS, [300_000, 60_000, 30_000]) assert.equal(validateSameCapWave(fastWave()).drainPaceWindowMs, 60_000) assert.equal(validateSameCapWave(fastWave({ drainPaceWindowMs: '30000', confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${usGeneral} drain-pace-window-ms=30000` })).drainPaceWindowMs, 30_000) for (const pace of [undefined, '', '0', '120000', '299999', '600000', '60000.0', '060000', ' 60000']) { assert.throws( () => validateSameCapWave(fastWave({ drainPaceWindowMs: pace })), /drain pace window must be one of 300000, 60000, 30000 ms/, String(pace) ) } // Verify reads production, never drains, but still states the pace it would roll at. assert.equal(validateSameCapWave(fastWave({ mode: 'verify', confirmation: '', canaryRunId: '' })).drainPaceWindowMs, 60_000) }) test('keeps Asia and migration-only cells on the default pace', () => { assert.deepEqual( SAME_CAP_FAST_DRAIN_PACE_CELLS.filter((cell) => !SAME_CAP_CELLS.includes(cell)), [] ) for (const cellId of SAME_CAP_CELLS) { const fast = SAME_CAP_FAST_DRAIN_PACE_CELLS.includes(cellId) const asia = ['c27', 'c28', 'c29', 'c30', 'c31', 'c34'].some( (cell) => cellId === `production-gce-${cell}` ) assert.equal(fast, entryAdmission(cellId) === 'general' && !asia, cellId) } const asia = 'production-gce-c30,production-gce-c31' assert.throws(() => validateSameCapWave(fastWave({ cellIds: asia, confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${asia} drain-pace-window-ms=60000` })), /US general cells only, not production-gce-c30,production-gce-c31/) // One Asia cell in an otherwise US batch is enough to refuse the whole batch. const mixed = 'production-gce-c8,production-gce-c29' assert.throws(() => validateSameCapWave(fastWave({ cellIds: mixed, confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${mixed} drain-pace-window-ms=60000` })), /not production-gce-c29$/) assert.throws(() => validateSameCapWave(fastWave({ mode: 'canary-apply', cellIds: 'production-gce-c17', confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c17 drain-pace-window-ms=60000`, canaryRunId: '' })), /not production-gce-c17/) }) test('binds a non-default pace into the confirmation, in both directions', () => { // A confirmation that does not name the pace is a confirmation of the default. assert.throws(() => validateSameCapWave(fastWave({ confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${usGeneral}` })), /confirmation does not match/) assert.throws(() => validateSameCapWave(fastWave({ confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${usGeneral} drain-pace-window-ms=30000` })), /confirmation does not match/) assert.throws(() => validateSameCapWave(fastWave({ drainPaceWindowMs: '300000' })), /confirmation does not match/) assert.throws(() => validateSameCapWave(fastWave({ drainPaceWindowMs: '300000', confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${usGeneral} drain-pace-window-ms=300000` })), /confirmation does not match/) assert.deepEqual(validateSameCapWave(fastWave({ mode: 'rollback', cellIds: 'production-gce-c8', confirmation: `ROLL_BACK_RELAY_SAME_CAP ${rollbackDigest} production-gce-c8 drain-pace-window-ms=60000`, canaryRunId: '' })).cells, ['production-gce-c8']) }) function shadowReport(pace, paceVerdict = 'PASS', overrides = {}) { return { cellId: 'production-gce-c7', paceVerdict, drain: { paceWindowMs: Number(pace), appliedPaceWindowMs: Number(pace), targetHosts: 692 }, ...overrides } } test('a canary authorizes batches at its own pace or slower, never faster', () => { const seal = (pace, report = shadowReport(pace)) => canaryAuthority({ cellIds: 'production-gce-c7', targetDigest, rollbackDigest, drainPaceWindowMs: pace, shadowReport: report, confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7` + (pace === '300000' ? '' : ` drain-pace-window-ms=${pace}`), commitSha: 'c'.repeat(40), runId: '42', selectorGeneration: '11', rehomeGeneration: '4' }) assert.equal(seal('60000').v, 2) assert.equal(seal('60000').drainPaceWindowMs, 60_000) const verify = (authority, pace) => verifyCanaryAuthority(authority, { commitSha: 'c'.repeat(40), runId: '42', cellIds: usGeneral, targetDigest, rollbackDigest, drainPaceWindowMs: pace, selectorGeneration: '13', rehomeGeneration: '4' }) // Stepping back to the default after a bad rung needs no new canary. for (const pace of ['60000', '300000']) assert.equal(verify(seal('60000'), pace).cellId, 'production-gce-c7') assert.throws(() => verify(seal('60000'), '30000'), /drained over 60000 ms/) assert.throws(() => verify(seal('300000'), '60000'), /drained over 300000 ms/) // An authority sealed before the pace was recorded proves nothing about it. const { drainPaceWindowMs: _dropped, ...unpaced } = seal('300000') assert.throws(() => verify({ ...unpaced, v: 1 }, '300000'), /does not match/) assert.throws(() => verify({ ...seal('300000'), drainPaceWindowMs: 0 }, '300000'), /does not match/) assert.throws(() => verify({ ...seal('300000'), paceVerdict: 'pass' }, '300000'), /does not match/) }) // A canary whose own drain looked bad still succeeds as a job, so the seal must carry what its pace // checks said, and a faster batch must refuse anything but PASS. test('only a canary whose pace checks passed authorizes a faster batch', () => { const seal = (report) => canaryAuthority({ cellIds: 'production-gce-c7', targetDigest, rollbackDigest, drainPaceWindowMs: '60000', shadowReport: report, confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7 drain-pace-window-ms=60000`, commitSha: 'c'.repeat(40), runId: '42', selectorGeneration: '11', rehomeGeneration: '4' }) const verify = (authority, pace) => verifyCanaryAuthority(authority, { commitSha: 'c'.repeat(40), runId: '42', cellIds: usGeneral, targetDigest, rollbackDigest, drainPaceWindowMs: pace, selectorGeneration: '13', rehomeGeneration: '4' }) assert.equal(seal(shadowReport('60000')).paceVerdict, 'PASS') assert.equal(CANARY_MIN_DRAINED_HOSTS, 400) assert.equal(seal(shadowReport('60000', 'PASS', { drain: { paceWindowMs: 60000, appliedPaceWindowMs: 60000, targetHosts: 400 } })).paceVerdict, 'PASS') for (const [report, sealed] of [ [shadowReport('60000', 'WARN'), 'WARN'], [shadowReport('60000', 'WOULD_BLOCK'), 'WOULD_BLOCK'], [null, 'UNVERIFIED'], // Another cell's report, another pace's, or a cell that fell back to an unpaced drain. [shadowReport('60000', 'PASS', { cellId: 'production-gce-c8' }), 'UNVERIFIED'], [shadowReport('300000'), 'UNVERIFIED'], [shadowReport('60000', 'PASS', { drain: { paceWindowMs: 60000, appliedPaceWindowMs: 0, targetHosts: 692 } }), 'UNVERIFIED'], // Too few hosts to have tested the pace, or no host count at all. [shadowReport('60000', 'PASS', { drain: { paceWindowMs: 60000, appliedPaceWindowMs: 60000, targetHosts: 399 } }), 'UNVERIFIED'], [shadowReport('60000', 'PASS', { drain: { paceWindowMs: 60000, appliedPaceWindowMs: 60000, targetHosts: 0 } }), 'UNVERIFIED'], [shadowReport('60000', 'PASS', { drain: { paceWindowMs: 60000, appliedPaceWindowMs: 60000, targetHosts: null } }), 'UNVERIFIED'], [shadowReport('60000', 'MAYBE'), 'UNVERIFIED'] ]) { const authority = seal(report) assert.equal(authority.paceVerdict, sealed) assert.throws(() => verify(authority, '60000'), new RegExp(`pace checks were ${sealed}`)) // Falling back to the default never needs a passing canary. assert.equal(verify(authority, '300000').cellId, 'production-gce-c7') } }) test('create-canary seals the verdict from the canary cell\'s shadow report file', async () => { const temporary = await mkdtemp(join(tmpdir(), 'relay-same-cap-seal-')) const printed = [] const write = process.stdout.write.bind(process.stdout) try { const path = join(temporary, 'report.json') await writeFile(path, JSON.stringify(shadowReport('60000'))) process.stdout.write = (chunk) => printed.push(String(chunk)) for (const report of [path, join(temporary, 'missing.json')]) { main([ 'create-canary', '--cell-id', 'production-gce-c7', '--target-digest', targetDigest, '--rollback-digest', rollbackDigest, '--confirmation', `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7 drain-pace-window-ms=60000`, '--drain-pace-window-ms', '60000', '--shadow-report', report, '--commit-sha', 'c'.repeat(40), '--run-id', '42', '--selector-generation', '11', '--rehome-generation', '4' ]) } } finally { process.stdout.write = write await rm(temporary, { recursive: true, force: true }) } assert.deepEqual(printed.map((line) => JSON.parse(line).paceVerdict), ['PASS', 'UNVERIFIED']) }) // The dispatch choice list, the validator's closed set, and the job's transition wait have to // agree, or the form offers a pace the gate refuses or the wait outlives the 20-min token. test('the workflows offer exactly the closed set and scale the drain wait with it', () => { const dispatch = readRelayWorkflow('deploy-relay-production-same-cap.yml') const input = dispatch.slice(dispatch.indexOf(' drain-pace-window-ms:\n')) assert.match(input, /default: '300000'\n/) assert.deepEqual( JSON.parse(/options: (\[[^\]]+\])/.exec(input)[1].replaceAll("'", '"')).map(Number), SAME_CAP_DRAIN_PACE_WINDOWS_MS ) const seal = dispatch.slice(dispatch.indexOf('\n seal_canary:')) assert.match( seal, /name: relay-same-cap-shadow-gate-\$\{\{ fromJSON\(needs\.gate\.outputs\.cells\)\[0\] \}\}-\$\{\{ github\.run_id \}\}\.json\n/ ) assert.ok(seal.indexOf('download-artifact') < seal.indexOf('create-canary')) assert.match(seal, /--shadow-report "\$\{RUNNER_TEMP\}\/relay-same-cap-shadow-gate\/relay-same-cap-shadow-gate-\$\{\{ fromJSON\(needs\.gate\.outputs\.cells\)\[0\] \}\}-\$\{GITHUB_RUN_ID\}\.json"/) // The seal names the cell the gate normalized, never the raw input a padded form could carry. assert.doesNotMatch(seal.slice(0, seal.indexOf('\n release_lease:')), /inputs\.cell-ids/) // The job uploads under the same normalized cell, which cell_1 receives as target-cell-id. assert.match(dispatch, /target-cell-id: \$\{\{ fromJSON\(needs\.gate\.outputs\.cells\)\[0\] \}\}/) for (const command of ['validate', 'verify-canary', 'create-canary']) { const at = dispatch.indexOf(`relay-production-same-cap-wave.mjs ${command}`) assert.notEqual(at, -1, command) const end = dispatch.indexOf('\n\n', at) assert.match(dispatch.slice(at, end), /--drain-pace-window-ms "\$\{/, command) } assert.equal( dispatch.match(/\n drain-pace-window-ms: \$\{\{ inputs\.drain-pace-window-ms \}\}\n/g).length, 10 ) const job = readRelayWorkflow('deploy-relay-production-same-cap-job.yml') assert.match(job, /\n DRAIN_PACE_WINDOW_MS: \$\{\{ inputs\.drain-pace-window-ms \}\}\n/) assert.doesNotMatch(job, /DRAIN_PACE_WINDOW_MS: '/) assert.match(job, /cell-class \\\n\s+--cell-id "\$\{TARGET_CELL_ID\}" --drain-pace-window-ms "\$\{DRAIN_PACE_WINDOW_MS\}"/) // The 15-min migration lease plus the window: exactly today's 20 min at the default. assert.match(job, /--timeout-ms "\$\(\(900000 \+ DRAIN_PACE_WINDOW_MS\)\)"/) assert.equal(900_000 + SAME_CAP_DRAIN_PACE_WINDOWS_MS[0], 1_200_000) assert.ok(Math.max(...SAME_CAP_DRAIN_PACE_WINDOWS_MS) === DEFAULT_SAME_CAP_DRAIN_PACE_WINDOW_MS) })