Files
orca/cloud/dev/scripts/relay-production-same-cap-wave.mjs
T
Jinwoo Hong 09622f0c28 feat(relay): add a break-glass override for the same-cap monitor gate (#21270)
* feat(relay): add a break-glass override for the same-cap monitor gate

Every mutating same-cap wave consumes a fresh 15-minute aggregate monitor
dry-run. When a chronic fault is what the gate freezes on, waiting for a green
window means waiting for the condition the wave removes: the gate froze 44
consecutive times on the recurring Cloud SQL stall the rolling image fixes.

Add `gate-override-reason` and `gate-override-confirmation`
(`SKIP_RELAY_MONITOR_GATE <target-image-digest>`) to the same-cap dispatch. A
valid pair skips only the aggregate evidence download, provenance verification,
and single-use marker. A partial or mismatched override fails closed before any
mutation, in both the caller and the reusable job. Record the actor, reason, and
confirmation in the gate run summary and, for a canary, in the sealed artifact.

The live per-wave preflight still runs. Give it a `--no-monitor-state` source
that takes the expected selector from the dispatch inputs and pins the migration
policy to `strict`, rather than synthesising a state file that would claim a
dry-run it never ran.

Also give `director.instances` the two-consecutive-sample tolerance the cell
probes have: Cloud Run replaces an instance in place, so the count leaves the
[5, 6] band for one sample roughly twice a day, and a deploy overlap raises it
the same way. Min and max share one streak so an alternating count still freezes.

* fix(relay): canonicalise the break-glass preflight membership

The override path parsed the operator's membership with a bare schema parse,
while the live selector read from the director is normalised and the comparison
is an ordered `JSON.stringify`. Unsorted dispatch input would therefore read as
selector drift on a healthy fleet, and the every-configured-cell-exactly-once
check was lost with it.

Normalise through the same `normalizeSelectorMembership` call the monitor CLI
uses when it seals evidence, against the same durable Terraform cell set.

Tests use a collect stub that returns the director's canonical selector rather
than echoing the expected one, so the ordering is actually exercised: unsorted
input must canonicalise, and a duplicated, missing, or unknown cell must be
rejected.
2026-09-17 15:05:46 -04:00

205 lines
8.4 KiB
JavaScript

import { readFileSync } from 'node:fs'
import { pathToFileURL } from 'node:url'
import { requireSameEvidenceCode } from './relay-evidence-code-provenance.mjs'
export const SAME_CAP_CELLS = [
'production-gce-c7', 'production-gce-c8', 'production-gce-c9', 'production-gce-c10',
'production-gce-c13', 'production-gce-c14', 'production-gce-c15', 'production-gce-c16',
'production-gce-c19', 'production-gce-c20', 'production-gce-c21', 'production-gce-c22',
'production-gce-c23', 'production-gce-c24', 'production-gce-c25', 'production-gce-c26',
'production-gce-c27', 'production-gce-c28', 'production-gce-c29'
]
function digest(value, name) {
if (!/^sha256:[a-f0-9]{64}$/.test(value ?? '')) throw new Error(`${name} is invalid`)
return value
}
function cells(value) {
const parsed = value.split(',').map((cell) => cell.trim()).filter(Boolean)
if (
parsed.length < 1 ||
parsed.length > 4 ||
new Set(parsed).size !== parsed.length ||
parsed.some((cell) => !SAME_CAP_CELLS.includes(cell))
) throw new Error('same-cap wave cells are invalid')
return parsed
}
// Break-glass: the aggregate 15-minute monitor gate is skipped, nothing else is.
// Returns null when no override was requested, and throws on a partial or
// mismatched one so a malformed override can never reach a mutation.
export function gateOverrideAuthorization(input, targetDigest, mutation) {
const reason = input.gateOverrideReason ?? ''
const confirmation = input.gateOverrideConfirmation ?? ''
if (!reason && !confirmation) return null
if (!mutation) throw new Error('verify does not accept a monitor gate override')
if (confirmation !== `SKIP_RELAY_MONITOR_GATE ${targetDigest}`) {
throw new Error('gate override confirmation does not match the exact target digest')
}
// Printable single-line only: this reason is rendered into the run summary and
// sealed into the canary artifact.
if (!/^[\x20-\x7e]{12,500}$/.test(reason)) {
throw new Error('gate override reason must be 12 to 500 printable characters on one line')
}
return { reason, confirmation }
}
export function validateSameCapWave(input) {
if (!['verify', 'canary-apply', 'batch-apply', 'rollback'].includes(input.mode)) {
throw new Error('same-cap wave mode is invalid')
}
const selected = cells(input.cellIds)
const targetDigest = digest(input.targetDigest, 'target digest')
const rollbackDigest = digest(input.rollbackDigest, 'rollback digest')
if (targetDigest === rollbackDigest) throw new Error('target and rollback digests must differ')
if (input.mode === 'canary-apply' && selected.length !== 1) {
throw new Error('canary mode requires exactly one cell')
}
if (input.mode === 'batch-apply' && (selected.length < 2 || selected.length > 4)) {
throw new Error('batch mode requires two to four cells')
}
// Later waves expect the selector to advance by exactly 2 per predecessor,
// which a resumed rollback cell (isolate skipped, +1) violates.
if (input.mode === 'rollback' && selected.length !== 1) {
throw new Error('rollback mode requires exactly one cell')
}
const mutation = input.mode !== 'verify'
const expectedConfirmation = input.mode === 'rollback'
? `ROLL_BACK_RELAY_SAME_CAP ${rollbackDigest} ${selected.join(',')}`
: `ROLL_RELAY_SAME_CAP ${targetDigest} ${selected.join(',')}`
if (mutation && input.confirmation !== expectedConfirmation) {
throw new Error('same-cap confirmation does not match the exact digest and cells')
}
if (!mutation && input.confirmation) throw new Error('verify does not accept confirmation')
const gateOverride = gateOverrideAuthorization(input, targetDigest, mutation)
if (input.mode === 'batch-apply' && !/^[1-9][0-9]*$/.test(input.canaryRunId ?? '')) {
throw new Error('batch mode requires a canary run ID')
}
if (input.mode !== 'batch-apply' && input.canaryRunId) {
throw new Error('only batch mode accepts a canary run ID')
}
return { cells: selected, targetDigest, rollbackDigest, gateOverride }
}
export function canaryAuthority(input) {
const wave = validateSameCapWave({ ...input, mode: 'canary-apply', canaryRunId: '' })
if (!/^[0-9a-f]{40}$/.test(input.commitSha ?? '')) throw new Error('commit SHA is invalid')
if (!/^[1-9][0-9]*$/.test(input.runId ?? '')) throw new Error('run ID is invalid')
const selectorGeneration = Number(input.selectorGeneration)
const rehomeGeneration = Number(input.rehomeGeneration)
if (!Number.isSafeInteger(selectorGeneration) || selectorGeneration < 0) {
throw new Error('selector generation is invalid')
}
if (!Number.isSafeInteger(rehomeGeneration) || rehomeGeneration < 0) {
throw new Error('rehome generation is invalid')
}
return {
v: 1,
commitSha: input.commitSha,
runId: input.runId,
cellId: wave.cells[0],
targetDigest: wave.targetDigest,
rollbackDigest: wave.rollbackDigest,
selectorGeneration: selectorGeneration + 2,
rehomeGeneration,
// Audit trail, not authority: a batch reusing this canary is authorized by
// its own confirmation, so verification below neither requires nor forbids it.
gateOverride: wave.gateOverride === null ? null : {
...wave.gateOverride,
actor: input.actor ?? ''
}
}
}
export function verifyCanaryAuthority(authority, expected, repositoryRoot) {
const selectorGeneration = Number(expected.selectorGeneration)
if (
authority?.v !== 1 ||
!/^[0-9a-f]{40}$/.test(authority.commitSha ?? '') ||
authority.runId !== expected.runId ||
authority.targetDigest !== expected.targetDigest ||
authority.rollbackDigest !== expected.rollbackDigest ||
!Number.isSafeInteger(authority.selectorGeneration) ||
authority.selectorGeneration < 0 ||
!Number.isSafeInteger(selectorGeneration) ||
selectorGeneration < authority.selectorGeneration ||
authority.rehomeGeneration !== Number(expected.rehomeGeneration) ||
!SAME_CAP_CELLS.includes(authority.cellId)
) throw new Error('canary authority does not match this batch')
// Each cell checks exact live selector state; later batches may reuse this control epoch's canary.
requireSameEvidenceCode({
sealedSha: authority.commitSha,
currentSha: expected.commitSha,
label: 'relay same-cap canary authority',
repositoryRoot
})
return authority
}
function values(argv) {
const result = {}
for (let index = 0; index < argv.length; index += 2) {
if (!argv[index]?.startsWith('--') || argv[index + 1] === undefined) {
throw new Error('invalid arguments')
}
result[argv[index].slice(2)] = argv[index + 1]
}
return result
}
export function main(argv = process.argv.slice(2)) {
const command = argv.shift()
const input = values(argv)
if (command === 'validate') {
const wave = validateSameCapWave({
mode: input.mode,
cellIds: input['cell-ids'],
targetDigest: input['target-digest'],
rollbackDigest: input['rollback-digest'],
confirmation: input.confirmation,
canaryRunId: input['canary-run-id'],
gateOverrideReason: input['gate-override-reason'],
gateOverrideConfirmation: input['gate-override-confirmation']
})
process.stdout.write(`${JSON.stringify(wave.cells)}\n`)
return
}
if (command === 'create-canary') {
process.stdout.write(`${JSON.stringify(canaryAuthority({
mode: 'canary-apply',
cellIds: input['cell-id'],
targetDigest: input['target-digest'],
rollbackDigest: input['rollback-digest'],
confirmation: input.confirmation,
commitSha: input['commit-sha'],
runId: input['run-id'],
selectorGeneration: input['selector-generation'],
rehomeGeneration: input['rehome-generation'],
gateOverrideReason: input['gate-override-reason'],
gateOverrideConfirmation: input['gate-override-confirmation'],
actor: input.actor
}))}\n`)
return
}
if (command === 'verify-canary') {
verifyCanaryAuthority(JSON.parse(readFileSync(input.file, 'utf8')), {
commitSha: input['commit-sha'],
runId: input['run-id'],
targetDigest: input['target-digest'],
rollbackDigest: input['rollback-digest'],
selectorGeneration: input['selector-generation'],
rehomeGeneration: input['rehome-generation']
})
return
}
throw new Error('unknown same-cap wave command')
}
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
try { main() } catch (error) {
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
process.exitCode = 1
}
}