chore(cloud): add the relay fence broker, ops console, Terraform root, scripts, and 24 cloud-* workflows (#18413)

Phase 6 of the relay split: the relay's deploy/operate surface moves under cloud/ with 24 cloud-* workflows gated on ORCA_CLOUD_OPERATIONS_ENABLED, the Cloud SQL rollout lease action, the relay Terraform root (dual-accept identities for both repositories), scripts, docs, CODEOWNERS, and a terraform validate job in Cloud Verify.
This commit is contained in:
Jinwoo Hong
2026-09-03 06:55:14 -04:00
committed by GitHub
parent 4cc0b8de61
commit 3eec77c11a
379 changed files with 102741 additions and 8 deletions
@@ -0,0 +1,83 @@
#!/usr/bin/env node
import { execFileSync } from 'node:child_process'
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'
import { join } from 'node:path'
// Why: the relay root can never plan zero-diff (cell templates roll one at a time by design),
// so the split is gated on plan EQUIVALENCE: the normalized change set for a root's addresses
// must be identical before and after a state move. This captures that set deterministically.
// Read-only: -lock=false, -refresh=false, no apply. Forgets from `removed` blocks are excluded
// because the baseline has none.
const usage =
'usage: capture-terraform-plan-baseline.mjs --root <dir> --env <staging|production> --out <dir> [--tag <name>]'
export function normalizePlan(planJson) {
const changes = (planJson.resource_changes ?? [])
.filter((entry) => !entry.change.actions.includes('forget'))
.map((entry) => ({
address: entry.address,
actions: entry.change.actions,
before: entry.change.before ?? null,
after: entry.change.after ?? null,
after_unknown: entry.change.after_unknown ?? null
}))
.sort((left, right) => (left.address < right.address ? -1 : left.address > right.address ? 1 : 0))
return changes
}
export function summarize(changes) {
const counts = { create: 0, update: 0, delete: 0, replace: 0, 'no-op': 0, read: 0 }
for (const change of changes) {
const key = change.actions.join('-')
if (key === 'create') counts.create += 1
else if (key === 'update') counts.update += 1
else if (key === 'delete') counts.delete += 1
else if (key === 'delete-create' || key === 'create-delete') counts.replace += 1
else if (key === 'read') counts.read += 1
else counts['no-op'] += 1
}
return counts
}
function argument(flag) {
const index = process.argv.indexOf(flag)
return index >= 0 ? process.argv[index + 1] : undefined
}
if (process.argv[1] && import.meta.url.endsWith(process.argv[1].split('/').pop())) {
const root = argument('--root')
const environment = argument('--env')
const out = argument('--out')
const tag = argument('--tag') ?? `${environment}-${root.replaceAll('/', '_')}`
if (!root || !['staging', 'production'].includes(environment) || !out) {
process.stderr.write(`${usage}\n`)
process.exit(2)
}
// The Cloudflare override only applies to the root that still declares the records; the relay
// root dropped them in the carve and errors on a -var for an undeclared variable.
const declaresArtifactDns = readFileSync(join(root, 'variables.tf'), 'utf8').includes(
'variable "manage_artifact_dns"'
)
mkdirSync(out, { recursive: true })
const planFile = join(out, `${tag}.tfplan`)
execFileSync(
'terraform',
[
`-chdir=${root}`, 'plan', '-input=false', '-lock=false', '-refresh=false', '-no-color',
`-var-file=environments/${environment}.tfvars`,
...(declaresArtifactDns ? ['-var', 'manage_artifact_dns=false'] : []),
`-out=${planFile}`
],
{ stdio: ['ignore', 'inherit', 'inherit'] }
)
const json = JSON.parse(
execFileSync('terraform', [`-chdir=${root}`, 'show', '-json', planFile], {
encoding: 'utf8',
maxBuffer: 256 * 1024 * 1024
})
)
const normalized = normalizePlan(json)
writeFileSync(join(out, `${tag}.norm.json`), `${JSON.stringify(normalized, null, 1)}\n`)
process.stdout.write(`${tag}: ${JSON.stringify(summarize(normalized))}\n`)
}
@@ -0,0 +1,15 @@
import assert from 'node:assert/strict'
import { test } from 'node:test'
import { normalizePlan, summarize } from './capture-terraform-plan-baseline.mjs'
test('normalizes, sorts, and drops forgets so a removed block cannot skew equivalence', () => {
const changes = normalizePlan({
resource_changes: [
{ address: 'b.two', change: { actions: ['update'], before: { x: 1 }, after: { x: 2 } } },
{ address: 'a.one', change: { actions: ['forget'], before: {}, after: null } },
{ address: 'c.three', change: { actions: ['delete', 'create'], before: {}, after: {}, after_unknown: { id: true } } }
]
})
assert.deepEqual(changes.map((change) => change.address), ['b.two', 'c.three'])
assert.deepEqual(summarize(changes), { create: 0, update: 1, delete: 0, replace: 1, 'no-op': 0, read: 0 })
})
@@ -0,0 +1,109 @@
import { readFileSync } from 'node:fs'
import { pathToFileURL } from 'node:url'
import { isDeepStrictEqual } from 'node:util'
function parseArguments(argv) {
if (argv.length !== 2 || argv[0] !== '--capacity-service-account' || !argv[1]) {
throw new Error('missing --capacity-service-account')
}
return argv[1]
}
export function classifyProductionCapacityDirector(state, capacityServiceAccount) {
const currentIdentity = state.currentCapacityServiceAccount
if (currentIdentity !== null && currentIdentity !== capacityServiceAccount) {
throw new Error('director has an unexpected capacity identity')
}
const {
baseCells,
currentCells,
capacityCellIds,
targetCellId,
targetHardCap
} = state
if (
!Array.isArray(baseCells) ||
!Array.isArray(currentCells) ||
!Array.isArray(capacityCellIds) ||
![600, 1000].includes(targetHardCap) ||
new Set(capacityCellIds).size !== capacityCellIds.length ||
!capacityCellIds.includes(targetCellId) ||
baseCells.length !== currentCells.length ||
new Set(baseCells.map((cell) => cell?.id)).size !== baseCells.length ||
new Set(currentCells.map((cell) => cell?.id)).size !== currentCells.length
) {
throw new Error('director topology transition input is invalid')
}
const capacityCells = new Set(capacityCellIds)
const normalizedCurrent = currentCells.map((current, index) => {
const base = baseCells[index]
if (
typeof current?.id !== 'string' ||
current.id !== base?.id
) {
throw new Error('director topology cell identity is invalid')
}
if (!capacityCells.has(current.id)) {
if (!isDeepStrictEqual(current, base)) {
throw new Error('director topology changed outside the capacity rollout')
}
return current
}
if (
base.connectionHardCap !== 1000 ||
base.connectionUnobservedBound !== 60 ||
![600, 1000].includes(current.connectionHardCap) ||
current.connectionUnobservedBound !== 60
) {
throw new Error('director capacity rollout state is invalid')
}
return {
...current,
connectionHardCap: base.connectionHardCap,
connectionUnobservedBound: base.connectionUnobservedBound
}
})
if (
!isDeepStrictEqual(normalizedCurrent, baseCells) ||
capacityCellIds.some((cellId) => !baseCells.some((cell) => cell.id === cellId))
) {
throw new Error('director topology is outside the reviewed capacity envelope')
}
const withTargetCap = (hardCap) => currentCells.map((cell) =>
cell.id === targetCellId
? { ...cell, connectionHardCap: hardCap, connectionUnobservedBound: 60 }
: cell
)
const desiredCells = withTargetCap(targetHardCap)
const predecessorCells = withTargetCap(targetHardCap === 600 ? 1000 : 600)
const topologyPhase = isDeepStrictEqual(currentCells, desiredCells)
? 'desired'
: isDeepStrictEqual(currentCells, predecessorCells)
? 'predecessor'
: null
if (!topologyPhase) throw new Error('director topology is not a reviewed transition state')
return {
topologyPhase,
directorReady:
topologyPhase === 'desired' && currentIdentity === capacityServiceAccount,
desiredCells
}
}
export function main(argv = process.argv.slice(2)) {
const capacityServiceAccount = parseArguments(argv)
const state = JSON.parse(readFileSync(0, 'utf8'))
process.stdout.write(`${JSON.stringify({
event: 'relay_production_capacity_director_classified',
...classifyProductionCapacityDirector(state, capacityServiceAccount)
})}\n`)
}
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
try {
main()
} catch (error) {
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
process.exitCode = 1
}
}
@@ -0,0 +1,93 @@
import assert from 'node:assert/strict'
import test from 'node:test'
import {
classifyProductionCapacityDirector
} from './classify-relay-production-capacity-director.mjs'
const capacityServiceAccount =
'orca-cloud-gha-relay-cap@onorca-cloud.iam.gserviceaccount.com'
const capacityCellIds = ['production-gce-c25', 'production-gce-c26']
const baseCells = [
{ id: 'production-gce-c17', connectionHardCap: 600, connectionUnobservedBound: 60 },
{ id: 'production-gce-c25', connectionHardCap: 1000, connectionUnobservedBound: 60 },
{ id: 'production-gce-c26', connectionHardCap: 1000, connectionUnobservedBound: 60 }
]
const mixedCells = [
baseCells[0],
{ ...baseCells[1], connectionHardCap: 600 },
baseCells[2]
]
function state(overrides = {}) {
return {
baseCells,
currentCells: mixedCells,
capacityCellIds,
targetCellId: 'production-gce-c25',
targetHardCap: 1000,
currentCapacityServiceAccount: capacityServiceAccount,
...overrides
}
}
test('classifies one target while preserving completed rollout cells', () => {
assert.deepEqual(
classifyProductionCapacityDirector(state(), capacityServiceAccount),
{
topologyPhase: 'predecessor',
directorReady: false,
desiredCells: baseCells
}
)
})
test('skips deployment only for exact topology and identity', () => {
assert.deepEqual(
classifyProductionCapacityDirector(state({ currentCells: baseCells }), capacityServiceAccount),
{ topologyPhase: 'desired', directorReady: true, desiredCells: baseCells }
)
assert.deepEqual(
classifyProductionCapacityDirector(state({
currentCells: baseCells,
targetHardCap: 600
}), capacityServiceAccount),
{
topologyPhase: 'predecessor',
directorReady: false,
desiredCells: mixedCells
}
)
})
test('rejects an unknown topology or capacity identity', () => {
assert.throws(
() => classifyProductionCapacityDirector(state({
currentCells: [baseCells[0], { ...baseCells[1], connectionHardCap: 700 }, baseCells[2]]
}), capacityServiceAccount),
/rollout state is invalid/
)
assert.throws(
() => classifyProductionCapacityDirector(state({
currentCapacityServiceAccount: 'unexpected@onorca-cloud.iam.gserviceaccount.com'
}), capacityServiceAccount),
/unexpected capacity identity/
)
assert.throws(
() => classifyProductionCapacityDirector(state({
currentCells: [{ ...baseCells[0], connectionHardCap: 1000 }, mixedCells[1], mixedCells[2]]
}), capacityServiceAccount),
/outside the capacity rollout/
)
assert.throws(
() => classifyProductionCapacityDirector(state({
currentCells: [baseCells[0], { ...mixedCells[1], url: 'https://wrong.invalid' }, baseCells[2]]
}), capacityServiceAccount),
/outside the reviewed capacity envelope/
)
assert.throws(
() => classifyProductionCapacityDirector(state({
targetCellId: 'production-gce-c17'
}), capacityServiceAccount),
/transition input is invalid/
)
})
@@ -0,0 +1,47 @@
import { pathToFileURL } from 'node:url'
export function classifyStagingBootstrap({ c2Kind, c2Admission, c3Kind, c3Admission }) {
for (const kind of [c2Kind, c3Kind]) {
if (!['legacy', 'modern'].includes(kind)) throw new Error('bootstrap runtime kind is invalid')
}
for (const admission of [c2Admission, c3Admission]) {
if (!['general', 'migration-only'].includes(admission)) {
throw new Error('bootstrap admission is not recoverable')
}
}
if (c2Kind === 'modern' && c3Kind === 'modern') return 'complete'
if (c2Kind === 'modern') return 'roll-c3'
if (c3Kind === 'modern') return 'roll-c2'
if (c2Admission === 'general') return 'normalize-and-roll-both'
if (c3Admission === 'general') return 'resume-c2-then-c3'
throw new Error('bootstrap has no general fallback')
}
function parseArguments(argv) {
const values = {}
for (let index = 0; index < argv.length; index += 2) {
const key = argv[index]
const value = argv[index + 1]
if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments')
values[key.slice(2)] = value
}
return {
c2Kind: values['c2-kind'],
c2Admission: values['c2-admission'],
c3Kind: values['c3-kind'],
c3Admission: values['c3-admission']
}
}
export function main(argv = process.argv.slice(2)) {
process.stdout.write(`${classifyStagingBootstrap(parseArguments(argv))}\n`)
}
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
try {
main()
} catch (error) {
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
process.exitCode = 1
}
}
@@ -0,0 +1,60 @@
import assert from 'node:assert/strict'
import { test } from 'node:test'
import { classifyStagingBootstrap } from './classify-relay-staging-bootstrap.mjs'
const state = (c2Kind, c2Admission, c3Kind, c3Admission) => ({
c2Kind,
c2Admission,
c3Kind,
c3Admission
})
test('classifies the fresh legacy bootstrap', () => {
assert.equal(
classifyStagingBootstrap(state('legacy', 'general', 'legacy', 'general')),
'normalize-and-roll-both'
)
})
test('retries normalization after a failed C3 restart', () => {
assert.equal(
classifyStagingBootstrap(state('legacy', 'general', 'legacy', 'migration-only')),
'normalize-and-roll-both'
)
})
test('resumes after C2 isolation', () => {
assert.equal(
classifyStagingBootstrap(state('legacy', 'migration-only', 'legacy', 'general')),
'resume-c2-then-c3'
)
})
test('resumes after C2 apply or a partial C3 transition', () => {
for (const c2Admission of ['general', 'migration-only']) {
for (const c3Admission of ['general', 'migration-only']) {
assert.equal(
classifyStagingBootstrap(state('modern', c2Admission, 'legacy', c3Admission)),
'roll-c3'
)
}
}
})
test('repairs an unexpected modern C3 before rolling legacy C2', () => {
assert.equal(
classifyStagingBootstrap(state('legacy', 'migration-only', 'modern', 'general')),
'roll-c2'
)
})
test('accepts already complete modern cells and rejects no-fallback legacy state', () => {
assert.equal(
classifyStagingBootstrap(state('modern', 'migration-only', 'modern', 'general')),
'complete'
)
assert.throws(
() => classifyStagingBootstrap(state('legacy', 'migration-only', 'legacy', 'migration-only')),
/no general fallback/
)
})
@@ -0,0 +1,305 @@
// Derives, from workflow and script content, which workflows roll out against the shared Cloud SQL
// instance. Hand lists go stale silently; everything here is read back off disk.
import { readFileSync, readdirSync } from 'node:fs'
import {
RELAY_WORKFLOW_DIRECTORY,
RELAY_WORKFLOW_FILE_PREFIX,
relayWorkflowFile
} from './relay-repository.mjs'
export const WORKFLOW_ROOT = RELAY_WORKFLOW_DIRECTORY
export const SCRIPT_ROOT = new URL('./', import.meta.url)
export const LEASE_ACTION = './.github/actions/cloud-sql-rollout-lease'
export const PRODUCTION_LEASE = {
bucket: 'onorca-cloud-terraform-state',
object: 'terraform/state/cloud-sql-rollout/production.lock'
}
export const STAGING_LEASE = {
bucket: 'onorca-cloud-staging-terraform-state',
object: 'terraform/state/cloud-sql-rollout/staging.lock'
}
export const PRODUCTION_GROUP = 'production-cloud-sql-rollout'
export const STAGING_GROUP = 'relay-staging-mutation'
export const SELECTABLE_GROUP =
"${{ inputs.environment == 'production' && 'production-cloud-sql-rollout' || 'relay-staging-mutation' }}"
const selectable = (production, staging) =>
`\${{ inputs.environment == 'production' && '${production}' || '${staging}' }}`
export const SELECTABLE_LEASE = {
bucket: selectable(PRODUCTION_LEASE.bucket, STAGING_LEASE.bucket),
object: selectable(PRODUCTION_LEASE.object, STAGING_LEASE.object)
}
export const LOCK_GROUPS = new Set([PRODUCTION_GROUP, STAGING_GROUP, SELECTABLE_GROUP])
export function readWorkflow(file) {
return readFileSync(new URL(file, WORKFLOW_ROOT), 'utf8')
}
export function workflowFiles() {
return readdirSync(WORKFLOW_ROOT)
.filter((name) => name.endsWith('.yml') && name.startsWith(RELAY_WORKFLOW_FILE_PREFIX))
.sort()
}
// --- YAML-shaped readers (line based; the workflows are hand-written and uniformly indented) ---
function indentOf(line) {
return line.length - line.trimStart().length
}
function blockAfter(lines, index) {
const base = indentOf(lines[index])
const body = []
for (let i = index + 1; i < lines.length; i += 1) {
if (lines[i].trim() === '') {
body.push(lines[i])
continue
}
if (indentOf(lines[i]) <= base) break
body.push(lines[i])
}
return body
}
export function concurrencyBlocks(text) {
const lines = text.split('\n')
const blocks = []
lines.forEach((line, index) => {
if (line.trim() !== 'concurrency:') return
const body = blockAfter(lines, index)
blocks.push({
group: body.find((l) => l.trim().startsWith('group:'))?.trim().slice('group:'.length).trim(),
cancelInProgress: body
.find((l) => l.trim().startsWith('cancel-in-progress:'))
?.trim()
.slice('cancel-in-progress:'.length)
.trim()
})
})
return blocks
}
export function jobs(text) {
const lines = text.split('\n')
const start = lines.findIndex((line) => line === 'jobs:')
if (start === -1) return []
const found = []
for (let i = start + 1; i < lines.length; i += 1) {
const match = /^ {2}([A-Za-z0-9_-]+):\s*$/.exec(lines[i])
if (!match) continue
found.push({ id: match[1], start: i, body: blockAfter(lines, i) })
}
return found.map((job) => ({ ...job, text: job.body.join('\n') }))
}
function scalarField(jobText, key) {
const lines = jobText.split('\n')
const index = lines.findIndex((line) => /^ {4}[A-Za-z-]+:/.test(line) && line.trim().startsWith(`${key}:`))
if (index === -1) return undefined
const inline = lines[index].trim().slice(`${key}:`.length).trim()
if (inline !== '' && inline !== '>-' && inline !== '|') return inline
return blockAfter(lines, index).join(' ').replace(/\s+/g, ' ').trim()
}
export function jobNeeds(jobText) {
const raw = scalarField(jobText, 'needs')
if (!raw) return []
return raw
.replace(/^\[|\]$/g, '')
.split(/[,\n]|\s+-\s+/)
.map((entry) => entry.replace(/^-/, '').trim())
.filter(Boolean)
}
export function jobIf(jobText) {
return scalarField(jobText, 'if') ?? ''
}
export function leaseSteps(text) {
const lines = text.split('\n')
const steps = []
lines.forEach((line, index) => {
if (line.trim() !== `- uses: ${LEASE_ACTION}`) return
const body = blockAfter(lines, index)
const read = (key) =>
body.find((l) => l.trim().startsWith(`${key}:`))?.trim().slice(`${key}:`.length).trim()
steps.push({
line: index + 1,
bucket: read('bucket'),
object: read('object'),
release: read('release')
})
})
return steps
}
export function leaseStepsByJob(file) {
const text = readWorkflow(file)
const steps = leaseSteps(text)
return jobs(text).map((job) => ({
id: job.id,
steps: steps.filter((step) => step.line > job.start + 1 && step.line <= job.start + 1 + job.body.length)
}))
}
// --- trigger and reusable-call graph ---
export function triggers(text) {
const lines = text.split('\n')
const index = lines.findIndex((line) => line === 'on:')
if (index === -1) return []
return blockAfter(lines, index)
.map((line) => /^ {2}([a-z_]+):/.exec(line)?.[1])
.filter(Boolean)
}
export function isEntrypoint(text) {
return triggers(text).some((trigger) => trigger !== 'workflow_call')
}
export function reusableCalls(text) {
const counts = new Map()
for (const match of text.matchAll(/uses: \.\/\.github\/workflows\/([A-Za-z0-9._-]+\.yml)/g)) {
counts.set(match[1], (counts.get(match[1]) ?? 0) + 1)
}
return counts
}
export function entrypointsFor(file, seen = new Set()) {
if (seen.has(file)) return new Set()
seen.add(file)
if (isEntrypoint(readWorkflow(file))) return new Set([file])
const reached = new Set()
for (const candidate of workflowFiles()) {
if (candidate === file) continue
if (!reusableCalls(readWorkflow(candidate)).has(file)) continue
for (const entry of entrypointsFor(candidate, seen)) reached.add(entry)
}
return reached
}
// --- what counts as a Cloud SQL connection-budget rollout ---
const COMMAND_PREFIX = /^(?:-\s+)?(?:run:\s*)?(?:[a-z_]+\s*=\s*"?\$\(\s*)?(?:if\s+|then\s+|else\s+|&&\s+|\|\|\s+|!\s+)*/
function commandLines(text) {
return text.split('\n').map((line) => line.trim().replace(COMMAND_PREFIX, ''))
}
export function appliesTerraform(text) {
return commandLines(text).some((line) => /^terraform\b.*\bapply\b/.test(line))
}
export function runsCloudRunMutation(text) {
return commandLines(text).some((line) =>
/^gcloud run (?:deploy\b|services (?:update|replace)\b|jobs (?:update|deploy)\b)/.test(line)
)
}
// Scripts that mint a Cloud Run revision, plus every script that re-exports one of them.
export function revisionMintingScripts() {
const self = new URL(import.meta.url).pathname.split('/').pop()
const names = readdirSync(SCRIPT_ROOT).filter(
(name) => name.endsWith('.mjs') && !name.endsWith('.test.mjs') && name !== self
)
const source = new Map(
names.map((name) => [name, readFileSync(new URL(name, SCRIPT_ROOT), 'utf8')])
)
const minting = new Set(
names.filter((name) => {
const text = source.get(name)
return (
text.includes("'--no-traffic'") ||
/'run',\s*'services',\s*'update'/.test(text) ||
/'run',\s*'deploy'/.test(text)
)
})
)
for (let changed = true; changed; ) {
changed = false
for (const name of names) {
if (minting.has(name)) continue
const imports = [...source.get(name).matchAll(/from '\.\/([A-Za-z0-9._-]+\.mjs)'/g)].map(
(match) => match[1]
)
if (!imports.some((imported) => minting.has(imported))) continue
minting.add(name)
changed = true
}
}
return minting
}
export function mutatesSharedInstance(text, minters = revisionMintingScripts()) {
if (appliesTerraform(text)) return 'terraform apply against the reviewed relay cell templates'
if (runsCloudRunMutation(text)) return 'gcloud mints or replaces a Cloud Run revision'
for (const script of minters) {
if (text.includes(`dev/scripts/${script}`)) return `runs ${script}, which mints a Cloud Run revision`
}
return undefined
}
// --- the declared contract ---
const production = (extra = {}) => ({ env: 'production', group: PRODUCTION_GROUP, ...extra })
const staging = (extra = {}) => ({ env: 'staging', group: STAGING_GROUP, ...extra })
const eitherEnvironment = () => ({ env: 'selectable', group: SELECTABLE_GROUP })
// Keys are workflow filenames, which the public copy prefixes; the prefix lives in one place.
const named = (entries) =>
Object.fromEntries(
entries.map(([file, entry]) => [
relayWorkflowFile(file),
entry.leaseFiles
? { ...entry, leaseFiles: entry.leaseFiles.map((member) => relayWorkflowFile(member)) }
: entry
])
)
export const LEASED_WORKFLOWS = named([
['deploy-relay-fence-broker.yml', production()],
['deploy-relay-production.yml', production()],
['deploy-relay-production-director.yml', production()],
['deploy-relay-production-multi-target.yml', production()],
[
'deploy-relay-production-capacity.yml',
production({
leaseFiles: ['deploy-relay-production-capacity-job.yml'],
reentrant: true
})
],
[
'deploy-relay-production-same-cap.yml',
production({
leaseFiles: ['deploy-relay-production-same-cap-job.yml'],
reentrant: true
})
],
[
'operate-relay-production-rehome.yml',
production({ leaseFiles: ['operate-relay-production-rehome-job.yml'] })
],
['deploy-relay-asia-topology.yml', eitherEnvironment()],
['operate-relay-asia-admission.yml', eitherEnvironment()],
['deploy-relay-staging.yml', staging()],
['deploy-relay-staging-gce-candidate.yml', staging()],
['bootstrap-relay-staging-capacity.yml', staging()],
['power-relay-staging.yml', staging()],
['prove-relay-asia-staging.yml', staging()],
[
'prove-relay-staging-capacity.yml',
staging({ exclusiveBy: "inputs.mode == 'refresh-asia-c4-image'" })
],
['recover-relay-staging-c4-image.yml', staging()]
])
export const NOT_A_CLOUD_SQL_CANDIDATE = named([
[
'monitor-relay-production.yml',
'Read-only. Its identity holds monitoring, logging, Cloud SQL and compute viewer roles only, and it runs `gcloud sql instances describe`, never a mutation. It consumes no connection budget, so the durable lease would only let monitoring block a rollout and a rollout block monitoring.'
]
])
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,814 @@
import assert from 'node:assert/strict'
import { readFileSync } from 'node:fs'
import { test } from 'node:test'
import { fileURLToPath } from 'node:url'
import {
activeRevision,
cloudRunTrafficTag,
DIRECTOR_ADMISSION_ENVIRONMENT,
DIRECTOR_REGIONAL_PLACEMENT_ENV,
DIRECTOR_REGIONAL_PLACEMENT_SECRET,
DIRECTOR_REHOME_AUDIENCE_ENV,
DIRECTOR_REHOME_IDENTITY_ENV,
assertRegionalRehomeDisabled,
deployDirector,
directorDeploymentEnvironment,
directorCellSetAddition,
directorStartupProbeArguments,
directorTopologyChange,
environmentUpdateValue,
parseArguments,
revisionEnvironment,
revisionSecretEnvironment,
suppliedAdminIdentityToken,
taggedRevisionOrigin,
taggedTraffic,
trafficTags,
waitForEvacuationCapacity
} from './deploy-relay-blue-green.mjs'
// Terraform declares these values; audited blue/green stamps the same values without targeting
// the drifted service, so this contract must fail before either side can silently diverge.
function terraformDirectorEnvironment(names) {
const read = (name) =>
readFileSync(fileURLToPath(new URL(`../../infra/terraform/${name}`, import.meta.url)), 'utf8')
const relay = read('relay.tf')
const variables = read('variables.tf')
const production = read('environments/production.tfvars')
return Object.fromEntries(
names.map((name) => {
const block = new RegExp(`name\\s*=\\s*"${name}"\\s*\\n\\s*value\\s*=\\s*([^\\n]+)`).exec(relay)
assert.ok(block, `${name} is not set by relay.tf`)
const variable = /var\.([a-z_]+)/.exec(block[1])
assert.ok(variable, `${name} is not sourced from a Terraform variable`)
// An environment override wins over the variable default, as Terraform resolves it.
const override = new RegExp(`^${variable[1]}\\s*=\\s*(\\S+)`, 'm').exec(production)
const fallback = new RegExp(
`variable\\s+"${variable[1]}"[\\s\\S]*?default\\s*=\\s*(\\S+)`
).exec(variables)
assert.ok(override || fallback, `${variable[1]} has neither an override nor a default`)
return [name, String((override ?? fallback)[1]).replace(/"/g, '')]
})
)
}
test('director admission environment matches what Terraform deploys', () => {
const names = Object.keys(DIRECTOR_ADMISSION_ENVIRONMENT)
assert.deepEqual(terraformDirectorEnvironment(names), { ...DIRECTOR_ADMISSION_ENVIRONMENT })
const relay = readFileSync(
fileURLToPath(new URL('../../infra/terraform/relay.tf', import.meta.url)),
'utf8'
)
assert.match(
relay,
/name = "ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED"[\s\S]*?secret\s+= google_secret_manager_secret\.relay_regional_placement_enabled\.secret_id[\s\S]*?version = data\.external\.relay_serving_regional_placement_version\.result\.version/
)
assert.match(relay, /data "external" "relay_serving_regional_placement_version"/)
assert.match(relay, /read-relay-serving-regional-placement-version\.mjs/)
assert.doesNotMatch(relay, /template\[0\]\.containers\[0\]\.env/)
})
test('validates the final stamped-cell admission state', () => {
const required = [
'--project',
'project',
'--region',
'region',
'--service',
'service',
'--image',
'image',
'--role',
'cell',
'--release-id',
'release',
'--director-origin',
'https://relay.example.com',
'--admin-audience',
'https://relay.example.com/v1/admin/drain'
]
assert.equal(parseArguments(required)['final-admission'], undefined)
assert.equal(
parseArguments([...required, '--final-admission', 'disabled'])['final-admission'],
'disabled'
)
assert.throws(() => parseArguments([...required, '--final-admission', 'sometimes']))
assert.equal(parseArguments([...required, '--min-instances', '0'])['min-instances'], '0')
assert.throws(() => parseArguments([...required, '--min-instances', '-1']))
assert.throws(() =>
parseArguments([...required, '--capacity-service-account', 'relay@example.com'])
)
})
test('validates optional director capacity configuration', () => {
const cells = [
{
id: 'staging-gce-c3',
url: 'https://c3.relay-staging.onorca.dev',
capacityRequests: 4_000,
initiallyEnabled: false,
region: 'us-central1',
connectionHardCap: 600,
connectionUnobservedBound: 60
}
]
const config = {
project: 'onorca-cloud-staging',
'capacity-service-account':
'orca-cloud-staging-gha-cap@onorca-cloud-staging.iam.gserviceaccount.com',
'director-cells-json': JSON.stringify(cells)
}
assert.deepEqual(directorDeploymentEnvironment(config), {
...DIRECTOR_ADMISSION_ENVIRONMENT,
ORCA_RELAY_ADMISSION_SELECTOR_VERSION: '3',
ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT:
'orca-cloud-staging-gha-cap@onorca-cloud-staging.iam.gserviceaccount.com',
ORCA_RELAY_CELLS_JSON: JSON.stringify([
{
id: cells[0].id,
url: cells[0].url,
capacityRequests: cells[0].capacityRequests,
region: cells[0].region,
initiallyEnabled: cells[0].initiallyEnabled,
connectionHardCap: cells[0].connectionHardCap,
connectionUnobservedBound: cells[0].connectionUnobservedBound
}
])
})
assert.throws(
() =>
directorDeploymentEnvironment({
...config,
'capacity-service-account': 'foreign@other-project.iam.gserviceaccount.com'
}),
/selected project/
)
assert.throws(
() =>
directorDeploymentEnvironment({
...config,
'director-cells-json': JSON.stringify([{ ...cells[0], unexpected: true }])
}),
/invalid cell/
)
assert.match(
environmentUpdateValue(directorDeploymentEnvironment(config)),
/^\^~\^ORCA_RELAY_DATABASE_POOL_MAX=/
)
assert.equal(environmentUpdateValue({ FIRST: 'one', SECOND: 'two' }), 'FIRST=one,SECOND=two')
assert.deepEqual(
directorTopologyChange(
JSON.stringify([
{
capacityRequests: 4_000,
connectionHardCap: 600,
connectionUnobservedBound: 60,
id: 'staging-gce-c3',
initiallyEnabled: false,
region: 'us-central1',
url: 'https://c3.relay-staging.onorca.dev'
}
]),
JSON.stringify([{ ...cells[0], connectionHardCap: 1_000 }]),
'staging-gce-c3'
),
{
changed: true,
value: directorDeploymentEnvironment({
'director-cells-json': JSON.stringify([{ ...cells[0], connectionHardCap: 1_000 }])
}).ORCA_RELAY_CELLS_JSON
}
)
assert.throws(
() =>
directorTopologyChange(
JSON.stringify(cells),
JSON.stringify([{ ...cells[0], url: 'https://wrong.relay-staging.onorca.dev' }]),
'staging-gce-c3'
),
/outside the reviewed capacity pair/
)
})
test('validates exact director runtime and regional rehome identities', () => {
const base = [
'--project',
'onorca-cloud',
'--region',
'us-central1',
'--service',
'orca-cloud-relay',
'--image',
`relay@sha256:${'a'.repeat(64)}`,
'--role',
'director',
'--release-id',
'rehome',
'--runtime-service-account',
'relay-director@onorca-cloud.iam.gserviceaccount.com',
'--rehome-director-service-account',
'relay-director@onorca-cloud.iam.gserviceaccount.com',
'--rehome-audience',
'https://relay.onorca.dev/v1/admin/host-drain',
'--expected-rehome-generation',
'7',
'--rehome-control-origin',
'https://relay.onorca.dev',
'--admin-audience',
'https://relay.onorca.dev/v1/admin/drain'
]
const config = parseArguments(base)
assert.deepEqual(directorDeploymentEnvironment(config), {
...DIRECTOR_ADMISSION_ENVIRONMENT,
ORCA_RELAY_ADMISSION_SELECTOR_VERSION: '3',
ORCA_RELAY_IMAGE_DIGEST: `sha256:${'a'.repeat(64)}`,
[DIRECTOR_REHOME_IDENTITY_ENV]:
'relay-director@onorca-cloud.iam.gserviceaccount.com',
[DIRECTOR_REHOME_AUDIENCE_ENV]:
'https://relay.onorca.dev/v1/admin/host-drain'
})
const missingAudience = [...base]
missingAudience.splice(missingAudience.indexOf('--rehome-audience'), 2)
assert.throws(() => parseArguments(missingAudience), /configured together/)
const invalidOrigin = [...base]
invalidOrigin[invalidOrigin.indexOf('--rehome-control-origin') + 1] =
'http://relay.onorca.dev'
assert.throws(
() => parseArguments(invalidOrigin),
/HTTPS origin/
)
const mutableImage = [...base]
mutableImage[mutableImage.indexOf('--image') + 1] = 'relay:latest'
assert.throws(() => parseArguments(mutableImage), /immutable digest/)
})
test('requires durable regional rehome control to be disabled at the exact generation', async () => {
const config = {
'admin-audience': 'https://relay.onorca.dev/v1/admin/drain',
'expected-rehome-generation': '7'
}
const environment = process.env.ORCA_RELAY_ADMIN_ID_TOKEN
process.env.ORCA_RELAY_ADMIN_ID_TOKEN = 'aaa.bbb.ccc'
try {
const control = await assertRegionalRehomeDisabled(
config,
'https://candidate.example.test',
async (url, init) => {
assert.equal(url, 'https://candidate.example.test/v1/admin/regional-rehome-control')
assert.equal(init.headers.authorization, 'Bearer aaa.bbb.ccc')
return new Response(JSON.stringify({
v: 1,
control: { generation: 7, enabled: false }
}))
}
)
assert.equal(control.generation, 7)
await assert.rejects(
assertRegionalRehomeDisabled(config, 'https://candidate.example.test', async () =>
new Response(JSON.stringify({
v: 1,
control: { generation: 8, enabled: false }
}))
),
/expected generation/
)
await assert.rejects(
assertRegionalRehomeDisabled(config, 'https://candidate.example.test', async () =>
new Response(JSON.stringify({
v: 1,
control: { generation: 7, enabled: true }
}))
),
/durably disabled/
)
} finally {
if (environment === undefined) delete process.env.ORCA_RELAY_ADMIN_ID_TOKEN
else process.env.ORCA_RELAY_ADMIN_ID_TOKEN = environment
}
})
test('rejects literal regional placement changes outside the runtime-setting step', () => {
const base = {
project: 'onorca-cloud',
region: 'us-central1',
service: 'orca-cloud-relay',
image: `us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:${'a'.repeat(64)}`,
role: 'director',
'release-id': 'regional-kill-switch'
}
const args = Object.entries(base).flatMap(([key, value]) => [`--${key}`, value])
assert.doesNotThrow(() => parseArguments(args))
assert.throws(
() => parseArguments([...args, '--regional-placement-enabled', 'false']),
/audited runtime-setting step/
)
})
test('appends disabled Asia cells without changing the existing director topology', () => {
const current = [
{
id: 'production-gce-c26',
url: 'https://c26.relay.onorca.dev',
capacityRequests: 4_000,
initiallyEnabled: true,
connectionHardCap: 1_000,
connectionUnobservedBound: 60
}
]
const asia = {
id: 'production-gce-c27',
url: 'https://c27.relay.onorca.dev',
region: 'asia-east2',
capacityRequests: 6_000,
initiallyEnabled: false,
connectionHardCap: 3_000,
connectionUnobservedBound: 60
}
assert.deepEqual(
directorCellSetAddition(
JSON.stringify(current),
JSON.stringify([asia, { ...current[0], region: 'us-central1' }])
),
{
changed: true,
value: directorDeploymentEnvironment({
'director-cells-json': JSON.stringify([asia, { ...current[0], region: 'us-central1' }])
}).ORCA_RELAY_CELLS_JSON
}
)
const exact = JSON.stringify([{ ...current[0], region: 'us-central1' }, asia])
assert.deepEqual(directorCellSetAddition(exact, exact), {
changed: false,
value: directorDeploymentEnvironment({ 'director-cells-json': exact })
.ORCA_RELAY_CELLS_JSON
})
assert.throws(
() => directorCellSetAddition(JSON.stringify(current), JSON.stringify([{ ...current[0], region: 'us-central1', capacityRequests: 6_000 }, asia])),
/changes an existing cell/
)
assert.throws(
() => directorCellSetAddition(JSON.stringify(current), JSON.stringify([{ ...current[0], region: 'us-central1' }, { ...asia, initiallyEnabled: true }])),
/must start disabled/
)
})
test('pins a director startup probe above the bounded reconciliation window', () => {
assert.deepEqual(directorStartupProbeArguments('director'), [
'--startup-probe',
'tcpSocket.port=8080,timeoutSeconds=120,periodSeconds=120,failureThreshold=1'
])
assert.deepEqual(directorStartupProbeArguments('cell'), [])
})
test('bounds traffic tags by the Cloud Run service-plus-tag contract', () => {
const service = 'orca-cloud-relay-staging-c1'
const candidate = cloudRunTrafficTag(service, 'candidate', '29247170608-1-19cc312a')
assert.match(candidate, /^candidate-[a-f0-9]{9}$/)
assert.equal(service.length + candidate.length, 46)
assert.equal(candidate, cloudRunTrafficTag(service, 'candidate', '29247170608-1-19cc312a'))
assert.notEqual(candidate, cloudRunTrafficTag(service, 'candidate', '29247170608-2-19cc312a'))
assert.throws(() => cloudRunTrafficTag(`${service}-too-long`, 'candidate', 'release'))
})
test('derives and validates a Cloud Run tagged revision origin', () => {
assert.equal(
taggedRevisionOrigin(
'https://orca-cloud-relay-staging-c1-gjzz5mc7ka-uc.a.run.app',
'candidate-123'
),
'https://candidate-123---orca-cloud-relay-staging-c1-gjzz5mc7ka-uc.a.run.app'
)
assert.throws(() => taggedRevisionOrigin('https://relay-staging.onorca.dev', 'candidate-123'))
assert.throws(() =>
taggedRevisionOrigin(
'https://orca-cloud-relay-staging-c1-gjzz5mc7ka-uc.a.run.app',
'123-invalid'
)
)
})
test('requires exactly one active revision and reads queried tag metadata', () => {
const service = {
status: {
traffic: [
{ percent: 100, revisionName: 'relay-00001-old' },
{
percent: 0,
revisionName: 'relay-00002-new',
tag: 'candidate-123',
url: 'https://candidate-123---relay-hash-uc.a.run.app'
}
]
}
}
assert.equal(activeRevision(service), 'relay-00001-old')
assert.deepEqual(taggedTraffic(service, 'candidate-123'), {
origin: 'https://candidate-123---relay-hash-uc.a.run.app',
revision: 'relay-00002-new'
})
assert.throws(() =>
activeRevision({ status: { traffic: [{ percent: 50 }, { percent: 50 }] } })
)
assert.deepEqual(trafficTags(service), ['candidate-123'])
})
function directorHarness({
deployFailure,
deleteFailure,
cleanupReportsFailure = false,
cleanupFailsBeforeRemoval = false,
servingMinimum = 1,
servingMaximum = 5,
// Reproduces gcloud dropping minScale from a newly created revision.
dropRequestedMinimum = false,
servingServiceAccount,
servingImageDigest = `sha256:${'f'.repeat(64)}`
} = {}) {
const state = {
activeRevision: 'relay-00001-old',
tags: new Map([['candidate-old', 'relay-00000-stale']]),
revisions: new Map([
['relay-00000-stale', { env: { ORCA_RELAY_ROLE: 'director' }, minimum: 1, maximum: 5 }],
[
'relay-00001-old',
{
env: {
ORCA_RELAY_ROLE: 'director'
},
secrets: {
[DIRECTOR_REGIONAL_PLACEMENT_ENV]: {
secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET,
version: '1'
}
},
minimum: servingMinimum,
maximum: servingMaximum,
serviceAccount: servingServiceAccount,
image: `relay@${servingImageDigest}`
}
]
]),
nextRevision: 2
}
const removed = []
const healthProtocols = []
let pendingCleanupFailure = cleanupFailsBeforeRemoval
const operations = {
describeService: () => ({
status: {
traffic: [
{ percent: 100, revisionName: state.activeRevision },
...[...state.tags].map(([tag, revisionName]) => ({
tag,
revisionName,
url: `https://${tag}---relay-hash-uc.a.run.app`
}))
]
}
}),
describeRevision: (_config, revision) => {
const value = state.revisions.get(revision)
return {
metadata: {
annotations: {
'autoscaling.knative.dev/minScale': String(value?.minimum ?? 0),
'autoscaling.knative.dev/maxScale': String(value?.maximum ?? 5)
}
},
spec: {
serviceAccountName: value?.serviceAccount,
containers: [
{
image: value?.image,
env: [
...Object.entries(value?.env ?? {}).map(([name, value]) => ({ name, value })),
...Object.entries(value?.secrets ?? {}).map(([name, secretKeyRef]) => ({
name,
valueSource: { secretKeyRef }
}))
]
}
]
}
}
},
deployCandidate: (config, tag, env, image, minimum, maximum, regionalVersion) => {
const revision = `relay-${String(state.nextRevision++).padStart(5, '0')}-new`
state.revisions.set(revision, {
env: { ORCA_RELAY_ROLE: 'director', ...env },
secrets: {
[DIRECTOR_REGIONAL_PLACEMENT_ENV]: {
secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET,
version: regionalVersion
}
},
minimum: dropRequestedMinimum ? 0 : Number(minimum ?? 1),
maximum,
serviceAccount:
config['runtime-service-account'] ??
state.revisions.get(state.activeRevision)?.serviceAccount,
image: image ?? state.revisions.get(state.activeRevision)?.image
})
state.tags.set(tag, revision)
if (deployFailure) throw deployFailure
},
listRevisions: () =>
[...state.revisions].map(([name]) => ({ metadata: { name } })),
deleteRevision: (_config, revision) => {
if (deleteFailure) throw deleteFailure
state.revisions.delete(revision)
},
updateTraffic: (_config, args) => {
const remove = args.find((argument) => argument.startsWith('--remove-tags='))
if (remove) {
const tags = remove.slice('--remove-tags='.length).split(',')
removed.push(tags)
if (pendingCleanupFailure && tags.includes('candidate-new')) {
pendingCleanupFailure = false
throw new Error('failed to remove promoted candidate tag')
}
for (const tag of tags) state.tags.delete(tag)
if (cleanupReportsFailure) throw new Error('gcloud reported failed latest revision')
return
}
const promote = args.find((argument) => argument.startsWith('--to-tags='))
assert.ok(promote)
const tag = promote.slice('--to-tags='.length).split('=')[0]
state.activeRevision = state.tags.get(tag)
},
waitForHealth: async (_origin, connectionCapacityProtocol) => {
healthProtocols.push(connectionCapacityProtocol)
}
}
return { state, removed, healthProtocols, operations }
}
test('director deploy removes stale and promoted Cloud Run tags', async () => {
const harness = directorHarness()
const config = {
project: 'onorca-cloud-staging',
'capacity-service-account':
'orca-cloud-staging-gha-cap@onorca-cloud-staging.iam.gserviceaccount.com'
}
await deployDirector(config, 'candidate-new', harness.operations)
assert.deepEqual(harness.removed, [['candidate-old'], ['candidate-new']])
assert.equal(harness.state.activeRevision, 'relay-00003-new')
assert.deepEqual([...harness.state.tags.keys()], ['selector-rollback'])
assert.deepEqual([...harness.state.revisions.keys()], [
'relay-00000-stale',
'relay-00001-old',
'relay-00002-new',
'relay-00003-new'
])
for (const revision of ['relay-00002-new', 'relay-00003-new']) {
assert.deepEqual(
Object.fromEntries(
Object.entries(harness.state.revisions.get(revision).env).filter(([key]) =>
key in DIRECTOR_ADMISSION_ENVIRONMENT
)
),
DIRECTOR_ADMISSION_ENVIRONMENT
)
assert.equal(
harness.state.revisions.get(revision).env.ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT,
config['capacity-service-account']
)
}
assert.deepEqual(harness.healthProtocols, [undefined, undefined])
})
test('director deploy stamps the durable regional placement secret reference', async () => {
const harness = directorHarness()
await deployDirector({}, 'candidate-new', harness.operations)
for (const revision of ['relay-00002-new', 'relay-00003-new']) {
assert.deepEqual(
harness.state.revisions.get(revision).secrets[DIRECTOR_REGIONAL_PLACEMENT_ENV],
{ secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET, version: '1' }
)
}
})
test('bootstraps both rollback and candidate onto the distinct director identity', async () => {
const predecessor = 'relay-runtime@onorca-cloud.iam.gserviceaccount.com'
const director = 'relay-director@onorca-cloud.iam.gserviceaccount.com'
const harness = directorHarness({ servingServiceAccount: predecessor })
const inspected = []
harness.operations.assertRegionalRehomeDisabled = async (_config, origin) => {
inspected.push(origin)
}
await deployDirector({
project: 'onorca-cloud',
'runtime-service-account': director,
'predecessor-runtime-service-account': predecessor,
'predecessor-image-digest': `sha256:${'f'.repeat(64)}`,
'bootstrap-runtime-identity': 'true',
'expected-rehome-generation': '0',
'rehome-control-origin': 'https://relay.onorca.dev'
}, 'candidate-new', harness.operations)
assert.equal(
harness.state.revisions.get(harness.state.activeRevision).serviceAccount,
director
)
assert.equal(
harness.state.revisions.get(harness.state.tags.get('selector-rollback')).serviceAccount,
director
)
assert.deepEqual(inspected, [
'https://selector-rollback---relay-hash-uc.a.run.app',
'https://candidate-new---relay-hash-uc.a.run.app'
])
})
test('steady-state director deploy rejects the predecessor identity', async () => {
const harness = directorHarness({
servingServiceAccount: 'relay-runtime@onorca-cloud.iam.gserviceaccount.com'
})
await assert.rejects(deployDirector({
'runtime-service-account': 'relay-director@onorca-cloud.iam.gserviceaccount.com'
}, 'candidate-new', harness.operations), /unexpected runtime service account/)
})
test('director deploy prunes old revisions when requested', async () => {
const harness = directorHarness()
await deployDirector({ 'prune-revisions': 'true' }, 'candidate-new', harness.operations)
assert.deepEqual([...harness.state.revisions.keys()], [
'relay-00002-new',
'relay-00003-new'
])
assert.deepEqual(harness.healthProtocols, [2, 2])
})
test('a prune failure preserves the active and rollback traffic pair', async () => {
const harness = directorHarness({ deleteFailure: new Error('injected delete failure') })
await assert.rejects(
deployDirector({ 'prune-revisions': 'true' }, 'candidate-new', harness.operations),
/injected delete failure/
)
assert.equal(harness.state.activeRevision, 'relay-00003-new')
assert.deepEqual([...harness.state.tags.keys()], ['selector-rollback'])
})
test('director deploy removes a candidate tag left by a failed update', async () => {
const failure = new Error('candidate failed to become ready')
const harness = directorHarness({ deployFailure: failure })
await assert.rejects(deployDirector({}, 'candidate-new', harness.operations), failure)
assert.deepEqual(harness.removed, [['candidate-old'], ['selector-rollback']])
assert.equal(harness.state.tags.size, 0)
})
test('director candidate inherits the serving warm-instance floor', async () => {
const harness = directorHarness({ servingMinimum: 5 })
await deployDirector({}, 'candidate-new', harness.operations)
const serving = harness.state.revisions.get(harness.state.activeRevision)
assert.equal(serving.minimum, 5)
// The standby rollback revision must stay cold.
const rollback = harness.state.revisions.get(harness.state.tags.get('selector-rollback'))
assert.equal(rollback.minimum, 0)
})
test('director deploy rejects a serving maximum above the checked budget', async () => {
const harness = directorHarness({ servingMaximum: 6 })
await assert.rejects(
deployDirector({ 'max-instances': '5' }, 'candidate-new', harness.operations),
/holds 6 maximum instances, expected 5/
)
})
test('an explicit --min-instances still overrides the serving floor', async () => {
const harness = directorHarness({ servingMinimum: 5 })
await deployDirector({ 'min-instances': '0' }, 'candidate-new', harness.operations)
assert.equal(harness.state.revisions.get(harness.state.activeRevision).minimum, 0)
})
test('director deploy refuses to move traffic onto a candidate that lost the floor', async () => {
const harness = directorHarness({ servingMinimum: 5, dropRequestedMinimum: true })
await assert.rejects(
deployDirector({}, 'candidate-new', harness.operations),
/candidate holds 0 minimum instances, expected 5/
)
// Traffic never moved, so the original revision still serves.
assert.equal(harness.state.activeRevision, 'relay-00001-old')
})
test('director deploy rejects unrelated revision-shape drift', async () => {
const harness = directorHarness()
const describeRevision = harness.operations.describeRevision
harness.operations.describeRevision = (config, revision) => {
const described = describeRevision(config, revision)
described.spec.containerConcurrency = revision === 'relay-00001-old' ? 80 : 1_000
return described
}
await assert.rejects(
deployDirector({}, 'candidate-new', harness.operations),
/unrelated revision shape/
)
assert.equal(harness.state.activeRevision, 'relay-00001-old')
})
test('director cleanup verifies success when gcloud reports a stale revision failure', async () => {
const harness = directorHarness({ cleanupReportsFailure: true })
await deployDirector({}, 'candidate-new', harness.operations)
assert.deepEqual(harness.removed, [['candidate-old'], ['candidate-new']])
assert.deepEqual([...harness.state.tags.keys()], ['selector-rollback'])
})
test('director deploy restores rollback traffic after promoted-tag cleanup fails', async () => {
const harness = directorHarness({ cleanupFailsBeforeRemoval: true })
await assert.rejects(
deployDirector({}, 'candidate-new', harness.operations),
/failed to remove promoted candidate tag/
)
assert.equal(
harness.state.activeRevision,
harness.state.tags.get('selector-rollback')
)
assert.deepEqual([...harness.state.tags.keys()], ['selector-rollback'])
})
test('reads only literal revision environment values', () => {
assert.deepEqual(
revisionEnvironment({
spec: {
containers: [
{
env: [
{ name: 'ORCA_RELAY_CELL_ID', value: 'staging-c1' },
{ name: 'ORCA_RELAY_CELL_CAPACITY', value: '900' },
{ name: 'DATABASE_URL', valueFrom: { secretKeyRef: { name: 'database' } } }
]
}
]
}
}),
{ ORCA_RELAY_CELL_ID: 'staging-c1', ORCA_RELAY_CELL_CAPACITY: '900' }
)
})
test('reads only Secret Manager revision environment references', () => {
assert.deepEqual(
revisionSecretEnvironment({
spec: {
containers: [{ env: [
{ name: 'LITERAL', value: 'true' },
{
name: DIRECTOR_REGIONAL_PLACEMENT_ENV,
valueSource: { secretKeyRef: {
secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET,
version: 'latest'
} }
},
{
name: 'GCP_SECRET_SHAPE',
valueFrom: { secretKeyRef: { name: 'gcp-secret', key: '2' } }
}
] }]
}
}),
{
[DIRECTOR_REGIONAL_PLACEMENT_ENV]: {
secret: DIRECTOR_REGIONAL_PLACEMENT_SECRET,
version: 'latest'
},
GCP_SECRET_SHAPE: {
secret: 'gcp-secret',
version: '2'
}
}
)
})
test('accepts only a bounded JWT-shaped supplied admin identity token', () => {
assert.equal(suppliedAdminIdentityToken({}), null)
assert.equal(suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }), 'aaa.bbb.ccc')
assert.throws(() => suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: '' }))
assert.throws(() => suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: 'not-a-jwt' }))
assert.throws(() =>
suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: `aaa.${'b'.repeat(8_190)}.ccc` })
)
})
test('waits for authenticated target readiness without hiding other capacity errors', async () => {
let attempts = 0
const capacity = await waitForEvacuationCapacity(
async () => {
attempts += 1
if (attempts < 3) throw new Error('/v1/admin/evacuation-capacity failed: target_cell_unavailable')
return { requiredTargetUnits: 2, availableTargetUnits: 4_000 }
},
'source',
'target',
{ pollIntervalMs: 1, timeoutMs: 100 }
)
assert.equal(attempts, 3)
assert.equal(capacity.availableTargetUnits, 4_000)
await assert.rejects(
waitForEvacuationCapacity(async () => {
throw new Error('/v1/admin/evacuation-capacity failed: forbidden')
}, 'source', 'target'),
/forbidden/
)
})
@@ -0,0 +1,871 @@
import { readFileSync } from 'node:fs'
import { spawnSync } from 'node:child_process'
import { pathToFileURL } from 'node:url'
import {
inspectAdmissionSelector,
selectorCellState,
transitionAdmissionSelector
} from './relay-admission-selector.mjs'
const DEFAULT_POLL_INTERVAL_MS = 5_000
const DEFAULT_TIMEOUT_MS = 14 * 60 * 1_000
const ADMIN_RETRY_ATTEMPTS = 3
const ADMIN_RETRY_BASE_MS = 250
const CONNECTION_CONTROL_REBIND_RESERVE = 100
const SUPPORTED_CONNECTION_HARD_CAPS = new Set([600, 1_000, 3_000])
const RETRYABLE_ADMIN_PATHS = new Set([
'/v1/admin/runtime-status',
'/v1/admin/cell-status',
'/v1/admin/evacuation-capacity',
'/v1/admin/evacuation-status'
])
function canonicalOrigin(value, name) {
const url = new URL(value)
if (url.protocol !== 'https:' || url.origin !== value || url.pathname !== '/') {
throw new Error(`${name} must be a canonical HTTPS origin`)
}
return value
}
function adminAudience(value) {
const url = new URL(value)
if (
url.protocol !== 'https:' ||
url.pathname !== '/v1/admin/drain' ||
url.search ||
url.hash ||
url.toString() !== value
) {
throw new Error('--admin-audience must be the canonical HTTPS director drain URL')
}
return value
}
function positiveInteger(value, name, maximum = Number.MAX_SAFE_INTEGER) {
const parsed = Number(value)
if (!Number.isInteger(parsed) || parsed <= 0 || parsed > maximum) {
throw new Error(`${name} must be a positive integer`)
}
return parsed
}
function nonnegativeInteger(value, name) {
const parsed = Number(value)
if (!Number.isInteger(parsed) || parsed < 0) {
throw new Error(`${name} must be a nonnegative integer`)
}
return parsed
}
export function parseArguments(argv) {
const values = {}
for (let index = 0; index < argv.length; index += 2) {
const key = argv[index]
const value = argv[index + 1]
if (!key?.startsWith('--') || value === undefined) throw new Error(`invalid argument ${key ?? ''}`)
values[key.slice(2)] = value
}
for (const key of [
'project',
'director-origin',
'admin-audience',
'topology-file',
'source-cell-id',
'target-cell-id',
'runtime-service-account',
'mode'
]) {
if (!values[key]) throw new Error(`missing --${key}`)
}
if (
![
'audit',
'preflight',
'recover-forward',
'continue-evacuation',
'disable-cell',
'execute',
'reset-empty-candidate',
'enable-empty-cell'
].includes(values.mode)
) {
throw new Error(
'--mode must be audit, preflight, recover-forward, continue-evacuation, disable-cell, execute, reset-empty-candidate, or enable-empty-cell'
)
}
return {
project: values.project,
directorOrigin: canonicalOrigin(values['director-origin'], '--director-origin'),
adminAudience: adminAudience(values['admin-audience']),
topologyFile: values['topology-file'],
sourceCellId: values['source-cell-id'],
targetCellId: values['target-cell-id'],
runtimeServiceAccount: values['runtime-service-account'],
mode: values.mode,
batchSize: positiveInteger(values['batch-size'] ?? 100, '--batch-size', 100),
drainGraceMs: positiveInteger(
values['drain-grace-ms'] ?? 120_000,
'--drain-grace-ms',
60 * 60 * 1_000
),
pollIntervalMs: positiveInteger(
values['poll-interval-ms'] ?? DEFAULT_POLL_INTERVAL_MS,
'--poll-interval-ms',
60_000
),
timeoutMs: positiveInteger(
values['timeout-ms'] ?? DEFAULT_TIMEOUT_MS,
'--timeout-ms',
60 * 60 * 1_000
)
}
}
export function deployment(value, cellId) {
if (!value || typeof value !== 'object') throw new Error(`missing topology for ${cellId}`)
const expected = {
cellId,
origin: canonicalOrigin(value.origin, `${cellId} origin`),
region: String(value.region ?? 'us-central1'),
zone: String(value.zone ?? ''),
migName: String(value.mig_name ?? ''),
instanceGroup: String(value.instance_group ?? ''),
backendName: String(value.backend_name ?? ''),
backendId: String(value.backend_id ?? ''),
urlMapName: String(value.url_map_name ?? ''),
generationIdentity: String(value.generation_identity ?? ''),
image: String(value.image ?? ''),
imageDigest: String(value.image ?? '').split('@')[1] ?? '',
capacityRequests: positiveInteger(value.capacity_requests, `${cellId} capacity`),
databasePoolMax: positiveInteger(
value.database_pool_max ?? 10,
`${cellId} database pool maximum`,
100
),
connectionHardCap:
value.connection_hard_cap === null || value.connection_hard_cap === undefined
? undefined
: positiveInteger(value.connection_hard_cap, `${cellId} connection hard cap`),
connectionUnobservedBound:
value.connection_unobserved_bound === null ||
value.connection_unobserved_bound === undefined
? undefined
: nonnegativeInteger(
value.connection_unobserved_bound,
`${cellId} unobserved connection bound`
),
initiallyEnabled: value.initially_enabled,
fenced: value.fenced,
desiredTargetSize: value.desired_target_size
}
if (!/^[a-z0-9-]+$/.test(expected.zone)) throw new Error(`${cellId} has an invalid zone`)
if (!['us-central1', 'asia-east2'].includes(expected.region) || !expected.zone.startsWith(`${expected.region}-`)) {
throw new Error(`${cellId} has an invalid region`)
}
for (const [name, resource] of [
['MIG', expected.migName],
['instance group', expected.instanceGroup],
['backend', expected.backendName],
['backend ID', expected.backendId]
]) {
if (!resource) throw new Error(`${cellId} has no ${name}`)
}
if (!/^[a-z0-9.-]+\/[a-z0-9._/-]+@sha256:[a-f0-9]{64}$/.test(expected.image)) {
throw new Error(`${cellId} image is not digest-pinned`)
}
if (typeof expected.initiallyEnabled !== 'boolean') {
throw new Error(`${cellId} has no initial admission state`)
}
if (
(expected.connectionHardCap === undefined) !==
(expected.connectionUnobservedBound === undefined) ||
(expected.connectionHardCap !== undefined &&
(!SUPPORTED_CONNECTION_HARD_CAPS.has(expected.connectionHardCap) ||
expected.connectionUnobservedBound >=
expected.connectionHardCap - CONNECTION_CONTROL_REBIND_RESERVE))
) {
throw new Error(`${cellId} has invalid connection capacity`)
}
return expected
}
export function assertDeploymentConnectionCapacity(expected, runtime, director) {
if (expected.connectionHardCap === undefined) {
if (runtime !== null || director !== null) {
throw new Error(`${expected.cellId} connection capacity differs from Terraform`)
}
return
}
const hardCap = expected.connectionHardCap
const unobservedBound = expected.connectionUnobservedBound
const ordinaryConnectionLimit = hardCap - CONNECTION_CONTROL_REBIND_RESERVE
const normalAdmissionPause = ordinaryConnectionLimit - unobservedBound
const matches = (capacity) =>
capacity?.hardCap === hardCap &&
capacity.controlRebindReserve === CONNECTION_CONTROL_REBIND_RESERVE &&
capacity.ordinaryConnectionLimit === ordinaryConnectionLimit &&
capacity.unobservedBound === unobservedBound &&
capacity.normalAdmissionPause === normalAdmissionPause
if (!matches(runtime) || !matches(director) || director.heartbeatFresh !== true) {
throw new Error(`${expected.cellId} connection capacity differs from Terraform`)
}
}
export function selectDeployments(topology, sourceCellId, targetCellId) {
if (sourceCellId === targetCellId) throw new Error('source and target cell IDs must differ')
const source = deployment(topology[sourceCellId], sourceCellId)
const target = deployment(topology[targetCellId], targetCellId)
for (const key of ['origin', 'migName', 'instanceGroup', 'backendName', 'backendId']) {
if (source[key] === target[key]) throw new Error(`source and target ${key} overlap`)
}
if (target.initiallyEnabled) throw new Error('candidate must be declared initially disabled')
return { source, target }
}
export function validateMig(mig, instances, expected) {
if (Number(mig.targetSize) !== 1) throw new Error(`${expected.cellId} MIG is not fixed-one`)
const policy = mig.updatePolicy ?? {}
if (
policy.replacementMethod !== 'RECREATE' ||
Number(policy.maxSurge?.fixed ?? policy.maxSurge) !== 0 ||
Number(policy.maxUnavailable?.fixed ?? policy.maxUnavailable) !== 1
) {
throw new Error(`${expected.cellId} MIG replacement policy is unsafe`)
}
const serving = instances.filter(
(entry) => entry.instanceStatus === 'RUNNING' && entry.currentAction === 'NONE'
)
if (instances.length !== 1 || serving.length !== 1) {
throw new Error(`${expected.cellId} MIG must have one running endpoint`)
}
return serving[0].instance.split('/').at(-1)
}
export function validateInstance(instance, expected, runtimeServiceAccount) {
const publicConfigs = (instance.networkInterfaces ?? []).flatMap(
(network) => network.accessConfigs ?? []
)
if (publicConfigs.length !== 0) throw new Error(`${expected.cellId} instance has a public IP`)
const serviceAccounts = (instance.serviceAccounts ?? []).map((entry) => entry.email)
if (serviceAccounts.length !== 1 || serviceAccounts[0] !== runtimeServiceAccount) {
throw new Error(`${expected.cellId} runtime service account mismatch`)
}
}
export function validateBackend(backend, expected) {
if (
backend.protocol !== 'HTTP' ||
Number(backend.timeoutSec) !== 86_400 ||
(backend.backends ?? []).length !== 1 ||
backend.backends[0].group !== expected.instanceGroup
) {
throw new Error(`${expected.cellId} backend topology mismatch`)
}
}
export function defaultCommandJson(args) {
const result = spawnSync('gcloud', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] })
if (result.status !== 0) {
throw new Error(`gcloud ${args.slice(0, 4).join(' ')} failed: ${result.stderr.trim()}`)
}
return JSON.parse(result.stdout)
}
export function suppliedAdminIdentityToken(environment = process.env) {
const token = environment.ORCA_RELAY_ADMIN_ID_TOKEN
if (token === undefined) return null
return validatedIdentityToken(token, 'admin')
}
export function suppliedFenceMutationIdentityToken(environment = process.env) {
const token = environment.ORCA_RELAY_FENCE_MUTATION_ID_TOKEN
if (token === undefined) return null
return validatedIdentityToken(token, 'fence mutation')
}
function validatedIdentityToken(token, label) {
// WIF supplies a masked Google ID token because external-account gcloud cannot mint one directly.
if (token.length > 8_192 || !/^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/.test(token)) {
throw new Error(`invalid supplied ${label} identity token`)
}
return token
}
export function defaultIdentityToken(audience) {
const supplied = suppliedAdminIdentityToken()
if (supplied !== null) return supplied
const result = spawnSync(
'gcloud',
['auth', 'print-identity-token', `--audiences=${audience}`],
{ encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }
)
if (result.status !== 0) throw new Error('gcloud identity-token command failed')
return result.stdout.trim()
}
async function responseJson(response, label) {
const body = await response.json().catch(() => ({ error: `http_${response.status}` }))
if (!response.ok) throw new Error(`${label} failed: ${body.error ?? response.status}`)
return body
}
export function createAdminPost(config, deps, token) {
return async (origin, path, body) => {
const requestToken = typeof token === 'function' ? token(path) : token
for (let attempt = 1; attempt <= ADMIN_RETRY_ATTEMPTS; attempt++) {
let response
try {
response = await deps.fetch(`${origin}${path}`, {
method: 'POST',
headers: {
authorization: `Bearer ${requestToken}`,
'content-type': 'application/json'
},
body: JSON.stringify(body),
signal: AbortSignal.timeout(30_000)
})
} catch (error) {
if (!RETRYABLE_ADMIN_PATHS.has(path) || attempt === ADMIN_RETRY_ATTEMPTS) throw error
deps.emit({ event: 'candidate_admin_retry', path, attempt, reason: 'transport' })
await deps.wait(deps.random() * ADMIN_RETRY_BASE_MS * 2 ** (attempt - 1))
continue
}
if (
RETRYABLE_ADMIN_PATHS.has(path) &&
([502, 503, 504].includes(response.status) ||
(path === '/v1/admin/evacuation-status' && response.status === 500)) &&
attempt < ADMIN_RETRY_ATTEMPTS
) {
// These endpoints are read-only or transactionally idempotent, so a
// lost response may be retried without widening deployment authority.
deps.emit({
event: 'candidate_admin_retry',
path,
attempt,
reason: `http_${response.status}`
})
await response.arrayBuffer().catch(() => undefined)
await deps.wait(deps.random() * ADMIN_RETRY_BASE_MS * 2 ** (attempt - 1))
continue
}
return await responseJson(response, path)
}
throw new Error(`${path} retry attempts exhausted`)
}
}
async function checkHttp(deps, origin, path) {
const response = await deps.fetch(`${origin}${path}`, { signal: AbortSignal.timeout(15_000) })
const body = await response.json().catch(() => ({}))
if (!response.ok || body.ok !== true) throw new Error(`${origin}${path} is unavailable`)
}
export async function inspectCell(config, deps, adminPost, expected) {
const common = ['--project', config.project, '--zone', expected.zone, '--format=json']
const mig = deps.commandJson([
'compute',
'instance-groups',
'managed',
'describe',
expected.migName,
...common
])
const instances = deps.commandJson([
'compute',
'instance-groups',
'managed',
'list-instances',
expected.migName,
...common
])
const instanceName = validateMig(mig, instances, expected)
const instance = deps.commandJson([
'compute',
'instances',
'describe',
instanceName,
...common
])
validateInstance(instance, expected, config.runtimeServiceAccount)
const backend = deps.commandJson([
'compute',
'backend-services',
'describe',
expected.backendName,
'--global',
'--project',
config.project,
'--format=json'
])
validateBackend(backend, expected)
await checkHttp(deps, expected.origin, '/health')
await checkHttp(deps, expected.origin, '/ready')
const runtime = await adminPost(expected.origin, '/v1/admin/runtime-status', { v: 1 })
if (
runtime.role !== 'cell' ||
runtime.cellId !== expected.cellId ||
runtime.cellUrl !== expected.origin ||
(runtime.region ?? 'us-central1') !== expected.region ||
runtime.imageDigest !== expected.imageDigest
) {
throw new Error(`${expected.cellId} served runtime does not match Terraform topology`)
}
const status = await adminPost(config.directorOrigin, '/v1/admin/cell-status', {
v: 1,
cellId: expected.cellId
})
if (
status.status?.cellUrl !== expected.origin ||
(status.status?.region ?? 'us-central1') !== expected.region ||
status.status?.runtime?.cellUrl !== expected.origin ||
status.status?.runtime?.ready !== true ||
status.status?.runtime?.heartbeatFresh !== true
) {
throw new Error(`${expected.cellId} has no fresh ready authenticated heartbeat`)
}
assertDeploymentConnectionCapacity(
expected,
runtime.connectionCapacity ?? null,
status.status.connectionCapacity ?? null
)
return {
...status.status,
draining: runtime.draining === true,
process: runtime.runtime ?? null,
runtimeConnectionCapacity: runtime.connectionCapacity ?? null
}
}
async function waitForMigration(config, deps, adminPost, completeReady) {
const deadline = deps.now() + config.timeoutMs
while (deps.now() < deadline) {
const status = await adminPost(config.directorOrigin, '/v1/admin/evacuation-status', {
v: 1,
sourceCellId: config.sourceCellId,
targetCellId: config.targetCellId,
completeReady
})
deps.emit({ event: completeReady ? 'migration_completion' : 'migration_registration', ...status })
if (completeReady ? status.inProgress === 0 : status.inProgress === status.targetRegistered) {
return status
}
if (
completeReady &&
status.targetRegistered === status.inProgress &&
status.registeredSourceActive === 0 &&
status.registeredCompletable === 0 &&
status.registeredTargetInactive === status.inProgress
) {
// CI waiting cannot revive an offline desktop; keep its proven migration
// pending until that target control reconnects.
return status
}
await deps.wait(config.pollIntervalMs)
}
throw new Error('timed out waiting for candidate migration')
}
export async function setCellState(config, adminPost, cellId, enabled) {
const post = async (path, body) => await adminPost(config.directorOrigin, path, body)
const inspected = await inspectAdmissionSelector(post)
if (inspected.selector.generation > 0) {
await transitionAdmissionSelector(post, {
[cellId]: enabled ? 'general' : 'existing-only'
})
return
}
await post('/v1/admin/cell-state', { v: 1, cellId, enabled })
}
function assertNoDurableActivity(status, operation) {
const activity = [
status.assignments,
status.activityLeases,
status.reservedRequests,
status.outgoingMigrations,
status.incomingMigrations
]
if (activity.some((value) => Number(value) !== 0)) {
throw new Error(`${operation} requires zero durable activity`)
}
}
async function recoverCandidateFailure(
config,
deps,
adminPost,
source,
target,
allowEmptyAdmissionRollback,
selectorActive
) {
const status = await adminPost(config.directorOrigin, '/v1/admin/evacuation-status', {
v: 1,
sourceCellId: source.cellId,
targetCellId: target.cellId,
completeReady: false
}).catch(() => null)
if (!selectorActive && allowEmptyAdmissionRollback && status?.inProgress === 0) {
await setCellState(config, adminPost, source.cellId, true).catch(() => undefined)
await setCellState(config, adminPost, target.cellId, false).catch(() => undefined)
return
}
if (!selectorActive && status && status.inProgress > 0 && status.targetRegistered === 0) {
await setCellState(config, adminPost, source.cellId, true).catch(() => undefined)
deps.emit({
event: 'candidate_rollback_waiting_for_lease_expiry',
sourceCellId: source.cellId,
targetCellId: target.cellId,
inProgress: status.inProgress
})
return
}
deps.emit({
event: 'candidate_forward_recovery_required',
sourceCellId: source.cellId,
targetCellId: target.cellId,
targetRegistered: status?.targetRegistered ?? null
})
}
export async function drainSource(
config,
deps,
token,
source,
graceMs = config.drainGraceMs,
traceValue
) {
const response = await deps.fetch(`${source.origin}/v1/admin/drain`, {
method: 'POST',
headers: {
authorization: `Bearer ${token}`,
'content-type': 'application/json',
...(traceValue ? { 'x-orca-drain-trace': traceValue } : {})
},
body: JSON.stringify({ v: 1, graceMs }),
signal: AbortSignal.timeout(30_000)
})
await responseJson(response, 'source drain')
return {
backendStatus: response.status,
backendInstance: response.headers.get('x-orca-backend-instance') ?? undefined
}
}
async function verifyCandidateCompletion(
config,
adminPost,
source,
target,
event,
eventName = 'candidate_complete'
) {
const finalSource = await adminPost(config.directorOrigin, '/v1/admin/cell-status', {
v: 1,
cellId: source.cellId
})
const finalTarget = await adminPost(config.directorOrigin, '/v1/admin/cell-status', {
v: 1,
cellId: target.cellId
})
if (
finalSource.status.activityLeases !== 0 ||
finalSource.status.reservedRequests !== 0 ||
finalSource.status.outgoingMigrations !== 0 ||
finalSource.status.runtime?.observedRequests !== 0 ||
finalTarget.status.incomingMigrations !== 0 ||
finalTarget.status.reservedRequests !== finalTarget.status.activityRequestUnits
) {
throw new Error('aggregate post-migration counts are not reconciled')
}
event({
event: eventName,
sourceCellId: source.cellId,
targetCellId: target.cellId,
dormantSourceAssignments: finalSource.status.assignments,
targetAssignments: finalTarget.status.assignments,
targetActivityLeases: finalTarget.status.activityLeases,
targetReservedRequests: finalTarget.status.reservedRequests
})
}
export async function runCandidateDeployment(config, overrides = {}) {
const deps = {
commandJson: overrides.commandJson ?? defaultCommandJson,
identityToken: overrides.identityToken ?? defaultIdentityToken,
fetch: overrides.fetch ?? fetch,
emit:
overrides.emit ??
((event) => process.stdout.write(`${JSON.stringify(event)}\n`)),
now: overrides.now ?? Date.now,
wait: overrides.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))),
random: overrides.random ?? Math.random
}
const topology = JSON.parse(readFileSync(config.topologyFile, 'utf8'))
const { source, target } = selectDeployments(
topology,
config.sourceCellId,
config.targetCellId
)
const token = deps.identityToken(config.adminAudience)
const adminPost = createAdminPost(config, deps, token)
const selectorPost = async (path, body) =>
await adminPost(config.directorOrigin, path, body)
const selectorInspection = await inspectAdmissionSelector(selectorPost)
const selectorActive = selectorInspection.selector.generation > 0
const sourceStatus = await inspectCell(config, deps, adminPost, source)
const targetStatus = await inspectCell(config, deps, adminPost, target)
const sourceAdmission = selectorActive
? selectorCellState(selectorInspection.selector, source.cellId)
: sourceStatus.enabled
? 'general'
: 'existing-only'
const targetAdmission = selectorActive
? selectorCellState(selectorInspection.selector, target.cellId)
: targetStatus.enabled
? 'general'
: 'existing-only'
if (config.mode === 'audit') {
const migration = await adminPost(config.directorOrigin, '/v1/admin/evacuation-status', {
v: 1,
sourceCellId: source.cellId,
targetCellId: target.cellId,
completeReady: false
})
// Forward recovery needs durable aggregate evidence without exposing assignment identities.
deps.emit({
event: 'candidate_audit',
source: aggregateCellStatus(sourceStatus),
target: aggregateCellStatus(targetStatus),
migration
})
return
}
if (config.mode === 'recover-forward') {
if (
selectorActive
? sourceAdmission !== 'existing-only' || targetAdmission !== 'migration-only'
: sourceStatus.enabled || !targetStatus.enabled
) {
throw new Error(
selectorActive
? 'forward recovery requires existing-only source and migration-only target'
: 'forward recovery requires disabled source and enabled target'
)
}
await drainSource(config, deps, token, source)
await waitForMigration(config, deps, adminPost, false)
const completion = await waitForMigration(config, deps, adminPost, true)
if (completion.inProgress > 0) {
deps.emit({
event: 'candidate_forward_pending',
sourceCellId: source.cellId,
targetCellId: target.cellId,
inProgress: completion.inProgress,
registeredSourceActive: completion.registeredSourceActive,
registeredCompletable: completion.registeredCompletable,
registeredTargetInactive: completion.registeredTargetInactive
})
throw new Error('forward recovery remains pending for inactive target controls')
}
await verifyCandidateCompletion(
config,
adminPost,
source,
target,
deps.emit,
'candidate_forward_recovered'
)
return
}
if (config.mode === 'disable-cell') {
if (targetStatus.enabled) await setCellState(config, adminPost, target.cellId, false)
// Disabling new admission preserves origin-owned sessions and durable recovery work.
deps.emit({
event: 'cell_admission_disabled',
targetCellId: target.cellId,
changed: targetStatus.enabled,
assignments: targetStatus.assignments,
activityLeases: targetStatus.activityLeases,
reservedRequests: targetStatus.reservedRequests,
outgoingMigrations: targetStatus.outgoingMigrations,
incomingMigrations: targetStatus.incomingMigrations
})
return
}
// Repair is safe only before a candidate owns assignments or origin-scoped work.
if (config.mode === 'reset-empty-candidate') {
assertNoDurableActivity(targetStatus, 'candidate admission reset')
if (targetStatus.enabled) await setCellState(config, adminPost, target.cellId, false)
deps.emit({
event: 'candidate_admission_reset',
targetCellId: target.cellId,
changed: targetStatus.enabled
})
return
}
if (config.mode === 'enable-empty-cell') {
assertNoDurableActivity(targetStatus, 'cell admission enable')
if (selectorActive ? targetAdmission === 'general' : targetStatus.enabled) {
deps.emit({ event: 'cell_admission_enabled', targetCellId: target.cellId, changed: false })
return
}
}
const continuingEvacuation = config.mode === 'continue-evacuation'
if (continuingEvacuation) {
if (
selectorActive
? targetAdmission !== 'migration-only'
: !targetStatus.enabled
) {
throw new Error(
selectorActive
? 'continued evacuation requires migration-only target'
: 'continued evacuation requires enabled target'
)
}
} else if (!selectorActive && targetStatus.enabled) {
throw new Error('candidate cell is already enabled')
} else if (
selectorActive &&
!['migration-only', 'existing-only'].includes(targetAdmission)
) {
throw new Error('candidate cell must not be generally admitted')
}
const capacity = await adminPost(config.directorOrigin, '/v1/admin/evacuation-capacity', {
v: 1,
sourceCellId: source.cellId,
targetCellId: target.cellId
})
if (capacity.requiredTargetUnits > capacity.availableTargetUnits) {
throw new Error('candidate lacks survivor request-unit headroom')
}
deps.emit({
event: 'candidate_preflight',
mode: config.mode,
sourceCellId: source.cellId,
targetCellId: target.cellId,
sourceOrigin: source.origin,
targetOrigin: target.origin,
sourceMig: source.migName,
targetMig: target.migName,
sourceBackend: source.backendName,
targetBackend: target.backendName,
sourceDigest: source.imageDigest,
targetDigest: target.imageDigest,
sourceAssignments: capacity.sourceAssignments,
requiredTargetUnits: capacity.requiredTargetUnits,
availableTargetUnits: capacity.availableTargetUnits
})
if (config.mode === 'preflight') return
if (config.mode === 'enable-empty-cell') {
await setCellState(config, adminPost, target.cellId, true)
deps.emit({ event: 'cell_admission_enabled', targetCellId: target.cellId, changed: true })
return
}
// Fresh execution starts from source-only admission; continuation preserves its target.
if (
!continuingEvacuation &&
(selectorActive ? sourceAdmission !== 'existing-only' : !sourceStatus.enabled)
) {
throw new Error(
selectorActive ? 'source cell is not existing-only' : 'source cell is not enabled'
)
}
if (selectorActive && targetAdmission !== 'migration-only') {
throw new Error('target cell is not migration-only')
}
let migrationsStarted = 0
try {
if (!selectorActive) {
if (sourceStatus.enabled) await setCellState(config, adminPost, source.cellId, false)
if (!targetStatus.enabled) await setCellState(config, adminPost, target.cellId, true)
}
for (;;) {
const result = await adminPost(config.directorOrigin, '/v1/admin/evacuate-cell', {
v: 1,
sourceCellId: source.cellId,
targetCellId: target.cellId,
limit: config.batchSize
})
migrationsStarted += result.started
deps.emit({ event: 'migration_batch', started: result.started, totalStarted: migrationsStarted })
if (result.started === 0) break
}
} catch (error) {
await recoverCandidateFailure(
config,
deps,
adminPost,
source,
target,
!continuingEvacuation,
selectorActive
)
throw error
}
try {
if (selectorActive) {
const currentSelector = await inspectAdmissionSelector(selectorPost)
if (
currentSelector.selector.generation !== selectorInspection.selector.generation ||
JSON.stringify(currentSelector.selector.membership) !==
JSON.stringify(selectorInspection.selector.membership)
) {
throw new Error('admission selector changed before drain')
}
}
await drainSource(config, deps, token, source)
await waitForMigration(config, deps, adminPost, false)
const completion = await waitForMigration(config, deps, adminPost, true)
if (completion.inProgress > 0) {
throw new Error('candidate migration remains pending for inactive target controls')
}
} catch (error) {
// A completion response can be lost after its transaction commits. Never
// reverse admission here merely because no in-progress row remains.
await recoverCandidateFailure(
config,
deps,
adminPost,
source,
target,
false,
selectorActive
)
throw error
}
await verifyCandidateCompletion(config, adminPost, source, target, deps.emit)
}
export function aggregateCellStatus(status) {
return {
cellId: status.cellId,
enabled: status.enabled,
assignments: status.assignments,
activityLeases: status.activityLeases,
activityRequestUnits: status.activityRequestUnits,
reservedRequests: status.reservedRequests,
outgoingMigrations: status.outgoingMigrations,
incomingMigrations: status.incomingMigrations,
runtimeReady: status.runtime?.ready ?? false,
heartbeatFresh: status.runtime?.heartbeatFresh ?? false,
observedRequests: status.runtime?.observedRequests ?? null
}
}
export async function main(argv = process.argv.slice(2)) {
await runCandidateDeployment(parseArguments(argv))
}
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
main().catch((error) => {
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
process.exitCode = 1
})
}
@@ -0,0 +1,889 @@
import assert from 'node:assert/strict'
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { test } from 'node:test'
import {
assertDeploymentConnectionCapacity,
createAdminPost,
deployment,
parseArguments,
runCandidateDeployment,
selectDeployments,
suppliedAdminIdentityToken,
validateBackend,
validateInstance,
validateMig
} from './deploy-relay-gce-candidate.mjs'
const runtimeServiceAccount = 'orca-relay@example.iam.gserviceaccount.com'
const digestA = `sha256:${'a'.repeat(64)}`
const digestB = `sha256:${'b'.repeat(64)}`
test('retries evacuation-status HTTP 500 without widening other admin retries', async () => {
const events = []
let calls = 0
const adminPost = createAdminPost({}, {
fetch: async () => {
calls++
return calls === 1
? new Response(JSON.stringify({ error: 'transient' }), { status: 500 })
: new Response(JSON.stringify({ ok: true }), { status: 200 })
},
emit: (event) => events.push(event),
wait: async () => {},
random: () => 0
}, 'token')
assert.deepEqual(
await adminPost('https://relay.example', '/v1/admin/evacuation-status', {}),
{ ok: true }
)
assert.equal(calls, 2)
assert.deepEqual(events, [{
event: 'candidate_admin_retry',
path: '/v1/admin/evacuation-status',
attempt: 1,
reason: 'http_500'
}])
calls = 0
await assert.rejects(
createAdminPost({}, {
fetch: async () => {
calls++
return new Response(JSON.stringify({ error: 'persistent' }), { status: 500 })
},
emit: () => {},
wait: async () => {},
random: () => 0
}, 'token')('https://relay.example', '/v1/admin/runtime-status', {}),
/runtime-status failed: persistent/
)
assert.equal(calls, 1)
})
test('verifies a 1,000-cap cell against both runtime and director telemetry', () => {
const expected = deployment(
{
...topology().target,
connection_hard_cap: 1_000,
connection_unobserved_bound: 60
},
'target'
)
const capacity = {
hardCap: 1_000,
controlRebindReserve: 100,
ordinaryConnectionLimit: 900,
unobservedBound: 60,
normalAdmissionPause: 840
}
assert.doesNotThrow(() =>
assertDeploymentConnectionCapacity(expected, capacity, {
...capacity,
heartbeatFresh: true
})
)
assert.throws(
() =>
assertDeploymentConnectionCapacity(expected, capacity, {
...capacity,
hardCap: 600,
heartbeatFresh: true
}),
/differs from Terraform/
)
})
test('verifies a 3,000-cap regional cell with a 2,840 placement boundary', () => {
const expected = deployment(
{
...topology().target,
connection_hard_cap: 3_000,
connection_unobserved_bound: 60
},
'target'
)
const capacity = {
hardCap: 3_000,
controlRebindReserve: 100,
ordinaryConnectionLimit: 2_900,
unobservedBound: 60,
normalAdmissionPause: 2_840
}
assert.doesNotThrow(() =>
assertDeploymentConnectionCapacity(expected, capacity, {
...capacity,
heartbeatFresh: true
})
)
})
function topology() {
return {
source: {
origin: 'https://c1.relay.example.com',
zone: 'us-central1-b',
mig_name: 'relay-c1',
instance_group: 'https://compute.example/instanceGroups/relay-c1',
backend_name: 'relay-c1',
backend_id: 'https://compute.example/backendServices/relay-c1',
image: `us-central1-docker.pkg.dev/project/repo/relay@${digestA}`,
capacity_requests: 4_000,
initially_enabled: true
},
target: {
origin: 'https://c2.relay.example.com',
zone: 'us-central1-c',
mig_name: 'relay-c2',
instance_group: 'https://compute.example/instanceGroups/relay-c2',
backend_name: 'relay-c2',
backend_id: 'https://compute.example/backendServices/relay-c2',
image: `us-central1-docker.pkg.dev/project/repo/relay@${digestB}`,
capacity_requests: 4_000,
initially_enabled: false
}
}
}
function withTopology(operation) {
const directory = mkdtempSync(join(tmpdir(), 'relay-gce-candidate-'))
const file = join(directory, 'topology.json')
writeFileSync(file, JSON.stringify(topology()))
return Promise.resolve(operation(file)).finally(() => rmSync(directory, { recursive: true }))
}
function config(topologyFile, mode = 'preflight') {
return {
project: 'test-project',
directorOrigin: 'https://relay.example.com',
adminAudience: 'https://relay.example.com/v1/admin/drain',
topologyFile,
sourceCellId: 'source',
targetCellId: 'target',
runtimeServiceAccount,
mode,
batchSize: 100,
drainGraceMs: 120_000,
pollIntervalMs: 1,
timeoutMs: 1_000
}
}
function response(body, status = 200) {
return new Response(JSON.stringify(body), {
status,
headers: { 'content-type': 'application/json' }
})
}
function migrationResponse(body) {
return response({
registeredSourceActive: 0,
registeredCompletable: 0,
registeredTargetInactive: 0,
expiredUnregistered: 0,
repairableExpiredUnregistered: 0,
abortableExpiredUnregistered: 0,
blockedExpiredUnregistered: 0,
blockedExpiredOnNewerTargetAssignment: 0,
...body
})
}
function fakeCommand(args) {
const name = args[args.indexOf('describe') + 1] ?? args[args.indexOf('list-instances') + 1]
if (args.includes('list-instances')) {
return [
{
instance: `https://compute.example/instances/${name}-vm`,
instanceStatus: 'RUNNING',
currentAction: 'NONE'
}
]
}
if (args.includes('instance-groups')) {
return {
targetSize: 1,
updatePolicy: {
replacementMethod: 'RECREATE',
maxSurge: { fixed: 0 },
maxUnavailable: { fixed: 1 }
}
}
}
if (args.includes('instances')) {
return {
networkInterfaces: [{ networkIP: '10.42.0.2' }],
serviceAccounts: [{ email: runtimeServiceAccount }]
}
}
const cell = name === 'relay-c1' ? topology().source : topology().target
return { protocol: 'HTTP', timeoutSec: 86_400, backends: [{ group: cell.instance_group }] }
}
function harness({
failTargetEnable = false,
failDrain = false,
failAfterRegistration = false,
failCompletionResponse = false,
sourceEnabled = true,
targetEnabled = false,
targetAssignments = 0,
migrationInProgress = 0,
migrationTargetRegistered = 0,
migrationTargetInactive = 0,
transientCompletionFailures = 0,
dormantSourceAssignments = 0,
selectorGeneration = 0
} = {}) {
const state = {
source: {
enabled: selectorGeneration > 0 ? false : sourceEnabled,
assignments: 2,
activityLeases: 2
},
target: {
enabled: selectorGeneration > 0 ? true : targetEnabled,
assignments: targetAssignments,
activityLeases: targetAssignments
}
}
const events = []
const stateChanges = []
let batch = 0
let migrationCompleted = false
let remainingTransientCompletionFailures = transientCompletionFailures
const fetch = async (url, options = {}) => {
const parsed = new URL(url)
if (parsed.pathname === '/health' || parsed.pathname === '/ready') return response({ ok: true })
const body = JSON.parse(options.body ?? '{}')
if (parsed.pathname === '/v1/admin/runtime-status') {
const target = parsed.origin.includes('c2.')
return response({
v: 1,
role: 'cell',
cellId: target ? 'target' : 'source',
cellUrl: parsed.origin,
imageDigest: target ? digestB : digestA
})
}
if (parsed.pathname === '/v1/admin/admission-selector/status') {
return response({
v: 1,
selector: {
generation: selectorGeneration,
attemptId: null,
membership: {
existingOnly:
selectorGeneration > 0
? ['source']
: Object.keys(state).filter((cellId) => !state[cellId].enabled),
migrationOnly: selectorGeneration > 0 ? ['target'] : [],
general:
selectorGeneration > 0
? []
: Object.keys(state).filter((cellId) => state[cellId].enabled)
}
},
intent: null
})
}
if (parsed.pathname === '/v1/admin/cell-status') {
const cell = state[body.cellId]
return response({
v: 1,
status: {
cellId: body.cellId,
cellUrl: topology()[body.cellId].origin,
enabled: cell.enabled,
admissionState:
selectorGeneration > 0
? body.cellId === 'source'
? 'existing-only'
: 'migration-only'
: cell.enabled
? 'general'
: 'existing-only',
assignments: cell.assignments,
activityLeases: cell.activityLeases,
activityRequestUnits: cell.activityLeases,
reservedRequests: cell.activityLeases,
outgoingMigrations: 0,
incomingMigrations: 0,
runtime: {
cellUrl: topology()[body.cellId].origin,
ready: true,
heartbeatFresh: true,
observedRequests: cell.activityLeases
}
}
})
}
if (parsed.pathname === '/v1/admin/evacuation-capacity') {
return response({ sourceAssignments: 2, requiredTargetUnits: 4, availableTargetUnits: 4_000 })
}
if (parsed.pathname === '/v1/admin/cell-state') {
stateChanges.push([body.cellId, body.enabled])
if (failTargetEnable && body.cellId === 'target' && body.enabled) {
return response({ error: 'injected_enable_failure' }, 409)
}
state[body.cellId].enabled = body.enabled
return response({ ok: true })
}
if (parsed.pathname === '/v1/admin/evacuate-cell') {
if (failAfterRegistration && batch > 0) {
return response({ error: 'injected_batch_failure' }, 503)
}
const started = batch++ === 0 ? 2 : 0
return response({ v: 1, started })
}
if (parsed.pathname === '/v1/admin/evacuation-status') {
if (failAfterRegistration) {
return migrationResponse({
v: 1,
inProgress: 2,
targetRegistered: 1,
registeredSourceActive: 1,
completed: 0,
blocked: 1
})
}
if (failDrain) {
return migrationResponse({
v: 1,
inProgress: 2,
targetRegistered: 0,
completed: 0,
blocked: 0
})
}
if (body.completeReady) {
state.source.assignments = dormantSourceAssignments
state.source.activityLeases = 0
state.target.assignments = 2
state.target.activityLeases = 2
if (remainingTransientCompletionFailures > 0) {
remainingTransientCompletionFailures--
throw new TypeError('injected transient fetch failure')
}
if (migrationTargetInactive > 0) {
return migrationResponse({
v: 1,
inProgress: migrationTargetInactive,
targetRegistered: migrationTargetInactive,
registeredTargetInactive: migrationTargetInactive,
completed: 0,
blocked: migrationTargetInactive
})
}
migrationCompleted = true
if (failCompletionResponse) {
return response({ error: 'injected_completion_response_failure' }, 503)
}
return migrationResponse({
v: 1,
inProgress: 0,
targetRegistered: 0,
completed: 2,
blocked: 0
})
}
if (migrationCompleted) {
return migrationResponse({
v: 1,
inProgress: 0,
targetRegistered: 0,
completed: 0,
blocked: 0
})
}
if (batch === 0) {
return migrationResponse({
v: 1,
inProgress: migrationInProgress,
targetRegistered: migrationTargetRegistered,
completed: 0,
blocked: 0
})
}
return migrationResponse({
v: 1,
inProgress: 2,
targetRegistered: 2,
registeredCompletable: 2,
completed: 0,
blocked: 0
})
}
if (parsed.pathname === '/v1/admin/drain') {
return failDrain ? response({ error: 'injected_drain_failure' }, 503) : response({ ok: true })
}
return response({ error: 'unexpected_request' }, 500)
}
return {
overrides: {
commandJson: fakeCommand,
identityToken: () => 'secret-token-never-emitted',
fetch,
emit: (event) => events.push(event),
wait: async () => undefined,
random: () => 0
},
events,
stateChanges,
state
}
}
test('requires explicit dry-run/execute inputs and a distinct disabled candidate', () => {
assert.throws(() => parseArguments([]), /missing --project/)
assert.throws(
() =>
parseArguments([
'--project',
'project',
'--director-origin',
'https://relay.example.com',
'--admin-audience',
'https://relay.example.com/not-drain',
'--topology-file',
'topology.json',
'--source-cell-id',
'source',
'--target-cell-id',
'target',
'--runtime-service-account',
runtimeServiceAccount,
'--mode',
'preflight'
]),
/director drain URL/
)
assert.throws(() => selectDeployments(topology(), 'source', 'source'), /must differ/)
const overlapping = topology()
overlapping.target.backend_id = overlapping.source.backend_id
assert.throws(() => selectDeployments(overlapping, 'source', 'target'), /backendId overlap/)
const enabled = topology()
enabled.target.initially_enabled = true
assert.throws(() => selectDeployments(enabled, 'source', 'target'), /initially disabled/)
})
test('accepts only a bounded JWT-shaped supplied admin identity token', () => {
assert.equal(suppliedAdminIdentityToken({}), null)
assert.equal(suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }), 'aaa.bbb.ccc')
assert.throws(() => suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: '' }))
assert.throws(() => suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: 'not-a-jwt' }))
assert.throws(() =>
suppliedAdminIdentityToken({ ORCA_RELAY_ADMIN_ID_TOKEN: `aaa.${'b'.repeat(8_190)}.ccc` })
)
})
test('rejects unsafe fixed-one topology, public IPs, and backend overlap', () => {
const expected = selectDeployments(topology(), 'source', 'target').target
assert.throws(() => validateMig({ targetSize: 2 }, [], expected), /fixed-one/)
assert.throws(
() =>
validateInstance(
{
networkInterfaces: [{ accessConfigs: [{ natIP: '203.0.113.1' }] }],
serviceAccounts: [{ email: runtimeServiceAccount }]
},
expected,
runtimeServiceAccount
),
/public IP/
)
assert.throws(
() => validateBackend({ protocol: 'HTTP', timeoutSec: 86_400, backends: [] }, expected),
/topology mismatch/
)
})
test('preflights exact served digests and survivor headroom without mutating admission', async () => {
await withTopology(async (file) => {
const { overrides, events, stateChanges } = harness()
await runCandidateDeployment(config(file), overrides)
assert.deepEqual(stateChanges, [])
assert.equal(events[0].event, 'candidate_preflight')
assert.equal(events[0].targetDigest, digestB)
assert.equal(JSON.stringify(events).includes('secret-token'), false)
})
})
test('audits a partially committed migration without changing admission or completing rows', async () => {
await withTopology(async (file) => {
const { overrides, events, stateChanges } = harness({
sourceEnabled: false,
targetEnabled: true,
targetAssignments: 1,
migrationInProgress: 2,
migrationTargetRegistered: 2
})
await runCandidateDeployment(config(file, 'audit'), overrides)
assert.deepEqual(stateChanges, [])
assert.deepEqual(events, [
{
event: 'candidate_audit',
source: {
cellId: 'source',
enabled: false,
assignments: 2,
activityLeases: 2,
activityRequestUnits: 2,
reservedRequests: 2,
outgoingMigrations: 0,
incomingMigrations: 0,
runtimeReady: true,
heartbeatFresh: true,
observedRequests: 2
},
target: {
cellId: 'target',
enabled: true,
assignments: 1,
activityLeases: 1,
activityRequestUnits: 1,
reservedRequests: 1,
outgoingMigrations: 0,
incomingMigrations: 0,
runtimeReady: true,
heartbeatFresh: true,
observedRequests: 1
},
migration: {
v: 1,
inProgress: 2,
targetRegistered: 2,
registeredSourceActive: 0,
registeredCompletable: 0,
registeredTargetInactive: 0,
completed: 0,
blocked: 0,
expiredUnregistered: 0,
repairableExpiredUnregistered: 0,
abortableExpiredUnregistered: 0,
blockedExpiredUnregistered: 0,
blockedExpiredOnNewerTargetAssignment: 0
}
}
])
})
})
test('preflights with source admission disabled but still refuses execution', async () => {
await withTopology(async (file) => {
const { overrides, events, stateChanges } = harness({ sourceEnabled: false })
await runCandidateDeployment(config(file), overrides)
assert.deepEqual(stateChanges, [])
assert.equal(events.at(-1).event, 'candidate_preflight')
await assert.rejects(
runCandidateDeployment(config(file, 'execute'), overrides),
/source cell is not enabled/
)
assert.deepEqual(stateChanges, [])
})
})
test('explicitly resets only an empty declared candidate to disabled admission', async () => {
await withTopology(async (file) => {
const { overrides, events, stateChanges } = harness({
sourceEnabled: false,
targetEnabled: true
})
await runCandidateDeployment(config(file, 'reset-empty-candidate'), overrides)
assert.deepEqual(stateChanges, [['target', false]])
assert.deepEqual(events.at(-1), {
event: 'candidate_admission_reset',
targetCellId: 'target',
changed: true
})
})
})
test('refuses to reset candidate admission while it owns durable activity', async () => {
await withTopology(async (file) => {
const { overrides, stateChanges } = harness({ targetEnabled: true, targetAssignments: 1 })
await assert.rejects(
runCandidateDeployment(config(file, 'reset-empty-candidate'), overrides),
/requires zero durable activity/
)
assert.deepEqual(stateChanges, [])
})
})
test('disables only new admission while preserving durable candidate activity', async () => {
await withTopology(async (file) => {
const { overrides, events, stateChanges } = harness({
targetEnabled: true,
targetAssignments: 1
})
await runCandidateDeployment(config(file, 'disable-cell'), overrides)
assert.deepEqual(stateChanges, [['target', false]])
assert.deepEqual(events.at(-1), {
event: 'cell_admission_disabled',
targetCellId: 'target',
changed: true,
assignments: 1,
activityLeases: 1,
reservedRequests: 1,
outgoingMigrations: 0,
incomingMigrations: 0
})
})
})
test('explicitly enables only an empty preflighted cell for admission', async () => {
await withTopology(async (file) => {
const { overrides, events, stateChanges } = harness({ sourceEnabled: false })
await runCandidateDeployment(config(file, 'enable-empty-cell'), overrides)
assert.deepEqual(stateChanges, [['target', true]])
assert.equal(events.at(-2).event, 'candidate_preflight')
assert.deepEqual(events.at(-1), {
event: 'cell_admission_enabled',
targetCellId: 'target',
changed: true
})
})
})
test('refuses to enable cell admission while it owns durable activity', async () => {
await withTopology(async (file) => {
const { overrides, stateChanges } = harness({ targetAssignments: 1 })
await assert.rejects(
runCandidateDeployment(config(file, 'enable-empty-cell'), overrides),
/requires zero durable activity/
)
assert.deepEqual(stateChanges, [])
})
})
test('executes target-first evacuation and verifies aggregate drained counts', async () => {
await withTopology(async (file) => {
const { overrides, events, stateChanges } = harness()
await runCandidateDeployment(config(file, 'execute'), overrides)
assert.deepEqual(stateChanges.slice(0, 2), [
['source', false],
['target', true]
])
assert.equal(events.at(-1).event, 'candidate_complete')
assert.equal(events.at(-1).targetAssignments, 2)
})
})
test('executes within selector membership without legacy admission writes', async () => {
await withTopology(async (file) => {
const testHarness = harness({ selectorGeneration: 1 })
await runCandidateDeployment(config(file, 'execute'), testHarness.overrides)
assert.deepEqual(testHarness.stateChanges, [])
assert.equal(testHarness.state.source.enabled, false)
assert.equal(testHarness.state.target.enabled, true)
})
})
test('never restores legacy general admission after selector-era failure', async () => {
await withTopology(async (file) => {
const testHarness = harness({
selectorGeneration: 1,
failAfterRegistration: true
})
await assert.rejects(
runCandidateDeployment(config(file, 'execute'), testHarness.overrides),
/injected_batch_failure/
)
assert.deepEqual(testHarness.stateChanges, [])
assert.equal(testHarness.state.source.enabled, false)
})
})
test('continues a partial evacuation without resetting target admission', async () => {
await withTopology(async (file) => {
const { overrides, events, stateChanges } = harness({
sourceEnabled: true,
targetEnabled: true,
targetAssignments: 1,
migrationInProgress: 1,
migrationTargetRegistered: 1
})
await runCandidateDeployment(config(file, 'continue-evacuation'), overrides)
assert.deepEqual(stateChanges, [['source', false]])
assert.equal(events.at(-1).event, 'candidate_complete')
})
})
test('refuses continued evacuation unless target admission is already enabled', async () => {
await withTopology(async (file) => {
const { overrides, stateChanges } = harness()
await assert.rejects(
runCandidateDeployment(config(file, 'continue-evacuation'), overrides),
/requires enabled target/
)
assert.deepEqual(stateChanges, [])
})
})
test('preserves partial target admission when continued batching fails', async () => {
await withTopology(async (file) => {
const { overrides, events, stateChanges } = harness({
failAfterRegistration: true,
sourceEnabled: true,
targetEnabled: true,
targetAssignments: 1,
migrationInProgress: 1,
migrationTargetRegistered: 1
})
await assert.rejects(
runCandidateDeployment(config(file, 'continue-evacuation'), overrides),
/injected_batch_failure/
)
assert.deepEqual(stateChanges, [['source', false]])
assert.equal(events.at(-1).event, 'candidate_forward_recovery_required')
})
})
test('resumes only a committed forward migration and preserves dormant source assignments', async () => {
await withTopology(async (file) => {
const { overrides, events, stateChanges } = harness({
sourceEnabled: false,
targetEnabled: true,
migrationInProgress: 2,
migrationTargetRegistered: 2,
dormantSourceAssignments: 7
})
await runCandidateDeployment(config(file, 'recover-forward'), overrides)
assert.deepEqual(stateChanges, [])
assert.deepEqual(events.at(-1), {
event: 'candidate_forward_recovered',
sourceCellId: 'source',
targetCellId: 'target',
dormantSourceAssignments: 7,
targetAssignments: 2,
targetActivityLeases: 2,
targetReservedRequests: 2
})
})
})
test('retries a transient idempotent completion request without reversing admission', async () => {
await withTopology(async (file) => {
const { overrides, events, stateChanges } = harness({
sourceEnabled: false,
targetEnabled: true,
migrationInProgress: 2,
migrationTargetRegistered: 2,
transientCompletionFailures: 1
})
await runCandidateDeployment(config(file, 'recover-forward'), overrides)
assert.deepEqual(stateChanges, [])
assert.deepEqual(
events.filter(({ event }) => event === 'candidate_admin_retry'),
[
{
event: 'candidate_admin_retry',
path: '/v1/admin/evacuation-status',
attempt: 1,
reason: 'transport'
}
]
)
assert.equal(events.at(-1).event, 'candidate_forward_recovered')
})
})
test('stops forward recovery promptly when only registered offline targets remain', async () => {
await withTopology(async (file) => {
const { overrides, events, stateChanges } = harness({
sourceEnabled: false,
targetEnabled: true,
migrationInProgress: 2,
migrationTargetRegistered: 2,
migrationTargetInactive: 2
})
await assert.rejects(
runCandidateDeployment(config(file, 'recover-forward'), overrides),
/pending for inactive target controls/
)
assert.deepEqual(stateChanges, [])
assert.deepEqual(events.at(-1), {
event: 'candidate_forward_pending',
sourceCellId: 'source',
targetCellId: 'target',
inProgress: 2,
registeredSourceActive: 0,
registeredCompletable: 0,
registeredTargetInactive: 2
})
})
})
test('refuses forward recovery unless source and target admission match committed direction', async () => {
await withTopology(async (file) => {
const { overrides, stateChanges } = harness()
await assert.rejects(
runCandidateDeployment(config(file, 'recover-forward'), overrides),
/requires disabled source and enabled target/
)
assert.deepEqual(stateChanges, [])
})
})
test('re-enables an intact source when candidate admission fails before migration', async () => {
await withTopology(async (file) => {
const { overrides, stateChanges } = harness({ failTargetEnable: true })
await assert.rejects(
runCandidateDeployment(config(file, 'execute'), overrides),
/injected_enable_failure/
)
assert.deepEqual(stateChanges, [
['source', false],
['target', true],
['source', true],
['target', false]
])
})
})
test('re-enables the source and waits for lease rollback when no target registered', async () => {
await withTopology(async (file) => {
const { overrides, events, stateChanges } = harness({ failDrain: true })
await assert.rejects(
runCandidateDeployment(config(file, 'execute'), overrides),
/injected_drain_failure/
)
assert.deepEqual(stateChanges.slice(-1), [['source', true]])
assert.equal(events.at(-1).event, 'candidate_rollback_waiting_for_lease_expiry')
})
})
test('preserves both routes for forward recovery after a target registration', async () => {
await withTopology(async (file) => {
const { overrides, events, stateChanges } = harness({ failAfterRegistration: true })
await assert.rejects(
runCandidateDeployment(config(file, 'execute'), overrides),
/injected_batch_failure/
)
assert.deepEqual(stateChanges, [
['source', false],
['target', true]
])
assert.equal(events.at(-1).event, 'candidate_forward_recovery_required')
assert.equal(events.at(-1).targetRegistered, 1)
})
})
test('does not reverse admission after completion commits but its response is lost', async () => {
await withTopology(async (file) => {
const { overrides, events, stateChanges } = harness({ failCompletionResponse: true })
await assert.rejects(
runCandidateDeployment(config(file, 'execute'), overrides),
/injected_completion_response_failure/
)
assert.deepEqual(stateChanges, [
['source', false],
['target', true]
])
assert.equal(events.at(-1).event, 'candidate_forward_recovery_required')
assert.equal(events.at(-1).targetRegistered, 0)
})
})
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+133
View File
@@ -0,0 +1,133 @@
const REQUEST_TIMEOUT_MS = 15_000
const JWT_PATTERN = /^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/
export async function requestGitHubSmokeTokens(
authOrigin,
fetchImpl = fetch,
environment = process.env,
options = {}
) {
const origin = canonicalHttpsOrigin(authOrigin)
const requestUrl = environment.ACTIONS_ID_TOKEN_REQUEST_URL
const requestToken = environment.ACTIONS_ID_TOKEN_REQUEST_TOKEN
if (!requestUrl || !requestToken) throw new Error('GitHub OIDC request context is unavailable')
const audience = `${origin}/v1/internal/github-smoke-token`
const oidcUrl = new URL(requestUrl)
oidcUrl.searchParams.set('audience', audience)
const oidcResponse = await fetchImpl(oidcUrl, {
headers: { authorization: `Bearer ${requestToken}` },
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS)
})
const oidc = await readJson(oidcResponse, 'GitHub OIDC request')
if (!oidcResponse.ok || !validJwt(oidc.value)) {
throw new Error(`GitHub OIDC request failed with ${oidcResponse.status}`)
}
const exchangeResponse = await fetchImpl(audience, {
method: 'POST',
headers: {
authorization: `Bearer ${oidc.value}`,
...(options.relayAsiaLoad ? { 'content-type': 'application/json' } : {})
},
...(options.relayAsiaLoad
? { body: JSON.stringify({ relayAsiaLoad: parseRelayAsiaLoadOptions(options.relayAsiaLoad) }) }
: {}),
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS)
})
const exchange = await readJson(exchangeResponse, 'Orca smoke identity exchange')
if (!exchangeResponse.ok) {
throw new Error(`Orca smoke identity exchange failed with ${exchangeResponse.status}`)
}
return {
...parseAccessTokens(exchange.accessTokens),
...(options.relayAsiaLoad
? {
relayAsiaLoadPrincipals: parseRelayAsiaLoadPrincipals(
exchange.relayAsiaLoadPrincipals,
options.relayAsiaLoad.principalCount
)
}
: {})
}
}
function parseRelayAsiaLoadOptions(value) {
if (
!value || typeof value !== 'object' ||
!Number.isSafeInteger(value.shardIndex) || value.shardIndex < 0 || value.shardIndex > 3 ||
!Number.isSafeInteger(value.principalCount) || value.principalCount < 1 ||
value.principalCount > 32
) throw new Error('Relay Asia load principal request is invalid')
return { v: 1, shardIndex: value.shardIndex, principalCount: value.principalCount }
}
function canonicalHttpsOrigin(value) {
const url = new URL(value)
if (url.protocol !== 'https:' || url.pathname !== '/' || url.search || url.hash) {
throw new Error('auth origin must be canonical HTTPS')
}
return url.origin
}
function validJwt(value) {
return typeof value === 'string' && value.length <= 8192 && JWT_PATTERN.test(value)
}
function parseAccessTokens(value) {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new Error('Orca smoke identity response is malformed')
}
const expected = ['outsider', 'owner', 'recipient']
if (Object.keys(value).sort().join(',') !== expected.join(',')) {
throw new Error('Orca smoke identity response principals are invalid')
}
return Object.fromEntries(
expected.map((name) => {
const principal = value[name]
if (
!principal ||
typeof principal !== 'object' ||
typeof principal.userId !== 'string' ||
!/^[A-Za-z0-9_-]{1,128}$/.test(principal.userId) ||
typeof principal.accessToken !== 'string' ||
!validJwt(principal.accessToken) ||
typeof principal.expiresAt !== 'number' ||
principal.expiresAt <= Date.now() ||
principal.expiresAt > Date.now() + 610_000
) {
throw new Error('Orca smoke identity response principal is malformed')
}
return [name, principal]
})
)
}
function parseRelayAsiaLoadPrincipals(value, expectedCount) {
if (!Array.isArray(value) || value.length !== expectedCount) {
throw new Error('Relay Asia load principal response is invalid')
}
const userIds = new Set()
return value.map((principal, principalIndex) => {
if (
!principal || typeof principal !== 'object' ||
principal.principalIndex !== principalIndex ||
typeof principal.userId !== 'string' ||
!/^usr_relay_asia_load_[A-Za-z0-9_-]{32}$/.test(principal.userId) ||
typeof principal.profileId !== 'string' ||
principal.profileId !== principal.userId.replace(/^usr_/, 'prof_') ||
userIds.has(principal.userId) ||
typeof principal.accessToken !== 'string' || !validJwt(principal.accessToken) ||
typeof principal.expiresAt !== 'number' || principal.expiresAt <= Date.now() ||
principal.expiresAt > Date.now() + 610_000
) throw new Error('Relay Asia load principal response is malformed')
userIds.add(principal.userId)
return principal
})
}
async function readJson(response, label) {
try {
return await response.json()
} catch {
throw new Error(`${label} did not return JSON`)
}
}
@@ -0,0 +1,111 @@
import assert from 'node:assert/strict'
import test from 'node:test'
import { requestGitHubSmokeTokens } from './github-smoke-token.mjs'
const jwt = (value) => `${value}.${value}.${value}`
test('exchanges the runner OIDC token without returning request credentials', async () => {
const requests = []
const fetchImpl = async (url, init) => {
requests.push({ url: String(url), init })
if (requests.length === 1) return Response.json({ value: jwt('github') })
return Response.json({
accessTokens: Object.fromEntries(
['owner', 'recipient', 'outsider'].map((name) => [
name,
{ userId: `usr_${name}`, accessToken: jwt(name), expiresAt: Date.now() + 600_000 }
])
)
})
}
const result = await requestGitHubSmokeTokens(
'https://auth-staging.onorca.dev',
fetchImpl,
{
ACTIONS_ID_TOKEN_REQUEST_URL: 'https://actions.example.test/token?api-version=1',
ACTIONS_ID_TOKEN_REQUEST_TOKEN: 'runner-request-token'
}
)
assert.equal(result.owner.userId, 'usr_owner')
assert.match(requests[0].url, /audience=https%3A%2F%2Fauth-staging\.onorca\.dev/)
assert.equal(requests[0].init.headers.authorization, 'Bearer runner-request-token')
assert.equal(requests[1].init.headers.authorization, `Bearer ${jwt('github')}`)
})
test('fails with bounded errors and never includes credentials', async () => {
await assert.rejects(
requestGitHubSmokeTokens(
'https://auth-staging.onorca.dev',
async () => new Response('denied', { status: 403 }),
{
ACTIONS_ID_TOKEN_REQUEST_URL: 'https://actions.example.test/token',
ACTIONS_ID_TOKEN_REQUEST_TOKEN: 'private-request-token'
}
),
(error) => {
assert.doesNotMatch(String(error), /private-request-token|denied/)
return true
}
)
})
test('requests and validates an exact Relay Asia principal batch', async () => {
const requests = []
const principals = Array.from({ length: 32 }, (_, principalIndex) => {
const suffix = String(principalIndex).padStart(32, 'a')
return {
principalIndex,
userId: `usr_relay_asia_load_${suffix}`,
profileId: `prof_relay_asia_load_${suffix}`,
accessToken: jwt(`load${principalIndex}`),
expiresAt: Date.now() + 600_000
}
})
const result = await requestGitHubSmokeTokens(
'https://auth-staging.onorca.dev',
async (url, init) => {
requests.push({ url: String(url), init })
return requests.length === 1
? Response.json({ value: jwt('github') })
: Response.json({
accessTokens: Object.fromEntries(['owner', 'recipient', 'outsider'].map((name) => [
name,
{ userId: `usr_${name}`, accessToken: jwt(name), expiresAt: Date.now() + 600_000 }
])),
relayAsiaLoadPrincipals: principals
})
},
{
ACTIONS_ID_TOKEN_REQUEST_URL: 'https://actions.example.test/token',
ACTIONS_ID_TOKEN_REQUEST_TOKEN: 'runner-request-token'
},
{ relayAsiaLoad: { shardIndex: 3, principalCount: 32 } }
)
assert.equal(result.relayAsiaLoadPrincipals.length, 32)
assert.deepEqual(JSON.parse(requests[1].init.body), {
relayAsiaLoad: { v: 1, shardIndex: 3, principalCount: 32 }
})
assert.equal(requests[1].init.headers['content-type'], 'application/json')
})
test('rejects malformed or duplicate Relay Asia principal batches', async () => {
const environment = {
ACTIONS_ID_TOKEN_REQUEST_URL: 'https://actions.example.test/token',
ACTIONS_ID_TOKEN_REQUEST_TOKEN: 'runner-request-token'
}
let request = 0
await assert.rejects(requestGitHubSmokeTokens(
'https://auth-staging.onorca.dev',
async () => ++request === 1
? Response.json({ value: jwt('github') })
: Response.json({
accessTokens: Object.fromEntries(['owner', 'recipient', 'outsider'].map((name) => [
name,
{ userId: `usr_${name}`, accessToken: jwt(name), expiresAt: Date.now() + 600_000 }
])),
relayAsiaLoadPrincipals: []
}),
environment,
{ relayAsiaLoad: { shardIndex: 0, principalCount: 32 } }
), /principal response is invalid/)
})
+146
View File
@@ -0,0 +1,146 @@
import { execFileSync } from 'node:child_process'
import { existsSync } from 'node:fs'
import { homedir } from 'node:os'
import { join } from 'node:path'
import { assertStagingRelayAwake } from './staging-relay-apply-guard.mjs'
// The relay root keeps its historical path so every existing caller — 9 workflows, the fence
// broker, and the infra:* scripts — is unchanged when --root is omitted. The foundation and apps
// roots stay in the private repository with the services they own.
const ROOT_DIRECTORIES = {
relay: join('infra', 'terraform')
}
const command = process.argv[2]
const environment = readEnvironment(process.argv.slice(3))
const root = readRoot(process.argv.slice(3))
const tool = process.env.IAC_TOOL || findTool()
if (!command || !['init', 'plan', 'apply'].includes(command)) {
exitWithUsage()
}
if (!environment) {
exitWithUsage('Missing --env staging|production')
}
if (!root) {
exitWithUsage(`Unknown --root; expected one of ${Object.keys(ROOT_DIRECTORIES).join('|')}`)
}
const rootDirectory = ROOT_DIRECTORIES[root]
const terraformDir = join(process.cwd(), rootDirectory)
const backendConfig = join(terraformDir, 'backend', `${environment}.hcl`)
const varFile = join(terraformDir, 'environments', `${environment}.tfvars`)
if (!existsSync(backendConfig)) {
throw new Error(`Backend config not found: ${backendConfig}`)
}
if (!existsSync(varFile)) {
throw new Error(`Variable file not found: ${varFile}`)
}
const chdir = `-chdir=${rootDirectory}`
if (command === 'init') {
run([chdir, 'init', `-backend-config=backend/${environment}.hcl`])
} else if (command === 'plan') {
run([
chdir,
'plan',
`-var-file=environments/${environment}.tfvars`,
`-out=${environment}.tfplan`
])
} else {
// A normal staging apply must not implicitly wake or partially mutate a sleeping data plane.
// Only the relay root can touch that data plane; the guard would refuse app work for no reason.
if (environment === 'staging' && root === 'relay') assertStagingRelayAwake()
run([chdir, 'apply', `${environment}.tfplan`])
}
function readEnvironment(args) {
const envIndex = args.indexOf('--env')
if (envIndex >= 0) {
return args[envIndex + 1]
}
return process.env.ORCA_CLOUD_ENV
}
function readRoot(args) {
const rootIndex = args.indexOf('--root')
const requested = rootIndex >= 0 ? args[rootIndex + 1] : 'relay'
return requested in ROOT_DIRECTORIES ? requested : undefined
}
function findTool() {
for (const candidate of ['tofu', 'terraform']) {
try {
execFileSync(candidate, ['version'], { stdio: 'ignore' })
return candidate
} catch {
// Try the next compatible IaC binary.
}
}
throw new Error('Install Terraform or OpenTofu, or set IAC_TOOL.')
}
function run(args) {
execFileSync(tool, args, { env: terraformEnv(), stdio: 'inherit' })
}
function terraformEnv() {
if (
process.env.GOOGLE_APPLICATION_CREDENTIALS ||
process.env.GOOGLE_CREDENTIALS ||
process.env.GOOGLE_OAUTH_ACCESS_TOKEN
) {
return process.env
}
const token = readGcloudAccessToken()
if (!token) {
return process.env
}
// Local convenience: Terraform uses ADC, while engineers often only have
// gcloud CLI auth. CI should use Workload Identity instead.
return { ...process.env, GOOGLE_OAUTH_ACCESS_TOKEN: token }
}
function readGcloudAccessToken() {
for (const candidate of gcloudCandidates()) {
try {
return execFileSync(candidate, ['auth', 'print-access-token'], {
encoding: 'utf8',
stdio: ['ignore', 'pipe', 'ignore']
}).trim()
} catch {
// Try the next common gcloud location.
}
}
return null
}
function gcloudCandidates() {
return [
process.env.GCLOUD_PATH,
'gcloud',
join(homedir(), 'Downloads', 'google-cloud-sdk', 'bin', 'gcloud'),
join(homedir(), 'google-cloud-sdk', 'bin', 'gcloud')
].filter(Boolean)
}
function exitWithUsage(message) {
if (message) {
console.error(message)
}
console.error(
'Usage: pnpm infra:<init|plan|apply> --env staging|production [--root relay]'
)
process.exit(1)
}
+77
View File
@@ -0,0 +1,77 @@
import assert from 'node:assert/strict'
import { execFileSync } from 'node:child_process'
import { readFileSync } from 'node:fs'
import { test } from 'node:test'
import { fileURLToPath } from 'node:url'
const repository = fileURLToPath(new URL('../../', import.meta.url))
const script = 'dev/scripts/infra.mjs'
// IAC_TOOL=echo prints the argv the real binary would have received, so the root a flag selects
// is observable without running Terraform.
function invoke(args) {
return execFileSync('node', [script, ...args], {
cwd: repository,
encoding: 'utf8',
env: { ...process.env, IAC_TOOL: 'echo' }
}).trim()
}
function rejects(args) {
try {
execFileSync('node', [script, ...args], {
cwd: repository,
encoding: 'utf8',
stdio: ['ignore', 'pipe', 'pipe'],
env: { ...process.env, IAC_TOOL: 'echo' }
})
} catch (error) {
return error.stderr
}
throw new Error(`expected ${args.join(' ')} to exit non-zero`)
}
// Why: 9 relay workflows, the fence broker, and the three infra:* package scripts all invoke this
// without --root. If the default ever moves off infra/terraform they break silently at the plan.
test('omitting --root keeps every existing caller on the relay root', () => {
for (const environment of ['staging', 'production']) {
assert.equal(
invoke(['init', '--env', environment]),
`-chdir=infra/terraform init -backend-config=backend/${environment}.hcl`
)
assert.equal(
invoke(['plan', '--env', environment]),
`-chdir=infra/terraform plan -var-file=environments/${environment}.tfvars -out=${environment}.tfplan`
)
}
})
// Only the relay root ships here; the foundation and apps roots stay in the private repository.
test('each root name selects exactly its own directory', () => {
const directories = { relay: 'infra/terraform' }
for (const [root, directory] of Object.entries(directories)) {
for (const environment of ['staging', 'production']) {
assert.equal(
invoke(['init', '--env', environment, '--root', root]),
`-chdir=${directory} init -backend-config=backend/${environment}.hcl`
)
}
}
})
test('an unknown root fails closed rather than falling back to the relay root', () => {
const stderr = rejects(['plan', '--env', 'staging', '--root', 'releay'])
assert.match(stderr, /Unknown --root/)
assert.doesNotMatch(stderr, /infra\/terraform /)
})
test('a missing environment still fails before any root is resolved', () => {
assert.match(rejects(['plan']), /Missing --env/)
})
// Why: the guard refuses a staging apply while the relay data plane is asleep. Applying it to the
// app or foundation roots would block work that never touches that data plane.
test('the sleeping staging relay guard is scoped to the relay root', () => {
const source = readFileSync(new URL('./infra.mjs', import.meta.url), 'utf8')
assert.match(source, /environment === 'staging' && root === 'relay'/)
})
+570
View File
@@ -0,0 +1,570 @@
import { createHash, createPrivateKey, createPublicKey } from 'node:crypto'
import { readFileSync } from 'node:fs'
import { monitorEventLoopDelay } from 'node:perf_hooks'
import { setTimeout as delay } from 'node:timers/promises'
import { RelayLoadControlPeer } from './relay-load-control-peer.mjs'
import { requestGitHubSmokeTokens } from './github-smoke-token.mjs'
import { relayLoadFailureReason } from './relay-load-connection-failure.mjs'
import {
assertRelayLoadDirectorCapacityToken,
waitForRelayLoadDirectorCapacity,
waitForRelayLoadRequestUnits
} from './relay-load-director-capacity-gate.mjs'
import { waitForRelayLoadPhaseBarrier } from './relay-load-phase-barrier.mjs'
import {
proveRelayLoadPlacementBoundary,
proveRelayLoadRegionalFallback
} from './relay-load-placement-boundary.mjs'
import {
proveRelayLoadRebindBoundary,
waitForRelayLoadRebindGate
} from './relay-load-rebind-boundary.mjs'
import { proveRelayLoadRegionBehavior } from './relay-load-region-behavior.mjs'
import {
openRelayLoadInviteOffers,
proveRelayLoadRequestUnitBoundary
} from './relay-load-request-unit-boundary.mjs'
import {
assertRelayLoadRampAccepted,
relayLoadRunHasDisallowedFailures,
runRelayLoadWithShutdown
} from './relay-load-run-lifecycle.mjs'
import { createRelayLoadReaderEvidence } from './relay-load-reader-evidence.mjs'
import {
parseRelayLoadArguments,
relayLoadPrincipalIndex,
relayLoadReaderEvidenceError,
relayLoadSpliceIndexes,
relayLoadSpliceProfile,
relayLoadSpliceStartDelayMs
} from './relay-load-profile.mjs'
function signingKey(path) {
if (!path) return {}
const key = createPrivateKey(readFileSync(path, 'utf8'))
const signingKeyId = createHash('sha256')
.update(createPublicKey(key).export({ type: 'spki', format: 'der' }))
.digest('base64url')
.slice(0, 16)
return { signingKey: key, signingKeyId }
}
function report(state, final = false) {
const elapsedSeconds = Math.max(1, (Date.now() - state.startedAt) / 1000)
const memory = process.memoryUsage()
const cpu = process.cpuUsage(state.generatorBaselineCpu)
const rssMiB = memory.rss / 1_048_576
state.generatorPeakRssMiB = Math.max(state.generatorPeakRssMiB, rssMiB)
const readerQueueEvidence = state.readerEvidence?.snapshot() ?? []
const output = {
event: final ? 'relay_load_complete' : 'relay_load_progress',
controls: state.controls,
shardCount: state.shardCount,
shardIndex: state.shardIndex,
configuredRampSeconds: state.rampMs / 1000,
configuredSteadySeconds: state.durationMs / 1000,
configuredSpliceHoldSeconds: state.spliceHoldMs / 1000,
requiredLeaseHorizons: state.requiredLeaseHorizons,
configuredSplices: state.splices,
configuredSlowReaderSplices: state.slowReaderSplices,
configuredWedgedReaderSplices: state.wedgedReaderSplices,
active: state.active.size,
peakActive: state.peakActive,
steadyMinimumActive: state.steadyMinimumActive,
connected: state.connected,
connectionFailures: state.connectionFailures,
rampConnectionFailures: state.rampConnectionFailures,
steadyConnectionFailures: state.steadyConnectionFailures,
transitionConnectionFailures: state.transitionConnectionFailures,
connectionFailuresByReason: state.connectionFailuresByReason,
closes: state.closes,
unexpectedCloses: state.unexpectedCloses,
unexpectedClosesByCode: state.unexpectedClosesByCode,
drains: state.drains,
pings: state.pings,
pingRate: Number((state.pings / elapsedSeconds).toFixed(2)),
tokens: state.tokens,
tokenRate: Number((state.tokens / elapsedSeconds).toFixed(2)),
refreshes: state.refreshes,
refreshErrors: state.refreshErrors,
protocolErrors: state.protocolErrors,
socketErrors: state.socketErrors,
rebindProbesOpened: state.rebindProbesOpened,
rebindOverflowReason: state.rebindOverflowReason,
placementOverflowReason: state.placementOverflowReason,
regionalFallbacksProved: state.regionalFallbacksProved,
oldClientUsFirstProved: state.oldClientUsFirstProved,
stickyAssignmentProved: state.stickyAssignmentProved,
requestUnitInvitesOpened: state.requestUnitInvitesOpened,
requestUnitPrincipalCount: state.requestUnitPrincipalCount,
relayAsiaLoadPrincipalCount: state.relayAsiaLoadPrincipalCount,
requestUnitOverflowReason: state.requestUnitOverflowReason,
requestUnitCleanupProved: state.requestUnitCleanupProved,
phaseBarrierPassed: state.phaseBarrierPassed,
activeSplices: state.activeSplices,
peakActiveSplices: state.peakActiveSplices,
completedSplices: state.completedSplices,
failedSplices: state.failedSplices,
slowReaderSplicesCompleted: state.slowReaderSplicesCompleted,
wedgedReaderSplicesClosed: state.wedgedReaderSplicesClosed,
readerQueueEvidence,
readerQueuedBytesPeak: Math.max(
0,
...readerQueueEvidence.map(({ increaseBytes }) => increaseBytes)
),
readerClosesByCode: state.readerClosesByCode,
controlHeadroom: Math.max(0, state.controls - state.active.size),
generatorRssMiB: Number(rssMiB.toFixed(1)),
generatorPeakRssMiB: Number(state.generatorPeakRssMiB.toFixed(1)),
generatorRssGrowthMiB: Number(
Math.max(0, state.generatorPeakRssMiB - state.generatorBaselineRssMiB).toFixed(1)
),
generatorHeapUsedMiB: Number((memory.heapUsed / 1_048_576).toFixed(1)),
generatorCpuPercent: Number(
(((cpu.user + cpu.system) / 1_000_000 / elapsedSeconds) * 100).toFixed(1)
),
generatorEventLoopP99Ms: Number((state.eventLoopDelay.percentile(99) / 1_000_000).toFixed(2)),
shutdownEvidence: final ? state.shutdownEvidence : undefined,
elapsedSeconds: Number(elapsedSeconds.toFixed(1))
}
console.log(JSON.stringify(output))
return output
}
const config = parseRelayLoadArguments(process.argv.slice(2))
let accessToken = process.env.ORCA_RELAY_LOAD_ACCESS_TOKEN
let accessTokenProviderForIndex
const adminToken = process.env.ORCA_RELAY_ADMIN_ID_TOKEN
if (
config.placementOverflowProbes > 0 || config.regionalFallbackProbes > 0 ||
config.slowReaderSplices + config.wedgedReaderSplices > 0 ||
config.requestUnitOverflowProbes > 0 || config.requestUnitCleanupTimeoutMs > 0
) {
assertRelayLoadDirectorCapacityToken({
directorOrigin: config.directorOrigin,
adminToken
}, Date.now,
config.rampMs + config.durationMs + config.wedgedReaderHoldMs +
(config.phaseBarrierDir ? 2 * config.phaseBarrierTimeoutMs : 0) +
config.spliceRampMs + config.requestUnitCleanupTimeoutMs + 120_000)
}
const key = signingKey(config.signingKeyFile)
if (!accessToken && !key.signingKey && process.env.ACTIONS_ID_TOKEN_REQUEST_URL &&
process.env.ACTIONS_ID_TOKEN_REQUEST_TOKEN) {
let tokens
let refresh
const loadOptions = config.relayAsiaLoadPrincipalCount > 0
? {
relayAsiaLoad: {
shardIndex: config.shardIndex,
principalCount: config.relayAsiaLoadPrincipalCount
}
}
: undefined
const smokeTokens = async () => {
const expiresAt = config.relayAsiaLoadPrincipalCount > 0
? tokens?.relayAsiaLoadPrincipals?.[0]?.expiresAt
: tokens?.owner?.expiresAt
if (expiresAt > Date.now() + 60_000) return tokens
refresh ??= requestGitHubSmokeTokens(
config.authOrigin,
fetch,
process.env,
loadOptions
)
try {
tokens = await refresh
return tokens
} finally {
refresh = undefined
}
}
accessTokenProviderForIndex = (index) => async () => {
const current = await smokeTokens()
return config.relayAsiaLoadPrincipalCount > 0
? current.relayAsiaLoadPrincipals[
relayLoadPrincipalIndex(
index,
config.shardCount,
current.relayAsiaLoadPrincipals.length
)
].accessToken
: current.owner.accessToken
}
await smokeTokens()
}
if (!accessToken && !accessTokenProviderForIndex && !key.signingKey) {
throw new Error('provide GitHub OIDC, ORCA_RELAY_LOAD_ACCESS_TOKEN, or --signing-key-file')
}
const eventLoopDelay = monitorEventLoopDelay({ resolution: 20 })
eventLoopDelay.enable()
const generatorBaselineRssMiB = process.memoryUsage().rss / 1_048_576
const generatorBaselineCpu = process.cpuUsage()
const state = {
...config,
startedAt: Date.now(),
active: new Set(),
peakActive: 0,
steadyMinimumActive: null,
steadyStarted: false,
connected: 0,
connectionFailures: 0,
rampConnectionFailures: 0,
steadyConnectionFailures: 0,
transitionConnectionFailures: 0,
connectionFailuresByReason: {},
closes: 0,
unexpectedCloses: 0,
unexpectedClosesByCode: {},
drains: 0,
pings: 0,
tokens: 0,
refreshes: 0,
refreshErrors: 0,
protocolErrors: 0,
socketErrors: 0,
rebindProbesOpened: 0,
rebindOverflowReason: null,
placementOverflowReason: null,
regionalFallbacksProved: 0,
oldClientUsFirstProved: 0,
stickyAssignmentProved: 0,
requestUnitInvitesOpened: 0,
requestUnitOverflowReason: null,
requestUnitCleanupProved: 0,
phaseBarrierPassed: false,
activeSplices: 0,
peakActiveSplices: 0,
completedSplices: 0,
failedSplices: 0,
slowReaderSplicesCompleted: 0,
wedgedReaderSplicesClosed: 0,
readerEvidence: null,
readerClosesByCode: {},
generatorBaselineRssMiB,
generatorBaselineCpu,
generatorPeakRssMiB: generatorBaselineRssMiB,
peerShutdowns: 0,
shutdownEvidence: null,
eventLoopDelay,
stopping: false,
transitionWindow: false
}
const peers = new Map()
const reconnectTimers = new Set()
async function readRuntimeQueuedBytes(origin) {
const response = await fetch(`${origin}/v1/admin/runtime-status`, {
method: 'POST',
headers: { authorization: `Bearer ${adminToken}`, 'content-type': 'application/json' },
body: JSON.stringify({ v: 1 }),
signal: AbortSignal.timeout(5_000)
})
if (response.status === 401 || response.status === 403) {
throw new Error('reader evidence identity was rejected')
}
if (!response.ok) throw new Error(`reader runtime status returned ${response.status}`)
const status = await response.json()
const queuedBytes = status?.runtime?.queuedBytes
if (!Number.isSafeInteger(queuedBytes) || queuedBytes < 0) {
throw new Error('reader runtime queued bytes are invalid')
}
return queuedBytes
}
async function observeReaderPressure(input) {
if (!state.readerEvidence) throw new Error('reader evidence baseline is unavailable')
await state.readerEvidence.observe(input)
state.generatorPeakRssMiB = Math.max(
state.generatorPeakRssMiB,
process.memoryUsage().rss / 1_048_576
)
}
function recordSteadyMinimum() {
if (!state.steadyStarted || state.stopping) return
state.steadyMinimumActive = Math.min(state.steadyMinimumActive, state.active.size)
}
function scheduleReconnect(peer) {
if (state.stopping) return
const timeout = setTimeout(() => {
reconnectTimers.delete(timeout)
void connect(peer)
}, Math.floor(Math.random() * (config.reconnectMaxMs + 1)))
reconnectTimers.add(timeout)
}
function observe(type, detail) {
if (type === 'connected') {
state.active.add(detail.index)
state.connected++
state.peakActive = Math.max(state.peakActive, state.active.size)
recordSteadyMinimum()
} else if (type === 'closed') {
state.active.delete(detail.index)
state.closes++
if (!detail.stopped && !detail.expectedDrain) {
state.unexpectedCloses++
const code = String(detail.code)
state.unexpectedClosesByCode[code] = (state.unexpectedClosesByCode[code] ?? 0) + 1
}
if (!detail.stopped) scheduleReconnect(peers.get(detail.index))
recordSteadyMinimum()
} else if (type === 'drain') state.drains++
else if (type === 'ping') state.pings++
else if (type === 'token') state.tokens++
else if (type === 'refresh') state.refreshes++
else if (type === 'refreshError') state.refreshErrors++
else if (type === 'protocolError') state.protocolErrors++
else if (type === 'socketError') state.socketErrors++
else if (type === 'spliceOpened') {
state.activeSplices++
state.peakActiveSplices = Math.max(state.peakActiveSplices, state.activeSplices)
} else if (type === 'spliceCompleted') {
state.completedSplices++
if (detail.readerMode === 'slow') state.slowReaderSplicesCompleted++
} else if (type === 'spliceWedged') {
state.wedgedReaderSplicesClosed++
const code = String(detail.code)
state.readerClosesByCode[code] = (state.readerClosesByCode[code] ?? 0) + 1
} else if (type === 'spliceClosed') state.activeSplices--
else if (type === 'spliceFailed') state.failedSplices++
else if (type === 'shutdown') state.peerShutdowns++
}
async function connect(peer) {
try {
await peer.connect()
} catch (error) {
state.connectionFailures++
if (state.steadyStarted) state.steadyConnectionFailures++
else if (state.transitionWindow) state.transitionConnectionFailures++
else state.rampConnectionFailures++
const reason = relayLoadFailureReason(error)
state.connectionFailuresByReason[reason] =
(state.connectionFailuresByReason[reason] ?? 0) + 1
scheduleReconnect(peer)
}
}
const peerOptions = (index, overrides = {}) => ({
...config,
...key,
accessToken,
...(accessTokenProviderForIndex
? { accessTokenProvider: accessTokenProviderForIndex(index) }
: {}),
seed: 0x4f524341 ^ config.shardIndex,
...overrides
})
if (config.regionBehaviorProbes > 0) {
const proofIndex = config.controls * config.shardCount + 10_000
const regionProof = await proveRelayLoadRegionBehavior({
oldClientPeer: new RelayLoadControlPeer(
proofIndex,
peerOptions(proofIndex, { preferredRegion: undefined }),
() => undefined
),
stickyPeer: new RelayLoadControlPeer(
proofIndex + 1,
peerOptions(proofIndex + 1, { preferredRegion: 'asia-east2' }),
() => undefined
),
asiaOrigin: config.capacityCellOrigin
})
state.oldClientUsFirstProved = regionProof.oldClientUsFirst ? 1 : 0
state.stickyAssignmentProved = regionProof.stickyAssignmentPreserved ? 1 : 0
}
const initialConnections = []
for (let localIndex = 0; localIndex < config.controls; localIndex++) {
const globalIndex = localIndex * config.shardCount + config.shardIndex
const peer = new RelayLoadControlPeer(globalIndex, peerOptions(globalIndex), observe)
peers.set(globalIndex, peer)
const rampOffset =
config.controls === 1 ? 0 : Math.floor((localIndex / (config.controls - 1)) * config.rampMs)
const offset = config.rampStartDelayMs + rampOffset
initialConnections.push(delay(offset).then(() => connect(peer)))
}
const progressTimer = setInterval(() => report(state), 10_000)
progressTimer.unref()
await runRelayLoadWithShutdown(async () => {
await Promise.all(initialConnections)
assertRelayLoadRampAccepted(state.rampConnectionFailures, config.maxRampConnectionFailures)
if (
config.rebindProbes > 0 || config.placementOverflowProbes > 0 ||
config.regionalFallbackProbes > 0
) {
state.transitionWindow = config.rebindDelayMs > 0
await waitForRelayLoadRebindGate({
delay,
delayMs: config.rebindDelayMs,
activeCount: () => state.active.size,
requiredCount: config.controls
})
state.transitionWindow = false
}
if (config.placementOverflowProbes > 0 || config.regionalFallbackProbes > 0) {
const closesBeforeBoundary = state.closes
await waitForRelayLoadDirectorCapacity({
directorOrigin: config.directorOrigin,
adminToken,
cellId: config.capacityCellId,
hardCap: config.capacityHardCap,
unobservedBound: config.capacityUnobservedBound,
requiredConnections: config.aggregateControls
})
if (state.active.size !== config.controls || state.closes !== closesBeforeBoundary) {
throw new Error('ordinary controls changed during the director capacity gate')
}
const overflowIndex = config.controls * config.shardCount + config.shardIndex
if (config.placementOverflowProbes > 0) {
state.placementOverflowReason = await proveRelayLoadPlacementBoundary({
peer: new RelayLoadControlPeer(overflowIndex, peerOptions(overflowIndex), observe),
failureReason: relayLoadFailureReason
})
}
if (config.regionalFallbackProbes > 0) {
await proveRelayLoadRegionalFallback({
peer: new RelayLoadControlPeer(overflowIndex, peerOptions(overflowIndex), () => undefined),
blockedOrigin: config.capacityCellOrigin
})
state.regionalFallbacksProved = 1
}
if (state.active.size !== config.controls || state.closes !== closesBeforeBoundary) {
throw new Error('ordinary controls changed during the placement boundary probe')
}
await waitForRelayLoadDirectorCapacity({
directorOrigin: config.directorOrigin,
adminToken,
cellId: config.capacityCellId,
hardCap: config.capacityHardCap,
unobservedBound: config.capacityUnobservedBound,
requiredConnections: config.aggregateControls,
requiredSamples: 1
})
if (state.active.size !== config.controls || state.closes !== closesBeforeBoundary) {
throw new Error('ordinary controls changed before post-probe capacity verification')
}
}
const rebindResult = await proveRelayLoadRebindBoundary({
peers: [...state.active].map((index) => peers.get(index)),
probeCount: config.rebindProbes,
holdMs: config.rebindHoldMs,
delay,
failureReason: relayLoadFailureReason,
requireOverflow: config.requireRebindOverflow
})
state.rebindProbesOpened = rebindResult.opened
state.rebindOverflowReason = rebindResult.overflowReason
if (config.requestUnitInvites > 0) {
state.requestUnitInvitesOpened = await openRelayLoadInviteOffers({
peers: [...state.active].sort((left, right) => left - right).map((index) => peers.get(index)),
count: config.requestUnitInvites,
ratePerSecond: config.requestUnitInvitesPerSecond
})
}
if (config.requestUnitOverflowProbes > 0) {
await waitForRelayLoadRequestUnits({
directorOrigin: config.directorOrigin,
adminToken,
cellId: config.capacityCellId,
capacityRequests: config.requestUnitCapacity,
expectedRequestUnits: config.requestUnitCapacity,
expectedActivityLeases: config.requestUnitCapacity
})
state.requestUnitOverflowReason = await proveRelayLoadRequestUnitBoundary(
peers.get([...state.active][0])
)
}
if (config.phaseBarrierDir) {
await waitForRelayLoadPhaseBarrier({
directory: config.phaseBarrierDir,
shardCount: config.shardCount,
shardIndex: config.shardIndex,
timeoutMs: config.phaseBarrierTimeoutMs
})
state.phaseBarrierPassed = true
}
state.steadyStarted = true
state.steadyMinimumActive = state.active.size
const spliceIndexes = relayLoadSpliceIndexes(config)
const readerOrigins = spliceIndexes.flatMap((index, spliceIndex) =>
relayLoadSpliceProfile(config, spliceIndex).readerMode === 'normal'
? []
: [peers.get(index).lastAssignment.cellUrl]
)
state.readerEvidence = await createRelayLoadReaderEvidence(readerOrigins, {
readQueuedBytes: readRuntimeQueuedBytes,
delay
})
if (config.phaseBarrierDir) {
await waitForRelayLoadPhaseBarrier({
directory: `${config.phaseBarrierDir}-splices`,
shardCount: config.shardCount,
shardIndex: config.shardIndex,
timeoutMs: config.phaseBarrierTimeoutMs
})
}
const splicePromises = spliceIndexes.map((index, spliceIndex) =>
delay(relayLoadSpliceStartDelayMs(config, spliceIndex)).then(() =>
peers.get(index).openSplice({
payloadBytes: config.splicePayloadBytes,
...relayLoadSpliceProfile(config, spliceIndex),
observeReaderPressure,
holdMs: config.spliceHoldMs
})
)
)
await Promise.all([...splicePromises, delay(config.durationMs)])
}, async () => {
state.stopping = true
clearInterval(progressTimer)
for (const timeout of reconnectTimers) clearTimeout(timeout)
reconnectTimers.clear()
await Promise.all([...peers.values()].map((peer) => peer.shutdown()))
eventLoopDelay.disable()
state.shutdownEvidence = {
peerShutdowns: state.peerShutdowns,
activeControls: state.active.size,
activeSplices: state.activeSplices,
reconnectTimers: reconnectTimers.size
}
})
if (config.requestUnitCleanupTimeoutMs > 0) {
await waitForRelayLoadRequestUnits({
directorOrigin: config.directorOrigin,
adminToken,
cellId: config.capacityCellId,
capacityRequests: config.requestUnitCapacity,
expectedRequestUnits: 0,
expectedActivityLeases: 0,
timeoutMs: config.requestUnitCleanupTimeoutMs
})
state.requestUnitCleanupProved = 1
}
const result = report(state, true)
const minimumPeak = config.allowPartial ? 1 : Math.ceil(config.controls * 0.95)
if (result.peakActive < minimumPeak) {
throw new Error(`peak active controls ${result.peakActive} below required ${minimumPeak}`)
}
if (result.steadyMinimumActive < minimumPeak) {
throw new Error(
`steady minimum active controls ${result.steadyMinimumActive} below required ${minimumPeak}`
)
}
if (relayLoadRunHasDisallowedFailures(result, config)) {
throw new Error('relay load run observed connection, protocol, refresh, or socket errors')
}
const readerEvidenceError = relayLoadReaderEvidenceError(result, config)
if (readerEvidenceError) throw new Error(readerEvidenceError)
if (
result.failedSplices > 0 ||
result.completedSplices + result.wedgedReaderSplicesClosed !== config.splices ||
result.shutdownEvidence.peerShutdowns !== config.controls ||
result.shutdownEvidence.activeControls !== 0 ||
result.shutdownEvidence.activeSplices !== 0 ||
result.shutdownEvidence.reconnectTimers !== 0
) {
throw new Error('relay load run did not complete splices or shut down cleanly')
}
@@ -0,0 +1,440 @@
import { createHash } from 'node:crypto'
import { fileURLToPath } from 'node:url'
import {
addExactMigrationCells,
applyExactAdmissionSelector,
inspectAdmissionSelector,
membershipWithStates,
selectorCellState
} from './relay-admission-selector.mjs'
const SHAPES = {
staging: {
directorOrigin: 'https://relay-staging.onorca.dev',
domain: 'relay-staging.onorca.dev',
allCells: ['staging-gce-c4']
},
production: {
directorOrigin: 'https://relay.onorca.dev',
domain: 'relay.onorca.dev',
allCells: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29']
}
}
function parseArguments(argv) {
const values = {}
for (let index = 0; index < argv.length; index += 2) {
const key = argv[index]
const value = argv[index + 1]
if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments')
values[key.slice(2)] = value
}
for (const key of ['environment', 'mode', 'cell-ids', 'image-digest']) {
if (!values[key]) throw new Error(`missing --${key}`)
}
if (!/^sha256:[a-f0-9]{64}$/.test(values['image-digest'])) {
throw new Error('--image-digest is invalid')
}
if (![
'inspect', 'initialize', 'verify', 'registered', 'register',
'promote', 'recover-promotion', 'rollback'
].includes(values.mode)) {
throw new Error('--mode is invalid')
}
const expectedGeneration = values.mode === 'inspect'
? undefined
: Number(values['expected-generation'])
if (
values.mode !== 'inspect' &&
(!Number.isSafeInteger(expectedGeneration) || expectedGeneration < 0)
) {
throw new Error('--expected-generation is invalid')
}
const shape = SHAPES[values.environment]
if (!shape) throw new Error('--environment is invalid')
const cells = values['cell-ids'].split(',').map((value) => value.trim()).filter(Boolean)
const distinct = new Set(cells)
if (distinct.size !== cells.length || cells.some((cell) => !shape.allCells.includes(cell))) {
throw new Error('--cell-ids are invalid')
}
const exact = (expected) => JSON.stringify([...cells].sort()) === JSON.stringify([...expected].sort())
if (
(['inspect', 'initialize', 'register', 'registered', 'verify'].includes(values.mode) &&
!exact(shape.allCells)) ||
(['promote', 'recover-promotion'].includes(values.mode) && values.environment === 'production' &&
!exact(['production-gce-c27']) && !exact(['production-gce-c28', 'production-gce-c29'])) ||
(['promote', 'recover-promotion'].includes(values.mode) && values.environment === 'staging' && !exact(shape.allCells)) ||
(values.mode === 'rollback' && cells.length === 0)
) throw new Error('--cell-ids do not match the reviewed admission wave')
const attemptId = values['attempt-id']
if (!['inspect', 'verify', 'registered'].includes(values.mode) &&
!/^[A-Za-z0-9_-]{8,128}$/.test(attemptId ?? '')) {
throw new Error('--attempt-id is invalid')
}
return {
environment: values.environment,
mode: values.mode,
cells,
expectedGeneration,
expectedMembershipSha256: values['expected-membership-sha256'],
imageDigest: values['image-digest'],
attemptId,
token: process.env.ORCA_RELAY_ADMIN_ID_TOKEN ?? ''
}
}
function hostname(cellId) {
return cellId.split('-').at(-1)
}
function cellOrigin(shape, cellId) {
return `https://${hostname(cellId)}.${shape.domain}`
}
async function responseJson(response, label) {
const body = await response.json().catch(() => ({}))
if (!response.ok) throw new Error(`${label} returned ${response.status}`)
return body
}
function defaultPost(fetchImpl, token) {
return async (url, body) => await responseJson(await fetchImpl(url, {
method: 'POST',
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
body: JSON.stringify(body),
signal: AbortSignal.timeout(30_000)
}), new URL(url).pathname)
}
async function verifyRuntime(fetchImpl, post, shape, cellId, imageDigest, requireDirector) {
const origin = cellOrigin(shape, cellId)
const [health, ready, runtime] = await Promise.all([
fetchImpl(`${origin}/health`, { redirect: 'error', signal: AbortSignal.timeout(8_000) }),
fetchImpl(`${origin}/ready`, { redirect: 'error', signal: AbortSignal.timeout(8_000) }),
post(`${origin}/v1/admin/runtime-status`, { v: 1 })
])
if (!health.ok || !ready.ok) throw new Error(`${cellId} is not ready`)
if (
runtime.cellId !== cellId ||
runtime.cellUrl !== origin ||
runtime.region !== 'asia-east2' ||
runtime.imageDigest !== imageDigest ||
runtime.draining !== false ||
runtime.connectionCapacity?.hardCap !== 3_000 ||
runtime.connectionCapacity?.unobservedBound !== 60
) throw new Error(`${cellId} runtime does not match the reviewed Asia shape`)
if (requireDirector) {
const result = await post(`${shape.directorOrigin}/v1/admin/cell-status`, { v: 1, cellId })
if (
result.status?.cellUrl !== origin ||
result.status?.runtime?.heartbeatFresh !== true ||
result.status?.runtime?.ready !== true
) throw new Error(`${cellId} has no fresh ready director heartbeat`)
}
}
function membershipStates(selector, cells) {
return Object.fromEntries(cells.map((cellId) => [cellId, selectorCellState(selector, cellId)]))
}
function inspectedMembershipStates(selector, cells) {
const known = new Set([
...selector.membership.existingOnly,
...selector.membership.migrationOnly,
...selector.membership.general
])
return Object.fromEntries(cells.map((cellId) => [
cellId,
known.has(cellId) ? selectorCellState(selector, cellId) : 'absent'
]))
}
function sameMembership(left, right) {
return JSON.stringify(left) === JSON.stringify(right)
}
function membershipSha256(membership) {
return createHash('sha256').update(JSON.stringify(membership)).digest('hex')
}
async function initializeAdmissionBoundary(post, selectorPost, shape, config, current) {
if (config.expectedGeneration !== 0) {
throw new Error('admission boundary initialization requires generation 0')
}
if (
!/^[a-f0-9]{64}$/.test(config.expectedMembershipSha256 ?? '') ||
membershipSha256(current.selector.membership) !== config.expectedMembershipSha256
) {
throw new Error('admission membership changed before boundary initialization')
}
const targetStates = inspectedMembershipStates(current.selector, config.cells)
if (Object.values(targetStates).some((state) => state !== 'absent')) {
throw new Error('Asia cell exists before admission boundary initialization')
}
const intendedMembership = current.intent?.previousMembership ?? current.selector.membership
const exactCommitted = (inspection) =>
inspection.intent?.state === 'committed' &&
inspection.intent.expectedGeneration === 0 &&
inspection.selector.generation === 1 &&
inspection.selector.attemptId === config.attemptId &&
sameMembership(inspection.intent.previousMembership, intendedMembership) &&
sameMembership(inspection.intent.membership, intendedMembership) &&
sameMembership(inspection.selector.membership, intendedMembership)
const exactUnchanged = (inspection) =>
inspection.intent?.state === 'unchanged' &&
inspection.intent.expectedGeneration === 0 &&
inspection.selector.generation === 0 &&
sameMembership(inspection.intent.previousMembership, intendedMembership) &&
sameMembership(inspection.intent.membership, intendedMembership) &&
sameMembership(inspection.selector.membership, intendedMembership)
if (exactCommitted(current)) {
return {
mode: config.mode,
generation: current.selector.generation,
states: inspectedMembershipStates(current.selector, config.cells),
recovered: true
}
}
if (current.intent && !exactUnchanged(current)) {
throw new Error('admission boundary initialization attempt diverged')
}
const request = {
v: 1,
attemptId: config.attemptId,
expectedGeneration: 0,
expectedMembershipSha256: config.expectedMembershipSha256,
membership: intendedMembership
}
let applyError
for (let attempt = 0; attempt < 2; attempt++) {
try {
await post(`${shape.directorOrigin}/v1/admin/admission-selector/apply`, request)
} catch (error) {
applyError = error
}
const verified = await inspectAdmissionSelector(selectorPost, config.attemptId)
if (exactCommitted(verified)) {
return {
mode: config.mode,
generation: verified.selector.generation,
states: targetStates,
recovered: current.intent !== null || applyError !== undefined || attempt > 0
}
}
if (!exactUnchanged(verified)) {
throw new Error('admission boundary initialization did not commit exactly', {
cause: applyError
})
}
}
throw new Error('admission boundary initialization remained unchanged after retry', {
cause: applyError
})
}
export async function operateRelayAsiaAdmission(config, dependencies = {}) {
const shape = SHAPES[config.environment]
const fetchImpl = dependencies.fetch ?? fetch
const post = dependencies.post ?? defaultPost(fetchImpl, config.token)
const selectorPost = (path, body) => {
if (config.environment !== 'staging' || path !== '/v1/admin/admission-selector/apply') {
return post(`${shape.directorOrigin}${path}`, body)
}
const state = selectorCellState({ membership: body.membership }, 'staging-gce-c4')
if (!['general', 'migration-only'].includes(state)) {
throw new Error('staging proof can only transition C4 between reviewed states')
}
return post(`${shape.directorOrigin}/v1/admin/admission-selector/apply-staging-asia-proof`, {
v: 1,
attemptId: body.attemptId,
expectedGeneration: body.expectedGeneration,
state
})
}
const current = await inspectAdmissionSelector(
selectorPost,
['inspect', 'verify', 'registered'].includes(config.mode) ? undefined : config.attemptId
)
if (config.mode === 'inspect') {
return {
mode: config.mode,
generation: current.selector.generation,
membership: current.selector.membership,
membershipSha256: membershipSha256(current.selector.membership),
states: inspectedMembershipStates(current.selector, config.cells)
}
}
if (
!current.intent &&
current.selector.generation !== config.expectedGeneration
) {
throw new Error('admission selector generation changed')
}
if (current.intent && current.intent.expectedGeneration !== config.expectedGeneration) {
throw new Error('admission attempt generation does not match')
}
if (config.mode === 'initialize') {
return await initializeAdmissionBoundary(post, selectorPost, shape, config, current)
}
if (config.mode === 'recover-promotion') {
if (config.cells.every(
(cellId) => selectorCellState(current.selector, cellId) === 'migration-only'
)) {
return {
mode: config.mode,
promoted: false,
generation: current.selector.generation,
states: membershipStates(current.selector, config.cells)
}
}
if (!current.intent) {
if (
current.selector.generation !== config.expectedGeneration
) throw new Error('promotion state changed without the reviewed attempt')
return {
mode: config.mode,
promoted: false,
generation: current.selector.generation,
states: membershipStates(current.selector, config.cells)
}
}
const expectedMembership = membershipWithStates(
{ membership: current.intent.previousMembership },
Object.fromEntries(config.cells.map((cellId) => [cellId, 'general']))
)
if (
current.intent.state !== 'committed' ||
JSON.stringify(current.intent.membership) !== JSON.stringify(expectedMembership) ||
config.cells.some((cellId) => selectorCellState(current.selector, cellId) !== 'general')
) throw new Error('promotion attempt is not the current general state')
return {
mode: config.mode,
promoted: true,
generation: current.selector.generation,
states: membershipStates(current.selector, config.cells)
}
}
if (config.mode === 'rollback') {
for (const cellId of config.cells) {
if (!['general', 'migration-only'].includes(selectorCellState(current.selector, cellId))) {
throw new Error(`${cellId} cannot roll back to migration-only`)
}
}
} else if (config.mode === 'register') {
const known = new Set([
...current.selector.membership.existingOnly,
...current.selector.membership.migrationOnly,
...current.selector.membership.general
])
if (!current.intent && config.cells.some((cellId) => known.has(cellId))) {
throw new Error('Asia cell is already registered')
}
await Promise.all(config.cells.map((cellId) =>
verifyRuntime(fetchImpl, post, shape, cellId, config.imageDigest, false)
))
} else if (config.mode !== 'registered') {
await Promise.all(config.cells.map((cellId) =>
verifyRuntime(fetchImpl, post, shape, cellId, config.imageDigest, true)
))
}
if (config.mode === 'registered') {
if (config.cells.some(
(cellId) => selectorCellState(current.selector, cellId) !== 'migration-only'
)) throw new Error('Asia cells are not registered migration-only')
await Promise.all(config.cells.map((cellId) =>
verifyRuntime(fetchImpl, post, shape, cellId, config.imageDigest, false)
))
}
if (['verify', 'registered'].includes(config.mode)) {
return {
mode: config.mode,
generation: current.selector.generation,
states: membershipStates(current.selector, config.cells)
}
}
if (config.mode === 'register') {
if (current.intent) {
const expectedCells = new Set(config.cells)
const addedCells = current.intent.membership.migrationOnly.filter(
(cellId) => !current.intent.previousMembership.migrationOnly.includes(cellId)
)
if (
current.intent.state !== 'committed' ||
addedCells.length !== expectedCells.size ||
addedCells.some((cellId) => !expectedCells.has(cellId)) ||
current.selector.generation !== config.expectedGeneration + 1 ||
JSON.stringify(current.selector.membership) !== JSON.stringify(current.intent.membership)
) {
throw new Error('admission attempt does not match the requested Asia registration')
}
return {
mode: config.mode,
generation: current.selector.generation,
states: membershipStates(current.selector, config.cells),
recovered: true
}
}
const result = await addExactMigrationCells(
selectorPost,
{
attemptId: config.attemptId,
cells: config.cells.map((cellId) => ({
cellId,
cellUrl: cellOrigin(shape, cellId),
region: 'asia-east2',
capacityRequests: 6_000,
connectionHardCap: 3_000,
connectionUnobservedBound: 60
}))
},
{ expectedCurrentSelector: current.selector }
)
return { mode: config.mode, generation: result.selector.generation, states: membershipStates(result.selector, config.cells) }
}
const desiredState = config.mode === 'promote' ? 'general' : 'migration-only'
if (current.intent) {
const expectedMembership = membershipWithStates(
{ membership: current.intent.previousMembership },
Object.fromEntries(config.cells.map((cellId) => [cellId, desiredState]))
)
if (
current.intent.state !== 'committed' ||
JSON.stringify(current.intent.membership) !== JSON.stringify(expectedMembership) ||
current.selector.generation !== config.expectedGeneration + 1 ||
JSON.stringify(current.selector.membership) !== JSON.stringify(current.intent.membership)
) {
throw new Error('admission attempt does not match the requested Asia transition')
}
return {
mode: config.mode,
generation: current.selector.generation,
states: membershipStates(current.selector, config.cells),
recovered: true
}
}
if (config.mode === 'promote' && config.cells.some(
(cellId) => selectorCellState(current.selector, cellId) !== 'migration-only'
)) throw new Error('Asia promotion requires migration-only cells')
if (
config.mode === 'promote' &&
config.environment === 'production' &&
config.cells.includes('production-gce-c28') &&
selectorCellState(current.selector, 'production-gce-c27') !== 'general'
) {
throw new Error('Asia expansion requires the C27 canary to be general')
}
const result = await applyExactAdmissionSelector(
selectorPost,
membershipWithStates(current.selector, Object.fromEntries(
config.cells.map((cellId) => [cellId, desiredState])
)),
{ attemptId: config.attemptId, expectedCurrentSelector: current.selector }
)
return { mode: config.mode, generation: result.selector.generation, states: membershipStates(result.selector, config.cells) }
}
if (process.argv[1] === fileURLToPath(import.meta.url)) {
const config = parseArguments(process.argv.slice(2))
if (!config.token) throw new Error('ORCA_RELAY_ADMIN_ID_TOKEN is required')
console.log(JSON.stringify(await operateRelayAsiaAdmission(config)))
}
@@ -0,0 +1,512 @@
import assert from 'node:assert/strict'
import { createHash } from 'node:crypto'
import { test } from 'node:test'
import { operateRelayAsiaAdmission } from './operate-relay-asia-admission.mjs'
const digest = `sha256:${'a'.repeat(64)}`
const membershipDigest = (membership) =>
createHash('sha256').update(JSON.stringify(membership)).digest('hex')
function harness(initialSelector) {
const initialMembership = structuredClone(initialSelector.membership)
let selector = structuredClone(initialSelector)
const intents = new Map()
const requests = []
let fetches = 0
let failAfterIntent = false
const post = async (url, body) => {
const parsed = new URL(url)
requests.push({ path: parsed.pathname, body })
if (parsed.pathname === '/v1/admin/runtime-status') {
const cell = parsed.hostname.split('.')[0]
return {
cellId: `production-gce-${cell}`,
cellUrl: parsed.origin,
region: 'asia-east2',
imageDigest: digest,
draining: false,
connectionCapacity: { hardCap: 3_000, unobservedBound: 60 }
}
}
if (parsed.pathname === '/v1/admin/cell-status') {
return {
status: {
cellUrl: `https://${body.cellId.split('-').at(-1)}.relay.onorca.dev`,
runtime: { heartbeatFresh: true, ready: true }
}
}
}
if (parsed.pathname.endsWith('/status')) {
return { selector, intent: body.attemptId ? intents.get(body.attemptId) ?? null : null }
}
if (parsed.pathname.endsWith('/add-migration-cells')) {
selector = {
generation: selector.generation + 1,
attemptId: body.attemptId,
membership: {
...selector.membership,
migrationOnly: [...selector.membership.migrationOnly, ...body.cells.map((cell) => cell.cellId)].sort()
}
}
} else if (parsed.pathname.endsWith('/apply-staging-asia-proof')) {
const membership = structuredClone(selector.membership)
membership.migrationOnly = membership.migrationOnly.filter((cell) => cell !== 'staging-gce-c4')
membership.general = membership.general.filter((cell) => cell !== 'staging-gce-c4')
membership[body.state === 'general' ? 'general' : 'migrationOnly'].push('staging-gce-c4')
selector = { generation: selector.generation + 1, attemptId: body.attemptId, membership }
} else if (parsed.pathname.endsWith('/apply')) {
if (
body.expectedMembershipSha256 &&
body.expectedMembershipSha256 !== membershipDigest(selector.membership)
) throw new Error('admission_selector_membership_mismatch')
if (failAfterIntent) {
failAfterIntent = false
intents.set(body.attemptId, {
state: 'unchanged',
expectedGeneration: body.expectedGeneration,
previousMembership: structuredClone(selector.membership),
membership: structuredClone(body.membership)
})
throw new Error('failure after intent persistence')
}
selector = { generation: selector.generation + 1, attemptId: body.attemptId, membership: body.membership }
} else throw new Error(`unexpected ${parsed.pathname}`)
intents.set(body.attemptId, {
state: 'committed', expectedGeneration: body.expectedGeneration,
previousMembership: initialSelector.membership,
membership: selector.membership
})
return { changed: true, selector }
}
const fetch = async () => {
fetches++
return new Response(null, { status: 200 })
}
const commitWithoutResponse = async (path, body) => {
await post(`https://relay.onorca.dev${path}`, body)
throw new Error('response lost after commit')
}
return {
post, fetch, requests, commitWithoutResponse,
failNextApplyAfterIntent: () => (failAfterIntent = true),
apply: async (attemptId, membership) => await post(
'https://relay.onorca.dev/v1/admin/admission-selector/apply',
{ attemptId, expectedGeneration: selector.generation, membership }
),
fetchCount: () => fetches, selector: () => selector
}
}
const baseSelector = {
generation: 7,
membership: { existingOnly: [], migrationOnly: [], general: ['production-gce-c26'] }
}
test('inspects generation zero without requiring target registration or making a mutation', async () => {
const subject = harness({
generation: 0,
membership: {
existingOnly: ['staging-gce-c3'],
migrationOnly: [],
general: ['staging-gce-c1', 'staging-gce-c2']
}
})
const result = await operateRelayAsiaAdmission({
environment: 'staging', mode: 'inspect', cells: ['staging-gce-c4'],
imageDigest: digest, token: 'not-logged'
}, subject)
assert.equal(result.generation, 0)
assert.equal(result.states['staging-gce-c4'], 'absent')
assert.deepEqual(result.membership, subject.selector().membership)
assert.equal(result.membershipSha256, membershipDigest(subject.selector().membership))
assert.deepEqual(subject.requests.map(({ path }) => path), [
'/v1/admin/admission-selector/status'
])
})
test('initializes generation zero without changing membership', async () => {
const membership = {
existingOnly: ['staging-gce-c3'],
migrationOnly: [],
general: ['staging-gce-c1', 'staging-gce-c2']
}
const subject = harness({ generation: 0, membership })
const result = await operateRelayAsiaAdmission({
environment: 'staging', mode: 'initialize', cells: ['staging-gce-c4'],
expectedGeneration: 0, imageDigest: digest,
expectedMembershipSha256: membershipDigest(membership),
attemptId: 'asia_boundary_0', token: 'not-logged'
}, subject)
const request = subject.requests.find(({ path }) => path.endsWith('/apply'))
assert.deepEqual(request.body, {
v: 1,
attemptId: 'asia_boundary_0',
expectedGeneration: 0,
expectedMembershipSha256: membershipDigest(membership),
membership
})
assert.equal(result.generation, 1)
assert.equal(result.states['staging-gce-c4'], 'absent')
assert.deepEqual(subject.selector().membership, membership)
})
test('retries the same fingerprint-bound initialization after intent persistence', async () => {
const membership = {
existingOnly: ['staging-gce-c3'],
migrationOnly: [],
general: ['staging-gce-c1', 'staging-gce-c2']
}
const subject = harness({ generation: 0, membership })
subject.failNextApplyAfterIntent()
const result = await operateRelayAsiaAdmission({
environment: 'staging', mode: 'initialize', cells: ['staging-gce-c4'],
expectedGeneration: 0, imageDigest: digest,
expectedMembershipSha256: membershipDigest(membership),
attemptId: 'asia_boundary_intent_retry', token: 'not-logged'
}, subject)
const applies = subject.requests.filter(({ path }) => path.endsWith('/apply'))
assert.equal(applies.length, 2)
assert.deepEqual(applies[1].body, applies[0].body)
assert.equal(result.recovered, true)
assert.equal(result.generation, 1)
assert.deepEqual(subject.selector().membership, membership)
})
test('recovers a committed generation-zero initialization', async () => {
const membership = {
existingOnly: ['staging-gce-c3'],
migrationOnly: [],
general: ['staging-gce-c1', 'staging-gce-c2']
}
const subject = harness({ generation: 0, membership })
const config = {
environment: 'staging', mode: 'initialize', cells: ['staging-gce-c4'],
expectedGeneration: 0, imageDigest: digest,
expectedMembershipSha256: membershipDigest(membership),
attemptId: 'asia_boundary_retry', token: 'not-logged'
}
await assert.rejects(subject.commitWithoutResponse(
'/v1/admin/admission-selector/apply',
{
v: 1,
attemptId: config.attemptId,
expectedGeneration: 0,
expectedMembershipSha256: membershipDigest(membership),
membership
}
), /response lost after commit/)
const recovered = await operateRelayAsiaAdmission(config, subject)
assert.equal(recovered.recovered, true)
assert.equal(recovered.generation, 1)
assert.deepEqual(subject.selector().membership, membership)
})
test('rejects generation-zero membership drift after inspect', async () => {
const inspected = {
existingOnly: ['staging-gce-c3'],
migrationOnly: [],
general: ['staging-gce-c1', 'staging-gce-c2']
}
const changed = {
existingOnly: ['staging-gce-c2', 'staging-gce-c3'],
migrationOnly: [],
general: ['staging-gce-c1']
}
const subject = harness({ generation: 0, membership: changed })
await assert.rejects(operateRelayAsiaAdmission({
environment: 'staging', mode: 'initialize', cells: ['staging-gce-c4'],
expectedGeneration: 0, imageDigest: digest,
expectedMembershipSha256: membershipDigest(inspected),
attemptId: 'asia_boundary_drift', token: 'not-logged'
}, subject), /membership changed/)
assert.equal(subject.requests.some(({ path }) => path.endsWith('/apply')), false)
})
test('registers all three Asia cells atomically with region and exact limits', async () => {
const subject = harness(baseSelector)
const result = await operateRelayAsiaAdmission({
environment: 'production', mode: 'register',
cells: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'],
expectedGeneration: 7, imageDigest: digest, attemptId: 'asia_register_7', token: 'not-logged'
}, subject)
const request = subject.requests.find(({ path }) => path.endsWith('/add-migration-cells'))
assert.equal(request.body.cells.length, 3)
assert.ok(request.body.cells.every((cell) =>
cell.region === 'asia-east2' && cell.capacityRequests === 6_000 &&
cell.connectionHardCap === 3_000 && cell.connectionUnobservedBound === 60
))
assert.equal(result.generation, 8)
assert.deepEqual(new Set(Object.values(result.states)), new Set(['migration-only']))
})
test('promotes the canary only after runtime and director-heartbeat checks', async () => {
const subject = harness({
generation: 8,
membership: { existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26'] }
})
const result = await operateRelayAsiaAdmission({
environment: 'production', mode: 'promote', cells: ['production-gce-c27'],
expectedGeneration: 8, imageDigest: digest, attemptId: 'asia_promote_8', token: 'not-logged'
}, subject)
assert.equal(subject.fetchCount(), 2)
assert.equal(result.states['production-gce-c27'], 'general')
})
test('checks registered migration-only cells before director configuration without requiring heartbeat', async () => {
const subject = harness({
generation: 8,
membership: {
existingOnly: [],
migrationOnly: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'],
general: ['production-gce-c26']
}
})
const result = await operateRelayAsiaAdmission({
environment: 'production', mode: 'registered',
cells: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'],
expectedGeneration: 8, imageDigest: digest, token: 'not-logged'
}, subject)
assert.equal(subject.fetchCount(), 6)
assert.equal(subject.requests.filter(({ path }) => path === '/v1/admin/cell-status').length, 0)
assert.deepEqual(new Set(Object.values(result.states)), new Set(['migration-only']))
})
test('rolls back admission without requiring an unhealthy runtime to answer', async () => {
const subject = harness({
generation: 9,
membership: { existingOnly: [], migrationOnly: [], general: ['production-gce-c26', 'production-gce-c27'] }
})
const result = await operateRelayAsiaAdmission({
environment: 'production', mode: 'rollback', cells: ['production-gce-c27'],
expectedGeneration: 9, imageDigest: digest, attemptId: 'asia_rollback_9', token: 'not-logged'
}, subject)
assert.equal(subject.fetchCount(), 0)
assert.equal(result.states['production-gce-c27'], 'migration-only')
})
test('uses the server-enforced C4-only route for staging proof transitions', async () => {
const subject = harness({
generation: 3,
membership: {
existingOnly: ['staging-gce-c1'],
migrationOnly: [],
general: ['staging-gce-c2', 'staging-gce-c4']
}
})
const result = await operateRelayAsiaAdmission({
environment: 'staging', mode: 'rollback', cells: ['staging-gce-c4'],
expectedGeneration: 3, imageDigest: digest, attemptId: 'asia_staging_rollback',
token: 'not-logged'
}, subject)
const request = subject.requests.find(
({ path }) => path.endsWith('/apply-staging-asia-proof')
)
assert.deepEqual(request.body, {
v: 1,
attemptId: 'asia_staging_rollback',
expectedGeneration: 3,
state: 'migration-only'
})
assert.deepEqual(subject.selector().membership, {
existingOnly: ['staging-gce-c1'],
migrationOnly: ['staging-gce-c4'],
general: ['staging-gce-c2']
})
assert.equal(result.states['staging-gce-c4'], 'migration-only')
})
test('fails closed when the exact selector generation moved', async () => {
const subject = harness(baseSelector)
await assert.rejects(operateRelayAsiaAdmission({
environment: 'production', mode: 'register',
cells: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'],
expectedGeneration: 6, imageDigest: digest, attemptId: 'asia_register_6', token: 'not-logged'
}, subject), /generation changed/)
assert.equal(subject.fetchCount(), 0)
})
test('recovers a committed registration when the workflow retries the original generation', async () => {
const subject = harness(baseSelector)
const config = {
environment: 'production', mode: 'register',
cells: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'],
expectedGeneration: 7, imageDigest: digest, attemptId: 'asia_register_retry',
token: 'not-logged'
}
await assert.rejects(subject.commitWithoutResponse(
'/v1/admin/admission-selector/add-migration-cells',
{
v: 1,
attemptId: config.attemptId,
expectedGeneration: config.expectedGeneration,
cells: config.cells.map((cellId) => ({ cellId }))
}
), /response lost after commit/)
const recovered = await operateRelayAsiaAdmission(config, subject)
assert.equal(recovered.recovered, true)
assert.equal(recovered.generation, 8)
})
test('recovers a committed promotion when the workflow retries the original generation', async () => {
const subject = harness({
generation: 8,
membership: {
existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26']
}
})
const config = {
environment: 'production', mode: 'promote', cells: ['production-gce-c27'],
expectedGeneration: 8, imageDigest: digest, attemptId: 'asia_promote_retry',
token: 'not-logged'
}
await assert.rejects(subject.commitWithoutResponse(
'/v1/admin/admission-selector/apply',
{
v: 1,
attemptId: config.attemptId,
expectedGeneration: config.expectedGeneration,
membership: {
existingOnly: [], migrationOnly: [],
general: ['production-gce-c26', 'production-gce-c27']
}
}
), /response lost after commit/)
const recovered = await operateRelayAsiaAdmission(config, subject)
assert.equal(recovered.recovered, true)
assert.equal(recovered.generation, 9)
assert.equal(recovered.states['production-gce-c27'], 'general')
})
test('inspects an ambiguous promotion without creating a new transition', async () => {
const untouched = harness({
generation: 8,
membership: {
existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26']
}
})
const config = {
environment: 'production', mode: 'recover-promotion', cells: ['production-gce-c27'],
expectedGeneration: 8, imageDigest: digest, attemptId: 'asia_recover_promote',
token: 'not-logged'
}
const absent = await operateRelayAsiaAdmission(config, untouched)
assert.equal(absent.promoted, false)
assert.equal(untouched.requests.some(({ path }) => path.endsWith('/apply')), false)
const committed = harness({
generation: 8,
membership: {
existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26']
}
})
await assert.rejects(committed.commitWithoutResponse('/v1/admin/admission-selector/apply', {
v: 1,
attemptId: config.attemptId,
expectedGeneration: 8,
membership: {
existingOnly: [], migrationOnly: [], general: ['production-gce-c26', 'production-gce-c27']
}
}), /response lost after commit/)
const recovered = await operateRelayAsiaAdmission(config, committed)
assert.equal(recovered.promoted, true)
assert.equal(recovered.generation, 9)
})
test('treats an already rolled-back promotion as recovered', async () => {
const subject = harness({
generation: 8,
membership: {
existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26']
}
})
const config = {
environment: 'production', mode: 'recover-promotion', cells: ['production-gce-c27'],
expectedGeneration: 8, imageDigest: digest, attemptId: 'asia_recover_after_rollback',
token: 'not-logged'
}
await assert.rejects(subject.commitWithoutResponse('/v1/admin/admission-selector/apply', {
v: 1, attemptId: config.attemptId, expectedGeneration: 8,
membership: {
existingOnly: [], migrationOnly: [], general: ['production-gce-c26', 'production-gce-c27']
}
}), /response lost after commit/)
await subject.apply('later_rollback', {
existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26']
})
const recovered = await operateRelayAsiaAdmission(config, subject)
assert.equal(recovered.promoted, false)
assert.equal(recovered.generation, 10)
})
test('recovers a committed rollback when the workflow retries the original generation', async () => {
const subject = harness({
generation: 9,
membership: {
existingOnly: [], migrationOnly: [], general: ['production-gce-c26', 'production-gce-c27']
}
})
const config = {
environment: 'production', mode: 'rollback', cells: ['production-gce-c27'],
expectedGeneration: 9, imageDigest: digest, attemptId: 'asia_rollback_retry',
token: 'not-logged'
}
await assert.rejects(subject.commitWithoutResponse(
'/v1/admin/admission-selector/apply',
{
v: 1,
attemptId: config.attemptId,
expectedGeneration: config.expectedGeneration,
membership: {
existingOnly: [], migrationOnly: ['production-gce-c27'],
general: ['production-gce-c26']
}
}
), /response lost after commit/)
const recovered = await operateRelayAsiaAdmission(config, subject)
assert.equal(recovered.recovered, true)
assert.equal(recovered.generation, 10)
assert.equal(recovered.states['production-gce-c27'], 'migration-only')
})
test('rejects a committed transition retry after a later selector change', async () => {
const subject = harness({
generation: 8,
membership: {
existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26']
}
})
const config = {
environment: 'production', mode: 'promote', cells: ['production-gce-c27'],
expectedGeneration: 8, imageDigest: digest, attemptId: 'asia_stale_promote',
token: 'not-logged'
}
await assert.rejects(subject.commitWithoutResponse('/v1/admin/admission-selector/apply', {
v: 1, attemptId: config.attemptId, expectedGeneration: 8,
membership: {
existingOnly: [], migrationOnly: [], general: ['production-gce-c26', 'production-gce-c27']
}
}), /response lost after commit/)
await subject.apply('later_rollback', {
existingOnly: [], migrationOnly: ['production-gce-c27'], general: ['production-gce-c26']
})
await assert.rejects(
operateRelayAsiaAdmission(config, subject),
/does not match the requested Asia transition/
)
})
test('requires the C27 canary before promoting C28 and C29', async () => {
const subject = harness({
generation: 8,
membership: {
existingOnly: [],
migrationOnly: ['production-gce-c27', 'production-gce-c28', 'production-gce-c29'],
general: ['production-gce-c26']
}
})
await assert.rejects(operateRelayAsiaAdmission({
environment: 'production', mode: 'promote',
cells: ['production-gce-c28', 'production-gce-c29'], expectedGeneration: 8,
imageDigest: digest, attemptId: 'asia_wave_before_canary', token: 'not-logged'
}, subject), /C27 canary/)
})
@@ -0,0 +1,312 @@
import { pathToFileURL } from 'node:url'
import { inspectAdmissionSelector } from './relay-admission-selector.mjs'
const DIRECTOR_ORIGIN = 'https://relay.onorca.dev'
const MODES = new Set(['inspect', 'enable', 'pause', 'disable', 'recover-enable'])
function canonicalCells(value) {
if (value === 'none') return []
const cells = value.split(',').map((cell) => cell.trim()).filter(Boolean).sort()
if (
cells.length === 0 ||
new Set(cells).size !== cells.length ||
cells.some((cell) => !/^production-gce-c(?:[1-9]|[12][0-9])$/.test(cell))
) throw new Error('selector membership is invalid')
return cells
}
function integer(value, name, { minimum = 0, maximum = Number.MAX_SAFE_INTEGER } = {}) {
const parsed = Number(value)
if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) {
throw new Error(`${name} is invalid`)
}
return parsed
}
export function parseRegionalRehomeArguments(argv, environment = process.env) {
const values = {}
for (let index = 0; index < argv.length; index += 2) {
const key = argv[index]
const value = argv[index + 1]
if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments')
values[key.slice(2)] = value
}
for (const key of ['mode', 'director-origin', 'expected-control-generation']) {
if (!values[key]) throw new Error(`missing --${key}`)
}
if (!MODES.has(values.mode)) throw new Error('--mode is invalid')
if (values['director-origin'] !== DIRECTOR_ORIGIN) {
throw new Error('--director-origin must be the production Relay origin')
}
const recovery = values.mode === 'recover-enable'
const mutation = values.mode !== 'inspect' && !recovery
const mutationKeys = [
'not-before',
'rate-per-minute',
'preference-max-age-ms',
'drain-grace-ms',
'confirmation'
]
if (mutation && mutationKeys.some((key) => values[key] === undefined)) {
throw new Error('mutations require the complete durable control shape')
}
if (!mutation && !recovery && mutationKeys.some((key) => values[key] !== undefined)) {
throw new Error('inspect cannot carry mutation arguments')
}
const selectorKeys = [
'expected-selector-generation',
'expected-existing-only-cells',
'expected-migration-only-cells',
'expected-general-cells'
]
if (!recovery && selectorKeys.some((key) => values[key] === undefined)) {
throw new Error('operation requires exact selector state')
}
if (recovery && selectorKeys.some((key) => values[key] !== undefined)) {
throw new Error('enable recovery cannot depend on selector diagnostics')
}
const expectedConfirmation = {
enable: 'ENABLE_REGIONAL_REHOMING',
pause: 'PAUSE_REGIONAL_REHOMING',
disable: 'DISABLE_REGIONAL_REHOMING'
}[values.mode]
if (mutation && values.confirmation !== expectedConfirmation) {
throw new Error('confirmation does not match the requested control action')
}
if (recovery && values.confirmation !== 'RECOVER_FAILED_REGIONAL_REHOME_ENABLE') {
throw new Error('confirmation does not authorize failed-enable recovery')
}
if (
recovery &&
mutationKeys
.filter((key) => key !== 'confirmation')
.some((key) => values[key] !== undefined)
) throw new Error('enable recovery cannot carry durable control shape arguments')
const token = environment.ORCA_RELAY_ADMIN_ID_TOKEN
if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable')
return {
mode: values.mode,
directorOrigin: DIRECTOR_ORIGIN,
...(!recovery
? {
expectedSelectorGeneration: integer(
values['expected-selector-generation'],
'--expected-selector-generation'
),
expectedMembership: {
existingOnly: canonicalCells(values['expected-existing-only-cells']),
migrationOnly: canonicalCells(values['expected-migration-only-cells']),
general: canonicalCells(values['expected-general-cells'])
}
}
: {}),
expectedControlGeneration: integer(
values['expected-control-generation'],
'--expected-control-generation'
),
...(mutation
? {
notBefore: integer(values['not-before'], '--not-before'),
ratePerMinute: integer(values['rate-per-minute'], '--rate-per-minute', {
minimum: 1,
maximum: 120
}),
preferenceMaxAgeMs: integer(
values['preference-max-age-ms'],
'--preference-max-age-ms',
{ minimum: 60_000, maximum: 30 * 24 * 60 * 60_000 }
),
drainGraceMs: integer(values['drain-grace-ms'], '--drain-grace-ms', {
minimum: 60_000,
maximum: 60 * 60_000
})
}
: {}),
token
}
}
async function responseJson(response, label) {
const body = await response.json().catch(() => ({}))
if (!response.ok) throw new Error(`${label} returned ${response.status}: ${body.error ?? 'unknown'}`)
return body
}
function exactMembership(actual, expected) {
return ['existingOnly', 'migrationOnly', 'general'].every(
(key) => JSON.stringify(actual[key]) === JSON.stringify(expected[key])
)
}
function assertControl(control, expected) {
if (
(expected.generation !== undefined && control?.generation !== expected.generation) ||
typeof control.enabled !== 'boolean' ||
!Number.isSafeInteger(control.observationStartedAt) ||
!Number.isSafeInteger(control.notBefore) ||
!Number.isSafeInteger(control.ratePerMinute) ||
!Number.isSafeInteger(control.preferenceMaxAgeMs) ||
!Number.isSafeInteger(control.drainGraceMs)
) throw new Error('director returned an invalid regional rehome control')
if (expected.enabled !== undefined && control.enabled !== expected.enabled) {
throw new Error('regional rehome enabled state does not match')
}
return control
}
async function verifiedDisabledControl(post, generation) {
return assertControl((await post('/v1/admin/regional-rehome-control', {
v: 1,
action: 'inspect'
})).control, { generation, enabled: false })
}
async function applyDisabledControl(post, before) {
return assertControl((await post('/v1/admin/regional-rehome-control', {
v: 1,
action: 'apply',
expectedGeneration: before.generation,
enabled: false,
notBefore: before.notBefore,
ratePerMinute: before.ratePerMinute,
preferenceMaxAgeMs: before.preferenceMaxAgeMs,
drainGraceMs: before.drainGraceMs,
confirmation: 'DISABLE_REGIONAL_REHOMING'
})).control, { generation: before.generation + 1, enabled: false })
}
async function resolveAmbiguousDisable(post, before, firstError) {
const observed = assertControl((await post('/v1/admin/regional-rehome-control', {
v: 1,
action: 'inspect'
})).control, {})
if (observed.generation === before.generation + 1 && !observed.enabled) {
return observed
}
if (observed.generation !== before.generation || !observed.enabled) {
throw new AggregateError(
[firstError],
'failed-enable recovery reached an unexpected control generation'
)
}
try {
return await applyDisabledControl(post, before)
} catch (retryError) {
try {
return await verifiedDisabledControl(post, before.generation + 1)
} catch (readbackError) {
throw new AggregateError(
[firstError, retryError, readbackError],
'failed-enable recovery exhausted two bounded CAS attempts'
)
}
}
}
export async function recoverRegionalRehomeEnable(config, post) {
const before = assertControl((await post('/v1/admin/regional-rehome-control', {
v: 1,
action: 'inspect'
})).control, {})
if (
before.generation < config.expectedControlGeneration ||
(before.generation === config.expectedControlGeneration && before.enabled)
) throw new Error('durable control cannot belong to the failed enable attempt')
if (!before.enabled) {
const verified = await verifiedDisabledControl(post, before.generation)
return { mode: config.mode, recovered: false, control: verified }
}
let applied
try {
applied = await applyDisabledControl(post, before)
} catch (error) {
applied = await resolveAmbiguousDisable(post, before, error)
}
const verified = await verifiedDisabledControl(post, applied.generation)
return { mode: config.mode, recovered: true, control: verified }
}
export async function operateRegionalRehome(config, dependencies = {}) {
const fetchImpl = dependencies.fetch ?? fetch
const post = dependencies.post ?? (async (path, body) => await responseJson(
await fetchImpl(`${config.directorOrigin}${path}`, {
method: 'POST',
headers: {
authorization: `Bearer ${config.token}`,
'content-type': 'application/json'
},
body: JSON.stringify(body),
signal: AbortSignal.timeout(30_000)
}),
path
))
if (config.mode === 'recover-enable') {
return await recoverRegionalRehomeEnable(config, post)
}
const selector = (await inspectAdmissionSelector(post)).selector
if (
selector.generation !== config.expectedSelectorGeneration ||
!exactMembership(selector.membership, config.expectedMembership)
) throw new Error('admission selector does not match the reviewed generation and membership')
const inspected = await post('/v1/admin/regional-rehome-control', {
v: 1,
action: 'inspect'
})
const before = assertControl(inspected.control, {
generation: config.expectedControlGeneration
})
if (config.mode === 'inspect') return { mode: config.mode, selector, control: before }
if (config.mode === 'enable' && before.enabled) {
throw new Error('regional rehome is already enabled; inspect before changing its rate')
}
if (config.mode === 'pause' && !before.enabled) {
throw new Error('regional rehome is already paused')
}
const enabled = config.mode === 'enable'
const applied = await post('/v1/admin/regional-rehome-control', {
v: 1,
action: 'apply',
expectedGeneration: config.expectedControlGeneration,
enabled,
notBefore: config.notBefore,
ratePerMinute: config.ratePerMinute,
preferenceMaxAgeMs: config.preferenceMaxAgeMs,
drainGraceMs: config.drainGraceMs,
confirmation: enabled
? 'ENABLE_REGIONAL_REHOMING'
: 'DISABLE_REGIONAL_REHOMING'
})
const after = assertControl(applied.control, {
generation: config.expectedControlGeneration + 1,
enabled
})
const verified = assertControl((await post('/v1/admin/regional-rehome-control', {
v: 1,
action: 'inspect'
})).control, {
generation: after.generation,
enabled
})
return { mode: config.mode, selector, control: verified }
}
export async function main(
argv = process.argv.slice(2),
environment = process.env,
dependencies = {},
write = (value) => process.stdout.write(value)
) {
const result = await operateRegionalRehome(
parseRegionalRehomeArguments(argv, environment),
dependencies
)
write(`${JSON.stringify({ event: 'relay_regional_rehome_control', ...result })}\n`)
}
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
main().catch((error) => {
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
process.exitCode = 1
})
}
@@ -0,0 +1,265 @@
import assert from 'node:assert/strict'
import { test } from 'node:test'
import {
main,
operateRegionalRehome,
parseRegionalRehomeArguments,
recoverRegionalRehomeEnable
} from './operate-relay-regional-rehome.mjs'
const membership = {
existingOnly: ['production-gce-c1'],
migrationOnly: ['production-gce-c2'],
general: ['production-gce-c7', 'production-gce-c27']
}
function argumentsFor(mode, confirmation) {
return [
'--mode', mode,
'--director-origin', 'https://relay.onorca.dev',
'--expected-selector-generation', '11',
'--expected-existing-only-cells', membership.existingOnly.join(','),
'--expected-migration-only-cells', membership.migrationOnly.join(','),
'--expected-general-cells', membership.general.join(','),
'--expected-control-generation', '4',
...(mode === 'inspect' ? [] : [
'--not-before', '2000000000000',
'--rate-per-minute', '10',
'--preference-max-age-ms', '86400000',
'--drain-grace-ms', '60000',
'--confirmation', confirmation
])
]
}
function control(generation, enabled) {
return {
generation,
enabled,
observationStartedAt: 1,
notBefore: 2_000_000_000_000,
ratePerMinute: 10,
preferenceMaxAgeMs: 86_400_000,
drainGraceMs: 60_000
}
}
test('parses exact selector and typed control confirmation', () => {
const parsed = parseRegionalRehomeArguments(
argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING'),
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
)
assert.equal(parsed.expectedSelectorGeneration, 11)
assert.equal(parsed.expectedControlGeneration, 4)
assert.equal(parsed.ratePerMinute, 10)
assert.throws(
() => parseRegionalRehomeArguments(
argumentsFor('pause', 'DISABLE_REGIONAL_REHOMING'),
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
),
/confirmation/
)
assert.throws(
() => parseRegionalRehomeArguments(
argumentsFor('inspect').concat('--rate-per-minute', '10'),
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
),
/inspect cannot/
)
})
test('binds enable to exact selector and durable control generations', async () => {
const requests = []
const controls = [
{ generation: 4, enabled: false },
{ generation: 5, enabled: true },
{ generation: 5, enabled: true }
].map((control) => ({
observationStartedAt: 1,
notBefore: 0,
ratePerMinute: 10,
preferenceMaxAgeMs: 86_400_000,
drainGraceMs: 60_000,
...control
}))
const config = parseRegionalRehomeArguments(
argumentsFor('enable', 'ENABLE_REGIONAL_REHOMING'),
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
)
const result = await operateRegionalRehome(config, {
post: async (path, body) => {
requests.push({ path, body })
if (path === '/v1/admin/admission-selector/status') {
return { selector: { generation: 11, membership } }
}
return { v: 1, control: controls.shift() }
}
})
assert.equal(result.control.generation, 5)
assert.deepEqual(requests[2].body, {
v: 1,
action: 'apply',
expectedGeneration: 4,
enabled: true,
notBefore: 2_000_000_000_000,
ratePerMinute: 10,
preferenceMaxAgeMs: 86_400_000,
drainGraceMs: 60_000,
confirmation: 'ENABLE_REGIONAL_REHOMING'
})
})
test('fails closed on selector drift before reading or mutating control', async () => {
let calls = 0
const config = parseRegionalRehomeArguments(
argumentsFor('disable', 'DISABLE_REGIONAL_REHOMING'),
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
)
await assert.rejects(
operateRegionalRehome(config, {
post: async () => {
calls += 1
return { selector: { generation: 12, membership } }
}
}),
/selector/
)
assert.equal(calls, 1)
})
test('failed-enable recovery CAS-disables an advanced enabled generation', async () => {
const requests = []
let current = control(7, true)
const result = await recoverRegionalRehomeEnable({
mode: 'recover-enable',
expectedControlGeneration: 4
}, async (_path, body) => {
requests.push(body)
if (body.action === 'inspect') return { control: current }
assert.equal(body.expectedGeneration, 7)
current = control(8, false)
throw new Error('enable recovery response was lost')
})
assert.equal(result.recovered, true)
assert.deepEqual(result.control, control(8, false))
assert.deepEqual(requests[1], {
v: 1,
action: 'apply',
expectedGeneration: 7,
enabled: false,
notBefore: 2_000_000_000_000,
ratePerMinute: 10,
preferenceMaxAgeMs: 86_400_000,
drainGraceMs: 60_000,
confirmation: 'DISABLE_REGIONAL_REHOMING'
})
})
test('failed-enable recovery retries once when the first CAS never commits', async () => {
let current = control(7, true)
const applyRequests = []
const result = await recoverRegionalRehomeEnable({
mode: 'recover-enable',
expectedControlGeneration: 4
}, async (_path, body) => {
if (body.action === 'inspect') return { control: current }
applyRequests.push(body)
if (applyRequests.length === 1) {
throw new Error('disable request was lost before commit')
}
current = control(8, false)
throw new Error('retry response was lost after commit')
})
assert.equal(applyRequests.length, 2)
assert.deepEqual(applyRequests[1], applyRequests[0])
assert.equal(result.recovered, true)
assert.deepEqual(result.control, control(8, false))
})
test('failed-enable recovery stops after two uncommitted CAS attempts', async () => {
let applyCalls = 0
await assert.rejects(
recoverRegionalRehomeEnable({
mode: 'recover-enable',
expectedControlGeneration: 4
}, async (_path, body) => {
if (body.action === 'inspect') return { control: control(7, true) }
applyCalls += 1
throw new Error(`disable attempt ${applyCalls} was lost before commit`)
}),
/exhausted two bounded CAS attempts/
)
assert.equal(applyCalls, 2)
})
test('failed-enable recovery is a verified no-op before enable and after cleanup', async () => {
for (const current of [control(4, false), control(8, false)]) {
const requests = []
const result = await recoverRegionalRehomeEnable({
mode: 'recover-enable',
expectedControlGeneration: 4
}, async (_path, body) => {
requests.push(body)
return { control: current }
})
assert.equal(result.recovered, false)
assert.equal(result.control.enabled, false)
assert.deepEqual(requests.map(({ action }) => action), ['inspect', 'inspect'])
}
})
test('failed-enable recovery rejects an unchanged pre-existing enabled state', async () => {
await assert.rejects(
recoverRegionalRehomeEnable({
mode: 'recover-enable',
expectedControlGeneration: 4
}, async () => ({ control: control(4, true) })),
/cannot belong to the failed enable attempt/
)
})
test('parses recovery without depending on selector diagnostics', () => {
const recoveryArguments = [
'--mode', 'recover-enable',
'--director-origin', 'https://relay.onorca.dev',
'--expected-control-generation', '4',
'--confirmation', 'RECOVER_FAILED_REGIONAL_REHOME_ENABLE'
]
const parsed = parseRegionalRehomeArguments(
recoveryArguments,
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
)
assert.equal(parsed.expectedControlGeneration, 4)
assert.equal(parsed.expectedMembership, undefined)
assert.throws(
() => parseRegionalRehomeArguments(
recoveryArguments.concat('--not-before', '2000000000000'),
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
),
/cannot carry durable control shape/
)
})
test('main executes recovery mode and emits verified disabled control', async () => {
let current = control(5, true)
let output = ''
await main([
'--mode', 'recover-enable',
'--director-origin', 'https://relay.onorca.dev',
'--expected-control-generation', '4',
'--confirmation', 'RECOVER_FAILED_REGIONAL_REHOME_ENABLE'
], { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }, {
post: async (_path, body) => {
if (body.action === 'apply') current = control(6, false)
return { control: current }
}
}, (value) => {
output += value
})
assert.deepEqual(JSON.parse(output), {
event: 'relay_regional_rehome_control',
mode: 'recover-enable',
recovered: true,
control: control(6, false)
})
})
+487
View File
@@ -0,0 +1,487 @@
import { readFileSync } from 'node:fs'
import { spawnSync } from 'node:child_process'
import { pathToFileURL } from 'node:url'
import {
inspectAdmissionSelector,
selectorCellState
} from './relay-admission-selector.mjs'
const PROJECT = 'onorca-cloud-staging'
const REGION = 'us-central1'
const DIRECTOR_ORIGIN = 'https://relay-staging.onorca.dev'
const ADMIN_AUDIENCE = `${DIRECTOR_ORIGIN}/v1/admin/drain`
const SQL_INSTANCE = 'orca-cloud-staging-auth-db'
const CLOUD_RUN_SERVICES = [
{ name: 'orca-cloud-relay-staging', healthOrigin: DIRECTOR_ORIGIN },
{ name: 'orca-cloud-auth-staging', healthOrigin: 'https://auth-staging.onorca.dev' }
]
const POLL_INTERVAL_MS = 5_000
const WAKE_TIMEOUT_MS = 12 * 60 * 1_000
export function parseArguments(argv) {
const values = {}
for (let index = 0; index < argv.length; index += 2) {
const key = argv[index]
const value = argv[index + 1]
if (!key?.startsWith('--') || value === undefined) throw new Error(`invalid argument ${key ?? ''}`)
const name = key.slice(2)
if (!['mode', 'wake-cells', 'topology-file'].includes(name)) {
throw new Error(`unsupported argument --${name}`)
}
values[name] = value
}
if (!['status', 'sleep', 'wake'].includes(values.mode)) {
throw new Error('--mode must be status, sleep, or wake')
}
if (!['configured', 'all'].includes(values['wake-cells'])) {
throw new Error('--wake-cells must be configured or all')
}
if (!values['topology-file']) throw new Error('missing --topology-file')
return {
mode: values.mode,
wakeCells: values['wake-cells'],
topologyFile: values['topology-file']
}
}
function canonicalStagingCell(cellId, value) {
if (!/^staging-gce-[a-z0-9-]+$/.test(cellId)) throw new Error(`unsafe staging cell ID ${cellId}`)
if (!value || typeof value !== 'object') throw new Error(`missing topology for ${cellId}`)
const cell = {
cellId,
migName: String(value.mig_name ?? ''),
zone: String(value.zone ?? ''),
origin: String(value.origin ?? ''),
initiallyEnabled: value.initially_enabled
}
if (!/^orca-cloud-staging-relay-gce-[a-z0-9-]+$/.test(cell.migName)) {
throw new Error(`${cellId} has an unsafe MIG name`)
}
if (!/^(?:us-central1|asia-east2)-[a-z]$/.test(cell.zone)) {
throw new Error(`${cellId} has an unsafe zone`)
}
const origin = new URL(cell.origin)
if (
origin.protocol !== 'https:' ||
origin.origin !== cell.origin ||
!origin.hostname.endsWith('.relay-staging.onorca.dev')
) {
throw new Error(`${cellId} has an unsafe origin`)
}
if (typeof cell.initiallyEnabled !== 'boolean') {
throw new Error(`${cellId} has no initial admission state`)
}
return cell
}
export function readStagingTopology(file) {
const parsed = JSON.parse(readFileSync(file, 'utf8'))
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
throw new Error('staging topology must be an object')
}
const cells = Object.entries(parsed)
.map(([cellId, value]) => canonicalStagingCell(cellId, value))
.sort((left, right) => left.cellId.localeCompare(right.cellId))
if (cells.length < 2 || cells.length > 10) {
throw new Error('staging topology must contain 2..10 cells')
}
if (cells.filter((cell) => cell.initiallyEnabled).length < 2) {
throw new Error('staging topology must retain two configured admission cells')
}
return cells
}
function defaultCommand(args, json) {
const result = spawnSync('gcloud', args, {
encoding: 'utf8',
stdio: ['ignore', 'pipe', 'pipe']
})
if (result.status !== 0) {
throw new Error(`gcloud ${args.slice(0, 5).join(' ')} failed: ${result.stderr.trim()}`)
}
return json ? JSON.parse(result.stdout) : result.stdout.trim()
}
function suppliedAdminToken(environment = process.env) {
const token = environment.ORCA_RELAY_ADMIN_ID_TOKEN
if (!token || token.length > 8_192 || !/^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/.test(token)) {
throw new Error('workflow did not supply a valid masked staging admin token')
}
return token
}
async function responseJson(response, label) {
const body = await response.json().catch(() => ({ error: `http_${response.status}` }))
if (!response.ok) throw new Error(`${label} failed: ${body.error ?? response.status}`)
return body
}
function createAdminPost(deps) {
const token = deps.adminToken()
return async (origin, path, body) =>
await responseJson(
await deps.fetch(`${origin}${path}`, {
method: 'POST',
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
body: JSON.stringify(body),
signal: AbortSignal.timeout(30_000)
}),
path
)
}
async function waitUntil(deps, label, operation, timeoutMs = WAKE_TIMEOUT_MS) {
const deadline = deps.now() + timeoutMs
let lastError
while (deps.now() < deadline) {
try {
const result = await operation()
if (result) return result
} catch (error) {
lastError = error
}
await deps.wait(POLL_INTERVAL_MS)
}
const detail = lastError instanceof Error ? `: ${lastError.message}` : ''
throw new Error(`timed out waiting for ${label}${detail}`)
}
async function checkHealth(deps, origin, path) {
const response = await deps.fetch(`${origin}${path}`, { signal: AbortSignal.timeout(15_000) })
const body = await response.json().catch(() => ({}))
return response.ok && body.ok === true
}
function sqlActivationPolicy(instance) {
return String(instance.settings?.activationPolicy ?? '')
}
function describeSql(deps) {
return deps.commandJson([
'sql',
'instances',
'describe',
SQL_INSTANCE,
'--project',
PROJECT,
'--format=json'
])
}
async function ensureSqlPolicy(deps, policy) {
if (sqlActivationPolicy(describeSql(deps)) === policy) return false
deps.command([
'sql',
'instances',
'patch',
SQL_INSTANCE,
'--project',
PROJECT,
`--activation-policy=${policy}`,
'--quiet'
])
await waitUntil(deps, `Cloud SQL activation policy ${policy}`, () =>
sqlActivationPolicy(describeSql(deps)) === policy
)
return true
}
function describeMig(deps, cell) {
return deps.commandJson([
'compute',
'instance-groups',
'managed',
'describe',
cell.migName,
'--project',
PROJECT,
'--zone',
cell.zone,
'--format=json'
])
}
async function setMigSize(deps, cell, size) {
const before = describeMig(deps, cell)
if (Number(before.targetSize) !== size) {
deps.command([
'compute',
'instance-groups',
'managed',
'resize',
cell.migName,
'--project',
PROJECT,
'--zone',
cell.zone,
`--size=${size}`,
'--quiet'
])
}
await waitUntil(deps, `${cell.cellId} size ${size}`, () => {
const current = describeMig(deps, cell)
return Number(current.targetSize) === size && current.status?.isStable === true
})
}
function activeRevisionName(service) {
const active = (service.status?.traffic ?? []).filter((entry) => Number(entry.percent ?? 0) > 0)
if (active.length !== 1 || Number(active[0].percent) !== 100 || !active[0].revisionName) {
throw new Error('Cloud Run service must have exactly one active revision')
}
return active[0].revisionName
}
function describeRunService(deps, name) {
return deps.commandJson([
'run',
'services',
'describe',
name,
'--project',
PROJECT,
'--region',
REGION,
'--format=json'
])
}
function describeRunRevision(deps, name) {
return deps.commandJson([
'run',
'revisions',
'describe',
name,
'--project',
PROJECT,
'--region',
REGION,
'--format=json'
])
}
function revisionMinimum(revision) {
return Number(revision.metadata?.annotations?.['autoscaling.knative.dev/minScale'] ?? 0)
}
async function ensureCloudRunScaleToZero(deps, service) {
const before = describeRunService(deps, service.name)
const activeRevision = describeRunRevision(deps, activeRevisionName(before))
if (revisionMinimum(activeRevision) === 0) return false
const latestName = before.status?.latestReadyRevisionName
const latest = latestName ? describeRunRevision(deps, latestName) : null
if (!latest || revisionMinimum(latest) !== 0) {
deps.command([
'run',
'services',
'update',
service.name,
'--project',
PROJECT,
'--region',
REGION,
'--min-instances=0',
'--quiet'
])
}
deps.command([
'run',
'services',
'update-traffic',
service.name,
'--project',
PROJECT,
'--region',
REGION,
'--to-latest',
'--quiet'
])
await waitUntil(deps, `${service.name} scale-to-zero revision`, async () => {
const current = describeRunService(deps, service.name)
const revision = describeRunRevision(deps, activeRevisionName(current))
return revisionMinimum(revision) === 0 && (await checkHealth(deps, service.healthOrigin, '/health'))
})
return true
}
async function cellStatus(adminPost, cell) {
const response = await adminPost(DIRECTOR_ORIGIN, '/v1/admin/cell-status', {
v: 1,
cellId: cell.cellId
})
if (!response.status || response.status.cellId !== cell.cellId) {
throw new Error(`${cell.cellId} returned an invalid status`)
}
return response.status
}
function assertQuiescent(status) {
const active = {
activityLeases: status.activityLeases,
activityRequestUnits: status.activityRequestUnits,
outgoingMigrations: status.outgoingMigrations,
incomingMigrations: status.incomingMigrations,
observedRequests: status.runtime?.observedRequests ?? 0
}
if (Object.values(active).some((value) => Number(value) !== 0)) {
throw new Error(`${status.cellId} still has active Relay work: ${JSON.stringify(active)}`)
}
}
async function setCellState(adminPost, cell, enabled) {
await adminPost(DIRECTOR_ORIGIN, '/v1/admin/cell-state', {
v: 1,
cellId: cell.cellId,
enabled
})
}
async function stagingStatus(deps, cells) {
return {
event: 'staging_relay_power_status',
project: PROJECT,
sqlActivationPolicy: sqlActivationPolicy(describeSql(deps)),
cells: cells.map((cell) => ({
cellId: cell.cellId,
initiallyEnabled: cell.initiallyEnabled,
targetSize: Number(describeMig(deps, cell).targetSize)
})),
cloudRun: CLOUD_RUN_SERVICES.map((service) => {
const described = describeRunService(deps, service.name)
const revision = describeRunRevision(deps, activeRevisionName(described))
return { service: service.name, activeRevisionMinimum: revisionMinimum(revision) }
})
}
}
async function sleepStaging(deps, cells) {
const sqlPolicy = sqlActivationPolicy(describeSql(deps))
if (sqlPolicy === 'NEVER') {
const runningCells = cells.filter((cell) => Number(describeMig(deps, cell).targetSize) !== 0)
if (runningCells.length > 0) {
// SQL-off plus running workers is an unknown partial state; never kill those workers blindly.
throw new Error(
`staging is partially asleep with running cells: ${runningCells.map((cell) => cell.cellId).join(', ')}`
)
}
deps.emit({ event: 'staging_relay_sleep_reconciled', alreadyAsleep: true })
return
}
const adminPost = createAdminPost(deps)
const initial = await Promise.all(cells.map((cell) => cellStatus(adminPost, cell)))
for (const status of initial) assertQuiescent(status)
const selector = await inspectAdmissionSelector(
async (path, body) => await adminPost(DIRECTOR_ORIGIN, path, body)
)
if (selector.selector.generation > 0) {
throw new Error(
'staging sleep cannot reverse the monotonic admission selector; keep staging awake'
)
}
const previouslyEnabled = new Set(initial.filter((status) => status.enabled).map((status) => status.cellId))
for (const cell of cells) await setCellState(adminPost, cell, false)
await deps.wait(15_000)
try {
const disabled = await Promise.all(cells.map((cell) => cellStatus(adminPost, cell)))
for (const status of disabled) {
if (status.enabled) throw new Error(`${status.cellId} admission did not disable`)
assertQuiescent(status)
}
for (const service of CLOUD_RUN_SERVICES) await ensureCloudRunScaleToZero(deps, service)
const final = await Promise.all(cells.map((cell) => cellStatus(adminPost, cell)))
for (const status of final) assertQuiescent(status)
} catch (error) {
for (const cell of cells.filter((candidate) => previouslyEnabled.has(candidate.cellId))) {
await setCellState(adminPost, cell, true).catch(() => undefined)
}
throw error
}
await Promise.all(cells.map((cell) => setMigSize(deps, cell, 0)))
await ensureSqlPolicy(deps, 'NEVER')
deps.emit({ event: 'staging_relay_slept', stoppedCells: cells.map((cell) => cell.cellId) })
}
async function wakeStaging(deps, cells, wakeCells) {
await ensureSqlPolicy(deps, 'ALWAYS')
await waitUntil(deps, 'staging director health', () => checkHealth(deps, DIRECTOR_ORIGIN, '/health'))
for (const service of CLOUD_RUN_SERVICES) await ensureCloudRunScaleToZero(deps, service)
const adminPost = createAdminPost(deps)
const selector = await inspectAdmissionSelector(
async (path, body) => await adminPost(DIRECTOR_ORIGIN, path, body)
)
const selectorActive = selector.selector.generation > 0
// Existing-only cells may still own live or dormant assignments, so a
// selector-era wake restores the complete retained topology.
const selected = selectorActive
? cells
: cells.filter((cell) => wakeCells === 'all' || cell.initiallyEnabled)
await Promise.all(
cells.map((cell) => setMigSize(deps, cell, selected.includes(cell) ? 1 : 0))
)
for (const cell of selected) {
await waitUntil(deps, `${cell.cellId} health`, () => checkHealth(deps, cell.origin, '/health'))
await waitUntil(deps, `${cell.cellId} readiness`, () => checkHealth(deps, cell.origin, '/ready'))
}
for (const cell of cells) {
if (selectorActive) {
const status = await waitUntil(deps, `${cell.cellId} authenticated heartbeat`, async () => {
const current = await cellStatus(adminPost, cell)
return current.runtime?.heartbeatFresh && current.runtime.ready ? current : null
})
if (status.admissionState !== selectorCellState(selector.selector, cell.cellId)) {
throw new Error(`${cell.cellId} admission does not match selector`)
}
continue
}
if (!selected.includes(cell)) {
await setCellState(adminPost, cell, false)
continue
}
const status = await waitUntil(deps, `${cell.cellId} authenticated heartbeat`, async () => {
const current = await cellStatus(adminPost, cell)
return current.runtime?.heartbeatFresh && current.runtime.ready ? current : null
})
if (cell.initiallyEnabled && !status.enabled) await setCellState(adminPost, cell, true)
if (!cell.initiallyEnabled && status.enabled) await setCellState(adminPost, cell, false)
}
deps.emit({
event: 'staging_relay_woke',
runningCells: selected.map((cell) => cell.cellId),
admissionCells: selectorActive
? selector.selector.membership.general
: selected.filter((cell) => cell.initiallyEnabled).map((cell) => cell.cellId)
})
}
export async function runStagingRelayPower(config, overrides = {}) {
const deps = {
command: overrides.command ?? ((args) => defaultCommand(args, false)),
commandJson: overrides.commandJson ?? ((args) => defaultCommand(args, true)),
fetch: overrides.fetch ?? fetch,
adminToken: overrides.adminToken ?? suppliedAdminToken,
emit: overrides.emit ?? ((event) => process.stdout.write(`${JSON.stringify(event)}\n`)),
now: overrides.now ?? Date.now,
wait: overrides.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)))
}
const cells = readStagingTopology(config.topologyFile)
if (config.mode === 'status') deps.emit(await stagingStatus(deps, cells))
else if (config.mode === 'sleep') await sleepStaging(deps, cells)
else await wakeStaging(deps, cells, config.wakeCells)
}
export async function main(argv = process.argv.slice(2)) {
await runStagingRelayPower(parseArguments(argv))
}
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
main().catch((error) => {
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
process.exitCode = 1
})
}
@@ -0,0 +1,359 @@
import assert from 'node:assert/strict'
import { mkdtempSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { test } from 'node:test'
import {
parseArguments,
readStagingTopology,
runStagingRelayPower
} from './power-staging-relay.mjs'
function topologyFile(overrides = {}) {
const directory = mkdtempSync(join(tmpdir(), 'staging-relay-power-'))
const topology = {
'staging-gce-c1': {
mig_name: 'orca-cloud-staging-relay-gce-c1',
zone: 'us-central1-b',
origin: 'https://c1.relay-staging.onorca.dev',
initially_enabled: true
},
'staging-gce-c2': {
mig_name: 'orca-cloud-staging-relay-gce-c2',
zone: 'us-central1-c',
origin: 'https://c2.relay-staging.onorca.dev',
initially_enabled: true
},
'staging-gce-c3': {
mig_name: 'orca-cloud-staging-relay-gce-c3',
zone: 'us-central1-a',
origin: 'https://c3.relay-staging.onorca.dev',
initially_enabled: false
},
'staging-gce-c4': {
mig_name: 'orca-cloud-staging-relay-gce-c4',
zone: 'asia-east2-a',
origin: 'https://c4.relay-staging.onorca.dev',
initially_enabled: false
},
...overrides
}
const file = join(directory, 'topology.json')
writeFileSync(file, JSON.stringify(topology))
return file
}
function argumentConfig(file, mode, wakeCells = 'configured') {
return parseArguments([
'--mode',
mode,
'--wake-cells',
wakeCells,
'--topology-file',
file
])
}
function response(body, status = 200) {
return new Response(JSON.stringify(body), {
status,
headers: { 'content-type': 'application/json' }
})
}
function harness({
sqlPolicy = 'ALWAYS',
migSize = 1,
observedRequests = 0,
selectorGeneration = 0
} = {}) {
const cells = new Map(
['staging-gce-c1', 'staging-gce-c2', 'staging-gce-c3', 'staging-gce-c4'].map((cellId, index) => [
cellId,
{
enabled: index < 2,
targetSize: migSize,
observedRequests,
initiallyEnabled: index < 2
}
])
)
const revisions = new Map([
['orca-cloud-relay-staging', { active: 'relay-00001', latest: 'relay-00001', min: 1 }],
['orca-cloud-auth-staging', { active: 'auth-00001', latest: 'auth-00001', min: 1 }]
])
const revisionMinimums = new Map([
['relay-00001', 1],
['auth-00001', 1]
])
const commands = []
const events = []
let activationPolicy = sqlPolicy
let clock = 0
function cellForMig(name) {
return [...cells.entries()].find(([, value], index) => {
const suffix = `c${index + 1}`
return name.endsWith(suffix) && value
})
}
function commandJson(args) {
if (args[0] === 'sql') return { settings: { activationPolicy } }
if (args[0] === 'compute') {
const entry = cellForMig(args[4])
return { targetSize: entry[1].targetSize, status: { isStable: true } }
}
if (args[0] === 'run' && args[1] === 'services') {
const state = revisions.get(args[3])
return {
status: {
latestReadyRevisionName: state.latest,
traffic: [{ percent: 100, revisionName: state.active }]
}
}
}
if (args[0] === 'run' && args[1] === 'revisions') {
return {
metadata: {
annotations: {
'autoscaling.knative.dev/minScale': String(revisionMinimums.get(args[3]) ?? 0)
}
}
}
}
throw new Error(`unexpected JSON command ${args.join(' ')}`)
}
function command(args) {
commands.push(args)
if (args[0] === 'sql') {
activationPolicy = args.find((arg) => arg.startsWith('--activation-policy='))?.split('=')[1]
return
}
if (args[0] === 'compute') {
const entry = cellForMig(args[4])
entry[1].targetSize = Number(args.find((arg) => arg.startsWith('--size='))?.split('=')[1])
return
}
if (args[0] === 'run' && args[1] === 'services' && args[2] === 'update') {
const state = revisions.get(args[3])
state.latest = `${args[3]}-power`
revisionMinimums.set(state.latest, 0)
return
}
if (args[0] === 'run' && args[1] === 'services' && args[2] === 'update-traffic') {
const state = revisions.get(args[3])
state.active = state.latest
return
}
throw new Error(`unexpected command ${args.join(' ')}`)
}
async function fetchImpl(url, options = {}) {
const parsed = new URL(url)
if (!options.method) return response({ ok: true })
const body = JSON.parse(options.body)
if (parsed.pathname === '/v1/admin/cell-status') {
const state = cells.get(body.cellId)
const index = [...cells.keys()].indexOf(body.cellId)
return response({
v: 1,
status: {
cellId: body.cellId,
enabled: state.enabled,
admissionState:
selectorGeneration > 0
? index === 0
? 'existing-only'
: index === 1
? 'migration-only'
: index === 2
? 'general'
: 'migration-only'
: state.enabled
? 'general'
: 'existing-only',
assignments: 0,
reservedRequests: 0,
activityLeases: 0,
activityRequestUnits: 0,
outgoingMigrations: 0,
incomingMigrations: 0,
runtime: {
ready: state.targetSize === 1,
heartbeatFresh: state.targetSize === 1,
observedRequests: state.observedRequests
}
}
})
}
if (parsed.pathname === '/v1/admin/admission-selector/status') {
return response({
v: 1,
selector: {
generation: selectorGeneration,
attemptId: null,
membership: {
existingOnly:
selectorGeneration > 0
? ['staging-gce-c1']
: [...cells]
.filter(([, state]) => !state.enabled)
.map(([cellId]) => cellId),
migrationOnly:
selectorGeneration > 0 ? ['staging-gce-c2', 'staging-gce-c4'] : [],
general:
selectorGeneration > 0
? ['staging-gce-c3']
: [...cells]
.filter(([, state]) => state.enabled)
.map(([cellId]) => cellId)
}
},
intent: null
})
}
if (parsed.pathname === '/v1/admin/cell-state') {
cells.get(body.cellId).enabled = body.enabled
return response({ ok: true })
}
throw new Error(`unexpected fetch ${parsed.pathname}`)
}
return {
cells,
commands,
events,
deps: {
command,
commandJson,
fetch: fetchImpl,
adminToken: () => 'header.payload.signature',
emit: (event) => events.push(event),
now: () => clock,
wait: async (ms) => {
clock += ms
}
},
sqlPolicy: () => activationPolicy
}
}
test('accepts only explicit staging power arguments and topology', () => {
const file = topologyFile()
assert.equal(argumentConfig(file, 'status').mode, 'status')
assert.equal(readStagingTopology(file).at(-1).zone, 'asia-east2-a')
assert.throws(() => argumentConfig(file, 'destroy'))
assert.throws(() => parseArguments(['--mode', 'sleep', '--wake-cells', 'configured']))
const unsafe = topologyFile({
'staging-gce-c1': {
mig_name: 'orca-cloud-relay-gce-c1',
zone: 'us-central1-a',
origin: 'https://c1.relay.onorca.dev',
initially_enabled: true
}
})
assert.throws(() => readStagingTopology(unsafe), /unsafe/)
const unreviewedRegion = topologyFile({
'staging-gce-c4': {
mig_name: 'orca-cloud-staging-relay-gce-c4',
zone: 'europe-west1-b',
origin: 'https://c4.relay-staging.onorca.dev',
initially_enabled: false
}
})
assert.throws(() => readStagingTopology(unreviewedRegion), /unsafe zone/)
})
test('refuses sleep before changing admission when a cell has active requests', async () => {
const testHarness = harness({ observedRequests: 1 })
await assert.rejects(
runStagingRelayPower(argumentConfig(topologyFile(), 'sleep'), testHarness.deps),
/still has active Relay work/
)
assert.equal(testHarness.commands.length, 0)
assert.equal(testHarness.cells.get('staging-gce-c1').enabled, true)
})
test('sleeps only after disabling admission and proving zero active work', async () => {
const testHarness = harness()
await runStagingRelayPower(argumentConfig(topologyFile(), 'sleep'), testHarness.deps)
assert.equal(testHarness.sqlPolicy(), 'NEVER')
assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.targetSize), [0, 0, 0, 0])
assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.enabled), [false, false, false, false])
assert.equal(testHarness.events.at(-1).event, 'staging_relay_slept')
assert.equal(
testHarness.commands.filter((args) => args[0] === 'run' && args[2] === 'update').length,
2
)
})
test('refuses staging sleep after the monotonic selector boundary', async () => {
const testHarness = harness({ selectorGeneration: 1 })
await assert.rejects(
runStagingRelayPower(argumentConfig(topologyFile(), 'sleep'), testHarness.deps),
/cannot reverse the monotonic admission selector/
)
assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.targetSize), [1, 1, 1, 1])
})
test('refuses to terminate workers from an unknown partially asleep state', async () => {
const testHarness = harness({ sqlPolicy: 'NEVER', migSize: 1 })
await assert.rejects(
runStagingRelayPower(argumentConfig(topologyFile(), 'sleep'), testHarness.deps),
/partially asleep with running cells/
)
assert.equal(testHarness.commands.length, 0)
assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.targetSize), [1, 1, 1, 1])
})
test('wakes SQL and configured cells while leaving the candidate off and disabled', async () => {
const testHarness = harness({ sqlPolicy: 'NEVER', migSize: 0 })
for (const cell of testHarness.cells.values()) cell.enabled = false
for (const state of testHarness.cells.values()) state.observedRequests = 0
await runStagingRelayPower(argumentConfig(topologyFile(), 'wake'), testHarness.deps)
assert.equal(testHarness.sqlPolicy(), 'ALWAYS')
assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.targetSize), [1, 1, 0, 0])
assert.deepEqual([...testHarness.cells.values()].map((cell) => cell.enabled), [true, true, false, false])
assert.deepEqual(testHarness.events.at(-1), {
event: 'staging_relay_woke',
runningCells: ['staging-gce-c1', 'staging-gce-c2'],
admissionCells: ['staging-gce-c1', 'staging-gce-c2']
})
})
test('wakes every retained cell without rewriting selector-era admission', async () => {
const testHarness = harness({
sqlPolicy: 'NEVER',
migSize: 0,
selectorGeneration: 1
})
const states = [...testHarness.cells.values()]
states[0].enabled = false
states[1].enabled = true
states[2].enabled = true
states[3].enabled = true
await runStagingRelayPower(argumentConfig(topologyFile(), 'wake'), testHarness.deps)
assert.deepEqual(states.map((cell) => cell.targetSize), [1, 1, 1, 1])
assert.deepEqual(states.map((cell) => cell.enabled), [false, true, true, true])
assert.deepEqual(testHarness.events.at(-1), {
event: 'staging_relay_woke',
runningCells: ['staging-gce-c1', 'staging-gce-c2', 'staging-gce-c3', 'staging-gce-c4'],
admissionCells: ['staging-gce-c3']
})
})
test('status is read-only and reports the current billable floor controls', async () => {
const testHarness = harness()
await runStagingRelayPower(argumentConfig(topologyFile(), 'status'), testHarness.deps)
assert.equal(testHarness.commands.length, 0)
assert.equal(testHarness.events[0].project, 'onorca-cloud-staging')
assert.equal(testHarness.events[0].sqlActivationPolicy, 'ALWAYS')
assert.equal(testHarness.events[0].cells.length, 4)
})
@@ -0,0 +1,78 @@
import { readFileSync, writeFileSync } from 'node:fs'
import { fileURLToPath } from 'node:url'
function argumentsFrom(argv) {
const values = {}
for (let index = 0; index < argv.length; index += 2) {
const key = argv[index]
const value = argv[index + 1]
if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments')
values[key.slice(2)] = value
}
for (const key of ['current-json', 'topology-json', 'output', 'cell-ids', 'image-digest']) {
if (!values[key]) throw new Error(`missing --${key}`)
}
return values
}
export function prepareRelayAsiaDirectorCells({ currentCells, topology, cellIds, imageDigest }) {
if (!Array.isArray(currentCells) || !topology || Array.isArray(topology)) {
throw new Error('director inputs are invalid')
}
const additions = cellIds.split(',').map((value) => value.trim()).filter(Boolean)
if (
additions.length === 0 ||
new Set(additions).size !== additions.length
) throw new Error('director Asia additions are invalid')
const currentIds = new Set(currentCells.map((cell) => cell.id))
if (currentIds.size !== currentCells.length) throw new Error('current director cells contain duplicates')
const normalizedCurrent = currentCells.map((cell) => ({
...cell,
region: cell.region ?? 'us-central1'
}))
const desiredCells = additions.map((cellId) => {
const cell = topology[cellId]
if (
!cell ||
cell.region !== 'asia-east2' ||
cell.capacity_requests !== 6_000 ||
cell.database_pool_max !== 10 ||
cell.connection_hard_cap !== 3_000 ||
cell.connection_unobserved_bound !== 60 ||
cell.initially_enabled !== false ||
cell.image?.split('@')[1] !== imageDigest
) throw new Error(`${cellId} state output does not match the reviewed Asia shape`)
return {
id: cellId,
url: cell.origin,
capacityRequests: cell.capacity_requests,
region: cell.region,
initiallyEnabled: false,
connectionHardCap: cell.connection_hard_cap,
connectionUnobservedBound: cell.connection_unobserved_bound
}
})
const desiredById = new Map(desiredCells.map((cell) => [cell.id, cell]))
for (const current of normalizedCurrent) {
const desired = desiredById.get(current.id)
if (!desired) continue
for (const [key, value] of Object.entries(desired)) {
if (current[key] !== value) {
throw new Error(`${current.id} director configuration differs from the reviewed Asia shape`)
}
}
}
const configured = new Set(normalizedCurrent.map((cell) => cell.id))
return [...normalizedCurrent, ...desiredCells.filter((cell) => !configured.has(cell.id))]
}
if (process.argv[1] === fileURLToPath(import.meta.url)) {
const values = argumentsFrom(process.argv.slice(2))
const result = prepareRelayAsiaDirectorCells({
currentCells: JSON.parse(readFileSync(values['current-json'], 'utf8')),
topology: JSON.parse(readFileSync(values['topology-json'], 'utf8')),
cellIds: values['cell-ids'],
imageDigest: values['image-digest']
})
writeFileSync(values.output, `${JSON.stringify(result)}\n`, { mode: 0o600 })
}
@@ -0,0 +1,60 @@
import assert from 'node:assert/strict'
import { test } from 'node:test'
import { prepareRelayAsiaDirectorCells } from './prepare-relay-asia-director-cells.mjs'
const digest = `sha256:${'a'.repeat(64)}`
const topologyCell = (ordinal, zone) => ({
origin: `https://c${ordinal}.relay.onorca.dev`, region: 'asia-east2', zone,
capacity_requests: 6_000, database_pool_max: 10,
connection_hard_cap: 3_000, connection_unobserved_bound: 60,
initially_enabled: false,
image: `us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@${digest}`
})
test('preserves current order, defaults predecessor regions, and appends exact Asia cells', () => {
const current = [{
id: 'production-gce-c1', url: 'https://c1.relay.onorca.dev',
capacityRequests: 4_000, initiallyEnabled: false
}]
const result = prepareRelayAsiaDirectorCells({
currentCells: current,
topology: {
'production-gce-c27': topologyCell(27, 'asia-east2-a'),
'production-gce-c28': topologyCell(28, 'asia-east2-b'),
'production-gce-c29': topologyCell(29, 'asia-east2-c')
},
cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29',
imageDigest: digest
})
assert.equal(result[0].region, 'us-central1')
assert.deepEqual(result.slice(1).map(({ id }) => id), [
'production-gce-c27', 'production-gce-c28', 'production-gce-c29'
])
assert.ok(result.slice(1).every((cell) =>
cell.region === 'asia-east2' && cell.initiallyEnabled === false &&
cell.connectionHardCap === 3_000
))
})
test('is idempotent for an exact existing Asia cell and rejects director drift', () => {
const topology = { 'production-gce-c27': topologyCell(27, 'asia-east2-a') }
const current = prepareRelayAsiaDirectorCells({
currentCells: [], topology, cellIds: 'production-gce-c27', imageDigest: digest
})
assert.deepEqual(prepareRelayAsiaDirectorCells({
currentCells: current, topology, cellIds: 'production-gce-c27', imageDigest: digest
}), current)
assert.throws(() => prepareRelayAsiaDirectorCells({
currentCells: [{ ...current[0], capacityRequests: 5_999 }],
topology, cellIds: 'production-gce-c27', imageDigest: digest
}), /director configuration differs/)
})
test('rejects a mismatching topology state output', () => {
const wrong = topologyCell(27, 'asia-east2-a')
wrong.database_pool_max = 20
assert.throws(() => prepareRelayAsiaDirectorCells({
currentCells: [], topology: { 'production-gce-c27': wrong },
cellIds: 'production-gce-c27', imageDigest: digest
}), /does not match/)
})
@@ -0,0 +1,113 @@
import { readFileSync } from 'node:fs'
import { fileURLToPath } from 'node:url'
const BOOT_IMAGE = 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21'
const SHAPES = {
staging: { project: 'onorca-cloud-staging', cells: { 'staging-gce-c4': 'asia-east2-a' } },
production: {
project: 'onorca-cloud',
cells: {
'production-gce-c27': 'asia-east2-a',
'production-gce-c28': 'asia-east2-b',
'production-gce-c29': 'asia-east2-c'
}
}
}
function argumentsFrom(argv) {
const values = {}
for (let index = 0; index < argv.length; index += 2) {
const key = argv[index]
const value = argv[index + 1]
if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments')
values[key.slice(2)] = value
}
for (const key of ['existing-json', 'environment', 'cell-ids', 'image']) {
if (!values[key]) throw new Error(`missing --${key}`)
}
return values
}
function canonical(value) {
if (Array.isArray(value)) return value.map(canonical)
if (value && typeof value === 'object') {
return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonical(value[key])]))
}
return value
}
export function prepareRelayAsiaTopologyInput({
existingCells,
existingAdditionalRegions,
environment,
cellIds,
image
}) {
const shape = SHAPES[environment]
if (!shape) throw new Error('invalid environment')
const requested = cellIds.split(',').map((value) => value.trim()).filter(Boolean).sort()
const expected = Object.keys(shape.cells).sort()
if (new Set(requested).size !== requested.length || JSON.stringify(requested) !== JSON.stringify(expected)) {
throw new Error('cell IDs do not match the reviewed Asia topology')
}
const prefix = `us-central1-docker.pkg.dev/${shape.project}/orca-cloud/relay@sha256:`
if (!image.startsWith(prefix) || !/sha256:[a-f0-9]{64}$/.test(image)) {
throw new Error('image is not the environment Relay image pinned by digest')
}
if (!existingCells || Array.isArray(existingCells) || typeof existingCells !== 'object') {
throw new Error('existing Relay cells must be an object')
}
if (
!existingAdditionalRegions ||
Array.isArray(existingAdditionalRegions) ||
typeof existingAdditionalRegions !== 'object' ||
JSON.stringify(canonical(existingAdditionalRegions)) !==
JSON.stringify(canonical({ 'asia-east2': '10.42.1.0/24' }))
) {
throw new Error('Asia subnet must be committed before topology planning')
}
const additions = Object.fromEntries(expected.map((cellId) => {
const hostname = cellId.split('-').at(-1)
return [cellId, {
hostname,
region: 'asia-east2',
zone: shape.cells[cellId],
machine_type: 'e2-standard-4',
boot_disk_gb: 30,
boot_image: BOOT_IMAGE,
capacity_requests: 6_000,
database_pool_max: 10,
image,
initially_enabled: false,
connection_hard_cap: 3_000,
connection_unobserved_bound: 60
}]
}))
for (const cellId of expected) {
if (!existingCells[cellId]) {
throw new Error('Asia cells must be committed before topology planning')
}
if (
JSON.stringify(canonical(existingCells[cellId])) !==
JSON.stringify(canonical(additions[cellId]))
) {
throw new Error('committed Asia cell differs from the reviewed topology')
}
}
return {
relay_gce_additional_region_subnetwork_cidrs: existingAdditionalRegions,
relay_gce_cells: existingCells
}
}
if (process.argv[1] === fileURLToPath(import.meta.url)) {
const values = argumentsFrom(process.argv.slice(2))
const existing = JSON.parse(readFileSync(values['existing-json'], 'utf8'))
prepareRelayAsiaTopologyInput({
existingCells: existing.relay_gce_cells,
existingAdditionalRegions: existing.relay_gce_additional_region_subnetwork_cidrs,
environment: values.environment,
cellIds: values['cell-ids'],
image: values.image
})
}
@@ -0,0 +1,87 @@
import assert from 'node:assert/strict'
import { test } from 'node:test'
import { prepareRelayAsiaTopologyInput } from './prepare-relay-asia-topology-input.mjs'
const image = `us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@sha256:${'a'.repeat(64)}`
const additionalRegions = { 'asia-east2': '10.42.1.0/24' }
const productionCells = () => Object.fromEntries([
[27, 'asia-east2-a'],
[28, 'asia-east2-b'],
[29, 'asia-east2-c']
].map(([ordinal, zone]) => [`production-gce-c${ordinal}`, {
hostname: `c${ordinal}`, region: 'asia-east2', zone,
machine_type: 'e2-standard-4', boot_disk_gb: 30,
boot_image: 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21',
capacity_requests: 6_000, database_pool_max: 10, image, initially_enabled: false,
connection_hard_cap: 3_000, connection_unobserved_bound: 60
}]))
test('accepts the exact production topology only after it is durably committed', () => {
const existing = {
'production-gce-c26': { hostname: 'c26', image: 'existing' },
...productionCells()
}
const result = prepareRelayAsiaTopologyInput({ existingCells: existing,
existingAdditionalRegions: additionalRegions, environment: 'production',
cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29', image })
assert.equal(result.relay_gce_cells, existing)
assert.equal(result.relay_gce_additional_region_subnetwork_cidrs, additionalRegions)
assert.deepEqual(existing['production-gce-c27'], {
hostname: 'c27', region: 'asia-east2', zone: 'asia-east2-a',
machine_type: 'e2-standard-4', boot_disk_gb: 30,
boot_image: 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21',
capacity_requests: 6_000, database_pool_max: 10, image, initially_enabled: false,
connection_hard_cap: 3_000, connection_unobserved_bound: 60
})
})
test('accepts the one exact committed staging Asia cell', () => {
const stagingImage = image.replace('onorca-cloud/', 'onorca-cloud-staging/')
const stagingCell = {
hostname: 'c4', region: 'asia-east2', zone: 'asia-east2-a',
machine_type: 'e2-standard-4', boot_disk_gb: 30,
boot_image: 'https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21',
capacity_requests: 6_000, database_pool_max: 10, image: stagingImage,
initially_enabled: false, connection_hard_cap: 3_000,
connection_unobserved_bound: 60
}
const result = prepareRelayAsiaTopologyInput({
existingCells: { 'staging-gce-c3': { hostname: 'c3' }, 'staging-gce-c4': stagingCell },
existingAdditionalRegions: additionalRegions,
environment: 'staging',
cellIds: 'staging-gce-c4',
image: stagingImage
})
assert.equal(result.relay_gce_cells['staging-gce-c4'].zone, 'asia-east2-a')
assert.equal(result.relay_gce_cells['staging-gce-c4'].image, stagingImage)
})
test('rejects an uncommitted subnet or cell, partial wave, wrong image, and drift', () => {
assert.throws(() => prepareRelayAsiaTopologyInput({
existingCells: productionCells(), existingAdditionalRegions: additionalRegions,
environment: 'production', cellIds: 'production-gce-c27', image
}), /cell IDs/)
assert.throws(() => prepareRelayAsiaTopologyInput({
existingCells: productionCells(), existingAdditionalRegions: additionalRegions,
environment: 'production',
cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29',
image: image.replace('onorca-cloud/', 'other-project/')
}), /environment Relay image/)
assert.throws(() => prepareRelayAsiaTopologyInput({
existingCells: productionCells(), existingAdditionalRegions: {}, environment: 'production',
cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29', image
}), /subnet must be committed/)
const missing = productionCells()
delete missing['production-gce-c29']
assert.throws(() => prepareRelayAsiaTopologyInput({
existingCells: missing, existingAdditionalRegions: additionalRegions, environment: 'production',
cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29', image
}), /cells must be committed/)
assert.throws(() => prepareRelayAsiaTopologyInput({
existingCells: { ...productionCells(), 'production-gce-c27': {} },
existingAdditionalRegions: additionalRegions, environment: 'production',
cellIds: 'production-gce-c27,production-gce-c28,production-gce-c29', image
}), /differs from the reviewed topology/)
})
@@ -0,0 +1,183 @@
import { pathToFileURL } from 'node:url'
import {
applyExactAdmissionSelector,
inspectAdmissionSelector,
membershipWithStates,
selectorCellState
} from './relay-admission-selector.mjs'
function parseArguments(argv) {
const values = {}
for (let index = 0; index < argv.length; index += 2) {
const key = argv[index]
const value = argv[index + 1]
if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments')
values[key.slice(2)] = value
}
for (const key of ['director-origin', 'cell-id', 'mode']) {
if (!values[key]) throw new Error(`missing --${key}`)
}
const origin = new URL(values['director-origin'])
if (origin.protocol !== 'https:' || origin.origin !== values['director-origin']) {
throw new Error('--director-origin must be a canonical HTTPS origin')
}
if (!['isolate', 'activate', 'restore-fallback', 'restore'].includes(values.mode)) {
throw new Error('--mode must be isolate, activate, restore-fallback, or restore')
}
const cellOrigin = values['cell-origin'] ? new URL(values['cell-origin']) : null
if (
values.mode === 'isolate' &&
(!cellOrigin || cellOrigin.protocol !== 'https:' || cellOrigin.origin !== values['cell-origin'])
) {
throw new Error('--cell-origin must be a canonical HTTPS origin for isolate mode')
}
const restoreGeneralCellIds = values['general-cell-ids']?.split(',').filter(Boolean) ?? []
if (
['restore-fallback', 'restore'].includes(values.mode) &&
restoreGeneralCellIds.length === 0
) {
throw new Error('--general-cell-ids is required for restore modes')
}
if (values.mode === 'restore' && !restoreGeneralCellIds.includes(values['cell-id'])) {
throw new Error('--general-cell-ids must include the canary for restore mode')
}
if (values.mode === 'restore-fallback' && restoreGeneralCellIds.includes(values['cell-id'])) {
throw new Error('--general-cell-ids cannot include the canary for fallback restore')
}
if (new Set(restoreGeneralCellIds).size !== restoreGeneralCellIds.length) {
throw new Error('--general-cell-ids must be distinct')
}
return {
directorOrigin: origin.origin,
cellOrigin: cellOrigin?.origin,
cellId: values['cell-id'],
mode: values.mode,
restoreGeneralCellIds
}
}
async function responseJson(response, label) {
const body = await response.json().catch(() => ({}))
if (!response.ok) throw new Error(`${label} returned ${response.status}`)
return body
}
function sameMembership(left, right) {
return JSON.stringify(left) === JSON.stringify(right)
}
async function legacyCellState(post, cellId) {
const result = await post('/v1/admin/cell-status', { v: 1, cellId })
if (result.status?.cellId !== cellId) throw new Error('legacy admission status is invalid')
const state = result.status.admissionState
if (!['existing-only', 'migration-only', 'general'].includes(state)) {
throw new Error('legacy admission status is invalid')
}
return state
}
async function applyLegacyStates(post, before, states, order) {
const expected = membershipWithStates(before.selector, states)
let changed = false
for (const cellId of order) {
const desired = states[cellId]
const current = await legacyCellState(post, cellId)
if (current === 'existing-only' && desired !== current) {
throw new Error(`legacy admission cannot re-enable existing-only cell ${cellId}`)
}
if (current === desired) continue
let cause
try {
await post('/v1/admin/cell-state', { v: 1, cellId, state: desired })
} catch (error) {
cause = error
}
if ((await legacyCellState(post, cellId)) !== desired) {
const detail = cause instanceof Error ? `: ${cause.message}` : ''
throw new Error(`legacy admission did not commit ${cellId} exactly${detail}`, { cause })
}
changed = true
}
const verified = await inspectAdmissionSelector(post)
if (verified.selector.generation !== 0 ||
!sameMembership(verified.selector.membership, expected)) {
throw new Error('legacy admission membership changed unexpectedly')
}
return { changed, selector: verified.selector }
}
export async function prepareCapacityCanary(config, overrides = {}) {
const fetchImpl = overrides.fetch ?? fetch
const token = overrides.token ?? process.env.ORCA_RELAY_ADMIN_ID_TOKEN
if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable')
const postAt = async (origin, path, body) =>
await responseJson(
await fetchImpl(`${origin}${path}`, {
method: 'POST',
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
body: JSON.stringify(body),
signal: AbortSignal.timeout(30_000)
}),
path
)
const post = async (path, body) => await postAt(config.directorOrigin, path, body)
const before = await inspectAdmissionSelector(post)
const state = selectorCellState(before.selector, config.cellId)
if (state === 'existing-only' && config.mode !== 'restore-fallback') {
throw new Error('capacity canary cannot restore existing-only admission')
}
const states =
config.mode === 'isolate'
? { [config.cellId]: 'migration-only' }
: config.mode === 'activate'
? Object.fromEntries([
...before.selector.membership.general.map((cellId) => [
cellId,
cellId === config.cellId ? 'general' : 'migration-only'
]),
[config.cellId, 'general']
])
: Object.fromEntries([
...config.restoreGeneralCellIds.map((cellId) => [cellId, 'general']),
...(config.mode === 'restore-fallback'
? [[config.cellId, state === 'existing-only' ? 'existing-only' : 'migration-only']]
: [])
])
const membership = membershipWithStates(before.selector, states)
const legacyOrder =
config.mode === 'activate'
? [config.cellId, ...before.selector.membership.general.filter((id) => id !== config.cellId)]
: config.mode === 'restore-fallback'
? [...config.restoreGeneralCellIds, config.cellId]
: Object.keys(states)
const result = before.selector.generation === 0
? await applyLegacyStates(post, before, states, legacyOrder)
: sameMembership(membership, before.selector.membership)
? { changed: false, selector: before.selector }
: await applyExactAdmissionSelector(post, membership, {
expectedCurrentSelector: before.selector
})
if (config.mode === 'isolate') {
await postAt(config.cellOrigin, '/v1/admin/drain', { v: 1, graceMs: 0 })
}
return {
changed: result.changed,
generation: result.selector.generation,
...(config.mode === 'isolate' ? { drained: true } : {})
}
}
export async function main(argv = process.argv.slice(2)) {
const config = parseArguments(argv)
const result = await prepareCapacityCanary(config)
process.stdout.write(
`${JSON.stringify({ event: 'relay_capacity_canary_admission', cellId: config.cellId, mode: config.mode, ...result })}\n`
)
}
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
main().catch((error) => {
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
process.exitCode = 1
})
}
@@ -0,0 +1,300 @@
import assert from 'node:assert/strict'
import { readFileSync } from 'node:fs'
import { test } from 'node:test'
import { relayWorkflowUrl } from './relay-repository.mjs'
import { prepareCapacityCanary } from './prepare-relay-capacity-canary.mjs'
function harness(initialState, options = {}) {
const {
generation = 4,
ambiguousCellState = false,
rejectCellState = false,
fallbackState = 'general',
extraGeneralCellIds = []
} = options
let selector = {
generation,
attemptId: 'initial',
membership: {
existingOnly: [
'staging-gce-c1',
...(initialState === 'existing-only' ? ['staging-gce-c3'] : [])
],
migrationOnly: [
...(fallbackState === 'migration-only' ? ['staging-gce-c2'] : []),
...(initialState === 'migration-only' ? ['staging-gce-c3'] : [])
],
general: [
...extraGeneralCellIds,
...(fallbackState === 'general' ? ['staging-gce-c2'] : []),
...(initialState === 'general' ? ['staging-gce-c3'] : [])
].sort()
}
}
let intent = null
let applies = 0
let drains = 0
const cellStateChanges = []
const fetch = async (url, options) => {
const path = new URL(url).pathname
const body = JSON.parse(options.body)
if (path === '/v1/admin/drain') {
assert.deepEqual(body, { v: 1, graceMs: 0 })
drains++
return Response.json({ ok: true })
}
if (path === '/v1/admin/cell-status') {
const state = selector.membership.existingOnly.includes(body.cellId)
? 'existing-only'
: selector.membership.migrationOnly.includes(body.cellId)
? 'migration-only'
: 'general'
return Response.json({ status: { cellId: body.cellId, admissionState: state } })
}
if (path === '/v1/admin/cell-state') {
assert.equal(selector.generation, 0)
if (rejectCellState) {
return Response.json({ error: 'invalid_token' }, { status: 401 })
}
const keys = {
'existing-only': 'existingOnly',
'migration-only': 'migrationOnly',
general: 'general'
}
for (const cells of Object.values(selector.membership)) {
const index = cells.indexOf(body.cellId)
if (index !== -1) cells.splice(index, 1)
}
selector.membership[keys[body.state]].push(body.cellId)
for (const cells of Object.values(selector.membership)) cells.sort()
cellStateChanges.push({ cellId: body.cellId, state: body.state })
if (ambiguousCellState) throw new Error('response lost')
return Response.json({ ok: true })
}
if (path.endsWith('/status')) return Response.json({ selector, intent })
applies++
selector = {
generation: body.expectedGeneration + 1,
attemptId: body.attemptId,
membership: body.membership
}
intent = {
attemptId: body.attemptId,
expectedGeneration: body.expectedGeneration,
intendedGeneration: selector.generation,
membership: selector.membership,
state: 'committed'
}
return Response.json({ changed: true, selector })
}
return {
fetch,
selector: () => selector,
applies: () => applies,
drains: () => drains,
cellStateChanges
}
}
const config = {
directorOrigin: 'https://relay.example.com',
cellOrigin: 'https://c3.relay.example.com',
cellId: 'staging-gce-c3',
mode: 'isolate',
restoreGeneralCellIds: []
}
test('isolates a general canary as migration-only', async () => {
const testHarness = harness('general')
assert.deepEqual(
await prepareCapacityCanary(config, { fetch: testHarness.fetch, token: 'masked' }),
{ changed: true, generation: 5, drained: true }
)
assert.deepEqual(testHarness.selector().membership.migrationOnly, [config.cellId])
assert.equal(testHarness.applies(), 1)
assert.equal(testHarness.drains(), 1)
})
test('activates the canary as the only general cell', async () => {
const testHarness = harness('migration-only')
assert.deepEqual(
await prepareCapacityCanary(
{ ...config, mode: 'activate' },
{ fetch: testHarness.fetch, token: 'masked' }
),
{ changed: true, generation: 5 }
)
assert.deepEqual(testHarness.selector().membership, {
existingOnly: ['staging-gce-c1'],
migrationOnly: ['staging-gce-c2'],
general: [config.cellId]
})
assert.equal(testHarness.applies(), 1)
})
test('restores the reviewed staging general membership', async () => {
const testHarness = harness('migration-only')
assert.deepEqual(
await prepareCapacityCanary(
{
...config,
mode: 'restore',
restoreGeneralCellIds: ['staging-gce-c2', config.cellId]
},
{ fetch: testHarness.fetch, token: 'masked' }
),
{ changed: true, generation: 5 }
)
assert.deepEqual(testHarness.selector().membership, {
existingOnly: ['staging-gce-c1'],
migrationOnly: [],
general: ['staging-gce-c2', config.cellId]
})
})
test('restores the fallback without promoting a possibly drained canary', async () => {
const testHarness = harness('general')
await prepareCapacityCanary(
{
...config,
mode: 'restore-fallback',
restoreGeneralCellIds: ['staging-gce-c2']
},
{ fetch: testHarness.fetch, token: 'masked' }
)
assert.deepEqual(testHarness.selector().membership, {
existingOnly: ['staging-gce-c1'],
migrationOnly: [config.cellId],
general: ['staging-gce-c2']
})
})
test('restores the fallback while preserving an irreversible canary', async () => {
const testHarness = harness('existing-only')
await prepareCapacityCanary(
{
...config,
mode: 'restore-fallback',
restoreGeneralCellIds: ['staging-gce-c2']
},
{ fetch: testHarness.fetch, token: 'masked' }
)
assert.deepEqual(testHarness.selector().membership, {
existingOnly: ['staging-gce-c1', config.cellId],
migrationOnly: [],
general: ['staging-gce-c2']
})
})
test('uses exact legacy admission writes before the selector boundary', async () => {
const testHarness = harness('general', { generation: 0 })
assert.deepEqual(
await prepareCapacityCanary(config, { fetch: testHarness.fetch, token: 'masked' }),
{ changed: true, generation: 0, drained: true }
)
assert.deepEqual(testHarness.cellStateChanges, [
{ cellId: config.cellId, state: 'migration-only' }
])
assert.equal(testHarness.drains(), 1)
})
test('promotes a legacy canary before demoting its fallback', async () => {
const testHarness = harness('migration-only', { generation: 0 })
await prepareCapacityCanary(
{ ...config, mode: 'activate' },
{ fetch: testHarness.fetch, token: 'masked' }
)
assert.deepEqual(testHarness.cellStateChanges, [
{ cellId: config.cellId, state: 'general' },
{ cellId: 'staging-gce-c2', state: 'migration-only' }
])
})
test('makes the canary sole general with the live legacy membership shape', async () => {
const extraGeneralCellIds = ['combined', 'staging-c1', 'staging-c2']
const testHarness = harness('migration-only', { generation: 0, extraGeneralCellIds })
const activate = { ...config, mode: 'activate' }
await prepareCapacityCanary(activate, { fetch: testHarness.fetch, token: 'masked' })
assert.deepEqual(testHarness.cellStateChanges, [
{ cellId: config.cellId, state: 'general' },
{ cellId: 'combined', state: 'migration-only' },
{ cellId: 'staging-c1', state: 'migration-only' },
{ cellId: 'staging-c2', state: 'migration-only' },
{ cellId: 'staging-gce-c2', state: 'migration-only' }
])
})
test('restores a legacy fallback before demoting the target', async () => {
const testHarness = harness('general', {
generation: 0,
fallbackState: 'migration-only'
})
await prepareCapacityCanary(
{
...config,
mode: 'restore-fallback',
restoreGeneralCellIds: ['staging-gce-c2']
},
{ fetch: testHarness.fetch, token: 'masked' }
)
assert.deepEqual(testHarness.cellStateChanges, [
{ cellId: 'staging-gce-c2', state: 'general' },
{ cellId: config.cellId, state: 'migration-only' }
])
})
test('keeps an already restored legacy fallback unchanged', async () => {
const testHarness = harness('migration-only', { generation: 0 })
assert.deepEqual(
await prepareCapacityCanary(
{
...config,
mode: 'restore-fallback',
restoreGeneralCellIds: ['staging-gce-c2']
},
{ fetch: testHarness.fetch, token: 'masked' }
),
{ changed: false, generation: 0 }
)
assert.deepEqual(testHarness.cellStateChanges, [])
assert.deepEqual(testHarness.selector().membership, {
existingOnly: ['staging-gce-c1'],
migrationOnly: [config.cellId],
general: ['staging-gce-c2']
})
})
test('recovers an ambiguous legacy admission response by exact readback', async () => {
const testHarness = harness('general', { generation: 0, ambiguousCellState: true })
await assert.doesNotReject(
prepareCapacityCanary(config, { fetch: testHarness.fetch, token: 'masked' })
)
assert.equal(testHarness.drains(), 1)
})
test('reports a rejected legacy admission write without draining', async () => {
const testHarness = harness('general', { generation: 0, rejectCellState: true })
const operation = prepareCapacityCanary(config, { fetch: testHarness.fetch, token: 'masked' })
await assert.rejects(operation, /cell-state returned 401/)
assert.equal(testHarness.drains(), 0)
})
test('the staging workflow supplies every required capacity transition argument', () => {
const workflow = readFileSync(
relayWorkflowUrl('prove-relay-staging-capacity.yml'),
'utf8'
)
const verifyCalls = workflow.match(
/node dev\/scripts\/verify-relay-capacity-transition\.mjs[\s\S]*?(?=\n\s*\n|\n\s*- name:)/g
)
assert.ok(verifyCalls?.length >= 5)
for (const call of verifyCalls) {
for (const flag of ['--cell-origin', '--heartbeat', '--admission', '--draining', '--activity']) {
assert.match(call, new RegExp(flag))
}
}
const isolate = workflow.match(
/node dev\/scripts\/prepare-relay-capacity-canary\.mjs[\s\S]*?--mode isolate/
)?.[0]
assert.match(isolate, /--cell-origin/)
})
@@ -0,0 +1,116 @@
import { pathToFileURL } from 'node:url'
import {
applyExactAdmissionSelector,
inspectAdmissionSelector,
membershipWithStates,
selectorCellState
} from './relay-admission-selector.mjs'
const DIRECTOR_ORIGIN = 'https://relay.onorca.dev'
export const PRODUCTION_CAPACITY_CELL_IDS = [
'production-gce-c7',
'production-gce-c8',
'production-gce-c9',
'production-gce-c10',
'production-gce-c13',
'production-gce-c14',
'production-gce-c15',
'production-gce-c16',
'production-gce-c19',
'production-gce-c20',
'production-gce-c21',
'production-gce-c22',
'production-gce-c23',
'production-gce-c24',
'production-gce-c25',
'production-gce-c26'
]
function cellOrigin(cellId) {
return `https://${cellId.slice('production-gce-'.length)}.relay.onorca.dev`
}
export function parseProductionCapacityCellArguments(argv) {
const values = {}
for (let index = 0; index < argv.length; index += 2) {
const key = argv[index]
const value = argv[index + 1]
if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments')
values[key.slice(2)] = value
}
if (!['isolate', 'drain', 'activate'].includes(values.mode)) {
throw new Error('--mode must be isolate, drain, or activate')
}
const cellId = values['cell-id']
if (!PRODUCTION_CAPACITY_CELL_IDS.includes(cellId)) {
throw new Error('production capacity target is not approved')
}
const expectedCellOrigin = cellOrigin(cellId)
if (
values['director-origin'] !== DIRECTOR_ORIGIN ||
values['cell-origin'] !== expectedCellOrigin
) {
throw new Error('production capacity target origin is not exact')
}
return {
directorOrigin: DIRECTOR_ORIGIN,
cellOrigin: expectedCellOrigin,
cellId,
mode: values.mode
}
}
async function responseJson(response, label) {
const body = await response.json().catch(() => ({}))
if (!response.ok) throw new Error(`${label} returned ${response.status}`)
return body
}
export async function prepareProductionCapacityCell(config, overrides = {}) {
const fetchImpl = overrides.fetch ?? fetch
const token = overrides.token ?? process.env.ORCA_RELAY_ADMIN_ID_TOKEN
if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable')
const postAt = async (origin, path, body) =>
await responseJson(
await fetchImpl(`${origin}${path}`, {
method: 'POST',
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
body: JSON.stringify(body),
signal: AbortSignal.timeout(30_000)
}),
path
)
const post = async (path, body) => await postAt(config.directorOrigin, path, body)
if (config.mode === 'drain') {
await postAt(config.cellOrigin, '/v1/admin/drain', { v: 1, graceMs: 0 })
return { changed: false, drained: true }
}
const before = await inspectAdmissionSelector(post)
const state = selectorCellState(before.selector, config.cellId)
if (state === 'existing-only') throw new Error('production capacity target is irreversible')
const desiredState = config.mode === 'isolate' ? 'migration-only' : 'general'
const membership = membershipWithStates(before.selector, { [config.cellId]: desiredState })
const result = await applyExactAdmissionSelector(post, membership, {
expectedCurrentSelector: before.selector
})
return {
changed: result.changed,
generation: result.selector.generation,
admissionState: desiredState
}
}
export async function main(argv = process.argv.slice(2)) {
const config = parseProductionCapacityCellArguments(argv)
const result = await prepareProductionCapacityCell(config)
process.stdout.write(
`${JSON.stringify({ event: 'relay_production_capacity_canary', cellId: config.cellId, mode: config.mode, ...result })}\n`
)
}
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
main().catch((error) => {
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
process.exitCode = 1
})
}
@@ -0,0 +1,173 @@
import assert from 'node:assert/strict'
import { describe, it } from 'node:test'
import {
parseProductionCapacityCellArguments,
prepareProductionCapacityCell,
PRODUCTION_CAPACITY_CELL_IDS
} from './prepare-relay-production-capacity-canary.mjs'
const config = {
directorOrigin: 'https://relay.onorca.dev',
cellOrigin: 'https://c26.relay.onorca.dev',
cellId: 'production-gce-c26'
}
const membership = {
existingOnly: ['production-gce-c1'],
migrationOnly: ['production-gce-c17'],
general: ['production-gce-c25', 'production-gce-c26']
}
function response(body, status = 200) {
return new Response(JSON.stringify(body), {
status,
headers: { 'content-type': 'application/json' }
})
}
function canaryFetch() {
let selector = { generation: 20, attemptId: null, membership }
const calls = []
const fetch = async (url, init) => {
const path = new URL(url).pathname
const body = JSON.parse(init.body)
calls.push({ path, body })
if (path === '/v1/admin/admission-selector/status') {
return response({
v: 1,
selector,
intent: body.attemptId
? {
attemptId: body.attemptId,
state: 'committed',
expectedGeneration: selector.generation - 1,
intendedGeneration: selector.generation,
membership: selector.membership
}
: null
})
}
if (path === '/v1/admin/admission-selector/apply') {
selector = {
generation: selector.generation + 1,
attemptId: body.attemptId,
membership: body.membership
}
return response({ v: 1, changed: true, selector })
}
if (path === '/v1/admin/drain') return response({ v: 1, draining: true })
throw new Error(`unexpected ${path}`)
}
return { calls, fetch, selector: () => selector }
}
describe('production Relay capacity cell admission', () => {
it('allows only the serving rollout cells', () => {
assert.deepEqual(PRODUCTION_CAPACITY_CELL_IDS, [
'production-gce-c7',
'production-gce-c8',
'production-gce-c9',
'production-gce-c10',
'production-gce-c13',
'production-gce-c14',
'production-gce-c15',
'production-gce-c16',
'production-gce-c19',
'production-gce-c20',
'production-gce-c21',
'production-gce-c22',
'production-gce-c23',
'production-gce-c24',
'production-gce-c25',
'production-gce-c26'
])
assert.deepEqual(parseProductionCapacityCellArguments([
'--director-origin', 'https://relay.onorca.dev',
'--cell-origin', 'https://c7.relay.onorca.dev',
'--cell-id', 'production-gce-c7',
'--mode', 'isolate'
]), {
directorOrigin: 'https://relay.onorca.dev',
cellOrigin: 'https://c7.relay.onorca.dev',
cellId: 'production-gce-c7',
mode: 'isolate'
})
assert.throws(() => parseProductionCapacityCellArguments([
'--director-origin', 'https://relay.onorca.dev',
'--cell-origin', 'https://c17.relay.onorca.dev',
'--cell-id', 'production-gce-c17',
'--mode', 'isolate'
]), /not approved/)
assert.throws(() => parseProductionCapacityCellArguments([
'--director-origin', 'https://relay.onorca.dev',
'--cell-origin', 'https://c8.relay.onorca.dev',
'--cell-id', 'production-gce-c7',
'--mode', 'isolate'
]), /origin is not exact/)
})
it('isolates only the selected cell without depending on its runtime', async () => {
const fake = canaryFetch()
const result = await prepareProductionCapacityCell(
{ ...config, mode: 'isolate' },
{ fetch: fake.fetch, token: 'token' }
)
assert.equal(result.admissionState, 'migration-only')
assert.deepEqual(fake.selector().membership, {
existingOnly: ['production-gce-c1'],
migrationOnly: ['production-gce-c17', 'production-gce-c26'],
general: ['production-gce-c25']
})
assert.doesNotMatch(fake.calls.map(({ path }) => path).join(','), /\/v1\/admin\/drain/)
})
it('drains the selected cell independently after durable isolation', async () => {
const fake = canaryFetch()
const result = await prepareProductionCapacityCell(
{ ...config, mode: 'drain' },
{ fetch: fake.fetch, token: 'token' }
)
assert.deepEqual(result, { changed: false, drained: true })
assert.deepEqual(fake.calls, [{
path: '/v1/admin/drain',
body: { v: 1, graceMs: 0 }
}])
})
it('restores only the selected cell to general admission', async () => {
const fake = canaryFetch()
await prepareProductionCapacityCell(
{ ...config, mode: 'isolate' },
{ fetch: fake.fetch, token: 'token' }
)
const result = await prepareProductionCapacityCell(
{ ...config, mode: 'activate' },
{ fetch: fake.fetch, token: 'token' }
)
assert.equal(result.admissionState, 'general')
assert.deepEqual(fake.selector().membership, membership)
})
it('refuses an irreversible existing-only target', async () => {
const fetch = async () => response({
v: 1,
selector: {
generation: 20,
attemptId: null,
membership: {
existingOnly: ['production-gce-c26'],
migrationOnly: ['production-gce-c17'],
general: ['production-gce-c25']
}
},
intent: null
})
await assert.rejects(
prepareProductionCapacityCell(
{ ...config, mode: 'isolate' },
{ fetch, token: 'token' }
),
/irreversible/
)
})
})
@@ -0,0 +1,73 @@
import { randomBytes } from 'node:crypto'
import { pathToFileURL } from 'node:url'
const WRONG_CELL = 4409
const DRAINING = 4503
function once(socket, event, listener) {
if (typeof socket.once === 'function') {
socket.once(event, listener)
return
}
if (typeof socket.addEventListener !== 'function') {
throw new Error('WebSocket event API is unavailable')
}
socket.addEventListener(event, (value) => {
if (event === 'close') listener(value.code)
else if (event === 'error') listener(value.error ?? new Error(value.message))
else listener()
}, { once: true })
}
export function parseLegacyAdmissionProbeArguments(argv) {
if (argv.length !== 2 || argv[0] !== '--cell-origin') throw new Error('invalid arguments')
const origin = new URL(argv[1])
if (origin.protocol !== 'https:' || origin.origin !== argv[1]) {
throw new Error('--cell-origin must be a canonical HTTPS origin')
}
return { cellOrigin: origin.origin }
}
export async function probeLegacyAdmission(config, overrides = {}) {
const Socket = overrides.WebSocket ?? globalThis.WebSocket
if (typeof Socket !== 'function') throw new Error('WebSocket is unavailable')
const random = overrides.randomBytes ?? randomBytes
const timeoutMs = overrides.timeoutMs ?? 15_000
const hostId = random(12).toString('base64url')
const credential = random(32).toString('base64url')
const url = `${config.cellOrigin.replace('https://', 'wss://')}/v1/connect/${hostId}`
await new Promise((resolve, reject) => {
const socket = new Socket(url)
const timer = setTimeout(() => {
if (typeof socket.terminate === 'function') socket.terminate()
else socket.close()
reject(new Error('legacy admission probe timed out'))
}, timeoutMs)
once(socket, 'open', () => {
socket.send(JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential }))
})
once(socket, 'close', (code) => {
clearTimeout(timer)
if (code === WRONG_CELL) resolve()
else if (code === DRAINING) reject(new Error('legacy cell is draining'))
else reject(new Error(`legacy admission probe closed with ${code}`))
})
once(socket, 'error', (error) => {
clearTimeout(timer)
reject(new Error(`legacy admission probe failed: ${error.message}`))
})
})
return { accepting: true }
}
export async function main(argv = process.argv.slice(2)) {
const result = await probeLegacyAdmission(parseLegacyAdmissionProbeArguments(argv))
process.stdout.write(`${JSON.stringify({ event: 'relay_legacy_admission_verified', ...result })}\n`)
}
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
main().catch((error) => {
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
process.exitCode = 1
})
}
@@ -0,0 +1,89 @@
import assert from 'node:assert/strict'
import { EventEmitter } from 'node:events'
import { test } from 'node:test'
import {
parseLegacyAdmissionProbeArguments,
probeLegacyAdmission
} from './probe-relay-legacy-admission.mjs'
function socketClosingWith(code, observed) {
return class extends EventEmitter {
constructor(url) {
super()
observed.url = url
queueMicrotask(() => this.emit('open'))
}
send(payload) {
observed.payload = JSON.parse(payload)
queueMicrotask(() => this.emit('close', code))
}
terminate() {}
}
}
function nativeSocketClosingWith(code) {
return class extends EventTarget {
constructor() {
super()
queueMicrotask(() => this.dispatchEvent(new Event('open')))
}
send() {
const event = new Event('close')
Object.defineProperty(event, 'code', { value: code })
queueMicrotask(() => this.dispatchEvent(event))
}
close() {}
}
}
const config = { cellOrigin: 'https://c2.relay.example.com' }
const random = (length) => Buffer.alloc(length, length)
test('accepts only a canonical cell origin', () => {
assert.deepEqual(
parseLegacyAdmissionProbeArguments(['--cell-origin', config.cellOrigin]),
config
)
assert.throws(
() => parseLegacyAdmissionProbeArguments(['--cell-origin', `${config.cellOrigin}/path`]),
/canonical/
)
})
test('proves admission with a synthetic invalid credential and exposes no identifier', async () => {
const observed = {}
assert.deepEqual(
await probeLegacyAdmission(config, {
WebSocket: socketClosingWith(4409, observed),
randomBytes: random
}),
{ accepting: true }
)
assert.match(observed.url, /^wss:\/\/c2\.relay\.example\.com\/v1\/connect\/[A-Za-z0-9_-]{16}$/)
assert.deepEqual(Object.keys(observed.payload).sort(), ['credential', 'mode', 'type', 'v'])
})
test('uses the dependency-free Node WebSocket event API', async () => {
await assert.doesNotReject(
probeLegacyAdmission(config, {
WebSocket: nativeSocketClosingWith(4409),
randomBytes: random
})
)
})
test('rejects the legacy draining close and any unknown outcome', async () => {
for (const [code, message] of [[4503, /draining/], [4401, /closed with 4401/]]) {
await assert.rejects(
probeLegacyAdmission(config, {
WebSocket: socketClosingWith(code, {}),
randomBytes: random
}),
message
)
}
})
@@ -0,0 +1,80 @@
import { pathToFileURL } from 'node:url'
const PRODUCTION_CELL = /^production-gce-c(?:7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26)$/
const DIRECTOR_ORIGIN = 'https://relay.onorca.dev'
export function parseRehomeTrustProbeArguments(argv, environment = process.env) {
const values = {}
for (let index = 0; index < argv.length; index += 2) {
const key = argv[index]
const value = argv[index + 1]
if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments')
values[key.slice(2)] = value
}
for (const key of ['director-origin', 'cell-id', 'cell-incarnation']) {
if (!values[key]) throw new Error(`missing --${key}`)
}
if (values['director-origin'] !== DIRECTOR_ORIGIN) {
throw new Error('--director-origin must be the production Relay origin')
}
if (!PRODUCTION_CELL.test(values['cell-id'])) throw new Error('--cell-id is not approved')
if (!/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(
values['cell-incarnation']
)) throw new Error('--cell-incarnation is invalid')
const token = environment.ORCA_RELAY_ADMIN_ID_TOKEN
if (!token || token.length > 8_192 || !/^[^.]+\.[^.]+\.[^.]+$/.test(token)) {
throw new Error('admin identity token is unavailable')
}
return {
directorOrigin: DIRECTOR_ORIGIN,
cellId: values['cell-id'],
cellIncarnation: values['cell-incarnation'],
token
}
}
export async function probeRehomeTrust(config, dependencies = {}) {
const fetchImpl = dependencies.fetch ?? fetch
const response = await fetchImpl(
`${config.directorOrigin}/v1/admin/regional-rehome-trust-probe`,
{
method: 'POST',
headers: {
authorization: `Bearer ${config.token}`,
'content-type': 'application/json'
},
body: JSON.stringify({
v: 1,
sourceCellId: config.cellId,
sourceCellIncarnation: config.cellIncarnation
}),
signal: AbortSignal.timeout(30_000)
}
)
const body = await response.json().catch(() => ({}))
if (!response.ok) {
throw new Error(`application-mediated rehome trust probe returned ${response.status}`)
}
if (
body.v !== 1 ||
body.dedicatedIdentity?.firstOutcome !== 'host-not-connected' ||
body.dedicatedIdentity?.secondOutcome !== 'host-not-connected' ||
body.dedicatedIdentity?.accepted !== true ||
body.dedicatedIdentity?.idempotent !== true ||
body.sharedRuntimeIdentityRejected !== true ||
body.proven !== true
) throw new Error('application-mediated rehome trust proof is incomplete')
return body
}
export async function main(argv = process.argv.slice(2)) {
const result = await probeRehomeTrust(parseRehomeTrustProbeArguments(argv))
process.stdout.write(`${JSON.stringify({ event: 'relay_rehome_trust_verified', ...result })}\n`)
}
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
main().catch((error) => {
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
process.exitCode = 1
})
}
@@ -0,0 +1,70 @@
import assert from 'node:assert/strict'
import { test } from 'node:test'
import {
parseRehomeTrustProbeArguments,
probeRehomeTrust
} from './probe-relay-rehome-trust.mjs'
const argv = [
'--director-origin', 'https://relay.onorca.dev',
'--cell-id', 'production-gce-c7',
'--cell-incarnation', '11111111-1111-4111-8111-111111111111'
]
const environment = { ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }
test('binds the application-mediated probe to an exact approved cell incarnation', () => {
assert.equal(parseRehomeTrustProbeArguments(argv, environment).cellId, 'production-gce-c7')
assert.throws(() => parseRehomeTrustProbeArguments(
argv.with(1, 'https://other.example.test'),
environment
))
assert.throws(() => parseRehomeTrustProbeArguments(argv, {
ORCA_RELAY_ADMIN_ID_TOKEN: 'not-a-token'
}))
})
test('requires complete aggregate application-mediated trust proof', async () => {
const config = parseRehomeTrustProbeArguments(argv, environment)
const result = await probeRehomeTrust(config, {
fetch: async (url, init) => {
assert.equal(url, 'https://relay.onorca.dev/v1/admin/regional-rehome-trust-probe')
assert.deepEqual(JSON.parse(init.body), {
v: 1,
sourceCellId: 'production-gce-c7',
sourceCellIncarnation: '11111111-1111-4111-8111-111111111111'
})
return Response.json({
v: 1,
dedicatedIdentity: {
firstOutcome: 'host-not-connected',
secondOutcome: 'host-not-connected',
accepted: true,
idempotent: true
},
sharedRuntimeIdentityRejected: true,
proven: true
})
}
})
assert.equal(result.proven, true)
})
test('rejects partial or mismatched proof', async () => {
const config = parseRehomeTrustProbeArguments(argv, environment)
await assert.rejects(
probeRehomeTrust(config, {
fetch: async () => Response.json({
v: 1,
dedicatedIdentity: {
firstOutcome: 'host-not-connected',
secondOutcome: 'host-not-connected',
accepted: true,
idempotent: true
},
sharedRuntimeIdentityRejected: false,
proven: false
})
}),
/incomplete/
)
})
@@ -0,0 +1,85 @@
import assert from 'node:assert/strict'
import { readFileSync } from 'node:fs'
import test from 'node:test'
import { PRODUCTION_CAPACITY_CELL_IDS } from './prepare-relay-production-capacity-canary.mjs'
import { readRelayWorkflow } from './relay-repository.mjs'
const production = source('infra/terraform/environments/production.tfvars')
const dispatchWorkflow = readRelayWorkflow('deploy-relay-production-capacity.yml')
const jobWorkflow = readRelayWorkflow('deploy-relay-production-capacity-job.yml')
const RELAY_REPOSITORY = 'us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay'
function source(path) {
return readFileSync(new URL(`../../${path}`, import.meta.url), 'utf8')
}
// Slice each "<cell-id>" = { ... } entry out of relay_gce_cells.
function productionCells() {
const block = production.slice(production.indexOf('relay_gce_cells = {'))
const cells = new Map()
for (const match of block.matchAll(/"(production-gce-c\d+)" = \{([\s\S]*?)\n {2}\}/g)) {
cells.set(match[1], match[2])
}
assert.ok(cells.size > 0, 'relay_gce_cells parsed empty')
return cells
}
function hardCap(body) {
const match = body.match(/connection_hard_cap\s*=\s*(\d+)/)
return match ? Number(match[1]) : undefined
}
function imageDigest(body) {
const match = body.match(/^\s*image\s*=\s*"([^"]+)"/m)
assert.ok(match, 'cell entry has no image')
const [repository, digest] = match[1].split('@')
assert.equal(repository, RELAY_REPOSITORY)
assert.match(digest, /^sha256:[0-9a-f]{64}$/)
return digest
}
function workflowPin(workflow, name) {
const match = workflow.match(new RegExp(`${name}: (sha256:[0-9a-f]{64})`))
assert.ok(match, `${name} is missing or not a full digest`)
return match[1]
}
test('every production cell pins a full relay image digest', () => {
for (const [cellId, body] of productionCells()) {
assert.match(imageDigest(body), /^sha256:[0-9a-f]{64}$/, `${cellId} image digest`)
}
})
test('the 1,000-cap cells are exactly the canonical capacity set', () => {
const thousandCap = [...productionCells()]
.filter(([, body]) => hardCap(body) === 1000)
.map(([cellId]) => cellId)
assert.deepEqual([...thousandCap].sort(), [...PRODUCTION_CAPACITY_CELL_IDS].sort())
})
test('the 1,000-cap cells all serve one image digest', () => {
const digests = new Map()
for (const [cellId, body] of productionCells()) {
if (hardCap(body) !== 1000) continue
const digest = imageDigest(body)
if (!digests.has(digest)) digests.set(digest, [])
digests.get(digest).push(cellId)
}
assert.equal(
digests.size,
1,
`1,000-cap cells split across digests: ${JSON.stringify([...digests])}`
)
assert.equal([...digests.values()][0].length, PRODUCTION_CAPACITY_CELL_IDS.length)
})
// COMPATIBLE_CELL_IMAGE_DIGEST is one half of a reviewed (director, cell) skew pair, not a
// claim about what the fleet serves; it is re-derived by hand for each capacity wave. So it
// is deliberately NOT tied to the tfvars digest — only to its twin in the dispatch workflow.
test('both capacity workflows declare the same reviewed image pins', () => {
for (const name of ['PREDECESSOR_IMAGE_DIGEST', 'COMPATIBLE_CELL_IMAGE_DIGEST']) {
assert.equal(workflowPin(dispatchWorkflow, name), workflowPin(jobWorkflow, name), name)
}
workflowPin(jobWorkflow, 'COMPATIBLE_DIRECTOR_IMAGE_DIGEST')
})
@@ -0,0 +1,210 @@
import assert from 'node:assert/strict'
import { readFileSync } from 'node:fs'
import { test } from 'node:test'
import {
LEASED_WORKFLOWS,
LOCK_GROUPS,
NOT_A_CLOUD_SQL_CANDIDATE,
PRODUCTION_LEASE,
SELECTABLE_LEASE,
STAGING_LEASE,
concurrencyBlocks,
entrypointsFor,
jobIf,
jobNeeds,
jobs,
leaseSteps,
leaseStepsByJob,
mutatesSharedInstance,
readWorkflow,
reusableCalls,
revisionMintingScripts,
workflowFiles
} from './cloud-sql-rollout-lock-census.mjs'
import { relayWorkflowFile } from './relay-repository.mjs'
const expectedLease = { production: PRODUCTION_LEASE, staging: STAGING_LEASE, selectable: SELECTABLE_LEASE }
const leasedFiles = Object.keys(LEASED_WORKFLOWS)
function contractFiles(file) {
return [file, ...(LEASED_WORKFLOWS[file].leaseFiles ?? [])]
}
test('every locked workflow declares exactly one lock group that never cancels', () => {
for (const file of leasedFiles) {
const blocks = concurrencyBlocks(readWorkflow(file))
assert.equal(blocks.length, 1, `${file} must declare exactly one concurrency block`)
assert.equal(blocks[0].group, LEASED_WORKFLOWS[file].group, file)
assert.equal(blocks[0].cancelInProgress, 'false', file)
}
})
test('every locked workflow takes the lease for its own environment', () => {
for (const file of leasedFiles) {
const lease = expectedLease[LEASED_WORKFLOWS[file].env]
const steps = contractFiles(file).flatMap((member) => leaseSteps(readWorkflow(member)))
assert.ok(steps.length > 0, `${file} must use the Cloud SQL rollout lease action`)
for (const step of steps) {
assert.equal(step.bucket, lease.bucket, file)
assert.equal(step.object, lease.object, file)
}
}
})
test('the lease step runs after the credential that authorizes it', () => {
for (const file of leasedFiles) {
for (const member of contractFiles(file)) {
const text = readWorkflow(member)
const lines = text.split('\n')
for (const step of leaseSteps(text)) {
const before = lines.slice(0, step.line - 1)
const gcloud = before.lastIndexOf(' - uses: google-github-actions/setup-gcloud@v2')
assert.notEqual(gcloud, -1, `${member}: lease step at line ${step.line} has no setup-gcloud before it`)
assert.ok(
before.lastIndexOf(' - uses: actions/checkout@v4') !== -1,
`${member}: lease step at line ${step.line} runs before the local action is checked out`
)
}
}
}
})
test('multi-wave workflows hold one lease per run and free it exactly once', () => {
for (const file of leasedFiles) {
const entry = LEASED_WORKFLOWS[file]
const waveFiles = entry.leaseFiles ?? []
const callCount = waveFiles.reduce(
(total, member) => total + (reusableCalls(readWorkflow(file)).get(member) ?? 0),
0
)
if (!entry.reentrant) {
assert.ok(callCount <= 1, `${file} calls a leased reusable job ${callCount} times; it needs a release job`)
for (const member of contractFiles(file)) {
for (const step of leaseSteps(readWorkflow(member))) {
assert.equal(step.release, undefined, `${member} must leave release at its default`)
}
}
continue
}
assert.ok(callCount > 1, `${file} no longer calls its reusable job more than once`)
const steps = contractFiles(file).flatMap((member) => leaseSteps(readWorkflow(member)))
const released = steps.filter((step) => step.release === "'true'")
assert.equal(released.length, 1, `${file} must free the run lease exactly once`)
for (const step of steps) {
if (step === released[0]) continue
assert.equal(step.release, "'false'", `${file} wave jobs must hold the lease`)
}
const callerJobs = jobs(readWorkflow(file))
const releaseJob = leaseStepsByJob(file).find((job) =>
job.steps.some((step) => step.release === "'true'")
)
assert.ok(releaseJob, `${file} must free the lease from its own job`)
const guard = jobIf(callerJobs.find((job) => job.id === releaseJob.id).text)
assert.match(guard, /always\(\)/, `${file}: the release job must run on failure and cancellation`)
const holders = callerJobs
.filter(
(job) =>
job.id !== releaseJob.id &&
(waveFiles.some((member) => job.text.includes(`uses: ./.github/workflows/${member}`)) ||
leaseStepsByJob(file).find((entry) => entry.id === job.id)?.steps.length > 0)
)
.map((job) => job.id)
const needs = jobNeeds(callerJobs.find((job) => job.id === releaseJob.id).text)
for (const holder of holders) {
assert.ok(needs.includes(holder), `${file}: the release job must need ${holder}`)
}
}
})
test('workflows with two lease-holding jobs can never run them together', () => {
for (const file of leasedFiles) {
const entry = LEASED_WORKFLOWS[file]
if (entry.reentrant) continue
const holding = leaseStepsByJob(file).filter((job) => job.steps.length > 0)
if (holding.length <= 1) continue
assert.ok(entry.exclusiveBy, `${file} has ${holding.length} lease-holding jobs and no exclusivity guard`)
const guards = holding.map((job) => jobIf(jobs(readWorkflow(file)).find((j) => j.id === job.id).text))
assert.equal(
guards.filter((guard) => guard.includes(entry.exclusiveBy)).length,
1,
`${file}: exactly one job may run when ${entry.exclusiveBy}`
)
assert.equal(
guards.filter((guard) => guard.includes(entry.exclusiveBy.replace('==', '!='))).length,
guards.length - 1,
`${file}: every other lease-holding job must be excluded when ${entry.exclusiveBy}`
)
}
})
test('census: no workflow rolls out against the shared instance outside the lease', () => {
const minters = revisionMintingScripts()
assert.ok(minters.size > 0, 'the revision-minting script scan found nothing and is vacuous')
const flagged = new Map()
for (const file of workflowFiles()) {
const reason = mutatesSharedInstance(readWorkflow(file), minters)
if (reason) flagged.set(file, reason)
}
assert.ok(flagged.size > 0, 'the rollout census found no candidates and is vacuous')
for (const [file, reason] of flagged) {
for (const entrypoint of entrypointsFor(file)) {
assert.ok(
entrypoint in LEASED_WORKFLOWS || entrypoint in NOT_A_CLOUD_SQL_CANDIDATE,
`${entrypoint} reaches ${file} (${reason}) but is neither leased nor recorded as a non-candidate`
)
}
}
for (const file of workflowFiles()) {
const groups = concurrencyBlocks(readWorkflow(file)).map((block) => block.group)
if (!groups.some((group) => LOCK_GROUPS.has(group))) continue
assert.ok(
file in LEASED_WORKFLOWS || file in NOT_A_CLOUD_SQL_CANDIDATE,
`${file} sits in a Cloud SQL lock group but is neither leased nor recorded as a non-candidate`
)
}
for (const [file, reason] of Object.entries(NOT_A_CLOUD_SQL_CANDIDATE)) {
assert.ok(typeof reason === 'string' && reason.length > 40, `${file} needs a real reason`)
const groups = concurrencyBlocks(readWorkflow(file)).map((block) => block.group)
assert.ok(
flagged.has(file) || groups.some((group) => LOCK_GROUPS.has(group)),
`${file} is recorded as a non-candidate but nothing would have flagged it`
)
}
for (const file of leasedFiles) {
assert.doesNotThrow(() => readWorkflow(file), `${file} is leased but does not exist`)
assert.ok(!(file in NOT_A_CLOUD_SQL_CANDIDATE), `${file} cannot be both leased and a non-candidate`)
}
})
// The API and auth deploy scripts share this contract but stay in the private repository.
const serviceCapScripts = ['dev/scripts/deploy-relay-blue-green.mjs']
test('budgets tagged Cloud Run candidates outside the service-wide instance cap', () => {
for (const file of serviceCapScripts) {
const script = readFileSync(new URL(`../../${file}`, import.meta.url), 'utf8')
assert.match(script, /'--no-traffic'/, file)
assert.match(script, /'--max'/, file)
}
const budget = readFileSync(
new URL('../../dev/scripts/relay-cloud-sql-connection-budget.mjs', import.meta.url),
'utf8'
)
assert.match(budget, /directly addressable tagged revisions outside service-level caps/)
assert.match(
budget,
/apiCandidate: retainedDirectorRollback \+ inputs\.apiInstances \* inputs\.apiPoolMax/
)
const director = readWorkflow(relayWorkflowFile('deploy-relay-production-director.yml'))
const capacity = readWorkflow(relayWorkflowFile('deploy-relay-production-capacity-job.yml'))
const asia = readWorkflow(relayWorkflowFile('operate-relay-asia-admission.yml'))
assert.match(director, /--max-instances "\$\{DIRECTOR_MAX_INSTANCES\}"/)
assert.match(capacity, /--max-instances 5/)
assert.match(asia, /--max-instances "\$\{DIRECTOR_MAX_INSTANCES\}"/)
})
@@ -0,0 +1,31 @@
import { readFileSync } from 'node:fs'
import { pathToFileURL } from 'node:url'
const CAPACITY_IDENTITY_NAME = 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT'
export function readProductionCapacityIdentity(revision) {
const env = revision?.spec?.containers?.[0]?.env
if (!Array.isArray(env)) throw new Error('director revision environment is missing')
const matches = env.filter((entry) => entry?.name === CAPACITY_IDENTITY_NAME)
if (matches.length === 0) return null
if (matches.length !== 1) throw new Error('duplicate capacity identity')
const value = matches[0]?.value
if (typeof value !== 'string' || value.length === 0) {
throw new Error('capacity identity is not a literal string')
}
return value
}
export function main() {
const revision = JSON.parse(readFileSync(0, 'utf8'))
process.stdout.write(`${JSON.stringify(readProductionCapacityIdentity(revision))}\n`)
}
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
try {
main()
} catch (error) {
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
process.exitCode = 1
}
}
@@ -0,0 +1,44 @@
import assert from 'node:assert/strict'
import test from 'node:test'
import { readProductionCapacityIdentity } from './read-relay-production-capacity-identity.mjs'
function revision(env) {
return { spec: { containers: [{ env }] } }
}
test('reads absent, exact, and foreign literal capacity identities', () => {
assert.equal(readProductionCapacityIdentity(revision([])), null)
assert.equal(
readProductionCapacityIdentity(revision([
{ name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', value: 'capacity@example.test' }
])),
'capacity@example.test'
)
assert.equal(
readProductionCapacityIdentity(revision([
{ name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', value: 'foreign@example.test' }
])),
'foreign@example.test'
)
})
test('rejects malformed or duplicate capacity identity entries', () => {
for (const entry of [
{ name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', value: null },
{ name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', value: '' },
{ name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', valueSource: { secretKeyRef: {} } }
]) {
assert.throws(
() => readProductionCapacityIdentity(revision([entry])),
/not a literal string/
)
}
assert.throws(
() => readProductionCapacityIdentity(revision([
{ name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', value: 'one@example.test' },
{ name: 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT', value: 'two@example.test' }
])),
/duplicate capacity identity/
)
assert.throws(() => readProductionCapacityIdentity({}), /environment is missing/)
})
@@ -0,0 +1,106 @@
import { spawnSync } from 'node:child_process'
import { fileURLToPath } from 'node:url'
const SECRET = 'orca-cloud-relay-regional-placement-enabled'
function validate(input) {
for (const key of ['project', 'region', 'service', 'bootstrap_version']) {
if (typeof input?.[key] !== 'string' || !input[key] || /[\r\n]/.test(input[key])) {
throw new Error(`invalid ${key}`)
}
}
if (!/^[a-z][a-z0-9-]{0,62}$/.test(input.service)) throw new Error('invalid service')
if (!/^[1-9][0-9]*$/.test(input.bootstrap_version)) {
throw new Error('invalid bootstrap_version')
}
}
export function classifyRelayServiceDescribeFailure(args, stderr) {
const serviceDescribe = args[0] === 'run' && args[1] === 'services' && args[2] === 'describe'
if (serviceDescribe && (stderr.includes('NOT_FOUND') || /Cannot find service \[[^\]\r\n]+\]/.test(stderr))) {
return 'NOT_FOUND'
}
return 'GCLOUD_FAILED'
}
function defaultRun(args) {
const result = spawnSync('gcloud', args, {
encoding: 'utf8',
stdio: ['ignore', 'pipe', 'pipe'],
maxBuffer: 10 * 1024 * 1024
})
if (result.status !== 0) {
const error = new Error('gcloud read failed')
error.code = classifyRelayServiceDescribeFailure(args, result.stderr)
throw error
}
return JSON.parse(result.stdout)
}
function gcloudArguments(kind, input, revision) {
return [
'run', kind, 'describe', revision ?? input.service,
'--project', input.project,
'--region', input.region,
'--format=json'
]
}
export function readRelayServingRegionalPlacementVersion(input, dependencies = {}) {
validate(input)
const run = dependencies.run ?? defaultRun
let service
try {
service = run(gcloudArguments('services', input))
} catch (error) {
if (error?.code === 'NOT_FOUND') return { version: input.bootstrap_version }
throw error
}
const serving = (service.status?.traffic ?? []).filter(
(entry) => Number(entry.percent ?? 0) > 0
)
if (
serving.length !== 1 ||
Number(serving[0].percent) !== 100 ||
typeof serving[0].revisionName !== 'string'
) {
throw new Error('Relay director must have exactly one revision serving 100% traffic')
}
const revision = run(gcloudArguments('revisions', input, serving[0].revisionName))
const references = (revision.spec?.containers ?? []).flatMap((container) =>
(container.env ?? []).filter(
(environment) => environment.name === 'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED'
)
)
if (references.length === 0) return { version: input.bootstrap_version }
const reference = normalizeSecretReference(references[0])
if (
references.length !== 1 ||
reference?.secret !== SECRET ||
!/^[1-9][0-9]*$/.test(reference?.version ?? '')
) {
throw new Error('serving regional placement secret reference is invalid')
}
return { version: reference.version }
}
// Why: the v2 API reports `valueSource.secretKeyRef.{secret,version}`, but
// `gcloud run revisions describe --format=json` emits the Knative v1 shape
// `valueFrom.secretKeyRef.{name,key}`, where `name` may be a full resource path.
// A `key` of "latest" is deliberately left invalid: the director's serving
// version must be a pinned integer for this data source to mean anything.
export function normalizeSecretReference(environment) {
const v2 = environment?.valueSource?.secretKeyRef
if (v2) return { secret: v2.secret, version: v2.version }
const v1 = environment?.valueFrom?.secretKeyRef
if (!v1) return undefined
const secret = typeof v1.name === 'string' ? v1.name.replace(/^projects\/[^/]+\/secrets\//, '') : undefined
return { secret, version: v1.key }
}
if (process.argv[1] === fileURLToPath(import.meta.url)) {
const chunks = []
for await (const chunk of process.stdin) chunks.push(chunk)
const input = JSON.parse(Buffer.concat(chunks).toString('utf8'))
process.stdout.write(`${JSON.stringify(readRelayServingRegionalPlacementVersion(input))}\n`)
}
@@ -0,0 +1,138 @@
import assert from 'node:assert/strict'
import test from 'node:test'
import {
classifyRelayServiceDescribeFailure,
readRelayServingRegionalPlacementVersion
} from './read-relay-serving-regional-placement-version.mjs'
const input = {
project: 'onorca-cloud',
region: 'us-central1',
service: 'orca-cloud-relay',
bootstrap_version: '7'
}
function revision(version = '11') {
return {
spec: {
containers: [{
env: [{
name: 'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED',
valueSource: {
secretKeyRef: {
secret: 'orca-cloud-relay-regional-placement-enabled',
version
}
}
}]
}]
}
}
}
// Why: `gcloud run revisions describe --format=json` emits the Knative v1 shape, where the
// secret lives in `name` and the version in `key`, and `name` may be the full resource path.
function v1Revision(name, key) {
return {
spec: {
containers: [{
env: [{
name: 'ORCA_RELAY_REGIONAL_PLACEMENT_ENABLED',
valueFrom: { secretKeyRef: { name, key } }
}]
}]
}
}
}
function serving() {
return { status: { traffic: [{ revisionName: 'relay-serving', percent: 100 }] } }
}
test('reads the exact version from the sole traffic-serving revision', () => {
const calls = []
const result = readRelayServingRegionalPlacementVersion(input, {
run: (args) => {
calls.push(args)
return calls.length === 1
? {
status: {
traffic: [
{ revisionName: 'relay-failed-latest', tag: 'candidate' },
{ revisionName: 'relay-serving', percent: 100 }
]
}
}
: revision()
}
})
assert.deepEqual(result, { version: '11' })
assert.equal(calls[1][3], 'relay-serving')
})
test('reads the gcloud v1 secret reference shape by bare id and by full resource path', () => {
for (const name of [
'orca-cloud-relay-regional-placement-enabled',
'projects/120364513935/secrets/orca-cloud-relay-regional-placement-enabled'
]) {
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
run: (args) => args[1] === 'services' ? serving() : v1Revision(name, '1')
}), { version: '1' })
}
})
test('rejects a v1 reference that names another secret or a floating version', () => {
assert.throws(() => readRelayServingRegionalPlacementVersion(input, {
run: (args) => args[1] === 'services'
? serving()
: v1Revision('projects/120364513935/secrets/some-other-secret', '1')
}), /secret reference is invalid/)
assert.throws(() => readRelayServingRegionalPlacementVersion(input, {
run: (args) => args[1] === 'services'
? serving()
: v1Revision('orca-cloud-relay-regional-placement-enabled', 'latest')
}), /secret reference is invalid/)
})
test('falls back only when the service or setting is absent', () => {
const notFound = new Error('not found')
notFound.code = 'NOT_FOUND'
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
run: () => { throw notFound }
}), { version: '7' })
assert.deepEqual(readRelayServingRegionalPlacementVersion(input, {
run: (args) => args[1] === 'services'
? { status: { traffic: [{ revisionName: 'relay-serving', percent: 100 }] } }
: { spec: { containers: [{ env: [] }] } }
}), { version: '7' })
})
test('classifies real absent-service stderr without weakening revision failures', () => {
const serviceArgs = ['run', 'services', 'describe', 'missing-service']
const stderr = 'ERROR: (gcloud.run.services.describe) Cannot find service [missing-service]'
assert.equal(classifyRelayServiceDescribeFailure(serviceArgs, stderr), 'NOT_FOUND')
assert.equal(
classifyRelayServiceDescribeFailure(['run', 'revisions', 'describe', 'missing-revision'], stderr),
'GCLOUD_FAILED'
)
assert.equal(classifyRelayServiceDescribeFailure(serviceArgs, 'PERMISSION_DENIED'), 'GCLOUD_FAILED')
})
test('rejects ambiguous traffic, malformed references, and read failures', () => {
assert.throws(() => readRelayServingRegionalPlacementVersion(input, {
run: () => ({
status: { traffic: [{ revisionName: 'a', percent: 50 }, { revisionName: 'b', percent: 50 }] }
})
}), /exactly one revision/)
assert.throws(() => readRelayServingRegionalPlacementVersion(input, {
run: (args) => args[1] === 'services'
? { status: { traffic: [{ revisionName: 'relay-serving', percent: 100 }] } }
: revision('latest')
}), /secret reference is invalid/)
const denied = new Error('denied')
denied.code = 'GCLOUD_FAILED'
assert.throws(() => readRelayServingRegionalPlacementVersion(input, {
run: () => { throw denied }
}), denied)
})
@@ -0,0 +1,277 @@
import { createHash } from 'node:crypto'
const STATES = ['existing-only', 'migration-only', 'general']
function normalizeMembership(input) {
const membership = {
existingOnly: [...input.existingOnly].sort(),
migrationOnly: [...input.migrationOnly].sort(),
general: [...input.general].sort()
}
const all = [...membership.existingOnly, ...membership.migrationOnly, ...membership.general]
if (new Set(all).size !== all.length) throw new Error('selector membership contains duplicates')
return membership
}
function encodedMembership(membership) {
return JSON.stringify(normalizeMembership(membership))
}
function membershipSha256(membership) {
return createHash('sha256').update(encodedMembership(membership)).digest('hex')
}
function normalizeMigrationCells(input) {
const cells = [...input]
.map((cell) => ({
cellId: cell.cellId,
cellUrl: cell.cellUrl,
capacityRequests: cell.capacityRequests,
...(cell.region ? { region: cell.region } : {}),
connectionHardCap: cell.connectionHardCap,
connectionUnobservedBound: cell.connectionUnobservedBound
}))
.sort((left, right) => left.cellId.localeCompare(right.cellId))
if (
cells.length === 0 ||
new Set(cells.map(({ cellId }) => cellId)).size !== cells.length ||
new Set(cells.map(({ cellUrl }) => cellUrl)).size !== cells.length
) {
throw new Error('migration cell registration must contain distinct cells')
}
return cells
}
function membershipWithMigrationCells(membership, cells) {
const known = new Set([
...membership.existingOnly,
...membership.migrationOnly,
...membership.general
])
if (cells.some(({ cellId }) => known.has(cellId))) {
throw new Error('migration cell registration contains an existing selector cell')
}
return normalizeMembership({
existingOnly: membership.existingOnly,
migrationOnly: [...membership.migrationOnly, ...cells.map(({ cellId }) => cellId)],
general: membership.general
})
}
function assertSelector(value) {
if (
!value ||
!Number.isSafeInteger(value.generation) ||
value.generation < 0 ||
!value.membership
) {
throw new Error('director returned an invalid admission selector')
}
return {
generation: value.generation,
attemptId: value.attemptId ?? null,
membership: normalizeMembership(value.membership)
}
}
export function selectorAttemptId(expectedGeneration, membership) {
const digest = createHash('sha256')
.update(`${expectedGeneration}:${encodedMembership(membership)}`)
.digest('hex')
.slice(0, 24)
return `selector_${expectedGeneration}_${digest}`
}
export function membershipWithStates(selector, states) {
const byCell = new Map()
for (const [state, key] of [
['existing-only', 'existingOnly'],
['migration-only', 'migrationOnly'],
['general', 'general']
]) {
for (const cellId of selector.membership[key]) byCell.set(cellId, state)
}
for (const [cellId, state] of Object.entries(states)) {
if (!byCell.has(cellId)) throw new Error(`selector does not contain ${cellId}`)
if (!STATES.includes(state)) throw new Error(`invalid admission state for ${cellId}`)
if (byCell.get(cellId) === 'existing-only' && state !== 'existing-only') {
throw new Error(`selector cannot re-enable existing-only cell ${cellId}`)
}
byCell.set(cellId, state)
}
return normalizeMembership({
existingOnly: [...byCell].filter(([, state]) => state === 'existing-only').map(([id]) => id),
migrationOnly: [...byCell].filter(([, state]) => state === 'migration-only').map(([id]) => id),
general: [...byCell].filter(([, state]) => state === 'general').map(([id]) => id)
})
}
export async function inspectAdmissionSelector(post, attemptId) {
const result = await post('/v1/admin/admission-selector/status', {
v: 1,
...(attemptId ? { attemptId } : {})
})
return {
selector: assertSelector(result.selector),
intent: result.intent
? {
...result.intent,
previousMembership: result.intent.previousMembership
? normalizeMembership(result.intent.previousMembership)
: undefined,
membership: normalizeMembership(result.intent.membership)
}
: null
}
}
function exactSelector(actual, expected) {
return (
actual.generation === expected.generation &&
encodedMembership(actual.membership) === encodedMembership(expected.membership)
)
}
export async function applyExactAdmissionSelector(post, membership, options = {}) {
const before = await inspectAdmissionSelector(post)
const desired = normalizeMembership(membership)
if (
options.expectedCurrentSelector &&
!exactSelector(before.selector, options.expectedCurrentSelector)
) {
throw new Error('admission selector changed before exact apply')
}
if (options.requireBoundary !== false && before.selector.generation < 1) {
throw new Error('admission selector boundary is not active')
}
if (encodedMembership(before.selector.membership) === encodedMembership(desired)) {
return { changed: false, selector: before.selector }
}
const attemptId =
options.attemptId ?? selectorAttemptId(before.selector.generation, desired)
const expected = {
generation: before.selector.generation + 1,
membership: desired
}
let result
try {
result = await post('/v1/admin/admission-selector/apply', {
v: 1,
attemptId,
expectedGeneration: before.selector.generation,
...(before.selector.generation === 0
? { expectedMembershipSha256: membershipSha256(before.selector.membership) }
: {}),
membership: desired
})
} catch (error) {
const inspected = await inspectAdmissionSelector(post, attemptId)
if (
inspected.intent?.state === 'committed' &&
exactSelector(inspected.selector, expected)
) {
return { changed: true, selector: inspected.selector, recovered: true }
}
if (
inspected.intent?.state === 'unchanged' &&
exactSelector(inspected.selector, before.selector)
) {
throw new Error('admission selector apply remained unchanged after an ambiguous response', {
cause: error
})
}
throw new Error('admission selector apply diverged after an ambiguous response', {
cause: error
})
}
const applied = assertSelector(result.selector)
if (!exactSelector(applied, expected)) {
throw new Error('admission selector apply returned unexpected membership')
}
const verified = await inspectAdmissionSelector(post, attemptId)
if (
verified.intent?.state !== 'committed' ||
!exactSelector(verified.selector, expected)
) {
throw new Error('admission selector commit could not be verified')
}
return { changed: result.changed === true, selector: verified.selector }
}
export async function addExactMigrationCells(post, input, options = {}) {
const cells = normalizeMigrationCells(input.cells)
const attemptId = input.attemptId
if (!/^[A-Za-z0-9_-]{8,128}$/.test(attemptId ?? '')) {
throw new Error('migration cell registration requires an exact attempt ID')
}
const before = await inspectAdmissionSelector(post, attemptId)
let expectedGeneration
let expectedMembership
if (before.intent) {
expectedGeneration = before.intent.expectedGeneration
expectedMembership = normalizeMembership(before.intent.membership)
} else {
if (before.selector.generation < 1) {
throw new Error('admission selector boundary is not active')
}
if (
options.expectedCurrentSelector &&
!exactSelector(before.selector, options.expectedCurrentSelector)
) {
throw new Error('admission selector changed before cell registration')
}
expectedGeneration = before.selector.generation
expectedMembership = membershipWithMigrationCells(before.selector.membership, cells)
}
const expected = {
generation: expectedGeneration + 1,
membership: expectedMembership
}
let result
try {
result = await post('/v1/admin/admission-selector/add-migration-cells', {
v: 1,
attemptId,
expectedGeneration,
cells
})
} catch (error) {
const inspected = await inspectAdmissionSelector(post, attemptId)
if (
!before.intent &&
inspected.intent?.state === 'committed' &&
exactSelector(inspected.selector, expected)
) {
return { changed: true, selector: inspected.selector, recovered: true }
}
throw new Error('migration cell registration did not commit exactly', { cause: error })
}
const applied = assertSelector(result.selector)
if (!exactSelector(applied, expected)) {
throw new Error('migration cell registration returned unexpected membership')
}
const verified = await inspectAdmissionSelector(post, attemptId)
if (verified.intent?.state !== 'committed' || !exactSelector(verified.selector, expected)) {
throw new Error('migration cell registration commit could not be verified')
}
return { changed: result.changed === true, selector: verified.selector }
}
export async function transitionAdmissionSelector(post, states, options = {}) {
const current = await inspectAdmissionSelector(post)
if (current.selector.generation < 1) {
throw new Error('admission selector boundary is not active')
}
return await applyExactAdmissionSelector(
post,
membershipWithStates(current.selector, states),
options
)
}
export function selectorCellState(selector, cellId) {
if (selector.membership.existingOnly.includes(cellId)) return 'existing-only'
if (selector.membership.migrationOnly.includes(cellId)) return 'migration-only'
if (selector.membership.general.includes(cellId)) return 'general'
throw new Error(`selector does not contain ${cellId}`)
}
@@ -0,0 +1,232 @@
import assert from 'node:assert/strict'
import { createHash } from 'node:crypto'
import { test } from 'node:test'
import {
addExactMigrationCells,
applyExactAdmissionSelector,
membershipWithStates,
selectorAttemptId,
transitionAdmissionSelector
} from './relay-admission-selector.mjs'
const initialMembership = {
existingOnly: ['legacy'],
migrationOnly: ['target'],
general: ['general']
}
function selectorHarness({ ambiguous = null, generation = 1 } = {}) {
let selector = { generation, attemptId: 'initial', membership: initialMembership }
const intents = new Map()
const requests = []
let applies = 0
const post = async (path, body) => {
if (path.endsWith('/status')) {
return {
selector,
intent: body.attemptId ? intents.get(body.attemptId) ?? null : null
}
}
applies++
requests.push(body)
const before = selector
const committed = {
generation: body.expectedGeneration + 1,
attemptId: body.attemptId,
membership: body.membership
}
intents.set(body.attemptId, {
attemptId: body.attemptId,
expectedGeneration: body.expectedGeneration,
intendedGeneration: committed.generation,
previousMembership: before.membership,
membership: body.membership,
state: ambiguous === 'unchanged' ? 'unchanged' : 'committed'
})
if (ambiguous !== 'unchanged') selector = committed
if (ambiguous) throw new Error('lost selector response')
return { changed: true, selector }
}
return { post, selector: () => selector, applies: () => applies, requests }
}
test('derives deterministic attempts and applies exact selector transitions', async () => {
const harness = selectorHarness()
const desired = membershipWithStates(harness.selector(), { target: 'general' })
assert.equal(
selectorAttemptId(1, desired),
selectorAttemptId(1, {
existingOnly: ['legacy'],
migrationOnly: [],
general: ['target', 'general']
})
)
const result = await transitionAdmissionSelector(harness.post, { target: 'general' })
assert.equal(result.selector.generation, 2)
assert.deepEqual(result.selector.membership.general, ['general', 'target'])
})
test('accepts only an exact committed result after an ambiguous response', async () => {
const harness = selectorHarness({ ambiguous: 'committed' })
const result = await applyExactAdmissionSelector(harness.post, {
existingOnly: ['legacy', 'target'],
migrationOnly: [],
general: ['general']
})
assert.equal(result.recovered, true)
assert.equal(harness.applies(), 1)
assert.equal(result.selector.generation, 2)
})
test('binds a generation-zero cutover to the inspected membership', async () => {
const harness = selectorHarness({ generation: 0 })
await applyExactAdmissionSelector(
harness.post,
{
existingOnly: ['legacy', 'target'],
migrationOnly: [],
general: ['general']
},
{ requireBoundary: false }
)
assert.equal(
harness.requests[0].expectedMembershipSha256,
createHash('sha256').update(JSON.stringify(initialMembership)).digest('hex')
)
})
test('stops on an unchanged ambiguous result without replaying', async () => {
const harness = selectorHarness({ ambiguous: 'unchanged' })
await assert.rejects(
applyExactAdmissionSelector(harness.post, {
existingOnly: ['legacy', 'target'],
migrationOnly: [],
general: ['general']
}),
/remained unchanged/
)
assert.equal(harness.applies(), 1)
assert.equal(harness.selector().generation, 1)
})
test('never restores an existing-only cell', () => {
assert.throws(
() => membershipWithStates({ membership: initialMembership }, { legacy: 'general' }),
/cannot re-enable/
)
})
test('refuses an exact apply after the inspected selector changes', async () => {
const harness = selectorHarness()
await assert.rejects(
applyExactAdmissionSelector(
harness.post,
{
existingOnly: ['legacy', 'target'],
migrationOnly: [],
general: ['general']
},
{
expectedCurrentSelector: {
generation: 0,
membership: {
existingOnly: ['target'],
migrationOnly: [],
general: ['general', 'legacy']
}
}
}
),
/changed before exact apply/
)
assert.equal(harness.applies(), 0)
})
test('adds exact migration cells and recovers a committed response loss', async () => {
let selector = { generation: 1, attemptId: 'initial', membership: initialMembership }
const intents = new Map()
let additions = 0
const post = async (path, body) => {
if (path.endsWith('/status')) {
return {
selector,
intent: body.attemptId ? intents.get(body.attemptId) ?? null : null
}
}
additions++
selector = {
generation: body.expectedGeneration + 1,
attemptId: body.attemptId,
membership: {
...selector.membership,
migrationOnly: [
...selector.membership.migrationOnly,
...body.cells.map(({ cellId }) => cellId)
].sort()
}
}
intents.set(body.attemptId, {
attemptId: body.attemptId,
expectedGeneration: body.expectedGeneration,
intendedGeneration: selector.generation,
membership: selector.membership,
state: 'committed'
})
throw new Error('lost cell registration response')
}
const result = await addExactMigrationCells(post, {
attemptId: 'add_cells_exact',
cells: [
{
cellId: 'target-2',
cellUrl: 'https://target-2.example.com',
capacityRequests: 4_000,
connectionHardCap: 600,
connectionUnobservedBound: 60
}
]
})
assert.equal(result.recovered, true)
assert.equal(additions, 1)
assert.deepEqual(result.selector.membership.migrationOnly, ['target', 'target-2'])
})
test('does not recover an attempt owned by another selector operation', async () => {
const selector = {
generation: 2,
attemptId: 'selector_collision',
membership: initialMembership
}
const post = async (path, body) => {
if (path.endsWith('/status')) {
return {
selector,
intent: body.attemptId
? {
attemptId: body.attemptId,
expectedGeneration: 1,
intendedGeneration: 2,
membership: initialMembership,
state: 'committed'
}
: null
}
}
throw new Error('admission_selector_attempt_mismatch')
}
await assert.rejects(
addExactMigrationCells(post, {
attemptId: 'selector_collision',
cells: [
{
cellId: 'target-2',
cellUrl: 'https://target-2.example.com',
capacityRequests: 4_000,
connectionHardCap: 600,
connectionUnobservedBound: 60
}
]
}),
/did not commit exactly/
)
})
@@ -0,0 +1,323 @@
import assert from 'node:assert/strict'
import { readFileSync } from 'node:fs'
import { test } from 'node:test'
import { relayWorkflowUrl } from './relay-repository.mjs'
const workflow = readFileSync(
relayWorkflowUrl('operate-relay-asia-admission.yml'),
'utf8'
)
const iam = readFileSync(new URL('../../infra/terraform/relay-github-actions.tf', import.meta.url), 'utf8')
const stagingProof = readFileSync(
relayWorkflowUrl('prove-relay-asia-staging.yml'),
'utf8'
)
const directorWorkflow = readFileSync(
relayWorkflowUrl('deploy-relay-production-director.yml'),
'utf8'
)
const terraformReadme = readFileSync(
new URL('../../infra/terraform/README.md', import.meta.url),
'utf8'
)
const proofIam = readFileSync(
new URL('../../infra/terraform/relay-asia-proof-iam.tf', import.meta.url),
'utf8'
)
const relayTerraform = readFileSync(
new URL('../../infra/terraform/relay.tf', import.meta.url),
'utf8'
)
const rolloutEvidence = readFileSync(
new URL('./relay-asia-rollout-evidence.mjs', import.meta.url),
'utf8'
)
const admissionBudgets = readFileSync(
new URL('../../packages/relay-contract/src/admission-budgets.ts', import.meta.url),
'utf8'
)
test('offers the exact audited admission modes under the shared deployment lock', () => {
for (const mode of [
'inspect', 'initialize', 'verify', 'register', 'configure', 'promote', 'rollback'
]) {
assert.match(workflow, new RegExp(`\\b${mode}\\b`))
}
assert.match(workflow, /production-cloud-sql-rollout/)
assert.match(workflow, /relay-staging-mutation/)
assert.match(workflow, /selector-generation/)
assert.match(workflow, /selector-attempt-id/)
})
test('requires exact confirmations and uses the existing admin identity', () => {
assert.match(workflow, /INITIALIZE_ADMISSION_SELECTOR/)
assert.match(workflow, /REGISTER_ASIA_MIGRATION_ONLY/)
assert.match(workflow, /PROMOTE_ASIA_GENERAL/)
assert.match(workflow, /ROLLBACK_ASIA_MIGRATION_ONLY/)
assert.match(workflow, /CONFIGURE_ASIA_DIRECTOR/)
assert.match(workflow, /GCP_RELAY_DEPLOY_SERVICE_ACCOUNT/)
assert.match(workflow, /id_token_audience: \$\{\{ env\.DIRECTOR_ORIGIN \}\}\/v1\/admin\/drain/)
assert.match(iam, /"operate-relay-asia-admission\.yml"/)
})
test('discovers generation read-only and explicitly initializes only generation zero', () => {
assert.match(workflow, /leave empty only for inspect/)
assert.match(workflow, /test -z "\$\{EXPECTED_SELECTOR_GENERATION\}"/)
assert.match(workflow, /test "\$\{EXPECTED_SELECTOR_GENERATION\}" = 0/)
assert.match(workflow, /\^\(0\|\[1-9\]\[0-9\]\*\)\$/)
assert.match(workflow, /selector-membership-sha256/)
assert.match(workflow, /\^\[a-f0-9\]\{64\}\$/)
assert.match(workflow, /director-image-digest/)
assert.match(workflow, /\.spec\.containers\[0\]\.image == \$image/)
})
test('uploads one sanitized machine-readable admission result', () => {
assert.match(workflow, /sanitize-relay-asia-admission-result\.mjs/)
const upload = /- name: Upload sanitized admission result\n([\s\S]*?)(?=\n - name:)/
.exec(workflow)?.[1]
assert.ok(upload)
assert.match(
upload,
/if: \$\{\{ inputs\.mode != 'configure' && steps\.admission-operation\.outcome == 'success' \}\}/
)
assert.match(upload, /uses: actions\/upload-artifact@v4/)
assert.match(
upload,
/relay-asia-admission-result-\$\{\{ github\.run_id \}\}-\$\{\{ github\.run_attempt \}\}/
)
assert.match(upload, /path: \$\{\{ runner\.temp \}\}\/relay-asia-admission-result\/result\.json/)
assert.match(upload, /if-no-files-found: error/)
assert.match(upload, /retention-days: 7/)
assert.ok(
workflow.indexOf('Upload sanitized admission result') >
workflow.indexOf('Upload immutable C27 canary evidence')
)
})
test('binds selector operations and director configuration to reviewed implementations', () => {
assert.match(workflow, /operate-relay-asia-admission\.mjs/)
assert.match(workflow, /prepare-relay-asia-director-cells\.mjs/)
assert.match(workflow, /deploy-relay-blue-green\.mjs/)
assert.match(workflow, /--prune-revisions false/)
assert.doesNotMatch(workflow, /gcloud secrets versions add/)
assert.match(workflow, /orca-cloud-relay-regional-placement-enabled/)
assert.match(workflow, /\.valueSource\.secretKeyRef/)
assert.match(workflow, /jq -er --arg secret "\$\{REGIONAL_PLACEMENT_SECRET\}"/)
assert.doesNotMatch(workflow, /jq -e --arg secret "\$\{REGIONAL_PLACEMENT_SECRET\}"/)
assert.doesNotMatch(workflow, /--regional-placement-enabled/)
assert.doesNotMatch(workflow, /"\$\{\{ inputs\./)
assert.doesNotMatch(workflow, /dns/i)
})
test('requires immutable staged evidence and a timed C27 canary before expansion', () => {
assert.match(workflow, /actions: read/)
assert.match(workflow, /actions\/download-artifact@v4/)
assert.match(workflow, /relay-asia-staging-\$\{EVIDENCE_RUN_ID\}-\$\{EVIDENCE_RUN_ATTEMPT\}/)
assert.match(workflow, /evidence_kind=staging/)
assert.match(workflow, /load-relay-controls\.mjs/)
assert.match(workflow, /--controls 1/)
assert.match(workflow, /--splices 1/)
assert.match(workflow, /--splice-hold-seconds 60/)
assert.match(workflow, /--relay-asia-load-principals 1/)
assert.match(workflow, /--duration-seconds 300/)
assert.match(workflow, /--required-lease-horizons 2/)
assert.match(workflow, /pnpm\/action-setup@v4/)
assert.match(workflow, /Install exact C27 canary dependencies/)
assert.match(workflow, /pnpm install --frozen-lockfile/)
assert.match(workflow, /pnpm --filter @orca-cloud\/relay-contract build/)
assert.ok(
workflow.indexOf('Build the C27 canary Relay contract') <
workflow.indexOf('Run a real five-minute C27 control and splice canary')
)
assert.match(workflow, /--load-report "\$\{RUNNER_TEMP\}\/relay-asia-c27-load\.json"/)
assert.match(workflow, /states\["production-gce-c28"\].*= migration-only/)
assert.match(workflow, /states\["production-gce-c29"\].*= migration-only/)
assert.match(workflow, /relay-asia-c27-canary-\$\{\{ github\.run_id \}\}-\$\{\{ github\.run_attempt \}\}/)
assert.match(workflow, /id: c27-evidence-upload/)
assert.match(workflow, /Return an unproven C27 canary to migration-only/)
assert.match(workflow, /steps\.c27-evidence-upload\.outcome != 'success'/)
assert.match(workflow, /--mode recover-promotion[\s\S]*?--attempt-id "\$\{SELECTOR_ATTEMPT_ID\}"/)
assert.match(workflow, /--attempt-id "\$\{SELECTOR_ATTEMPT_ID\}-rollback"/)
assert.match(workflow, /evidence_kind=c27/)
assert.match(workflow, /orca_relay_runtime_metrics/)
assert.match(workflow, /relay-asia-rollout-evidence\.mjs create-c27/)
assert.match(workflow, /retention-days: 7/)
assert.match(workflow, /Require the exact director image before promotion/)
assert.match(workflow, /DIRECTOR_ORIGIN.*\/v1\/admin\/runtime-status/)
assert.match(workflow, /\.imageDigest.*IMAGE_DIGEST/)
const provenance = /- name: Verify evidence provenance and rollout binding before authentication\n([\s\S]*?)(?=\n - id: auth)/
.exec(workflow)?.[1]
assert.ok(provenance)
assert.match(provenance, /\.head_sha \| select\(type == "string" and test\("\^\[a-f0-9\]\{40\}\$"\)\)/)
assert.match(provenance, /--commit-sha "\$\{evidence_commit_sha\}"/)
assert.doesNotMatch(provenance, /--commit-sha "\$\{GITHUB_SHA\}"/)
})
test('creates staging evidence only after the bounded launch-path load and rollback', () => {
assert.match(stagingProof, /runs-on: \[self-hosted, linux, x64, relay-asia-east2-load\]/)
assert.doesNotMatch(stagingProof, /group: relay-asia-east2-load/)
assert.match(stagingProof, /pnpm\/action-setup@v4/)
assert.match(stagingProof, /pnpm install --frozen-lockfile/)
assert.match(stagingProof, /pnpm --filter @orca-cloud\/relay-contract build/)
assert.match(stagingProof, /run_phase launch 5 5/)
assert.doesNotMatch(stagingProof, /run_phase control|run_phase mixed/)
assert.match(stagingProof, /--aggregate-controls "\$\(\(controls \* 4\)\)"/)
assert.match(stagingProof, /--aggregate-splices "\$\(\(splices \* 4\)\)"/)
assert.match(stagingProof, /--required-lease-horizons 2/)
assert.match(stagingProof, /--splice-ramp-seconds 120/)
assert.match(stagingProof, /--max-generator-rss-growth-mib 512/)
assert.match(stagingProof, /--relay-asia-load-principals 32/)
assert.match(stagingProof, /ulimit -n/)
assert.match(stagingProof, /--region-behavior-probes 1/)
assert.match(stagingProof, /--capacity-cell-origin https:\/\/c4\.relay-staging\.onorca\.dev/)
assert.match(stagingProof, /--rebind-probes 2/)
assert.match(stagingProof, /--skip-rebind-overflow-check/)
assert.doesNotMatch(stagingProof, /--request-unit-invites|--regional-fallback-probes/)
assert.match(stagingProof, /--aggregate-reader-splices.*echo 5/)
assert.match(stagingProof, /--aggregate-reader-bytes.*echo 12582912/)
assert.match(stagingProof, /--phase-barrier-dir "\$\{proof_dir\}\/\$\{phase\}-barrier"/)
assert.match(stagingProof, /--duration-seconds 210/)
assert.match(stagingProof, /trap stop_shards EXIT/)
assert.match(stagingProof, /if ! wait "\$\{pid\}"; then failed=1; break; fi/)
assert.match(stagingProof, /connectionFailuresByReason/)
assert.match(stagingProof, /--launch-report "\$\{proof_dir\}\/launch\.json"/)
assert.match(stagingProof, /id-token: write/)
assert.match(stagingProof, /STAGING_GCP_RELAY_ASIA_PROOF_WORKLOAD_IDENTITY_PROVIDER/)
assert.match(stagingProof, /STAGING_GCP_RELAY_ASIA_PROOF_SERVICE_ACCOUNT/)
assert.doesNotMatch(stagingProof, /STAGING_GCP_DEPLOY_SERVICE_ACCOUNT/)
assert.doesNotMatch(stagingProof, /STAGING_RELAY_LOAD_ACCESS_TOKEN/)
assert.doesNotMatch(stagingProof, /secrets versions access|signing-key-file/)
assert.match(stagingProof, /relay-asia-rollout-evidence\.mjs create-staging/)
assert.match(stagingProof, /Require the exact staging director image before promotion/)
assert.match(stagingProof, /DIRECTOR_ORIGIN.*\/v1\/admin\/runtime-status/)
assert.match(stagingProof, /\.imageDigest.*IMAGE_DIGEST/)
assert.match(stagingProof, /Return staging C4 to migration-only/)
assert.match(stagingProof, /steps\.promote\.outcome != 'skipped'/)
assert.match(stagingProof, /--mode recover-promotion[\s\S]*?--attempt-id "\$\{PROMOTE_ATTEMPT_ID\}"/)
assert.match(stagingProof, /--mode rollback[\s\S]*?--expected-generation "\$\{promoted_generation\}"/)
assert.match(stagingProof, /if: \$\{\{ success\(\) \}\}/)
assert.match(
stagingProof,
/recover:\n if: \$\{\{ always\(\) && github\.ref == 'refs\/heads\/main' \}\}/
)
assert.match(stagingProof, /needs: prove/)
assert.match(stagingProof, /Recover staging C4 with a fresh identity/)
assert.equal((stagingProof.match(/google-github-actions\/auth@v2/g) ?? []).length, 2)
assert.equal((stagingProof.match(/--mode recover-promotion/g) ?? []).length, 2)
assert.equal((stagingProof.match(/--mode rollback/g) ?? []).length, 2)
})
test('keeps the private runner below its 64-port Cloud NAT allocation', () => {
const profile = /run_phase launch (\d+) (\d+)/.exec(stagingProof)
const controlsPerShard = Number(profile?.[1])
const splicesPerShard = Number(profile?.[2])
const rebindProbes = Number(/--rebind-probes (\d+)/.exec(stagingProof)?.[1])
const runtimeStatusSockets = 1
assert.ok(
controlsPerShard * 4 + splicesPerShard * 4 * 2 + rebindProbes + runtimeStatusSockets < 64
)
})
test('paces one-source staging upgrades below the Relay anti-abuse ceiling', () => {
const splicesPerShard = Number(/run_phase launch \d+ (\d+)/.exec(stagingProof)?.[1])
const rebindProbes = Number(/--rebind-probes (\d+)/.exec(stagingProof)?.[1])
const spliceRampMs = Number(/--splice-ramp-seconds (\d+)/.exec(stagingProof)?.[1]) * 1000
const ceiling = Number(
/maxPreAuthAttemptsPerSourcePerMinute: (\d+)/.exec(admissionBudgets)?.[1]
)
const totalSplices = splicesPerShard * 4
const attempts = Array.from({ length: totalSplices }, (_, ordinal) =>
Math.floor(ordinal * spliceRampMs / (totalSplices - 1))
).flatMap((startedAt) => [startedAt, startedAt])
attempts.push(...Array.from({ length: 4 + rebindProbes }, () => 0))
const busiestMinute = Math.max(...attempts.map((startedAt) =>
attempts.filter((attempt) => attempt >= startedAt && attempt < startedAt + 60_000).length
))
assert.ok(busiestMinute < ceiling)
})
test('reserves rollback time beyond the complete bounded staging proof envelope', () => {
const timeoutMinutes = Number(/timeout-minutes: (\d+)/.exec(stagingProof)?.[1])
assert.equal(timeoutMinutes, 75)
const spliceRampSeconds = Number(/--splice-ramp-seconds (\d+)/.exec(stagingProof)?.[1])
const launchSeconds = 180 + spliceRampSeconds + 210 + 60
const setupEvidenceAndRollbackSeconds = 10 * 60
const envelopeMinutes = Math.ceil((launchSeconds + setupEvidenceAndRollbackSeconds) / 60)
assert.ok(timeoutMinutes - envelopeMinutes >= 30)
assert.match(stagingProof, /--ramp-seconds 180/)
assert.match(stagingProof, /--duration-seconds 210/)
})
test('binds the staging proof to one least-privilege Google identity', () => {
assert.match(
proofIam,
/github_relay_asia_proof_workflow_file = "prove-relay-asia-staging\.yml"/
)
assert.match(
proofIam,
/assertion\.workflow_ref == '\$\{prefix\}\$\{local\.github_relay_asia_proof_workflow_file\}@refs\/heads\/main'/
)
assert.match(proofIam, /assertion\.environment == 'staging'/)
assert.match(proofIam, /assertion\.event_name == 'workflow_dispatch'/)
assert.match(proofIam, /roles\/logging\.viewer/)
assert.match(proofIam, /roles\/monitoring\.viewer/)
assert.match(rolloutEvidence, /readCloudSqlBackends/)
assert.match(rolloutEvidence, /cloudSql: await readCloudSqlBackends/)
assert.doesNotMatch(proofIam, /compute\.|cloudsql\.|secretmanager\.|roles\/editor|roles\/run\./)
})
test('keeps the production US-first switch in durable Secret Manager state', () => {
assert.match(directorWorkflow, /options: \[preserve, enable, disable\]/)
assert.match(directorWorkflow, /default: preserve/)
assert.match(directorWorkflow, /gcloud secrets versions add/)
assert.match(directorWorkflow, /preserve\) desired="\$\{current\}"/)
assert.match(directorWorkflow, /--regional-placement-secret-version "\$\{target_version\}"/)
assert.match(directorWorkflow, /test "\$\{CEILING\}" = "\$\{DIRECTOR_MAX_INSTANCES\}"/)
assert.match(directorWorkflow, /orca-cloud-relay-regional-placement-enabled/)
assert.match(directorWorkflow, /\.valueSource\.secretKeyRef \/\/ \.valueFrom\.secretKeyRef/)
assert.match(directorWorkflow, /\.version \/\/ \.key/)
assert.match(directorWorkflow, /\.secret \/\/ \.name/)
assert.match(workflow, /\.valueSource\.secretKeyRef \/\/ \.valueFrom\.secretKeyRef/)
assert.doesNotMatch(directorWorkflow, /--regional-placement-enabled/)
assert.doesNotMatch(workflow, /inputs\.regional-placement-enabled/)
})
test('prunes incompatible production revisions only when explicitly confirmed', () => {
assert.match(
directorWorkflow,
/prune-incompatible-revisions:[\s\S]*?default: false[\s\S]*?type: boolean/
)
assert.match(directorWorkflow, /PRUNE_INCOMPATIBLE_RELAY_DIRECTOR_REVISIONS/)
assert.match(
directorWorkflow,
/test "\$\{REGIONAL_PLACEMENT_MODE\}" = preserve[\s\S]*?test "\$\{CONFIRMATION\}" = PRUNE_INCOMPATIBLE_RELAY_DIRECTOR_REVISIONS/
)
assert.match(
directorWorkflow,
/--prune-revisions "\$\{PRUNE_INCOMPATIBLE_REVISIONS\}"/
)
})
test('documents the exact regional-placement secret bootstrap before director rollout', () => {
for (const address of [
'google_secret_manager_secret.relay_regional_placement_enabled',
'google_secret_manager_secret_version.relay_regional_placement_enabled',
'google_secret_manager_secret_iam_member.relay_regional_placement_runtime_accessor',
'google_secret_manager_secret_iam_member.relay_regional_placement_deploy_accessor[0]',
'google_secret_manager_secret_iam_member.relay_regional_placement_deploy_adder[0]',
'google_secret_manager_secret_iam_member.relay_regional_placement_deploy_viewer[0]'
]) {
assert.match(terraformReadme, new RegExp(address.replaceAll(/[.[\]]/g, '\\$&')))
}
assert.match(terraformReadme, /Before the first director deployment/)
assert.match(terraformReadme, /Pass the exact environment tfvars/)
// The Cloudflare records left with the apps root; a -var for a variable this root no longer
// declares is a hard error, so no relay procedure may still tell an operator to pass it.
assert.doesNotMatch(terraformReadme, /manage_artifact_dns/)
assert.match(terraformReadme, /exactly these six additions/)
assert.match(terraformReadme, /version metadata/)
assert.match(
relayTerraform,
/resource "google_secret_manager_secret_iam_member" "relay_regional_placement_deploy_viewer"[\s\S]*?role\s+= "roles\/secretmanager\.viewer"/
)
})
@@ -0,0 +1,33 @@
import { spawnSync } from 'node:child_process'
function accessToken() {
const result = spawnSync('gcloud', ['auth', 'print-access-token'], {
encoding: 'utf8', timeout: 30_000
})
const token = result.stdout.trim()
if (result.status !== 0 || token.length < 20) {
throw new Error('Google access token is unavailable')
}
return token
}
export async function readCloudSqlBackends(environment, startedAt, endedAt) {
const production = environment === 'production'
if (!production && environment !== 'staging') throw new Error('Cloud SQL environment is invalid')
const project = production ? 'onorca-cloud' : 'onorca-cloud-staging'
const instance = production ? 'orca-cloud-auth-db' : 'orca-cloud-staging-auth-db'
const url = new URL(`https://monitoring.googleapis.com/v3/projects/${project}/timeSeries`)
url.searchParams.set('filter', `metric.type = "cloudsql.googleapis.com/database/postgresql/num_backends" AND resource.labels.database_id = "${project}:${instance}"`)
url.searchParams.set('interval.startTime', startedAt)
url.searchParams.set('interval.endTime', endedAt)
url.searchParams.set('aggregation.alignmentPeriod', '60s')
url.searchParams.set('aggregation.perSeriesAligner', 'ALIGN_MAX')
url.searchParams.set('aggregation.crossSeriesReducer', 'REDUCE_MAX')
url.searchParams.set('view', 'FULL')
const response = await fetch(url, {
headers: { authorization: `Bearer ${accessToken()}` },
redirect: 'error', signal: AbortSignal.timeout(30_000)
})
if (!response.ok) throw new Error(`Cloud SQL metric query returned ${response.status}`)
return await response.json()
}
@@ -0,0 +1,544 @@
import { readFileSync, writeFileSync } from 'node:fs'
import { fileURLToPath } from 'node:url'
import { readCloudSqlBackends } from './relay-asia-cloud-sql-metrics.mjs'
import { RELAY_GITHUB_REPOSITORY, relayWorkflowPath } from './relay-repository.mjs'
const REPOSITORY = RELAY_GITHUB_REPOSITORY
const ADMISSION_WORKFLOW = relayWorkflowPath('operate-relay-asia-admission.yml')
const STAGING_WORKFLOW = relayWorkflowPath('prove-relay-asia-staging.yml')
const STAGING_CELL = 'staging-gce-c4'
const C27 = 'production-gce-c27'
const DIGEST_PATTERN = /^sha256:[a-f0-9]{64}$/
const SHA_PATTERN = /^[a-f0-9]{40}$/
const MAX_LOG_EDGE_GAP_MS = 120_000
const MAX_LOG_SAMPLE_GAP_MS = 120_000
const CLOUD_SQL_LIMIT = 320
const C27_CANARY_MINIMUM_MS = 5 * 60_000
const GENERATOR_CPU_PERCENT_LIMIT = 80
const GENERATOR_EVENT_LOOP_P99_MS_LIMIT = 100
const GENERATOR_RSS_GROWTH_MIB_LIMIT = 512
const DATABASE_POOL_TRANSIENT_WAITERS_MAX = 4
const DATABASE_POOL_TRANSIENT_WAIT_MS_MAX = 50
function object(value, label) {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new Error(`${label} is invalid`)
}
return value
}
function positiveInteger(value, label) {
const number = Number(value)
if (!Number.isSafeInteger(number) || number < 1) throw new Error(`${label} is invalid`)
return number
}
function instant(value, label) {
const date = new Date(value)
if (!Number.isFinite(date.valueOf())) throw new Error(`${label} is invalid`)
return date
}
function exactCells(actual, expected) {
return Array.isArray(actual) &&
JSON.stringify([...actual].sort()) === JSON.stringify([...expected].sort())
}
function source(input, environment, workflow) {
if (input.repository !== REPOSITORY) throw new Error('repository is invalid')
if (!SHA_PATTERN.test(input.commitSha)) throw new Error('commit SHA is invalid')
return {
repository: input.repository,
workflow,
environment,
runId: positiveInteger(input.runId, 'run ID'),
runAttempt: positiveInteger(input.runAttempt, 'run attempt'),
commitSha: input.commitSha
}
}
function baseEvidence(input, environment, cells, workflow = ADMISSION_WORKFLOW) {
if (!DIGEST_PATTERN.test(input.imageDigest)) throw new Error('image digest is invalid')
return {
version: 1,
source: source(input, environment, workflow),
imageDigest: input.imageDigest,
topology: {
cellIds: cells,
selectorGeneration: positiveInteger(input.selectorGeneration, 'selector generation')
}
}
}
export function buildStagingEvidence(input) {
const start = instant(input.startedAt, 'staging proof start')
const end = instant(input.endedAt, 'staging proof end')
const launch = loadReports(input.launchReport, 'launch load report')
const expectedLoad = {
controls: 20, splices: 20, slowReaders: 4, wedgedReaders: 1,
minimumSeconds: 210
}
assertLoadReports(launch, expectedLoad)
const metrics = runtimeMetrics(input.logs, start, end, STAGING_CELL)
assertPassingRuntimeMetrics(metrics, 'staging proof', 1)
const minimumConcurrentSplices = expectedLoad.splices - expectedLoad.wedgedReaders
if (
metrics.targetControlsMax < expectedLoad.controls ||
metrics.targetSplicesMax < minimumConcurrentSplices
) {
throw new Error('staging launch load did not reach C4 at the reviewed levels')
}
metrics.cloudSqlBackendsMax = cloudSqlMaximum(input.cloudSql, start, end)
if (metrics.cloudSqlBackendsMax >= CLOUD_SQL_LIMIT) {
throw new Error(`Cloud SQL backends must remain below ${CLOUD_SQL_LIMIT}`)
}
return {
...baseEvidence(input, 'staging', [STAGING_CELL], STAGING_WORKFLOW),
kind: 'staging-asia-readiness',
window: { startedAt: start.toISOString(), endedAt: end.toISOString() },
load: { launch: loadSummary(launch) },
metrics
}
}
function loadReports(value, label) {
if (!Array.isArray(value) || value.length < 2) throw new Error(`${label} must be sharded`)
return value.map((report) => object(report, label))
}
function total(reports, key) {
return reports.reduce((sum, report) => sum + number(report[key], key), 0)
}
function loadSummary(reports) {
return {
shards: reports.length,
controls: total(reports, 'controls'),
peakActive: total(reports, 'peakActive'),
steadyMinimumActive: total(reports, 'steadyMinimumActive'),
peakActiveSplices: total(reports, 'peakActiveSplices'),
completedSplices: total(reports, 'completedSplices'),
slowReaderSplicesCompleted: total(reports, 'slowReaderSplicesCompleted'),
wedgedReaderSplicesClosed: total(reports, 'wedgedReaderSplicesClosed'),
regionalFallbacksProved: total(reports, 'regionalFallbacksProved'),
oldClientUsFirstProved: total(reports, 'oldClientUsFirstProved'),
stickyAssignmentProved: total(reports, 'stickyAssignmentProved'),
requestUnitInvitesOpened: total(reports, 'requestUnitInvitesOpened'),
requestUnitPrincipalCounts: reports.map((report) => report.requestUnitPrincipalCount),
requestUnitOverflowReasons:
reports.map((report) => report.requestUnitOverflowReason).filter(Boolean),
requestUnitCleanupProved: total(reports, 'requestUnitCleanupProved'),
phaseBarrierPassed: reports.every((report) => report.phaseBarrierPassed === true),
rebindProbesOpened: total(reports, 'rebindProbesOpened'),
rebindOverflowReasons: reports.map((report) => report.rebindOverflowReason).filter(Boolean),
readerQueueEvidence: reports.flatMap((report) => report.readerQueueEvidence),
readerQueuedBytesPeak: Math.max(...reports.map((report) => report.readerQueuedBytesPeak)),
generatorCpuPercentMax: Math.max(...reports.map((report) => report.generatorCpuPercent)),
generatorEventLoopP99MsMax: Math.max(
...reports.map((report) => report.generatorEventLoopP99Ms)
),
generatorRssGrowthMiBMax: Math.max(...reports.map((report) => report.generatorRssGrowthMiB)),
configuredSteadySeconds: Math.min(...reports.map((report) => report.configuredSteadySeconds))
}
}
function assertLoadReports(reports, expected) {
const shardCount = reports.length
if (
reports.some((report, index) =>
report.event !== 'relay_load_complete' ||
report.shardCount !== shardCount || report.shardIndex !== index ||
report.relayAsiaLoadPrincipalCount !== 32 ||
number(report.configuredSteadySeconds, 'staging steady seconds') < expected.minimumSeconds ||
number(report.configuredSpliceHoldSeconds, 'staging splice hold seconds') <
expected.minimumSeconds ||
report.requiredLeaseHorizons !== 2 || report.phaseBarrierPassed !== true
) ||
total(reports, 'controls') !== expected.controls
) {
throw new Error('staging load profile does not match')
}
if (
total(reports, 'peakActive') !== expected.controls ||
total(reports, 'steadyMinimumActive') !== expected.controls
) {
throw new Error('staging load did not sustain the required controls')
}
for (const key of [
'rampConnectionFailures', 'steadyConnectionFailures', 'transitionConnectionFailures',
'unexpectedCloses', 'protocolErrors', 'refreshErrors', 'socketErrors', 'failedSplices'
]) {
if (total(reports, key) !== 0) throw new Error(`staging load ${key} must be zero`)
}
const peakActiveSplices = total(reports, 'peakActiveSplices')
if (
total(reports, 'configuredSplices') !== expected.splices ||
total(reports, 'configuredSlowReaderSplices') !== expected.slowReaders ||
total(reports, 'configuredWedgedReaderSplices') !== expected.wedgedReaders ||
peakActiveSplices < expected.splices - expected.wedgedReaders ||
peakActiveSplices > expected.splices ||
total(reports, 'completedSplices') !== expected.splices - expected.wedgedReaders ||
total(reports, 'slowReaderSplicesCompleted') !== expected.slowReaders ||
total(reports, 'wedgedReaderSplicesClosed') !== expected.wedgedReaders
) throw new Error('staging mixed load evidence does not match')
if (
total(reports, 'regionalFallbacksProved') !== 0 ||
total(reports, 'oldClientUsFirstProved') !== 1 ||
total(reports, 'stickyAssignmentProved') !== 1 ||
total(reports, 'requestUnitInvitesOpened') !== 0 ||
reports.some((report) => report.requestUnitPrincipalCount !== 0) ||
total(reports, 'requestUnitCleanupProved') !== 0 ||
reports.some((report) => report.requestUnitOverflowReason !== null) ||
total(reports, 'rebindProbesOpened') !== 2 ||
reports.some((report) => report.rebindOverflowReason !== null)
) throw new Error('staging launch-path evidence does not match')
const readerReports = reports.filter(
(report) => report.configuredSlowReaderSplices + report.configuredWedgedReaderSplices > 0
)
if (expected.slowReaders > 0 && readerReports.length !== 1) {
throw new Error('staging reader pressure must have one causal owner')
}
for (const report of reports) {
const queue = report.readerQueueEvidence
if (!Array.isArray(queue)) throw new Error('staging reader queue evidence is invalid')
if (expected.slowReaders === 0 && queue.length !== 0) {
throw new Error('control load unexpectedly contains reader queue evidence')
}
const ownsReaderPressure = readerReports.includes(report)
if (expected.slowReaders > 0 && ownsReaderPressure && (
queue.length !== 1 ||
queue[0]?.origin !== 'https://c4.relay-staging.onorca.dev' ||
number(queue[0]?.baselineBytes, 'reader queue baseline') >
number(queue[0]?.peakBytes, 'reader queue peak') ||
number(queue[0]?.increaseBytes, 'reader queue increase') <= 0 ||
queue[0].peakBytes - queue[0].baselineBytes !== queue[0].increaseBytes ||
report.readerQueuedBytesPeak !== queue[0].increaseBytes
)) throw new Error('staging reader queue evidence is not causal')
if (expected.slowReaders > 0 && !ownsReaderPressure && (
queue.length !== 0 || report.readerQueuedBytesPeak !== 0
)) throw new Error('non-owner shard contains reader queue evidence')
}
for (const report of reports) {
if (
number(report.generatorCpuPercent, 'generator CPU') >= GENERATOR_CPU_PERCENT_LIMIT ||
number(report.generatorEventLoopP99Ms, 'generator event loop') >=
GENERATOR_EVENT_LOOP_P99_MS_LIMIT ||
number(report.generatorRssGrowthMiB, 'generator RSS growth') >=
GENERATOR_RSS_GROWTH_MIB_LIMIT
) throw new Error('staging load generator has insufficient headroom')
const shutdown = object(report.shutdownEvidence, 'load shutdown evidence')
if (
shutdown.peerShutdowns !== report.controls || shutdown.activeControls !== 0 ||
shutdown.activeSplices !== 0 || shutdown.reconnectTimers !== 0
) throw new Error('staging load cleanup is incomplete')
}
}
function number(value, label) {
if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
throw new Error(`${label} is invalid`)
}
return value
}
function sumMap(value, label) {
const entries = Object.entries(object(value ?? {}, label))
return entries.reduce((total, [key, count]) => {
if (!key) throw new Error(`${label} is invalid`)
return total + number(count, label)
}, 0)
}
function assertCoverage(timestamps, start, end, label) {
if (timestamps.length === 0) throw new Error(`${label} has no samples`)
const ordered = timestamps.map((value) => instant(value, `${label} timestamp`).valueOf())
.sort((left, right) => left - right)
if (
ordered[0] > start.valueOf() + MAX_LOG_EDGE_GAP_MS ||
ordered.at(-1) < end.valueOf() - MAX_LOG_EDGE_GAP_MS
) throw new Error(`${label} does not cover the canary window`)
if (ordered.some((timestamp, index) => index > 0 && timestamp - ordered[index - 1] > MAX_LOG_SAMPLE_GAP_MS)) {
throw new Error(`${label} has a sampling gap`)
}
}
function pointValue(point) {
const value = point?.value
if (typeof value?.doubleValue === 'number') return value.doubleValue
if (typeof value?.int64Value === 'string') return Number(value.int64Value)
return NaN
}
function cloudSqlMaximum(response, start, end) {
const points = (object(response, 'Cloud SQL response').timeSeries ?? [])
.flatMap((series) => series.points ?? [])
assertCoverage(points.map((point) => point.interval?.endTime), start, end, 'Cloud SQL metrics')
const values = points.map(pointValue)
if (values.some((value) => !Number.isFinite(value) || value < 0)) {
throw new Error('Cloud SQL backend metric is invalid')
}
return Math.max(...values)
}
export function buildC27CanaryEvidence(input) {
const start = instant(input.startedAt, 'canary start')
const end = instant(input.endedAt, 'canary end')
if (end.valueOf() - start.valueOf() < C27_CANARY_MINIMUM_MS) {
throw new Error('C27 canary window is shorter than 5 minutes')
}
const load = object(input.loadReport, 'C27 load report')
assertC27CanaryLoad(load)
const metrics = runtimeMetrics(input.logs, start, end, C27)
assertPassingRuntimeMetrics(metrics, 'C27 canary')
metrics.cloudSqlBackendsMax = cloudSqlMaximum(input.cloudSql, start, end)
assertPassingCanary(metrics)
return {
...baseEvidence(input, 'production', [C27]),
kind: 'production-c27-canary',
window: { startedAt: start.toISOString(), endedAt: end.toISOString() },
load,
metrics
}
}
function assertC27CanaryLoad(report) {
if (
report.event !== 'relay_load_complete' ||
report.controls !== 1 || report.shardCount !== 1 || report.shardIndex !== 0 ||
report.relayAsiaLoadPrincipalCount !== 1 ||
number(report.configuredSteadySeconds, 'canary steady seconds') < 300 ||
number(report.configuredSpliceHoldSeconds, 'canary splice hold seconds') < 60 ||
number(report.requiredLeaseHorizons, 'canary lease horizons') < 2 ||
report.peakActive !== 1 || report.steadyMinimumActive !== 1 ||
report.configuredSplices !== 1 || report.peakActiveSplices !== 1 ||
report.completedSplices !== 1 || report.failedSplices !== 0
) throw new Error('C27 control and splice canary did not match')
for (const key of [
'connectionFailures', 'unexpectedCloses', 'protocolErrors',
'refreshErrors', 'socketErrors'
]) {
if (number(report[key], key) !== 0) throw new Error(`C27 canary ${key} must be zero`)
}
const shutdown = object(report.shutdownEvidence, 'C27 load shutdown evidence')
if (
shutdown.peerShutdowns !== 1 || shutdown.activeControls !== 0 ||
shutdown.activeSplices !== 0 || shutdown.reconnectTimers !== 0
) throw new Error('C27 load cleanup is incomplete')
}
function runtimeMetrics(logs, start, end, targetCellId) {
const entries = logs.map((entry) => object(entry, 'runtime metric entry'))
const directorEntries = entries.filter((entry) => entry.jsonPayload?.role === 'director')
const cellEntries = entries.filter((entry) =>
entry.jsonPayload?.role === 'cell' &&
entry.jsonPayload?.cellId === targetCellId &&
entry.jsonPayload?.region === 'asia-east2'
)
assertCoverage(directorEntries.map((entry) => entry.timestamp), start, end, 'director metrics')
assertCoverage(cellEntries.map((entry) => entry.timestamp), start, end, `${targetCellId} metrics`)
const identifiedDirectors = Map.groupBy(
directorEntries.filter((entry) => entry.resource?.labels?.instance_id),
(entry) => entry.resource.labels.instance_id
)
for (const [instanceId, instanceEntries] of identifiedDirectors) {
assertCoverage(instanceEntries.map((entry) => entry.timestamp), start, end, `director ${instanceId}`)
}
const directorPayloads = directorEntries.map((entry) => entry.jsonPayload)
const cellPayloads = cellEntries.map((entry) => entry.jsonPayload)
const payloads = [...directorPayloads, ...cellPayloads]
return {
asiaSelections: directorPayloads.reduce(
(total, payload) => total + number(payload.selectedRegionsDelta?.['asia-east2'] ?? 0, 'Asia selections'), 0
),
regionFallbacks: directorPayloads.reduce(
(total, payload) => total + sumMap(payload.regionFallbacksDelta, 'region fallbacks'), 0
),
usRegionFallbacks: directorPayloads.reduce(
(total, payload) => total + number(
payload.regionFallbacksDelta?.['us-central1'] ?? 0,
'US region fallbacks'
), 0
),
unavailableRegions: directorPayloads.reduce(
(total, payload) => total + sumMap(payload.unavailableRegionsDelta, 'unavailable regions'), 0
),
relaySqlFailures: payloads.reduce(
(total, payload) => total + number(payload.sqlFailuresDelta, 'Relay SQL failures'), 0
),
databasePoolWaitingMax: Math.max(...payloads.map(
(payload) => number(payload.databasePoolWaiting, 'database pool waiting')
)),
databasePoolWaitersMax: Math.max(...payloads.map(
(payload) => number(payload.databasePoolWaitersMax, 'database pool waiters')
)),
databasePoolWaitMsMax: Math.max(...payloads.map(
(payload) => number(payload.databasePoolWaitMsMax, 'database pool wait time')
)),
targetControlsMax: Math.max(...cellPayloads.map(
(payload) => number(payload.controls, 'target controls')
)),
targetSplicesMax: Math.max(...cellPayloads.map(
(payload) => number(payload.splices, 'target splices')
))
}
}
function assertPassingRuntimeMetrics(metrics, label, expectedRegionFallbacks = 0) {
if (number(metrics.asiaSelections, 'Asia selections') < 1) {
throw new Error(`${label} observed no Asia selections`)
}
if (
number(metrics.regionFallbacks, 'regionFallbacks') !== expectedRegionFallbacks ||
number(metrics.usRegionFallbacks, 'usRegionFallbacks') !== expectedRegionFallbacks
) {
throw new Error(`${label} regionFallbacks did not match the intentional probes`)
}
for (const key of [
'unavailableRegions', 'relaySqlFailures', 'databasePoolWaitingMax'
]) {
if (number(metrics[key], key) !== 0) throw new Error(`${label} ${key} must be zero`)
}
if (
number(metrics.databasePoolWaitersMax, 'databasePoolWaitersMax') >
DATABASE_POOL_TRANSIENT_WAITERS_MAX ||
number(metrics.databasePoolWaitMsMax, 'databasePoolWaitMsMax') >
DATABASE_POOL_TRANSIENT_WAIT_MS_MAX
) throw new Error(`${label} transient database pool pressure exceeded its bound`)
}
function assertPassingCanary(metrics) {
if (
number(metrics.targetControlsMax, 'C27 controls') < 1 ||
number(metrics.targetSplicesMax, 'C27 splices') < 1
) throw new Error('C27 canary traffic did not reach C27')
if (number(metrics.cloudSqlBackendsMax, 'Cloud SQL backends') >= CLOUD_SQL_LIMIT) {
throw new Error(`Cloud SQL backends must remain below ${CLOUD_SQL_LIMIT}`)
}
}
function assertProvenance(evidence, run, expected) {
const evidenceSource = object(evidence.source, 'evidence source')
if (
run.id !== evidenceSource.runId ||
run.run_attempt !== evidenceSource.runAttempt ||
run.conclusion !== 'success' ||
run.event !== 'workflow_dispatch' ||
run.head_branch !== 'main' ||
run.head_sha !== evidenceSource.commitSha ||
run.repository?.full_name !== evidenceSource.repository ||
run.path?.split('@')[0] !== expected.workflow ||
evidenceSource.workflow !== expected.workflow ||
evidenceSource.repository !== REPOSITORY ||
expected.repository !== REPOSITORY ||
evidenceSource.environment !== expected.environment ||
evidenceSource.commitSha !== expected.commitSha
) throw new Error('evidence workflow provenance does not match')
}
export function verifyRolloutEvidence(evidence, run, expected) {
object(evidence, 'evidence')
object(run, 'workflow run')
if (evidence.version !== 1 || evidence.kind !== expected.kind) {
throw new Error('evidence kind is invalid')
}
assertProvenance(evidence, run, expected)
if (evidence.imageDigest !== expected.imageDigest) throw new Error('evidence image digest does not match')
if (!exactCells(evidence.topology?.cellIds, expected.cellIds)) {
throw new Error('evidence topology does not match')
}
positiveInteger(evidence.topology?.selectorGeneration, 'evidence selector generation')
if (
expected.selectorGeneration !== undefined &&
evidence.topology.selectorGeneration !== expected.selectorGeneration
) throw new Error('evidence selector generation does not match')
const now = instant(expected.now, 'verification time')
const proofTime = instant(evidence.window?.endedAt, 'evidence time')
const maxAgeMs = expected.kind === 'staging-asia-readiness' ? 24 * 60 * 60_000 : 6 * 60 * 60_000
if (proofTime > now || now.valueOf() - proofTime.valueOf() > maxAgeMs) {
throw new Error('rollout evidence is stale')
}
if (expected.kind === 'production-c27-canary') {
const start = instant(evidence.window?.startedAt, 'canary start')
if (proofTime.valueOf() - start.valueOf() < C27_CANARY_MINIMUM_MS) {
throw new Error('C27 canary window is shorter than 5 minutes')
}
assertC27CanaryLoad(object(evidence.load, 'canary load'))
assertPassingCanary(object(evidence.metrics, 'canary metrics'))
}
return evidence
}
function argumentsMap(argv) {
const values = new Map()
for (let index = 1; index < argv.length; index += 2) {
const key = argv[index]
const value = argv[index + 1]
if (!key?.startsWith('--') || value === undefined || values.has(key.slice(2))) {
throw new Error('invalid rollout evidence arguments')
}
values.set(key.slice(2), value)
}
return { command: argv[0], values }
}
function required(values, key) {
const value = values.get(key)
if (!value) throw new Error(`missing --${key}`)
return value
}
function commonInput(values) {
return {
repository: required(values, 'repository'), runId: required(values, 'run-id'),
runAttempt: required(values, 'run-attempt'), commitSha: required(values, 'commit-sha'),
imageDigest: required(values, 'image-digest'),
selectorGeneration: required(values, 'selector-generation')
}
}
async function main(argv) {
const { command, values } = argumentsMap(argv)
const output = required(values, 'output')
if (command === 'create-staging') {
writeFileSync(output, `${JSON.stringify(buildStagingEvidence({
...commonInput(values), startedAt: required(values, 'started-at'),
endedAt: required(values, 'ended-at'),
launchReport: JSON.parse(readFileSync(required(values, 'launch-report'), 'utf8')),
logs: JSON.parse(readFileSync(required(values, 'logs-json'), 'utf8')),
cloudSql: await readCloudSqlBackends(
'staging', required(values, 'started-at'), required(values, 'ended-at')
)
}), null, 2)}\n`)
return
}
if (command === 'create-c27') {
const startedAt = required(values, 'started-at')
const endedAt = required(values, 'ended-at')
writeFileSync(output, `${JSON.stringify(buildC27CanaryEvidence({
...commonInput(values), startedAt, endedAt,
loadReport: JSON.parse(readFileSync(required(values, 'load-report'), 'utf8')),
logs: JSON.parse(readFileSync(required(values, 'logs-json'), 'utf8')),
cloudSql: await readCloudSqlBackends('production', startedAt, endedAt)
}), null, 2)}\n`)
return
}
if (!['verify-staging', 'verify-c27'].includes(command)) throw new Error('invalid evidence command')
const kind = command === 'verify-staging' ? 'staging-asia-readiness' : 'production-c27-canary'
verifyRolloutEvidence(
JSON.parse(readFileSync(required(values, 'evidence'), 'utf8')),
JSON.parse(readFileSync(required(values, 'run-json'), 'utf8')),
{
kind, repository: REPOSITORY,
workflow: kind === 'staging-asia-readiness' ? STAGING_WORKFLOW : ADMISSION_WORKFLOW,
environment: kind === 'staging-asia-readiness' ? 'staging' : 'production',
commitSha: required(values, 'commit-sha'), imageDigest: required(values, 'image-digest'),
cellIds: [kind === 'staging-asia-readiness' ? STAGING_CELL : C27],
selectorGeneration: values.has('selector-generation')
? positiveInteger(values.get('selector-generation'), 'selector generation') : undefined,
now: required(values, 'now')
}
)
writeFileSync(output, 'verified\n')
}
if (process.argv[1] === fileURLToPath(import.meta.url)) await main(process.argv.slice(2))
@@ -0,0 +1,357 @@
import assert from 'node:assert/strict'
import { test } from 'node:test'
import { RELAY_GITHUB_REPOSITORY, relayWorkflowPath } from './relay-repository.mjs'
import {
buildC27CanaryEvidence,
buildStagingEvidence,
verifyRolloutEvidence
} from './relay-asia-rollout-evidence.mjs'
const digest = `sha256:${'a'.repeat(64)}`
const commitSha = 'b'.repeat(40)
const repository = RELAY_GITHUB_REPOSITORY
const start = new Date('2026-08-13T12:00:00.000Z')
const end = new Date(start.valueOf() + 15 * 60_000)
const canaryEnd = new Date(start.valueOf() + 5 * 60_000)
function sourceInput(overrides = {}) {
return {
repository, runId: 123, runAttempt: 2, commitSha, imageDigest: digest,
selectorGeneration: 9, ...overrides
}
}
function metricLog(timestamp, role, cellId, overrides = {}) {
return {
timestamp: timestamp.toISOString(),
resource: { labels: role === 'director' ? { instance_id: 'director-1' } : {} },
jsonPayload: {
role, cellId, region: role === 'cell' ? 'asia-east2' : 'us-central1',
controls: role === 'cell' ? 2_840 : 0,
splices: role === 'cell' ? 120 : 0,
selectedRegionsDelta: role === 'director' ? { 'asia-east2': 1 } : {},
regionFallbacksDelta: {}, unavailableRegionsDelta: {}, sqlFailuresDelta: 0,
databasePoolWaiting: 0, databasePoolWaitersMax: 0, databasePoolWaitMsMax: 0,
...overrides
}
}
}
function completeLogs(cellId = 'production-gce-c27', windowEnd = end) {
const samples = (windowEnd.valueOf() - start.valueOf()) / 60_000 + 1
return Array.from({ length: samples }, (_, minute) => {
const timestamp = new Date(start.valueOf() + minute * 60_000)
return [
metricLog(timestamp, 'director', 'production-director'),
metricLog(timestamp, 'cell', cellId)
]
}).flat()
}
function cloudSql(max = 319, windowEnd = end) {
const samples = (windowEnd.valueOf() - start.valueOf()) / 60_000
return {
timeSeries: [{
points: Array.from({ length: samples }, (_, minute) => ({
interval: { endTime: new Date(start.valueOf() + (minute + 1) * 60_000).toISOString() },
value: { int64Value: String(minute === samples - 1 ? max : 300) }
}))
}]
}
}
function loadReport({
controls, shardIndex, splices = 0, slow = 0, wedged = 0,
launch = false
}) {
return {
event: 'relay_load_complete', controls, shardCount: 4, shardIndex,
configuredRampSeconds: 180, configuredSteadySeconds: 210, requiredLeaseHorizons: 2,
configuredSpliceHoldSeconds: 210,
configuredSplices: splices, configuredSlowReaderSplices: slow,
configuredWedgedReaderSplices: wedged,
peakActive: controls, steadyMinimumActive: controls,
connectionFailures: 0, rampConnectionFailures: 0, steadyConnectionFailures: 0,
transitionConnectionFailures: 0, unexpectedCloses: 0, protocolErrors: 0,
refreshErrors: 0, socketErrors: 0, failedSplices: 0,
regionalFallbacksProved: 0,
oldClientUsFirstProved: launch ? 1 : 0,
stickyAssignmentProved: launch ? 1 : 0,
requestUnitInvitesOpened: 0,
requestUnitPrincipalCount: 0,
relayAsiaLoadPrincipalCount: 32,
requestUnitOverflowReason: null,
requestUnitCleanupProved: 0,
phaseBarrierPassed: true,
rebindProbesOpened: launch ? 2 : 0,
rebindOverflowReason: null,
peakActiveSplices: splices, completedSplices: splices - wedged,
slowReaderSplicesCompleted: slow, wedgedReaderSplicesClosed: wedged,
readerQueuedBytesPeak: slow > 0 ? 1_024 : 0,
generatorCpuPercent: 25, generatorEventLoopP99Ms: 20, generatorRssGrowthMiB: 10,
readerQueueEvidence: slow > 0 ? [{
origin: 'https://c4.relay-staging.onorca.dev',
baselineBytes: 128,
peakBytes: 1_152,
increaseBytes: 1_024
}] : [],
shutdownEvidence: {
peerShutdowns: controls, activeControls: 0, activeSplices: 0, reconnectTimers: 0
}
}
}
function stagingInput(overrides = {}) {
const logs = completeLogs('staging-gce-c4')
logs[0].jsonPayload.regionFallbacksDelta = { 'us-central1': 1 }
return {
...sourceInput({ selectorGeneration: 4 }),
startedAt: start.toISOString(), endedAt: end.toISOString(),
launchReport: Array.from({ length: 4 }, (_, shardIndex) => loadReport({
controls: 5, shardIndex, splices: 5, launch: shardIndex === 0,
slow: shardIndex === 0 ? 4 : 0, wedged: shardIndex === 0 ? 1 : 0
})),
logs, cloudSql: cloudSql(), ...overrides
}
}
function canaryInput(overrides = {}) {
const load = loadReport({ controls: 1, shardIndex: 0, splices: 1 })
Object.assign(load, {
shardCount: 1,
configuredSteadySeconds: 300,
configuredSpliceHoldSeconds: 60,
relayAsiaLoadPrincipalCount: 1
})
return {
...sourceInput(), startedAt: start.toISOString(), endedAt: canaryEnd.toISOString(),
loadReport: load,
logs: completeLogs('production-gce-c27', canaryEnd), cloudSql: cloudSql(319, canaryEnd),
...overrides
}
}
function workflowRun(evidence, overrides = {}) {
return {
id: evidence.source.runId, run_attempt: evidence.source.runAttempt,
conclusion: 'success', event: 'workflow_dispatch', head_branch: 'main',
head_sha: evidence.source.commitSha,
repository: { full_name: evidence.source.repository }, path: evidence.source.workflow,
...overrides
}
}
function verifyExpected(kind, overrides = {}) {
const staging = kind === 'staging-asia-readiness'
return {
kind, repository,
workflow: staging
? relayWorkflowPath('prove-relay-asia-staging.yml')
: relayWorkflowPath('operate-relay-asia-admission.yml'),
environment: staging ? 'staging' : 'production', commitSha, imageDigest: digest,
cellIds: [staging ? 'staging-gce-c4' : 'production-gce-c27'],
now: new Date(end.valueOf() + 60_000).toISOString(), ...overrides
}
}
test('accepts the exact sharded staging load, telemetry, and provenance proof', () => {
const evidence = buildStagingEvidence(stagingInput())
assert.equal(evidence.load.launch.controls, 20)
assert.equal(evidence.load.launch.peakActiveSplices, 20)
assert.equal(evidence.load.launch.readerQueueEvidence.length, 1)
assert.equal(verifyRolloutEvidence(
evidence, workflowRun(evidence), verifyExpected('staging-asia-readiness')
), evidence)
})
test('ignores unrelated legacy cell metrics outside the Asia proof', () => {
const input = stagingInput()
const legacy = metricLog(start, 'cell', 'staging-gce-c1', {
region: undefined,
sqlFailuresDelta: 1
})
delete legacy.jsonPayload.databasePoolWaiting
delete legacy.jsonPayload.databasePoolWaitersMax
delete legacy.jsonPayload.databasePoolWaitMsMax
input.logs.push(legacy)
assert.equal(buildStagingEvidence(input).metrics.relaySqlFailures, 0)
})
test('accepts bounded transient pool waits without a sampled queue', () => {
const input = stagingInput()
input.logs[0].jsonPayload.databasePoolWaitersMax = 4
input.logs[0].jsonPayload.databasePoolWaitMsMax = 50
const metrics = buildStagingEvidence(input).metrics
assert.equal(metrics.databasePoolWaitingMax, 0)
assert.equal(metrics.databasePoolWaitersMax, 4)
assert.equal(metrics.databasePoolWaitMsMax, 50)
})
test('requires exactly the intentional sticky-assignment fallback in staging', () => {
const missing = stagingInput()
missing.logs[0].jsonPayload.regionFallbacksDelta = {}
assert.throws(() => buildStagingEvidence(missing), /intentional probes/)
const extra = stagingInput()
extra.logs[2].jsonPayload.regionFallbacksDelta = { 'asia-east2': 1 }
assert.throws(() => buildStagingEvidence(extra), /intentional probes/)
const substituted = stagingInput()
substituted.logs[0].jsonPayload.regionFallbacksDelta = { 'asia-east2': 1 }
assert.throws(() => buildStagingEvidence(substituted), /intentional probes/)
})
test('accepts the wedged splice outside the sustained non-wedged peak', () => {
const input = stagingInput()
input.launchReport[0].peakActiveSplices = 4
input.logs.filter((entry) => entry.jsonPayload.role === 'cell')
.forEach((entry) => { entry.jsonPayload.splices = 19 })
const evidence = buildStagingEvidence(input)
assert.equal(evidence.load.launch.peakActiveSplices, 19)
input.launchReport[0].peakActiveSplices = 3
assert.throws(() => buildStagingEvidence(input), /mixed load evidence/)
})
test('rejects staging evidence with incomplete load or cleanup', () => {
const input = stagingInput()
input.launchReport[0].steadyMinimumActive--
assert.throws(() => buildStagingEvidence(input), /required controls/)
const cleanup = stagingInput()
cleanup.launchReport[0].shutdownEvidence.activeControls = 1
assert.throws(() => buildStagingEvidence(cleanup), /cleanup/)
const nonCausal = stagingInput()
nonCausal.launchReport[0].readerQueueEvidence[0].increaseBytes = 0
assert.throws(() => buildStagingEvidence(nonCausal), /not causal/)
const multipleOwners = stagingInput()
multipleOwners.launchReport[0].configuredSlowReaderSplices--
multipleOwners.launchReport[0].slowReaderSplicesCompleted--
multipleOwners.launchReport[1] = loadReport({
controls: 5, shardIndex: 1, splices: 5, slow: 1
})
assert.throws(() => buildStagingEvidence(multipleOwners), /one causal owner/)
const overloaded = stagingInput()
overloaded.launchReport[0].generatorCpuPercent = 80
assert.throws(() => buildStagingEvidence(overloaded), /insufficient headroom/)
const missingLaunchProof = stagingInput()
missingLaunchProof.launchReport[0].rebindProbesOpened = 0
assert.throws(() => buildStagingEvidence(missingLaunchProof), /launch-path/)
const offTarget = stagingInput()
offTarget.logs.filter((entry) => entry.jsonPayload.role === 'cell')
.forEach((entry) => { entry.jsonPayload.controls = 19 })
assert.throws(() => buildStagingEvidence(offTarget), /did not reach C4/)
})
test('rejects staging evidence with a shortened splice hold', () => {
const input = stagingInput()
input.launchReport[0].configuredSpliceHoldSeconds = 60
assert.throws(() => buildStagingEvidence(input), /load profile does not match/)
})
for (const [label, value] of [['missing', undefined], ['malformed', 'invalid']]) {
test(`rejects staging evidence with a ${label} splice hold`, () => {
const input = stagingInput()
input.launchReport[0].configuredSpliceHoldSeconds = value
assert.throws(() => buildStagingEvidence(input), /staging splice hold seconds is invalid/)
})
}
for (const [label, mutate, message] of [
['phase barrier', (input) => { input.launchReport[0].phaseBarrierPassed = false }, /profile/],
['old-client routing', (input) => { input.launchReport[0].oldClientUsFirstProved = 0 }, /launch-path/],
['sticky routing', (input) => { input.launchReport[0].stickyAssignmentProved = 0 }, /launch-path/]
]) {
test(`rejects staging evidence without ${label} proof`, () => {
const input = stagingInput()
mutate(input)
assert.throws(() => buildStagingEvidence(input), message)
})
}
test('rejects staging evidence from a non-canonical repository', () => {
assert.throws(() => buildStagingEvidence(stagingInput({ repository: 'fork/orca-cloud' })), /repository/)
})
test('rejects mismatched staging provenance, digest, topology, or age', () => {
const evidence = buildStagingEvidence(stagingInput())
const expected = verifyExpected('staging-asia-readiness')
assert.throws(() => verifyRolloutEvidence(evidence, workflowRun(evidence, {
head_sha: 'c'.repeat(40)
}), expected), /provenance/)
assert.throws(() => verifyRolloutEvidence(evidence, workflowRun(evidence), {
...expected, commitSha: 'c'.repeat(40)
}), /provenance/)
assert.throws(() => verifyRolloutEvidence(evidence, workflowRun(evidence), {
...expected, imageDigest: `sha256:${'c'.repeat(64)}`
}), /image digest/)
assert.throws(() => verifyRolloutEvidence({
...evidence, topology: { ...evidence.topology, cellIds: ['staging-gce-c3'] }
}, workflowRun(evidence), expected), /topology/)
assert.throws(() => verifyRolloutEvidence(evidence, workflowRun(evidence), {
...expected, now: new Date(end.valueOf() + 25 * 60 * 60_000).toISOString()
}), /stale/)
})
test('builds and verifies a passing continuous 5-minute C27 canary', () => {
const evidence = buildC27CanaryEvidence(canaryInput())
assert.equal(evidence.load.completedSplices, 1)
assert.equal(evidence.metrics.asiaSelections, 6)
assert.equal(evidence.metrics.cloudSqlBackendsMax, 319)
assert.equal(verifyRolloutEvidence(
evidence, workflowRun(evidence),
verifyExpected('production-c27-canary', { selectorGeneration: 9 })
), evidence)
})
test('rejects short, sparse, or unrelated-cell-only C27 coverage', () => {
assert.throws(() => buildC27CanaryEvidence(canaryInput({
endedAt: new Date(canaryEnd.valueOf() - 1).toISOString()
})), /shorter than 5 minutes/)
const sparse = completeLogs('production-gce-c27', canaryEnd).filter((entry) =>
entry.timestamp === start.toISOString() || entry.timestamp === canaryEnd.toISOString()
)
assert.throws(() => buildC27CanaryEvidence(canaryInput({ logs: sparse })), /sampling gap/)
assert.throws(() => buildC27CanaryEvidence(canaryInput({
logs: completeLogs('production-gce-c26', canaryEnd)
})), /production-gce-c27 metrics has no samples/)
})
test('rejects a C27 canary without a real control and splice', () => {
const input = canaryInput()
input.loadReport.completedSplices = 0
assert.throws(() => buildC27CanaryEvidence(input), /did not match/)
const shortHold = canaryInput()
shortHold.loadReport.configuredSpliceHoldSeconds = 59
assert.throws(() => buildC27CanaryEvidence(shortHold), /did not match/)
})
for (const [label, mutation, message] of [
['Asia selections', (input) => input.logs.forEach((entry) => { entry.jsonPayload.selectedRegionsDelta = {} }), /no Asia selections/],
['region fallbacks', (input) => { input.logs[0].jsonPayload.regionFallbacksDelta = { 'asia-east2': 1 } }, /regionFallbacks/],
['unavailable regions', (input) => { input.logs[0].jsonPayload.unavailableRegionsDelta = { 'asia-east2': 1 } }, /unavailableRegions/],
['Relay SQL failures', (input) => { input.logs[0].jsonPayload.sqlFailuresDelta = 1 }, /relaySqlFailures/],
['pool waiting', (input) => { input.logs[0].jsonPayload.databasePoolWaiting = 1 }, /databasePoolWaitingMax/],
['pool waiters', (input) => { input.logs[0].jsonPayload.databasePoolWaitersMax = 5 }, /transient database pool pressure/],
['pool wait time', (input) => { input.logs[0].jsonPayload.databasePoolWaitMsMax = 51 }, /transient database pool pressure/],
['C27 controls', (input) => input.logs.filter((entry) => entry.jsonPayload.role === 'cell')
.forEach((entry) => { entry.jsonPayload.controls = 0 }), /did not reach C27/],
['C27 splices', (input) => input.logs.filter((entry) => entry.jsonPayload.role === 'cell')
.forEach((entry) => { entry.jsonPayload.splices = 0 }), /did not reach C27/],
['Cloud SQL headroom', (input) => { input.cloudSql = cloudSql(320, canaryEnd) }, /below 320/]
]) {
test(`rejects C27 evidence with ${label}`, () => {
const input = canaryInput()
mutation(input)
assert.throws(() => buildC27CanaryEvidence(input), message)
})
}
test('rejects C27 evidence from a different selector generation', () => {
const evidence = buildC27CanaryEvidence(canaryInput())
assert.throws(() => verifyRolloutEvidence(
evidence, workflowRun(evidence),
verifyExpected('production-c27-canary', { selectorGeneration: 10 })
), /selector generation/)
})
@@ -0,0 +1,124 @@
import assert from 'node:assert/strict'
import { readFileSync } from 'node:fs'
import { test } from 'node:test'
import { relayWorkflowUrl } from './relay-repository.mjs'
const workflow = readFileSync(
relayWorkflowUrl('deploy-relay-asia-topology.yml'),
'utf8'
)
const iam = readFileSync(
new URL('../../infra/terraform/relay-asia-topology-iam.tf', import.meta.url),
'utf8'
)
const cells = readFileSync(
new URL('../../infra/terraform/relay-gce-cells.tf', import.meta.url),
'utf8'
)
const variables = readFileSync(
new URL('../../infra/terraform/variables.tf', import.meta.url),
'utf8'
)
test('uses only its exact workflow-bound topology identity', () => {
assert.match(workflow, /production-cloud-sql-rollout/)
assert.match(workflow, /relay-staging-mutation/)
assert.match(workflow, /RELAY_ASIA_TOPOLOGY_WORKLOAD_IDENTITY_PROVIDER/)
assert.match(workflow, /RELAY_ASIA_TOPOLOGY_SERVICE_ACCOUNT/)
assert.doesNotMatch(workflow, /GCP_DEPLOY_SERVICE_ACCOUNT/)
assert.match(
iam,
/assertion\.workflow_ref == '\$\{prefix\}\$\{local\.github_relay_asia_topology_workflow_file\}@refs\/heads\/main'/
)
assert.match(iam, /assertion\.ref == 'refs\/heads\/main'/)
assert.match(iam, /assertion\.event_name == 'workflow_dispatch'/)
assert.match(iam, /assertion\.environment == '\$\{var\.environment\}'/)
})
test('plans only additive Asia topology and applies the saved plan', () => {
assert.equal((workflow.match(/manage_artifact_dns=false/g) ?? []).length, 2)
for (const target of [
'relay_gce_additional',
'google_compute_instance_template.relay_gce_cell',
'google_compute_instance_group_manager.relay_gce_cell',
'google_compute_backend_service.relay_gce_cell',
'google_compute_url_map.relay_gce'
]) assert.match(workflow, new RegExp(target.replaceAll('.', '\\.')))
assert.match(workflow, /apply -input=false -auto-approve "\$\{\{ steps\.plan\.outputs\.plan \}\}"/)
assert.match(workflow, /prepare-relay-asia-topology-input\.mjs/)
assert.doesNotMatch(workflow, /steps\.variables\.outputs\.file/)
assert.match(workflow, /\.variables\.relay_gce_cells\.value/)
assert.match(
workflow,
/\.variables\.relay_gce_additional_region_subnetwork_cidrs\.value/
)
assert.doesNotMatch(workflow, /terraform -chdir=infra\/terraform console/)
assert.equal((workflow.match(/-var-file="\$\{TF_VARS\}"/g) ?? []).length, 2)
assert.doesNotMatch(workflow, /terraform[^\n]*apply[^\n]*-target/)
assert.doesNotMatch(workflow, /google_(?:sql|cloudflare|dns|certificate_manager)/)
})
test('validates before apply and proves convergence afterward', () => {
assert.equal((workflow.match(/validate-relay-asia-topology-plan\.mjs/g) ?? []).length, 2)
assert.match(workflow, /APPLY_RELAY_ASIA_TOPOLOGY/)
assert.match(workflow, /test "\$\(jq -er '\.changes'/)
assert.match(workflow, /Register the exact new cells atomically as migration-only/)
})
test('checks the connection budget and production live ceiling before planning', () => {
assert.match(workflow, /relay-cloud-sql-connection-budget\.mjs/)
assert.match(workflow, /gcloud sql instances describe "\$\{CLOUD_SQL_INSTANCE\}"/)
assert.match(workflow, /select\(\.name == "max_connections"\)/)
assert.match(workflow, /VERIFIED_DEFAULT_MAX_CONNECTIONS_TIER: db-custom-4-15360/)
assert.match(workflow, /VERIFIED_DEFAULT_MAX_CONNECTIONS_DATABASE_VERSION: POSTGRES_17/)
assert.match(workflow, /live_source=verified-shape-default/)
assert.match(workflow, /test "\$\(jq -er '\.settings\.tier'/)
assert.match(workflow, /test "\$\(jq -er '\.databaseVersion'/)
assert.match(workflow, /test "\$\{live_max\}" = "\$\{checked_max\}"/)
assert.ok(
workflow.indexOf('relay-cloud-sql-connection-budget.mjs') <
workflow.indexOf('terraform -chdir=infra/terraform plan')
)
})
test('binds computed Asia references to the matching Terraform cell resources', () => {
assert.match(cells, /instance_template = google_compute_instance_template\.relay_gce_cell\[each\.key\]\.self_link/)
assert.match(cells, /group\s+= google_compute_instance_group_manager\.relay_gce_cell\[each\.key\]\.instance_group/)
assert.match(cells, /default_service = google_compute_backend_service\.relay_gce_cell\[cell\.key\]\.id/)
assert.match(cells, /subnetwork = local\.relay_gce_subnetworks\[each\.value\.region\]/)
})
test('keeps cross-variable region constraints in Terraform 1.5 check blocks', () => {
assert.doesNotMatch(variables, /region != var\.region/)
assert.doesNotMatch(variables, /cell\.region == var\.region/)
assert.match(cells, /check "relay_gce_fixed_one_topology"[\s\S]*?region != var\.region/)
assert.match(cells, /cell\.region == var\.region[\s\S]*?configured subnetwork/)
})
test('the custom role cannot delete topology or mutate SQL and DNS', () => {
assert.doesNotMatch(iam, /compute\.[A-Za-z]+\.delete/)
assert.doesNotMatch(
iam,
/roles\/viewer|cloudsql\.instances\.(?:update|delete)|dns\.|certificatemanager|cloudflare/i
)
assert.match(iam, /resource "google_project_iam_custom_role" "github_relay_asia_topology_read"/)
assert.match(iam, /"cloudsql\.instances\.get"/)
assert.match(iam, /"run\.revisions\.get"/)
assert.match(iam, /"run\.services\.get"/)
assert.match(iam, /"serviceusage\.services\.list"/)
assert.match(iam, /"compute\.networks\.updatePolicy"/)
assert.match(iam, /"compute\.healthChecks\.useReadOnly"/)
assert.match(iam, /"compute\.instanceGroups\.create"/)
assert.match(iam, /"compute\.instances\.use"/)
assert.match(iam, /roles\/storage\.objectAdmin/)
assert.match(iam, /default\.tfstate/)
assert.match(iam, /default\.tflock/)
assert.match(
iam,
/resource "google_project_iam_custom_role" "github_relay_asia_topology_state_list"[\s\S]*?permissions = \["storage\.objects\.list"\]/
)
assert.match(
iam,
/resource "google_storage_bucket_iam_member" "github_relay_asia_topology_state_list"[\s\S]*?role\s+= google_project_iam_custom_role\.github_relay_asia_topology_state_list\[0\]\.id/
)
})
@@ -0,0 +1,148 @@
import { readFileSync } from 'node:fs'
import { fileURLToPath } from 'node:url'
const DEFAULT_PATHS = {
productionTfvars: new URL('../../infra/terraform/environments/production.tfvars', import.meta.url),
terraformVariables: new URL('../../infra/terraform/variables.tf', import.meta.url),
relayConfig: new URL('../../apps/relay/src/config.ts', import.meta.url)
}
// Auth and API live outside the Relay tree, so their consumption is published as a contract
// rather than parsed from their source. production-cloud-sql-app-consumers.test.mjs binds it back.
const APP_CONSUMERS_CONTRACT = new URL(
'../contracts/production-cloud-sql-app-consumers.json',
import.meta.url
)
const APP_CONSUMER_FIELDS = ['authInstances', 'authPoolMax', 'apiInstances', 'apiPoolMax', 'maxConnections']
export function readProductionCloudSqlAppConsumers(contract) {
const parsed = contract ?? JSON.parse(readFileSync(APP_CONSUMERS_CONTRACT, 'utf8'))
for (const field of APP_CONSUMER_FIELDS) {
const value = parsed[field]
if (!Number.isSafeInteger(value) || value <= 0) {
throw new Error(`could not read ${field} from the app consumer contract`)
}
}
return parsed
}
function requiredInteger(source, pattern, label) {
const value = Number(source.match(pattern)?.[1])
if (!Number.isSafeInteger(value) || value < 0) throw new Error(`could not read ${label}`)
return value
}
function productionCells(source, defaultPoolMax) {
const fencedMatch = source.match(/relay_gce_fenced_cells\s*=\s*\[([^\]]*)\]/)
if (!fencedMatch) throw new Error('could not read fenced Relay cells')
const fenced = new Set([...fencedMatch[1].matchAll(/"([^"]+)"/g)].map((match) => match[1]))
const cells = [...source.matchAll(/"(production-gce-[^"]+)"\s*=\s*\{([\s\S]*?)\n\s*\}/g)].map(
([, id, body]) => ({
id,
fenced: fenced.has(id),
region: body.match(/\bregion\s*=\s*"([^"]+)"/)?.[1] ?? 'us-central1',
poolMax: body.match(/\bdatabase_pool_max\s*=\s*(\d+)/)
? Number(body.match(/\bdatabase_pool_max\s*=\s*(\d+)/)[1])
: defaultPoolMax
})
)
if (cells.length === 0) throw new Error('could not read production Relay cells')
return cells
}
export function calculateRelayCloudSqlConnectionBudget(inputs) {
const consumers = {
cells: inputs.cellPoolTotal + inputs.asiaCellCount * inputs.asiaPoolMax,
directors: inputs.directorInstances * inputs.directorPoolMax,
auth: inputs.authInstances * inputs.authPoolMax,
api: inputs.apiInstances * inputs.apiPoolMax
}
const configuredMaximum = Object.values(consumers).reduce((total, value) => total + value, 0)
const retainedDirectorRollback = inputs.directorInstances * inputs.directorPoolMax
const candidateOverlap = {
relayDirectorCandidate: retainedDirectorRollback * 2,
apiCandidate: retainedDirectorRollback + inputs.apiInstances * inputs.apiPoolMax,
authCandidate: retainedDirectorRollback + inputs.authInstances * inputs.authPoolMax,
relayCells: retainedDirectorRollback
}
const rolloutOverlap = Math.max(...Object.values(candidateOverlap))
const operatingMaximum = configuredMaximum + rolloutOverlap + inputs.maintenanceAdminAllowance
const usableCeiling = inputs.maxConnections - inputs.explicitReserve
const budgetedTotal = operatingMaximum + inputs.explicitReserve
return {
maxConnections: inputs.maxConnections,
consumers,
asia: { cells: inputs.asiaCellCount, poolMax: inputs.asiaPoolMax },
configuredMaximum,
rolloutOverlap: {
...candidateOverlap,
retainedDirectorRollback,
maximum: rolloutOverlap,
reason: 'serialized rollouts include directly addressable tagged revisions outside service-level caps'
},
maintenanceAdminAllowance: inputs.maintenanceAdminAllowance,
maintenanceAdminAllowanceReason: 'covers bounded work outside configured services',
explicitReserve: inputs.explicitReserve,
explicitReserveReason: 'remains unavailable to configured services and planned rollouts',
usableCeiling,
operatingMaximum,
remainingWithinUsableCeiling: usableCeiling - operatingMaximum,
budgetedTotal,
unallocated: inputs.maxConnections - budgetedTotal,
withinBudget: operatingMaximum <= usableCeiling && budgetedTotal < inputs.maxConnections
}
}
export function readRelayCloudSqlConnectionBudget({
sources,
appConsumers,
proposedAsiaCellCount = 3,
asiaPoolMax = 10,
maxConnections,
maintenanceAdminAllowance = 5,
explicitReserve = 10
} = {}) {
const read = (name) => sources?.[name] ?? readFileSync(DEFAULT_PATHS[name], 'utf8')
const apps = readProductionCloudSqlAppConsumers(appConsumers)
const productionTfvars = read('productionTfvars')
const terraformVariables = read('terraformVariables')
const relayConfig = read('relayConfig')
const cells = productionCells(
productionTfvars,
requiredInteger(relayConfig, /RELAY_DATABASE_POOL_MAX\s*=\s*(\d+)/, 'Relay pool maximum')
)
const poweredCells = cells.filter(({ fenced }) => !fenced)
const configuredAsiaCells = poweredCells.filter(({ region }) => region === 'asia-east2')
const nonAsiaCells = poweredCells.filter(({ region }) => region !== 'asia-east2')
const cellPoolTotal = nonAsiaCells.reduce((total, cell) => total + cell.poolMax, 0)
const asiaCellCount = configuredAsiaCells.length || proposedAsiaCellCount
const configuredAsiaPoolMax = configuredAsiaCells[0]?.poolMax ?? asiaPoolMax
if (configuredAsiaCells.some(({ poolMax }) => poolMax !== configuredAsiaPoolMax)) {
throw new Error('Asia Relay cells must use one checked pool maximum')
}
return calculateRelayCloudSqlConnectionBudget({
cellPoolTotal,
asiaCellCount,
asiaPoolMax: configuredAsiaPoolMax,
directorInstances: requiredInteger(productionTfvars, /relay_max_instances\s*=\s*(\d+)/, 'director instances'),
directorPoolMax: requiredInteger(
terraformVariables,
/variable\s+"relay_director_database_pool_max"[\s\S]*?default\s*=\s*(\d+)/,
'director pool maximum'
),
authInstances: apps.authInstances,
authPoolMax: apps.authPoolMax,
apiInstances: apps.apiInstances,
apiPoolMax: apps.apiPoolMax,
maxConnections: maxConnections ?? apps.maxConnections,
maintenanceAdminAllowance,
explicitReserve
})
}
if (process.argv[1] === fileURLToPath(import.meta.url)) {
const report = readRelayCloudSqlConnectionBudget()
console.log(JSON.stringify({ event: 'relay_cloud_sql_connection_budget', ...report }, null, 2))
if (!report.withinBudget) process.exitCode = 1
}
@@ -0,0 +1,110 @@
import assert from 'node:assert/strict'
import { readFileSync } from 'node:fs'
import test from 'node:test'
import {
calculateRelayCloudSqlConnectionBudget,
readRelayCloudSqlConnectionBudget
} from './relay-cloud-sql-connection-budget.mjs'
test('production plus three Asia pools preserves allowance and reserve below the ceiling', () => {
const report = readRelayCloudSqlConnectionBudget()
assert.deepEqual(report.consumers, { cells: 230, directors: 15, auth: 20, api: 50 })
assert.deepEqual(report.asia, { cells: 3, poolMax: 10 })
assert.equal(report.configuredMaximum, 315)
assert.equal(report.rolloutOverlap.relayDirectorCandidate, 30)
assert.equal(report.rolloutOverlap.apiCandidate, 65)
assert.equal(report.rolloutOverlap.authCandidate, 35)
assert.equal(report.rolloutOverlap.relayCells, 15)
assert.equal(report.rolloutOverlap.retainedDirectorRollback, 15)
assert.equal(report.rolloutOverlap.maximum, 65)
assert.equal(report.maintenanceAdminAllowance, 5)
assert.equal(report.explicitReserve, 10)
assert.equal(report.usableCeiling, 390)
assert.equal(report.operatingMaximum, 385)
assert.equal(report.remainingWithinUsableCeiling, 5)
assert.equal(report.budgetedTotal, 395)
assert.equal(report.unallocated, 5)
assert.equal(report.withinBudget, true)
})
test('fails closed when pool growth consumes the explicit reserve', () => {
const report = calculateRelayCloudSqlConnectionBudget({
cellPoolTotal: 200,
asiaCellCount: 3,
asiaPoolMax: 20,
directorInstances: 5,
directorPoolMax: 3,
authInstances: 2,
authPoolMax: 10,
apiInstances: 20,
apiPoolMax: 5,
maxConnections: 400,
maintenanceAdminAllowance: 5,
explicitReserve: 10
})
assert.equal(report.operatingMaximum, 515)
assert.equal(report.withinBudget, false)
})
test('excludes fenced cell pools and reads per-cell pool overrides', () => {
const report = readRelayCloudSqlConnectionBudget({
proposedAsiaCellCount: 1,
appConsumers: { authInstances: 1, authPoolMax: 10, apiInstances: 1, apiPoolMax: 5, maxConnections: 100 },
sources: {
productionTfvars: `
relay_max_instances = 1
relay_gce_fenced_cells = ["production-gce-c1"]
relay_gce_cells = {
"production-gce-c1" = { database_pool_max = 99
}
"production-gce-c2" = { database_pool_max = 4
}
}
`,
terraformVariables: 'variable "relay_director_database_pool_max" { default = 3 }',
relayConfig: 'export const RELAY_DATABASE_POOL_MAX = 10'
},
maxConnections: 100,
maintenanceAdminAllowance: 1,
explicitReserve: 1
})
assert.equal(report.consumers.cells, 14)
assert.equal(report.operatingMaximum, 46)
assert.equal(report.budgetedTotal, 47)
})
test('requires strict headroom below the physical ceiling', () => {
const report = calculateRelayCloudSqlConnectionBudget({
cellPoolTotal: 20,
asiaCellCount: 0,
asiaPoolMax: 10,
directorInstances: 1,
directorPoolMax: 3,
authInstances: 1,
authPoolMax: 10,
apiInstances: 1,
apiPoolMax: 5,
maxConnections: 50,
maintenanceAdminAllowance: 9,
explicitReserve: 3
})
assert.equal(report.budgetedTotal, 63)
assert.equal(report.withinBudget, false)
})
test('pages Relay channels when Cloud SQL backends consume headroom', () => {
const terraform = readFileSync(
new URL('../../infra/terraform/relay-observability.tf', import.meta.url),
'utf8'
)
const policy = terraform.match(
/resource "google_monitoring_alert_policy" "relay_cloud_sql_backends" \{([\s\S]*?)\n\}/
)?.[1]
assert.ok(policy)
assert.match(policy, /notification_channels\s*=\s*var\.relay_alert_notification_channels/)
})
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,37 @@
export function relayLoadFailureReason(error) {
const message = error instanceof Error ? error.message : String(error)
const tokenExchange = /^relay token exchange failed: ([1-5][0-9]{2})$/.exec(message)
if (tokenExchange) return `token_http_${tokenExchange[1]}`
const assignment =
/^relay assignment failed: ([1-5][0-9]{2})(?: (relay_capacity_exhausted|relay_connection_headroom_exhausted))?$/.exec(
message
)
if (assignment?.[1] === '503' && assignment[2]) return 'assignment_capacity_exhausted'
if (assignment) return `assignment_http_${assignment[1]}`
const closed = /^control closed: ([0-9]{4})\b/.exec(message)
if (closed) return `control_close_${closed[1]}`
if (message === 'control open timeout') return 'control_open_timeout'
if (message === 'control response timeout') return 'control_response_timeout'
if (message === 'relay token exchange timeout') return 'token_timeout'
if (message === 'relay assignment timeout') return 'assignment_timeout'
if (message === 'WebSocket was closed before the connection was established') {
return 'socket_closed_before_open'
}
if (message === 'relay token exchange omitted token') return 'token_response_invalid'
if (message === 'relay assignment response invalid') return 'assignment_response_invalid'
if (message === 'expected host challenge') return 'host_challenge_invalid'
if (message === 'host proof challenge did not decrypt') return 'host_challenge_decrypt_failed'
if (message === 'expected host hello acknowledgement') return 'host_ack_invalid'
const socketResponse = /^Unexpected server response: ([1-5][0-9]{2})\b/.exec(message)
if (socketResponse) return `socket_http_${socketResponse[1]}`
if (/\b(?:ECONNREFUSED|ECONNRESET|EHOSTUNREACH|ETIMEDOUT)\b/.test(message)) {
return 'socket_transport'
}
return 'unknown'
}
export function discardFailedLoadSocket(socket) {
if (!socket) return
socket.on('error', () => undefined)
socket.terminate()
}
@@ -0,0 +1,57 @@
import assert from 'node:assert/strict'
import { EventEmitter } from 'node:events'
import test from 'node:test'
import {
discardFailedLoadSocket,
relayLoadFailureReason
} from './relay-load-connection-failure.mjs'
test('classifies only bounded aggregate connection failure reasons', () => {
assert.equal(relayLoadFailureReason(new Error('relay token exchange failed: 503')), 'token_http_503')
assert.equal(relayLoadFailureReason(new Error('relay assignment failed: 503')), 'assignment_http_503')
assert.equal(
relayLoadFailureReason(
new Error('relay assignment failed: 503 relay_connection_headroom_exhausted')
),
'assignment_capacity_exhausted'
)
for (const status of [400, 429, 500]) {
assert.equal(
relayLoadFailureReason(
new Error(`${`relay assignment failed: ${status}`} relay_capacity_exhausted`)
),
`assignment_http_${status}`
)
}
assert.equal(relayLoadFailureReason(new Error('control closed: 4404 wrong cell')), 'control_close_4404')
assert.equal(relayLoadFailureReason(new Error('control open timeout')), 'control_open_timeout')
assert.equal(relayLoadFailureReason(new Error('relay token exchange timeout')), 'token_timeout')
assert.equal(relayLoadFailureReason(new Error('relay assignment timeout')), 'assignment_timeout')
assert.equal(
relayLoadFailureReason(new Error('relay token exchange omitted token')),
'token_response_invalid'
)
assert.equal(
relayLoadFailureReason(new Error('relay assignment response invalid')),
'assignment_response_invalid'
)
assert.equal(
relayLoadFailureReason(new Error('Unexpected server response: 503 Service Unavailable')),
'socket_http_503'
)
assert.equal(relayLoadFailureReason(new Error('connect ECONNRESET 127.0.0.1')), 'socket_transport')
assert.equal(relayLoadFailureReason(new Error('expected host challenge')), 'host_challenge_invalid')
assert.equal(
relayLoadFailureReason(new Error('host proof challenge did not decrypt')),
'host_challenge_decrypt_failed'
)
assert.equal(relayLoadFailureReason(new Error('expected host hello acknowledgement')), 'host_ack_invalid')
assert.equal(relayLoadFailureReason(new Error('host-sensitive detail')), 'unknown')
})
test('absorbs the setup error emitted while discarding a failed socket', () => {
const socket = new EventEmitter()
socket.terminate = () => socket.emit('error', new Error('closed before open'))
assert.doesNotThrow(() => discardFailedLoadSocket(socket))
})
@@ -0,0 +1,891 @@
import { createHash, createHmac } from 'node:crypto'
import { createRequire } from 'node:module'
import { controlPhase } from './relay-load-model.mjs'
import { discardFailedLoadSocket } from './relay-load-connection-failure.mjs'
const requireFromRelay = createRequire(new URL('../../apps/relay/package.json', import.meta.url))
const nacl = requireFromRelay('tweetnacl')
const WebSocket = requireFromRelay('ws')
const { SignJWT } = await import(requireFromRelay.resolve('jose'))
const { buildHostProofMacInput, HOST_CHALLENGE_PLAINTEXT_DOMAIN } = await import(
requireFromRelay.resolve('@orca-cloud/relay-contract')
)
const CAPACITY_ASSIGNMENT_ERRORS = [
'relay_capacity_exhausted',
'relay_connection_headroom_exhausted'
]
function waitForOpen(socket, timeoutMs = 10_000) {
return new Promise((resolve, reject) => {
const timer = setTimeout(() => finish(new Error('control open timeout')), timeoutMs)
const finish = (error) => {
clearTimeout(timer)
socket.off('open', onOpen)
socket.off('close', onClose)
socket.off('error', onError)
if (error) reject(error)
else resolve()
}
const onOpen = () => finish()
const onClose = (code, reason) => finish(new Error(`control closed: ${code} ${reason}`))
const onError = (error) => finish(error)
socket.once('open', onOpen)
socket.once('close', onClose)
socket.once('error', onError)
})
}
function nextJson(socket, timeoutMs = 10_000) {
return new Promise((resolve, reject) => {
const timer = setTimeout(() => finish(new Error('control response timeout')), timeoutMs)
const finish = (error, value) => {
clearTimeout(timer)
socket.off('message', onMessage)
socket.off('close', onClose)
socket.off('error', onError)
if (error) reject(error)
else resolve(value)
}
const onMessage = (data) => {
try {
finish(undefined, JSON.parse(data.toString()))
} catch (error) {
finish(error)
}
}
const onClose = (code, reason) => finish(new Error(`control closed: ${code} ${reason}`))
const onError = (error) => finish(error)
socket.once('message', onMessage)
socket.once('close', onClose)
socket.once('error', onError)
})
}
function nextFrame(socket, timeoutMs = 10_000) {
return new Promise((resolve, reject) => {
const timer = setTimeout(() => finish(new Error('relay frame timeout')), timeoutMs)
const finish = (error, value) => {
clearTimeout(timer)
socket.off('message', onMessage)
socket.off('close', onClose)
socket.off('error', onError)
if (error) reject(error)
else resolve(value)
}
const onMessage = (data, binary) => finish(undefined, { bytes: Buffer.from(data), binary })
const onClose = (code, reason) => finish(new Error(`splice closed: ${code} ${reason}`))
const onError = (error) => finish(error)
socket.once('message', onMessage)
socket.once('close', onClose)
socket.once('error', onError)
})
}
function receiveBinaryStream(socket, expectedBytes, timeoutMs) {
return new Promise((resolve, reject) => {
let receivedBytes = 0
const hash = createHash('sha256')
const timer = setTimeout(() => finish(new Error('relay stream timeout')), timeoutMs)
const finish = (error, value) => {
clearTimeout(timer)
socket.off('message', onMessage)
socket.off('close', onClose)
socket.off('error', onError)
if (error) reject(error)
else resolve(value)
}
const onMessage = (data, binary) => {
if (!binary) return finish(new Error('relay changed stream opcode'))
const bytes = Buffer.from(data)
receivedBytes += bytes.byteLength
hash.update(bytes)
if (receivedBytes > expectedBytes) return finish(new Error('relay expanded reader stream'))
if (receivedBytes === expectedBytes) {
finish(undefined, { bytes: receivedBytes, digest: hash.digest('hex') })
}
}
const onClose = (code, reason) => finish(new Error(`splice closed: ${code} ${reason}`))
const onError = (error) => finish(error)
socket.on('message', onMessage)
socket.once('close', onClose)
socket.once('error', onError)
})
}
function closeInfo(socket, timeoutMs) {
return new Promise((resolve, reject) => {
const timer = setTimeout(() => finish(new Error('reader close timeout')), timeoutMs)
const finish = (error, value) => {
clearTimeout(timer)
socket.off('close', onClose)
socket.off('error', onError)
if (error) reject(error)
else resolve(value)
}
const onClose = (code, reason) => finish(undefined, { code, reason: reason.toString() })
const onError = (error) => finish(error)
socket.once('close', onClose)
socket.once('error', onError)
})
}
export function relayLoadWedgedCloseAccepted(closeCodes) {
return closeCodes.length === 2 && closeCodes[1] === 4429 &&
(closeCodes[0] === 4429 || closeCodes[0] === 1006)
}
function waitForClose(socket, timeoutMs = 10_000) {
if (!socket || socket.readyState === socket.CLOSED) return Promise.resolve()
return new Promise((resolve, reject) => {
const timer = setTimeout(() => {
socket.off('close', onClose)
discardFailedLoadSocket(socket)
reject(new Error('control close timeout'))
}, timeoutMs)
const onClose = () => {
clearTimeout(timer)
resolve()
}
socket.once('close', onClose)
})
}
async function cancelResponse(response) {
try {
await response.body?.cancel()
} catch {
// Preserve the bounded failure classification.
}
}
function proofForChallenge(challenge, hostSecretKey) {
const plaintext = nacl.box.open(
Buffer.from(challenge.ciphertextB64, 'base64'),
Buffer.from(challenge.nonceB64, 'base64'),
Buffer.from(challenge.relayEphemeralPublicKeyB64, 'base64'),
hostSecretKey
)
if (!plaintext) throw new Error('host proof challenge did not decrypt')
const domain = new TextEncoder().encode(`${HOST_CHALLENGE_PLAINTEXT_DOMAIN}\0`)
const transcriptLength = new DataView(
plaintext.buffer,
plaintext.byteOffset + domain.length,
4
).getUint32(0, false)
const transcriptStart = domain.length + 4
const transcript = plaintext.slice(transcriptStart, transcriptStart + transcriptLength)
const secret = plaintext.slice(transcriptStart + transcriptLength)
return createHmac('sha256', secret).update(buildHostProofMacInput(transcript)).digest('base64')
}
export class RelayLoadControlPeer {
constructor(index, options, observe) {
if (options.directorOrigin && options.targetOrigin) {
throw new Error('provide either directorOrigin or targetOrigin, not both')
}
this.index = index
this.options = options
this.observe = observe
this.keys = nacl.box.keyPair()
this.relayHostId = createHash('sha256')
.update(this.keys.publicKey)
.digest('base64url')
.slice(0, 16)
this.phase = controlPhase(index, options.seed)
this.socket = null
this.generation = undefined
this.controlResumeSecret = undefined
this.lastAssignment = undefined
this.refreshTimer = null
this.stopped = false
this.connecting = false
this.inFlight = new Set()
this.shutdownPromise = null
this.abortController = new AbortController()
this.drainExpected = false
this.controlWaiters = new Set()
this.spliceSockets = new Set()
this.spliceSequence = 0
}
connect() {
if (
this.stopped ||
this.connecting ||
(this.socket !== null && this.socket.readyState === this.socket.OPEN)
) {
return Promise.resolve()
}
this.connecting = true
const operation = this.connectOnce()
this.inFlight.add(operation)
const finish = () => {
this.connecting = false
this.inFlight.delete(operation)
}
operation.then(finish, finish)
return operation
}
assignedCellUrl() {
return this.lastAssignment?.cellUrl
}
async connectOnce() {
let socket = null
try {
const relayToken = await this.relayToken()
if (this.stopped) return
const assignment = await this.assignment(relayToken)
if (this.stopped) return
this.lastAssignment = assignment
socket = this.createSocket(assignment, relayToken)
this.socket = socket
this.drainExpected = false
await waitForOpen(socket)
if (this.stopped || this.socket !== socket) return
socket.send(
JSON.stringify({
type: 'host-hello',
v: 1,
relayHostId: this.relayHostId,
assignmentEpoch: assignment.assignmentEpoch,
hostPublicKeyB64: Buffer.from(this.keys.publicKey).toString('base64'),
appVersion: 'relay-load',
...(this.generation === undefined ? {} : { previousGeneration: this.generation }),
...(this.controlResumeSecret === undefined
? {}
: { controlResumeSecret: this.controlResumeSecret })
})
)
const challenge = await nextJson(socket)
if (this.stopped || this.socket !== socket) return
if (challenge.type !== 'host-challenge') throw new Error('expected host challenge')
socket.send(
JSON.stringify({
type: 'host-challenge-ack',
challengeId: challenge.challengeId,
proofB64: proofForChallenge(challenge, this.keys.secretKey)
})
)
const ack = await nextJson(socket)
if (this.stopped || this.socket !== socket) return
if (ack.type !== 'host-hello-ack') throw new Error('expected host hello acknowledgement')
this.generation = ack.generation
this.controlResumeSecret = ack.controlResumeSecret
socket.on('message', (data) => this.onMessage(socket, data))
socket.once('close', (code) => this.onClose(socket, code))
socket.once('error', (error) => this.observe('socketError', { index: this.index, error }))
this.observe('connected', { index: this.index })
this.scheduleRefresh(this.phase.refreshOffsetMs)
} catch (error) {
discardFailedLoadSocket(socket)
if (this.socket === socket) this.socket = null
if (this.stopped) return
throw error
}
}
createSocket(assignment, relayToken) {
return new WebSocket(`${assignment.cellUrl.replace(/^http/, 'ws')}/v1/host/control`, {
headers: { authorization: `Bearer ${relayToken}` },
perMessageDeflate: false
})
}
shutdown() {
if (this.shutdownPromise) return this.shutdownPromise
this.stopped = true
this.abortController.abort()
if (this.refreshTimer) clearTimeout(this.refreshTimer)
this.refreshTimer = null
this.shutdownPromise = this.shutdownOnce()
return this.shutdownPromise
}
async shutdownOnce() {
const socket = this.socket
const closed = waitForClose(socket)
for (const spliceSocket of this.spliceSockets) {
if (
spliceSocket.readyState !== spliceSocket.CLOSED &&
spliceSocket.readyState !== spliceSocket.CLOSING
) {
spliceSocket.close(1000, 'load complete')
}
}
this.rejectControlWaiters(new Error('control stopped'))
if (socket && socket.readyState !== socket.CLOSED && socket.readyState !== socket.CLOSING) {
socket.close(1000, 'load complete')
}
const settled = async () => {
while (this.inFlight.size > 0) {
await Promise.allSettled([...this.inFlight])
}
}
await Promise.all([closed, settled()])
this.observe('shutdown', {
index: this.index,
activeControls: this.socket?.readyState === this.socket?.OPEN ? 1 : 0,
activeSpliceSockets: this.spliceSockets.size,
inFlightOperations: this.inFlight.size,
refreshTimerActive: this.refreshTimer !== null
})
}
openSplice(options = {}) {
if (this.stopped) return Promise.reject(new Error('control stopped'))
const operation = this.openSpliceOnce(options)
this.inFlight.add(operation)
const finish = () => this.inFlight.delete(operation)
operation.then(finish, finish)
return operation
}
openInviteOffer() {
if (this.stopped) return Promise.reject(new Error('control stopped'))
const operation = this.openInviteOfferOnce()
this.inFlight.add(operation)
const finish = () => this.inFlight.delete(operation)
operation.then(finish, finish)
return operation
}
async openInviteOfferOnce() {
if (!this.socket || this.socket.readyState !== this.socket.OPEN) {
throw new Error('active control required for invite offer')
}
const sequence = this.spliceSequence++
const reqId = `load-offer-${this.index}-${sequence}`
const response = this.waitForControlMessage(
(message) =>
message.reqId === reqId &&
(message.type === 'invite-created' || message.type === 'control-error')
)
this.socket.send(JSON.stringify({
type: 'invite-create',
reqId,
relayDeviceId: `load-offer-device-${this.index}-${sequence}`
}))
const result = await response
if (result.type === 'control-error') throw new Error(`invite offer failed: ${result.code}`)
if (
typeof result.inviteToken !== 'string' ||
!Number.isSafeInteger(result.expiresAt) ||
result.expiresAt <= Date.now()
) throw new Error('relay invite offer response invalid')
}
async openSpliceOnce({
payloadBytes = 64,
readerMode = 'normal',
readerHoldMs = 0,
streamBytes = payloadBytes,
frameBytes = payloadBytes,
observeReaderPressure = async () => undefined,
readerDelay = async (ms) => await new Promise((resolve) => setTimeout(resolve, ms)),
slowReaderHoldMs = 0,
holdMs = 0
} = {}) {
if (
!this.socket ||
this.socket.readyState !== this.socket.OPEN ||
this.generation === undefined ||
this.lastAssignment === undefined
) {
throw new Error('active control required for splice')
}
if (!Number.isSafeInteger(payloadBytes) || payloadBytes < 1) {
throw new Error('splice payload bytes must be positive')
}
const sequence = this.spliceSequence++
const reqId = `load-invite-${this.index}-${sequence}`
const relayDeviceId = `load-device-${this.index}-${sequence}`
let phone
let data
let opened = false
try {
const invitePromise = this.waitForControlMessage(
(message) => message.type === 'invite-created' && message.reqId === reqId
)
this.socket.send(JSON.stringify({ type: 'invite-create', reqId, relayDeviceId }))
const invite = await invitePromise
if (typeof invite.inviteToken !== 'string') throw new Error('relay invite response invalid')
phone = this.createClientSocket(this.lastAssignment)
this.trackSpliceSocket(phone)
await waitForOpen(phone)
const connectionPromise = this.waitForControlMessage(
(message) => message.type === 'conn-open' && message.relayDeviceId === relayDeviceId
)
phone.send(
JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential: invite.inviteToken })
)
const connection = await connectionPromise
if (typeof connection.connId !== 'string' || typeof connection.connTicket !== 'string') {
throw new Error('relay connection response invalid')
}
data = this.createHostDataSocket(this.lastAssignment, connection.connId)
this.trackSpliceSocket(data)
await waitForOpen(data)
const phoneHello = nextJson(phone)
data.send(
JSON.stringify({
type: 'host-data-auth',
v: 1,
connTicket: connection.connTicket,
generation: this.generation
})
)
if ((await phoneHello).ok !== true) throw new Error('relay rejected load splice')
if (slowReaderHoldMs > 0 && readerMode === 'normal') {
readerMode = 'slow'
readerHoldMs = slowReaderHoldMs
streamBytes = payloadBytes
frameBytes = payloadBytes
}
if (!['normal', 'slow', 'wedged'].includes(readerMode)) {
throw new Error('reader mode is invalid')
}
const pausedSocket = readerMode === 'normal' ? undefined : phone._socket
if (readerMode !== 'normal' && !pausedSocket) throw new Error('reader transport unavailable')
if (readerMode === 'wedged') {
const closes = [closeInfo(phone, readerHoldMs + 10_000), closeInfo(data, readerHoldMs + 10_000)]
pausedSocket.pause()
const readerPausedAt = Date.now()
const readerPressure = observeReaderPressure({
cellOrigin: this.lastAssignment.cellUrl,
readerMode,
streamBytes
})
const [sent] = await Promise.all([
this.sendReaderStream(data, sequence, streamBytes, frameBytes, readerDelay),
readerPressure
])
await readerDelay(Math.max(0, readerHoldMs - (Date.now() - readerPausedAt)))
pausedSocket.resume()
const closeEvidence = await Promise.all(closes)
const closeCodes = closeEvidence.map(({ code }) => code)
if (!relayLoadWedgedCloseAccepted(closeCodes)) {
throw new Error(`wedged reader close codes: ${closeCodes.join(',')}`)
}
opened = true
this.observe('spliceOpened', { index: this.index, readerMode })
this.observe('spliceWedged', { index: this.index, code: 4429, streamBytes: sent.bytes })
return
}
const expectedStreamBytes = readerMode === 'slow' ? streamBytes : payloadBytes
const expectedFrameBytes = readerMode === 'slow' ? frameBytes : payloadBytes
const phoneStream = receiveBinaryStream(
phone,
expectedStreamBytes,
readerMode === 'slow' ? readerHoldMs + 10_000 : 10_000
)
const readerPausedAt = pausedSocket ? Date.now() : 0
if (pausedSocket) pausedSocket.pause()
const readerPressure = readerMode === 'slow'
? observeReaderPressure({
cellOrigin: this.lastAssignment.cellUrl,
readerMode,
streamBytes: expectedStreamBytes
})
: Promise.resolve()
const [sent] = await Promise.all([
this.sendReaderStream(
data,
sequence,
expectedStreamBytes,
expectedFrameBytes,
readerDelay
),
readerPressure
])
if (readerMode === 'slow') {
await readerDelay(Math.max(0, readerHoldMs - (Date.now() - readerPausedAt)))
pausedSocket.resume()
}
const receivedByPhone = await phoneStream
if (receivedByPhone.bytes !== sent.bytes || receivedByPhone.digest !== sent.digest) {
throw new Error('relay changed host-to-client splice payload')
}
const textPayload = `orca-relay-load:${this.index}:${sequence}:${payloadBytes}`
const dataFrame = nextFrame(data)
phone.send(textPayload)
const receivedByHost = await dataFrame
if (receivedByHost.binary || receivedByHost.bytes.toString() !== textPayload) {
throw new Error('relay changed client-to-host splice payload')
}
opened = true
this.observe('spliceOpened', { index: this.index, readerMode })
if (holdMs > 0 && !(await this.waitForSpliceHold(holdMs, [phone, data]))) return
this.observe('spliceCompleted', { index: this.index, readerMode })
} catch (error) {
if (!this.stopped) this.observe('spliceFailed', { index: this.index, error })
throw error
} finally {
await Promise.all([this.closeSpliceSocket(phone), this.closeSpliceSocket(data)])
if (opened) this.observe('spliceClosed', { index: this.index })
}
}
waitForSpliceHold(holdMs, sockets) {
if (this.stopped) return Promise.resolve(false)
return new Promise((resolve, reject) => {
const finish = (error, completed = false) => {
clearTimeout(timer)
this.abortController.signal.removeEventListener('abort', onAbort)
for (const socket of sockets) {
socket.off('close', onClose)
socket.off('error', onError)
}
if (error) reject(error)
else resolve(completed)
}
const onAbort = () => finish(undefined, false)
const onClose = (code, reason) => finish(new Error(`splice closed: ${code} ${reason}`))
const onError = (error) => finish(error)
const timer = setTimeout(() => finish(undefined, true), holdMs)
this.abortController.signal.addEventListener('abort', onAbort, { once: true })
for (const socket of sockets) {
socket.once('close', onClose)
socket.once('error', onError)
}
})
}
createClientSocket(assignment) {
return new WebSocket(
`${assignment.cellUrl.replace(/^http/, 'ws')}/v1/connect/${this.relayHostId}`,
{ perMessageDeflate: false }
)
}
createHostDataSocket(assignment, connId) {
return new WebSocket(
`${assignment.cellUrl.replace(/^http/, 'ws')}/v1/host/data/${connId}`,
{ perMessageDeflate: false }
)
}
splicePayload(sequence, payloadBytes) {
const seed = createHash('sha256')
.update(`orca-relay-load:${this.index}:${sequence}`)
.digest()
return Buffer.allocUnsafe(payloadBytes).map((_, index) => seed[index % seed.length])
}
async sendReaderStream(socket, sequence, streamBytes, frameBytes, delay) {
const hash = createHash('sha256')
let sentBytes = 0
let frameIndex = 0
while (sentBytes < streamBytes) {
const bytes = Math.min(frameBytes, streamBytes - sentBytes)
const payload = this.splicePayload(sequence + frameIndex, bytes)
await this.sendReaderFrame(socket, payload)
hash.update(payload)
sentBytes += bytes
frameIndex++
while (socket.bufferedAmount > frameBytes) await delay(10)
}
return { bytes: sentBytes, digest: hash.digest('hex') }
}
sendReaderFrame(socket, payload) {
if (socket.send.length < 2) {
socket.send(payload)
return Promise.resolve()
}
return new Promise((resolve, reject) => {
const timer = setTimeout(() => reject(new Error('reader send timeout')), 10_000)
socket.send(payload, (error) => {
clearTimeout(timer)
if (error) reject(error)
else resolve()
})
})
}
trackSpliceSocket(socket) {
this.spliceSockets.add(socket)
socket.once('close', () => this.spliceSockets.delete(socket))
}
async closeSpliceSocket(socket) {
if (!socket) return
const closed = waitForClose(socket).catch(() => undefined)
if (socket.readyState !== socket.CLOSED && socket.readyState !== socket.CLOSING) {
socket.close(1000, 'splice complete')
}
await closed
this.spliceSockets.delete(socket)
}
async openRebindProbe() {
if (
!this.socket ||
this.socket.readyState !== this.socket.OPEN ||
this.generation === undefined ||
this.controlResumeSecret === undefined ||
this.lastAssignment === undefined
) {
throw new Error('active control required for rebind probe')
}
const relayToken = await this.relayToken()
const socket = new WebSocket(
`${this.lastAssignment.cellUrl.replace(/^http/, 'ws')}/v1/host/control`,
{
headers: { authorization: `Bearer ${relayToken}` },
perMessageDeflate: false
}
)
try {
await waitForOpen(socket)
socket.send(
JSON.stringify({
type: 'host-hello',
v: 1,
relayHostId: this.relayHostId,
assignmentEpoch: this.lastAssignment.assignmentEpoch,
hostPublicKeyB64: Buffer.from(this.keys.publicKey).toString('base64'),
appVersion: 'relay-load-rebind-proof',
previousGeneration: this.generation,
controlResumeSecret: this.controlResumeSecret
})
)
const challenge = await nextJson(socket)
if (challenge.type !== 'host-challenge') throw new Error('expected host challenge')
socket.on('error', () => undefined)
const closed = new Promise((resolve) => socket.once('close', resolve))
return {
close: async () => {
const closeCompleted = waitForClose(socket)
if (socket.readyState !== socket.CLOSED && socket.readyState !== socket.CLOSING) {
socket.close(1000, 'rebind boundary proved')
}
await closeCompleted
},
closed,
isOpen: () => socket.readyState === socket.OPEN
}
} catch (error) {
discardFailedLoadSocket(socket)
await waitForClose(socket).catch(() => undefined)
throw error
}
}
async relayToken() {
const accessToken = this.options.accessTokenProvider
? await this.options.accessTokenProvider()
: this.options.accessToken
if (accessToken) {
const body = await this.requestJson(
`${this.options.authOrigin}/v1/desktop/auth/relay-token`,
{
method: 'POST',
headers: {
authorization: `Bearer ${accessToken}`,
'content-type': 'application/json'
},
body: JSON.stringify({
relayHostId: this.relayHostId,
hostPublicKeyB64: Buffer.from(this.keys.publicKey).toString('base64')
})
},
'relay token exchange timeout',
(status) => `relay token exchange failed: ${status}`
)
if (typeof body.relayToken !== 'string') throw new Error('relay token exchange omitted token')
if (!this.stopped) this.observe('token', { index: this.index })
return body.relayToken
}
const token = await new SignJWT({
prof: `load-profile-${this.index}`,
org: 'relay-load',
purpose: 'host-control',
relayHostId: this.relayHostId
})
.setProtectedHeader({ alg: 'ES256', kid: this.options.signingKeyId })
.setIssuer(this.options.authOrigin)
.setAudience('orca-relay')
.setSubject(`load-user-${this.index}`)
.setIssuedAt()
.setExpirationTime('5m')
.sign(this.options.signingKey)
if (!this.stopped) this.observe('token', { index: this.index })
return token
}
async requestAssignment(preferredRegion) {
return await this.assignment(await this.relayToken(), preferredRegion)
}
async assignment(relayToken, preferredRegion = this.options.preferredRegion) {
if (!this.options.directorOrigin) {
if (!this.options.targetOrigin) throw new Error('relay target origin missing')
return { cellUrl: this.options.targetOrigin, assignmentEpoch: 1 }
}
const body = await this.requestJson(
`${this.options.directorOrigin}/v1/assign`,
{
method: 'POST',
headers: { authorization: `Bearer ${relayToken}`, 'content-type': 'application/json' },
body: JSON.stringify({
v: 1,
relayHostId: this.relayHostId,
...(preferredRegion ? { preferredRegion } : {})
})
},
'relay assignment timeout',
(status, errorCode) =>
`relay assignment failed: ${status}${errorCode ? ` ${errorCode}` : ''}`,
CAPACITY_ASSIGNMENT_ERRORS
)
if (
typeof body.cellUrl !== 'string' ||
!Number.isSafeInteger(body.assignmentEpoch) ||
body.assignmentEpoch < 1
) {
throw new Error('relay assignment response invalid')
}
return body
}
async requestJson(url, init, timeoutMessage, httpErrorMessage, allowedErrorCodes = []) {
const controller = new AbortController()
const onShutdown = () => controller.abort()
if (this.abortController.signal.aborted) controller.abort()
else this.abortController.signal.addEventListener('abort', onShutdown, { once: true })
let timedOut = false
const timer = setTimeout(() => {
timedOut = true
controller.abort()
}, this.options.requestTimeoutMs ?? 10_000)
try {
const response = await fetch(url, { ...init, signal: controller.signal })
if (!response.ok) {
let bodyConsumed = false
let errorCode
if (allowedErrorCodes.length > 0) {
try {
const body = await response.json()
bodyConsumed = true
if (allowedErrorCodes.includes(body?.error)) errorCode = body.error
} catch {
// Preserve the bounded status-only classification.
}
}
if (!bodyConsumed) await cancelResponse(response)
throw new Error(httpErrorMessage(response.status, errorCode))
}
return await response.json()
} catch (error) {
if (timedOut) throw new Error(timeoutMessage, { cause: error })
throw error
} finally {
clearTimeout(timer)
this.abortController.signal.removeEventListener('abort', onShutdown)
}
}
onMessage(socket, data) {
let message
try {
message = JSON.parse(data.toString())
} catch {
this.observe('protocolError', { index: this.index })
return
}
for (const waiter of this.controlWaiters) {
if (waiter.matches(message)) {
this.controlWaiters.delete(waiter)
clearTimeout(waiter.timer)
waiter.resolve(message)
return
}
}
if (message.type === 'ping') {
socket.send(JSON.stringify({ type: 'pong', t: message.t }))
this.observe('ping', { index: this.index })
} else if (message.type === 'drain') {
this.drainExpected = true
this.observe('drain', { index: this.index })
}
}
onClose(socket, code) {
if (this.socket !== socket) return
this.socket = null
if (this.refreshTimer) clearTimeout(this.refreshTimer)
this.refreshTimer = null
this.rejectControlWaiters(new Error(`control closed: ${code}`))
this.observe('closed', {
index: this.index,
code,
stopped: this.stopped,
expectedDrain: this.drainExpected
})
this.drainExpected = false
}
waitForControlMessage(matches, timeoutMs = 10_000) {
if (this.stopped) return Promise.reject(new Error('control stopped'))
return new Promise((resolve, reject) => {
const waiter = {
matches,
resolve,
reject,
timer: setTimeout(() => {
this.controlWaiters.delete(waiter)
reject(new Error('control response timeout'))
}, timeoutMs)
}
this.controlWaiters.add(waiter)
})
}
rejectControlWaiters(error) {
for (const waiter of this.controlWaiters) {
clearTimeout(waiter.timer)
waiter.reject(error)
}
this.controlWaiters.clear()
}
scheduleRefresh(delayMs) {
if (this.stopped) return
this.refreshTimer = setTimeout(() => {
void this.refresh().then(
() => this.scheduleRefresh(this.phase.refreshIntervalMs),
() => this.scheduleRefresh(this.phase.refreshIntervalMs)
)
}, delayMs)
}
refresh() {
if (this.stopped) return Promise.resolve()
const operation = this.refreshOnce()
this.inFlight.add(operation)
const finish = () => this.inFlight.delete(operation)
operation.then(finish, finish)
return operation
}
async refreshOnce() {
const socket = this.socket
if (this.stopped || !socket || socket.readyState !== socket.OPEN) return
try {
const relayJwt = await this.relayToken()
if (this.stopped || this.socket !== socket || socket.readyState !== socket.OPEN) return
socket.send(JSON.stringify({ type: 'auth-refresh', relayJwt }))
this.observe('refresh', { index: this.index })
} catch (error) {
if (!this.stopped) this.observe('refreshError', { index: this.index, error })
}
}
}
@@ -0,0 +1,903 @@
import assert from 'node:assert/strict'
import { generateKeyPairSync } from 'node:crypto'
import { EventEmitter } from 'node:events'
import { createRequire } from 'node:module'
import test from 'node:test'
import {
RelayLoadControlPeer,
relayLoadWedgedCloseAccepted
} from './relay-load-control-peer.mjs'
const requireFromRelay = createRequire(new URL('../../apps/relay/package.json', import.meta.url))
const nacl = requireFromRelay('tweetnacl')
const { buildHostChallengePlaintext } = await import(
requireFromRelay.resolve('@orca-cloud/relay-contract')
)
function deferred() {
let resolve
let reject
const promise = new Promise((resolvePromise, rejectPromise) => {
resolve = resolvePromise
reject = rejectPromise
})
return { promise, resolve, reject }
}
function peerOptions(overrides = {}) {
const { privateKey } = generateKeyPairSync('ec', { namedCurve: 'P-256' })
return {
authOrigin: 'https://auth.test',
directorOrigin: 'https://director.test',
reconnectMaxMs: 0,
seed: 1,
signingKey: privateKey,
signingKeyId: 'test-key',
...overrides
}
}
function response(body) {
return { ok: true, status: 200, json: async () => body }
}
function fakeOpenSocket() {
const socket = new EventEmitter()
socket.OPEN = 1
socket.CLOSING = 2
socket.CLOSED = 3
socket.readyState = socket.OPEN
socket.sent = []
socket.send = (message) => socket.sent.push(message)
socket.close = (code = 1000, reason = '') => {
socket.readyState = socket.CLOSING
queueMicrotask(() => {
socket.readyState = socket.CLOSED
socket.emit('close', code, Buffer.from(reason))
})
}
socket.terminate = socket.close
return socket
}
function fakeHandshakeSocket() {
const socket = fakeOpenSocket()
socket.CONNECTING = 0
socket.readyState = socket.CONNECTING
socket.open = () => {
socket.readyState = socket.OPEN
socket.emit('open')
}
socket.message = (message) => socket.emit('message', Buffer.from(JSON.stringify(message)))
return socket
}
function openOnNextTurn(socket) {
queueMicrotask(() => socket.open())
return socket
}
function validChallenge(peer) {
const relayKeys = nacl.box.keyPair()
const nonce = nacl.randomBytes(nacl.box.nonceLength)
const plaintext = buildHostChallengePlaintext(
new Uint8Array([1, 2, 3]),
nacl.randomBytes(32)
)
const ciphertext = nacl.box(plaintext, nonce, peer.keys.publicKey, relayKeys.secretKey)
return {
type: 'host-challenge',
challengeId: 'test-challenge',
ciphertextB64: Buffer.from(ciphertext).toString('base64'),
nonceB64: Buffer.from(nonce).toString('base64'),
relayEphemeralPublicKeyB64: Buffer.from(relayKeys.publicKey).toString('base64')
}
}
test('shutdown waits for pending assignment and prevents a late connection', async (context) => {
const originalFetch = global.fetch
context.after(() => {
global.fetch = originalFetch
})
const assignment = deferred()
const assignmentStarted = deferred()
global.fetch = () => {
assignmentStarted.resolve()
return assignment.promise
}
const observations = []
const peer = new RelayLoadControlPeer(0, peerOptions(), (type) => observations.push(type))
const connecting = peer.connect()
await assignmentStarted.promise
let shutdownFinished = false
const shutdown = peer.shutdown().then(() => {
shutdownFinished = true
})
await new Promise((resolve) => setImmediate(resolve))
assert.equal(shutdownFinished, false)
assignment.resolve(response({ cellUrl: 'https://cell.test', assignmentEpoch: 1 }))
await Promise.all([connecting, shutdown])
assert.equal(peer.socket, null)
assert.equal(observations.includes('connected'), false)
})
test('shutdown after socket open prevents a late handshake', async () => {
const observations = []
const peer = new RelayLoadControlPeer(
0,
peerOptions({ directorOrigin: undefined, targetOrigin: 'https://cell.test' }),
(type) => observations.push(type)
)
const socket = fakeHandshakeSocket()
const socketCreated = deferred()
peer.createSocket = () => {
socketCreated.resolve()
return socket
}
const connecting = peer.connect()
await socketCreated.promise
socket.open()
await Promise.all([connecting, peer.shutdown()])
assert.deepEqual(socket.sent, [])
assert.equal(observations.includes('connected'), false)
})
test('shutdown after challenge prevents a late acknowledgement', async () => {
const observations = []
const peer = new RelayLoadControlPeer(
0,
peerOptions({ directorOrigin: undefined, targetOrigin: 'https://cell.test' }),
(type) => observations.push(type)
)
const socket = fakeHandshakeSocket()
const socketCreated = deferred()
const helloSent = deferred()
socket.send = (message) => {
socket.sent.push(message)
helloSent.resolve()
}
peer.createSocket = () => {
socketCreated.resolve()
return socket
}
const connecting = peer.connect()
await socketCreated.promise
socket.open()
await helloSent.promise
socket.message({ type: 'host-challenge' })
await Promise.all([connecting, peer.shutdown()])
assert.equal(socket.sent.length, 1)
assert.equal(observations.includes('connected'), false)
})
test('shutdown after host acknowledgement prevents a late connected observation', async () => {
const observations = []
const peer = new RelayLoadControlPeer(
0,
peerOptions({ directorOrigin: undefined, targetOrigin: 'https://cell.test' }),
(type) => observations.push(type)
)
const socket = fakeHandshakeSocket()
const socketCreated = deferred()
const helloSent = deferred()
const proofSent = deferred()
socket.send = (message) => {
socket.sent.push(message)
if (socket.sent.length === 1) helloSent.resolve()
else proofSent.resolve()
}
peer.createSocket = () => {
socketCreated.resolve()
return socket
}
const connecting = peer.connect()
await socketCreated.promise
socket.open()
await helloSent.promise
socket.message(validChallenge(peer))
await proofSent.promise
socket.message({ type: 'host-hello-ack', generation: 1, controlResumeSecret: 'test-secret' })
await Promise.all([connecting, peer.shutdown()])
assert.equal(socket.sent.length, 2)
assert.equal(observations.includes('connected'), false)
})
test('shutdown prevents a pending refresh from sending or reporting success', async (context) => {
const originalFetch = global.fetch
context.after(() => {
global.fetch = originalFetch
})
const token = deferred()
const tokenStarted = deferred()
global.fetch = () => {
tokenStarted.resolve()
return token.promise
}
const observations = []
const peer = new RelayLoadControlPeer(
0,
peerOptions({ accessToken: 'test-access-token', directorOrigin: undefined }),
(type) => observations.push(type)
)
const socket = fakeOpenSocket()
peer.socket = socket
const refreshing = peer.refresh()
await tokenStarted.promise
const shutdown = peer.shutdown()
token.resolve(response({ relayToken: 'test-relay-token' }))
await Promise.all([refreshing, shutdown])
assert.deepEqual(socket.sent, [])
assert.equal(observations.includes('refresh'), false)
assert.equal(observations.includes('refreshError'), false)
})
test('shutdown suppresses a late refresh error', async (context) => {
const originalFetch = global.fetch
context.after(() => {
global.fetch = originalFetch
})
const token = deferred()
const tokenStarted = deferred()
global.fetch = () => {
tokenStarted.resolve()
return token.promise
}
const observations = []
const peer = new RelayLoadControlPeer(
0,
peerOptions({ accessToken: 'test-access-token', directorOrigin: undefined }),
(type) => observations.push(type)
)
peer.socket = fakeOpenSocket()
const refreshing = peer.refresh()
await tokenStarted.promise
const shutdown = peer.shutdown()
token.reject(new Error('late token failure'))
await Promise.all([refreshing, shutdown])
assert.equal(observations.includes('refreshError'), false)
})
test('shutdown aborts an HTTP request that would otherwise remain pending', async (context) => {
const originalFetch = global.fetch
context.after(() => {
global.fetch = originalFetch
})
const requestStarted = deferred()
let aborted = false
global.fetch = (_url, { signal }) =>
new Promise((_resolve, reject) => {
requestStarted.resolve()
signal.addEventListener(
'abort',
() => {
aborted = true
reject(new Error('request aborted'))
},
{ once: true }
)
})
const observations = []
const peer = new RelayLoadControlPeer(
0,
peerOptions({ accessToken: 'test-access-token', directorOrigin: undefined }),
(type) => observations.push(type)
)
const connecting = peer.connect()
await requestStarted.promise
await Promise.all([connecting, peer.shutdown()])
assert.equal(aborted, true)
assert.equal(observations.includes('connected'), false)
})
test('bounds a pending HTTP request with a classified timeout', async (context) => {
const originalFetch = global.fetch
context.after(() => {
global.fetch = originalFetch
})
global.fetch = (_url, { signal }) =>
new Promise((_resolve, reject) => {
signal.addEventListener('abort', () => reject(new Error('request aborted')), { once: true })
})
const peer = new RelayLoadControlPeer(
0,
peerOptions({
accessToken: 'test-access-token',
directorOrigin: undefined,
requestTimeoutMs: 1
}),
() => undefined
)
await assert.rejects(peer.connect(), /relay token exchange timeout/)
await peer.shutdown()
})
test('bounds a stalled token response body', async (context) => {
const originalFetch = global.fetch
context.after(() => {
global.fetch = originalFetch
})
global.fetch = async (_url, { signal }) => ({
ok: true,
status: 200,
json: () =>
new Promise((_resolve, reject) => {
signal.addEventListener('abort', () => reject(new Error('body aborted')), { once: true })
})
})
const peer = new RelayLoadControlPeer(
0,
peerOptions({
accessToken: 'test-access-token',
directorOrigin: undefined,
requestTimeoutMs: 1
}),
() => undefined
)
await assert.rejects(peer.connect(), /relay token exchange timeout/)
await peer.shutdown()
})
test('bounds a stalled assignment response body', async (context) => {
const originalFetch = global.fetch
context.after(() => {
global.fetch = originalFetch
})
global.fetch = async (_url, { signal }) => ({
ok: true,
status: 200,
json: () =>
new Promise((_resolve, reject) => {
signal.addEventListener('abort', () => reject(new Error('body aborted')), { once: true })
})
})
const peer = new RelayLoadControlPeer(
0,
peerOptions({ requestTimeoutMs: 1 }),
() => undefined
)
await assert.rejects(peer.connect(), /relay assignment timeout/)
await peer.shutdown()
})
test('shutdown aborts a stalled successful response body', async (context) => {
const originalFetch = global.fetch
context.after(() => {
global.fetch = originalFetch
})
const bodyStarted = deferred()
let bodyAborted = false
global.fetch = async (_url, { signal }) => ({
ok: true,
status: 200,
json: () =>
new Promise((_resolve, reject) => {
bodyStarted.resolve()
signal.addEventListener(
'abort',
() => {
bodyAborted = true
reject(new Error('body aborted'))
},
{ once: true }
)
})
})
const observations = []
const peer = new RelayLoadControlPeer(
0,
peerOptions({ accessToken: 'test-access-token', directorOrigin: undefined }),
(type) => observations.push(type)
)
const connecting = peer.connect()
await bodyStarted.promise
await Promise.all([connecting, peer.shutdown()])
assert.equal(bodyAborted, true)
assert.equal(observations.includes('connected'), false)
})
test('cancels a rejected HTTP response body', async (context) => {
const originalFetch = global.fetch
context.after(() => {
global.fetch = originalFetch
})
let canceled = false
global.fetch = async () => ({
ok: false,
status: 503,
body: {
cancel: async () => {
canceled = true
}
}
})
const peer = new RelayLoadControlPeer(
0,
peerOptions({ accessToken: 'test-access-token', directorOrigin: undefined }),
() => undefined
)
await assert.rejects(peer.connect(), /relay token exchange failed: 503/)
await peer.shutdown()
assert.equal(canceled, true)
})
test('preserves only an exact capacity assignment rejection reason', async (context) => {
const originalFetch = global.fetch
context.after(() => {
global.fetch = originalFetch
})
global.fetch = async () => ({
ok: false,
status: 503,
json: async () => ({ error: 'relay_connection_headroom_exhausted' })
})
const peer = new RelayLoadControlPeer(0, peerOptions(), () => undefined)
await assert.rejects(
peer.connect(),
/relay assignment failed: 503 relay_connection_headroom_exhausted/
)
await peer.shutdown()
})
test('sends preferred region and preserves the genuine director epoch', async (context) => {
const originalFetch = global.fetch
context.after(() => {
global.fetch = originalFetch
})
let assignmentRequest
global.fetch = async (_url, init) => {
assignmentRequest = JSON.parse(init.body)
return response({ cellUrl: 'https://asia-cell.test', assignmentEpoch: 47 })
}
const peer = new RelayLoadControlPeer(
0,
peerOptions({ preferredRegion: 'asia-east2' }),
() => undefined
)
const assignment = await peer.assignment('relay-token')
assert.deepEqual(assignmentRequest, {
v: 1,
relayHostId: peer.relayHostId,
preferredRegion: 'asia-east2'
})
assert.equal(assignment.assignmentEpoch, 47)
await peer.shutdown()
})
test('omits preferred region and rejects a fabricated director epoch', async (context) => {
const originalFetch = global.fetch
context.after(() => {
global.fetch = originalFetch
})
let assignmentRequest
global.fetch = async (_url, init) => {
assignmentRequest = JSON.parse(init.body)
return response({ cellUrl: 'https://cell.test', assignmentEpoch: 0 })
}
const peer = new RelayLoadControlPeer(0, peerOptions(), () => undefined)
await assert.rejects(peer.assignment('relay-token'), /assignment response invalid/)
assert.equal('preferredRegion' in assignmentRequest, false)
await peer.shutdown()
})
test('rejects ambiguous direct and director assignment modes', () => {
assert.throws(
() =>
new RelayLoadControlPeer(
0,
peerOptions({ targetOrigin: 'https://cell.test' }),
() => undefined
),
/either directorOrigin or targetOrigin/
)
})
test('opens a genuine splice and verifies payloads in both directions', async () => {
const observations = []
const peer = new RelayLoadControlPeer(3, peerOptions(), (type, detail) => {
observations.push({ type, detail })
})
const control = fakeOpenSocket()
const phone = fakeHandshakeSocket()
const data = fakeHandshakeSocket()
let dataPayloadBytes = 0
let observationStarted = false
let sentBeforeObservation = false
let paused = 0
let resumed = 0
phone._socket = {
pause: () => paused++,
resume: () => resumed++
}
control.send = (raw) => {
const message = JSON.parse(raw)
if (message.type === 'invite-create') {
queueMicrotask(() =>
control.emit(
'message',
Buffer.from(
JSON.stringify({
type: 'invite-created',
reqId: message.reqId,
inviteToken: 'invite-token'
})
)
)
)
}
}
phone.send = (raw) => {
if (typeof raw === 'string' && raw.startsWith('{') && JSON.parse(raw).type === 'relay-auth') {
queueMicrotask(() =>
control.emit(
'message',
Buffer.from(
JSON.stringify({
type: 'conn-open',
relayDeviceId: 'load-device-3-0',
connId: 'connection-1',
connTicket: 'connection-ticket'
})
)
)
)
return
}
queueMicrotask(() => data.emit('message', Buffer.from(raw), false))
}
data.send = (raw) => {
if (typeof raw === 'string') {
queueMicrotask(() => phone.emit('message', Buffer.from(JSON.stringify({ ok: true })), false))
return
}
const dataPayload = Buffer.from(raw)
if (!observationStarted) sentBeforeObservation = true
dataPayloadBytes += dataPayload.byteLength
queueMicrotask(() => phone.emit('message', dataPayload, true))
}
peer.socket = control
peer.generation = 9
peer.lastAssignment = { cellUrl: 'https://cell.test', assignmentEpoch: 47 }
control.on('message', (raw) => peer.onMessage(control, raw))
peer.createClientSocket = () => openOnNextTurn(phone)
peer.createHostDataSocket = () => openOnNextTurn(data)
await peer.openSplice({
readerMode: 'slow',
readerHoldMs: 1,
streamBytes: 300 * 1024,
frameBytes: 64 * 1024,
observeReaderPressure: async () => { observationStarted = true }
})
assert.equal(dataPayloadBytes, 300 * 1024)
assert.equal(sentBeforeObservation, false)
assert.equal(paused, 1)
assert.equal(resumed, 1)
assert.equal(observations.filter(({ type }) => type === 'spliceCompleted').length, 1)
assert.equal(observations.some(({ type }) => type === 'spliceFailed'), false)
await peer.shutdown()
assert.deepEqual(observations.at(-1), {
type: 'shutdown',
detail: {
index: 3,
activeControls: 0,
activeSpliceSockets: 0,
inFlightOperations: 0,
refreshTimerActive: false
}
})
})
test('opens invitation leases and preserves exact capacity errors', async () => {
const peer = new RelayLoadControlPeer(4, peerOptions(), () => undefined)
const control = fakeOpenSocket()
peer.socket = control
control.on('message', (raw) => peer.onMessage(control, raw))
let calls = 0
control.send = (raw) => {
const request = JSON.parse(raw)
calls++
queueMicrotask(() => control.emit('message', Buffer.from(JSON.stringify(
calls === 1
? {
type: 'invite-created', reqId: request.reqId,
inviteToken: 'invite-token', expiresAt: Date.now() + 60_000
}
: { type: 'control-error', reqId: request.reqId, code: 'relay_capacity_exhausted' }
))))
}
await peer.openInviteOffer()
await assert.rejects(peer.openInviteOffer(), /invite offer failed: relay_capacity_exhausted/)
await peer.shutdown()
})
test('can request the same assignment with an explicit replacement preference', async (context) => {
const originalFetch = global.fetch
context.after(() => { global.fetch = originalFetch })
const requests = []
global.fetch = async (url, init) => {
if (String(url).endsWith('/v1/assign')) {
requests.push(JSON.parse(init.body))
return response({ cellUrl: 'https://asia-cell.test', assignmentEpoch: 3 })
}
return response({ relayToken: 'relay-token' })
}
const peer = new RelayLoadControlPeer(
5,
peerOptions({ accessToken: 'access-token', preferredRegion: 'asia-east2' }),
() => undefined
)
assert.equal((await peer.requestAssignment('us-central1')).cellUrl, 'https://asia-cell.test')
assert.equal(requests[0].preferredRegion, 'us-central1')
await peer.shutdown()
})
test('uses a refreshable workflow access-token provider', async (context) => {
const originalFetch = global.fetch
context.after(() => { global.fetch = originalFetch })
let providerCalls = 0
let authorization
global.fetch = async (url, init) => {
if (String(url).endsWith('/v1/desktop/auth/relay-token')) {
authorization = init.headers.authorization
return response({ relayToken: 'relay-token' })
}
return response({ cellUrl: 'https://cell.test', assignmentEpoch: 1 })
}
const peer = new RelayLoadControlPeer(6, peerOptions({
accessTokenProvider: async () => {
providerCalls++
return 'refreshed-access-token'
}
}), () => undefined)
await peer.requestAssignment('asia-east2')
assert.equal(providerCalls, 1)
assert.equal(authorization, 'Bearer refreshed-access-token')
await peer.shutdown()
})
test('accepts a forced close only when the responsive splice leg receives 4429', async () => {
const observations = []
const peer = new RelayLoadControlPeer(7, peerOptions(), (type, detail) => {
observations.push({ type, detail })
})
const control = fakeOpenSocket()
const phone = fakeHandshakeSocket()
const data = fakeHandshakeSocket()
let streamStarted
const streamStartedPromise = new Promise((resolve) => { streamStarted = resolve })
phone._socket = { pause: () => undefined, resume: () => undefined }
control.send = (raw) => {
const message = JSON.parse(raw)
queueMicrotask(() => control.emit('message', Buffer.from(JSON.stringify({
type: 'invite-created', reqId: message.reqId, inviteToken: 'invite-token'
}))))
}
phone.send = (raw) => {
if (typeof raw === 'string' && raw.startsWith('{')) {
queueMicrotask(() => control.emit('message', Buffer.from(JSON.stringify({
type: 'conn-open', relayDeviceId: 'load-device-7-0',
connId: 'connection-7', connTicket: 'connection-ticket'
}))))
}
}
data.send = (raw) => {
if (typeof raw === 'string') {
queueMicrotask(() => phone.emit('message', Buffer.from(JSON.stringify({ ok: true })), false))
} else {
streamStarted()
}
}
peer.socket = control
peer.generation = 11
peer.lastAssignment = { cellUrl: 'https://cell.test', assignmentEpoch: 9 }
control.on('message', (raw) => peer.onMessage(control, raw))
peer.createClientSocket = () => openOnNextTurn(phone)
peer.createHostDataSocket = () => openOnNextTurn(data)
await peer.openSplice({
readerMode: 'wedged',
readerHoldMs: 10_001,
streamBytes: 300 * 1024,
frameBytes: 64 * 1024,
observeReaderPressure: async () => {
await streamStartedPromise
phone.close(1006)
data.close(4429, 'wedged relay link')
},
readerDelay: async () => undefined
})
assert.equal(observations.filter(({ type }) => type === 'spliceWedged').length, 1)
assert.equal(observations.find(({ type }) => type === 'spliceWedged').detail.code, 4429)
await peer.shutdown()
})
test('requires one 4429 and rejects unrelated wedged close codes', () => {
assert.equal(relayLoadWedgedCloseAccepted([4429, 4429]), true)
assert.equal(relayLoadWedgedCloseAccepted([1006, 4429]), true)
assert.equal(relayLoadWedgedCloseAccepted([4429, 1006]), false)
assert.equal(relayLoadWedgedCloseAccepted([1006, 1006]), false)
assert.equal(relayLoadWedgedCloseAccepted([1000, 4429]), false)
assert.equal(relayLoadWedgedCloseAccepted([4429]), false)
assert.equal(relayLoadWedgedCloseAccepted([1006, 4429, 4429]), false)
})
test('streams reader frames with bounded send-side backpressure', async () => {
const peer = new RelayLoadControlPeer(9, peerOptions(), () => undefined)
let inFlight = 0
let peakInFlight = 0
let sentBytes = 0
const socket = {
bufferedAmount: 0,
send(payload, callback) {
inFlight++
peakInFlight = Math.max(peakInFlight, inFlight)
sentBytes += payload.byteLength
this.bufferedAmount = payload.byteLength
queueMicrotask(() => {
this.bufferedAmount = 0
inFlight--
callback()
})
}
}
const result = await peer.sendReaderStream(
socket,
0,
1024 * 1024,
64 * 1024,
async () => undefined
)
assert.equal(result.bytes, 1024 * 1024)
assert.equal(sentBytes, 1024 * 1024)
assert.equal(peakInFlight, 1)
await peer.shutdown()
})
test('reports a bidirectional splice payload mismatch', async () => {
const observations = []
const peer = new RelayLoadControlPeer(2, peerOptions(), (type) => observations.push(type))
const control = fakeOpenSocket()
const phone = fakeHandshakeSocket()
const data = fakeHandshakeSocket()
control.send = (raw) => {
const message = JSON.parse(raw)
queueMicrotask(() =>
control.emit(
'message',
Buffer.from(
JSON.stringify({ type: 'invite-created', reqId: message.reqId, inviteToken: 'token' })
)
)
)
}
phone.send = (raw) => {
if (typeof raw === 'string' && raw.startsWith('{') && JSON.parse(raw).type === 'relay-auth') {
queueMicrotask(() =>
control.emit(
'message',
Buffer.from(
JSON.stringify({
type: 'conn-open',
relayDeviceId: 'load-device-2-0',
connId: 'connection-2',
connTicket: 'ticket'
})
)
)
)
}
}
data.send = (raw) => {
if (typeof raw === 'string') {
queueMicrotask(() => phone.emit('message', Buffer.from(JSON.stringify({ ok: true })), false))
} else {
queueMicrotask(() => phone.emit('message', Buffer.alloc(Buffer.from(raw).byteLength), true))
}
}
peer.socket = control
peer.generation = 4
peer.lastAssignment = { cellUrl: 'https://cell.test', assignmentEpoch: 2 }
control.on('message', (raw) => peer.onMessage(control, raw))
peer.createClientSocket = () => openOnNextTurn(phone)
peer.createHostDataSocket = () => openOnNextTurn(data)
await assert.rejects(peer.openSplice(), /changed host-to-client splice payload/)
assert.equal(observations.includes('spliceFailed'), true)
await peer.shutdown()
})
test('shutdown closes both splice legs and waits for the in-flight splice', async () => {
const spliceOpened = deferred()
const observations = []
const peer = new RelayLoadControlPeer(5, peerOptions(), (type, detail) => {
observations.push({ type, detail })
if (type === 'spliceOpened') spliceOpened.resolve()
})
const control = fakeOpenSocket()
const phone = fakeHandshakeSocket()
const data = fakeHandshakeSocket()
control.send = (raw) => {
const message = JSON.parse(raw)
queueMicrotask(() =>
control.emit(
'message',
Buffer.from(
JSON.stringify({ type: 'invite-created', reqId: message.reqId, inviteToken: 'token' })
)
)
)
}
phone.send = (raw) => {
if (typeof raw === 'string' && raw.startsWith('{')) {
queueMicrotask(() =>
control.emit(
'message',
Buffer.from(
JSON.stringify({
type: 'conn-open',
relayDeviceId: 'load-device-5-0',
connId: 'connection-5',
connTicket: 'ticket'
})
)
)
)
} else {
queueMicrotask(() => data.emit('message', Buffer.from(raw), false))
}
}
data.send = (raw) => {
if (typeof raw === 'string') {
queueMicrotask(() => phone.emit('message', Buffer.from(JSON.stringify({ ok: true })), false))
} else {
queueMicrotask(() => phone.emit('message', Buffer.from(raw), true))
}
}
peer.socket = control
peer.generation = 8
peer.lastAssignment = { cellUrl: 'https://cell.test', assignmentEpoch: 3 }
control.on('message', (raw) => peer.onMessage(control, raw))
peer.createClientSocket = () => openOnNextTurn(phone)
peer.createHostDataSocket = () => openOnNextTurn(data)
const splice = peer.openSplice({ holdMs: 60_000 })
await spliceOpened.promise
await Promise.all([splice, peer.shutdown()])
assert.equal(phone.readyState, phone.CLOSED)
assert.equal(data.readyState, data.CLOSED)
assert.equal(peer.inFlight.size, 0)
assert.equal(observations.at(-1).type, 'shutdown')
assert.equal(observations.at(-1).detail.activeSpliceSockets, 0)
})
@@ -0,0 +1,147 @@
import { setTimeout as delayDefault } from 'node:timers/promises'
function integer(value) {
return Number.isSafeInteger(value) && value >= 0 ? value : undefined
}
export function assertRelayLoadDirectorCapacityToken(config, now = Date.now, timeoutMs = 0) {
if (!config.adminToken || config.adminToken.length > 8_192) {
throw new Error('director capacity identity token is unavailable')
}
const origin = new URL(config.directorOrigin)
if (origin.protocol !== 'https:' || origin.origin !== config.directorOrigin) {
throw new Error('director capacity origin must be canonical HTTPS')
}
let claims
try {
const parts = config.adminToken.split('.')
if (parts.length !== 3) throw new Error('invalid token shape')
claims = JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf8'))
} catch {
throw new Error('director capacity identity token is invalid')
}
const expectedAudience = new URL('/v1/admin/drain', origin).toString()
const audiences = Array.isArray(claims.aud) ? claims.aud : [claims.aud]
const expiresAt = integer(claims.exp)
if (
!audiences.includes(expectedAudience) ||
typeof claims.email !== 'string' ||
claims.email.length === 0 ||
claims.email_verified !== true ||
expiresAt === undefined ||
expiresAt * 1_000 <= now() + timeoutMs
) {
throw new Error('director capacity identity token is not bound to this proof')
}
}
function matchingHeartbeat(status, config) {
const capacity = status?.connectionCapacity
const runtime = status?.runtime
const heartbeatAt = integer(runtime?.lastHeartbeatAt)
const matches =
status?.cellId === config.cellId &&
status?.admissionState === 'general' &&
runtime?.ready === true &&
runtime?.heartbeatFresh === true &&
capacity?.heartbeatFresh === true &&
integer(capacity?.hardCap) === config.hardCap &&
integer(capacity?.unobservedBound) === config.unobservedBound &&
integer(capacity?.normalAdmissionPause) === config.requiredConnections &&
integer(capacity?.observedConnections) === config.requiredConnections &&
integer(capacity?.enforcedConnectionUnits) === config.requiredConnections &&
integer(capacity?.inFlightConnections) === 0 &&
integer(capacity?.reservedConnectionUnits) === 0 &&
integer(capacity?.pendingControlReservations) === 0 &&
heartbeatAt !== undefined
return matches ? heartbeatAt : undefined
}
async function cellStatus(fetchImpl, config) {
const response = await fetchImpl(`${config.directorOrigin}/v1/admin/cell-status`, {
method: 'POST',
headers: {
authorization: `Bearer ${config.adminToken}`,
'content-type': 'application/json'
},
body: JSON.stringify({ v: 1, cellId: config.cellId }),
signal: AbortSignal.timeout(30_000)
})
if (response.status === 401 || response.status === 403) {
throw new Error('director capacity identity was rejected')
}
if (!response.ok) {
await response.arrayBuffer().catch(() => undefined)
return undefined
}
const result = await response.json().catch(() => undefined)
if (!result?.status) throw new Error('director capacity status is invalid')
return result.status
}
export async function waitForRelayLoadDirectorCapacity(config, overrides = {}) {
const fetchImpl = overrides.fetch ?? fetch
const delay = overrides.delay ?? delayDefault
const now = overrides.now ?? Date.now
const timeoutMs = overrides.timeoutMs ?? 120_000
const pollMs = overrides.pollMs ?? 1_000
assertRelayLoadDirectorCapacityToken(config, now, timeoutMs)
const deadline = now() + timeoutMs
let baselineHeartbeatAt = config.baselineHeartbeatAt
let previousHeartbeatAt
let matchingSamples = 0
const requiredSamples = config.requiredSamples ?? 2
for (;;) {
const status = await cellStatus(fetchImpl, config)
const currentHeartbeatAt = integer(status?.runtime?.lastHeartbeatAt)
if (baselineHeartbeatAt === undefined && currentHeartbeatAt !== undefined) {
baselineHeartbeatAt = currentHeartbeatAt
}
const heartbeatAt = status ? matchingHeartbeat(status, config) : undefined
if (heartbeatAt !== undefined && heartbeatAt > baselineHeartbeatAt) {
if (previousHeartbeatAt === undefined || heartbeatAt > previousHeartbeatAt) {
previousHeartbeatAt = heartbeatAt
matchingSamples++
if (matchingSamples === requiredSamples) return { heartbeatAt }
}
} else {
previousHeartbeatAt = undefined
matchingSamples = 0
}
if (now() >= deadline) throw new Error('director capacity did not converge after recovery')
await delay(pollMs)
}
}
function matchingRequestUnits(status, config) {
return (
status?.cellId === config.cellId &&
status?.admissionState === 'general' &&
status?.capacityRequests === config.capacityRequests &&
status?.reservedRequests === config.expectedRequestUnits &&
status?.activityRequestUnits === config.expectedRequestUnits &&
status?.activityLeases === config.expectedActivityLeases &&
status?.runtime?.observedRequests === config.expectedRequestUnits &&
status?.runtime?.ready === true &&
status?.runtime?.heartbeatFresh === true
)
}
export async function waitForRelayLoadRequestUnits(config, overrides = {}) {
const fetchImpl = overrides.fetch ?? fetch
const delay = overrides.delay ?? delayDefault
const now = overrides.now ?? Date.now
const timeoutMs = overrides.timeoutMs ?? config.timeoutMs ?? 120_000
const pollMs = overrides.pollMs ?? 1_000
const requiredSamples = overrides.requiredSamples ?? 2
assertRelayLoadDirectorCapacityToken(config, now, timeoutMs)
const deadline = now() + timeoutMs
let matches = 0
for (;;) {
const status = await cellStatus(fetchImpl, config)
matches = matchingRequestUnits(status, config) ? matches + 1 : 0
if (matches === requiredSamples) return
if (now() >= deadline) throw new Error('Relay request-unit accounting did not converge')
await delay(pollMs)
}
}
@@ -0,0 +1,260 @@
import assert from 'node:assert/strict'
import { test } from 'node:test'
import {
assertRelayLoadDirectorCapacityToken,
waitForRelayLoadDirectorCapacity,
waitForRelayLoadRequestUnits
} from './relay-load-director-capacity-gate.mjs'
function token(claims) {
const encode = (value) => Buffer.from(JSON.stringify(value)).toString('base64url')
return `${encode({ alg: 'none' })}.${encode(claims)}.signature`
}
const config = {
directorOrigin: 'https://relay-staging.example.com',
adminToken: token({
aud: 'https://relay-staging.example.com/v1/admin/drain',
email: 'capacity@example.com',
email_verified: true,
exp: 4_000_000_000
}),
cellId: 'staging-gce-c3',
hardCap: 1_000,
unobservedBound: 60,
requiredConnections: 840
}
function status(lastHeartbeatAt, overrides = {}) {
return {
cellId: config.cellId,
admissionState: 'general',
runtime: { ready: true, heartbeatFresh: true, lastHeartbeatAt, observedRequests: 0 },
connectionCapacity: {
hardCap: 1_000,
unobservedBound: 60,
normalAdmissionPause: 840,
observedConnections: 840,
enforcedConnectionUnits: 840,
inFlightConnections: 0,
reservedConnectionUnits: 0,
pendingControlReservations: 0,
heartbeatFresh: true,
...overrides
}
}
}
function response(value, responseStatus = 200) {
return {
ok: responseStatus >= 200 && responseStatus < 300,
status: responseStatus,
json: async () => value,
arrayBuffer: async () => new ArrayBuffer(0)
}
}
test('requires two advancing exact director capacity heartbeats', async () => {
const heartbeats = [101, 116, 131]
let calls = 0
const result = await waitForRelayLoadDirectorCapacity(config, {
fetch: async () => response({ status: status(heartbeats[calls++]) }),
delay: async () => undefined
})
assert.equal(calls, 3)
assert.deepEqual(result, { heartbeatAt: 131 })
})
test('resets after a newer heartbeat undercounts recovered controls', async () => {
const samples = [
status(101),
status(116),
status(131, { observedConnections: 839 }),
status(146),
status(161)
]
let calls = 0
await waitForRelayLoadDirectorCapacity(config, {
fetch: async () => response({ status: samples[calls++] }),
delay: async () => undefined
})
assert.equal(calls, 5)
})
test('fails closed when exact advancing telemetry never converges', async () => {
let elapsed = 0
await assert.rejects(
waitForRelayLoadDirectorCapacity(config, {
fetch: async () => response({ status: status(101) }),
delay: async (milliseconds) => {
elapsed += milliseconds
},
now: () => elapsed,
timeoutMs: 2_000,
pollMs: 1_000
}),
/did not converge/
)
})
test('rejects an unauthorized capacity identity without retrying', async () => {
let calls = 0
await assert.rejects(
waitForRelayLoadDirectorCapacity(config, {
fetch: async () => {
calls++
return response({}, 401)
},
delay: async () => undefined
}),
/identity was rejected/
)
assert.equal(calls, 1)
})
test('binds the admin token to the canonical director audience', async () => {
await assert.rejects(
waitForRelayLoadDirectorCapacity(
{
...config,
directorOrigin: 'https://relay-staging.example.com/path'
},
{ fetch: async () => response({ status: status(101) }), now: () => 0 }
),
/canonical HTTPS/
)
await assert.rejects(
waitForRelayLoadDirectorCapacity(
{
...config,
adminToken: token({
aud: 'https://other.example.com/v1/admin/drain',
email: 'capacity@example.com',
email_verified: true,
exp: 4_000_000_000
})
},
{ fetch: async () => response({ status: status(101) }), now: () => 0 }
),
/not bound/
)
})
test('rejects a missing or malformed admin token during startup preflight', () => {
assert.throws(
() => assertRelayLoadDirectorCapacityToken({ ...config, adminToken: undefined }, () => 0),
/unavailable/
)
assert.throws(
() => assertRelayLoadDirectorCapacityToken({ ...config, adminToken: 'not-a-jwt' }, () => 0),
/invalid/
)
assert.throws(
() =>
assertRelayLoadDirectorCapacityToken(
{
...config,
adminToken: token({
aud: 'https://relay-staging.example.com/v1/admin/drain',
exp: 4_000_000_000
})
},
() => 0
),
/not bound/
)
})
test('supports one newer exact post-probe heartbeat', async () => {
const heartbeats = [146, 161]
let calls = 0
const result = await waitForRelayLoadDirectorCapacity(
{ ...config, requiredSamples: 1 },
{
fetch: async () => response({ status: status(heartbeats[calls++]) }),
delay: async () => undefined,
now: () => 0
}
)
assert.equal(calls, 2)
assert.deepEqual(result, { heartbeatAt: 161 })
})
test('requires consecutive exact request-unit accounting samples', async () => {
const requestConfig = {
...config,
capacityRequests: 6_000,
expectedRequestUnits: 6_000,
expectedActivityLeases: 6_000
}
const samples = [5_999, 6_000, 6_000]
let calls = 0
await waitForRelayLoadRequestUnits(requestConfig, {
fetch: async () => response({
status: {
...status(100),
runtime: {
...status(100).runtime,
observedRequests: samples[calls]
},
capacityRequests: 6_000,
reservedRequests: samples[calls],
activityRequestUnits: samples[calls],
activityLeases: samples[calls++]
}
}),
delay: async () => undefined,
now: () => 0
})
assert.equal(calls, 3)
})
test('requires the cell runtime to observe every request unit', async () => {
let elapsed = 0
await assert.rejects(waitForRelayLoadRequestUnits({
...config,
capacityRequests: 6_000,
expectedRequestUnits: 6_000,
expectedActivityLeases: 6_000,
timeoutMs: 1_000
}, {
fetch: async () => response({
status: {
...status(100),
runtime: { ...status(100).runtime, observedRequests: 5_999 },
capacityRequests: 6_000,
reservedRequests: 6_000,
activityRequestUnits: 6_000,
activityLeases: 6_000
}
}),
delay: async (milliseconds) => { elapsed += milliseconds },
now: () => elapsed,
pollMs: 1_000
}), /did not converge/)
})
test('fails closed when request-unit accounting does not clean up', async () => {
let elapsed = 0
await assert.rejects(waitForRelayLoadRequestUnits({
...config,
capacityRequests: 6_000,
expectedRequestUnits: 0,
expectedActivityLeases: 0,
timeoutMs: 2_000
}, {
fetch: async () => response({
status: {
...status(100),
runtime: { ...status(100).runtime, observedRequests: 1 },
capacityRequests: 6_000,
reservedRequests: 1,
activityRequestUnits: 1,
activityLeases: 1
}
}),
delay: async (milliseconds) => { elapsed += milliseconds },
now: () => elapsed,
pollMs: 1_000
}), /did not converge/)
})
+76
View File
@@ -0,0 +1,76 @@
const HEARTBEAT_INTERVAL_MS = 15_000
const REFRESH_MIN_MS = 180_000
const REFRESH_MAX_MS = 240_000
function mix32(value) {
let mixed = value >>> 0
mixed = Math.imul(mixed ^ (mixed >>> 16), 0x21f0aaad)
mixed = Math.imul(mixed ^ (mixed >>> 15), 0x735a2d97)
return (mixed ^ (mixed >>> 15)) >>> 0
}
function fraction(seed, index, stream) {
return mix32(seed ^ Math.imul(index + 1, 0x9e3779b1) ^ stream) / 0x1_0000_0000
}
export function controlPhase(controlIndex, seed = 0x4f524341) {
const refreshIntervalMs = Math.round(
REFRESH_MIN_MS + fraction(seed, controlIndex, 2) * (REFRESH_MAX_MS - REFRESH_MIN_MS)
)
return {
heartbeatOffsetMs: Math.floor(fraction(seed, controlIndex, 1) * HEARTBEAT_INTERVAL_MS),
refreshIntervalMs,
refreshOffsetMs: Math.floor(fraction(seed, controlIndex, 3) * refreshIntervalMs),
reconnectJitterMs: Math.floor(fraction(seed, controlIndex, 4) * 30_000)
}
}
export function modeledRelayLoad(controlCount, durationMs = 15 * 60_000, seed) {
if (!Number.isInteger(controlCount) || controlCount < 1) throw new Error('controlCount must be positive')
const bins = Array.from({ length: Math.ceil(durationMs / 1000) }, () => ({ pings: 0, refreshes: 0 }))
for (let controlIndex = 0; controlIndex < controlCount; controlIndex++) {
const phase = controlPhase(controlIndex, seed)
for (let at = phase.heartbeatOffsetMs; at < durationMs; at += HEARTBEAT_INTERVAL_MS) {
bins[Math.floor(at / 1000)].pings++
}
for (let at = phase.refreshOffsetMs; at < durationMs; at += phase.refreshIntervalMs) {
bins[Math.floor(at / 1000)].refreshes++
}
}
const totals = bins.reduce(
(result, bin) => ({
pings: result.pings + bin.pings,
refreshes: result.refreshes + bin.refreshes
}),
{ pings: 0, refreshes: 0 }
)
const durationSeconds = durationMs / 1000
return {
controlCount,
durationMs,
expectedPingRate: controlCount / (HEARTBEAT_INTERVAL_MS / 1000),
expectedRefreshRate: controlCount / ((REFRESH_MIN_MS + REFRESH_MAX_MS) / 2 / 1000),
observedPingRate: totals.pings / durationSeconds,
observedRefreshRate: totals.refreshes / durationSeconds,
maxPingBurst: Math.max(...bins.map(({ pings }) => pings)),
maxRefreshBurst: Math.max(...bins.map(({ refreshes }) => refreshes))
}
}
export function assertSpreadModel(model) {
const pingTolerance = model.expectedPingRate * 0.03 + 1
const refreshTolerance = model.expectedRefreshRate * 0.08 + 1
if (Math.abs(model.observedPingRate - model.expectedPingRate) > pingTolerance) {
throw new Error('modeled heartbeat rate diverged from the 15-second contract')
}
if (Math.abs(model.observedRefreshRate - model.expectedRefreshRate) > refreshTolerance) {
throw new Error('modeled token refresh rate diverged from the 180-240 second contract')
}
if (model.maxPingBurst > model.expectedPingRate * 1.3 + 5) {
throw new Error('heartbeat phase spreading produced a reconnect cliff')
}
// One-second bins have Poisson-sized tails even with uniform phase spreading.
if (model.maxRefreshBurst > model.expectedRefreshRate * 1.75 + 5) {
throw new Error('refresh phase spreading produced an auth herd')
}
}
@@ -0,0 +1,25 @@
import assert from 'node:assert/strict'
import test from 'node:test'
import { assertSpreadModel, controlPhase, modeledRelayLoad } from './relay-load-model.mjs'
test('phases are deterministic, bounded, and separated by stream', () => {
assert.deepEqual(controlPhase(42), controlPhase(42))
assert.notDeepEqual(controlPhase(42), controlPhase(43))
const phase = controlPhase(42)
assert.ok(phase.heartbeatOffsetMs >= 0 && phase.heartbeatOffsetMs < 15_000)
assert.ok(phase.refreshIntervalMs >= 180_000 && phase.refreshIntervalMs <= 240_000)
assert.ok(phase.refreshOffsetMs >= 0 && phase.refreshOffsetMs < phase.refreshIntervalMs)
assert.ok(phase.reconnectJitterMs >= 0 && phase.reconnectJitterMs < 30_000)
})
for (const [controls, expectedPings, expectedRefreshes] of [
[4_000, 267, 19],
[10_000, 667, 48]
]) {
test(`${controls} modeled controls spread heartbeat and token refresh load`, () => {
const model = modeledRelayLoad(controls)
assert.equal(Math.round(model.expectedPingRate), expectedPings)
assert.equal(Math.round(model.expectedRefreshRate), expectedRefreshes)
assert.doesNotThrow(() => assertSpreadModel(model))
})
}
@@ -0,0 +1,37 @@
import { access, mkdir, open } from 'node:fs/promises'
import { join } from 'node:path'
import { setTimeout as delayDefault } from 'node:timers/promises'
export async function waitForRelayLoadPhaseBarrier(config, overrides = {}) {
const delay = overrides.delay ?? delayDefault
const now = overrides.now ?? Date.now
const timeoutMs = overrides.timeoutMs ?? config.timeoutMs
if (
typeof config.directory !== 'string' || config.directory.length === 0 ||
!Number.isSafeInteger(config.shardCount) || config.shardCount < 2 ||
!Number.isSafeInteger(config.shardIndex) || config.shardIndex < 0 ||
config.shardIndex >= config.shardCount ||
!Number.isSafeInteger(timeoutMs) || timeoutMs < 1
) throw new Error('invalid Relay load phase barrier')
await mkdir(config.directory, { recursive: true })
const marker = join(config.directory, `${config.shardIndex}.ready`)
const handle = await open(marker, 'wx')
await handle.close()
const deadline = now() + timeoutMs
for (;;) {
const ready = await Promise.all(
Array.from({ length: config.shardCount }, async (_, index) => {
try {
await access(join(config.directory, `${index}.ready`))
return true
} catch {
return false
}
})
)
if (ready.every(Boolean)) return
if (now() >= deadline) throw new Error('Relay load phase barrier timed out')
await delay(100)
}
}
@@ -0,0 +1,65 @@
import assert from 'node:assert/strict'
import { mkdtemp, readFile, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import test from 'node:test'
import { waitForRelayLoadPhaseBarrier } from './relay-load-phase-barrier.mjs'
const loadHarness = await readFile(new URL('./load-relay-controls.mjs', import.meta.url), 'utf8')
test('releases every shard only after all readiness markers exist', async () => {
const directory = await mkdtemp(join(tmpdir(), 'relay-load-barrier-'))
try {
let firstResolved = false
const first = waitForRelayLoadPhaseBarrier({
directory, shardCount: 2, shardIndex: 0, timeoutMs: 1_000
}).then(() => { firstResolved = true })
await new Promise((resolve) => setTimeout(resolve, 20))
assert.equal(firstResolved, false)
await Promise.all([
first,
waitForRelayLoadPhaseBarrier({
directory, shardCount: 2, shardIndex: 1, timeoutMs: 1_000
})
])
assert.equal(firstResolved, true)
} finally {
await rm(directory, { recursive: true, force: true })
}
})
test('fails closed on a duplicate shard or incomplete barrier', async () => {
const directory = await mkdtemp(join(tmpdir(), 'relay-load-barrier-'))
try {
const nowValues = [0, 2]
await assert.rejects(
waitForRelayLoadPhaseBarrier(
{ directory, shardCount: 2, shardIndex: 0, timeoutMs: 1 },
{ now: () => nowValues.shift() ?? 2, delay: async () => undefined }
),
/timed out/
)
await assert.rejects(
waitForRelayLoadPhaseBarrier({
directory, shardCount: 2, shardIndex: 0, timeoutMs: 1
}),
/EEXIST/
)
} finally {
await rm(directory, { recursive: true, force: true })
}
})
test('synchronizes splice ramps after every shard finishes reader baselines', () => {
assert.match(
loadHarness,
/createRelayLoadReaderEvidence[\s\S]*?phaseBarrierDir\}-splices[\s\S]*?splicePromises/
)
})
test('budgets both shard barriers and the splice ramp in token lifetime', () => {
assert.match(
loadHarness,
/phaseBarrierDir \? 2 \* config\.phaseBarrierTimeoutMs : 0[\s\S]*?config\.spliceRampMs/
)
})
@@ -0,0 +1,29 @@
export async function proveRelayLoadPlacementBoundary({ peer, failureReason }) {
let connected = false
try {
await peer.connect()
connected = true
} catch (error) {
const reason = failureReason(error)
if (reason !== 'assignment_capacity_exhausted') {
throw new Error(`placement overflow was not rejected: ${reason}`)
}
return reason
} finally {
await peer.shutdown()
}
if (connected) throw new Error('placement overflow unexpectedly connected')
}
export async function proveRelayLoadRegionalFallback({ peer, blockedOrigin }) {
try {
await peer.connect()
const assignedOrigin = peer.assignedCellUrl()
if (!assignedOrigin || assignedOrigin === blockedOrigin) {
throw new Error('regional fallback did not leave the full preferred cell')
}
return true
} finally {
await peer.shutdown()
}
}
@@ -0,0 +1,53 @@
import assert from 'node:assert/strict'
import { test } from 'node:test'
import {
proveRelayLoadPlacementBoundary,
proveRelayLoadRegionalFallback
} from './relay-load-placement-boundary.mjs'
function peer(connect) {
return { connect, shutdown: async () => undefined }
}
test('requires the next fresh placement to receive HTTP 503', async () => {
assert.equal(
await proveRelayLoadPlacementBoundary({
peer: peer(async () => {
throw new Error('relay assignment failed: 503 relay_connection_headroom_exhausted')
}),
failureReason: (error) =>
error.message === 'relay assignment failed: 503 relay_connection_headroom_exhausted'
? 'assignment_capacity_exhausted'
: 'unknown'
}),
'assignment_capacity_exhausted'
)
await assert.rejects(
proveRelayLoadPlacementBoundary({
peer: peer(async () => undefined),
failureReason: () => 'unknown'
}),
/placement overflow unexpectedly connected/
)
})
test('requires a preferred-region fallback to leave the full cell', async () => {
let shutdowns = 0
assert.equal(await proveRelayLoadRegionalFallback({
peer: {
connect: async () => undefined,
assignedCellUrl: () => 'https://c3.relay-staging.onorca.dev',
shutdown: async () => { shutdowns++ }
},
blockedOrigin: 'https://c4.relay-staging.onorca.dev'
}), true)
assert.equal(shutdowns, 1)
await assert.rejects(proveRelayLoadRegionalFallback({
peer: {
connect: async () => undefined,
assignedCellUrl: () => 'https://c4.relay-staging.onorca.dev',
shutdown: async () => undefined
},
blockedOrigin: 'https://c4.relay-staging.onorca.dev'
}), /did not leave/)
})
+404
View File
@@ -0,0 +1,404 @@
export const RELAY_CONTROL_LEASE_HORIZON_SECONDS = 105
export const RELAY_LOAD_SPLICE_HIGH_WATER_BYTES = 256 * 1024
export const RELAY_LOAD_SPLICE_WEDGED_TIMEOUT_MS = 10_000
export const RELAY_LOAD_MAX_AGGREGATE_READER_SPLICES = 16
export const RELAY_LOAD_MAX_AGGREGATE_READER_BYTES = 64 * 1024 * 1024
const DEFAULT_SLOW_READER_STREAM_BYTES = 1024 * 1024
const DEFAULT_WEDGED_READER_STREAM_BYTES = 8 * 1024 * 1024
const DEFAULT_READER_FRAME_BYTES = 64 * 1024
export function parseRelayLoadArguments(argv) {
const values = new Map()
const flags = new Set()
for (let index = 0; index < argv.length; index++) {
const argument = argv[index]
if (argument === '--') continue
if (
[
'--allow-partial',
'--allow-planned-transition-retries',
'--skip-rebind-overflow-check'
].includes(argument)
) {
flags.add(argument)
continue
}
if (!argument.startsWith('--') || index + 1 >= argv.length) {
throw new Error(`invalid argument: ${argument}`)
}
values.set(argument, argv[++index])
}
const integer = (name, fallback) => {
const parsed = Number(values.get(name) ?? fallback)
if (!Number.isSafeInteger(parsed) || parsed < 0) {
throw new Error(`${name} must be a nonnegative integer`)
}
return parsed
}
const targetOrigin = values.get('--target-origin')?.replace(/\/$/, '')
const directorOrigin = values.get('--director-origin')?.replace(/\/$/, '')
const authOrigin = values.get('--auth-origin')?.replace(/\/$/, '')
if ((!targetOrigin && !directorOrigin) || (targetOrigin && directorOrigin) || !authOrigin) {
throw new Error('provide --auth-origin and exactly one target or director origin')
}
const controls = integer('--controls', 100)
const maxRampConnectionFailures = integer('--max-ramp-connection-failures', 0)
const maxUnexpectedCloses = integer('--max-unexpected-closes', 0)
const rebindProbes = integer('--rebind-probes', 0)
const placementOverflowProbes = integer('--placement-overflow-probes', 0)
const regionalFallbackProbes = integer('--regional-fallback-probes', 0)
const regionBehaviorProbes = integer('--region-behavior-probes', 0)
const requestUnitInvites = integer('--request-unit-invites', 0)
const requestUnitInvitesPerSecond = integer('--request-unit-invites-per-second', 0)
const requestUnitPrincipalCount = integer('--request-unit-principals', 0)
const relayAsiaLoadPrincipalCount = integer('--relay-asia-load-principals', 0)
const requestUnitOverflowProbes = integer('--request-unit-overflow-probes', 0)
const requestUnitCapacity = values.has('--request-unit-capacity')
? integer('--request-unit-capacity', 0)
: undefined
const requestUnitCleanupTimeoutMs = integer(
'--request-unit-cleanup-timeout-seconds',
0
) * 1000
const rebindHoldMs = integer('--rebind-hold-ms', 4_000)
const rebindDelayMs = integer('--rebind-delay-seconds', 0) * 1000
const capacityCellId = values.get('--capacity-cell-id')
const capacityCellOrigin = values.get('--capacity-cell-origin')?.replace(/\/$/, '')
const capacityHardCap = values.has('--capacity-hard-cap')
? integer('--capacity-hard-cap', 0)
: undefined
const capacityUnobservedBound = values.has('--capacity-unobserved-bound')
? integer('--capacity-unobserved-bound', 0)
: undefined
const shardCount = integer('--shard-count', 1)
const shardIndex = integer('--shard-index', 0)
const durationMs = integer('--duration-seconds', 900) * 1000
const spliceHoldMs = integer(
'--splice-hold-seconds',
values.get('--duration-seconds') ?? 900
) * 1000
const splices = integer('--splices', 0)
const spliceRampMs = integer('--splice-ramp-seconds', 0) * 1000
const slowReaderSplices = integer('--slow-reader-splices', 0)
const wedgedReaderSplices = integer('--wedged-reader-splices', 0)
const splicePayloadBytes = integer('--splice-payload-bytes', 1_024)
const slowReaderStreamBytes = integer(
'--slow-reader-stream-bytes',
DEFAULT_SLOW_READER_STREAM_BYTES
)
const wedgedReaderStreamBytes = integer(
'--wedged-reader-stream-bytes',
DEFAULT_WEDGED_READER_STREAM_BYTES
)
const readerFrameBytes = integer('--reader-frame-bytes', DEFAULT_READER_FRAME_BYTES)
const slowReaderHoldMs = integer('--slow-reader-hold-ms', 2_000)
const wedgedReaderHoldMs = integer('--wedged-reader-hold-ms', 12_000)
const maxGeneratorRssGrowthMiB = integer('--max-generator-rss-growth-mib', 512)
const requiredLeaseHorizons = integer('--required-lease-horizons', 0)
const aggregateControls = values.has('--aggregate-controls')
? integer('--aggregate-controls', 0)
: controls
const aggregateSplices = values.has('--aggregate-splices')
? integer('--aggregate-splices', 0)
: splices
const aggregateRequestUnitInvites = values.has('--aggregate-request-unit-invites')
? integer('--aggregate-request-unit-invites', 0)
: requestUnitInvites
const phaseBarrierDir = values.get('--phase-barrier-dir')
const phaseBarrierTimeoutMs = integer('--phase-barrier-timeout-seconds', 180) * 1000
if (controls < 1 || controls > 10_000) throw new Error('--controls must be between 1 and 10000')
if (rebindProbes > controls) throw new Error('--rebind-probes cannot exceed --controls')
if (splices > controls) throw new Error('--splices cannot exceed --controls')
if (shardCount > 1 && capacityHardCap !== undefined) {
if (!values.has('--aggregate-controls') || !values.has('--aggregate-splices')) {
throw new Error('capacity-bound sharding requires explicit aggregate controls and splices')
}
if (aggregateControls !== controls * shardCount || aggregateSplices !== splices * shardCount) {
throw new Error('aggregate controls and splices must match every equal-sized shard')
}
} else if (aggregateControls !== controls || aggregateSplices !== splices) {
throw new Error('aggregate controls and splices require matching sharded local counts')
}
if (
capacityHardCap !== undefined &&
aggregateControls + 2 * aggregateSplices > capacityHardCap - 100
) {
throw new Error('controls plus splice connection units exceed ordinary cell admission')
}
if (slowReaderSplices + wedgedReaderSplices > splices) {
throw new Error('reader splice counts cannot exceed --splices')
}
if (splices > 0 && (spliceHoldMs < 1_000 || spliceHoldMs > durationMs)) {
throw new Error('--splice-hold-seconds must be between 1 and the steady duration')
}
if (slowReaderSplices + wedgedReaderSplices > 0 && !directorOrigin) {
throw new Error('reader evidence requires --director-origin')
}
if (splicePayloadBytes < 1 || splicePayloadBytes > 1_048_576) {
throw new Error('--splice-payload-bytes must be between 1 and 1048576')
}
if (
slowReaderSplices > 0 &&
slowReaderStreamBytes <= RELAY_LOAD_SPLICE_HIGH_WATER_BYTES
) {
throw new Error('--slow-reader-stream-bytes must exceed the 256 KiB splice high-water mark')
}
if (
wedgedReaderSplices > 0 &&
wedgedReaderStreamBytes <= RELAY_LOAD_SPLICE_HIGH_WATER_BYTES
) {
throw new Error('--wedged-reader-stream-bytes must exceed the 256 KiB splice high-water mark')
}
const localReaderSplices = slowReaderSplices + wedgedReaderSplices
const localReaderBytes = slowReaderSplices * slowReaderStreamBytes +
wedgedReaderSplices * wedgedReaderStreamBytes
const aggregateReaderSplices = values.has('--aggregate-reader-splices')
? integer('--aggregate-reader-splices', 0)
: localReaderSplices * shardCount
const aggregateReaderBytes = values.has('--aggregate-reader-bytes')
? integer('--aggregate-reader-bytes', 0)
: localReaderBytes * shardCount
if (
aggregateReaderSplices < localReaderSplices ||
aggregateReaderBytes < localReaderBytes ||
(shardCount === 1 &&
(aggregateReaderSplices !== localReaderSplices || aggregateReaderBytes !== localReaderBytes))
) {
throw new Error('aggregate reader bounds do not cover the local shard')
}
if (aggregateReaderSplices > RELAY_LOAD_MAX_AGGREGATE_READER_SPLICES) {
throw new Error('aggregate reader splice count exceeds the reviewed bound')
}
if (aggregateReaderBytes > RELAY_LOAD_MAX_AGGREGATE_READER_BYTES) {
throw new Error('aggregate reader stream bytes exceed the reviewed bound')
}
if (readerFrameBytes < 1 || readerFrameBytes > 1_048_576) {
throw new Error('--reader-frame-bytes must be between 1 and 1048576')
}
if (slowReaderSplices > 0 && slowReaderHoldMs >= RELAY_LOAD_SPLICE_WEDGED_TIMEOUT_MS) {
throw new Error('--slow-reader-hold-ms must stay below the wedged timeout')
}
if (wedgedReaderSplices > 0 && wedgedReaderHoldMs <= RELAY_LOAD_SPLICE_WEDGED_TIMEOUT_MS) {
throw new Error('--wedged-reader-hold-ms must exceed the wedged timeout')
}
if (wedgedReaderHoldMs > 30_000) {
throw new Error('--wedged-reader-hold-ms cannot exceed 30000')
}
if (maxGeneratorRssGrowthMiB < 1) {
throw new Error('--max-generator-rss-growth-mib must be positive')
}
if (placementOverflowProbes > 1) {
throw new Error('--placement-overflow-probes must be zero or one')
}
if (regionalFallbackProbes > 1) {
throw new Error('--regional-fallback-probes must be zero or one')
}
if (regionBehaviorProbes > 1 || requestUnitOverflowProbes > 1) {
throw new Error('regional behavior and request-unit overflow probes must be zero or one')
}
if (placementOverflowProbes > 0 && shardCount > 1) {
throw new Error('placement overflow proof requires one coordinated generator')
}
if (
placementOverflowProbes > 0 &&
(!directorOrigin ||
!capacityCellId ||
capacityHardCap === undefined ||
capacityUnobservedBound === undefined)
) {
throw new Error('placement overflow requires exact capacity cell, hard cap, and bound')
}
if (regionalFallbackProbes > 0 && (
!directorOrigin || !capacityCellId || !capacityCellOrigin ||
capacityHardCap === undefined || capacityUnobservedBound === undefined ||
(shardCount > 1 && shardIndex !== 0)
)) throw new Error('regional fallback requires the coordinating capacity shard')
if (regionBehaviorProbes > 0 && (
!directorOrigin || !capacityCellOrigin || preferredRegionValue(values) !== 'asia-east2' ||
(shardCount > 1 && shardIndex !== 0)
)) throw new Error('regional behavior proof requires the coordinating Asia shard')
if (phaseBarrierDir && shardCount < 2) {
throw new Error('phase barrier requires multiple shards')
}
if (phaseBarrierTimeoutMs < 1_000) {
throw new Error('phase barrier timeout must be at least one second')
}
if (requestUnitInvites > 0) {
if (
!directorOrigin || !capacityCellId || requestUnitCapacity === undefined ||
requestUnitCapacity < 1 || requestUnitInvitesPerSecond < 1 ||
requestUnitInvitesPerSecond > 20 || requestUnitPrincipalCount < 1 ||
requestUnitPrincipalCount > 32 || requestUnitInvites > requestUnitPrincipalCount * 30 ||
aggregateSplices !== 0 || !phaseBarrierDir ||
aggregateRequestUnitInvites !== requestUnitInvites * shardCount ||
aggregateControls + aggregateRequestUnitInvites !== requestUnitCapacity
) throw new Error('request-unit proof does not reach the exact reviewed capacity')
} else if (
requestUnitCapacity !== undefined || requestUnitInvitesPerSecond !== 0 ||
requestUnitPrincipalCount !== 0 ||
requestUnitOverflowProbes > 0 ||
requestUnitCleanupTimeoutMs > 0 || aggregateRequestUnitInvites !== 0
) throw new Error('request-unit proof options require invite offers')
if (
requestUnitOverflowProbes > 0 &&
(shardIndex !== 0 || requestUnitCleanupTimeoutMs < 600_000)
) throw new Error('request-unit overflow requires the cleanup-owning coordinator')
if (requestUnitCleanupTimeoutMs > 0 && requestUnitOverflowProbes !== 1) {
throw new Error('request-unit cleanup requires the overflow proof')
}
if (
relayAsiaLoadPrincipalCount > 32 ||
(relayAsiaLoadPrincipalCount > 0 &&
(!directorOrigin || preferredRegionValue(values) !== 'asia-east2'))
) throw new Error('Relay Asia load principals require a regional director proof')
if (capacityCellOrigin) {
const origin = new URL(capacityCellOrigin)
if (origin.protocol !== 'https:' || origin.origin !== capacityCellOrigin) {
throw new Error('--capacity-cell-origin must be canonical HTTPS')
}
}
if (shardCount < 1 || shardIndex >= shardCount) throw new Error('invalid shard index/count')
const minimumDurationMs = requiredLeaseHorizons * RELAY_CONTROL_LEASE_HORIZON_SECONDS * 1000
if (durationMs < minimumDurationMs) {
throw new Error(`--duration-seconds must cover ${requiredLeaseHorizons} lease horizons`)
}
return {
targetOrigin,
directorOrigin,
authOrigin,
preferredRegion: preferredRegionValue(values),
controls,
maxRampConnectionFailures,
maxUnexpectedCloses,
rebindProbes,
placementOverflowProbes,
regionalFallbackProbes,
regionBehaviorProbes,
requestUnitInvites,
requestUnitInvitesPerSecond,
requestUnitPrincipalCount,
relayAsiaLoadPrincipalCount,
requestUnitOverflowProbes,
requestUnitCapacity,
requestUnitCleanupTimeoutMs,
rebindHoldMs,
rebindDelayMs,
capacityCellId,
capacityCellOrigin,
capacityHardCap,
capacityUnobservedBound,
durationMs,
spliceHoldMs,
rampMs: integer('--ramp-seconds', 60) * 1000,
rampStartDelayMs: integer('--ramp-start-delay-ms', 0),
reconnectMaxMs: integer('--reconnect-max-seconds', 30) * 1000,
shardCount,
shardIndex,
aggregateControls,
aggregateSplices,
aggregateRequestUnitInvites,
phaseBarrierDir,
phaseBarrierTimeoutMs,
aggregateReaderSplices,
aggregateReaderBytes,
signingKeyFile: values.get('--signing-key-file'),
splices,
spliceRampMs,
splicePayloadBytes,
slowReaderSplices,
wedgedReaderSplices,
slowReaderStreamBytes,
wedgedReaderStreamBytes,
readerFrameBytes,
slowReaderHoldMs,
wedgedReaderHoldMs,
maxGeneratorRssGrowthMiB,
requiredLeaseHorizons,
allowPartial: flags.has('--allow-partial'),
allowPlannedTransitionRetries: flags.has('--allow-planned-transition-retries'),
requireRebindOverflow: !flags.has('--skip-rebind-overflow-check')
}
}
function preferredRegionValue(values) {
return values.get('--preferred-region')
}
export function relayLoadSpliceIndexes(config) {
return Array.from(
{ length: config.splices },
(_, localIndex) => localIndex * config.shardCount + config.shardIndex
)
}
export function relayLoadSpliceStartDelayMs(config, localIndex) {
if (!Number.isSafeInteger(localIndex) || localIndex < 0 || localIndex >= config.splices) {
throw new Error('invalid local splice index')
}
const totalSplices = config.splices * config.shardCount
if (totalSplices <= 1) return 0
const globalOrdinal = localIndex * config.shardCount + config.shardIndex
return Math.floor(globalOrdinal * config.spliceRampMs / (totalSplices - 1))
}
export function relayLoadPrincipalIndex(peerIndex, shardCount, principalCount) {
if (
!Number.isSafeInteger(peerIndex) || peerIndex < 0 ||
!Number.isSafeInteger(shardCount) || shardCount < 1 ||
!Number.isSafeInteger(principalCount) || principalCount < 1
) throw new Error('invalid Relay load principal mapping')
return Math.floor(peerIndex / shardCount) % principalCount
}
export function relayLoadSpliceProfile(config, spliceIndex) {
if (spliceIndex < config.wedgedReaderSplices) {
return {
readerMode: 'wedged',
readerHoldMs: config.wedgedReaderHoldMs,
streamBytes: config.wedgedReaderStreamBytes,
frameBytes: config.readerFrameBytes
}
}
if (spliceIndex < config.wedgedReaderSplices + config.slowReaderSplices) {
return {
readerMode: 'slow',
readerHoldMs: config.slowReaderHoldMs,
streamBytes: config.slowReaderStreamBytes,
frameBytes: config.readerFrameBytes
}
}
return {
readerMode: 'normal',
readerHoldMs: 0,
streamBytes: config.splicePayloadBytes,
frameBytes: config.splicePayloadBytes
}
}
export function relayLoadReaderEvidenceError(result, config) {
const readerSplices = config.slowReaderSplices + config.wedgedReaderSplices
if (readerSplices > 0 && result.generatorRssGrowthMiB > config.maxGeneratorRssGrowthMiB) {
return 'load generator exceeded its RSS growth budget'
}
if (result.slowReaderSplicesCompleted !== config.slowReaderSplices) {
return 'slow-reader streams did not all complete'
}
if (result.wedgedReaderSplicesClosed !== config.wedgedReaderSplices) {
return 'wedged-reader streams did not all close at the relay limit'
}
if (
readerSplices > 0 &&
(result.readerQueueEvidence.length === 0 ||
result.readerQueueEvidence.some(({ increaseBytes }) => increaseBytes < 1))
) {
return 'reader streams produced no causal Relay queued-byte evidence'
}
if (
config.wedgedReaderSplices > 0 &&
result.readerClosesByCode['4429'] !== config.wedgedReaderSplices
) {
return 'wedged-reader streams did not close with 4429'
}
return undefined
}
@@ -0,0 +1,388 @@
import assert from 'node:assert/strict'
import test from 'node:test'
import {
parseRelayLoadArguments,
relayLoadPrincipalIndex,
relayLoadReaderEvidenceError,
relayLoadSpliceIndexes,
relayLoadSpliceProfile,
relayLoadSpliceStartDelayMs
} from './relay-load-profile.mjs'
const required = ['--auth-origin', 'https://auth.test', '--director-origin', 'https://relay.test']
test('accepts the 2840-control two-lease-horizon Asia proof profile', () => {
const config = parseRelayLoadArguments([
...required,
'--controls',
'2840',
'--duration-seconds',
'210',
'--required-lease-horizons',
'2',
'--preferred-region',
'asia-east2',
'--relay-asia-load-principals',
'32',
'--capacity-hard-cap',
'3000'
])
assert.equal(config.controls, 2840)
assert.equal(config.durationMs, 210_000)
assert.equal(config.requiredLeaseHorizons, 2)
assert.equal(config.preferredRegion, 'asia-east2')
assert.equal(config.relayAsiaLoadPrincipalCount, 32)
assert.equal(config.splices, 0)
})
test('binds synthetic Relay principals to a bounded Asia director proof', () => {
assert.throws(() => parseRelayLoadArguments([
...required, '--relay-asia-load-principals', '1'
]), /require a regional director proof/)
assert.throws(() => parseRelayLoadArguments([
...required, '--preferred-region', 'asia-east2',
'--relay-asia-load-principals', '33'
]), /require a regional director proof/)
})
test('rejects a mixed profile beyond the ordinary 2900-unit boundary', () => {
assert.throws(
() => parseRelayLoadArguments([
...required,
'--controls', '2840',
'--splices', '31',
'--capacity-hard-cap', '3000'
]),
/exceed ordinary cell admission/
)
expectMixedProfile(parseRelayLoadArguments([
...required,
'--controls', '2600',
'--splices', '120',
'--capacity-hard-cap', '3000'
]))
})
test('requires reviewed aggregate totals for capacity-bound shards', () => {
assert.throws(
() => parseRelayLoadArguments([
...required, '--controls', '2840', '--capacity-hard-cap', '3000',
'--shard-count', '4', '--shard-index', '0'
]),
/requires explicit aggregate controls and splices/
)
assert.throws(
() => parseRelayLoadArguments([
...required, '--controls', '2840', '--capacity-hard-cap', '3000',
'--shard-count', '4', '--shard-index', '0',
'--aggregate-controls', '2840', '--aggregate-splices', '0'
]),
/must match every equal-sized shard/
)
const config = parseRelayLoadArguments([
...required, '--controls', '710', '--capacity-hard-cap', '3000',
'--shard-count', '4', '--shard-index', '0',
'--aggregate-controls', '2840', '--aggregate-splices', '0'
])
assert.equal(config.aggregateControls, 2840)
assert.equal(config.aggregateSplices, 0)
})
test('allows one coordinating shard to prove regional fallback', () => {
const config = parseRelayLoadArguments([
...required, '--controls', '710', '--capacity-hard-cap', '3000',
'--shard-count', '4', '--shard-index', '0',
'--aggregate-controls', '2840', '--aggregate-splices', '0',
'--regional-fallback-probes', '1', '--capacity-cell-id', 'staging-gce-c4',
'--capacity-cell-origin', 'https://c4.relay-staging.onorca.dev',
'--capacity-unobserved-bound', '60', '--rebind-probes', '160'
])
assert.equal(config.regionalFallbackProbes, 1)
assert.equal(config.capacityCellOrigin, 'https://c4.relay-staging.onorca.dev')
assert.throws(() => parseRelayLoadArguments([
...required, '--controls', '710', '--capacity-hard-cap', '3000',
'--shard-count', '4', '--shard-index', '1',
'--aggregate-controls', '2840', '--aggregate-splices', '0',
'--regional-fallback-probes', '1', '--capacity-cell-id', 'staging-gce-c4',
'--capacity-cell-origin', 'https://c4.relay-staging.onorca.dev',
'--capacity-unobserved-bound', '60'
]), /coordinating capacity shard/)
})
test('accepts the exact sharded request-unit and region behavior proof', () => {
const config = parseRelayLoadArguments([
...required, '--preferred-region', 'asia-east2',
'--controls', '710', '--capacity-hard-cap', '3000',
'--shard-count', '4', '--shard-index', '0',
'--aggregate-controls', '2840', '--aggregate-splices', '0',
'--capacity-cell-id', 'staging-gce-c4',
'--capacity-cell-origin', 'https://c4.relay-staging.onorca.dev',
'--request-unit-invites', '790', '--request-unit-invites-per-second', '2',
'--request-unit-principals', '32',
'--aggregate-request-unit-invites', '3160',
'--request-unit-capacity', '6000', '--request-unit-overflow-probes', '1',
'--request-unit-cleanup-timeout-seconds', '630',
'--region-behavior-probes', '1', '--phase-barrier-dir', '/tmp/load-barrier'
])
assert.equal(config.aggregateRequestUnitInvites, 3_160)
assert.equal(config.requestUnitCapacity, 6_000)
assert.equal(config.requestUnitPrincipalCount, 32)
assert.equal(config.requestUnitCleanupTimeoutMs, 630_000)
assert.equal(config.regionBehaviorProbes, 1)
assert.equal(config.phaseBarrierDir, '/tmp/load-barrier')
assert.throws(() => parseRelayLoadArguments([
...required, '--controls', '710', '--capacity-hard-cap', '3000',
'--shard-count', '4', '--shard-index', '0',
'--aggregate-controls', '2840', '--aggregate-splices', '0',
'--capacity-cell-id', 'staging-gce-c4', '--request-unit-invites', '789',
'--request-unit-invites-per-second', '2',
'--request-unit-principals', '32',
'--aggregate-request-unit-invites', '3156', '--request-unit-capacity', '6000',
'--phase-barrier-dir', '/tmp/load-barrier'
]), /does not reach the exact reviewed capacity/)
assert.throws(() => parseRelayLoadArguments([
...required, '--controls', '710', '--capacity-hard-cap', '3000',
'--shard-count', '4', '--shard-index', '0',
'--aggregate-controls', '2840', '--aggregate-splices', '0',
'--capacity-cell-id', 'staging-gce-c4', '--request-unit-invites', '790',
'--request-unit-invites-per-second', '2', '--request-unit-principals', '26',
'--aggregate-request-unit-invites', '3160', '--request-unit-capacity', '6000',
'--phase-barrier-dir', '/tmp/load-barrier'
]), /does not reach the exact reviewed capacity/)
})
function expectMixedProfile(config) {
assert.equal(config.controls + 2 * config.splices, 2840)
}
test('rejects a run shorter than its required lease horizons', () => {
assert.throws(
() =>
parseRelayLoadArguments([
...required,
'--duration-seconds',
'209',
'--required-lease-horizons',
'2'
]),
/must cover 2 lease horizons/
)
})
test('bounds an explicit splice hold within the steady window', () => {
const config = parseRelayLoadArguments([
...required,
'--controls', '1',
'--splices', '1',
'--duration-seconds', '300',
'--splice-hold-seconds', '60'
])
assert.equal(config.durationMs, 300_000)
assert.equal(config.spliceHoldMs, 60_000)
assert.throws(() => parseRelayLoadArguments([
...required,
'--controls', '1',
'--splices', '1',
'--duration-seconds', '300',
'--splice-hold-seconds', '301'
]), /between 1 and the steady duration/)
})
test('maps splice ownership deterministically within a shard', () => {
const config = parseRelayLoadArguments([
...required,
'--controls',
'4',
'--splices',
'3',
'--shard-count',
'4',
'--shard-index',
'2'
])
assert.deepEqual(relayLoadSpliceIndexes(config), [2, 6, 10])
})
test('staggered shards form one deterministic splice ramp', () => {
const config = parseRelayLoadArguments([
...required,
'--controls', '4',
'--splices', '3',
'--splice-ramp-seconds', '11',
'--shard-count', '4',
'--shard-index', '2'
])
assert.equal(config.spliceRampMs, 11_000)
assert.deepEqual(
[0, 1, 2].map((index) => relayLoadSpliceStartDelayMs(config, index)),
[2_000, 6_000, 10_000]
)
assert.throws(() => relayLoadSpliceStartDelayMs(config, 3), /invalid local splice index/)
})
test('distributes each shard invite wave below the account rate limit', () => {
const identities = Array.from(
{ length: 710 },
(_, localIndex) => relayLoadPrincipalIndex(localIndex * 4 + 2, 4, 32)
)
const offers = Array.from({ length: 790 }, (_, index) => identities[index % identities.length])
const counts = new Map()
for (const principal of offers) counts.set(principal, (counts.get(principal) ?? 0) + 1)
assert.equal(counts.size, 32)
assert.equal(Math.max(...counts.values()), 26)
})
test('requires exactly one assignment mode', () => {
assert.throws(
() =>
parseRelayLoadArguments([
...required,
'--target-origin',
'https://cell.test'
]),
/exactly one target or director origin/
)
})
test('requires separate recoverable and wedged reader profiles', () => {
const config = parseRelayLoadArguments([
...required,
'--controls', '4',
'--splices', '3',
'--slow-reader-splices', '1',
'--wedged-reader-splices', '1',
'--slow-reader-stream-bytes', '524288',
'--wedged-reader-stream-bytes', '1048576',
'--slow-reader-hold-ms', '9000',
'--wedged-reader-hold-ms', '11000'
])
assert.deepEqual(relayLoadSpliceProfile(config, 0), {
readerMode: 'wedged', readerHoldMs: 11_000, streamBytes: 1_048_576, frameBytes: 65_536
})
assert.deepEqual(relayLoadSpliceProfile(config, 1), {
readerMode: 'slow', readerHoldMs: 9_000, streamBytes: 524_288, frameBytes: 65_536
})
assert.equal(relayLoadSpliceProfile(config, 2).readerMode, 'normal')
})
test('bounds reader stream, timeout, and splice load per shard', () => {
assert.throws(
() => parseRelayLoadArguments([...required, '--controls', '2', '--splices', '3']),
/splices cannot exceed/
)
assert.throws(
() =>
parseRelayLoadArguments([
...required,
'--splices',
'1',
'--slow-reader-splices',
'2'
]),
/reader splice counts cannot exceed/
)
assert.throws(
() => parseRelayLoadArguments([
...required, '--splices', '1', '--slow-reader-splices', '1',
'--slow-reader-stream-bytes', '262144'
]),
/must exceed the 256 KiB/
)
assert.throws(
() => parseRelayLoadArguments([
...required, '--splices', '1', '--wedged-reader-splices', '1',
'--wedged-reader-stream-bytes', '262144'
]),
/must exceed the 256 KiB/
)
assert.throws(
() => parseRelayLoadArguments([
...required, '--splices', '1', '--slow-reader-splices', '1',
'--slow-reader-hold-ms', '10000'
]),
/must stay below the wedged timeout/
)
assert.throws(
() => parseRelayLoadArguments([
...required, '--splices', '1', '--wedged-reader-splices', '1',
'--wedged-reader-hold-ms', '10000'
]),
/must exceed the wedged timeout/
)
assert.throws(
() => parseRelayLoadArguments([
...required, '--controls', '17', '--splices', '17', '--slow-reader-splices', '17'
]),
/reader splice count exceeds/
)
assert.throws(
() => parseRelayLoadArguments([
...required, '--controls', '9', '--splices', '9', '--wedged-reader-splices', '9'
]),
/reader stream bytes exceed/
)
})
test('supports one bounded reader-owning shard without multiplying its pressure', () => {
const owner = parseRelayLoadArguments([
...required,
'--controls', '650', '--splices', '30', '--capacity-hard-cap', '3000',
'--shard-count', '4', '--shard-index', '0',
'--aggregate-controls', '2600', '--aggregate-splices', '120',
'--slow-reader-splices', '10', '--wedged-reader-splices', '1',
'--aggregate-reader-splices', '11', '--aggregate-reader-bytes', '18874368'
])
const peer = parseRelayLoadArguments([
...required,
'--controls', '650', '--splices', '30', '--capacity-hard-cap', '3000',
'--shard-count', '4', '--shard-index', '1',
'--aggregate-controls', '2600', '--aggregate-splices', '120',
'--aggregate-reader-splices', '11', '--aggregate-reader-bytes', '18874368'
])
assert.equal(owner.aggregateReaderSplices, 11)
assert.equal(owner.aggregateReaderBytes, 18 * 1024 * 1024)
assert.equal(peer.slowReaderSplices + peer.wedgedReaderSplices, 0)
assert.equal(peer.aggregateReaderSplices, 11)
})
test('requires queue, memory, and expected close evidence', () => {
const config = parseRelayLoadArguments([
...required, '--splices', '2', '--slow-reader-splices', '1',
'--wedged-reader-splices', '1', '--max-generator-rss-growth-mib', '100'
])
const passing = {
generatorRssGrowthMiB: 50,
slowReaderSplicesCompleted: 1,
wedgedReaderSplicesClosed: 1,
readerQueueEvidence: [
{ origin: 'https://cell.test', baselineBytes: 8, peakBytes: 65_544, increaseBytes: 65_536 }
],
readerClosesByCode: { '4429': 1 }
}
assert.equal(relayLoadReaderEvidenceError(passing, config), undefined)
assert.match(
relayLoadReaderEvidenceError({
...passing,
readerQueueEvidence: [
{ origin: 'https://cell.test', baselineBytes: 8, peakBytes: 8, increaseBytes: 0 }
]
}, config),
/no causal Relay queued-byte evidence/
)
assert.match(
relayLoadReaderEvidenceError({ ...passing, generatorRssGrowthMiB: 101 }, config),
/RSS growth budget/
)
assert.match(
relayLoadReaderEvidenceError({ ...passing, readerClosesByCode: { '4429': 0 } }, config),
/did not close with 4429/
)
})
@@ -0,0 +1,51 @@
const DEFAULT_TIMEOUT_MS = 8_000
const DEFAULT_POLL_MS = 100
export async function createRelayLoadReaderEvidence(origins, dependencies) {
const distinctOrigins = [...new Set(origins)].sort()
const baselines = new Map(await Promise.all(distinctOrigins.map(async (origin) => [
origin,
await dependencies.readQueuedBytes(origin)
])))
const peaks = new Map(baselines)
const pending = new Map()
const now = dependencies.now ?? Date.now
const delay = dependencies.delay
const timeoutMs = dependencies.timeoutMs ?? DEFAULT_TIMEOUT_MS
const pollMs = dependencies.pollMs ?? DEFAULT_POLL_MS
const observe = async ({ cellOrigin }) => {
if (!baselines.has(cellOrigin)) throw new Error('reader origin lacks a run baseline')
if (peaks.get(cellOrigin) > baselines.get(cellOrigin)) return
const current = pending.get(cellOrigin)
if (current) return await current
const proof = (async () => {
const baseline = baselines.get(cellOrigin)
const deadline = now() + timeoutMs
for (;;) {
const queuedBytes = await dependencies.readQueuedBytes(cellOrigin)
peaks.set(cellOrigin, Math.max(peaks.get(cellOrigin), queuedBytes))
if (queuedBytes > baseline) return
if (now() >= deadline) {
throw new Error('reader stream produced no causal Relay queued-byte increase')
}
await delay(pollMs)
}
})()
pending.set(cellOrigin, proof)
try {
await proof
} finally {
pending.delete(cellOrigin)
}
}
const snapshot = () => distinctOrigins.map((origin) => ({
origin,
baselineBytes: baselines.get(origin),
peakBytes: peaks.get(origin),
increaseBytes: peaks.get(origin) - baselines.get(origin)
}))
return { observe, snapshot }
}
@@ -0,0 +1,76 @@
import assert from 'node:assert/strict'
import test from 'node:test'
import { createRelayLoadReaderEvidence } from './relay-load-reader-evidence.mjs'
test('requires a queue increase above the pre-injection baseline for every origin', async () => {
const samples = new Map([
['https://a.test', [7, 7, 11]],
['https://b.test', [0, 3]]
])
let now = 0
const evidence = await createRelayLoadReaderEvidence([...samples.keys()], {
readQueuedBytes: async (origin) => samples.get(origin).shift(),
delay: async (ms) => { now += ms },
now: () => now
})
await Promise.all([
evidence.observe({ cellOrigin: 'https://a.test' }),
evidence.observe({ cellOrigin: 'https://b.test' })
])
assert.deepEqual(evidence.snapshot(), [
{ origin: 'https://a.test', baselineBytes: 7, peakBytes: 11, increaseBytes: 4 },
{ origin: 'https://b.test', baselineBytes: 0, peakBytes: 3, increaseBytes: 3 }
])
})
test('shares one causal proof across concurrent readers on the same cell', async () => {
const samples = [4, 4, 9]
let reads = 0
let now = 0
const evidence = await createRelayLoadReaderEvidence(['https://cell.test'], {
readQueuedBytes: async () => { reads++; return samples.shift() },
delay: async (ms) => { now += ms },
now: () => now
})
await Promise.all([
evidence.observe({ cellOrigin: 'https://cell.test' }),
evidence.observe({ cellOrigin: 'https://cell.test' })
])
assert.equal(reads, 3)
assert.equal(evidence.snapshot()[0].increaseBytes, 5)
})
test('reuses a completed causal proof for later readers on the same cell', async () => {
const samples = [4, 9]
let reads = 0
const evidence = await createRelayLoadReaderEvidence(['https://cell.test'], {
readQueuedBytes: async () => { reads++; return samples.shift() },
delay: async () => undefined
})
await evidence.observe({ cellOrigin: 'https://cell.test' })
await evidence.observe({ cellOrigin: 'https://cell.test' })
assert.equal(reads, 2)
assert.equal(evidence.snapshot()[0].increaseBytes, 5)
})
test('rejects a pre-existing nonzero queue that never increases', async () => {
let now = 0
const evidence = await createRelayLoadReaderEvidence(['https://cell.test'], {
readQueuedBytes: async () => 9,
delay: async (ms) => { now += ms },
now: () => now,
timeoutMs: 200,
pollMs: 100
})
await assert.rejects(
evidence.observe({ cellOrigin: 'https://cell.test' }),
/no causal Relay queued-byte increase/
)
})
@@ -0,0 +1,59 @@
export async function waitForRelayLoadRebindGate({
delay,
delayMs,
activeCount,
requiredCount
}) {
await delay(delayMs)
const active = activeCount()
if (active !== requiredCount) {
throw new Error(`rebind boundary requires ${requiredCount} active controls, found ${active}`)
}
}
export async function proveRelayLoadRebindBoundary({
peers,
probeCount,
holdMs,
delay,
failureReason,
requireOverflow = true
}) {
if (probeCount === 0) return { opened: 0, overflowReason: null }
if (peers.length < probeCount) throw new Error('insufficient active controls for rebind proof')
const probes = []
try {
const opened = await Promise.allSettled(
peers.slice(0, probeCount).map((peer) => peer.openRebindProbe())
)
for (const result of opened) {
if (result.status === 'fulfilled') probes.push(result.value)
}
const rejected = opened.find((result) => result.status === 'rejected')
if (rejected) throw rejected.reason
let overflowReason = null
if (requireOverflow) {
try {
const overflow = await peers[0].openRebindProbe()
await overflow.close()
} catch (error) {
overflowReason = failureReason(error)
}
if (overflowReason !== 'socket_http_503') {
throw new Error(`rebind overflow was not rejected at the hard cap: ${overflowReason}`)
}
}
const closedIndex = await Promise.race([
delay(holdMs).then(() => -1),
...probes.map((probe, index) => probe.closed.then(() => index))
])
if (closedIndex >= 0 || probes.some((probe) => !probe.isOpen())) {
throw new Error('rebind probe closed before the hold completed')
}
return { opened: probes.length, overflowReason }
} finally {
await Promise.all(probes.map((probe) => probe.close()))
}
}
@@ -0,0 +1,164 @@
import assert from 'node:assert/strict'
import test from 'node:test'
import {
proveRelayLoadRebindBoundary,
waitForRelayLoadRebindGate
} from './relay-load-rebind-boundary.mjs'
function peer(open) {
return { openRebindProbe: open }
}
function probe(onClose = () => undefined) {
let open = true
let resolveClosed
const closed = new Promise((resolve) => {
resolveClosed = resolve
})
return {
close: () => {
if (!open) return
open = false
onClose()
resolveClosed()
},
closed,
isOpen: () => open
}
}
test('holds the requested rebind overlap and requires a hard-cap rejection', async () => {
let openCalls = 0
let closes = 0
let heldFor = null
const peers = [
peer(async () => {
openCalls++
if (openCalls === 3) throw new Error('Unexpected server response: 503')
return probe(() => closes++)
}),
peer(async () => {
openCalls++
return probe(() => closes++)
})
]
const result = await proveRelayLoadRebindBoundary({
peers,
probeCount: 2,
holdMs: 4_000,
delay: async (milliseconds) => {
heldFor = milliseconds
},
failureReason: (error) =>
error.message.includes('503') ? 'socket_http_503' : 'unknown'
})
assert.deepEqual(result, { opened: 2, overflowReason: 'socket_http_503' })
assert.equal(heldFor, 4_000)
assert.equal(closes, 2)
})
test('closes successful probes when a boundary probe fails', async () => {
let closes = 0
const peers = [
peer(async () => probe(() => closes++)),
peer(async () => {
throw new Error('probe failed')
})
]
await assert.rejects(
proveRelayLoadRebindBoundary({
peers,
probeCount: 2,
holdMs: 0,
delay: async () => undefined,
failureReason: () => 'unknown'
}),
/probe failed/
)
assert.equal(closes, 1)
})
test('waits for every replacement socket to finish closing', async () => {
let finishClose
const closeFinished = new Promise((resolve) => {
finishClose = resolve
})
const closingProbe = probe()
closingProbe.close = () => closeFinished
let completed = false
const boundary = proveRelayLoadRebindBoundary({
peers: [peer(async () => closingProbe)],
probeCount: 1,
holdMs: 0,
delay: async () => undefined,
failureReason: () => 'unknown',
requireOverflow: false
}).then(() => {
completed = true
})
await new Promise((resolve) => setImmediate(resolve))
assert.equal(completed, false)
finishClose()
await boundary
assert.equal(completed, true)
})
test('can prove reserved replacement headroom below the physical cap', async () => {
let closes = 0
const result = await proveRelayLoadRebindBoundary({
peers: [peer(async () => probe(() => closes++))],
probeCount: 1,
holdMs: 0,
delay: async () => undefined,
failureReason: () => 'unknown',
requireOverflow: false
})
assert.deepEqual(result, { opened: 1, overflowReason: null })
assert.equal(closes, 1)
})
test('fails when a replacement closes before the hold completes', async () => {
let heldProbe
await assert.rejects(
proveRelayLoadRebindBoundary({
peers: [
peer(async () => {
heldProbe = probe()
return heldProbe
})
],
probeCount: 1,
holdMs: 4_000,
delay: async () => {
heldProbe.close()
},
failureReason: () => 'unknown',
requireOverflow: false
}),
/closed before the hold completed/
)
})
test('delays the boundary until every ordinary control has recovered', async () => {
let active = 899
await assert.rejects(
waitForRelayLoadRebindGate({
delay: async () => undefined,
delayMs: 0,
activeCount: () => active,
requiredCount: 900
}),
/requires 900 active controls/
)
await waitForRelayLoadRebindGate({
delay: async () => {
active = 900
},
delayMs: 1,
activeCount: () => active,
requiredCount: 900
})
})
@@ -0,0 +1,35 @@
const ASSIGNMENT_RETRY_DELAY_MS = 5_100
const waitPastAssignmentRateLimit = (schedule) =>
new Promise((resolve) => schedule(resolve, ASSIGNMENT_RETRY_DELAY_MS))
export async function proveRelayLoadRegionBehavior({
oldClientPeer,
stickyPeer,
asiaOrigin,
scheduleAssignmentRetry = setTimeout
}) {
try {
await oldClientPeer.connect()
if (!oldClientPeer.assignedCellUrl() || oldClientPeer.assignedCellUrl() === asiaOrigin) {
throw new Error('unhinted client did not use the US-first path')
}
} finally {
await oldClientPeer.shutdown()
}
try {
await stickyPeer.connect()
if (stickyPeer.assignedCellUrl() !== asiaOrigin) {
throw new Error('preferred Asia client did not reach the Asia cell')
}
await waitPastAssignmentRateLimit(scheduleAssignmentRetry)
const reassigned = await stickyPeer.requestAssignment('us-central1')
if (reassigned.cellUrl !== asiaOrigin) {
throw new Error('valid sticky assignment moved after preference changed')
}
} finally {
await stickyPeer.shutdown()
}
return { oldClientUsFirst: true, stickyAssignmentPreserved: true }
}
@@ -0,0 +1,47 @@
import assert from 'node:assert/strict'
import test from 'node:test'
import { proveRelayLoadRegionBehavior } from './relay-load-region-behavior.mjs'
function peer(origin, reassigned = origin) {
let shutdowns = 0
return {
connect: async () => undefined,
assignedCellUrl: () => origin,
requestAssignment: async () => ({ cellUrl: reassigned }),
shutdown: async () => { shutdowns++ },
shutdowns: () => shutdowns
}
}
test('proves unhinted US-first placement and sticky Asia preservation', async () => {
const oldClientPeer = peer('https://c3.relay-staging.onorca.dev')
const stickyPeer = peer('https://c4.relay-staging.onorca.dev')
let retryDelayMs = 0
assert.deepEqual(await proveRelayLoadRegionBehavior({
oldClientPeer,
stickyPeer,
asiaOrigin: 'https://c4.relay-staging.onorca.dev',
scheduleAssignmentRetry: (resolve, delayMs) => {
retryDelayMs = delayMs
resolve()
}
}), { oldClientUsFirst: true, stickyAssignmentPreserved: true })
assert.equal(retryDelayMs, 5_100)
assert.equal(oldClientPeer.shutdowns(), 1)
assert.equal(stickyPeer.shutdowns(), 1)
})
test('rejects Asia placement for an unhinted client or a moved sticky assignment', async () => {
await assert.rejects(proveRelayLoadRegionBehavior({
oldClientPeer: peer('https://c4.relay-staging.onorca.dev'),
stickyPeer: peer('https://c4.relay-staging.onorca.dev'),
asiaOrigin: 'https://c4.relay-staging.onorca.dev',
scheduleAssignmentRetry: (resolve) => resolve()
}), /US-first/)
await assert.rejects(proveRelayLoadRegionBehavior({
oldClientPeer: peer('https://c3.relay-staging.onorca.dev'),
stickyPeer: peer('https://c4.relay-staging.onorca.dev', 'https://c3.relay-staging.onorca.dev'),
asiaOrigin: 'https://c4.relay-staging.onorca.dev',
scheduleAssignmentRetry: (resolve) => resolve()
}), /sticky assignment moved/)
})
@@ -0,0 +1,43 @@
import { setTimeout as delayDefault } from 'node:timers/promises'
export async function openRelayLoadInviteOffers({
peers,
count,
ratePerSecond,
concurrency = 8,
delay = delayDefault,
now = Date.now
}) {
if (
!Array.isArray(peers) || peers.length === 0 ||
!Number.isSafeInteger(count) || count < 0 ||
!Number.isSafeInteger(ratePerSecond) || ratePerSecond < 1 || ratePerSecond > 20 ||
!Number.isSafeInteger(concurrency) || concurrency < 1
) throw new Error('invalid Relay invite-offer load')
let next = 0
let nextStartAt = now()
const workers = Array.from({ length: Math.min(concurrency, count) }, async () => {
for (;;) {
const index = next++
if (index >= count) return
const scheduledAt = Math.max(nextStartAt, now())
nextStartAt = scheduledAt + 1_000 / ratePerSecond
await delay(Math.max(0, scheduledAt - now()))
await peers[index % peers.length].openInviteOffer()
}
})
await Promise.all(workers)
return count
}
export async function proveRelayLoadRequestUnitBoundary(peer) {
try {
await peer.openInviteOffer()
} catch (error) {
if (error instanceof Error && error.message === 'invite offer failed: relay_capacity_exhausted') {
return 'relay_capacity_exhausted'
}
throw new Error('request-unit overflow was not rejected safely', { cause: error })
}
throw new Error('request-unit overflow unexpectedly succeeded')
}
@@ -0,0 +1,52 @@
import assert from 'node:assert/strict'
import test from 'node:test'
import {
openRelayLoadInviteOffers,
proveRelayLoadRequestUnitBoundary
} from './relay-load-request-unit-boundary.mjs'
test('distributes the exact invite count with bounded concurrency', async () => {
let active = 0
let peak = 0
const delays = []
const calls = [0, 0, 0]
const peers = calls.map((_, index) => ({
async openInviteOffer() {
calls[index]++
active++
peak = Math.max(peak, active)
await Promise.resolve()
active--
}
}))
assert.equal(await openRelayLoadInviteOffers({
peers,
count: 8,
ratePerSecond: 2,
concurrency: 2,
delay: async (milliseconds) => { delays.push(milliseconds) },
now: () => 0
}), 8)
assert.deepEqual(calls, [3, 3, 2])
assert.ok(peak <= 2)
assert.equal(delays.length, 8)
assert.ok(Math.max(...delays) >= 3_500)
})
test('accepts only the exact request-unit exhaustion error', async () => {
assert.equal(await proveRelayLoadRequestUnitBoundary({
openInviteOffer: async () => {
throw new Error('invite offer failed: relay_capacity_exhausted')
}
}), 'relay_capacity_exhausted')
await assert.rejects(
proveRelayLoadRequestUnitBoundary({
openInviteOffer: async () => { throw new Error('control response timeout') }
}),
/not rejected safely/
)
await assert.rejects(
proveRelayLoadRequestUnitBoundary({ openInviteOffer: async () => undefined }),
/unexpectedly succeeded/
)
})
@@ -0,0 +1,25 @@
export function assertRelayLoadRampAccepted(rampConnectionFailures, maximum) {
if (rampConnectionFailures > maximum) {
throw new Error('relay load ramp exceeded the allowed connection failures')
}
}
export async function runRelayLoadWithShutdown(operation, shutdown) {
try {
return await operation()
} finally {
await shutdown()
}
}
export function relayLoadRunHasDisallowedFailures(result, config) {
return (
result.rampConnectionFailures > config.maxRampConnectionFailures ||
(!config.allowPlannedTransitionRetries && result.transitionConnectionFailures > 0) ||
result.steadyConnectionFailures > 0 ||
result.unexpectedCloses > config.maxUnexpectedCloses ||
result.protocolErrors > 0 ||
result.refreshErrors > 0 ||
result.socketErrors > 0
)
}
@@ -0,0 +1,68 @@
import assert from 'node:assert/strict'
import test from 'node:test'
import {
assertRelayLoadRampAccepted,
relayLoadRunHasDisallowedFailures,
runRelayLoadWithShutdown
} from './relay-load-run-lifecycle.mjs'
test('always shuts down peers when a load phase fails', async () => {
const events = []
await assert.rejects(
runRelayLoadWithShutdown(
async () => {
events.push('run')
throw new Error('boundary failed')
},
async () => events.push('shutdown')
),
/boundary failed/
)
assert.deepEqual(events, ['run', 'shutdown'])
})
test('fails immediately when the strict ramp budget is exceeded', () => {
assert.doesNotThrow(() => assertRelayLoadRampAccepted(0, 0))
assert.throws(() => assertRelayLoadRampAccepted(1, 0), /ramp exceeded/)
})
test('rejects connection failures during the transition window', () => {
const result = {
rampConnectionFailures: 0,
transitionConnectionFailures: 1,
steadyConnectionFailures: 0,
unexpectedCloses: 0,
protocolErrors: 0,
refreshErrors: 0,
socketErrors: 0
}
assert.equal(
relayLoadRunHasDisallowedFailures(result, {
maxRampConnectionFailures: 0,
maxUnexpectedCloses: 0
}),
true
)
})
test('allows only explicitly planned transition retries', () => {
const result = {
rampConnectionFailures: 0,
transitionConnectionFailures: 1,
steadyConnectionFailures: 0,
unexpectedCloses: 0,
protocolErrors: 0,
refreshErrors: 0,
socketErrors: 0
}
const config = {
allowPlannedTransitionRetries: true,
maxRampConnectionFailures: 0,
maxUnexpectedCloses: 0
}
assert.equal(relayLoadRunHasDisallowedFailures(result, config), false)
assert.equal(
relayLoadRunHasDisallowedFailures({ ...result, steadyConnectionFailures: 1 }, config),
true
)
})
@@ -0,0 +1,355 @@
import { createHash } from 'node:crypto'
import { chmod, readFile, readdir, stat, writeFile } from 'node:fs/promises'
import { basename, join, resolve } from 'node:path'
import { pathToFileURL } from 'node:url'
const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{1,127}$/
const SHA = /^[a-f0-9]{40}$/
const JWT = /^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/
const EVIDENCE_MAX_AGE_MS = 5 * 60_000
// Matches the same-cap cell job timeout-minutes; bounds each predecessor wave.
const WAVE_PREDECESSOR_TIMEOUT_MS = 75 * 60_000
const WAVE_INDEX = /^[0-3]$/
const EVIDENCE_SAMPLE_INTERVAL_MS = 60_000
const EVIDENCE_MAX_LINEAGE_MS = 25 * 60_000
const MIGRATION_POLICIES = new Set([
'strict',
'recover-forward',
'capacity-transition'
])
const MUTATION_MODES = new Set([
'capacity-transition',
'continue-evacuation',
'disable-cell',
'enable-empty-cell',
'execute',
'fence-source',
'recover-forward',
'reset-empty-candidate'
])
function argumentsByName(argv) {
const values = {}
for (let index = 0; index < argv.length; index += 2) {
const name = argv[index]
const value = argv[index + 1]
if (!name?.startsWith('--') || !value || value.startsWith('--')) {
throw new Error('relay monitor evidence arguments are invalid')
}
values[name.slice(2)] = value
}
return values
}
async function sha256(path) {
return createHash('sha256').update(await readFile(path)).digest('hex')
}
async function regularFile(path) {
try {
return (await stat(path)).isFile()
} catch {
return false
}
}
function provenance(values) {
const runAttempt = Number(values['run-attempt'])
if (
!SAFE_ID.test(values['incident-id'] ?? '') ||
!SAFE_ID.test(values['run-id'] ?? '') ||
!Number.isSafeInteger(runAttempt) ||
runAttempt < 1 ||
!SHA.test(values['commit-sha'] ?? '') ||
!['dry-run', 'monitor'].includes(values.mode)
) {
throw new Error('relay monitor evidence provenance is invalid')
}
return {
incidentId: values['incident-id'],
runId: values['run-id'],
runAttempt,
commitSha: values['commit-sha'],
mode: values.mode
}
}
export async function createEvidenceManifest(argv) {
const values = argumentsByName(argv)
const directory = resolve(values.directory ?? '')
const expected = provenance(values)
const candidates = [
`${expected.incidentId}.state.json`,
`${expected.incidentId}.summaries.jsonl`,
`${expected.incidentId}.summary.md`
]
const files = {}
for (const name of candidates) {
const path = join(directory, name)
if (await regularFile(path)) files[name] = await sha256(path)
}
if (!files[`${expected.incidentId}.state.json`]) {
throw new Error('relay monitor durable state is missing')
}
const manifest = {
schemaVersion: 1,
...expected,
files
}
const path = join(directory, 'evidence-manifest.json')
await writeFile(path, `${JSON.stringify(manifest)}\n`, { mode: 0o600 })
await chmod(path, 0o600)
return manifest
}
async function readAndVerifyManifest(directory, expected) {
const manifest = JSON.parse(
await readFile(join(directory, 'evidence-manifest.json'), 'utf8')
)
if (
manifest.schemaVersion !== 1 ||
manifest.incidentId !== expected.incidentId ||
manifest.runId !== expected.runId ||
manifest.runAttempt !== expected.runAttempt ||
manifest.commitSha !== expected.commitSha ||
manifest.mode !== expected.mode
) {
throw new Error('relay monitor evidence provenance does not match')
}
const names = Object.keys(manifest.files ?? {})
if (!names.includes(`${expected.incidentId}.state.json`)) {
throw new Error('relay monitor evidence has no durable state')
}
for (const name of names) {
if (basename(name) !== name || !/^[A-Za-z0-9._-]+$/.test(name)) {
throw new Error('relay monitor evidence file name is invalid')
}
if (await sha256(join(directory, name)) !== manifest.files[name]) {
throw new Error('relay monitor evidence hash does not match')
}
}
const allowed = new Set([...names, 'evidence-manifest.json'])
const unexpected = (await readdir(directory)).filter((name) => !allowed.has(name))
if (unexpected.length > 0) throw new Error('relay monitor evidence has unexpected files')
return manifest
}
function validMigrationPolicyState(state) {
return (
(
state.migrationPolicy === 'strict' &&
state.recoverySourceCellId === null &&
state.capacityCellId === null
) ||
(
state.migrationPolicy === 'recover-forward' &&
state.capacityCellId === null &&
typeof state.recoverySourceCellId === 'string' &&
state.expectedSelector?.membership?.existingOnly?.includes(
state.recoverySourceCellId
)
) ||
(
state.migrationPolicy === 'capacity-transition' &&
state.recoverySourceCellId === null &&
typeof state.capacityCellId === 'string' &&
state.expectedSelector?.membership?.general?.includes(state.capacityCellId)
)
)
}
export async function verifyRestoredEvidence(argv) {
const values = argumentsByName(argv)
const directory = resolve(values.directory ?? '')
const expected = provenance(values)
await readAndVerifyManifest(directory, expected)
const state = JSON.parse(
await readFile(join(directory, `${expected.incidentId}.state.json`), 'utf8')
)
if (
state.schemaVersion !== 4 ||
state.incidentId !== expected.incidentId ||
state.environment !== 'production' ||
state.preDrainDryRun !== (expected.mode === 'dry-run') ||
!MIGRATION_POLICIES.has(state.migrationPolicy) ||
!validMigrationPolicyState(state)
) {
throw new Error('relay monitor restored state does not match provenance')
}
return state
}
function validCompletedDryRunState(state, expected, nowMs, maxAgeMs) {
const completedAt = Date.parse(state.completedAt)
const startedAt = Date.parse(state.startedAt)
const windowStartedAt = Date.parse(state.windowStartedAt)
const lastSampleAt = Date.parse(state.lastSampleAt)
const age = nowMs - completedAt
return (
state.schemaVersion === 4 &&
state.incidentId === expected.incidentId &&
state.environment === 'production' &&
state.preDrainDryRun === true &&
validMigrationPolicyState(state) &&
state.durationMinutes === 15 &&
state.intervalMs === EVIDENCE_SAMPLE_INTERVAL_MS &&
state.sampleCount >= 16 &&
state.frozenAt === null &&
Number.isFinite(startedAt) &&
completedAt - startedAt >= 0 &&
completedAt - startedAt <= EVIDENCE_MAX_LINEAGE_MS &&
Number.isFinite(windowStartedAt) &&
completedAt - windowStartedAt >= 15 * 60_000 &&
Number.isFinite(lastSampleAt) &&
lastSampleAt <= completedAt &&
completedAt - lastSampleAt <= state.intervalMs &&
Number.isFinite(completedAt) &&
age >= 0 &&
age <= maxAgeMs
)
}
export async function verifyDryRunAuthority(argv, now = Date.now) {
const values = argumentsByName(argv)
const directory = resolve(values.directory ?? '')
const expected = provenance(values)
if (expected.mode !== 'dry-run') throw new Error('relay mutation requires dry-run evidence')
const manifest = await readAndVerifyManifest(directory, expected)
const state = JSON.parse(
await readFile(join(directory, `${expected.incidentId}.state.json`), 'utf8')
)
const requiredMigrationPolicy = values['required-migration-policy']
// Later same-cap waves start after sequential predecessor cell rolls, so the
// freshness bound grows by one cell-job timeout per predecessor; single-use
// consumption, needs-chaining, and each wave's live preflight recheck keep
// holding the mutation to current health.
const waveIndex = values['wave-index'] ?? '0'
if (!WAVE_INDEX.test(waveIndex)) {
throw new Error('relay monitor wave index is invalid')
}
const maxAgeMs =
EVIDENCE_MAX_AGE_MS + Number(waveIndex) * WAVE_PREDECESSOR_TIMEOUT_MS
if (
!MIGRATION_POLICIES.has(requiredMigrationPolicy) ||
state.migrationPolicy !== requiredMigrationPolicy ||
!validCompletedDryRunState(state, expected, now(), maxAgeMs)
) {
throw new Error('relay monitor dry-run authority is incomplete or stale')
}
return { manifest, state }
}
function exactSelector(actual, expected) {
const membership = (selector) => {
if (
!selector?.membership ||
!['existingOnly', 'migrationOnly', 'general'].every((key) =>
Array.isArray(selector.membership[key])
)
) return null
const normalized = Object.fromEntries(
['existingOnly', 'migrationOnly', 'general'].map((key) => [
key,
[...selector.membership[key]].sort()
])
)
const all = Object.values(normalized).flat()
return new Set(all).size === all.length ? normalized : null
}
const actualMembership = membership(actual)
const expectedMembership = membership(expected)
return Boolean(
actualMembership &&
expectedMembership &&
actual?.generation === expected?.generation &&
JSON.stringify(actualMembership) === JSON.stringify(expectedMembership)
)
}
export async function verifyMutationEvidence(
argv,
environment = process.env,
fetchImpl = fetch,
now = Date.now
) {
const values = argumentsByName(argv)
const directory = resolve(values.directory ?? '')
const expected = provenance(values)
if (expected.mode !== 'dry-run') throw new Error('mutation requires dry-run evidence')
const manifest = await readAndVerifyManifest(directory, expected)
const state = JSON.parse(
await readFile(join(directory, `${expected.incidentId}.state.json`), 'utf8')
)
const mutationMode = values['mutation-mode']
if (!MUTATION_MODES.has(mutationMode)) {
throw new Error('relay monitor mutation mode is invalid')
}
const scopedRecoverySourceCellId =
values['scoped-recovery-source-cell-id']
const recoveryMutation = ['fence-source', 'recover-forward'].includes(mutationMode)
const scopedRecoveryMutation =
['execute', 'recover-forward'].includes(mutationMode) &&
Boolean(scopedRecoverySourceCellId)
if (scopedRecoverySourceCellId && !scopedRecoveryMutation) {
throw new Error('relay monitor scoped recovery evidence is invalid')
}
const requiredMigrationPolicy = mutationMode === 'capacity-transition'
? 'capacity-transition'
: recoveryMutation || scopedRecoveryMutation ? 'recover-forward' : 'strict'
if (state.migrationPolicy !== requiredMigrationPolicy) {
throw new Error('relay monitor migration policy does not match mutation')
}
const expectedRecoverySourceCellId = scopedRecoveryMutation
? scopedRecoverySourceCellId
: values['source-cell-id']
if (
(recoveryMutation || scopedRecoveryMutation) &&
state.recoverySourceCellId !== expectedRecoverySourceCellId
) {
throw new Error('relay monitor recovery source does not match mutation')
}
if (
mutationMode === 'capacity-transition' &&
state.capacityCellId !== values['source-cell-id']
) {
throw new Error('relay monitor capacity cell does not match mutation')
}
if (
!validCompletedDryRunState(state, expected, now(), EVIDENCE_MAX_AGE_MS)
) {
throw new Error('relay monitor dry-run evidence is incomplete or stale')
}
const token = environment.ORCA_RELAY_ADMIN_ID_TOKEN
const origin = values['director-origin']
if (!token || !JWT.test(token) || !origin?.startsWith('https://')) {
throw new Error('relay monitor live selector verification is unavailable')
}
const response = await fetchImpl(`${origin}/v1/admin/admission-selector/status`, {
method: 'POST',
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
body: JSON.stringify({ v: 1 }),
signal: AbortSignal.timeout(30_000)
})
if (!response.ok) throw new Error('relay monitor live selector verification failed')
const current = (await response.json()).selector
if (!exactSelector(current, state.expectedSelector)) {
throw new Error('relay admission selector changed after the dry run')
}
return { manifest, state }
}
async function main() {
const [command, ...argv] = process.argv.slice(2)
if (command === 'create') await createEvidenceManifest(argv)
else if (command === 'verify-restore') await verifyRestoredEvidence(argv)
else if (command === 'verify-authority') await verifyDryRunAuthority(argv)
else if (command === 'verify-mutation') await verifyMutationEvidence(argv)
else throw new Error('relay monitor evidence command is invalid')
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
main().catch((error) => {
console.error(error instanceof Error ? error.message : 'relay monitor evidence failed')
process.exitCode = 1
})
}
@@ -0,0 +1,515 @@
import assert from 'node:assert/strict'
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import test from 'node:test'
import { relayWorkflowPath, relayWorkflowUrl } from './relay-repository.mjs'
import {
createEvidenceManifest,
verifyDryRunAuthority,
verifyMutationEvidence,
verifyRestoredEvidence
} from './relay-monitor-evidence.mjs'
const now = Date.parse('2026-07-28T12:00:00.000Z')
const provenance = [
'--incident-id',
'relay-123',
'--run-id',
'123',
'--run-attempt',
'1',
'--commit-sha',
'a'.repeat(40),
'--mode',
'dry-run'
]
const selector = {
generation: 2,
membership: {
existingOnly: ['c1'],
migrationOnly: ['c2'],
general: ['c3']
}
}
async function evidenceDirectory(migrationPolicy = 'strict') {
const directory = await mkdtemp(join(tmpdir(), 'relay-monitor-evidence-'))
const state = {
schemaVersion: 4,
incidentId: 'relay-123',
environment: 'production',
preDrainDryRun: true,
migrationPolicy,
recoverySourceCellId: migrationPolicy === 'recover-forward' ? 'c1' : null,
capacityCellId: migrationPolicy === 'capacity-transition' ? 'c3' : null,
startedAt: new Date(now - 17 * 60_000).toISOString(),
durationMinutes: 15,
intervalMs: 60_000,
sampleCount: 16,
windowStartedAt: new Date(now - 16 * 60_000).toISOString(),
lastSampleAt: new Date(now - 60_007).toISOString(),
completedAt: new Date(now - 60_000).toISOString(),
frozenAt: null,
expectedSelector: selector
}
await writeFile(
join(directory, 'relay-123.state.json'),
`${JSON.stringify(state)}\n`
)
return directory
}
test('creates and verifies exact restart provenance and hashes', async () => {
const directory = await evidenceDirectory()
try {
await createEvidenceManifest(['--directory', directory, ...provenance])
await assert.doesNotReject(
verifyRestoredEvidence(['--directory', directory, ...provenance])
)
await writeFile(join(directory, 'relay-123.state.json'), '{}\n')
await assert.rejects(
verifyRestoredEvidence(['--directory', directory, ...provenance]),
/hash does not match/
)
} finally {
await rm(directory, { recursive: true, force: true })
}
})
test('requires fresh green evidence and rechecks the live selector', async () => {
const directory = await evidenceDirectory()
try {
await createEvidenceManifest(['--directory', directory, ...provenance])
const verifyAuthority = () => verifyDryRunAuthority(
[
'--directory',
directory,
...provenance,
'--required-migration-policy',
'strict'
],
() => now
)
await assert.doesNotReject(verifyAuthority())
const fetchImpl = async (_input, init) => {
assert.equal(
new Headers(init.headers).get('authorization'),
'Bearer aaa.bbb.ccc'
)
return Response.json({ selector })
}
await assert.doesNotReject(
verifyMutationEvidence(
[
'--directory',
directory,
...provenance,
'--mutation-mode',
'execute',
'--source-cell-id',
'c1',
'--director-origin',
'https://relay.example'
],
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' },
fetchImpl,
() => now
)
)
const statePath = join(directory, 'relay-123.state.json')
const state = JSON.parse(await readFile(statePath, 'utf8'))
state.completedAt = new Date(now - 300_001).toISOString()
await writeFile(statePath, `${JSON.stringify(state)}\n`)
await createEvidenceManifest(['--directory', directory, ...provenance])
await assert.rejects(
verifyAuthority(),
/authority is incomplete or stale/
)
await assert.rejects(
verifyMutationEvidence(
[
'--directory',
directory,
...provenance,
'--mutation-mode',
'execute',
'--source-cell-id',
'c1',
'--director-origin',
'https://relay.example'
],
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' },
fetchImpl,
() => now
),
/incomplete or stale/
)
state.completedAt = new Date(now - 60_000).toISOString()
state.lastSampleAt = new Date(now - 120_001).toISOString()
await writeFile(statePath, `${JSON.stringify(state)}\n`)
await createEvidenceManifest(['--directory', directory, ...provenance])
await assert.rejects(
verifyMutationEvidence(
[
'--directory',
directory,
...provenance,
'--mutation-mode',
'execute',
'--source-cell-id',
'c1',
'--director-origin',
'https://relay.example'
],
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' },
fetchImpl,
() => now
),
/incomplete or stale/
)
state.lastSampleAt = new Date(now - 60_007).toISOString()
state.startedAt = new Date(now - 26 * 60_000 - 1).toISOString()
await writeFile(statePath, `${JSON.stringify(state)}\n`)
await createEvidenceManifest(['--directory', directory, ...provenance])
await assert.rejects(
verifyAuthority(),
/authority is incomplete or stale/
)
await assert.rejects(
verifyMutationEvidence(
[
'--directory',
directory,
...provenance,
'--mutation-mode',
'execute',
'--source-cell-id',
'c1',
'--director-origin',
'https://relay.example'
],
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' },
fetchImpl,
() => now
),
/incomplete or stale/
)
} finally {
await rm(directory, { recursive: true, force: true })
}
})
test('later same-cap waves accept evidence aged by predecessor cell rolls', async () => {
const directory = await evidenceDirectory()
try {
const statePath = join(directory, 'relay-123.state.json')
const state = JSON.parse(await readFile(statePath, 'utf8'))
const authorityAt = (...waveArgs) => verifyDryRunAuthority(
[
'--directory',
directory,
...provenance,
'--required-migration-policy',
'strict',
...waveArgs.flatMap((waveIndex) => ['--wave-index', waveIndex])
],
() => now
)
const ageState = async (ageMs) => {
state.completedAt = new Date(now - ageMs).toISOString()
state.lastSampleAt = new Date(now - ageMs - 7).toISOString()
state.startedAt = new Date(now - ageMs - 17 * 60_000).toISOString()
state.windowStartedAt = new Date(now - ageMs - 16 * 60_000).toISOString()
await writeFile(statePath, `${JSON.stringify(state)}\n`)
await createEvidenceManifest(['--directory', directory, ...provenance])
}
// The wave-0 bound in isolation: exactly 5 minutes, flag or no flag.
await ageState(5 * 60_000)
await assert.doesNotReject(authorityAt())
await assert.doesNotReject(authorityAt('0'))
await ageState(5 * 60_000 + 1)
await assert.rejects(authorityAt(), /authority is incomplete or stale/)
await assert.rejects(authorityAt('0'), /authority is incomplete or stale/)
// One predecessor cell roll (~16 min) exceeds wave 0 but fits wave 1.
await ageState(17 * 60_000)
await assert.rejects(authorityAt('0'), /authority is incomplete or stale/)
await assert.doesNotReject(authorityAt('1'))
await assert.rejects(authorityAt('4'), /wave index is invalid/)
await assert.rejects(authorityAt('x'), /wave index is invalid/)
// Both edges of one predecessor job timeout: 5min + 75min exactly.
await ageState(80 * 60_000)
await assert.doesNotReject(authorityAt('1'))
await ageState(80 * 60_000 + 1)
await assert.rejects(authorityAt('1'), /authority is incomplete or stale/)
await assert.doesNotReject(authorityAt('2'))
// Wave 2 and wave 3 edges: 5min + 2 * 75min and 5min + 3 * 75min exactly.
await ageState(155 * 60_000)
await assert.doesNotReject(authorityAt('2'))
await ageState(155 * 60_000 + 1)
await assert.rejects(authorityAt('2'), /authority is incomplete or stale/)
await ageState(230 * 60_000)
await assert.doesNotReject(authorityAt('3'))
await ageState(230 * 60_000 + 1)
await assert.rejects(authorityAt('3'), /authority is incomplete or stale/)
} finally {
await rm(directory, { recursive: true, force: true })
}
})
test('binds migration policies to their exact mutations', async () => {
const strictDirectory = await evidenceDirectory()
const recoveryDirectory = await evidenceDirectory('recover-forward')
const capacityDirectory = await evidenceDirectory('capacity-transition')
const fetchImpl = async () => Response.json({ selector })
try {
await createEvidenceManifest(['--directory', strictDirectory, ...provenance])
await createEvidenceManifest(['--directory', recoveryDirectory, ...provenance])
await createEvidenceManifest(['--directory', capacityDirectory, ...provenance])
await assert.rejects(
verifyDryRunAuthority(
[
'--directory',
recoveryDirectory,
...provenance,
'--required-migration-policy',
'strict'
],
() => now
),
/authority is incomplete or stale/
)
const verify = (directory, mutationMode, sourceCellId = 'c1') => verifyMutationEvidence(
[
'--directory',
directory,
...provenance,
'--mutation-mode',
mutationMode,
'--source-cell-id',
sourceCellId,
'--director-origin',
'https://relay.example'
],
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' },
fetchImpl,
() => now
)
await assert.doesNotReject(verify(strictDirectory, 'execute'))
await assert.doesNotReject(
verify(capacityDirectory, 'capacity-transition', 'c3')
)
await assert.doesNotReject(verify(recoveryDirectory, 'recover-forward'))
await assert.doesNotReject(verify(recoveryDirectory, 'fence-source'))
await assert.doesNotReject(
verifyMutationEvidence(
[
'--directory',
recoveryDirectory,
...provenance,
'--mutation-mode',
'execute',
'--source-cell-id',
'c12',
'--scoped-recovery-source-cell-id',
'c1',
'--director-origin',
'https://relay.example'
],
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' },
fetchImpl,
() => now
)
)
await assert.doesNotReject(
verifyMutationEvidence(
[
'--directory',
recoveryDirectory,
...provenance,
'--mutation-mode',
'recover-forward',
'--source-cell-id',
'c12',
'--scoped-recovery-source-cell-id',
'c1',
'--director-origin',
'https://relay.example'
],
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' },
fetchImpl,
() => now
)
)
await assert.rejects(
verify(recoveryDirectory, 'execute'),
/migration policy does not match/
)
await assert.rejects(
verify(strictDirectory, 'recover-forward'),
/migration policy does not match/
)
await assert.rejects(
verify(strictDirectory, 'capacity-transition', 'c3'),
/migration policy does not match/
)
await assert.rejects(
verify(capacityDirectory, 'capacity-transition', 'c1'),
/capacity cell does not match/
)
await assert.rejects(
verifyMutationEvidence(
[
'--directory',
recoveryDirectory,
...provenance,
'--mutation-mode',
'recover-forward',
'--source-cell-id',
'c9',
'--director-origin',
'https://relay.example'
],
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' },
fetchImpl,
() => now
),
/recovery source does not match/
)
await assert.rejects(
verifyMutationEvidence(
[
'--directory',
recoveryDirectory,
...provenance,
'--mutation-mode',
'fence-source',
'--source-cell-id',
'c1',
'--scoped-recovery-source-cell-id',
'c1',
'--director-origin',
'https://relay.example'
],
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' },
fetchImpl,
() => now
),
/scoped recovery evidence is invalid/
)
await assert.rejects(
verifyMutationEvidence(
[
'--directory',
recoveryDirectory,
...provenance,
'--mutation-mode',
'execute',
'--source-cell-id',
'c12',
'--scoped-recovery-source-cell-id',
'c9',
'--director-origin',
'https://relay.example'
],
{ ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' },
fetchImpl,
() => now
),
/recovery source does not match/
)
} finally {
await rm(strictDirectory, { recursive: true, force: true })
await rm(recoveryDirectory, { recursive: true, force: true })
await rm(capacityDirectory, { recursive: true, force: true })
}
})
test('workflow reruns restore the prior attempt into one stable incident', async () => {
const workflow = await readFile(
relayWorkflowUrl('monitor-relay-production-job.yml'),
'utf8'
)
assert.match(workflow, /INCIDENT_ID: relay-\$\{\{ github\.run_id \}\}-\$\{\{ inputs\.mode \}\}/)
assert.doesNotMatch(workflow, /INCIDENT_ID:.*run_attempt/)
assert.match(workflow, /actions\/download-artifact@v4/)
assert.match(workflow, /verify-restore/)
assert.match(workflow, /RESTART_FLAG=--restart/)
assert.equal(workflow.match(/--capacity-cell-id/g)?.length, 3)
const dispatchWorkflow = await readFile(
relayWorkflowUrl('monitor-relay-production.yml'),
'utf8'
)
assert.match(dispatchWorkflow, /- capacity-transition/)
assert.match(dispatchWorkflow, /capacity-cell-id: \$\{\{ inputs\.capacity-cell-id \}\}/)
})
test('same-cap and rehome mutations require complete strict dry-run authority', async () => {
for (const name of [
'deploy-relay-production-same-cap-job.yml',
'operate-relay-production-rehome-job.yml'
]) {
const workflow = await readFile(
relayWorkflowUrl(name),
'utf8'
)
assert.match(workflow, /relay-monitor-evidence\.mjs verify-authority/)
assert.match(workflow, /--required-migration-policy strict/)
assert.doesNotMatch(workflow, /relay-monitor-evidence\.mjs verify-restore/)
}
})
test('production mutation workflows consume and live-recheck dry-run evidence', async () => {
for (const name of [
'deploy-relay-production.yml',
'deploy-relay-production-multi-target.yml'
]) {
const workflow = await readFile(
relayWorkflowUrl(name),
'utf8'
)
assert.match(workflow, /actions\/download-artifact@v4/)
assert.match(workflow, /verify-mutation/)
assert.match(workflow, /--mutation-mode "\$\{DEPLOY_MODE\}"/)
assert.match(workflow, /--source-cell-id "\$\{SOURCE_CELL_ID\}"/)
assert.match(workflow, /incident:relay-preflight/)
assert.match(workflow, /Reject previously consumed dry-run evidence/)
assert.match(workflow, /actions\/upload-artifact@v4/)
assert.match(workflow, /relay-monitor-consumed-/)
assert.match(workflow, /ORCA_RELAY_ADMIN_ID_TOKEN/)
assert.match(workflow, /github\.ref == 'refs\/heads\/main'/)
assert.ok(
workflow.indexOf('pnpm install --frozen-lockfile') <
workflow.indexOf('id: google-auth')
)
}
})
test('monitor and mutation workflows share the production Cloud SQL rollout lock', async () => {
for (const name of [
'monitor-relay-production.yml',
'deploy-relay-production.yml',
'deploy-relay-production-multi-target.yml',
'deploy-relay-production-capacity.yml'
]) {
const workflow = await readFile(
relayWorkflowUrl(name),
'utf8'
)
assert.match(workflow, /group: production-cloud-sql-rollout/)
}
})
test('monitor uses a reusable job so exact job_workflow_ref is present', async () => {
const wrapper = await readFile(
relayWorkflowUrl('monitor-relay-production.yml'),
'utf8'
)
assert.ok(wrapper.includes(`uses: ./${relayWorkflowPath('monitor-relay-production-job.yml')}`))
const job = await readFile(
relayWorkflowUrl('monitor-relay-production-job.yml'),
'utf8'
)
assert.match(job, /workflow_call:/)
assert.match(job, /environment: production/)
})
@@ -0,0 +1,172 @@
import { readFile, writeFile } from 'node:fs/promises'
import { pathToFileURL } from 'node:url'
import { PRODUCTION_CAPACITY_CELL_IDS } from './prepare-relay-production-capacity-canary.mjs'
const APPROVED_CELLS = new Set(PRODUCTION_CAPACITY_CELL_IDS)
function argumentsByName(argv, allowed) {
const values = {}
for (let index = 0; index < argv.length; index += 2) {
const argument = argv[index]
const value = argv[index + 1]
if (!argument?.startsWith('--') || !value || value.startsWith('--')) {
throw new Error('capacity wave arguments are invalid')
}
const name = argument.slice(2)
if (!allowed.has(name) || name in values) {
throw new Error(`capacity wave argument --${name} is invalid`)
}
values[name] = value
}
return values
}
export function parseCapacityWave(waveCellIds, confirmation) {
const cells = waveCellIds.split(',')
if (
cells.length < 2 ||
cells.length > 4 ||
cells.some((cell) => !APPROVED_CELLS.has(cell)) ||
new Set(cells).size !== cells.length ||
cells.join(',') !== waveCellIds
) {
throw new Error('capacity wave must contain two to four unique approved cells')
}
if (confirmation !== `RAISE_SELECTED_WAVE_TO_1000 ${waveCellIds}`) {
throw new Error('capacity wave confirmation does not match the selected cells')
}
return cells
}
function exactMembership(membership) {
const keys = ['existingOnly', 'migrationOnly', 'general']
if (!keys.every((key) => Array.isArray(membership?.[key]))) return false
const cells = keys.flatMap((key) => membership[key])
return cells.every((cell) => typeof cell === 'string') && new Set(cells).size === cells.length
}
export function capacityWavePreflightState(state, waveCellIds, waveIndex, targetCellId) {
const cells = parseCapacityWave(
waveCellIds,
`RAISE_SELECTED_WAVE_TO_1000 ${waveCellIds}`
)
const index = Number(waveIndex)
const membership = state.expectedSelector?.membership
if (
!Number.isSafeInteger(index) ||
index < 0 ||
index >= cells.length ||
targetCellId !== cells[index] ||
state.schemaVersion !== 4 ||
state.environment !== 'production' ||
state.preDrainDryRun !== true ||
state.migrationPolicy !== 'capacity-transition' ||
state.recoverySourceCellId !== null ||
state.capacityCellId !== cells[0] ||
!Number.isSafeInteger(state.expectedSelector?.generation) ||
!exactMembership(membership) ||
!cells.every((cell) => membership.general.includes(cell)) ||
state.sampleCount < 16 ||
state.frozenAt !== null ||
typeof state.completedAt !== 'string'
) {
throw new Error('capacity wave evidence does not match this step')
}
return {
schemaVersion: 4,
environment: 'production',
expectedSelector: {
generation: state.expectedSelector.generation + index * 2,
membership
},
migrationPolicy: 'capacity-transition',
recoverySourceCellId: null,
capacityCellId: targetCellId
}
}
export function capacityWaveResumePreflightState(state, waveCellIds, targetCellId) {
const cells = parseCapacityWave(
waveCellIds,
`RAISE_SELECTED_WAVE_TO_1000 ${waveCellIds}`
)
const index = cells.indexOf(targetCellId)
const base = capacityWavePreflightState(
state,
waveCellIds,
String(index),
targetCellId
)
const membership = base.expectedSelector.membership
const general = membership.general.filter((cell) => cell !== targetCellId)
const capacityCellId = general.includes(state.capacityCellId)
? state.capacityCellId
: cells.find((cell) => general.includes(cell))
if (!capacityCellId) throw new Error('capacity wave resume has no general evidence cell')
return {
...base,
expectedSelector: {
generation: base.expectedSelector.generation + 1,
membership: {
existingOnly: membership.existingOnly,
migrationOnly: [...membership.migrationOnly, targetCellId].sort(),
general
}
},
capacityCellId
}
}
async function main(argv) {
const [command, ...arguments_] = argv
if (command === 'validate') {
const values = argumentsByName(
arguments_,
new Set(['wave-cell-ids', 'confirmation'])
)
process.stdout.write(`${JSON.stringify(parseCapacityWave(
values['wave-cell-ids'] ?? '',
values.confirmation ?? ''
))}\n`)
return
}
if (command === 'build-preflight' || command === 'build-resume-preflight') {
const values = argumentsByName(
arguments_,
new Set([
'state-file',
'wave-cell-ids',
...(command === 'build-preflight' ? ['wave-index'] : []),
'target-cell-id',
'output-file'
])
)
const state = JSON.parse(await readFile(values['state-file'] ?? '', 'utf8'))
const preflight = command === 'build-preflight'
? capacityWavePreflightState(
state,
values['wave-cell-ids'] ?? '',
values['wave-index'] ?? '',
values['target-cell-id'] ?? ''
)
: capacityWaveResumePreflightState(
state,
values['wave-cell-ids'] ?? '',
values['target-cell-id'] ?? ''
)
await writeFile(
values['output-file'] ?? '',
`${JSON.stringify(preflight)}\n`,
{ mode: 0o600 }
)
return
}
throw new Error('capacity wave command is invalid')
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
main(process.argv.slice(2)).catch((error) => {
console.error(error instanceof Error ? error.message : 'capacity wave failed')
process.exitCode = 1
})
}
@@ -0,0 +1,129 @@
import assert from 'node:assert/strict'
import test from 'node:test'
import {
capacityWavePreflightState,
capacityWaveResumePreflightState,
parseCapacityWave
} from './relay-production-capacity-wave.mjs'
const wave = [
'production-gce-c22',
'production-gce-c21',
'production-gce-c20',
'production-gce-c19'
]
function evidence(overrides = {}) {
return {
schemaVersion: 4,
environment: 'production',
preDrainDryRun: true,
migrationPolicy: 'capacity-transition',
recoverySourceCellId: null,
capacityCellId: wave[0],
expectedSelector: {
generation: 39,
membership: {
existingOnly: ['production-gce-c1'],
migrationOnly: ['production-gce-c17', 'production-gce-c18'],
general: [...wave, 'production-gce-c23']
}
},
sampleCount: 16,
frozenAt: null,
completedAt: '2026-08-11T21:00:00.000Z',
...overrides
}
}
test('accepts only exact confirmed waves of two to four approved cells', () => {
for (const cells of [wave.slice(0, 2), wave]) {
const value = cells.join(',')
assert.deepEqual(
parseCapacityWave(value, `RAISE_SELECTED_WAVE_TO_1000 ${value}`),
cells
)
}
for (const [cells, confirmation] of [
[[wave[0]], `RAISE_SELECTED_WAVE_TO_1000 ${wave[0]}`],
[[...wave, 'production-gce-c16'], `RAISE_SELECTED_WAVE_TO_1000 ${wave.join(',')}`],
[[wave[0], wave[0]], `RAISE_SELECTED_WAVE_TO_1000 ${wave[0]},${wave[0]}`],
[[wave[0], 'production-gce-c17'], `RAISE_SELECTED_WAVE_TO_1000 ${wave[0]},production-gce-c17`],
[[wave[0], ` ${wave[1]}`], `RAISE_SELECTED_WAVE_TO_1000 ${wave[0]}, ${wave[1]}`],
[wave, 'RAISE_SELECTED_WAVE_TO_1000 production-gce-c22']
]) {
assert.throws(() => parseCapacityWave(cells.join(','), confirmation))
}
})
test('derives each continuation preflight from the sealed selector generation', () => {
for (const [index, cell] of wave.entries()) {
const state = capacityWavePreflightState(
evidence(),
wave.join(','),
String(index),
cell
)
assert.equal(state.expectedSelector.generation, 39 + index * 2)
assert.equal(state.capacityCellId, cell)
assert.deepEqual(state.expectedSelector.membership, evidence().expectedSelector.membership)
}
})
test('derives an exact isolated-cell resume state from sealed wave evidence', () => {
const state = capacityWaveResumePreflightState(
evidence(),
wave.join(','),
wave[3]
)
assert.equal(state.expectedSelector.generation, 46)
assert.equal(state.capacityCellId, wave[0])
assert.deepEqual(state.expectedSelector.membership, {
existingOnly: ['production-gce-c1'],
migrationOnly: ['production-gce-c17', 'production-gce-c18', wave[3]].sort(),
general: [wave[0], wave[1], wave[2], 'production-gce-c23']
})
})
test('resume rejects a target outside the exact sealed wave', () => {
assert.throws(
() => capacityWaveResumePreflightState(
evidence(),
wave.join(','),
'production-gce-c16'
),
/does not match/
)
})
test('resume rebinds first-cell evidence to another general wave cell', () => {
const state = capacityWaveResumePreflightState(
evidence(),
wave.join(','),
wave[0]
)
assert.equal(state.capacityCellId, wave[1])
assert.equal(state.expectedSelector.generation, 40)
assert.ok(state.expectedSelector.membership.migrationOnly.includes(wave[0]))
assert.ok(state.expectedSelector.membership.general.includes(wave[1]))
})
test('rejects reordered, incomplete, frozen, or mismatched wave evidence', () => {
const calls = [
() => capacityWavePreflightState(evidence(), wave.join(','), '1', wave[0]),
() => capacityWavePreflightState(evidence({ capacityCellId: wave[1] }), wave.join(','), '0', wave[0]),
() => capacityWavePreflightState(evidence({ sampleCount: 15 }), wave.join(','), '0', wave[0]),
() => capacityWavePreflightState(evidence({ frozenAt: '2026-08-11T20:59:00.000Z' }), wave.join(','), '0', wave[0]),
() => capacityWavePreflightState(evidence({ completedAt: null }), wave.join(','), '0', wave[0]),
() => capacityWavePreflightState(evidence({
expectedSelector: {
...evidence().expectedSelector,
membership: {
...evidence().expectedSelector.membership,
general: wave.slice(1)
}
}
}), wave.join(','), '0', wave[0])
]
for (const call of calls) assert.throws(call, /does not match/)
})
@@ -0,0 +1,440 @@
import assert from 'node:assert/strict'
import { spawnSync } from 'node:child_process'
import { readFileSync } from 'node:fs'
import test from 'node:test'
import { readRelayWorkflow, relayWorkflowPath } from './relay-repository.mjs'
import { PRODUCTION_CAPACITY_CELL_IDS } from './prepare-relay-production-capacity-canary.mjs'
const dispatchWorkflow = readRelayWorkflow('deploy-relay-production-capacity.yml')
const workflow = readRelayWorkflow('deploy-relay-production-capacity-job.yml')
const terraform = source('infra/terraform/relay-github-actions.tf')
const production = source('infra/terraform/environments/production.tfvars')
const capacityCells = PRODUCTION_CAPACITY_CELL_IDS
function source(path) {
return readFileSync(new URL(`../../${path}`, import.meta.url), 'utf8')
}
function resource(type, name) {
const start = terraform.indexOf(`resource "${type}" "${name}"`)
assert.notEqual(start, -1, `${type}.${name} is missing`)
const next = terraform.indexOf('\nresource "', start + 1)
return terraform.slice(start, next === -1 ? undefined : next)
}
function ordered(...markers) {
let previous = -1
for (const marker of markers) {
const current = workflow.indexOf(marker)
assert.ok(current > previous, `${marker} is missing or out of order`)
previous = current
}
}
function mutationConfirmation(mode, targetCellId, confirmation) {
const stepStart = workflow.indexOf(' - name: Require exact mutation confirmation')
const runMarker = ' run: |\n'
const runStart = workflow.indexOf(runMarker, stepStart) + runMarker.length
const runEnd = workflow.indexOf('\n - name:', runStart)
const script = workflow.slice(runStart, runEnd).replace(/^ {10}/gm, '')
return spawnSync('bash', ['-euo', 'pipefail', '-c', script], {
env: {
...process.env,
DEPLOY_MODE: mode,
TARGET_CELL_ID: targetCellId,
CONFIRMATION: confirmation
}
}).status
}
function imageCompatibility(activeImage, desiredImage) {
const start = workflow.indexOf(' ACTIVE_IMAGE_DIGEST="${ACTIVE_IMAGE##*@}"')
const end = workflow.indexOf('\n CURRENT_CELLS_JSON=', start)
const script = workflow.slice(start, end).replace(/^ {10}/gm, '')
return spawnSync('bash', ['-euo', 'pipefail', '-c', script], {
env: {
...process.env,
ACTIVE_IMAGE: activeImage,
DESIRED_IMAGE: desiredImage,
DESIRED_IMAGE_DIGEST: desiredImage.split('@').at(-1),
COMPATIBLE_DIRECTOR_IMAGE_DIGEST: 'sha256:01b7fc3e6dce66180034f268a2dc92c05458706c5b3a0dc4450dcdd6161f6e73',
COMPATIBLE_CELL_IMAGE_DIGEST: 'sha256:c77ec7aef565009fdb645b0989806859bfa40a7aa14e4a57ab55ac92fee6c34f'
}
}).status
}
test('production capacity mutation is restricted to the exact serving rollout set', () => {
assert.match(workflow, /TARGET_CELL_ID: \$\{\{ inputs\.target-cell-id \}\}/)
const targetInput = dispatchWorkflow.slice(
dispatchWorkflow.indexOf(' target-cell-id:'),
dispatchWorkflow.indexOf(' wave-cell-ids:')
)
assert.deepEqual(
[...targetInput.matchAll(/^\s+- (production-gce-c\d+)$/gm)].map((match) => match[1]),
capacityCells
)
assert.match(workflow, new RegExp(`CAPACITY_CELL_IDS: ${capacityCells.join(',')}`))
for (const cellId of capacityCells) {
assert.match(dispatchWorkflow, new RegExp(`^\\s+- ${cellId}$`, 'm'))
}
assert.match(workflow, /CELL_ORIGIN="https:\/\/\$\{TARGET_HOSTNAME\}\.relay\.onorca\.dev"/)
assert.match(workflow, /echo "TARGET_HOSTNAME=\$\{TARGET_HOSTNAME\}"/)
assert.match(workflow, /\} >> "\$\{GITHUB_ENV\}"/)
assert.match(workflow, /RAISE_SELECTED_CELL_TO_1000/)
assert.match(workflow, /ROLL_BACK_SELECTED_CELL_TO_600/)
assert.match(workflow, /TARGET_HARD_CAP=600[\s\S]*?else[\s\S]*?TARGET_HARD_CAP=1000/)
})
test('rollback confirmation is bound to the exact selected cell', () => {
assert.equal(
mutationConfirmation('apply', 'production-gce-c25', 'RAISE_SELECTED_CELL_TO_1000'),
0
)
assert.equal(
mutationConfirmation(
'rollback',
'production-gce-c25',
'ROLL_BACK_SELECTED_CELL_TO_600 production-gce-c25'
),
0
)
assert.notEqual(
mutationConfirmation('rollback', 'production-gce-c25', 'ROLL_BACK_SELECTED_CELL_TO_600'),
0
)
assert.notEqual(
mutationConfirmation(
'rollback',
'production-gce-c25',
'ROLL_BACK_SELECTED_CELL_TO_600 production-gce-c26'
),
0
)
})
test('production configuration selects only serving cells for 1,000 and the compatible image', () => {
const cell = (cellId) => production.slice(
production.indexOf(`"${cellId}"`),
production.indexOf('\n }', production.indexOf(`"${cellId}"`))
)
for (const cellId of capacityCells) {
assert.match(cell(cellId), /connection_hard_cap\s+= 1000/)
assert.match(
cell(cellId),
/sha256:5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563/
)
}
for (const cellId of ['production-gce-c17', 'production-gce-c18']) {
assert.match(cell(cellId), /connection_hard_cap\s+= 600/)
assert.doesNotMatch(cell(cellId), /connection_hard_cap\s+= 1000/)
assert.match(cell(cellId), /sha256:0e83408b0dc08531f1e8182019dc151afc38d63ddde4ad5cc01e40247ef3681d/)
}
assert.equal(production.match(/connection_hard_cap\s+= 1000/g)?.length, capacityCells.length)
// Asia cells legitimately share this digest, so scope the uniqueness check to the capacity set.
assert.equal(
new Set(capacityCells.map((cellId) => cell(cellId).match(/sha256:[0-9a-f]{64}/)[0])).size,
1
)
assert.match(
workflow,
/COMPATIBLE_DIRECTOR_IMAGE_DIGEST: sha256:01b7fc3e6dce66180034f268a2dc92c05458706c5b3a0dc4450dcdd6161f6e73/
)
assert.match(
workflow,
/test "\$\{ACTIVE_IMAGE_DIGEST\}" = "\$\{COMPATIBLE_DIRECTOR_IMAGE_DIGEST\}"/
)
assert.match(
workflow,
/test "\$\{DESIRED_IMAGE_DIGEST\}" = "\$\{COMPATIBLE_CELL_IMAGE_DIGEST\}"/
)
assert.match(workflow, /\.\[\$cell\]\.connection_hard_cap = \$cap/)
assert.match(workflow, /baseCells:\$baseCells/)
assert.match(workflow, /capacityCellIds:\(\$capacityCellIds \| split\(","\)\)/)
assert.match(
workflow,
/Verify current selected-cell capacity[\s\S]*?TOPOLOGY_PHASE.*predecessor[\s\S]*?CURRENT_CAP=600[\s\S]*?DESIRED_IMAGE_DIGEST.*PREDECESSOR_IMAGE_DIGEST/
)
})
test('director and cell image compatibility is an exact reviewed pair', () => {
const repository = 'us-central1-docker.pkg.dev/onorca-cloud/orca-cloud/relay@'
const director = `${repository}sha256:01b7fc3e6dce66180034f268a2dc92c05458706c5b3a0dc4450dcdd6161f6e73`
const cell = `${repository}sha256:c77ec7aef565009fdb645b0989806859bfa40a7aa14e4a57ab55ac92fee6c34f`
const other = `${repository}sha256:${'a'.repeat(64)}`
assert.equal(imageCompatibility(cell, cell), 0)
assert.equal(imageCompatibility(director, cell), 0)
assert.notEqual(imageCompatibility(director, other), 0)
assert.notEqual(imageCompatibility(other, cell), 0)
})
// The matching check on google_project_service.required lives with the foundation root, which
// stays in the private repository.
test('apply consumes fresh evidence before arming mutation cleanup', () => {
for (const marker of [
'Require fresh dry-run evidence reference',
'Verify dry-run artifact before cloud authentication',
'Reject previously consumed dry-run evidence',
'Verify fresh dry-run evidence against the live selector',
'Recheck every live safety signal',
'Publish the consumed-evidence marker'
]) {
assert.match(workflow, new RegExp(marker))
}
assert.match(workflow, /--mutation-mode capacity-transition/)
assert.match(workflow, /--source-cell-id "\$\{TARGET_CELL_ID\}"/)
ordered(
'Publish the consumed-evidence marker',
'Arm fail-closed mutation cleanup',
'Reversibly isolate only the selected cell',
'Deploy only the reviewed director topology',
'Plan and apply only the empty selected cell',
'Restore only the selected cell to general admission',
'Verify the live general selected cell'
)
})
test('wave apply consumes one proof and runs fail-closed cells sequentially', () => {
assert.match(dispatchWorkflow, /- wave-apply/)
assert.match(dispatchWorkflow, /group: production-cloud-sql-rollout/)
assert.match(dispatchWorkflow, /Validate the exact wave request/)
assert.match(dispatchWorkflow, /Verify wave evidence against the live selector/)
assert.match(dispatchWorkflow, /Require exact 600\/60 predecessor wave cells/)
assert.match(
dispatchWorkflow,
/COMPATIBLE_CELL_IMAGE_DIGEST: sha256:c77ec7aef565009fdb645b0989806859bfa40a7aa14e4a57ab55ac92fee6c34f/
)
assert.match(
dispatchWorkflow,
/Require exact 600\/60 predecessor wave cells[\s\S]*?--expected-image-digests \\\n\s+"\$\{PREDECESSOR_IMAGE_DIGEST\},\$\{COMPATIBLE_CELL_IMAGE_DIGEST\}"/
)
assert.match(dispatchWorkflow, /Publish the consumed-evidence marker/)
assert.match(
dispatchWorkflow,
/OUTPUT_DIRECTORY: \$\{\{ github\.workspace \}\}\/relay-monitor-evidence/
)
assert.match(
dispatchWorkflow,
/path: \$\{\{ github\.workspace \}\}\/relay-monitor-evidence/
)
assert.doesNotMatch(dispatchWorkflow, /strategy:/)
for (const [index, dependency] of [
[1, 'wave_gate'],
[2, 'wave_cell_1'],
[3, 'wave_cell_2'],
[4, 'wave_cell_3']
]) {
const start = dispatchWorkflow.indexOf(` wave_cell_${index}:`)
const end = dispatchWorkflow.indexOf(`\n wave_cell_${index + 1}:`, start)
const job = dispatchWorkflow.slice(start, end === -1 ? undefined : end)
assert.match(job, new RegExp(`needs: (?:\\[wave_gate, )?${dependency}`))
assert.match(job, /evidence-mode: continuation/)
assert.match(job, new RegExp(`wave-index: '${index - 1}'`))
}
assert.match(workflow, /Download this workflow's wave authority/)
assert.match(workflow, /run-id: \$\{\{ github\.run_id \}\}/)
assert.match(workflow, /relay-production-capacity-wave\.mjs build-preflight/)
assert.match(workflow, /Require the exact wave predecessor topology/)
assert.match(workflow, /test "\$\{TOPOLOGY_PHASE\}" = predecessor/)
assert.match(workflow, /Recheck exact wave state and every live safety signal/)
assert.match(workflow, /if test "\$\{WAVE_INDEX\}" != 0; then RETRY_ARGS=\(--retry-freshness\); fi/)
assert.equal(workflow.match(/--retry-freshness/g)?.length, 2)
ordered(
'Recheck exact wave state and every live safety signal',
'Arm fail-closed mutation cleanup',
'Reversibly isolate only the selected cell',
'Verify the live general selected cell'
)
})
test('Terraform mutation targets only the selected cell and has fail-closed recovery', () => {
assert.match(
workflow,
/google_compute_instance_template\.relay_gce_cell\[\\"\$\{TARGET_CELL_ID\}\\"\]/
)
assert.match(
workflow,
/google_compute_instance_group_manager\.relay_gce_cell\[\\"\$\{TARGET_CELL_ID\}\\"\]/
)
assert.doesNotMatch(workflow, /relay_gce_cell\["production-gce-c26"\]/)
assert.doesNotMatch(workflow, /target=google_cloud_run_v2_service\.relay/)
assert.match(workflow, /validate-relay-capacity-plan\.mjs/)
assert.match(workflow, /--mode bootstrap-cell/)
assert.match(workflow, /--capacity-service-account "\$\{CAPACITY_SERVICE_ACCOUNT\}"/)
assert.match(workflow, /failure\(\) && inputs\.mode != 'verify'/)
assert.match(workflow, /test "\$\{MUTATION_STARTED:-false\}" = true \|\| exit 0/)
assert.match(workflow, /--mode isolate/)
assert.doesNotMatch(workflow, /rolling-action restart/)
assert.equal(workflow.match(/manage_artifact_dns=false/g)?.length, 5)
assert.match(workflow, /OFFLINE_ROLLBACK=true/)
assert.match(workflow, /--runtime unavailable/)
assert.equal(workflow.match(/--expected-image-digests/g)?.length, 7)
assert.match(workflow, /PREDECESSOR_IMAGE_DIGEST: sha256:0e83408b/)
assert.match(workflow, /classify-relay-production-capacity-director\.mjs/)
assert.match(workflow, /CURRENT_CAPACITY_SERVICE_ACCOUNT_JSON/)
assert.match(workflow, /if test "\$\{DIRECTOR_READY\}" = true; then exit 0; fi/)
assert.match(
workflow,
/Keep the selected cell isolated after a failed mutation[\s\S]*?--mode isolate[\s\S]*?--mode drain/
)
const cleanup = workflow.slice(workflow.indexOf('id: cleanup-auth'))
assert.match(cleanup, /Keep the selected cell isolated after a failed mutation/)
assert.match(cleanup, /steps\.cleanup-auth\.outputs\.id_token/)
assert.doesNotMatch(cleanup, /steps\.deploy-auth\.outputs\.id_token/)
ordered(
'Reversibly isolate only the selected cell',
'Drain the selected cell or prove an offline rollback',
'id: restart-auth-one',
'Require restart-safe selected-cell activity',
'id: restart-auth-two',
'Require extended restart-safe selected-cell activity',
'Deploy only the reviewed director topology',
'id: director-transition-auth',
'Require fail-closed director transition',
'id: capacity-auth',
'Plan and apply only the empty selected cell',
'id: capacity-transition-auth',
'Verify fresh exact selected-cell heartbeat before admission'
)
const restartGate = workflow.slice(
workflow.indexOf('id: restart-auth-one'),
workflow.indexOf('Deploy only the reviewed director topology')
)
assert.equal(restartGate.match(/--timeout-ms 450000/g)?.length, 2)
assert.match(restartGate, /steps\.restart-auth-one\.outputs\.id_token/)
assert.match(restartGate, /steps\.restart-auth-two\.outputs\.id_token/)
assert.match(restartGate, /capacity transition verification timed out:/)
assert.match(workflow, /timeout-minutes: 75/)
})
test('wave resume is bound to the failed run and exact isolated selector state', () => {
assert.match(dispatchWorkflow, /- wave-resume/)
assert.match(dispatchWorkflow, /evidence-mode: resume/)
assert.match(dispatchWorkflow, /source-wave-run-id: \$\{\{ inputs\.source-wave-run-id \}\}/)
assert.match(workflow, /Download the failed wave authority for resume/)
assert.match(workflow, /test "\$\{SOURCE_SHA\}" = "\$\{MONITOR_SHA\}"/)
assert.match(workflow, /test "\$\{SOURCE_ATTEMPT\}" = "\$\{EXPECTED_SOURCE_ATTEMPT\}"/)
for (const boundary of [
'31554591366:31555510376:production-gce-c16,production-gce-c15,production-gce-c14,production-gce-c13:production-gce-c13',
'31562760783:31563664692:production-gce-c10,production-gce-c9,production-gce-c8,production-gce-c7:production-gce-c10',
'31571019947:31572080665:production-gce-c9,production-gce-c8,production-gce-c7:production-gce-c8',
'a917e8e1fc1a2654e8cb81ba39b57733ec56be9c',
'6082e9ca89a918ca51f0c87db003f5e8805b64b7',
'e59958130c9d9b7a6cd805df2678d08997842c7c',
'EXPECTED_SOURCE_ATTEMPT=2'
]) {
assert.match(workflow, new RegExp(boundary))
}
assert.match(workflow, /\*\) exit 1 ;;/)
assert.match(workflow, /test "\$\{MONITOR_SHA\}" = "\$\{EXPECTED_SHA\}"/)
assert.match(workflow, /\.head_branch == "main"/)
assert.match(workflow, /\.head_repository\.full_name == env\.GITHUB_REPOSITORY/)
assert.ok(workflow.includes(`.path == "${relayWorkflowPath('monitor-relay-production.yml')}"`))
assert.ok(workflow.includes(`.path == "${relayWorkflowPath('deploy-relay-production-capacity.yml')}"`))
assert.match(workflow, /build-resume-preflight/)
assert.match(workflow, /RESUME_SELECTED_CELL_TO_1000 \$\{TARGET_CELL_ID\}/)
assert.match(
workflow,
/Recheck exact isolated resume state[\s\S]*?--hard-cap 600[\s\S]*?--admission migration-only[\s\S]*?--draining required/
)
})
test('GCE capacity identity is exact-workflow and narrowly permissioned', () => {
const provider = resource(
'google_iam_workload_identity_pool_provider',
'github_production_relay_capacity'
)
assert.match(provider, /concat\(local\.relay_github_leading_repository_claims, \[/)
for (const boundary of [
"assertion.ref == 'refs/heads/main'",
"assertion.environment == 'production'",
'local.relay_github_workflow_conditions["github_production_relay_capacity"]'
]) {
assert.match(provider, new RegExp(boundary.replaceAll(/[.*+?^${}()|[\]\\]/g, '\\$&')))
}
// The workflow pair itself is pinned in the clause the provider renders, once per accepted
// repository, and each repository supplies its own workflow-ref head.
for (const boundary of [
"assertion.workflow_ref == '${prefix}${local.github_production_relay_capacity_workflow_file}@refs/heads/main'",
"assertion.job_workflow_ref == '${prefix}${local.github_production_relay_capacity_job_workflow_file}@refs/heads/main'"
]) {
assert.match(terraform, new RegExp(boundary.replaceAll(/[.*+?^${}()|[\]\\]/g, '\\$&')))
}
const role = resource(
'google_project_iam_custom_role',
'github_production_relay_capacity_mutation'
)
assert.match(role, /compute\.instanceGroupManagers\.update/)
assert.match(role, /compute\.instanceTemplates\.create/)
assert.doesNotMatch(
role,
/compute\.(?:disks\.delete|instances\.(?:delete|start|stop|update))|cloudsql|secretmanager/
)
const state = resource(
'google_storage_bucket_iam_member',
'github_production_relay_capacity_state'
)
assert.match(state, /objects\/terraform\/state\/default\.tfstate/)
assert.match(state, /objects\/terraform\/state\/default\.tflock/)
})
test('deploy and capacity identities are used in their intended phases', () => {
const jobStart = workflow.indexOf(' capacity:')
const stepsStart = workflow.indexOf(' steps:', jobStart)
const jobHeader = workflow.slice(jobStart, stepsStart)
assert.deepEqual(jobHeader.match(/^\s+if:.*$/gm), [
" if: ${{ github.ref == 'refs/heads/main' }}"
])
const configurationStart = workflow.indexOf('Require production workflow configuration')
const configurationEnd = workflow.indexOf('- uses: actions/checkout@v4', configurationStart)
assert.ok(configurationStart >= 0)
assert.ok(configurationEnd > configurationStart)
const configurationStep = workflow.slice(configurationStart, configurationEnd)
for (const name of [
'GCP_REGION',
'DEPLOY_WORKLOAD_IDENTITY_PROVIDER',
'DEPLOY_SERVICE_ACCOUNT',
'CAPACITY_WORKLOAD_IDENTITY_PROVIDER',
'CAPACITY_SERVICE_ACCOUNT'
]) {
assert.match(configurationStep, new RegExp(`test -n "\\$\\{${name}\\}"`))
}
ordered('Require production workflow configuration', 'id: deploy-auth')
ordered('id: deploy-auth', 'Reversibly isolate only the selected cell', 'id: capacity-auth')
assert.match(workflow, /steps\.deploy-auth\.outputs\.id_token/)
assert.doesNotMatch(workflow, /steps\.capacity-auth\.outputs\.id_token/)
assert.equal(
workflow.match(/steps\.capacity-transition-auth\.outputs\.id_token/g)?.length,
3
)
assert.match(workflow, /PRODUCTION_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER/)
assert.match(workflow, /PRODUCTION_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT/)
assert.match(workflow, /read-relay-production-capacity-identity\.mjs/)
assert.match(workflow, /\(\.directorReady \| type\) == "boolean"/)
assert.match(workflow, /\(\.directorReady \| tostring\)/)
})
test('director readiness extraction preserves only JSON booleans', {
skip: spawnSync('jq', ['--version']).status !== 0
}, () => {
const filter = `if (.directorReady | type) == "boolean" then
(.directorReady | tostring)
else error("invalid directorReady classification") end`
const extract = (input) => spawnSync('jq', ['-er', filter], {
encoding: 'utf8',
input: JSON.stringify(input)
})
for (const value of [true, false]) {
const result = extract({ directorReady: value })
assert.equal(result.status, 0)
assert.equal(result.stdout.trim(), String(value))
}
for (const input of [
{ directorReady: 'true' },
{ directorReady: 'false' },
{ directorReady: null },
{}
]) {
assert.notEqual(extract(input).status, 0)
}
})
@@ -0,0 +1,169 @@
import assert from 'node:assert/strict'
import { readFile } from 'node:fs/promises'
import test from 'node:test'
import { readRelayWorkflow, relayWorkflowFile } from './relay-repository.mjs'
import { readWorkflow, workflowFiles } from './cloud-sql-rollout-lock-census.mjs'
async function source(path) {
return await readFile(new URL(`../../${path}`, import.meta.url), 'utf8')
}
// Why: the shared deploy identity is relay-owned in production and moves with the relay
// extraction, so it needs a name the public repo can carry without touching the app pair. The
// generic names are retired; a workflow that still reads them would silently resolve to nothing.
test('no workflow names the retired generic production deploy identity', async () => {
const files = workflowFiles()
assert.ok(files.length > 20)
const relayReaders = []
for (const file of files) {
const workflow = readWorkflow(file)
assert.doesNotMatch(workflow, /PRODUCTION_GCP_WORKLOAD_IDENTITY_PROVIDER\b/, file)
assert.doesNotMatch(workflow, /PRODUCTION_GCP_DEPLOY_SERVICE_ACCOUNT\b/, file)
if (/PRODUCTION_GCP_RELAY_DEPLOY_/.test(workflow)) relayReaders.push(file)
}
assert.deepEqual(relayReaders.sort(), [
'deploy-relay-fence-broker.yml',
'deploy-relay-production-capacity-job.yml',
'deploy-relay-production-capacity.yml',
'deploy-relay-production-director.yml',
'deploy-relay-production-multi-target.yml',
'deploy-relay-production-same-cap-job.yml',
'deploy-relay-production-same-cap.yml',
'deploy-relay-production.yml',
'operate-relay-asia-admission.yml',
'operate-relay-production-rehome-job.yml',
'publish-relay-production.yml'
].map((name) => relayWorkflowFile(name)).sort())
})
test('monitor workflow has no shared deploy identity fallback', async () => {
const workflow = readRelayWorkflow('monitor-relay-production-job.yml')
assert.match(workflow, /PRODUCTION_GCP_RELAY_MONITOR_WORKLOAD_IDENTITY_PROVIDER/)
assert.match(workflow, /PRODUCTION_GCP_RELAY_MONITOR_SERVICE_ACCOUNT/)
assert.doesNotMatch(workflow, /PRODUCTION_GCP_DEPLOY_SERVICE_ACCOUNT/)
assert.doesNotMatch(workflow, /PRODUCTION_GCP_WORKLOAD_IDENTITY_PROVIDER/)
})
test('relay fencing uses the dedicated requester and private broker', async () => {
const workflow = readRelayWorkflow('deploy-relay-production-multi-target.yml')
assert.match(workflow, /Reject direct-runner Terraform fence aborts/)
assert.match(workflow, /inputs\.mode == 'fence-source'/)
assert.match(workflow, /inputs\.mode == 'abort-fence-source'/)
assert.match(workflow, /inputs\.mode == 'supersede-target'/)
assert.match(workflow, /PRODUCTION_GCP_RELAY_FENCE_WORKLOAD_IDENTITY_PROVIDER/)
assert.match(workflow, /PRODUCTION_GCP_RELAY_FENCE_SERVICE_ACCOUNT/)
assert.match(workflow, /PRODUCTION_GCP_RELAY_FENCE_BROKER_URI/)
assert.match(workflow, /Invoke private target-supersession broker/)
assert.match(workflow, /Invoke private source-fence broker/)
assert.match(workflow, /Require exact broker cell contract/)
assert.match(workflow, /Require exact source-fence broker contract/)
assert.match(workflow, /Require private fence-broker environment/)
assert.match(
workflow,
/DEPLOY_MODE\}" = "execute" \|\|\s+"\$\{DEPLOY_MODE\}" = "recover-forward"\) &&\s+"\$\{SOURCE_CELL_ID\}" = "production-gce-c12"/
)
assert.match(
workflow,
/--scoped-recovery-source-cell-id\s+production-gce-c3/
)
assert.match(
workflow,
/test "\$\{FAILED_TARGET_CELL_ID\}" = "production-gce-c12"/
)
assert.match(
workflow,
/test "\$\{REPLACEMENT_TARGET_CELL_ID\}" = "production-gce-c13"/
)
assert.match(
workflow,
/test "\$\{TARGET_CELL_IDS\}" = "production-gce-c12,production-gce-c13"/
)
assert.match(
workflow,
/test "\$\{TARGET_CELL_IDS\}" = "production-gce-c7,production-gce-c8,production-gce-c10,production-gce-c13,production-gce-c17,production-gce-c18"/
)
const jobGate = workflow.slice(
workflow.indexOf('jobs:'),
workflow.indexOf('runs-on:')
)
assert.doesNotMatch(jobGate, /PRODUCTION_GCP_RELAY_FENCE_/)
const brokerStep = workflow.slice(
workflow.indexOf('- name: Invoke private target-supersession broker'),
workflow.indexOf('- name: Preflight or run multi-target evacuation')
)
assert.match(brokerStep, /steps\.google-fence-broker-auth\.outputs\.id_token/)
assert.doesNotMatch(brokerStep, /PRODUCTION_GCP_DEPLOY_SERVICE_ACCOUNT/)
const sourceFenceStep = workflow.slice(
workflow.indexOf('- name: Invoke private source-fence broker'),
workflow.indexOf('- name: Preflight or run multi-target evacuation')
)
assert.match(sourceFenceStep, /steps\.google-fence-broker-auth\.outputs\.id_token/)
assert.match(sourceFenceStep, /\/v1\/fence-source/)
assert.doesNotMatch(sourceFenceStep, /PRODUCTION_GCP_DEPLOY_SERVICE_ACCOUNT/)
})
test('Terraform binds dedicated identities to exact OIDC and resource boundaries', async () => {
const terraform = await source('infra/terraform/relay-github-actions.tf')
for (const claim of ['job_workflow_ref', 'workflow_ref', 'ref', 'environment']) {
assert.match(terraform, new RegExp(`assertion\\.${claim}`))
}
assert.match(terraform, /github_monitor_workflow_file/)
assert.match(terraform, /github_fence_workflow_file/)
assert.match(terraform, /github_production_relay_capacity_job_workflow_file/)
assert.match(terraform, /google_service_account" "github_monitor"/)
assert.match(terraform, /google_service_account" "github_fence"/)
assert.match(terraform, /google_service_account\.github_monitor\[0\]\.member/)
assert.match(terraform, /service_account_id = google_service_account\.github_fence\[0\]\.name/)
assert.match(terraform, /attribute\.relay_ops_identity\/monitor/)
assert.match(terraform, /attribute\.relay_ops_identity\/fence/)
assert.doesNotMatch(terraform, /github_relay_fence_operator/)
assert.doesNotMatch(terraform, /github_terraform_fence_state_writer/)
const broker = await source('infra/terraform/relay-fence-broker.tf')
assert.match(broker, /max_instance_request_concurrency = 1/)
assert.match(broker, /max_instance_count = 1/)
assert.match(broker, /roles\/run\.invoker/)
assert.match(broker, /google_service_account\.github_fence\[0\]\.member/)
assert.doesNotMatch(broker, /allUsers/)
const brokerDeploy = readRelayWorkflow('deploy-relay-fence-broker.yml')
assert.match(brokerDeploy, /sha-\$\{GITHUB_SHA\}/)
assert.match(brokerDeploy, /gcloud run services update/)
assert.match(brokerDeploy, /\.status\.traffic/)
assert.doesNotMatch(brokerDeploy, /latestReadyRevisionName/)
assert.doesNotMatch(brokerDeploy, /--set-env-vars/)
})
test('Terraform exposes the audited production environment values', async () => {
const outputs = await source('infra/terraform/outputs.tf')
for (const output of [
'github_relay_monitor_workload_identity_provider',
'github_relay_monitor_service_account',
'github_relay_fence_workload_identity_provider',
'github_relay_fence_service_account'
]) {
assert.match(outputs, new RegExp(`output "${output}"`))
}
})
test('production mutations pass the minted admin token to live preflight', async () => {
const workflow = readRelayWorkflow('deploy-relay-production.yml')
const recheck = workflow.slice(
workflow.indexOf('- name: Recheck all live safety signals'),
workflow.indexOf('- name: Create single-use dry-run marker')
)
assert.match(
recheck,
/ORCA_RELAY_ADMIN_ID_TOKEN: \$\{\{ steps\.google-auth\.outputs\.id_token \}\}/
)
const multiTarget = readRelayWorkflow('deploy-relay-production-multi-target.yml')
const multiTargetRecheck = multiTarget.slice(
multiTarget.indexOf('- name: Recheck all live safety signals'),
multiTarget.indexOf('- name: Create single-use dry-run marker')
)
assert.match(multiTargetRecheck, /steps\.google-auth\.outputs\.id_token/)
assert.match(multiTargetRecheck, /inputs\.mode != 'supersede-target'/)
})
test('fence broker pins the production-proven Terraform planner', async () => {
const dockerfile = await source('apps/relay-fence-broker/Dockerfile')
assert.match(dockerfile, /FROM hashicorp\/terraform:1\.15\.8 AS terraform/)
})
@@ -0,0 +1,161 @@
import { readFileSync } from 'node:fs'
import { pathToFileURL } from 'node:url'
export const SAME_CAP_CELLS = [
'production-gce-c7', 'production-gce-c8', 'production-gce-c9', 'production-gce-c10',
'production-gce-c13', 'production-gce-c14', 'production-gce-c15', 'production-gce-c16',
'production-gce-c19', 'production-gce-c20', 'production-gce-c21', 'production-gce-c22',
'production-gce-c23', 'production-gce-c24', 'production-gce-c25', 'production-gce-c26',
'production-gce-c27', 'production-gce-c28', 'production-gce-c29'
]
function digest(value, name) {
if (!/^sha256:[a-f0-9]{64}$/.test(value ?? '')) throw new Error(`${name} is invalid`)
return value
}
function cells(value) {
const parsed = value.split(',').map((cell) => cell.trim()).filter(Boolean)
if (
parsed.length < 1 ||
parsed.length > 4 ||
new Set(parsed).size !== parsed.length ||
parsed.some((cell) => !SAME_CAP_CELLS.includes(cell))
) throw new Error('same-cap wave cells are invalid')
return parsed
}
export function validateSameCapWave(input) {
if (!['verify', 'canary-apply', 'batch-apply', 'rollback'].includes(input.mode)) {
throw new Error('same-cap wave mode is invalid')
}
const selected = cells(input.cellIds)
const targetDigest = digest(input.targetDigest, 'target digest')
const rollbackDigest = digest(input.rollbackDigest, 'rollback digest')
if (targetDigest === rollbackDigest) throw new Error('target and rollback digests must differ')
if (input.mode === 'canary-apply' && selected.length !== 1) {
throw new Error('canary mode requires exactly one cell')
}
if (input.mode === 'batch-apply' && (selected.length < 2 || selected.length > 4)) {
throw new Error('batch mode requires two to four cells')
}
// Later waves expect the selector to advance by exactly 2 per predecessor,
// which a resumed rollback cell (isolate skipped, +1) violates.
if (input.mode === 'rollback' && selected.length !== 1) {
throw new Error('rollback mode requires exactly one cell')
}
const mutation = input.mode !== 'verify'
const expectedConfirmation = input.mode === 'rollback'
? `ROLL_BACK_RELAY_SAME_CAP ${rollbackDigest} ${selected.join(',')}`
: `ROLL_RELAY_SAME_CAP ${targetDigest} ${selected.join(',')}`
if (mutation && input.confirmation !== expectedConfirmation) {
throw new Error('same-cap confirmation does not match the exact digest and cells')
}
if (!mutation && input.confirmation) throw new Error('verify does not accept confirmation')
if (input.mode === 'batch-apply' && !/^[1-9][0-9]*$/.test(input.canaryRunId ?? '')) {
throw new Error('batch mode requires a canary run ID')
}
if (input.mode !== 'batch-apply' && input.canaryRunId) {
throw new Error('only batch mode accepts a canary run ID')
}
return { cells: selected, targetDigest, rollbackDigest }
}
export function canaryAuthority(input) {
const wave = validateSameCapWave({ ...input, mode: 'canary-apply', canaryRunId: '' })
if (!/^[0-9a-f]{40}$/.test(input.commitSha ?? '')) throw new Error('commit SHA is invalid')
if (!/^[1-9][0-9]*$/.test(input.runId ?? '')) throw new Error('run ID is invalid')
const selectorGeneration = Number(input.selectorGeneration)
const rehomeGeneration = Number(input.rehomeGeneration)
if (!Number.isSafeInteger(selectorGeneration) || selectorGeneration < 0) {
throw new Error('selector generation is invalid')
}
if (!Number.isSafeInteger(rehomeGeneration) || rehomeGeneration < 0) {
throw new Error('rehome generation is invalid')
}
return {
v: 1,
commitSha: input.commitSha,
runId: input.runId,
cellId: wave.cells[0],
targetDigest: wave.targetDigest,
rollbackDigest: wave.rollbackDigest,
selectorGeneration: selectorGeneration + 2,
rehomeGeneration
}
}
export function verifyCanaryAuthority(authority, expected) {
if (
authority?.v !== 1 ||
authority.commitSha !== expected.commitSha ||
authority.runId !== expected.runId ||
authority.targetDigest !== expected.targetDigest ||
authority.rollbackDigest !== expected.rollbackDigest ||
authority.selectorGeneration !== Number(expected.selectorGeneration) ||
authority.rehomeGeneration !== Number(expected.rehomeGeneration) ||
!SAME_CAP_CELLS.includes(authority.cellId)
) throw new Error('canary authority does not match this batch')
return authority
}
function values(argv) {
const result = {}
for (let index = 0; index < argv.length; index += 2) {
if (!argv[index]?.startsWith('--') || argv[index + 1] === undefined) {
throw new Error('invalid arguments')
}
result[argv[index].slice(2)] = argv[index + 1]
}
return result
}
export function main(argv = process.argv.slice(2)) {
const command = argv.shift()
const input = values(argv)
if (command === 'validate') {
const wave = validateSameCapWave({
mode: input.mode,
cellIds: input['cell-ids'],
targetDigest: input['target-digest'],
rollbackDigest: input['rollback-digest'],
confirmation: input.confirmation,
canaryRunId: input['canary-run-id']
})
process.stdout.write(`${JSON.stringify(wave.cells)}\n`)
return
}
if (command === 'create-canary') {
process.stdout.write(`${JSON.stringify(canaryAuthority({
mode: 'canary-apply',
cellIds: input['cell-id'],
targetDigest: input['target-digest'],
rollbackDigest: input['rollback-digest'],
confirmation: input.confirmation,
commitSha: input['commit-sha'],
runId: input['run-id'],
selectorGeneration: input['selector-generation'],
rehomeGeneration: input['rehome-generation']
}))}\n`)
return
}
if (command === 'verify-canary') {
verifyCanaryAuthority(JSON.parse(readFileSync(input.file, 'utf8')), {
commitSha: input['commit-sha'],
runId: input['run-id'],
targetDigest: input['target-digest'],
rollbackDigest: input['rollback-digest'],
selectorGeneration: input['selector-generation'],
rehomeGeneration: input['rehome-generation']
})
return
}
throw new Error('unknown same-cap wave command')
}
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
try { main() } catch (error) {
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
process.exitCode = 1
}
}
@@ -0,0 +1,106 @@
import assert from 'node:assert/strict'
import { test } from 'node:test'
import {
canaryAuthority,
validateSameCapWave,
verifyCanaryAuthority
} from './relay-production-same-cap-wave.mjs'
const targetDigest = `sha256:${'a'.repeat(64)}`
const rollbackDigest = `sha256:${'b'.repeat(64)}`
test('requires one canary or a bounded reviewed batch', () => {
assert.deepEqual(validateSameCapWave({
mode: 'canary-apply',
cellIds: 'production-gce-c7',
targetDigest,
rollbackDigest,
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7`
}).cells, ['production-gce-c7'])
assert.throws(() => validateSameCapWave({
mode: 'canary-apply',
cellIds: 'production-gce-c7,production-gce-c8',
targetDigest,
rollbackDigest,
confirmation: 'wrong'
}), /canary/)
assert.deepEqual(validateSameCapWave({
mode: 'batch-apply',
cellIds: 'production-gce-c8,production-gce-c9',
targetDigest,
rollbackDigest,
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c8,production-gce-c9`,
canaryRunId: '42'
}).cells, ['production-gce-c8', 'production-gce-c9'])
assert.deepEqual(validateSameCapWave({
mode: 'canary-apply',
cellIds: 'production-gce-c28',
targetDigest,
rollbackDigest,
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c28`
}).cells, ['production-gce-c28'])
assert.throws(() => validateSameCapWave({
mode: 'canary-apply',
cellIds: 'production-gce-c30',
targetDigest,
rollbackDigest,
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c30`
}), /cells/)
})
test('binds rollback confirmation to the exact digest and ordered cells', () => {
assert.throws(() => validateSameCapWave({
mode: 'rollback',
cellIds: 'production-gce-c7',
targetDigest,
rollbackDigest,
confirmation: `ROLL_BACK_RELAY_SAME_CAP ${targetDigest} production-gce-c7`
}), /confirmation/)
})
test('rollback rolls exactly one cell so later waves stay unreachable', () => {
const cellIds = 'production-gce-c7,production-gce-c8'
assert.throws(() => validateSameCapWave({
mode: 'rollback',
cellIds,
targetDigest,
rollbackDigest,
confirmation: `ROLL_BACK_RELAY_SAME_CAP ${rollbackDigest} ${cellIds}`
}), /rollback mode requires exactly one cell/)
assert.deepEqual(validateSameCapWave({
mode: 'rollback',
cellIds: 'production-gce-c7',
targetDigest,
rollbackDigest,
confirmation: `ROLL_BACK_RELAY_SAME_CAP ${rollbackDigest} production-gce-c7`
}).cells, ['production-gce-c7'])
})
test('seals and verifies canary authority for later batches', () => {
const authority = canaryAuthority({
cellIds: 'production-gce-c7',
targetDigest,
rollbackDigest,
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} production-gce-c7`,
commitSha: 'c'.repeat(40),
runId: '42',
selectorGeneration: '11',
rehomeGeneration: '4'
})
assert.equal(verifyCanaryAuthority(authority, {
commitSha: 'c'.repeat(40),
runId: '42',
targetDigest,
rollbackDigest,
selectorGeneration: '13',
rehomeGeneration: '4'
}).cellId, 'production-gce-c7')
assert.throws(() => verifyCanaryAuthority(authority, {
commitSha: 'd'.repeat(40),
runId: '42',
targetDigest,
rollbackDigest,
selectorGeneration: '11',
rehomeGeneration: '4'
}), /does not match/)
})
@@ -0,0 +1,82 @@
import assert from 'node:assert/strict'
import test from 'node:test'
import {
isEntrypoint,
jobIf,
jobNeeds,
jobs,
readWorkflow,
workflowFiles
} from './cloud-sql-rollout-lock-census.mjs'
import { relayWorkflowFile } from './relay-repository.mjs'
// Why: this repository publishes the relay's operate surface next to the desktop app. Three
// invariants make that safe, and each of them is one careless edit away from being lost.
const OPERATIONS_GATE = "vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true'"
// Cloud Verify is the only cloud workflow that must run on every pull request.
const UNGATED = relayWorkflowFile('verify.yml')
const relayWorkflows = () => workflowFiles().filter((file) => file !== UNGATED)
test('the copy carries every relay workflow', () => {
assert.equal(relayWorkflows().length, 24)
})
// Why: workflow_run chains match by display name, not filename. Renaming a file is safe; renaming
// one of these silently breaks the recovery chain with no failing run to notice.
test('the recovery chain keeps the display names it is matched by', () => {
const names = Object.fromEntries(
['prove-relay-staging-capacity.yml', 'recover-relay-staging-c4-image.yml', 'requeue-relay-staging-c4-recovery.yml'].map(
(name) => [name, /^name: (.+)$/m.exec(readWorkflow(relayWorkflowFile(name)))?.[1]]
)
)
assert.deepEqual(names, {
'prove-relay-staging-capacity.yml': 'Prove Relay Staging Capacity',
'recover-relay-staging-c4-image.yml': 'Recover Relay Staging C4 Image',
'requeue-relay-staging-c4-recovery.yml': 'Requeue Relay Staging C4 Recovery'
})
const recover = readWorkflow(relayWorkflowFile('recover-relay-staging-c4-image.yml'))
const requeue = readWorkflow(relayWorkflowFile('requeue-relay-staging-c4-recovery.yml'))
assert.ok(recover.includes(`workflows: [${names['prove-relay-staging-capacity.yml']}]`))
assert.ok(requeue.includes(`workflows: [${names['recover-relay-staging-c4-image.yml']}]`))
})
// Why: this repository holds none of the GCP credentials these workflows would need. Every one
// authenticates through Workload Identity read from a variable, so any repository secret other
// than the automatic token would be a credential the owner has to store here.
test('no cloud workflow reads a repository secret', () => {
for (const file of workflowFiles()) {
for (const [, name] of readWorkflow(file).matchAll(/secrets\.([A-Za-z_][A-Za-z0-9_]*)/g)) {
assert.equal(name, 'GITHUB_TOKEN', `${file} reads secrets.${name}`)
}
}
})
// Why: the operations gate is what makes the whole surface inert until the owner enables it. A
// job that can start without a gated dependency would run the moment someone dispatches it.
test('every job that can start on its own is gated on the operations variable', () => {
const reachable = []
for (const file of relayWorkflows()) {
const text = readWorkflow(file)
if (!isEntrypoint(text)) continue
for (const job of jobs(text)) {
if (jobNeeds(job.text).length > 0) continue
reachable.push(`${file}:${job.id}`)
assert.ok(jobIf(job.text).includes(OPERATIONS_GATE), `${file}:${job.id} is not gated`)
}
}
assert.ok(reachable.length >= 20, `only ${reachable.length} root jobs were checked`)
})
// Why: reusable jobs inherit the caller's gate. Gating them again would be dead configuration
// that reads as protection, and every caller is already checked above.
test('reusable workflows carry no gate of their own', () => {
for (const file of relayWorkflows()) {
const text = readWorkflow(file)
if (isEntrypoint(text)) continue
for (const job of jobs(text)) {
assert.ok(!jobIf(job.text).includes(OPERATIONS_GATE), `${file}:${job.id} regates a reusable job`)
}
}
})
@@ -0,0 +1,517 @@
const EXPECTED_KEYS = {
report: ['schemaVersion', 'environment', 'load', 'outcomes'],
environment: [
'projectId',
'directorOrigin',
'databaseVcpu',
'databasePoolMax',
'publicConcurrentMax',
'resolvePrioritySlots',
'directorMinInstances',
'directorMaxInstances',
'cloudRunConcurrency',
'rolloutOldPublicConcurrentMax',
'rolloutOldResolvePrioritySlots',
'rolloutNewPublicConcurrentMax',
'rolloutNewResolvePrioritySlots'
],
load: [
'drainingDesktops',
'backgroundRequestsPerMinute',
'backgroundAssignmentRequestsPerMinute',
'backgroundAssignment503PerMinute',
'targetConnectionCap',
'targetCells'
],
targetCell: ['cellId', 'peakConnections', 'recoveredControls'],
outcomes: [
'migrationExpirations',
'migrationAborts',
'transactionRetryExhaustions',
'keyProvenTargetRegistrations',
'oldestMigrationLeaseRemainingAtDrainMs',
'targetRegistrationDurationMs',
'assignmentSuccessesPerMinuteBaseline',
'assignmentSuccessesPerMinuteMinimum',
'eligibleResolveRequests',
'resolve2xx',
'resolveOverload',
'readinessChecks',
'readinessFailures',
'maintenanceOperations',
'maintenanceFailures',
'directorPeakInstances',
'rolloutOverlapPeakInstances',
'rolloutOverlapPeakPublicOperations',
'rolloutOverlapEligibleResolveRequests',
'rolloutOverlapResolve2xx',
'rolloutOverlapResolveOverload',
'rolloutOverlapReadinessFailures',
'rolloutOverlapPoolWaitP95Ms',
'rolloutOverlapDatabaseCpuPercentMax',
'poolWaitP95Ms',
'poolWaitMaxMs',
'databaseCpuPercentP95',
'databaseCpuPercentMax',
'recoveryDurationMs'
]
}
const LIMITS = {
drainingDesktopsMin: 760,
drainingDesktopsMax: 840,
backgroundRequestsPerMinuteMin: 10_450,
backgroundRequestsPerMinuteMax: 11_550,
backgroundAssignment503PerMinuteMin: 8_500,
backgroundAssignment503PerMinuteMax: 10_500,
targetCellCount: 2,
targetConnectionCap: 600,
databaseVcpu: 2,
databasePoolMax: 3,
publicConcurrentMax: 2,
resolvePrioritySlots: 1,
directorMinInstances: 1,
directorMaxInstances: 2,
directorPeakInstances: 2,
rolloutOverlapPeakInstances: 4,
rolloutOverlapPeakPublicOperations: 8,
cloudRunConcurrency: 80,
rolloutOldPublicConcurrentMax: 2,
rolloutOldResolvePrioritySlots: 0,
rolloutNewPublicConcurrentMax: 2,
rolloutNewResolvePrioritySlots: 1,
assignmentThroughputRetentionMin: 0.9,
resolveSuccessRateMin: 0.95,
resolveOverloadRateMaxExclusive: 0.01,
poolWaitP95MsMaxExclusive: 500,
poolWaitMaxMsMaxExclusive: 5_000,
databaseCpuPercentP95MaxExclusive: 70,
databaseCpuPercentMaxMaxExclusive: 85,
oldestMigrationLeaseRemainingAtDrainMsMin: 10 * 60_000,
targetRegistrationDurationMsMax: 5 * 60_000,
recoveryDurationMsMax: 14 * 60_000
}
export function evaluateRecoveryWaveReport(input) {
const report = parseReport(input)
const recoveredControls = report.load.targetCells.reduce(
(total, cell) => total + cell.recoveredControls,
0
)
const peakTargetConnections = Math.max(
...report.load.targetCells.map((cell) => cell.peakConnections)
)
const assignmentThroughputRetention = ratio(
report.outcomes.assignmentSuccessesPerMinuteMinimum,
report.outcomes.assignmentSuccessesPerMinuteBaseline
)
const resolveSuccessRate = ratio(
report.outcomes.resolve2xx,
report.outcomes.eligibleResolveRequests
)
const resolveOverloadRate = ratio(
report.outcomes.resolveOverload,
report.outcomes.eligibleResolveRequests
)
const rolloutOverlapResolveSuccessRate = ratio(
report.outcomes.rolloutOverlapResolve2xx,
report.outcomes.rolloutOverlapEligibleResolveRequests
)
const rolloutOverlapResolveOverloadRate = ratio(
report.outcomes.rolloutOverlapResolveOverload,
report.outcomes.rolloutOverlapEligibleResolveRequests
)
const thresholds = [
equal('database_vcpu', report.environment.databaseVcpu, LIMITS.databaseVcpu),
equal('database_pool_max', report.environment.databasePoolMax, LIMITS.databasePoolMax),
equal(
'public_concurrent_max',
report.environment.publicConcurrentMax,
LIMITS.publicConcurrentMax
),
equal(
'resolve_priority_slots',
report.environment.resolvePrioritySlots,
LIMITS.resolvePrioritySlots
),
equal(
'director_min_instances',
report.environment.directorMinInstances,
LIMITS.directorMinInstances
),
equal(
'director_max_instances',
report.environment.directorMaxInstances,
LIMITS.directorMaxInstances
),
equal(
'cloud_run_concurrency',
report.environment.cloudRunConcurrency,
LIMITS.cloudRunConcurrency
),
equal(
'rollout_old_public_concurrent_max',
report.environment.rolloutOldPublicConcurrentMax,
LIMITS.rolloutOldPublicConcurrentMax
),
equal(
'rollout_old_resolve_priority_slots',
report.environment.rolloutOldResolvePrioritySlots,
LIMITS.rolloutOldResolvePrioritySlots
),
equal(
'rollout_new_public_concurrent_max',
report.environment.rolloutNewPublicConcurrentMax,
LIMITS.rolloutNewPublicConcurrentMax
),
equal(
'rollout_new_resolve_priority_slots',
report.environment.rolloutNewResolvePrioritySlots,
LIMITS.rolloutNewResolvePrioritySlots
),
between(
'draining_desktops',
report.load.drainingDesktops,
LIMITS.drainingDesktopsMin,
LIMITS.drainingDesktopsMax
),
between(
'background_requests_per_minute',
report.load.backgroundRequestsPerMinute,
LIMITS.backgroundRequestsPerMinuteMin,
LIMITS.backgroundRequestsPerMinuteMax
),
between(
'background_assignment_requests_per_minute',
report.load.backgroundAssignmentRequestsPerMinute,
LIMITS.backgroundRequestsPerMinuteMin,
LIMITS.backgroundRequestsPerMinuteMax
),
between(
'background_assignment_503_per_minute',
report.load.backgroundAssignment503PerMinute,
LIMITS.backgroundAssignment503PerMinuteMin,
LIMITS.backgroundAssignment503PerMinuteMax
),
atMost(
'background_assignment_requests_within_total',
report.load.backgroundAssignmentRequestsPerMinute,
report.load.backgroundRequestsPerMinute
),
atMost(
'background_assignment_503_within_assignments',
report.load.backgroundAssignment503PerMinute,
report.load.backgroundAssignmentRequestsPerMinute
),
equal('target_cell_count', report.load.targetCells.length, LIMITS.targetCellCount),
equal(
'target_connection_cap',
report.load.targetConnectionCap,
LIMITS.targetConnectionCap
),
atMost(
'peak_target_connections',
peakTargetConnections,
report.load.targetConnectionCap
),
equal('recovered_controls', recoveredControls, report.load.drainingDesktops),
equal('migration_expirations', report.outcomes.migrationExpirations, 0),
equal('migration_aborts', report.outcomes.migrationAborts, 0),
equal('transaction_retry_exhaustions', report.outcomes.transactionRetryExhaustions, 0),
equal(
'key_proven_target_registrations',
report.outcomes.keyProvenTargetRegistrations,
report.load.drainingDesktops
),
atLeast(
'oldest_migration_lease_remaining_at_drain_ms',
report.outcomes.oldestMigrationLeaseRemainingAtDrainMs,
LIMITS.oldestMigrationLeaseRemainingAtDrainMsMin
),
atMost(
'target_registration_duration_ms',
report.outcomes.targetRegistrationDurationMs,
LIMITS.targetRegistrationDurationMsMax
),
atLeast(
'assignment_throughput_retention',
assignmentThroughputRetention,
LIMITS.assignmentThroughputRetentionMin
),
atLeast('resolve_success_rate', resolveSuccessRate, LIMITS.resolveSuccessRateMin),
lessThan(
'resolve_overload_rate',
resolveOverloadRate,
LIMITS.resolveOverloadRateMaxExclusive
),
atLeast('eligible_resolve_requests', report.outcomes.eligibleResolveRequests, 100),
equal('readiness_failures', report.outcomes.readinessFailures, 0),
atLeast('readiness_checks', report.outcomes.readinessChecks, 1),
equal('maintenance_failures', report.outcomes.maintenanceFailures, 0),
atLeast('maintenance_operations', report.outcomes.maintenanceOperations, 1),
equal(
'director_peak_instances',
report.outcomes.directorPeakInstances,
LIMITS.directorPeakInstances
),
equal(
'rollout_overlap_peak_instances',
report.outcomes.rolloutOverlapPeakInstances,
LIMITS.rolloutOverlapPeakInstances
),
equal(
'rollout_overlap_peak_public_operations',
report.outcomes.rolloutOverlapPeakPublicOperations,
LIMITS.rolloutOverlapPeakPublicOperations
),
atLeast(
'rollout_overlap_eligible_resolve_requests',
report.outcomes.rolloutOverlapEligibleResolveRequests,
100
),
atLeast(
'rollout_overlap_resolve_success_rate',
rolloutOverlapResolveSuccessRate,
LIMITS.resolveSuccessRateMin
),
lessThan(
'rollout_overlap_resolve_overload_rate',
rolloutOverlapResolveOverloadRate,
LIMITS.resolveOverloadRateMaxExclusive
),
equal(
'rollout_overlap_readiness_failures',
report.outcomes.rolloutOverlapReadinessFailures,
0
),
lessThan(
'rollout_overlap_pool_wait_p95_ms',
report.outcomes.rolloutOverlapPoolWaitP95Ms,
LIMITS.poolWaitP95MsMaxExclusive
),
lessThan(
'rollout_overlap_database_cpu_percent_max',
report.outcomes.rolloutOverlapDatabaseCpuPercentMax,
LIMITS.databaseCpuPercentMaxMaxExclusive
),
lessThan(
'pool_wait_p95_ms',
report.outcomes.poolWaitP95Ms,
LIMITS.poolWaitP95MsMaxExclusive
),
lessThan(
'pool_wait_max_ms',
report.outcomes.poolWaitMaxMs,
LIMITS.poolWaitMaxMsMaxExclusive
),
lessThan(
'database_cpu_percent_p95',
report.outcomes.databaseCpuPercentP95,
LIMITS.databaseCpuPercentP95MaxExclusive
),
lessThan(
'database_cpu_percent_max',
report.outcomes.databaseCpuPercentMax,
LIMITS.databaseCpuPercentMaxMaxExclusive
),
atMost(
'recovery_duration_ms',
report.outcomes.recoveryDurationMs,
LIMITS.recoveryDurationMsMax
)
]
return {
schemaVersion: 1,
status: thresholds.every(({ pass }) => pass) ? 'PASS' : 'FAIL',
environment: {
projectId: report.environment.projectId,
directorOrigin: report.environment.directorOrigin
},
metrics: {
recoveredControls,
peakTargetConnections,
assignmentThroughputRetention,
resolveSuccessRate,
resolveOverloadRate,
rolloutOverlapResolveSuccessRate,
rolloutOverlapResolveOverloadRate
},
thresholds
}
}
function parseReport(input) {
const report = strictObject(input, EXPECTED_KEYS.report, 'report')
if (report.schemaVersion !== 1) throw new Error('unsupported report schemaVersion')
const environment = strictObject(
report.environment,
EXPECTED_KEYS.environment,
'environment'
)
assertSafeEnvironment(environment)
const load = strictObject(report.load, EXPECTED_KEYS.load, 'load')
if (!Array.isArray(load.targetCells)) throw new Error('load.targetCells must be an array')
const targetCells = load.targetCells.map((value, index) => {
const cell = strictObject(value, EXPECTED_KEYS.targetCell, `load.targetCells[${index}]`)
if (!/^[a-z0-9-]{1,128}$/.test(cell.cellId)) throw new Error('target cellId is invalid')
return {
cellId: cell.cellId,
peakConnections: nonnegativeNumber(cell.peakConnections, 'peakConnections'),
recoveredControls: nonnegativeNumber(cell.recoveredControls, 'recoveredControls')
}
})
if (new Set(targetCells.map(({ cellId }) => cellId)).size !== targetCells.length) {
throw new Error('target cell IDs must be unique')
}
const outcomes = strictObject(report.outcomes, EXPECTED_KEYS.outcomes, 'outcomes')
return {
schemaVersion: 1,
environment: {
projectId: environment.projectId,
directorOrigin: environment.directorOrigin,
databaseVcpu: positiveNumber(environment.databaseVcpu, 'databaseVcpu'),
databasePoolMax: positiveNumber(environment.databasePoolMax, 'databasePoolMax'),
publicConcurrentMax: positiveNumber(
environment.publicConcurrentMax,
'publicConcurrentMax'
),
resolvePrioritySlots: positiveNumber(
environment.resolvePrioritySlots,
'resolvePrioritySlots'
),
directorMinInstances: positiveNumber(
environment.directorMinInstances,
'directorMinInstances'
),
directorMaxInstances: positiveNumber(
environment.directorMaxInstances,
'directorMaxInstances'
),
cloudRunConcurrency: positiveNumber(
environment.cloudRunConcurrency,
'cloudRunConcurrency'
),
rolloutOldPublicConcurrentMax: positiveNumber(
environment.rolloutOldPublicConcurrentMax,
'rolloutOldPublicConcurrentMax'
),
rolloutOldResolvePrioritySlots: nonnegativeNumber(
environment.rolloutOldResolvePrioritySlots,
'rolloutOldResolvePrioritySlots'
),
rolloutNewPublicConcurrentMax: positiveNumber(
environment.rolloutNewPublicConcurrentMax,
'rolloutNewPublicConcurrentMax'
),
rolloutNewResolvePrioritySlots: positiveNumber(
environment.rolloutNewResolvePrioritySlots,
'rolloutNewResolvePrioritySlots'
)
},
load: {
drainingDesktops: positiveNumber(load.drainingDesktops, 'drainingDesktops'),
backgroundRequestsPerMinute: positiveNumber(
load.backgroundRequestsPerMinute,
'backgroundRequestsPerMinute'
),
backgroundAssignmentRequestsPerMinute: positiveNumber(
load.backgroundAssignmentRequestsPerMinute,
'backgroundAssignmentRequestsPerMinute'
),
backgroundAssignment503PerMinute: nonnegativeNumber(
load.backgroundAssignment503PerMinute,
'backgroundAssignment503PerMinute'
),
targetConnectionCap: positiveNumber(load.targetConnectionCap, 'targetConnectionCap'),
targetCells
},
outcomes: Object.fromEntries(
EXPECTED_KEYS.outcomes.map((key) => [key, nonnegativeNumber(outcomes[key], `outcomes.${key}`)])
)
}
}
function assertSafeEnvironment(environment) {
if (typeof environment.projectId !== 'string') throw new Error('projectId must be a string')
if (
environment.projectId !== 'local' &&
!environment.projectId.endsWith('-staging') &&
!environment.projectId.endsWith('-test')
) {
throw new Error('recovery-wave reports must come from an isolated non-production project')
}
if (typeof environment.directorOrigin !== 'string') {
throw new Error('directorOrigin must be a string')
}
const origin = new URL(environment.directorOrigin)
const loopback = ['localhost', '127.0.0.1', '::1', '[::1]'].includes(origin.hostname)
const isolatedHost =
loopback || origin.hostname.endsWith('.test') || origin.hostname.includes('staging')
if (
origin.origin !== environment.directorOrigin ||
origin.pathname !== '/' ||
(!loopback && origin.protocol !== 'https:') ||
!isolatedHost
) {
throw new Error('directorOrigin must identify a canonical isolated non-production origin')
}
}
function strictObject(value, keys, name) {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new Error(`${name} must be an object`)
}
const actual = Object.keys(value).sort()
const expected = [...keys].sort()
if (actual.length !== expected.length || actual.some((key, index) => key !== expected[index])) {
throw new Error(`${name} has unexpected or missing fields`)
}
return value
}
function positiveNumber(value, name) {
const number = nonnegativeNumber(value, name)
if (number <= 0) throw new Error(`${name} must be positive`)
return number
}
function nonnegativeNumber(value, name) {
if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
throw new Error(`${name} must be a finite nonnegative number`)
}
return value
}
function ratio(numerator, denominator) {
return denominator === 0 ? 0 : numerator / denominator
}
function equal(name, observed, limit) {
return threshold(name, observed, '==', limit, observed === limit)
}
function atLeast(name, observed, limit) {
return threshold(name, observed, '>=', limit, observed >= limit)
}
function atMost(name, observed, limit) {
return threshold(name, observed, '<=', limit, observed <= limit)
}
function lessThan(name, observed, limit) {
return threshold(name, observed, '<', limit, observed < limit)
}
function between(name, observed, minimum, maximum) {
return threshold(
name,
observed,
'between_inclusive',
[minimum, maximum],
observed >= minimum && observed <= maximum
)
}
function threshold(name, observed, operator, limit, pass) {
return { name, observed, operator, limit, pass }
}
@@ -0,0 +1,214 @@
import assert from 'node:assert/strict'
import test from 'node:test'
import { evaluateRecoveryWaveReport } from './relay-recovery-wave-gate.mjs'
test('passes a complete isolated production-shaped recovery report', () => {
const result = evaluateRecoveryWaveReport(passingReport())
assert.equal(result.status, 'PASS')
assert.equal(result.metrics.recoveredControls, 800)
assert.equal(result.metrics.peakTargetConnections, 425)
assert.equal(result.metrics.resolveSuccessRate, 0.99)
assert.equal(result.thresholds.every(({ pass }) => pass), true)
assert.equal(
result.thresholds.find(({ name }) => name === 'recovery_duration_ms')?.limit,
840_000
)
})
for (const [name, mutate, failedThreshold] of [
[
'target connection ceiling',
(report) => {
report.load.targetCells[0].peakConnections = 601
},
'peak_target_connections'
],
[
'migration expiration',
(report) => {
report.outcomes.migrationExpirations = 1
},
'migration_expirations'
],
[
'migration abort',
(report) => {
report.outcomes.migrationAborts = 1
},
'migration_aborts'
],
[
'full-wave registration deadline',
(report) => {
report.outcomes.targetRegistrationDurationMs = 300_001
},
'target_registration_duration_ms'
],
[
'legacy assignment background shape',
(report) => {
report.load.backgroundAssignment503PerMinute = 100
},
'background_assignment_503_per_minute'
],
[
'production director instance topology',
(report) => {
report.outcomes.directorPeakInstances = 1
},
'director_peak_instances'
],
[
'old/new rollout overlap topology',
(report) => {
report.outcomes.rolloutOverlapPeakInstances = 2
},
'rollout_overlap_peak_instances'
],
[
'old revision shared admission mode',
(report) => {
report.environment.rolloutOldResolvePrioritySlots = 1
},
'rollout_old_resolve_priority_slots'
],
[
'old/new rollout overlap resolve availability',
(report) => {
report.outcomes.rolloutOverlapResolveOverload = 2
},
'rollout_overlap_resolve_overload_rate'
],
[
'resolve availability',
(report) => {
report.outcomes.resolve2xx = 940
report.outcomes.resolveOverload = 20
},
'resolve_success_rate'
],
[
'pool wait',
(report) => {
report.outcomes.poolWaitP95Ms = 500
},
'pool_wait_p95_ms'
],
[
'database CPU',
(report) => {
report.outcomes.databaseCpuPercentMax = 85
},
'database_cpu_percent_max'
],
[
'recovery deadline',
(report) => {
report.outcomes.recoveryDurationMs = 840_001
},
'recovery_duration_ms'
],
[
'non-public database maintenance',
(report) => {
report.outcomes.maintenanceFailures = 1
},
'maintenance_failures'
]
]) {
test(`fails closed on ${name}`, () => {
const report = passingReport()
mutate(report)
const result = evaluateRecoveryWaveReport(report)
assert.equal(result.status, 'FAIL')
assert.equal(
result.thresholds.find(({ name: thresholdName }) => thresholdName === failedThreshold)?.pass,
false
)
})
}
test('rejects production provenance and unexpected report fields', () => {
const productionProject = passingReport()
productionProject.environment.projectId = 'onorca-cloud'
assert.throws(
() => evaluateRecoveryWaveReport(productionProject),
/isolated non-production project/
)
const productionOrigin = passingReport()
productionOrigin.environment.directorOrigin = 'https://relay.onorca.dev'
assert.throws(
() => evaluateRecoveryWaveReport(productionOrigin),
/isolated non-production origin/
)
const extraField = passingReport()
extraField.environment.accessToken = 'must-not-be-accepted'
assert.throws(() => evaluateRecoveryWaveReport(extraField), /unexpected or missing fields/)
})
function passingReport() {
return {
schemaVersion: 1,
environment: {
projectId: 'onorca-cloud-staging',
directorOrigin: 'https://relay-staging.onorca.dev',
databaseVcpu: 2,
databasePoolMax: 3,
publicConcurrentMax: 2,
resolvePrioritySlots: 1,
directorMinInstances: 1,
directorMaxInstances: 2,
cloudRunConcurrency: 80,
rolloutOldPublicConcurrentMax: 2,
rolloutOldResolvePrioritySlots: 0,
rolloutNewPublicConcurrentMax: 2,
rolloutNewResolvePrioritySlots: 1
},
load: {
drainingDesktops: 800,
backgroundRequestsPerMinute: 11_000,
backgroundAssignmentRequestsPerMinute: 10_950,
backgroundAssignment503PerMinute: 9_500,
targetConnectionCap: 600,
targetCells: [
{ cellId: 'target-a', peakConnections: 425, recoveredControls: 400 },
{ cellId: 'target-b', peakConnections: 419, recoveredControls: 400 }
]
},
outcomes: {
migrationExpirations: 0,
migrationAborts: 0,
transactionRetryExhaustions: 0,
keyProvenTargetRegistrations: 800,
oldestMigrationLeaseRemainingAtDrainMs: 660_000,
targetRegistrationDurationMs: 240_000,
assignmentSuccessesPerMinuteBaseline: 1_400,
assignmentSuccessesPerMinuteMinimum: 1_330,
eligibleResolveRequests: 1_000,
resolve2xx: 990,
resolveOverload: 5,
readinessChecks: 180,
readinessFailures: 0,
maintenanceOperations: 30,
maintenanceFailures: 0,
directorPeakInstances: 2,
rolloutOverlapPeakInstances: 4,
rolloutOverlapPeakPublicOperations: 8,
rolloutOverlapEligibleResolveRequests: 100,
rolloutOverlapResolve2xx: 99,
rolloutOverlapResolveOverload: 0,
rolloutOverlapReadinessFailures: 0,
rolloutOverlapPoolWaitP95Ms: 180,
rolloutOverlapDatabaseCpuPercentMax: 78,
poolWaitP95Ms: 120,
poolWaitMaxMs: 900,
databaseCpuPercentP95: 55,
databaseCpuPercentMax: 72,
recoveryDurationMs: 360_000
}
}
}
@@ -0,0 +1,152 @@
import { createHash } from 'node:crypto'
import { readFileSync } from 'node:fs'
import { pathToFileURL } from 'node:url'
const WINDOW_MS = 24 * 60 * 60_000
const BUCKET_MS = 60 * 60_000
const METRICS = [
'requestedRegionsDelta',
'selectedRegionsDelta',
'regionFallbacksDelta',
'unavailableRegionsDelta'
]
const REGION_KEYS = new Set(['asia-east2', 'us-central1', 'unhinted'])
function integer(value, name) {
const parsed = Number(value)
if (!Number.isSafeInteger(parsed) || parsed < 0) throw new Error(`${name} is invalid`)
return parsed
}
function digest(value, name) {
if (!/^sha256:[a-f0-9]{64}$/.test(value ?? '')) throw new Error(`${name} is invalid`)
return value
}
function metric(value, name) {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new Error(`${name} is invalid`)
}
return Object.fromEntries(Object.entries(value).map(([key, count]) => {
if (!REGION_KEYS.has(key)) throw new Error(`${name} has an unknown aggregate key`)
return [key, integer(count, `${name}.${key}`)]
}))
}
function sumMetric(total, value) {
for (const [key, count] of Object.entries(value)) total[key] = (total[key] ?? 0) + count
}
function evidenceSha256(evidence) {
return createHash('sha256').update(JSON.stringify(evidence)).digest('hex')
}
export function createRegionObservationEvidence(entries, bindings, now = Date.now()) {
if (!Array.isArray(entries)) throw new Error('runtime metrics response must be an array')
if (!/^[0-9a-f]{40}$/.test(bindings.commitSha ?? '')) throw new Error('commit SHA is invalid')
const directorImageDigest = digest(bindings.directorImageDigest, 'director digest')
const selectorGeneration = integer(bindings.selectorGeneration, 'selector generation')
const controlGeneration = integer(bindings.controlGeneration, 'control generation')
const start = now - WINDOW_MS
const buckets = Array.from({ length: 24 }, () => 0)
const totals = Object.fromEntries(METRICS.map((name) => [name, {}]))
let samples = 0
for (const entry of entries) {
const timestamp = Date.parse(entry?.timestamp ?? '')
const payload = entry?.jsonPayload
if (
!Number.isFinite(timestamp) ||
timestamp < start ||
timestamp > now + 60_000 ||
payload?.event !== 'orca_relay_runtime_metrics' ||
payload.role !== 'director'
) continue
const bucket = Math.min(23, Math.floor((timestamp - start) / BUCKET_MS))
buckets[bucket] += 1
samples += 1
for (const name of METRICS) sumMetric(totals[name], metric(payload[name], name))
}
if (buckets.some((count) => count === 0)) {
throw new Error('24-hour region evidence has a missing hourly bucket')
}
if (
!Number.isSafeInteger(totals.requestedRegionsDelta['asia-east2']) ||
totals.requestedRegionsDelta['asia-east2'] < 1 ||
!Number.isSafeInteger(totals.selectedRegionsDelta['asia-east2']) ||
totals.selectedRegionsDelta['asia-east2'] < 1
) throw new Error('24-hour region evidence has no Asia request and selection activity')
const evidence = {
v: 1,
commitSha: bindings.commitSha,
directorImageDigest,
selectorGeneration,
controlGeneration,
windowStartedAt: start,
windowEndedAt: now,
hourlySampleCounts: buckets,
samples,
totals
}
return { evidence, sha256: evidenceSha256(evidence) }
}
export function verifyRegionObservationEvidence(sealed, bindings) {
if (
sealed?.sha256 !== evidenceSha256(sealed?.evidence) ||
sealed.evidence?.commitSha !== bindings.commitSha ||
sealed.evidence?.directorImageDigest !== bindings.directorImageDigest ||
sealed.evidence?.selectorGeneration !== Number(bindings.selectorGeneration) ||
sealed.evidence?.controlGeneration !== Number(bindings.controlGeneration) ||
!Array.isArray(sealed.evidence?.hourlySampleCounts) ||
sealed.evidence.hourlySampleCounts.length !== 24 ||
sealed.evidence.hourlySampleCounts.some((count) => integer(count, 'bucket') < 1) ||
integer(sealed.evidence?.totals?.requestedRegionsDelta?.['asia-east2'], 'Asia requests') < 1 ||
integer(sealed.evidence?.totals?.selectedRegionsDelta?.['asia-east2'], 'Asia selections') < 1
) throw new Error('sealed 24-hour region evidence does not match enable authority')
return sealed.evidence
}
function values(argv) {
const result = {}
for (let index = 0; index < argv.length; index += 2) {
if (!argv[index]?.startsWith('--') || argv[index + 1] === undefined) {
throw new Error('invalid arguments')
}
result[argv[index].slice(2)] = argv[index + 1]
}
return result
}
async function stdinJson(input) {
const chunks = []
for await (const chunk of input) chunks.push(chunk)
return JSON.parse(Buffer.concat(chunks).toString('utf8'))
}
export async function main(argv = process.argv.slice(2), input = process.stdin) {
const command = argv.shift()
const args = values(argv)
const bindings = {
commitSha: args['commit-sha'],
directorImageDigest: args['director-image-digest'],
selectorGeneration: args['selector-generation'],
controlGeneration: args['control-generation']
}
if (command === 'create') {
const sealed = createRegionObservationEvidence(await stdinJson(input), bindings)
process.stdout.write(`${JSON.stringify(sealed)}\n`)
return
}
if (command === 'verify') {
verifyRegionObservationEvidence(JSON.parse(readFileSync(args.file, 'utf8')), bindings)
return
}
throw new Error('unknown region observation evidence command')
}
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
main().catch((error) => {
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
process.exitCode = 1
})
}
@@ -0,0 +1,53 @@
import assert from 'node:assert/strict'
import { test } from 'node:test'
import {
createRegionObservationEvidence,
verifyRegionObservationEvidence
} from './relay-region-observation-evidence.mjs'
const now = Date.parse('2026-08-14T12:00:00Z')
const bindings = {
commitSha: 'a'.repeat(40),
directorImageDigest: `sha256:${'b'.repeat(64)}`,
selectorGeneration: 11,
controlGeneration: 4
}
function entries() {
return Array.from({ length: 24 }, (_, index) => ({
timestamp: new Date(now - (index * 60 + 30) * 60_000).toISOString(),
jsonPayload: {
event: 'orca_relay_runtime_metrics',
role: 'director',
requestedRegionsDelta: { 'asia-east2': index === 0 ? 2 : 0 },
selectedRegionsDelta: { 'asia-east2': index === 0 ? 1 : 0 },
regionFallbacksDelta: { 'asia-east2': index === 0 ? 1 : 0 },
unavailableRegionsDelta: {}
}
}))
}
test('seals all 24 hourly aggregate region buckets', () => {
const sealed = createRegionObservationEvidence(entries(), bindings, now)
assert.equal(sealed.evidence.hourlySampleCounts.length, 24)
assert.equal(sealed.evidence.totals.requestedRegionsDelta['asia-east2'], 2)
assert.equal(verifyRegionObservationEvidence(sealed, bindings).samples, 24)
})
test('rejects missing coverage, missing Asia activity, and changed bindings', () => {
assert.throws(() => createRegionObservationEvidence(entries().slice(1), bindings, now), /missing hourly/)
const noAsia = entries().map((entry) => ({
...entry,
jsonPayload: {
...entry.jsonPayload,
requestedRegionsDelta: {},
selectedRegionsDelta: {}
}
}))
assert.throws(() => createRegionObservationEvidence(noAsia, bindings, now), /no Asia/)
const sealed = createRegionObservationEvidence(entries(), bindings, now)
assert.throws(() => verifyRegionObservationEvidence(sealed, {
...bindings,
commitSha: 'c'.repeat(40)
}), /does not match/)
})
@@ -0,0 +1,193 @@
import assert from 'node:assert/strict'
import { readFileSync } from 'node:fs'
import { test } from 'node:test'
import { fileURLToPath } from 'node:url'
import { relayWorkflowUrl } from './relay-repository.mjs'
function workflow(name) {
return readFileSync(
fileURLToPath(relayWorkflowUrl(name)),
'utf8'
)
}
test('same-cap wrapper is reusable, canary-bound, and sequential', () => {
const wrapper = workflow('deploy-relay-production-same-cap.yml')
const job = workflow('deploy-relay-production-same-cap-job.yml')
assert.match(wrapper, /options: \[verify, canary-apply, batch-apply, rollback\]/)
assert.match(wrapper, /relay-same-cap-canary-\$\{\{ inputs\.canary-run-id \}\}/)
assert.match(wrapper, /needs: \[gate, cell_1\]/)
assert.match(wrapper, /needs: \[gate, cell_2\]/)
assert.match(wrapper, /needs: \[gate, cell_3\]/)
assert.match(job, /on:\n workflow_call:/)
assert.match(job, /c27\|c28\|c29/)
assert.match(job, /EXPECTED_HARD_CAP=3000/)
assert.match(job, /EXPECTED_REGION=asia-east2/)
assert.match(job, /--hard-cap "\$\{EXPECTED_HARD_CAP\}"/)
assert.match(job, /--regional-rehome-protocol "\$\{DESIRED_REHOME_PROTOCOL\}"/)
assert.match(job, /--argjson protocol "\$\{PREDECESSOR_REHOME_PROTOCOL\}"/)
assert.match(job, /runtime predecessor mismatch fields=/)
// A rollback interrupted between apply and restore must be resumable.
assert.match(job, /ROLLBACK_RESUME=true/)
assert.match(job, /test "\$\{LIVE_IMAGE_DIGEST\}" = "\$\{DESIRED_IMAGE_DIGEST\}"/)
// Resume must skip BOTH the drain (no restart will clear the flag) and the
// apply (state already converged), and prove convergence instead.
assert.match(
job,
/Reversibly isolate and drain only the selected cell\n if: \$\{\{ inputs\.mode != 'verify' && env\.ROLLBACK_RESUME != 'true' \}\}/
)
assert.match(
job,
/Apply only the selected same-cap template and MIG\n if: \$\{\{ inputs\.mode != 'verify' && env\.ROLLBACK_RESUME != 'true' \}\}/
)
assert.match(
job,
/Require converged Terraform state and a stable MIG on resume\n if: \$\{\{ inputs\.mode != 'verify' && env\.ROLLBACK_RESUME == 'true' \}\}/
)
assert.match(job, /resume found unconverged resources/)
// A canary or batch cell that failed before its template apply also
// resumes here with template drift from repo changes since its last roll;
// only a plan the reviewed validator approves for the image the cell
// already serves may pass, and resume still applies nothing.
assert.match(job, /requiring reviewed rollback-image drift/)
assert.match(
job,
/--image "\$\{DESIRED_IMAGE\}" \\\n {16}--rollback-image "\$\{DESIRED_IMAGE\}"/
)
// The relaxation is only safe if the reviewed validator actually runs on
// the NON-converged branch, in same-cap-cell mode, with the trust config
// the validator requires, restricted to the template-and-MIG change pair.
assert.match(
job,
/if ! terraform -chdir=infra\/terraform show -json[\s\S]{0,220}\| length == 0' >\/dev\/null\n then\n/
)
assert.match(
job,
/requiring reviewed rollback-image drift'\n[\s\S]{0,400}?\n {16}--mode same-cap-cell --cell-id "\$\{TARGET_CELL_ID\}" \\\n/
)
assert.match(
job,
/Require converged Terraform state and a stable MIG on resume[\s\S]{0,200}DIRECTOR_RUNTIME_SERVICE_ACCOUNT: \$\{\{ vars\.PRODUCTION_GCP_RELAY_DIRECTOR_RUNTIME_SERVICE_ACCOUNT \}\}/
)
assert.match(
job,
/--rollback-image "\$\{DESIRED_IMAGE\}" \\\n {16}--rehome-director-service-account "\$\{DIRECTOR_RUNTIME_SERVICE_ACCOUNT\}"/
)
assert.match(job, /host-drain \\\n {14}\| jq -e '\.changes == 2' >\/dev\/null/)
assert.match(job, /resume requires the isolated migration-only cell/)
assert.match(job, /test "\$\{TARGET_INCARNATION\}" = "\$\{SOURCE_INCARNATION\}"/)
assert.match(job, /\(.regionalRehomeProtocol \/\/ 0\) == \$protocol/)
assert.match(job, /\(\.draining == false or \$drainingOk\)/)
// Selector expectations must follow the mutations' returned generations,
// not fixed offsets: isolate is a no-op on a cell a failed canary already
// isolated, and the restore inspect must expect post-restore membership.
assert.match(job, /SELECTOR_GENERATION_AFTER_ISOLATE=\$\{EFFECTIVE_SELECTOR_GENERATION\}/)
assert.match(job, /SELECTOR_GENERATION_AFTER_ISOLATE=\$\{ISOLATE_GENERATION\}/)
assert.match(job, /--expected-selector-generation "\$\{SELECTOR_GENERATION_AFTER_ISOLATE\}"/)
assert.match(job, /--expected-selector-generation "\$\{SELECTOR_GENERATION_AFTER_ACTIVATE\}"/)
assert.match(job, /--expected-migration-only-cells "\$\{RESTORED_MIGRATION_CELLS\}"/)
assert.match(job, /--expected-general-cells "\$\{RESTORED_GENERAL_CELLS\}"/)
assert.match(job, /FAILSAFE_GENERATION/)
// Later batch waves start after ~16-min predecessor rolls, so BOTH evidence
// age checks must scale by wave or cell_2+ can never pass; the bound's
// per-wave step is the cell job timeout, so the two must move together.
assert.match(job, /--required-migration-policy strict \\\n --wave-index "\$\{WAVE_INDEX\}"/)
assert.match(job, /dry-run\.state\.json" \\\n --wave-index "\$\{WAVE_INDEX\}" "\$\{RETRY_ARGS\[@\]\}"/)
assert.match(job, /timeout-minutes: 75/)
// Both age gates step by the cell job timeout above; the constant is
// duplicated across the two languages, so pin each copy to it.
for (const source of [
'../../dev/scripts/relay-monitor-evidence.mjs',
'../../apps/relay-ops/src/incident-live-preflight-cli.ts'
]) {
const body = readFileSync(fileURLToPath(new URL(source, import.meta.url)), 'utf8')
assert.match(body, /WAVE_PREDECESSOR_TIMEOUT_MS = 75 \* 60_000/)
assert.match(body, /\^\[0-3\]\$/)
}
// Aged-evidence replay via job re-runs is fenced: mutations are
// single-dispatch, so a failed cell needs a fresh gate and monitor run.
assert.match(job, /test "\$\{GITHUB_RUN_ATTEMPT\}" = 1/)
for (const index of [0, 1, 2, 3]) {
assert.match(wrapper, new RegExp(`wave-index: '${index}'`))
}
assert.doesNotMatch(job, /EFFECTIVE_SELECTOR_GENERATION \+ 1\)/)
assert.doesNotMatch(job, /EFFECTIVE_SELECTOR_GENERATION \+ 2\)/)
assert.match(job, /\$region == "us-central1" and \$protocol == 0 and [.]region == null/)
assert.match(job, /[.]regionalRehomeProtocol \/\/ 0/)
assert.match(job, /runtime predecessor normalized legacy fields=/)
assert.match(job, /probe-relay-rehome-trust[.]mjs/)
assert.doesNotMatch(job, /service_account: \$\{\{ vars\.PRODUCTION_GCP_RELAY_(?:DIRECTOR_)?RUNTIME_SERVICE_ACCOUNT/)
assert.doesNotMatch(job, /roles\/iam\.serviceAccountTokenCreator/)
})
// Why: the same-cap caller defines release_lease itself, and a caller-defined job presents the
// caller as job_workflow_ref, so the pair must admit the caller alongside its reusable job.
test('shared deploy WIF admits the exact same-cap reusable workflow pair and the caller itself', () => {
const terraform = readFileSync(
fileURLToPath(new URL('../../infra/terraform/relay-github-actions.tf', import.meta.url)),
'utf8'
)
const providerStart = terraform.indexOf(
'resource "google_iam_workload_identity_pool_provider" "github"'
)
const providerEnd = terraform.indexOf('\nresource "', providerStart + 1)
const sharedProvider = terraform.slice(providerStart, providerEnd)
assert.ok(providerStart >= 0 && providerEnd > providerStart)
assert.match(sharedProvider, /local\.relay_github_workflow_conditions\["github"\]/)
// The pairing itself now lives in the clause the provider renders, once per accepted repository.
assert.match(
terraform,
/assertion\.workflow_ref == '\$\{prefix\}\$\{local\.github_production_relay_same_cap_workflow_file\}@refs\/heads\/main' && \(assertion\.job_workflow_ref == '\$\{prefix\}\$\{local\.github_production_relay_same_cap_job_workflow_file\}@refs\/heads\/main' \|\| assertion\.job_workflow_ref == '\$\{prefix\}\$\{local\.github_production_relay_same_cap_workflow_file\}@refs\/heads\/main'\)/
)
})
test('pause and disable precede optional installation and cloud diagnostics', () => {
const job = workflow('operate-relay-production-rehome-job.yml')
const emergency = job.indexOf('Apply emergency durable pause or disable before diagnostics')
const install = job.indexOf('pnpm install --frozen-lockfile')
const revision = job.indexOf('Verify exact serving and rollback director identities')
assert.ok(emergency > 0)
assert.ok(emergency < install)
assert.ok(emergency < revision)
assert.match(job, /inputs\.mode == 'pause' \|\| inputs\.mode == 'disable'/)
assert.match(job, /Seal 24-hour aggregate region observation evidence/)
assert.match(job, /--freshness=25h --limit=30000/)
assert.match(job, /relay-region-observation-\$\{\{ github\.run_id \}\}-\$\{\{ github\.run_attempt \}\}/)
assert.match(job, /test "\$\{RATE_PER_MINUTE\}" = 10/)
})
test('a failed enable independently restores and verifies durable disabled state', () => {
const job = workflow('operate-relay-production-rehome-job.yml')
const enable = job.indexOf('Apply exact durable regional rehome enable')
const evidence = job.indexOf('Read fresh aggregate completion and abort evidence')
const summary = job.indexOf('Publish aggregate control evidence')
const recovery = job.indexOf('Fail closed after an unsuccessful enable run')
assert.ok(enable > 0 && enable < evidence && evidence < summary && summary < recovery)
const recoveryStep = job.slice(recovery)
assert.match(
recoveryStep,
/failure\(\) && inputs\.mode == 'enable' && steps\.google-auth\.outcome == 'success'/
)
assert.match(recoveryStep, /--mode recover-enable/)
assert.match(recoveryStep, /--expected-control-generation "\$\{EXPECTED_CONTROL_GENERATION\}"/)
assert.match(recoveryStep, /RECOVER_FAILED_REGIONAL_REHOME_ENABLE/)
assert.match(recoveryStep, /\.control\.enabled == false/)
assert.doesNotMatch(recoveryStep, /gcloud|pnpm/)
})
test('director rollout has a strict one-time identity bootstrap', () => {
const workflowBody = workflow('deploy-relay-production-director.yml')
const script = readFileSync(
fileURLToPath(new URL('./deploy-relay-blue-green.mjs', import.meta.url)),
'utf8'
)
assert.match(workflowBody, /BOOTSTRAP_RELAY_DIRECTOR_REHOME_IDENTITY/)
assert.match(workflowBody, /--predecessor-runtime-service-account/)
assert.match(workflowBody, /--expected-rehome-generation/)
assert.match(script, /args\.push\('--service-account', config\['runtime-service-account'\]\)/)
assert.match(script, /director predecessor runtime service account does not match/)
const candidateProof = script.indexOf('await verifyRehomeDisabled(candidate.origin)')
const trafficMove = script.indexOf('operations.updateTraffic(config, [`--to-tags=')
assert.ok(candidateProof > 0 && candidateProof < trafficMove)
assert.equal(script.indexOf('verifyRehomeDisabled', trafficMove), -1)
})
@@ -0,0 +1,65 @@
import { pathToFileURL } from 'node:url'
const INVENTORY = /^\[orca-relay\] regional rehome inventory active=(\d+) awaitingReceipt=(\d+) targetRegistered=(\d+) completedLast24Hours=(\d+) abortedLast24Hours=(\d+) oldestActiveAgeMs=(none|\d+)$/
function count(value, name) {
const parsed = Number(value)
if (!Number.isSafeInteger(parsed) || parsed < 0) throw new Error(`${name} is invalid`)
return parsed
}
export function parseRegionalRehomeInventory(entries, options = {}) {
if (!Array.isArray(entries)) throw new Error('logging response must be an array')
const parsed = entries.flatMap((entry) => {
const match = INVENTORY.exec(entry?.textPayload ?? '')
const timestamp = Date.parse(entry?.timestamp ?? '')
if (!match || !Number.isFinite(timestamp)) return []
return [{
timestamp,
active: count(match[1], 'active'),
awaitingReceipt: count(match[2], 'awaiting receipt'),
targetRegistered: count(match[3], 'target registered'),
completedLast24Hours: count(match[4], 'completed'),
abortedLast24Hours: count(match[5], 'aborted'),
oldestActiveAgeMs: match[6] === 'none' ? null : count(match[6], 'oldest active age')
}]
}).sort((left, right) => right.timestamp - left.timestamp)
if (parsed.length === 0) throw new Error('no aggregate regional rehome inventory evidence')
const latest = parsed[0]
const now = options.now ?? Date.now()
const maxAgeMs = options.maxAgeMs ?? 15 * 60_000
if (latest.timestamp > now + 60_000 || latest.timestamp < now - maxAgeMs) {
throw new Error('aggregate regional rehome inventory evidence is stale')
}
return latest
}
function argumentsMap(argv) {
const values = {}
for (let index = 0; index < argv.length; index += 2) {
if (!argv[index]?.startsWith('--') || argv[index + 1] === undefined) {
throw new Error('invalid arguments')
}
values[argv[index].slice(2)] = argv[index + 1]
}
return values
}
export async function main(argv = process.argv.slice(2), input = process.stdin) {
const values = argumentsMap(argv)
const maxAgeMs = count(values['max-age-ms'] ?? 900_000, '--max-age-ms')
const chunks = []
for await (const chunk of input) chunks.push(chunk)
const evidence = parseRegionalRehomeInventory(
JSON.parse(Buffer.concat(chunks).toString('utf8')),
{ maxAgeMs }
)
process.stdout.write(`${JSON.stringify({ event: 'relay_rehome_aggregate_evidence', ...evidence })}\n`)
}
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
main().catch((error) => {
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
process.exitCode = 1
})
}
@@ -0,0 +1,38 @@
import assert from 'node:assert/strict'
import { test } from 'node:test'
import { parseRegionalRehomeInventory } from './relay-rehome-aggregate-evidence.mjs'
const now = Date.parse('2026-08-14T12:00:00Z')
test('selects the newest fresh aggregate-only regional rehome inventory', () => {
const result = parseRegionalRehomeInventory([
{
timestamp: '2026-08-14T11:58:00Z',
textPayload: '[orca-relay] regional rehome inventory active=2 awaitingReceipt=1 targetRegistered=1 completedLast24Hours=9 abortedLast24Hours=0 oldestActiveAgeMs=30000'
},
{
timestamp: '2026-08-14T11:50:00Z',
textPayload: '[orca-relay] regional rehome inventory active=1 awaitingReceipt=0 targetRegistered=1 completedLast24Hours=8 abortedLast24Hours=0 oldestActiveAgeMs=none'
}
], { now, maxAgeMs: 5 * 60_000 })
assert.deepEqual(result, {
timestamp: Date.parse('2026-08-14T11:58:00Z'),
active: 2,
awaitingReceipt: 1,
targetRegistered: 1,
completedLast24Hours: 9,
abortedLast24Hours: 0,
oldestActiveAgeMs: 30_000
})
})
test('rejects stale, malformed, and identity-bearing lookalikes', () => {
assert.throws(() => parseRegionalRehomeInventory([{
timestamp: '2026-08-14T11:00:00Z',
textPayload: '[orca-relay] regional rehome inventory active=0 awaitingReceipt=0 targetRegistered=0 completedLast24Hours=0 abortedLast24Hours=0 oldestActiveAgeMs=none'
}], { now, maxAgeMs: 5 * 60_000 }), /stale/)
assert.throws(() => parseRegionalRehomeInventory([{
timestamp: '2026-08-14T11:59:00Z',
textPayload: '[orca-relay] regional rehome inventory active=0 hostId=secret'
}], { now }), /no aggregate/)
})
+29
View File
@@ -0,0 +1,29 @@
import { readFileSync } from 'node:fs'
// Single place naming the repository the Relay workflows live in and where their files sit. The
// public-repo copy moves this tree under cloud/, prefixes every workflow filename, and changes the
// owning repository, so only this module changes: nothing else may restate any of the three.
export const RELAY_GITHUB_REPOSITORY = 'stablyai/orca'
export const RELAY_WORKFLOW_FILE_PREFIX = 'cloud-'
// Where .github/workflows sits relative to this file. Workflows stay at the repository root while
// this tree moves under cloud/, so the depth changes at the copy even though the layout does not.
export const RELAY_WORKFLOW_DIRECTORY = new URL('../../../.github/workflows/', import.meta.url)
export function relayWorkflowFile(name) {
return `${RELAY_WORKFLOW_FILE_PREFIX}${name}`
}
// Repository-relative path, the shape GitHub reports in workflow_ref and evidence payloads.
export function relayWorkflowPath(name) {
return `.github/workflows/${relayWorkflowFile(name)}`
}
export function relayWorkflowUrl(name) {
return new URL(relayWorkflowFile(name), RELAY_WORKFLOW_DIRECTORY)
}
export function readRelayWorkflow(name) {
return readFileSync(relayWorkflowUrl(name), 'utf8')
}
@@ -0,0 +1,36 @@
import assert from 'node:assert/strict'
import { readdirSync, readFileSync } from 'node:fs'
import test from 'node:test'
import { fileURLToPath } from 'node:url'
import {
RELAY_GITHUB_REPOSITORY,
RELAY_WORKFLOW_FILE_PREFIX,
readRelayWorkflow,
relayWorkflowFile,
relayWorkflowPath,
relayWorkflowUrl
} from './relay-repository.mjs'
const directory = fileURLToPath(new URL('.', import.meta.url))
// The Relay copy takes the scripts named for it. Everything else stays with the applications.
const relayScripts = readdirSync(directory)
.filter((name) => name.includes('relay') && name.endsWith('.mjs'))
.filter((name) => !name.startsWith('relay-repository.'))
test('workflow identity is derived, never restated', () => {
assert.equal(relayWorkflowFile('deploy-relay-staging.yml'), `${RELAY_WORKFLOW_FILE_PREFIX}deploy-relay-staging.yml`)
assert.equal(relayWorkflowPath('deploy-relay-staging.yml'), `.github/workflows/${relayWorkflowFile('deploy-relay-staging.yml')}`)
assert.ok(relayWorkflowUrl('deploy-relay-staging.yml').pathname.endsWith(relayWorkflowPath('deploy-relay-staging.yml')))
assert.match(readRelayWorkflow('deploy-relay-staging.yml'), /^name:/m)
assert.match(RELAY_GITHUB_REPOSITORY, /^[\w.-]+\/[\w.-]+$/)
})
// Why: the public-repo copy changes the owning repository, the workflow filenames, and the depth
// this tree sits at. Each has to be one edit here, so no Relay script may restate any of them.
test('no Relay script restates the repository or the workflow directory', () => {
for (const name of relayScripts) {
const text = readFileSync(`${directory}${name}`, 'utf8')
assert.doesNotMatch(text, /stablyai\//, `${name} restates the GitHub repository`)
assert.doesNotMatch(text, /\.github\/workflows/, `${name} restates the workflow directory`)
}
})
@@ -0,0 +1,159 @@
import assert from 'node:assert/strict'
import { readFileSync } from 'node:fs'
import { test } from 'node:test'
import { relayWorkflowFile, relayWorkflowUrl } from './relay-repository.mjs'
const workflow = readFileSync(
relayWorkflowUrl('prove-relay-staging-capacity.yml'),
'utf8'
)
const recoveryWorkflow = readFileSync(
relayWorkflowUrl('recover-relay-staging-c4-image.yml'),
'utf8'
)
const requeueWorkflow = readFileSync(
relayWorkflowUrl('requeue-relay-staging-c4-recovery.yml'),
'utf8'
)
const githubActions = readFileSync(
new URL('../../infra/terraform/relay-github-actions.tf', import.meta.url),
'utf8'
)
const cells = readFileSync(
new URL('../../infra/terraform/relay-gce-cells.tf', import.meta.url),
'utf8'
)
const relay = readFileSync(new URL('../../infra/terraform/relay.tf', import.meta.url), 'utf8')
const stagingTfvars = readFileSync(
new URL('../../infra/terraform/environments/staging.tfvars', import.meta.url),
'utf8'
)
const productionTfvars = readFileSync(
new URL('../../infra/terraform/environments/production.tfvars', import.meta.url),
'utf8'
)
const launchDigest = '5aedbca5c86de24c8b4d4bf7e3b444b76c712f281ede916cb9d90f70cad1e563'
// Scoped to the Asia cells by name: the production capacity cells now serve this digest too,
// so a file-wide count no longer isolates Asia.
const asiaCells = ['production-gce-c27', 'production-gce-c28', 'production-gce-c29']
function productionCell(cellId) {
const start = productionTfvars.indexOf(`"${cellId}"`)
assert.notEqual(start, -1, `${cellId} is missing`)
return productionTfvars.slice(start, productionTfvars.indexOf('\n }', start))
}
test('pins staging C4 and all production Asia cells to the same launch image', () => {
assert.equal(stagingTfvars.match(new RegExp(launchDigest, 'g'))?.length, 1)
for (const cellId of asiaCells) {
assert.match(productionCell(cellId), new RegExp(`relay@sha256:${launchDigest}"`), cellId)
}
assert.match(recoveryWorkflow, new RegExp(`TARGET_IMAGE_DIGEST: sha256:${launchDigest}`))
})
test('refreshes only empty staging C4 through the trusted capacity identity', () => {
const refresh = workflow.slice(workflow.indexOf(' refresh-asia-c4-image:'))
assert.match(refresh, /terraform_version: 1\.15\.8/)
assert.doesNotMatch(refresh, /terraform_version: 1\.5\.7/)
assert.match(refresh, /github\.ref == 'refs\/heads\/main'/)
assert.match(refresh, /REFRESH_STAGING_ASIA_C4_IMAGE/)
assert.match(refresh, /APPROVED_PREDECESSOR_IMAGE_DIGEST/)
assert.match(refresh, /--activity quiescent/)
assert.match(refresh, /--admission migration-only/)
assert.match(refresh, /fence_digests="\$\{PREDECESSOR_IMAGE_DIGEST\}"/)
assert.match(refresh, /--expected-image-digests "\$\{TARGET_IMAGE_DIGEST\}"/)
assert.match(refresh, /--mode same-cap-image/)
assert.match(refresh, /test "\$\(jq -r '\.changes'/)
assert.match(refresh, /REFRESH_PHASE=\$\{refresh_phase\}/)
assert.match(refresh, /MUTATION_STARTED=false/)
assert.match(refresh, /MIG_STABLE_AT_MS=/)
assert.match(refresh, /PLAN_CHANGES=/)
assert.match(refresh, /obsolete-template-delete/)
assert.match(
refresh,
/\*:manager-convergence\|\*:replacement-with-obsolete-template\) refresh_phase=converging/
)
assert.match(refresh, /--mode isolate/)
assert.match(refresh, /--mode verify/)
assert.match(refresh, /\.status\.runtime\.ready/)
assert.match(refresh, /\.status\.runtime\.startedAt/)
assert.match(refresh, /\.status\.runtime\.lastHeartbeatAt/)
assert.match(refresh, /--runtime unavailable/)
const plan = refresh.indexOf('Save, validate, and classify the exact C4 plan')
const currentState = refresh.indexOf('Verify the exact selector and current C4 state')
const isolate = refresh.indexOf('--mode isolate')
const apply = refresh.indexOf('terraform -chdir=infra/terraform apply -auto-approve')
assert.ok(plan < currentState && currentState < isolate && isolate < apply)
assert.match(refresh, /Require an empty targeted Terraform readback/)
})
test('recovers a failed or cancelled C4 refresh from an independent workflow', () => {
assert.match(recoveryWorkflow, /terraform_version: 1\.15\.8/)
assert.doesNotMatch(recoveryWorkflow, /terraform_version: 1\.5\.7/)
assert.match(recoveryWorkflow, /workflow_run:/)
assert.match(recoveryWorkflow, /workflows: \[Prove Relay Staging Capacity\]/)
assert.match(recoveryWorkflow, /RECOVER_STAGING_ASIA_C4_IMAGE/)
assert.match(recoveryWorkflow, /outputs:\n\s+recover: \$\{\{ steps\.trigger\.outputs\.recover \}\}/)
assert.match(recoveryWorkflow, /if test "\$\{count\}" = 0; then\n\s+echo "recover=false"/)
assert.match(recoveryWorkflow, /needs: gate\n\s+if: \$\{\{ needs\.gate\.outputs\.recover == 'true' \}\}/)
assert.match(recoveryWorkflow, /concurrency:\n\s+group: relay-staging-mutation/)
assert.match(recoveryWorkflow, /group: relay-staging-mutation/)
assert.match(recoveryWorkflow, /\.name == "refresh-asia-c4-image"/)
assert.match(recoveryWorkflow, /PREDECESSOR_IMAGE_DIGEST: sha256:ce16d13/)
assert.match(recoveryWorkflow, /TARGET_IMAGE_DIGEST: sha256:5aedbca5/)
assert.match(recoveryWorkflow, /id: preflight-auth/)
assert.match(recoveryWorkflow, /id: verify-auth/)
assert.match(recoveryWorkflow, /\.status\.runtime\.ready/)
assert.match(recoveryWorkflow, /--runtime unavailable/)
assert.match(recoveryWorkflow, /current_digest.*\^sha256:\[a-f0-9\]\{64\}\$/)
assert.match(recoveryWorkflow, /expected_digests="\$\{expected_digests\},\$\{current_digest\}"/)
assert.match(recoveryWorkflow, /--timeout-ms 240000/)
assert.doesNotMatch(recoveryWorkflow, /--timeout-ms 900000/)
assert.match(recoveryWorkflow, /-var manage_artifact_dns=false -lock-timeout=5m/)
assert.match(recoveryWorkflow, /--mode same-cap-image/)
assert.match(recoveryWorkflow, /test "\$\(jq -r '\.changes'/)
assert.equal(recoveryWorkflow.match(/\*:replacement-with-obsolete-template/g)?.length, 2)
assert.match(
recoveryWorkflow,
/\^\(replacement\|replacement-with-obsolete-template\|manager-convergence\)\$/
)
const preflightAuth = recoveryWorkflow.indexOf('id: preflight-auth')
const preflight = recoveryWorkflow.indexOf('Inspect the exact C4 recovery state')
const recoveryPlan = recoveryWorkflow.indexOf('Classify both exact recovery end states')
const apply = recoveryWorkflow.indexOf('Apply and stabilize the saved predecessor plan')
const restore = recoveryWorkflow.indexOf('Restart only when the plan did not replace C4')
const verifyAuth = recoveryWorkflow.indexOf('id: verify-auth')
const verify = recoveryWorkflow.indexOf('Verify the recovered image and unchanged isolation')
assert.ok(preflightAuth < preflight && preflight < recoveryPlan && recoveryPlan < apply)
assert.ok(apply < restore)
assert.ok(restore < verifyAuth && verifyAuth < verify)
assert.match(githubActions, /"recover-relay-staging-c4-image\.yml"/)
})
test('requeues a protected C4 recovery cancelled while pending', () => {
assert.match(requeueWorkflow, /workflows: \[Recover Relay Staging C4 Image\]/)
assert.match(requeueWorkflow, /conclusion == 'cancelled'/)
assert.match(requeueWorkflow, /permissions:\n\s+actions: write\n\s+contents: read/)
assert.match(requeueWorkflow, /group: relay-staging-c4-recovery-requeue/)
assert.match(requeueWorkflow, /\.name == "recover" and\n\s+\.conclusion == "cancelled"/)
assert.match(requeueWorkflow, /\.started_at == null/)
assert.match(requeueWorkflow, /\.name == "gate" and \.conclusion == "success"/)
assert.match(requeueWorkflow, /\.name == "recover" and \.status != "completed"/)
assert.match(requeueWorkflow, /actions\/runs\/\$\{run_id\}\/jobs\?filter=latest/)
assert.match(requeueWorkflow, /if test "\$\{active\}" != 0; then exit 0; fi/)
assert.ok(requeueWorkflow.includes(`gh workflow run ${relayWorkflowFile('recover-relay-staging-c4-image.yml')}`))
assert.match(requeueWorkflow, /-f confirmation=RECOVER_STAGING_ASIA_C4_IMAGE/)
assert.doesNotMatch(requeueWorkflow, /id-token: write/)
assert.doesNotMatch(requeueWorkflow, /relay-staging-mutation/)
})
test('keeps cell-only plans independent from service-account description drift', () => {
assert.match(cells, /runtime_service_account\s+= local\.relay_runtime_service_account_email/)
assert.match(
cells,
/rehome_director_service_account\s+= local\.relay_director_runtime_service_account_email/
)
assert.match(relay, /var\.environment == "staging" \? "Orca Relay"/)
})
@@ -0,0 +1,269 @@
import assert from 'node:assert/strict'
import { readFileSync } from 'node:fs'
import test from 'node:test'
import { relayWorkflowUrl } from './relay-repository.mjs'
const terraform = readFileSync(
new URL('../../infra/terraform/relay-github-actions.tf', import.meta.url),
'utf8'
)
const outputs = readFileSync(new URL('../../infra/terraform/outputs.tf', import.meta.url), 'utf8')
const workflow = readFileSync(
relayWorkflowUrl('prove-relay-staging-capacity.yml'),
'utf8'
)
const deployWorkflow = readFileSync(
relayWorkflowUrl('deploy-relay-staging.yml'),
'utf8'
)
const publishWorkflow = readFileSync(
relayWorkflowUrl('publish-relay-production.yml'),
'utf8'
)
const bootstrapWorkflow = readFileSync(
relayWorkflowUrl('bootstrap-relay-staging-capacity.yml'),
'utf8'
)
function resource(type, name) {
const start = terraform.indexOf(`resource "${type}" "${name}"`)
assert.notEqual(start, -1, `${type}.${name} is missing`)
const next = terraform.indexOf('\nresource "', start + 1)
return terraform.slice(start, next === -1 ? undefined : next)
}
function terraformStringList(block, attribute) {
const match = block.match(new RegExp(`${attribute}\\s*=\\s*\\[([\\s\\S]*?)\\]`))
assert.ok(match, `${attribute} is missing`)
return [...match[1].matchAll(/"([^"]+)"/g)].map((entry) => entry[1])
}
test('capacity workflow uses only its exact staging identity', () => {
assert.match(workflow, /STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER/)
assert.match(workflow, /STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT/)
assert.doesNotMatch(workflow, /vars\.STAGING_GCP_WORKLOAD_IDENTITY_PROVIDER/)
assert.doesNotMatch(workflow, /vars\.STAGING_GCP_DEPLOY_SERVICE_ACCOUNT/)
const provider = resource(
'google_iam_workload_identity_pool_provider',
'github_staging_relay_capacity'
)
// The three repository claims are pinned once in relay-shared.tf; every provider concatenates
// that list rather than restating the repository on its own.
assert.match(provider, /concat\(local\.relay_github_leading_repository_claims, \[/)
for (const boundary of [
"assertion.ref == 'refs/heads/main'",
"assertion.environment == 'staging'"
]) {
assert.match(provider, new RegExp(boundary.replaceAll(/[.*+?^${}()|[\]\\]/g, '\\$&')))
}
assert.match(provider, /local\.relay_github_workflow_conditions\["github_staging_relay_capacity"\]/)
assert.deepEqual(
terraformStringList(terraform, 'github_staging_relay_capacity_workflow_files'),
[
'bootstrap-relay-staging-capacity.yml',
'prove-relay-staging-capacity.yml',
'recover-relay-staging-c4-image.yml'
]
)
assert.match(
terraform,
/for workflow_file in local\.github_staging_relay_capacity_workflow_files : "assertion\.workflow_ref == '\$\{prefix\}\$\{workflow_file\}@refs\/heads\/main'"/
)
assert.doesNotMatch(terraform, /github_staging_relay_capacity_workflow_file\s*=/)
})
test('job gates do not read environment variables before the environment is attached', () => {
for (const source of [workflow, deployWorkflow, bootstrapWorkflow]) {
const jobGate = source.match(/^\s{4}if:.*$/m)?.[0] ?? ''
assert.doesNotMatch(jobGate, /STAGING_GCP_RELAY_CAPACITY_/)
}
})
test('capacity identity has bounded mutation and state permissions', () => {
const role = resource(
'google_project_iam_custom_role',
'github_staging_relay_capacity_mutation'
)
assert.deepEqual(terraformStringList(role, 'permissions'), [
'compute.disks.create',
'compute.healthChecks.use',
'compute.images.useReadOnly',
'compute.instanceGroupManagers.get',
'compute.instanceGroupManagers.update',
'compute.instances.create',
'compute.instances.setLabels',
'compute.instances.setMetadata',
'compute.instances.setTags',
'compute.instanceTemplates.create',
'compute.instanceTemplates.delete',
'compute.instanceTemplates.get',
'compute.instanceTemplates.useReadOnly',
'compute.networks.use',
'compute.subnetworks.use',
'compute.zoneOperations.get'
])
assert.doesNotMatch(
role,
/compute\.(?:disks\.delete|instances\.(?:delete|start|stop|update))|cloudsql|secretmanager/
)
for (const source of [workflow, bootstrapWorkflow]) {
assert.match(source, /instance-groups managed recreate-instances/)
assert.match(source, /--instances/)
assert.doesNotMatch(source, /rolling-action restart/)
}
const state = resource(
'google_storage_bucket_iam_member',
'github_staging_relay_capacity_state'
)
assert.match(state, /roles\/storage\.objectAdmin/)
assert.match(state, /objects\/terraform\/state\/default\.tfstate/)
assert.match(state, /objects\/terraform\/state\/default\.tflock/)
assert.doesNotMatch(state, /resource\.name\.startsWith/)
const runtime = resource(
'google_service_account_iam_member',
'github_staging_relay_capacity_runtime_user'
)
assert.match(runtime, /google_service_account\.relay_runtime\.name/)
assert.match(runtime, /roles\/iam\.serviceAccountUser/)
const cloudRun = resource(
'google_cloud_run_v2_service_iam_member',
'github_staging_relay_capacity_developer'
)
assert.match(cloudRun, /name\s*=\s*var\.relay_cloud_run_service_name/)
assert.doesNotMatch(cloudRun, /google_cloud_run_v2_service\.relay/)
const relay = readFileSync(new URL('../../infra/terraform/relay.tf', import.meta.url), 'utf8')
const startup = readFileSync(
new URL('../../infra/terraform/relay-gce-startup.sh.tftpl', import.meta.url),
'utf8'
)
assert.match(relay, /ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT/)
assert.match(startup, /ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT/)
})
test('capacity identity exposes only its provider and service account', () => {
assert.match(outputs, /output "github_staging_relay_capacity_workload_identity_provider"/)
assert.match(outputs, /output "github_staging_relay_capacity_service_account"/)
})
test('director capacity configuration stays on the audited blue-green path', () => {
assert.match(deployWorkflow, /STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT/)
assert.match(deployWorkflow, /--capacity-service-account "\$\{CAPACITY_SERVICE_ACCOUNT\}"/)
assert.match(deployWorkflow, /expected-image-digest/)
assert.match(deployWorkflow, /var\.relay_gce_cells\["staging-gce-c4"\]\.image/)
assert.match(deployWorkflow, /init -reconfigure \\\n\s+-backend-config=backend\/staging\.hcl/)
assert.ok(
deployWorkflow.indexOf('id: google-auth') <
deployWorkflow.indexOf('Bind the request to the checked-in staging C4 image')
)
assert.match(deployWorkflow, /artifacts docker images describe "\$\{IMAGE\}"/)
assert.doesNotMatch(deployWorkflow, /docker (?:build|push)/)
assert.match(workflow, /--director-cells-json "\$\{DESIRED_CELLS_JSON\}"/)
assert.doesNotMatch(workflow, /target=google_cloud_run_v2_service\.relay/)
assert.doesNotMatch(workflow, /--mode director/)
})
test('mirrors the exact production manifest through the production deploy identity', () => {
assert.match(publishWorkflow, /options: \[publish, mirror-staging\]/)
assert.match(publishWorkflow, /MIRROR_RELAY_PRODUCTION_IMAGE_TO_STAGING/)
assert.match(publishWorkflow, /docker pull "\$\{source_image\}"/)
assert.match(publishWorkflow, /docker tag "\$\{source_image\}" "\$\{target_tag\}"/)
assert.match(publishWorkflow, /test "\$\{source_digest\}" = "\$\{MIRROR_DIGEST\}"/)
assert.match(publishWorkflow, /test "\$\{target_digest\}" = "\$\{MIRROR_DIGEST\}"/)
const mirrorWriter = resource(
'google_artifact_registry_repository_iam_member',
'github_production_relay_staging_mirror_writer'
)
assert.match(mirrorWriter, /var\.environment == "staging"/)
assert.match(mirrorWriter, /roles\/artifactregistry\.writer/)
assert.match(
mirrorWriter,
/serviceAccount:orca-cloud-gha-deploy@onorca-cloud\.iam\.gserviceaccount\.com/
)
})
test('cells bootstrap one at a time with bounded deploy and capacity identities', () => {
assert.match(bootstrapWorkflow, /STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER/)
assert.match(bootstrapWorkflow, /STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT/)
assert.match(bootstrapWorkflow, /STAGING_GCP_RELAY_CAPACITY_WORKLOAD_IDENTITY_PROVIDER/)
assert.match(bootstrapWorkflow, /STAGING_GCP_RELAY_CAPACITY_SERVICE_ACCOUNT/)
assert.match(bootstrapWorkflow, /--mode bootstrap-cell/)
assert.match(bootstrapWorkflow, /--cell-id "\$\{fallback_cell_id\}"/)
assert.match(bootstrapWorkflow, /google_compute_instance_template\.relay_gce_cell/)
assert.match(bootstrapWorkflow, /google_compute_instance_group_manager\.relay_gce_cell/)
assert.match(bootstrapWorkflow, /--mode restore-fallback/)
assert.doesNotMatch(bootstrapWorkflow, /target=google_cloud_run_v2_service\.relay/)
assert.ok(
bootstrapWorkflow.indexOf('id: deploy-auth') < bootstrapWorkflow.indexOf('id: capacity-auth')
)
assert.match(
bootstrapWorkflow,
/restore_fallback\(\) \{[\s\S]*?verify_fallback[\s\S]*?--mode restore-fallback/
)
const rollCell = bootstrapWorkflow.slice(bootstrapWorkflow.indexOf('roll_cell()'))
assert.ok(
rollCell.indexOf('verify_fallback\n trap restore_fallback EXIT') <
rollCell.indexOf('--mode isolate')
)
assert.match(
bootstrapWorkflow,
/staging-gce-c2 general[\s\S]*?trap restore_legacy_c3_fallback EXIT[\s\S]*?--mode isolate/
)
const normalize = bootstrapWorkflow.slice(
bootstrapWorkflow.indexOf('normalize_legacy_c3() {'),
bootstrapWorkflow.indexOf('\n roll_cell()', bootstrapWorkflow.indexOf('normalize_legacy_c3() {'))
)
const trapInstalled = normalize.indexOf('trap restore_legacy_c3_fallback EXIT')
const isolated = normalize.indexOf('--mode isolate', trapInstalled)
const recreated = normalize.indexOf('recreate-instances', isolated)
const restored = normalize.indexOf('--mode restore', recreated)
const c3Verified = normalize.indexOf('staging-gce-c3 general', restored)
const restoreDisabled = normalize.indexOf('legacy_c3_isolated=false', c3Verified)
const trapCleared = normalize.indexOf('trap - EXIT', restoreDisabled)
assert.ok(
trapInstalled < isolated &&
isolated < recreated &&
recreated < restored &&
restored < c3Verified &&
c3Verified < restoreDisabled &&
restoreDisabled < trapCleared
)
assert.equal(normalize.indexOf('trap - EXIT', trapInstalled), trapCleared)
assert.match(bootstrapWorkflow, /--heartbeat either/)
assert.doesNotMatch(bootstrapWorkflow, /heartbeat=stale/)
assert.match(
rollCell,
/"\$\{desired_cap\}" "\$\{desired_bound\}" absent-or-stale[\s\S]*?deploy-relay-blue-green\.mjs[\s\S]*?"\$\{desired_cap\}" "\$\{desired_bound\}"/
)
})
test('workflows read desired topology from configuration and gate exact predecessors', () => {
assert.match(workflow, /<<< 'local\.relay_director_cells_json' \| jq -r '\.'/)
assert.match(bootstrapWorkflow, /<<< 'local\.relay_director_cells_json' \| jq -r '\.'/)
assert.match(workflow, /1000\/0\)[\s\S]*?PREDECESSOR_C3_CAP=600/)
assert.match(workflow, /1000\/60\)[\s\S]*?PREDECESSOR_C3_BOUND=0/)
assert.match(workflow, /600\/60\)[\s\S]*?PREDECESSOR_C3_CAP=1000/)
assert.match(workflow, /Unsupported staging capacity transition/)
})
test('capacity apply resumes after director or cell success and preserves the no-op restart proof', () => {
for (const phase of ['predecessor', 'director-ready', 'cell-ready', 'cell-active']) {
assert.match(workflow, new RegExp(`TRANSITION_PHASE=${phase}`))
}
assert.match(workflow, /--argjson expected "\$\{PREDECESSOR_CELLS_JSON\}"/)
assert.match(workflow, /--argjson expected "\$\{DESIRED_CELLS_JSON\}"/)
assert.match(
workflow,
/test "\$\{TRANSITION_PHASE\}" = cell-ready; then[\s\S]*?test "\$\{CELL_PLAN_CHANGES\}" = 0/
)
assert.match(
workflow,
/test "\$\{TRANSITION_PHASE\}" = cell-active; then[\s\S]*?recreate_fixed_one_instance/
)
assert.match(workflow, /--admission migration-only/)
})
@@ -0,0 +1,228 @@
import assert from 'node:assert/strict'
import { readFileSync } from 'node:fs'
import test from 'node:test'
import { readWorkflow, workflowFiles } from './cloud-sql-rollout-lock-census.mjs'
import {
RELAY_WORKFLOW_FILE_PREFIX,
relayWorkflowFile,
relayWorkflowPath
} from './relay-repository.mjs'
const identity = readFileSync(
new URL('../../infra/terraform/relay-staging-deploy-iam.tf', import.meta.url),
'utf8'
)
const shared = readFileSync(
new URL('../../infra/terraform/relay-shared.tf', import.meta.url),
'utf8'
)
const variables = readFileSync(
new URL('../../infra/terraform/variables.tf', import.meta.url),
'utf8'
)
const outputs = readFileSync(new URL('../../infra/terraform/outputs.tf', import.meta.url), 'utf8')
const stagingTfvars = readFileSync(
new URL('../../infra/terraform/environments/staging.tfvars', import.meta.url),
'utf8'
)
// The exact five staging Relay workflows the relay-owned deploy identity serves.
const DEPLOY_WORKFLOWS = [
'bootstrap-relay-staging-capacity.yml',
'deploy-relay-staging-gce-candidate.yml',
'deploy-relay-staging.yml',
'operate-relay-asia-admission.yml',
'power-relay-staging.yml'
]
const GENERIC_STAGING_PAIR =
/vars\.STAGING_GCP_(?:WORKLOAD_IDENTITY_PROVIDER|DEPLOY_SERVICE_ACCOUNT)\b/
const workflowNames = workflowFiles
// DEPLOY_WORKFLOWS holds the names Terraform pins; the files on disk carry the copy's prefix.
const workflow = (name) => readWorkflow(relayWorkflowFile(name))
function block(type, name) {
const start = identity.indexOf(`resource "${type}" "${name}"`)
assert.notEqual(start, -1, `${type}.${name} is missing`)
const next = identity.indexOf('\nresource "', start + 1)
return identity.slice(start, next === -1 ? undefined : next)
}
function declaredFamilies() {
return [...identity.matchAll(/^resource "([a-z0-9_]+)" "([a-z0-9_]+)"/gm)]
.map((match) => `${match[1]}.${match[2]}`)
.sort()
}
function providerWorkflowFiles() {
const match = identity.match(
/github_staging_relay_deploy_workflow_files\s*=\s*\[([\s\S]*?)\n {2}\]/
)
assert.ok(match, 'github_staging_relay_deploy_workflow_files is missing')
return [...match[1].matchAll(/"([^"]+)"/g)].map((entry) => entry[1])
}
function variableDefault(name) {
const match = variables.match(
new RegExp(`variable "${name}" \\{[\\s\\S]*?default\\s*=\\s*"([^"]*)"`)
)
assert.ok(match, `variable ${name} has no default`)
return match[1]
}
test('no Relay workflow authenticates as the shared staging deploy identity', () => {
for (const name of workflowNames()) {
if (!name.includes('relay')) continue
assert.doesNotMatch(readWorkflow(name), GENERIC_STAGING_PAIR, name)
}
})
test('the five staging Relay workflows name the relay deploy pair', () => {
for (const name of DEPLOY_WORKFLOWS) {
const source = workflow(name)
assert.match(source, /vars\.STAGING_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER\b/, name)
assert.match(source, /vars\.STAGING_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT\b/, name)
}
})
// Why: the Asia workflow serves both environments from one job. Repointing its staging arm must
// not move production off the relay-owned shared account.
test('the Asia admission production arm keeps the production deploy pair', () => {
const source = workflow('operate-relay-asia-admission.yml')
assert.match(source, /vars\.PRODUCTION_GCP_RELAY_DEPLOY_WORKLOAD_IDENTITY_PROVIDER\b/)
assert.match(source, /vars\.PRODUCTION_GCP_RELAY_DEPLOY_SERVICE_ACCOUNT\b/)
})
test('the provider allowlists exactly those five workflow refs', () => {
const files = providerWorkflowFiles()
assert.deepEqual([...files].sort(), [...DEPLOY_WORKFLOWS].sort())
for (const file of files) {
assert.match(file, /^[a-z0-9-]+\.yml$/)
}
// Each accepted repository turns that file list into its own exact refs.
assert.match(
identity,
/for workflow_file in local\.github_staging_relay_deploy_workflow_files : "assertion\.workflow_ref == '\$\{prefix\}\$\{workflow_file\}@refs\/heads\/main'"/
)
const provider = block(
'google_iam_workload_identity_pool_provider',
'github_staging_relay_deploy'
)
assert.match(provider, /workload_identity_pool_provider_id\s*=\s*"github-relay-deploy"/)
assert.match(provider, /concat\(local\.relay_github_leading_repository_claims/)
assert.match(provider, /assertion\.ref == 'refs\/heads\/main'/)
assert.match(provider, /assertion\.environment == 'staging'/)
assert.match(provider, /local\.relay_github_workflow_conditions\["github_staging_relay_deploy"\]/)
// A prefix match would turn the allowlist into a namespace grant with no Terraform diff.
assert.doesNotMatch(provider, /startsWith|endsWith/)
})
// Why: the documented attribute_condition limit is 4096 characters and the expression grows with
// every workflow added. Render it the way Terraform does and keep the headroom visible.
test('the rendered attribute condition stays inside the provider limit', () => {
const repository = `${variableDefault('github_owner')}/${variableDefault('github_repo')}`
const claims = [
`assertion.repository == '${repository}'`,
`assertion.repository_id == '${variableDefault('github_repo_id')}'`,
`assertion.repository_owner_id == '${variableDefault('github_owner_id')}'`
]
const workflowRefs = providerWorkflowFiles().map(
(file) => `${repository}/${relayWorkflowPath(file)}@refs/heads/main`
)
const rendered = [
...claims,
"assertion.ref == 'refs/heads/main'",
"assertion.environment == 'staging'",
`(${workflowRefs.map((ref) => `assertion.workflow_ref == '${ref}'`).join(' || ')})`
].join(' && ')
assert.ok(rendered.length < 4096, `rendered condition is ${rendered.length} characters`)
// 797 is the private repository's rendered length. This copy prefixes every workflow filename,
// which is the only difference, so the pin still moves the moment a workflow is added or dropped.
assert.equal(rendered.length, 797 + workflowRefs.length * RELAY_WORKFLOW_FILE_PREFIX.length)
})
// Why: the census is the point. A binding added here without a workflow step behind it, or one
// silently dropped, changes what the staging Relay credential can reach.
test('the staging deploy identity declares exactly its enumerated grants', () => {
assert.deepEqual(declaredFamilies(), [
'google_artifact_registry_repository_iam_member.github_staging_relay_deploy_artifact_reader',
'google_cloud_run_v2_service_iam_member.github_staging_relay_deploy_auth_developer',
'google_cloud_run_v2_service_iam_member.github_staging_relay_deploy_director_developer',
'google_iam_workload_identity_pool_provider.github_staging_relay_deploy',
'google_project_iam_member.github_staging_relay_deploy_compute_viewer',
'google_service_account.github_staging_relay_deploy',
'google_service_account_iam_member.github_staging_relay_deploy_auth_runtime_user',
'google_service_account_iam_member.github_staging_relay_deploy_workload_identity_user',
'google_storage_bucket_iam_member.github_staging_relay_deploy_state',
'google_storage_bucket_iam_member.github_staging_relay_deploy_state_list'
])
// Each grant carries a comment naming the workflow step that needs it; the account, its
// provider, and the pool binding are the identity itself and are covered by the file header.
const identityFamilies = new Set([
'google_service_account.github_staging_relay_deploy',
'google_iam_workload_identity_pool_provider.github_staging_relay_deploy',
'google_service_account_iam_member.github_staging_relay_deploy_workload_identity_user'
])
for (const family of declaredFamilies()) {
if (identityFamilies.has(family)) continue
const [type, name] = family.split('.')
const preceding = identity.slice(0, identity.indexOf(`resource "${type}" "${name}"`)).trimEnd()
assert.match(preceding.slice(preceding.lastIndexOf('\n') + 1), /^#/, `${family} has no justifying comment`)
}
assert.match(identity, /var\.environment == "staging"/)
const state = block('google_storage_bucket_iam_member', 'github_staging_relay_deploy_state')
assert.match(state, /roles\/storage\.objectViewer/)
assert.match(state, /objects\/terraform\/state\/default\.tfstate/)
assert.match(state, /objects\/terraform\/state\/default\.tflock/)
assert.doesNotMatch(state, /resource\.name\.startsWith/)
assert.doesNotMatch(state, /objectAdmin/)
const director = block(
'google_cloud_run_v2_service_iam_member',
'github_staging_relay_deploy_director_developer'
)
assert.match(director, /name\s*=\s*var\.relay_cloud_run_service_name/)
// Project-wide run.developer or artifactregistry.writer would let the staging Relay credential
// deploy the API service or push images; the shared account holds both today.
const projectRoles = declaredFamilies()
.filter((family) => family.startsWith('google_project_iam_member.'))
.map((family) => block('google_project_iam_member', family.split('.')[1]).match(/role\s*=\s*"([^"]+)"/)[1])
assert.deepEqual(projectRoles, ['roles/compute.viewer'])
assert.doesNotMatch(identity, /roles\/artifactregistry\.writer/)
assert.doesNotMatch(identity, /"roles\/(?:owner|editor|viewer)"/)
})
// Why: the auth-plane grants are guarded on a variable, so an unset tfvars entry would drop them
// silently and Power Relay Staging would fail only on the sleep path.
test('staging pins the shared auth service the power workflow scales', () => {
assert.match(stagingTfvars, /relay_staging_power_auth_service_name\s*=\s*"orca-cloud-auth-staging"/)
assert.match(variables, /variable "relay_staging_power_auth_service_name"/)
for (const name of [
'github_staging_relay_deploy_auth_developer',
'github_staging_relay_deploy_auth_runtime_user'
]) {
const type = name.endsWith('runtime_user')
? 'google_service_account_iam_member'
: 'google_cloud_run_v2_service_iam_member'
assert.match(block(type, name), /var\.relay_staging_power_auth_service_name != ""/)
}
})
// Why: flipping this local is what moves the staging cells' startup metadata and the director's
// ORCA_RELAY_DEPLOY_SERVICE_ACCOUNT onto the new account. Production must keep the shared one.
test('the deploy account email is environment-conditional', () => {
assert.match(
shared,
/relay_github_deploy_service_account_email = \(\s*var\.environment == "production"\s*\? "\$\{var\.name_prefix\}-gha-deploy@\$\{var\.project_id\}\.iam\.gserviceaccount\.com"\s*: "\$\{var\.name_prefix\}-gha-relay@\$\{var\.project_id\}\.iam\.gserviceaccount\.com"\s*\)/
)
assert.match(identity, /account_id\s*=\s*"\$\{var\.name_prefix\}-gha-relay"/)
})
test('the identity is exposed through its own outputs', () => {
assert.match(outputs, /output "github_staging_relay_deploy_workload_identity_provider"/)
assert.match(outputs, /output "github_staging_relay_deploy_service_account"/)
})
@@ -0,0 +1,535 @@
// Renders every Workload Identity provider `attribute_condition` exactly as
// Terraform would, so contract tests can pin the resulting strings without a
// plan. Understands only the HCL subset those expressions use.
//
// Each root is loaded on its own: the relay and apps roots both declare a provider named
// `github` while the staging copy waits on its state surgery, and only separate scopes can
// show that the two render the same string.
//
// Only the relay root ships in this repository. The apps root is still declared so this stays a
// straight copy of the private original, and is skipped when its directory is absent.
import { existsSync } from 'node:fs'
import { readFile } from 'node:fs/promises'
const TERRAFORM_ROOTS = {
relay: {
directory: 'infra/terraform',
sources: [
'infra/terraform/relay-shared.tf',
'infra/terraform/relay-github-workflow-trust.tf',
'infra/terraform/relay-github-actions.tf',
'infra/terraform/relay-staging-deploy-iam.tf',
'infra/terraform/relay-asia-topology-iam.tf',
'infra/terraform/relay-asia-proof-iam.tf'
]
},
apps: {
directory: 'infra/terraform-apps',
sources: ['infra/terraform-apps/github-actions.tf']
}
}
export function hasTerraformRoot(root) {
const directory = TERRAFORM_ROOTS[root]?.directory
return directory !== undefined && existsSync(repoFile(directory))
}
export const TERRAFORM_ROOT_NAMES = Object.keys(TERRAFORM_ROOTS).filter(hasTerraformRoot)
const PROVIDER_RESOURCE = 'google_iam_workload_identity_pool_provider'
function repoFile(path) {
return new URL(`../../${path}`, import.meta.url)
}
function skipTrivia(src, index) {
let i = index
for (;;) {
while (i < src.length && /\s/.test(src[i])) i += 1
if (src[i] === '#') {
while (i < src.length && src[i] !== '\n') i += 1
continue
}
return i
}
}
// Returns the index just past the closing quote of the string starting at `i`.
function endOfString(src, i) {
let cursor = i + 1
while (src[cursor] !== '"') {
if (src[cursor] === '\\') {
cursor += 2
continue
}
if (src[cursor] === '$' && src[cursor + 1] === '{') {
cursor = endOfInterpolation(src, cursor + 2).next
continue
}
cursor += 1
}
return cursor + 1
}
function endOfInterpolation(src, i) {
let depth = 1
let cursor = i
while (depth > 0) {
const char = src[cursor]
if (char === undefined) throw new Error('unterminated interpolation')
if (char === '"') {
cursor = endOfString(src, cursor)
continue
}
if (char === '{') depth += 1
else if (char === '}') {
depth -= 1
if (depth === 0) break
}
cursor += 1
}
return { text: src.slice(i, cursor), next: cursor + 1 }
}
// Stands in for a loop variable when the collection is empty: the body still has to be parsed
// once to find where it ends, and any attribute of the probe is another probe.
const PROBE = new Proxy(
{},
{
get: (target, key) => (key === Symbol.toPrimitive ? () => '' : PROBE)
}
)
function readMember(value, key) {
if (value === PROBE) return PROBE
if (value === null || value === undefined) throw new Error(`cannot read ${String(key)} of ${value}`)
if (Array.isArray(value)) {
if (typeof key !== 'number') throw new Error(`list index must be a number, got ${String(key)}`)
if (!Number.isInteger(key) || key < 0 || key >= value.length) {
throw new Error(`list index ${key} is out of range`)
}
return value[key]
}
if (typeof value !== 'object') throw new Error(`cannot index ${typeof value}`)
if (!Object.hasOwn(value, key)) throw new Error(`unknown attribute ${String(key)}`)
return value[key]
}
// [key, value] pairs the way HCL iterates: list index and element, or object key and value.
function collectionEntries(collection) {
if (Array.isArray(collection)) return collection.map((item, index) => [index, item])
if (collection && typeof collection === 'object') return Object.entries(collection)
throw new Error(`cannot iterate ${typeof collection}`)
}
class ExpressionParser {
constructor(source, scope) {
this.source = source
this.scope = scope
this.index = 0
}
parse() {
const value = this.parseTernary()
this.index = skipTrivia(this.source, this.index)
if (this.index !== this.source.length) {
throw new Error(`trailing expression text: ${this.source.slice(this.index)}`)
}
return value
}
peek(token) {
this.index = skipTrivia(this.source, this.index)
return this.source.startsWith(token, this.index)
}
eat(token) {
if (!this.peek(token)) return false
this.index += token.length
return true
}
expect(token) {
if (!this.eat(token)) {
throw new Error(`expected ${token} at ${this.source.slice(this.index, this.index + 40)}`)
}
}
parseTernary() {
const condition = this.parseOr()
if (!this.eat('?')) return condition
const consequent = this.parseTernary()
this.expect(':')
const alternate = this.parseTernary()
return condition ? consequent : alternate
}
parseOr() {
let left = this.parseAnd()
while (this.eat('||')) left = Boolean(this.parseAnd()) || Boolean(left)
return left
}
parseAnd() {
let left = this.parseEquality()
while (this.eat('&&')) left = Boolean(this.parseEquality()) && Boolean(left)
return left
}
parseEquality() {
let left = this.parseUnary()
for (;;) {
if (this.eat('==')) left = left === this.parseUnary()
else if (this.eat('!=')) left = left !== this.parseUnary()
else return left
}
}
parseUnary() {
return this.parsePostfix(this.parsePrimary())
}
parsePrimary() {
if (this.eat('(')) {
const value = this.parseTernary()
this.expect(')')
return value
}
if (this.peek('"')) return this.parseString()
if (this.peek('[')) return this.parseList()
if (this.peek('{')) return this.parseObject()
const number = /^[0-9]+/.exec(this.source.slice(this.index))
if (number) {
this.index += number[0].length
return Number(number[0])
}
return this.parseIdentifier()
}
parsePostfix(value) {
let current = value
for (;;) {
if (this.eat('.')) {
current = readMember(current, this.readWord())
continue
}
if (this.peek('[')) {
this.index += 1
const key = this.parseTernary()
this.expect(']')
current = readMember(current, key)
continue
}
return current
}
}
// `for a in x : body` / `for a, b in x : body`, shared by list and object comprehensions.
parseComprehension(readBody) {
const names = [this.readWord()]
if (this.eat(',')) names.push(this.readWord())
this.expect('in')
const collection = this.parseUnary()
this.expect(':')
const bodyStart = skipTrivia(this.source, this.index)
const entries = collectionEntries(collection)
const bodyParser = ([key, item]) => {
const bindings = { ...this.scope.bindings }
if (names.length === 1) bindings[names[0]] = Array.isArray(collection) ? item : key
else {
bindings[names[0]] = key
bindings[names[1]] = item
}
const parser = new ExpressionParser(this.source, { ...this.scope, bindings })
parser.index = bodyStart
return parser
}
// Parse once with a probe binding to find where the body ends, because an empty
// collection would never parse it.
const probe = bodyParser(entries[0] ?? [PROBE, PROBE])
readBody(probe)
this.index = probe.index
return entries.map((entry) => readBody(bodyParser(entry)))
}
parseObject() {
this.expect('{')
if (this.eat('for')) {
const pairs = this.parseComprehension((parser) => {
const key = parser.parseTernary()
parser.expect('=>')
return [key, parser.parseTernary()]
})
this.expect('}')
return Object.fromEntries(pairs)
}
const object = {}
if (this.eat('}')) return object
for (;;) {
const key = this.peek('"') ? this.parseString() : this.readWord()
this.expect('=')
object[key] = this.parseTernary()
this.eat(',')
if (this.eat('}')) return object
}
}
parseString() {
this.index = skipTrivia(this.source, this.index)
const src = this.source
let cursor = this.index + 1
let rendered = ''
while (src[cursor] !== '"') {
if (src[cursor] === '\\') {
rendered += src[cursor + 1]
cursor += 2
continue
}
if (src[cursor] === '$' && src[cursor + 1] === '{') {
const { text, next } = endOfInterpolation(src, cursor + 2)
rendered += String(evaluate(text, this.scope))
cursor = next
continue
}
rendered += src[cursor]
cursor += 1
}
this.index = cursor + 1
return rendered
}
parseList() {
this.expect('[')
if (this.eat('for')) {
const items = this.parseComprehension((parser) => parser.parseTernary())
this.expect(']')
return items
}
const items = []
if (this.eat(']')) return items
for (;;) {
items.push(this.parseTernary())
if (this.eat(',')) {
if (this.eat(']')) return items
continue
}
this.expect(']')
return items
}
}
readWord() {
this.index = skipTrivia(this.source, this.index)
const match = /^[A-Za-z_][A-Za-z0-9_]*/.exec(this.source.slice(this.index))
if (!match) throw new Error(`expected identifier at ${this.source.slice(this.index, this.index + 40)}`)
this.index += match[0].length
return match[0]
}
parseIdentifier() {
const word = this.readWord()
if (word === 'join') {
this.expect('(')
const separator = this.parseTernary()
this.expect(',')
const parts = this.parseTernary()
this.eat(',')
this.expect(')')
return parts.join(separator)
}
if (word === 'concat') {
this.expect('(')
const lists = []
for (;;) {
lists.push(this.parseTernary())
if (this.eat(',')) {
if (this.eat(')')) break
continue
}
this.expect(')')
break
}
return lists.flat()
}
if (word === 'length') {
this.expect('(')
const value = this.parseTernary()
this.eat(',')
this.expect(')')
return collectionEntries(value).length
}
if (word === 'local') {
this.expect('.')
return resolveLocal(this.readWord(), this.scope)
}
if (word === 'var') {
this.expect('.')
const name = this.readWord()
if (!(name in this.scope.variables)) throw new Error(`unknown variable ${name}`)
return this.scope.variables[name]
}
if (word in this.scope.bindings) return this.scope.bindings[word]
if (word === 'true') return true
if (word === 'false') return false
throw new Error(`unsupported identifier ${word}`)
}
}
function evaluate(source, scope) {
return new ExpressionParser(source, scope).parse()
}
function resolveLocal(name, scope) {
if (scope.resolved.has(name)) return scope.resolved.get(name)
if (!scope.locals.has(name)) throw new Error(`unknown local ${name}`)
if (scope.resolving.has(name)) throw new Error(`local cycle at ${name}`)
scope.resolving.add(name)
const value = evaluate(scope.locals.get(name), { ...scope, bindings: {} })
scope.resolving.delete(name)
scope.resolved.set(name, value)
return value
}
function collectLocals(source, locals) {
const blockPattern = /^locals \{$/gm
let match
while ((match = blockPattern.exec(source)) !== null) {
const end = source.indexOf('\n}\n', match.index)
const body = source.slice(match.index + match[0].length, end)
let name = null
let buffer = []
const flush = () => {
if (name) locals.set(name, buffer.join('\n'))
}
for (const line of body.split('\n')) {
const assignment = /^ {2}([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(line)
if (assignment) {
flush()
name = assignment[1]
buffer = [assignment[2]]
continue
}
if (name && line.trim() !== '' && !line.trim().startsWith('#')) buffer.push(line)
}
flush()
}
}
function collectProviderFields(source, fields) {
const pattern = new RegExp(`resource "${PROVIDER_RESOURCE}" "([A-Za-z_0-9]+)" \\{`, 'g')
let match
while ((match = pattern.exec(source)) !== null) {
const end = source.indexOf('\n}\n', match.index)
const body = source.slice(match.index, end)
const conditionStart = body.indexOf(' attribute_condition = ')
const conditionEnd = body.indexOf('\n\n oidc {', conditionStart)
const countStart = body.indexOf(' count = ')
fields.set(match[1], {
count: body.slice(countStart + ' count = '.length, body.indexOf('\n', countStart)),
condition: body.slice(conditionStart + ' attribute_condition = '.length, conditionEnd)
})
}
}
// Values that are not a plain quoted string (a list of objects, say) are read with the
// expression parser; anything it cannot evaluate is left undefined, exactly as before.
function parseValueAt(source, index) {
const parser = new ExpressionParser(source, {
locals: new Map(),
variables: {},
bindings: {},
resolved: new Map(),
resolving: new Set()
})
parser.index = index
return parser.parseTernary()
}
function collectVariableDefaults(source, variables) {
const pattern = /variable "([A-Za-z_0-9]+)" \{([\s\S]*?)\n\}/g
let match
while ((match = pattern.exec(source)) !== null) {
const body = match[2]
const fallback = /\n\s*default\s*=\s*"([^"]*)"/.exec(body)
if (fallback) {
variables[match[1]] = fallback[1]
continue
}
const assignment = /\n\s*default\s*=\s*/.exec(body)
if (!assignment) continue
const start = match.index + match[0].indexOf(body) + assignment.index + assignment[0].length
try {
variables[match[1]] = parseValueAt(source, start)
} catch {
// A default this evaluator does not understand is not one any condition reads.
}
}
}
function collectTfvars(source, variables) {
let offset = 0
for (const line of source.split('\n')) {
const quoted = /^([A-Za-z_][A-Za-z0-9_]*)\s*=\s*"([^"]*)"\s*$/.exec(line)
if (quoted) {
variables[quoted[1]] = quoted[2]
offset += line.length + 1
continue
}
const structured = /^([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(?=[[{])/.exec(line)
if (structured) {
try {
variables[structured[1]] = parseValueAt(source, offset + structured[0].length)
} catch {
// Same as above: unreadable here means unread by every condition.
}
}
offset += line.length + 1
}
}
async function loadScope(root, environment) {
const { directory, sources } = TERRAFORM_ROOTS[root] ?? {}
if (!sources) throw new Error(`unknown terraform root ${root}`)
const locals = new Map()
const providers = new Map()
for (const path of sources) {
const source = await readFile(repoFile(path), 'utf8')
collectLocals(source, locals)
collectProviderFields(source, providers)
}
const variables = {}
collectVariableDefaults(await readFile(repoFile(`${directory}/variables.tf`), 'utf8'), variables)
collectTfvars(
await readFile(repoFile(`${directory}/environments/${environment}.tfvars`), 'utf8'),
variables
)
return {
providers,
scope: { locals, variables, bindings: {}, resolved: new Map(), resolving: new Set() }
}
}
// Rendered `attribute_condition` per provider that the given root creates in the environment.
export async function renderRootAttributeConditions(root, environment) {
const { providers, scope } = await loadScope(root, environment)
const rendered = {}
for (const [name, fields] of providers) {
if (evaluate(fields.count, scope) === 0) continue
rendered[name] = evaluate(fields.condition, scope)
}
return rendered
}
// Every root's rendered conditions, keyed by root and then by provider.
export async function renderAttributeConditions(environment) {
const rendered = {}
for (const root of TERRAFORM_ROOT_NAMES) {
rendered[root] = await renderRootAttributeConditions(root, environment)
}
return rendered
}
export async function readTerraformLocal(name, environment, root = 'relay') {
const { scope } = await loadScope(root, environment)
return resolveLocal(name, scope)
}
@@ -0,0 +1,8 @@
import { assertSpreadModel, modeledRelayLoad } from './relay-load-model.mjs'
const counts = process.argv.slice(2).length > 0 ? process.argv.slice(2).map(Number) : [4_000, 10_000]
for (const count of counts) {
const model = modeledRelayLoad(count)
assertSpreadModel(model)
console.log(JSON.stringify(model))
}

Some files were not shown because too many files have changed in this diff Show More